diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index ffb61be226b..ff934d46817 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -382,7 +382,7 @@ build/cuda101/nompi/clang/all/release/static:
     CUDA_ARCH: 35
 
 # clang-cuda with cuda 10.1 and friends
-build/clang-cuda101/nompi/gcc/cuda/release/shared:
+build/clang-cuda101/openmpi/gcc/cuda/release/shared:
   <<: *default_build
   extends:
     - .quick_test_condition
@@ -392,6 +392,8 @@ build/clang-cuda101/nompi/gcc/cuda/release/shared:
     CUDA_COMPILER: "clang++"
     BUILD_OMP: "ON"
     BUILD_CUDA: "ON"
+    BUILD_MPI: "ON"
+    MPI_AS_ROOT: "ON"
     BUILD_HIP: "OFF"
     BUILD_TYPE: "Release"
     CUDA_ARCH: 35
@@ -465,7 +467,7 @@ build/cuda102/nompi/intel/cuda/debug/static:
     CUDA_ARCH: 35
 
 # cuda 11.0 and friends
-build/cuda110/nompi/gcc/cuda/debug/shared:
+build/cuda110/mvapich2/gcc/cuda/debug/shared:
   <<: *default_build_with_test
   extends:
     - .full_test_condition
@@ -474,6 +476,7 @@ build/cuda110/nompi/gcc/cuda/debug/shared:
     <<: *default_variables
     BUILD_OMP: "ON"
     BUILD_CUDA: "ON"
+    BUILD_MPI: "ON"
     BUILD_TYPE: "Debug"
     FAST_TESTS: "ON"
     CUDA_ARCH: 61
@@ -541,7 +544,7 @@ build/amd/nompi/gcc/hip/debug/shared:
     BUILD_TYPE: "Debug"
     FAST_TESTS: "ON"
 
-build/amd/nompi/clang/hip/release/static:
+build/amd/openmpi/clang/hip/release/static:
   <<: *default_build_with_test
   extends:
     - .quick_test_condition
@@ -552,6 +555,8 @@ build/amd/nompi/clang/hip/release/static:
     CXX_COMPILER: "clang++"
     BUILD_OMP: "ON"
     BUILD_HIP: "ON"
+    MPI_AS_ROOT: "ON"
+    BUILD_MPI: "ON"
     BUILD_TYPE: "Release"
     BUILD_SHARED_LIBS: "OFF"
 
@@ -614,7 +619,7 @@ build/nocuda/nompi/gcc/omp/release/shared:
     BUILD_OMP: "ON"
     BUILD_TYPE: "Release"
 
-build/nocuda/nompi/clang/omp/debug/static:
+build/nocuda/openmpi/clang/omp/debug/static:
   <<: *default_build_with_test
   extends:
     - .full_test_condition
@@ -624,6 +629,8 @@ build/nocuda/nompi/clang/omp/debug/static:
     C_COMPILER: "clang"
     CXX_COMPILER: "clang++"
     BUILD_OMP: "ON"
+    MPI_AS_ROOT: "ON"
+    BUILD_MPI: "ON"
     BUILD_TYPE: "Debug"
     FAST_TESTS: "ON"
     BUILD_SHARED_LIBS: "OFF"
diff --git a/cmake/create_test.cmake b/cmake/create_test.cmake
index 32162604869..dccd16332e2 100644
--- a/cmake/create_test.cmake
+++ b/cmake/create_test.cmake
@@ -75,7 +75,7 @@ function(ginkgo_create_mpi_test test_name num_mpi_procs)
   else()
       set(OPENMPI_RUN_AS_ROOT_FLAG "")
   endif()
-  target_link_libraries(${TEST_TARGET_NAME} PRIVATE ginkgo GTest::MPI_main GTest::GTest ${ARGN})
+  target_link_libraries(${TEST_TARGET_NAME} PRIVATE ginkgo gtest_mpi_main GTest::GTest ${ARGN})
   target_link_libraries(${TEST_TARGET_NAME} PRIVATE MPI::MPI_CXX)
   set(test_param ${MPIEXEC_NUMPROC_FLAG} ${num_mpi_procs} ${OPENMPI_RUN_AS_ROOT_FLAG} ${CMAKE_BINARY_DIR}/${REL_BINARY_DIR}/${test_name})
   add_test(NAME ${REL_BINARY_DIR}/${test_name}
diff --git a/core/test/mpi/CMakeLists.txt b/core/test/mpi/CMakeLists.txt
index 1ad6a5575b2..8edc6781c4e 100644
--- a/core/test/mpi/CMakeLists.txt
+++ b/core/test/mpi/CMakeLists.txt
@@ -1 +1,7 @@
+add_library(gtest_mpi_main "")
+target_sources(gtest_mpi_main
+    PRIVATE
+    gtest/mpi_listener.cpp)
+find_package(MPI REQUIRED)
+target_link_libraries(gtest_mpi_main PRIVATE GTest::GTest MPI::MPI_CXX)
 add_subdirectory(base)
diff --git a/core/test/mpi/base/bindings.cpp b/core/test/mpi/base/bindings.cpp
index c8d9f5a885f..410df29b7a6 100644
--- a/core/test/mpi/base/bindings.cpp
+++ b/core/test/mpi/base/bindings.cpp
@@ -69,8 +69,10 @@ TYPED_TEST(MpiBindings, CanCreatewindow)
 {
     auto data = std::vector<TypeParam>{1, 2, 3, 4};
     auto comm = gko::mpi::communicator(MPI_COMM_WORLD);
+
     auto win =
         gko::mpi::window<TypeParam>(data.data(), 4 * sizeof(TypeParam), comm);
+
     ASSERT_NE(win.get_window(), MPI_WIN_NULL);
     win.lock_all();
     win.unlock_all();
@@ -83,6 +85,7 @@ TYPED_TEST(MpiBindings, CanSendAndRecvValues)
     auto my_rank = comm.rank();
     auto num_ranks = comm.size();
     auto recv_array = gko::Array<TypeParam>{this->ref};
+
     if (my_rank == 0) {
         auto send_array = std::vector<TypeParam>{1, 2, 3, 4};
         for (auto rank = 0; rank < num_ranks; ++rank) {
@@ -94,6 +97,7 @@ TYPED_TEST(MpiBindings, CanSendAndRecvValues)
         recv_array = gko::Array<TypeParam>{this->ref, 4};
         comm.recv(recv_array.get_data(), 4, 0, 40 + my_rank);
     }
+
     if (my_rank != 0) {
         auto ref_array = gko::Array<TypeParam>{this->ref, {1, 2, 3, 4}};
         GKO_ASSERT_ARRAY_EQ(ref_array, recv_array);
@@ -111,6 +115,7 @@ TYPED_TEST(MpiBindings, CanNonBlockingSendAndNonBlockingRecvValues)
     TypeParam* data;
     auto req1 = std::vector<gko::mpi::request>(num_ranks);
     auto req2 = gko::mpi::request();
+
     if (my_rank == 0) {
         send_array = std::vector<TypeParam>{1, 2, 3, 4};
         for (auto rank = 0; rank < num_ranks; ++rank) {
@@ -122,6 +127,7 @@ TYPED_TEST(MpiBindings, CanNonBlockingSendAndNonBlockingRecvValues)
         recv_array = gko::Array<TypeParam>{this->ref, 4};
         req2 = comm.i_recv(recv_array.get_data(), 4, 0, 40 + my_rank);
     }
+
     if (my_rank == 0) {
         auto stat1 = wait_all(req1);
     } else {
@@ -146,17 +152,52 @@ TYPED_TEST(MpiBindings, CanPutValuesWithLockAll)
     } else {
         data = std::vector<TypeParam>{0, 0, 0, 0};
     }
-    auto win = window(data.data(), 4, comm);
+
+    {
+        auto win = window(data.data(), 4, comm);
+        if (my_rank == 0) {
+            win.lock_all();
+            for (auto rank = 0; rank < num_ranks; ++rank) {
+                if (rank != my_rank) {
+                    win.put(data.data(), 4, rank, 0, 4);
+                }
+            }
+            win.unlock_all();
+        }
+    }
+
+    auto ref = std::vector<TypeParam>{1, 2, 3, 4};
+    ASSERT_EQ(data, ref);
+}
+
+
+TYPED_TEST(MpiBindings, CanNonBlockingPutValuesWithLockAll)
+{
+    using window = gko::mpi::window<TypeParam>;
+    auto comm = gko::mpi::communicator(MPI_COMM_WORLD);
+    auto my_rank = comm.rank();
+    auto num_ranks = comm.size();
+    std::vector<TypeParam> data;
     if (my_rank == 0) {
-        win.lock_all();
-        for (auto rank = 0; rank < num_ranks; ++rank) {
-            if (rank != my_rank) {
-                win.put(data.data(), 4, rank, 0, 4);
+        data = std::vector<TypeParam>{1, 2, 3, 4};
+    } else {
+        data = std::vector<TypeParam>{0, 0, 0, 0};
+    }
+
+    {
+        gko::mpi::request req;
+        auto win = window(data.data(), 4, comm);
+        if (my_rank == 0) {
+            win.lock_all();
+            for (auto rank = 0; rank < num_ranks; ++rank) {
+                if (rank != my_rank) {
+                    req = win.r_put(data.data(), 4, rank, 0, 4);
+                }
             }
+            req.wait();
+            win.unlock_all();
         }
-        win.unlock_all();
     }
-    win.fence();
 
     auto ref = std::vector<TypeParam>{1, 2, 3, 4};
     ASSERT_EQ(data, ref);
@@ -170,23 +211,59 @@ TYPED_TEST(MpiBindings, CanPutValuesWithExclusiveLock)
     auto my_rank = comm.rank();
     auto num_ranks = comm.size();
     std::vector<TypeParam> data;
+
     if (my_rank == 0) {
         data = std::vector<TypeParam>{1, 2, 3, 4};
     } else {
         data = std::vector<TypeParam>{0, 0, 0, 0};
     }
-    auto win = window(data.data(), 4, comm);
+
+    {
+        auto win = window(data.data(), 4, comm);
+        if (my_rank == 0) {
+            for (auto rank = 0; rank < num_ranks; ++rank) {
+                if (rank != my_rank) {
+                    win.lock(rank, window::lock_type::exclusive);
+                    win.put(data.data(), 4, rank, 0, 4);
+                    win.flush(0);
+                    win.unlock(rank);
+                }
+            }
+        }
+    }
+
+    auto ref = std::vector<TypeParam>{1, 2, 3, 4};
+    ASSERT_EQ(data, ref);
+}
+
+
+TYPED_TEST(MpiBindings, CanPutValuesWithSharedLock)
+{
+    using window = gko::mpi::window<TypeParam>;
+    auto comm = gko::mpi::communicator(MPI_COMM_WORLD);
+    auto my_rank = comm.rank();
+    auto num_ranks = comm.size();
+    std::vector<TypeParam> data;
+
     if (my_rank == 0) {
-        for (auto rank = 0; rank < num_ranks; ++rank) {
-            if (rank != my_rank) {
-                win.lock(rank, 0, window::lock_type::exclusive);
-                win.put(data.data(), 4, rank, 0, 4);
-                win.flush(0);
-                win.unlock(rank);
+        data = std::vector<TypeParam>{1, 2, 3, 4};
+    } else {
+        data = std::vector<TypeParam>{0, 0, 0, 0};
+    }
+
+    {
+        auto win = window(data.data(), 4, comm);
+        if (my_rank == 0) {
+            for (auto rank = 0; rank < num_ranks; ++rank) {
+                if (rank != my_rank) {
+                    win.lock(rank);
+                    win.put(data.data(), 4, rank, 0, 4);
+                    win.flush(0);
+                    win.unlock(rank);
+                }
             }
         }
     }
-    win.fence();
 
     auto ref = std::vector<TypeParam>{1, 2, 3, 4};
     ASSERT_EQ(data, ref);
@@ -206,6 +283,7 @@ TYPED_TEST(MpiBindings, CanPutValuesWithFence)
         data = std::vector<TypeParam>{0, 0, 0, 0};
     }
     auto win = window(data.data(), 4, comm);
+
     win.fence();
     if (my_rank == 0) {
         for (auto rank = 0; rank < num_ranks; ++rank) {
@@ -221,7 +299,7 @@ TYPED_TEST(MpiBindings, CanPutValuesWithFence)
 }
 
 
-TYPED_TEST(MpiBindings, CanGetValuesWithLockAll)
+TYPED_TEST(MpiBindings, CanAccumulateValues)
 {
     using window = gko::mpi::window<TypeParam>;
     auto comm = gko::mpi::communicator(MPI_COMM_WORLD);
@@ -230,26 +308,45 @@ TYPED_TEST(MpiBindings, CanGetValuesWithLockAll)
     std::vector<TypeParam> data;
     if (my_rank == 0) {
         data = std::vector<TypeParam>{1, 2, 3, 4};
+    } else if (my_rank == 1) {
+        data = std::vector<TypeParam>{5, 6, 7, 8};
+    } else if (my_rank == 2) {
+        data = std::vector<TypeParam>{9, 10, 11, 12};
     } else {
         data = std::vector<TypeParam>{0, 0, 0, 0};
     }
-    auto win = window(data.data(), 4, comm);
-    if (my_rank != 0) {
-        win.lock_all();
-        for (auto rank = 0; rank < num_ranks; ++rank) {
-            if (rank != my_rank) {
-                win.get(data.data(), 4, 0, 0, 4);
+
+    {
+        auto win = window(data.data(), 4, comm);
+        if (my_rank == 0) {
+            win.lock_all();
+            for (auto rank = 0; rank < num_ranks; ++rank) {
+                if (rank != my_rank) {
+                    win.accumulate(data.data(), 4, rank, 0, 4, MPI_SUM);
+                }
             }
+            win.unlock_all();
         }
-        win.unlock_all();
     }
 
-    auto ref = std::vector<TypeParam>{1, 2, 3, 4};
-    ASSERT_EQ(data, ref);
+    std::vector<TypeParam> ref;
+    if (my_rank == 0) {
+        ref = std::vector<TypeParam>{1, 2, 3, 4};
+        ASSERT_EQ(data, ref);
+    } else if (my_rank == 1) {
+        ref = std::vector<TypeParam>{6, 8, 10, 12};
+        ASSERT_EQ(data, ref);
+    } else if (my_rank == 2) {
+        ref = std::vector<TypeParam>{10, 12, 14, 16};
+        ASSERT_EQ(data, ref);
+    } else {
+        ref = std::vector<TypeParam>{1, 2, 3, 4};
+        ASSERT_EQ(data, ref);
+    }
 }
 
 
-TYPED_TEST(MpiBindings, CanGetValuesWithExclusiveLock)
+TYPED_TEST(MpiBindings, CanNonBlockingAccumulateValues)
 {
     using window = gko::mpi::window<TypeParam>;
     auto comm = gko::mpi::communicator(MPI_COMM_WORLD);
@@ -258,26 +355,47 @@ TYPED_TEST(MpiBindings, CanGetValuesWithExclusiveLock)
     std::vector<TypeParam> data;
     if (my_rank == 0) {
         data = std::vector<TypeParam>{1, 2, 3, 4};
+    } else if (my_rank == 1) {
+        data = std::vector<TypeParam>{5, 6, 7, 8};
+    } else if (my_rank == 2) {
+        data = std::vector<TypeParam>{9, 10, 11, 12};
     } else {
         data = std::vector<TypeParam>{0, 0, 0, 0};
     }
-    auto win = window(data.data(), 4, comm);
-    if (my_rank != 0) {
-        for (auto rank = 0; rank < num_ranks; ++rank) {
-            if (rank != my_rank) {
-                win.lock(0, 0, window::lock_type::exclusive);
-                win.get(data.data(), 4, 0, 0, 4);
-                win.unlock(0);
+
+    gko::mpi::request req;
+    {
+        auto win = window(data.data(), 4, comm);
+        if (my_rank == 0) {
+            win.lock_all();
+            for (auto rank = 0; rank < num_ranks; ++rank) {
+                if (rank != my_rank) {
+                    req = win.r_accumulate(data.data(), 4, rank, 0, 4, MPI_SUM);
+                }
             }
+            win.unlock_all();
         }
     }
 
-    auto ref = std::vector<TypeParam>{1, 2, 3, 4};
-    ASSERT_EQ(data, ref);
+    req.wait();
+    std::vector<TypeParam> ref;
+    if (my_rank == 0) {
+        ref = std::vector<TypeParam>{1, 2, 3, 4};
+        ASSERT_EQ(data, ref);
+    } else if (my_rank == 1) {
+        ref = std::vector<TypeParam>{6, 8, 10, 12};
+        ASSERT_EQ(data, ref);
+    } else if (my_rank == 2) {
+        ref = std::vector<TypeParam>{10, 12, 14, 16};
+        ASSERT_EQ(data, ref);
+    } else {
+        ref = std::vector<TypeParam>{1, 2, 3, 4};
+        ASSERT_EQ(data, ref);
+    }
 }
 
 
-TYPED_TEST(MpiBindings, CanGetValuesWithFence)
+TYPED_TEST(MpiBindings, CanGetValuesWithLockAll)
 {
     using window = gko::mpi::window<TypeParam>;
     auto comm = gko::mpi::communicator(MPI_COMM_WORLD);
@@ -290,174 +408,714 @@ TYPED_TEST(MpiBindings, CanGetValuesWithFence)
         data = std::vector<TypeParam>{0, 0, 0, 0};
     }
     auto win = window(data.data(), 4, comm);
-    win.fence();
+
     if (my_rank != 0) {
-        for (auto rank = 0; rank < num_ranks; ++rank) {
-            if (rank != my_rank) {
-                win.get(data.data(), 4, 0, 0, 4);
-            }
-        }
+        win.lock_all();
+        win.get(data.data(), 4, 0, 0, 4);
+        win.unlock_all();
     }
-    win.fence();
 
     auto ref = std::vector<TypeParam>{1, 2, 3, 4};
     ASSERT_EQ(data, ref);
 }
 
 
-TYPED_TEST(MpiBindings, CanBroadcastValues)
+TYPED_TEST(MpiBindings, CanNonBlockingGetValuesWithLockAll)
 {
+    using window = gko::mpi::window<TypeParam>;
     auto comm = gko::mpi::communicator(MPI_COMM_WORLD);
     auto my_rank = comm.rank();
     auto num_ranks = comm.size();
-    auto array = gko::Array<TypeParam>{this->ref, 8};
+    std::vector<TypeParam> data;
     if (my_rank == 0) {
-        array = gko::Array<TypeParam>(this->ref, {2, 3, 1, 3, -1, 0, 3, 1});
+        data = std::vector<TypeParam>{1, 2, 3, 4};
+    } else {
+        data = std::vector<TypeParam>{0, 0, 0, 0};
     }
-    comm.broadcast(array.get_data(), 8, 0);
-    auto comp_data = array.get_data();
-    ASSERT_EQ(comp_data[0], TypeParam{2});
-    ASSERT_EQ(comp_data[1], TypeParam{3});
-    ASSERT_EQ(comp_data[2], TypeParam{1});
-    ASSERT_EQ(comp_data[3], TypeParam{3});
-    ASSERT_EQ(comp_data[4], TypeParam{-1});
-    ASSERT_EQ(comp_data[5], TypeParam{0});
-    ASSERT_EQ(comp_data[6], TypeParam{3});
-    ASSERT_EQ(comp_data[7], TypeParam{1});
+    gko::mpi::request req;
+    auto win = window(data.data(), 4, comm);
+
+    if (my_rank != 0) {
+        win.lock_all();
+        req = win.r_get(data.data(), 4, 0, 0, 4);
+        win.unlock_all();
+    }
+
+    req.wait();
+    auto ref = std::vector<TypeParam>{1, 2, 3, 4};
+    ASSERT_EQ(data, ref);
 }
 
 
-TYPED_TEST(MpiBindings, CanReduceValues)
+TYPED_TEST(MpiBindings, CanGetValuesWithExclusiveLock)
 {
-    using TypeParam = TypeParam;
+    using window = gko::mpi::window<TypeParam>;
     auto comm = gko::mpi::communicator(MPI_COMM_WORLD);
     auto my_rank = comm.rank();
     auto num_ranks = comm.size();
-    TypeParam data, sum, max, min;
+    std::vector<TypeParam> data;
     if (my_rank == 0) {
-        data = 3;
-    } else if (my_rank == 1) {
-        data = 5;
-    } else if (my_rank == 2) {
-        data = 2;
-    } else if (my_rank == 3) {
-        data = 6;
+        data = std::vector<TypeParam>{1, 2, 3, 4};
+    } else {
+        data = std::vector<TypeParam>{0, 0, 0, 0};
     }
-    comm.reduce(&data, &sum, 1, MPI_SUM, 0);
-    comm.reduce(&data, &max, 1, MPI_MAX, 0);
-    comm.reduce(&data, &min, 1, MPI_MIN, 0);
-    if (my_rank == 0) {
-        EXPECT_EQ(sum, TypeParam{16});
-        EXPECT_EQ(max, TypeParam{6});
-        EXPECT_EQ(min, TypeParam{2});
+    auto win = window(data.data(), 4, comm);
+
+    if (my_rank != 0) {
+        win.lock(0, window::lock_type::exclusive);
+        win.get(data.data(), 4, 0, 0, 4);
+        win.unlock(0);
     }
+
+    auto ref = std::vector<TypeParam>{1, 2, 3, 4};
+    ASSERT_EQ(data, ref);
 }
 
 
-TYPED_TEST(MpiBindings, CanAllReduceValues)
+TYPED_TEST(MpiBindings, CanGetValuesWithSharedLock)
 {
+    using window = gko::mpi::window<TypeParam>;
     auto comm = gko::mpi::communicator(MPI_COMM_WORLD);
     auto my_rank = comm.rank();
     auto num_ranks = comm.size();
-    TypeParam data, sum;
+    std::vector<TypeParam> data;
     if (my_rank == 0) {
-        data = 3;
-    } else if (my_rank == 1) {
-        data = 5;
-    } else if (my_rank == 2) {
-        data = 2;
-    } else if (my_rank == 3) {
-        data = 6;
+        data = std::vector<TypeParam>{1, 2, 3, 4};
+    } else {
+        data = std::vector<TypeParam>{0, 0, 0, 0};
     }
-    comm.all_reduce(&data, &sum, 1, MPI_SUM);
-    ASSERT_EQ(sum, TypeParam{16});
+    auto win = window(data.data(), 4, comm);
+
+    if (my_rank != 0) {
+        win.lock(0);
+        win.get(data.data(), 4, 0, 0, 4);
+        win.unlock(0);
+    }
+
+    auto ref = std::vector<TypeParam>{1, 2, 3, 4};
+    ASSERT_EQ(data, ref);
 }
 
 
-TYPED_TEST(MpiBindings, CanAllReduceValuesInPlace)
+TYPED_TEST(MpiBindings, CanGetValuesWithFence)
 {
+    using window = gko::mpi::window<TypeParam>;
     auto comm = gko::mpi::communicator(MPI_COMM_WORLD);
     auto my_rank = comm.rank();
     auto num_ranks = comm.size();
-    TypeParam data;
+    std::vector<TypeParam> data;
     if (my_rank == 0) {
-        data = 3;
-    } else if (my_rank == 1) {
-        data = 5;
-    } else if (my_rank == 2) {
-        data = 2;
-    } else if (my_rank == 3) {
-        data = 6;
+        data = std::vector<TypeParam>{1, 2, 3, 4};
+    } else {
+        data = std::vector<TypeParam>{0, 0, 0, 0};
     }
-    comm.all_reduce(&data, 1, MPI_SUM);
-    ASSERT_EQ(data, TypeParam{16});
+    auto win = window(data.data(), 4, comm);
+
+    win.fence();
+    if (my_rank != 0) {
+        win.get(data.data(), 4, 0, 0, 4);
+    }
+    win.fence();
+
+    auto ref = std::vector<TypeParam>{1, 2, 3, 4};
+    ASSERT_EQ(data, ref);
 }
 
 
-TYPED_TEST(MpiBindings, CanScatterValues)
+TYPED_TEST(MpiBindings, CanGetAccumulateValuesWithLockAll)
 {
+    using window = gko::mpi::window<TypeParam>;
     auto comm = gko::mpi::communicator(MPI_COMM_WORLD);
     auto my_rank = comm.rank();
     auto num_ranks = comm.size();
-    auto scatter_from_array = gko::Array<TypeParam>{this->ref};
+    std::vector<TypeParam> data;
+    std::vector<TypeParam> target;
+    std::vector<TypeParam> result(4, 0);
     if (my_rank == 0) {
-        scatter_from_array =
-            gko::Array<TypeParam>{this->ref, {2, 3, 1, 3, -1, 0, 3, 1}};
+        data = std::vector<TypeParam>{1, 2, 3, 4};
+        target = std::vector<TypeParam>{1, 2, 3, 4};
+    } else if (my_rank == 1) {
+        data = std::vector<TypeParam>{5, 6, 7, 8};
+        target = std::vector<TypeParam>{5, 6, 7, 8};
+    } else if (my_rank == 2) {
+        data = std::vector<TypeParam>{9, 10, 11, 12};
+        target = std::vector<TypeParam>{9, 10, 11, 12};
+    } else {
+        data = std::vector<TypeParam>{0, 0, 0, 0};
+        target = std::vector<TypeParam>{0, 0, 0, 0};
     }
-    auto scatter_into_array = gko::Array<TypeParam>{this->ref, 2};
-    comm.scatter(scatter_from_array.get_data(), 2,
-                 scatter_into_array.get_data(), 2, 0);
-    auto comp_data = scatter_into_array.get_data();
-    if (my_rank == 0) {
-        ASSERT_EQ(comp_data[0], TypeParam{2});
-        ASSERT_EQ(comp_data[1], TypeParam{3});
 
-    } else if (my_rank == 1) {
-        ASSERT_EQ(comp_data[0], TypeParam{1});
-        ASSERT_EQ(comp_data[1], TypeParam{3});
+    {
+        auto win = window(target.data(), 4, comm);
+
+        if (my_rank == 2) {
+            win.lock_all();
+            win.get_accumulate(data.data(), 4, result.data(), 4, 0, 0, 4,
+                               MPI_SUM);
+            win.unlock_all();
+        }
+    }
+
+    std::vector<TypeParam> ref;
+    std::vector<TypeParam> ref2;
+    if (my_rank == 0) {
+        ref = std::vector<TypeParam>{10, 12, 14, 16};
+        EXPECT_EQ(target, ref);
     } else if (my_rank == 2) {
-        ASSERT_EQ(comp_data[0], TypeParam{-1});
-        ASSERT_EQ(comp_data[1], TypeParam{0});
-    } else if (my_rank == 3) {
-        ASSERT_EQ(comp_data[0], TypeParam{3});
-        ASSERT_EQ(comp_data[1], TypeParam{1});
+        ref = std::vector<TypeParam>{1, 2, 3, 4};
+        EXPECT_EQ(result, ref);
     }
 }
 
 
-TYPED_TEST(MpiBindings, CanGatherValues)
+TYPED_TEST(MpiBindings, CanNonBlockingGetAccumulateValuesWithLockAll)
 {
+    using window = gko::mpi::window<TypeParam>;
     auto comm = gko::mpi::communicator(MPI_COMM_WORLD);
     auto my_rank = comm.rank();
     auto num_ranks = comm.size();
-    TypeParam data;
+    std::vector<TypeParam> data;
+    std::vector<TypeParam> target;
+    std::vector<TypeParam> result(4, 0);
     if (my_rank == 0) {
-        data = 3;
+        data = std::vector<TypeParam>{1, 2, 3, 4};
+        target = std::vector<TypeParam>{1, 2, 3, 4};
     } else if (my_rank == 1) {
-        data = 5;
+        data = std::vector<TypeParam>{5, 6, 7, 8};
+        target = std::vector<TypeParam>{5, 6, 7, 8};
     } else if (my_rank == 2) {
-        data = 2;
-    } else if (my_rank == 3) {
-        data = 6;
+        data = std::vector<TypeParam>{9, 10, 11, 12};
+        target = std::vector<TypeParam>{9, 10, 11, 12};
+    } else {
+        data = std::vector<TypeParam>{0, 0, 0, 0};
+        target = std::vector<TypeParam>{0, 0, 0, 0};
     }
-    auto gather_array = gko::Array<TypeParam>{
-        this->ref, static_cast<gko::size_type>(num_ranks)};
-    comm.gather(&data, 1, gather_array.get_data(), 1, 0);
+    gko::mpi::request req;
+
+    {
+        auto win = window(target.data(), 4, comm);
+
+        if (my_rank == 2) {
+            win.lock_all();
+            req = win.r_get_accumulate(data.data(), 4, result.data(), 4, 0, 0,
+                                       4, MPI_SUM);
+            win.unlock_all();
+        }
+    }
+
+    req.wait();
+    std::vector<TypeParam> ref;
+    std::vector<TypeParam> ref2;
     if (my_rank == 0) {
-        ASSERT_EQ(gather_array.get_data()[0], TypeParam{3});
-        ASSERT_EQ(gather_array.get_data()[1], TypeParam{5});
-        ASSERT_EQ(gather_array.get_data()[2], TypeParam{2});
-        ASSERT_EQ(gather_array.get_data()[3], TypeParam{6});
+        ref = std::vector<TypeParam>{10, 12, 14, 16};
+        ref2 = std::vector<TypeParam>{1, 2, 3, 4};
+        EXPECT_EQ(target, ref);
+        EXPECT_EQ(data, ref2);
+    } else if (my_rank == 2) {
+        ref = std::vector<TypeParam>{1, 2, 3, 4};
+        ref2 = std::vector<TypeParam>{9, 10, 11, 12};
+        EXPECT_EQ(result, ref);
+        EXPECT_EQ(target, ref2);
+        EXPECT_EQ(data, ref2);
     }
 }
 
 
-TYPED_TEST(MpiBindings, CanScatterValuesWithDisplacements)
+TYPED_TEST(MpiBindings, CanFetchAndOperate)
 {
+    using window = gko::mpi::window<TypeParam>;
     auto comm = gko::mpi::communicator(MPI_COMM_WORLD);
     auto my_rank = comm.rank();
     auto num_ranks = comm.size();
-    auto scatter_from_array = gko::Array<TypeParam>{this->ref};
+    std::vector<TypeParam> data;
+    std::vector<TypeParam> target;
+    std::vector<TypeParam> result(4, 0);
+    if (my_rank == 0) {
+        data = std::vector<TypeParam>{1, 2, 3, 4};
+        target = std::vector<TypeParam>{1, 2, 3, 4};
+    } else if (my_rank == 1) {
+        data = std::vector<TypeParam>{5, 6, 7, 8};
+        target = std::vector<TypeParam>{5, 6, 7, 8};
+    } else if (my_rank == 2) {
+        data = std::vector<TypeParam>{9, 10, 11, 12};
+        target = std::vector<TypeParam>{9, 10, 11, 12};
+    } else {
+        data = std::vector<TypeParam>{0, 0, 0, 0};
+        target = std::vector<TypeParam>{0, 0, 0, 0};
+    }
+
+    {
+        auto win = window(target.data(), 4, comm);
+
+        if (my_rank == 2) {
+            win.lock_all();
+            win.fetch_and_op(data.data(), result.data(), 0, 1, MPI_SUM);
+            win.unlock_all();
+        }
+    }
+
+    std::vector<TypeParam> ref;
+    std::vector<TypeParam> ref2;
+    if (my_rank == 0) {
+        ref = std::vector<TypeParam>{1, 11, 3, 4};
+        EXPECT_EQ(target, ref);
+    } else if (my_rank == 2) {
+        ref = std::vector<TypeParam>{2, 0, 0, 0};
+        EXPECT_EQ(result, ref);
+    }
+}
+
+
+TYPED_TEST(MpiBindings, CanBroadcastValues)
+{
+    auto comm = gko::mpi::communicator(MPI_COMM_WORLD);
+    auto my_rank = comm.rank();
+    auto num_ranks = comm.size();
+    auto array = gko::Array<TypeParam>{this->ref, 8};
+    if (my_rank == 0) {
+        array = gko::Array<TypeParam>(this->ref, {2, 3, 1, 3, -1, 0, 3, 1});
+    }
+
+    comm.broadcast(array.get_data(), 8, 0);
+
+    auto ref = gko::Array<TypeParam>(this->ref, {2, 3, 1, 3, -1, 0, 3, 1});
+    GKO_ASSERT_ARRAY_EQ(ref, array);
+}
+
+
+TYPED_TEST(MpiBindings, CanNonBlockingBroadcastValues)
+{
+    auto comm = gko::mpi::communicator(MPI_COMM_WORLD);
+    auto my_rank = comm.rank();
+    auto num_ranks = comm.size();
+    auto array = gko::Array<TypeParam>{this->ref, 8};
+    if (my_rank == 0) {
+        array = gko::Array<TypeParam>(this->ref, {2, 3, 1, 3, -1, 0, 3, 1});
+    }
+
+    auto req = comm.i_broadcast(array.get_data(), 8, 0);
+
+    req.wait();
+    auto ref = gko::Array<TypeParam>(this->ref, {2, 3, 1, 3, -1, 0, 3, 1});
+    GKO_ASSERT_ARRAY_EQ(ref, array);
+}
+
+
+TYPED_TEST(MpiBindings, CanReduceValues)
+{
+    auto comm = gko::mpi::communicator(MPI_COMM_WORLD);
+    auto my_rank = comm.rank();
+    auto num_ranks = comm.size();
+    TypeParam data, sum, max, min;
+    if (my_rank == 0) {
+        data = 3;
+    } else if (my_rank == 1) {
+        data = 5;
+    } else if (my_rank == 2) {
+        data = 2;
+    } else if (my_rank == 3) {
+        data = 6;
+    }
+
+    comm.reduce(&data, &sum, 1, MPI_SUM, 0);
+    comm.reduce(&data, &max, 1, MPI_MAX, 0);
+    comm.reduce(&data, &min, 1, MPI_MIN, 0);
+
+    if (my_rank == 0) {
+        EXPECT_EQ(sum, TypeParam{16});
+        EXPECT_EQ(max, TypeParam{6});
+        EXPECT_EQ(min, TypeParam{2});
+    }
+}
+
+
+TYPED_TEST(MpiBindings, CanNonBlockingReduceValues)
+{
+    auto comm = gko::mpi::communicator(MPI_COMM_WORLD);
+    auto my_rank = comm.rank();
+    auto num_ranks = comm.size();
+    TypeParam data, sum, max, min;
+    if (my_rank == 0) {
+        data = 3;
+    } else if (my_rank == 1) {
+        data = 5;
+    } else if (my_rank == 2) {
+        data = 2;
+    } else if (my_rank == 3) {
+        data = 6;
+    }
+
+    auto req1 = comm.i_reduce(&data, &sum, 1, MPI_SUM, 0);
+    auto req2 = comm.i_reduce(&data, &max, 1, MPI_MAX, 0);
+    auto req3 = comm.i_reduce(&data, &min, 1, MPI_MIN, 0);
+
+    req1.wait();
+    req2.wait();
+    req3.wait();
+    if (my_rank == 0) {
+        EXPECT_EQ(sum, TypeParam{16});
+        EXPECT_EQ(max, TypeParam{6});
+        EXPECT_EQ(min, TypeParam{2});
+    }
+}
+
+
+TYPED_TEST(MpiBindings, CanAllReduceValues)
+{
+    auto comm = gko::mpi::communicator(MPI_COMM_WORLD);
+    auto my_rank = comm.rank();
+    auto num_ranks = comm.size();
+    TypeParam data, sum;
+    if (my_rank == 0) {
+        data = 3;
+    } else if (my_rank == 1) {
+        data = 5;
+    } else if (my_rank == 2) {
+        data = 2;
+    } else if (my_rank == 3) {
+        data = 6;
+    }
+
+    comm.all_reduce(&data, &sum, 1, MPI_SUM);
+
+    ASSERT_EQ(sum, TypeParam{16});
+}
+
+
+TYPED_TEST(MpiBindings, CanAllReduceValuesInPlace)
+{
+    auto comm = gko::mpi::communicator(MPI_COMM_WORLD);
+    auto my_rank = comm.rank();
+    auto num_ranks = comm.size();
+    TypeParam data;
+    if (my_rank == 0) {
+        data = 3;
+    } else if (my_rank == 1) {
+        data = 5;
+    } else if (my_rank == 2) {
+        data = 2;
+    } else if (my_rank == 3) {
+        data = 6;
+    }
+
+    comm.all_reduce(&data, 1, MPI_SUM);
+
+    ASSERT_EQ(data, TypeParam{16});
+}
+
+
+TYPED_TEST(MpiBindings, CanNonBlockingAllReduceValues)
+{
+    auto comm = gko::mpi::communicator(MPI_COMM_WORLD);
+    auto my_rank = comm.rank();
+    auto num_ranks = comm.size();
+    TypeParam data, sum;
+    if (my_rank == 0) {
+        data = 3;
+    } else if (my_rank == 1) {
+        data = 5;
+    } else if (my_rank == 2) {
+        data = 2;
+    } else if (my_rank == 3) {
+        data = 6;
+    }
+
+    auto req = comm.i_all_reduce(&data, &sum, 1, MPI_SUM);
+
+    req.wait();
+    ASSERT_EQ(sum, TypeParam{16});
+}
+
+
+TYPED_TEST(MpiBindings, CanNonBlockingAllReduceValuesInPlace)
+{
+    auto comm = gko::mpi::communicator(MPI_COMM_WORLD);
+    auto my_rank = comm.rank();
+    auto num_ranks = comm.size();
+    TypeParam data;
+    if (my_rank == 0) {
+        data = 3;
+    } else if (my_rank == 1) {
+        data = 5;
+    } else if (my_rank == 2) {
+        data = 2;
+    } else if (my_rank == 3) {
+        data = 6;
+    }
+
+    auto req = comm.i_all_reduce(&data, 1, MPI_SUM);
+
+    req.wait();
+    ASSERT_EQ(data, TypeParam{16});
+}
+
+
+TYPED_TEST(MpiBindings, CanGatherValues)
+{
+    auto comm = gko::mpi::communicator(MPI_COMM_WORLD);
+    auto my_rank = comm.rank();
+    auto num_ranks = comm.size();
+    TypeParam data;
+    if (my_rank == 0) {
+        data = 3;
+    } else if (my_rank == 1) {
+        data = 5;
+    } else if (my_rank == 2) {
+        data = 2;
+    } else if (my_rank == 3) {
+        data = 6;
+    }
+    auto gather_array = gko::Array<TypeParam>{
+        this->ref, static_cast<gko::size_type>(num_ranks)};
+
+    comm.gather(&data, 1, gather_array.get_data(), 1, 0);
+
+    if (my_rank == 0) {
+        auto ref = gko::Array<TypeParam>(this->ref, {3, 5, 2, 6});
+        GKO_ASSERT_ARRAY_EQ(ref, gather_array);
+    }
+}
+
+
+TYPED_TEST(MpiBindings, CanNonBlockingGatherValues)
+{
+    auto comm = gko::mpi::communicator(MPI_COMM_WORLD);
+    auto my_rank = comm.rank();
+    auto num_ranks = comm.size();
+    TypeParam data;
+    if (my_rank == 0) {
+        data = 3;
+    } else if (my_rank == 1) {
+        data = 5;
+    } else if (my_rank == 2) {
+        data = 2;
+    } else if (my_rank == 3) {
+        data = 6;
+    }
+    auto gather_array = gko::Array<TypeParam>{
+        this->ref, static_cast<gko::size_type>(num_ranks)};
+
+    auto req = comm.i_gather(&data, 1, gather_array.get_data(), 1, 0);
+
+    req.wait();
+    if (my_rank == 0) {
+        auto ref = gko::Array<TypeParam>(this->ref, {3, 5, 2, 6});
+        GKO_ASSERT_ARRAY_EQ(ref, gather_array);
+    }
+}
+
+
+TYPED_TEST(MpiBindings, CanAllGatherValues)
+{
+    auto comm = gko::mpi::communicator(MPI_COMM_WORLD);
+    auto my_rank = comm.rank();
+    auto num_ranks = comm.size();
+    TypeParam data;
+    if (my_rank == 0) {
+        data = 3;
+    } else if (my_rank == 1) {
+        data = 5;
+    } else if (my_rank == 2) {
+        data = 2;
+    } else if (my_rank == 3) {
+        data = 6;
+    }
+    auto gather_array = gko::Array<TypeParam>{
+        this->ref, static_cast<gko::size_type>(num_ranks)};
+
+    comm.all_gather(&data, 1, gather_array.get_data(), 1);
+
+    auto ref = gko::Array<TypeParam>(this->ref, {3, 5, 2, 6});
+    GKO_ASSERT_ARRAY_EQ(ref, gather_array);
+}
+
+
+TYPED_TEST(MpiBindings, CanNonBlockingAllGatherValues)
+{
+    auto comm = gko::mpi::communicator(MPI_COMM_WORLD);
+    auto my_rank = comm.rank();
+    auto num_ranks = comm.size();
+    TypeParam data;
+    if (my_rank == 0) {
+        data = 3;
+    } else if (my_rank == 1) {
+        data = 5;
+    } else if (my_rank == 2) {
+        data = 2;
+    } else if (my_rank == 3) {
+        data = 6;
+    }
+    auto gather_array = gko::Array<TypeParam>{
+        this->ref, static_cast<gko::size_type>(num_ranks)};
+
+    auto req = comm.i_all_gather(&data, 1, gather_array.get_data(), 1);
+
+    req.wait();
+    auto ref = gko::Array<TypeParam>(this->ref, {3, 5, 2, 6});
+    GKO_ASSERT_ARRAY_EQ(ref, gather_array);
+}
+
+
+TYPED_TEST(MpiBindings, CanGatherValuesWithDisplacements)
+{
+    auto comm = gko::mpi::communicator(MPI_COMM_WORLD);
+    auto my_rank = comm.rank();
+    auto num_ranks = comm.size();
+    auto gather_from_array = gko::Array<TypeParam>{this->ref};
+    auto gather_into_array = gko::Array<TypeParam>{this->ref};
+    auto r_counts =
+        gko::Array<int>{this->ref, static_cast<gko::size_type>(num_ranks)};
+    auto displacements = gko::Array<int>{this->ref};
+    int nelems;
+    if (my_rank == 0) {
+        gather_from_array = gko::Array<TypeParam>{this->ref, {2, 3}};
+        nelems = 2;
+        displacements = gko::Array<int>{this->ref, {0, 2, 6, 7}};
+        gather_into_array = gko::Array<TypeParam>{this->ref, 10};
+    } else if (my_rank == 1) {
+        nelems = 4;
+        gather_from_array = gko::Array<TypeParam>{this->ref, {1, 2, 1, 0}};
+    } else if (my_rank == 2) {
+        nelems = 1;
+        gather_from_array = gko::Array<TypeParam>{this->ref, {1}};
+    } else if (my_rank == 3) {
+        nelems = 3;
+        gather_from_array = gko::Array<TypeParam>{this->ref, {1, -4, 5}};
+    }
+
+    comm.gather(&nelems, 1, r_counts.get_data(), 1, 0);
+    comm.gather_v(gather_from_array.get_data(), nelems,
+                  gather_into_array.get_data(), r_counts.get_data(),
+                  displacements.get_data(), 0);
+
+    auto comp_data = gather_into_array.get_data();
+    if (my_rank == 0) {
+        auto ref_array =
+            gko::Array<TypeParam>(this->ref, {2, 3, 1, 2, 1, 0, 1, 1, -4, 5});
+        GKO_ASSERT_ARRAY_EQ(gather_into_array, ref_array);
+    } else {
+        ASSERT_EQ(comp_data, nullptr);
+    }
+}
+
+
+TYPED_TEST(MpiBindings, CanNonBlockingGatherValuesWithDisplacements)
+{
+    auto comm = gko::mpi::communicator(MPI_COMM_WORLD);
+    auto my_rank = comm.rank();
+    auto num_ranks = comm.size();
+    auto gather_from_array = gko::Array<TypeParam>{this->ref};
+    auto gather_into_array = gko::Array<TypeParam>{this->ref};
+    auto r_counts =
+        gko::Array<int>{this->ref, static_cast<gko::size_type>(num_ranks)};
+    auto displacements = gko::Array<int>{this->ref};
+    int nelems;
+    if (my_rank == 0) {
+        gather_from_array = gko::Array<TypeParam>{this->ref, {2, 3}};
+        nelems = 2;
+        displacements = gko::Array<int>{this->ref, {0, 2, 6, 7}};
+        gather_into_array = gko::Array<TypeParam>{this->ref, 10};
+    } else if (my_rank == 1) {
+        nelems = 4;
+        gather_from_array = gko::Array<TypeParam>{this->ref, {1, 2, 1, 0}};
+    } else if (my_rank == 2) {
+        nelems = 1;
+        gather_from_array = gko::Array<TypeParam>{this->ref, {1}};
+    } else if (my_rank == 3) {
+        nelems = 3;
+        gather_from_array = gko::Array<TypeParam>{this->ref, {1, -4, 5}};
+    }
+
+    comm.gather(&nelems, 1, r_counts.get_data(), 1, 0);
+    auto req = comm.i_gather_v(
+        gather_from_array.get_data(), nelems, gather_into_array.get_data(),
+        r_counts.get_data(), displacements.get_data(), 0);
+
+    req.wait();
+    auto comp_data = gather_into_array.get_data();
+    if (my_rank == 0) {
+        auto ref_array =
+            gko::Array<TypeParam>(this->ref, {2, 3, 1, 2, 1, 0, 1, 1, -4, 5});
+        GKO_ASSERT_ARRAY_EQ(gather_into_array, ref_array);
+    } else {
+        ASSERT_EQ(comp_data, nullptr);
+    }
+}
+
+
+TYPED_TEST(MpiBindings, CanScatterValues)
+{
+    auto comm = gko::mpi::communicator(MPI_COMM_WORLD);
+    auto my_rank = comm.rank();
+    auto num_ranks = comm.size();
+    auto scatter_from_array = gko::Array<TypeParam>{this->ref};
+    if (my_rank == 0) {
+        scatter_from_array =
+            gko::Array<TypeParam>{this->ref, {2, 3, 1, 3, -1, 0, 3, 1}};
+    }
+    auto scatter_into_array = gko::Array<TypeParam>{this->ref, 2};
+
+    comm.scatter(scatter_from_array.get_data(), 2,
+                 scatter_into_array.get_data(), 2, 0);
+
+    auto comp_data = scatter_into_array.get_data();
+    if (my_rank == 0) {
+        ASSERT_EQ(comp_data[0], TypeParam{2});
+        ASSERT_EQ(comp_data[1], TypeParam{3});
+
+    } else if (my_rank == 1) {
+        ASSERT_EQ(comp_data[0], TypeParam{1});
+        ASSERT_EQ(comp_data[1], TypeParam{3});
+    } else if (my_rank == 2) {
+        ASSERT_EQ(comp_data[0], TypeParam{-1});
+        ASSERT_EQ(comp_data[1], TypeParam{0});
+    } else if (my_rank == 3) {
+        ASSERT_EQ(comp_data[0], TypeParam{3});
+        ASSERT_EQ(comp_data[1], TypeParam{1});
+    }
+}
+
+
+TYPED_TEST(MpiBindings, CanNonBlockingScatterValues)
+{
+    auto comm = gko::mpi::communicator(MPI_COMM_WORLD);
+    auto my_rank = comm.rank();
+    auto num_ranks = comm.size();
+    auto scatter_from_array = gko::Array<TypeParam>{this->ref};
+    if (my_rank == 0) {
+        scatter_from_array =
+            gko::Array<TypeParam>{this->ref, {2, 3, 1, 3, -1, 0, 3, 1}};
+    }
+    auto scatter_into_array = gko::Array<TypeParam>{this->ref, 2};
+
+    auto req = comm.i_scatter(scatter_from_array.get_data(), 2,
+                              scatter_into_array.get_data(), 2, 0);
+
+    req.wait();
+    auto comp_data = scatter_into_array.get_data();
+    if (my_rank == 0) {
+        ASSERT_EQ(comp_data[0], TypeParam{2});
+        ASSERT_EQ(comp_data[1], TypeParam{3});
+
+    } else if (my_rank == 1) {
+        ASSERT_EQ(comp_data[0], TypeParam{1});
+        ASSERT_EQ(comp_data[1], TypeParam{3});
+    } else if (my_rank == 2) {
+        ASSERT_EQ(comp_data[0], TypeParam{-1});
+        ASSERT_EQ(comp_data[1], TypeParam{0});
+    } else if (my_rank == 3) {
+        ASSERT_EQ(comp_data[0], TypeParam{3});
+        ASSERT_EQ(comp_data[1], TypeParam{1});
+    }
+}
+
+
+TYPED_TEST(MpiBindings, CanScatterValuesWithDisplacements)
+{
+    auto comm = gko::mpi::communicator(MPI_COMM_WORLD);
+    auto my_rank = comm.rank();
+    auto num_ranks = comm.size();
+    auto scatter_from_array = gko::Array<TypeParam>{this->ref};
     auto scatter_into_array = gko::Array<TypeParam>{this->ref};
     auto s_counts = gko::Array<int>{this->ref->get_master(),
                                     static_cast<gko::size_type>(num_ranks)};
@@ -477,10 +1135,12 @@ TYPED_TEST(MpiBindings, CanScatterValuesWithDisplacements)
     }
     scatter_into_array =
         gko::Array<TypeParam>{this->ref, static_cast<gko::size_type>(nelems)};
+
     comm.gather(&nelems, 1, s_counts.get_data(), 1, 0);
     comm.scatter_v(scatter_from_array.get_data(), s_counts.get_data(),
                    displacements.get_data(), scatter_into_array.get_data(),
                    nelems, 0);
+
     auto comp_data = scatter_into_array.get_data();
     if (my_rank == 0) {
         ASSERT_EQ(comp_data[0], TypeParam{2});
@@ -501,44 +1161,54 @@ TYPED_TEST(MpiBindings, CanScatterValuesWithDisplacements)
 }
 
 
-TYPED_TEST(MpiBindings, CanGatherValuesWithDisplacements)
+TYPED_TEST(MpiBindings, CanNonBlockingScatterValuesWithDisplacements)
 {
     auto comm = gko::mpi::communicator(MPI_COMM_WORLD);
     auto my_rank = comm.rank();
     auto num_ranks = comm.size();
-    auto gather_from_array = gko::Array<TypeParam>{this->ref};
-    auto gather_into_array = gko::Array<TypeParam>{this->ref};
-    auto r_counts =
-        gko::Array<int>{this->ref, static_cast<gko::size_type>(num_ranks)};
-    auto displacements = gko::Array<int>{this->ref};
+    auto scatter_from_array = gko::Array<TypeParam>{this->ref};
+    auto scatter_into_array = gko::Array<TypeParam>{this->ref};
+    auto s_counts = gko::Array<int>{this->ref->get_master(),
+                                    static_cast<gko::size_type>(num_ranks)};
+    auto displacements = gko::Array<int>{this->ref->get_master()};
     int nelems;
     if (my_rank == 0) {
-        gather_from_array = gko::Array<TypeParam>{this->ref, {2, 3}};
+        scatter_from_array =
+            gko::Array<TypeParam>{this->ref, {2, 3, 1, 3, -1, 0, 2, -1, 0, 3}};
         nelems = 2;
-        displacements = gko::Array<int>{this->ref, {0, 2, 6, 7}};
-        gather_into_array = gko::Array<TypeParam>{this->ref, 10};
+        displacements = gko::Array<int>{this->ref, {0, 2, 6, 9}};
     } else if (my_rank == 1) {
         nelems = 4;
-        gather_from_array = gko::Array<TypeParam>{this->ref, {1, 2, 1, 0}};
     } else if (my_rank == 2) {
-        nelems = 1;
-        gather_from_array = gko::Array<TypeParam>{this->ref, {1}};
-    } else if (my_rank == 3) {
         nelems = 3;
-        gather_from_array = gko::Array<TypeParam>{this->ref, {1, -4, 5}};
+    } else if (my_rank == 3) {
+        nelems = 1;
     }
+    scatter_into_array =
+        gko::Array<TypeParam>{this->ref, static_cast<gko::size_type>(nelems)};
 
-    comm.gather(&nelems, 1, r_counts.get_data(), 1, 0);
-    comm.gather_v(gather_from_array.get_data(), nelems,
-                  gather_into_array.get_data(), r_counts.get_data(),
-                  displacements.get_data(), 0);
-    auto comp_data = gather_into_array.get_data();
+    comm.gather(&nelems, 1, s_counts.get_data(), 1, 0);
+    auto req = comm.i_scatter_v(scatter_from_array.get_data(),
+                                s_counts.get_data(), displacements.get_data(),
+                                scatter_into_array.get_data(), nelems, 0);
+
+    req.wait();
+    auto comp_data = scatter_into_array.get_data();
     if (my_rank == 0) {
-        auto ref_array =
-            gko::Array<TypeParam>(this->ref, {2, 3, 1, 2, 1, 0, 1, 1, -4, 5});
-        GKO_ASSERT_ARRAY_EQ(gather_into_array, ref_array);
-    } else {
-        ASSERT_EQ(comp_data, nullptr);
+        ASSERT_EQ(comp_data[0], TypeParam{2});
+        ASSERT_EQ(comp_data[1], TypeParam{3});
+
+    } else if (my_rank == 1) {
+        ASSERT_EQ(comp_data[0], TypeParam{1});
+        ASSERT_EQ(comp_data[1], TypeParam{3});
+        ASSERT_EQ(comp_data[2], TypeParam{-1});
+        ASSERT_EQ(comp_data[3], TypeParam{0});
+    } else if (my_rank == 2) {
+        ASSERT_EQ(comp_data[0], TypeParam{2});
+        ASSERT_EQ(comp_data[1], TypeParam{-1});
+        ASSERT_EQ(comp_data[2], TypeParam{0});
+    } else if (my_rank == 3) {
+        ASSERT_EQ(comp_data[0], TypeParam{3});
     }
 }
 
@@ -567,6 +1237,38 @@ TYPED_TEST(MpiBindings, AllToAllWorksCorrectly)
     }
 
     comm.all_to_all(send_array.get_data(), 1, recv_array.get_data(), 1);
+
+    GKO_ASSERT_ARRAY_EQ(recv_array, ref_array);
+}
+
+
+TYPED_TEST(MpiBindings, NonBlockingAllToAllWorksCorrectly)
+{
+    auto comm = gko::mpi::communicator(MPI_COMM_WORLD);
+    auto my_rank = comm.rank();
+    auto num_ranks = comm.size();
+    auto send_array = gko::Array<TypeParam>{this->ref};
+    auto recv_array = gko::Array<TypeParam>{this->ref};
+    auto ref_array = gko::Array<TypeParam>{this->ref};
+    recv_array = gko::Array<TypeParam>{this->ref, 4};
+    if (my_rank == 0) {
+        send_array = gko::Array<TypeParam>(this->ref, {2, 3, 1, 2});
+        ref_array = gko::Array<TypeParam>(this->ref, {2, 2, 2, 5});
+    } else if (my_rank == 1) {
+        send_array = gko::Array<TypeParam>(this->ref, {2, 3, 1, 2});
+        ref_array = gko::Array<TypeParam>(this->ref, {3, 3, 3, 3});
+    } else if (my_rank == 2) {
+        send_array = gko::Array<TypeParam>(this->ref, {2, 3, 1, 0});
+        ref_array = gko::Array<TypeParam>(this->ref, {1, 1, 1, 3});
+    } else if (my_rank == 3) {
+        send_array = gko::Array<TypeParam>(this->ref, {5, 3, 3, -2});
+        ref_array = gko::Array<TypeParam>(this->ref, {2, 2, 0, -2});
+    }
+
+    auto req =
+        comm.i_all_to_all(send_array.get_data(), 1, recv_array.get_data(), 1);
+
+    req.wait();
     GKO_ASSERT_ARRAY_EQ(recv_array, ref_array);
 }
 
@@ -598,6 +1300,35 @@ TYPED_TEST(MpiBindings, AllToAllInPlaceWorksCorrectly)
 }
 
 
+TYPED_TEST(MpiBindings, NonBlockingAllToAllInPlaceWorksCorrectly)
+{
+    auto comm = gko::mpi::communicator(MPI_COMM_WORLD);
+    auto my_rank = comm.rank();
+    auto num_ranks = comm.size();
+    auto recv_array = gko::Array<TypeParam>{this->ref};
+    auto ref_array = gko::Array<TypeParam>{this->ref};
+    recv_array = gko::Array<TypeParam>{this->ref, 4};
+    if (my_rank == 0) {
+        recv_array = gko::Array<TypeParam>(this->ref, {2, 3, 1, 2});
+        ref_array = gko::Array<TypeParam>(this->ref, {2, 2, 2, 5});
+    } else if (my_rank == 1) {
+        recv_array = gko::Array<TypeParam>(this->ref, {2, 3, 1, 2});
+        ref_array = gko::Array<TypeParam>(this->ref, {3, 3, 3, 3});
+    } else if (my_rank == 2) {
+        recv_array = gko::Array<TypeParam>(this->ref, {2, 3, 1, 0});
+        ref_array = gko::Array<TypeParam>(this->ref, {1, 1, 1, 3});
+    } else if (my_rank == 3) {
+        recv_array = gko::Array<TypeParam>(this->ref, {5, 3, 3, -2});
+        ref_array = gko::Array<TypeParam>(this->ref, {2, 2, 0, -2});
+    }
+
+    auto req = comm.i_all_to_all(recv_array.get_data(), 1);
+
+    req.wait();
+    GKO_ASSERT_ARRAY_EQ(recv_array, ref_array);
+}
+
+
 TYPED_TEST(MpiBindings, AllToAllVWorksCorrectly)
 {
     auto comm = gko::mpi::communicator(MPI_COMM_WORLD);
@@ -651,6 +1382,62 @@ TYPED_TEST(MpiBindings, AllToAllVWorksCorrectly)
 }
 
 
+TYPED_TEST(MpiBindings, NonBlockingAllToAllVWorksCorrectly)
+{
+    auto comm = gko::mpi::communicator(MPI_COMM_WORLD);
+    auto my_rank = comm.rank();
+    auto num_ranks = comm.size();
+    auto send_array = gko::Array<TypeParam>{this->ref};
+    auto recv_array = gko::Array<TypeParam>{this->ref};
+    auto ref_array = gko::Array<TypeParam>{this->ref};
+    auto scounts_array = gko::Array<int>{this->ref};
+    auto soffset_array = gko::Array<int>{this->ref};
+    auto rcounts_array = gko::Array<int>{this->ref};
+    auto roffset_array = gko::Array<int>{this->ref};
+    if (my_rank == 0) {
+        recv_array = gko::Array<TypeParam>{this->ref, {0, 0, 0, 0, 0, 0}};
+        send_array = gko::Array<TypeParam>{this->ref, {2, 3, 1, 2}};
+        scounts_array = gko::Array<int>{this->ref, {1, 2, 1, 0}};
+        rcounts_array = gko::Array<int>{this->ref, {1, 2, 2, 1}};
+        soffset_array = gko::Array<int>{this->ref, {0, 1, 1, 0}};
+        roffset_array = gko::Array<int>{this->ref, {0, 1, 3, 5}};
+        ref_array = gko::Array<TypeParam>{this->ref, {2, 2, 3, 1, 2, 5}};
+    } else if (my_rank == 1) {
+        recv_array = gko::Array<TypeParam>{this->ref, {0, 0, 0, 0, 0, 0}};
+        send_array = gko::Array<TypeParam>{this->ref, {2, 3, 1, 2}};
+        scounts_array = gko::Array<int>{this->ref, {2, 2, 1, 2}};
+        rcounts_array = gko::Array<int>{this->ref, {2, 2, 2, 0}};
+        soffset_array = gko::Array<int>{this->ref, {0, 1, 1, 0}};
+        roffset_array = gko::Array<int>{this->ref, {0, 2, 4, 5}};
+        ref_array = gko::Array<TypeParam>{this->ref, {3, 1, 3, 1, 3, 1}};
+    } else if (my_rank == 2) {
+        recv_array = gko::Array<TypeParam>{this->ref, {0, 0, 0, 0}};
+        send_array = gko::Array<TypeParam>{this->ref, {2, 3, 1, 2}};
+        scounts_array = gko::Array<int>{this->ref, {2, 2, 1, 1}};
+        rcounts_array = gko::Array<int>{this->ref, {1, 1, 1, 1}};
+        soffset_array = gko::Array<int>{this->ref, {2, 1, 1, 1}};
+        roffset_array = gko::Array<int>{this->ref, {0, 1, 2, 3}};
+        ref_array = gko::Array<TypeParam>{this->ref, {3, 3, 3, 3}};
+    } else if (my_rank == 3) {
+        recv_array = gko::Array<TypeParam>{this->ref, {0, 0, 0, 0}};
+        send_array = gko::Array<TypeParam>{this->ref, {5, 3, 3, -2}};
+        scounts_array = gko::Array<int>{this->ref, {1, 0, 1, 0}};
+        rcounts_array = gko::Array<int>{this->ref, {0, 2, 1, 0}};
+        soffset_array = gko::Array<int>{this->ref, {0, 1, 1, 0}};
+        roffset_array = gko::Array<int>{this->ref, {0, 1, 3, 3}};
+        ref_array = gko::Array<TypeParam>{this->ref, {0, 2, 3, 3}};
+    }
+
+    auto req =
+        comm.i_all_to_all_v(send_array.get_data(), scounts_array.get_data(),
+                            soffset_array.get_data(), recv_array.get_data(),
+                            rcounts_array.get_data(), roffset_array.get_data());
+
+    req.wait();
+    GKO_ASSERT_ARRAY_EQ(recv_array, ref_array);
+}
+
+
 TYPED_TEST(MpiBindings, CanScanValues)
 {
     auto comm = gko::mpi::communicator(MPI_COMM_WORLD);
@@ -666,9 +1453,54 @@ TYPED_TEST(MpiBindings, CanScanValues)
     } else if (my_rank == 3) {
         data = 6;
     }
+
     comm.scan(&data, &sum, 1, MPI_SUM);
     comm.scan(&data, &max, 1, MPI_MAX);
     comm.scan(&data, &min, 1, MPI_MIN);
+
+    if (my_rank == 0) {
+        EXPECT_EQ(sum, TypeParam{3});
+        EXPECT_EQ(max, TypeParam{3});
+        EXPECT_EQ(min, TypeParam{3});
+    } else if (my_rank == 1) {
+        EXPECT_EQ(sum, TypeParam{8});
+        EXPECT_EQ(max, TypeParam{5});
+        EXPECT_EQ(min, TypeParam{3});
+    } else if (my_rank == 2) {
+        EXPECT_EQ(sum, TypeParam{10});
+        EXPECT_EQ(max, TypeParam{5});
+        EXPECT_EQ(min, TypeParam{2});
+    } else if (my_rank == 3) {
+        EXPECT_EQ(sum, TypeParam{16});
+        EXPECT_EQ(max, TypeParam{6});
+        EXPECT_EQ(min, TypeParam{2});
+    }
+}
+
+
+TYPED_TEST(MpiBindings, CanNonBlockingScanValues)
+{
+    auto comm = gko::mpi::communicator(MPI_COMM_WORLD);
+    auto my_rank = comm.rank();
+    auto num_ranks = comm.size();
+    TypeParam data, sum, max, min;
+    if (my_rank == 0) {
+        data = 3;
+    } else if (my_rank == 1) {
+        data = 5;
+    } else if (my_rank == 2) {
+        data = 2;
+    } else if (my_rank == 3) {
+        data = 6;
+    }
+
+    auto req1 = comm.i_scan(&data, &sum, 1, MPI_SUM);
+    auto req2 = comm.i_scan(&data, &max, 1, MPI_MAX);
+    auto req3 = comm.i_scan(&data, &min, 1, MPI_MIN);
+
+    req1.wait();
+    req2.wait();
+    req3.wait();
     if (my_rank == 0) {
         EXPECT_EQ(sum, TypeParam{3});
         EXPECT_EQ(max, TypeParam{3});
diff --git a/third_party/gtest/gtest_mpi_listener.cpp b/core/test/mpi/gtest/mpi_listener.cpp
similarity index 100%
rename from third_party/gtest/gtest_mpi_listener.cpp
rename to core/test/mpi/gtest/mpi_listener.cpp
index 41e05a908d7..d74a77040aa 100644
--- a/third_party/gtest/gtest_mpi_listener.cpp
+++ b/core/test/mpi/gtest/mpi_listener.cpp
@@ -39,8 +39,8 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  *
  *******************************************************************************/
 
-#include <mpi.h>
 #include <cassert>
+#include <mpi.h>
 #include <sstream>
 #include <string>
 #include <vector>
diff --git a/include/ginkgo/core/base/mpi.hpp b/include/ginkgo/core/base/mpi.hpp
index b99081445ea..4d6df9544ab 100644
--- a/include/ginkgo/core/base/mpi.hpp
+++ b/include/ginkgo/core/base/mpi.hpp
@@ -381,7 +381,10 @@ class communicator {
      * This function is used to synchronize the ranks in the communicator.
      * Calls MPI_Barrier
      */
-    void synchronize() const { GKO_ASSERT_NO_MPI_ERRORS(MPI_Barrier(get())); }
+    void synchronize() const
+    {
+        GKO_ASSERT_NO_MPI_ERRORS(MPI_Barrier(this->get()));
+    }
 
     /**
      * Send (Blocking) data from calling process to destination rank.
@@ -395,9 +398,9 @@ class communicator {
     void send(const SendType* send_buffer, const int send_count,
               const int destination_rank, const int send_tag) const
     {
-        GKO_ASSERT_NO_MPI_ERRORS(MPI_Send(send_buffer, send_count,
-                                          type_impl<SendType>::get_type(),
-                                          destination_rank, send_tag, get()));
+        GKO_ASSERT_NO_MPI_ERRORS(
+            MPI_Send(send_buffer, send_count, type_impl<SendType>::get_type(),
+                     destination_rank, send_tag, this->get()));
     }
 
     /**
@@ -418,17 +421,17 @@ class communicator {
         request req;
         GKO_ASSERT_NO_MPI_ERRORS(
             MPI_Isend(send_buffer, send_count, type_impl<SendType>::get_type(),
-                      destination_rank, send_tag, get(), req.get()));
+                      destination_rank, send_tag, this->get(), req.get()));
         return req;
     }
 
     /**
      * Receive data from source rank.
      *
-     * @param recv_buffer  the buffer to send
-     * @param recv_count  the number of elements to send
-     * @param source_rank  the rank to send the data to
-     * @param recv_tag  the tag for the send call
+     * @param recv_buffer  the buffer to receive
+     * @param recv_count  the number of elements to receive
+     * @param source_rank  the rank to receive the data from
+     * @param recv_tag  the tag for the recv call
      *
      * @return  the status of completion of this call
      */
@@ -439,7 +442,7 @@ class communicator {
         status st;
         GKO_ASSERT_NO_MPI_ERRORS(
             MPI_Recv(recv_buffer, recv_count, type_impl<RecvType>::get_type(),
-                     source_rank, recv_tag, get(), st.get()));
+                     source_rank, recv_tag, this->get(), st.get()));
         return st;
     }
 
@@ -447,12 +450,11 @@ class communicator {
      * Receive (Non-blocking, Immediate return) data from source rank.
      *
      * @param recv_buffer  the buffer to send
-     * @param recv_count  the number of elements to send
-     * @param source_rank  the rank to send the data to
-     * @param recv_tag  the tag for the send call
-     * @param req  the request handle for the send call
+     * @param recv_count  the number of elements to receive
+     * @param source_rank  the rank to receive the data from
+     * @param recv_tag  the tag for the recv call
      *
-     * @return  the request handle for the send call
+     * @return  the request handle for the recv call
      */
     template <typename RecvType>
     request i_recv(RecvType* recv_buffer, const int recv_count,
@@ -461,7 +463,7 @@ class communicator {
         request req;
         GKO_ASSERT_NO_MPI_ERRORS(
             MPI_Irecv(recv_buffer, recv_count, type_impl<RecvType>::get_type(),
-                      source_rank, recv_tag, get(), req.get()));
+                      source_rank, recv_tag, this->get(), req.get()));
         return req;
     }
 
@@ -477,7 +479,27 @@ class communicator {
     {
         GKO_ASSERT_NO_MPI_ERRORS(MPI_Bcast(buffer, count,
                                            type_impl<BroadcastType>::get_type(),
-                                           root_rank, get()));
+                                           root_rank, this->get()));
+    }
+
+    /**
+     * (Non-blocking) Broadcast data from calling process to all ranks in the
+     * communicator
+     *
+     * @param buffer  the buffer to broadcsat
+     * @param count  the number of elements to broadcast
+     * @param root_rank  the rank to broadcast from
+     *
+     * @return  the request handle for the call
+     */
+    template <typename BroadcastType>
+    request i_broadcast(BroadcastType* buffer, int count, int root_rank) const
+    {
+        request req;
+        GKO_ASSERT_NO_MPI_ERRORS(
+            MPI_Ibcast(buffer, count, type_impl<BroadcastType>::get_type(),
+                       root_rank, this->get(), req.get()));
+        return req;
     }
 
     /**
@@ -495,12 +517,12 @@ class communicator {
     {
         GKO_ASSERT_NO_MPI_ERRORS(MPI_Reduce(send_buffer, recv_buffer, count,
                                             type_impl<ReduceType>::get_type(),
-                                            operation, root_rank, get()));
+                                            operation, root_rank, this->get()));
     }
 
     /**
-     * Reduce data into root from all calling processes on the same
-     * communicator.
+     * (Non-blocking) Reduce data into root from all calling processes on the
+     * same communicator.
      *
      * @param send_buffer  the buffer to reduce
      * @param recv_buffer  the reduced result
@@ -516,13 +538,13 @@ class communicator {
         request req;
         GKO_ASSERT_NO_MPI_ERRORS(MPI_Ireduce(
             send_buffer, recv_buffer, count, type_impl<ReduceType>::get_type(),
-            operation, root_rank, get(), req.get()));
+            operation, root_rank, this->get(), req.get()));
         return req;
     }
 
     /**
-     * Reduce data from all calling processes from all calling processes on same
-     * communicator.
+     * (In-place) Reduce data from all calling processes from all calling
+     * processes on same communicator.
      *
      * @param recv_buffer  the data to reduce and the reduced result
      * @param count  the number of elements to reduce
@@ -531,14 +553,14 @@ class communicator {
     template <typename ReduceType>
     void all_reduce(ReduceType* recv_buffer, int count, MPI_Op operation) const
     {
-        GKO_ASSERT_NO_MPI_ERRORS(
-            MPI_Allreduce(in_place<ReduceType>(), recv_buffer, count,
-                          type_impl<ReduceType>::get_type(), operation, get()));
+        GKO_ASSERT_NO_MPI_ERRORS(MPI_Allreduce(
+            in_place<ReduceType>(), recv_buffer, count,
+            type_impl<ReduceType>::get_type(), operation, this->get()));
     }
 
     /**
-     * Reduce data from all calling processes from all calling processes on same
-     * communicator.
+     * (In-place, non-blocking) Reduce data from all calling processes from all
+     * calling processes on same communicator.
      *
      * @param recv_buffer  the data to reduce and the reduced result
      * @param count  the number of elements to reduce
@@ -551,9 +573,10 @@ class communicator {
                          MPI_Op operation) const
     {
         request req;
-        GKO_ASSERT_NO_MPI_ERRORS(MPI_Iallreduce(
-            in_place<ReduceType>(), recv_buffer, count,
-            type_impl<ReduceType>::get_type(), operation, get(), req.get()));
+        GKO_ASSERT_NO_MPI_ERRORS(
+            MPI_Iallreduce(in_place<ReduceType>(), recv_buffer, count,
+                           type_impl<ReduceType>::get_type(), operation,
+                           this->get(), req.get()));
         return req;
     }
 
@@ -570,9 +593,9 @@ class communicator {
     void all_reduce(const ReduceType* send_buffer, ReduceType* recv_buffer,
                     int count, MPI_Op operation) const
     {
-        GKO_ASSERT_NO_MPI_ERRORS(
-            MPI_Allreduce(send_buffer, recv_buffer, count,
-                          type_impl<ReduceType>::get_type(), operation, get()));
+        GKO_ASSERT_NO_MPI_ERRORS(MPI_Allreduce(
+            send_buffer, recv_buffer, count, type_impl<ReduceType>::get_type(),
+            operation, this->get()));
     }
 
     /**
@@ -593,7 +616,7 @@ class communicator {
         request req;
         GKO_ASSERT_NO_MPI_ERRORS(MPI_Iallreduce(
             send_buffer, recv_buffer, count, type_impl<ReduceType>::get_type(),
-            operation, get(), req.get()));
+            operation, this->get(), req.get()));
         return req;
     }
 
@@ -614,7 +637,32 @@ class communicator {
         GKO_ASSERT_NO_MPI_ERRORS(
             MPI_Gather(send_buffer, send_count, type_impl<SendType>::get_type(),
                        recv_buffer, recv_count, type_impl<RecvType>::get_type(),
-                       root_rank, get()));
+                       root_rank, this->get()));
+    }
+
+    /**
+     * (Non-blocking) Gather data onto the root rank from all ranks in the
+     * communicator.
+     *
+     * @param send_buffer  the buffer to gather from
+     * @param send_count  the number of elements to send
+     * @param recv_buffer  the buffer to gather into
+     * @param recv_count  the number of elements to receive
+     * @param root_rank  the rank to gather into
+     *
+     * @return  the request handle for the call
+     */
+    template <typename SendType, typename RecvType>
+    request i_gather(const SendType* send_buffer, const int send_count,
+                     RecvType* recv_buffer, const int recv_count,
+                     int root_rank) const
+    {
+        request req;
+        GKO_ASSERT_NO_MPI_ERRORS(MPI_Igather(
+            send_buffer, send_count, type_impl<SendType>::get_type(),
+            recv_buffer, recv_count, type_impl<RecvType>::get_type(), root_rank,
+            this->get(), req.get()));
+        return req;
     }
 
     /**
@@ -636,7 +684,34 @@ class communicator {
         GKO_ASSERT_NO_MPI_ERRORS(MPI_Gatherv(
             send_buffer, send_count, type_impl<SendType>::get_type(),
             recv_buffer, recv_counts, displacements,
-            type_impl<RecvType>::get_type(), root_rank, get()));
+            type_impl<RecvType>::get_type(), root_rank, this->get()));
+    }
+
+    /**
+     * (Non-blocking) Gather data onto the root rank from all ranks in the
+     * communicator with offsets.
+     *
+     * @param send_buffer  the buffer to gather from
+     * @param send_count  the number of elements to send
+     * @param recv_buffer  the buffer to gather into
+     * @param recv_count  the number of elements to receive
+     * @param displacements  the offsets for the buffer
+     * @param root_rank  the rank to gather into
+     *
+     * @return  the request handle for the call
+     */
+    template <typename SendType, typename RecvType>
+    request i_gather_v(const SendType* send_buffer, const int send_count,
+                       RecvType* recv_buffer, const int* recv_counts,
+                       const int* displacements, int root_rank) const
+    {
+        request req;
+        GKO_ASSERT_NO_MPI_ERRORS(MPI_Igatherv(
+            send_buffer, send_count, type_impl<SendType>::get_type(),
+            recv_buffer, recv_counts, displacements,
+            type_impl<RecvType>::get_type(), root_rank, this->get(),
+            req.get()));
+        return req;
     }
 
     /**
@@ -653,7 +728,31 @@ class communicator {
     {
         GKO_ASSERT_NO_MPI_ERRORS(MPI_Allgather(
             send_buffer, send_count, type_impl<SendType>::get_type(),
-            recv_buffer, recv_count, type_impl<RecvType>::get_type(), get()));
+            recv_buffer, recv_count, type_impl<RecvType>::get_type(),
+            this->get()));
+    }
+
+    /**
+     * (Non-blocking) Gather data onto all ranks from all ranks in the
+     * communicator.
+     *
+     * @param send_buffer  the buffer to gather from
+     * @param send_count  the number of elements to send
+     * @param recv_buffer  the buffer to gather into
+     * @param recv_count  the number of elements to receive
+     *
+     * @return  the request handle for the call
+     */
+    template <typename SendType, typename RecvType>
+    request i_all_gather(const SendType* send_buffer, const int send_count,
+                         RecvType* recv_buffer, const int recv_count) const
+    {
+        request req;
+        GKO_ASSERT_NO_MPI_ERRORS(MPI_Iallgather(
+            send_buffer, send_count, type_impl<SendType>::get_type(),
+            recv_buffer, recv_count, type_impl<RecvType>::get_type(),
+            this->get(), req.get()));
+        return req;
     }
 
     /**
@@ -672,7 +771,31 @@ class communicator {
         GKO_ASSERT_NO_MPI_ERRORS(MPI_Scatter(
             send_buffer, send_count, type_impl<SendType>::get_type(),
             recv_buffer, recv_count, type_impl<RecvType>::get_type(), root_rank,
-            get()));
+            this->get()));
+    }
+
+    /**
+     * (Non-blocking) Scatter data from root rank to all ranks in the
+     * communicator.
+     *
+     * @param send_buffer  the buffer to gather from
+     * @param send_count  the number of elements to send
+     * @param recv_buffer  the buffer to gather into
+     * @param recv_count  the number of elements to receive
+     *
+     * @return  the request handle for the call
+     */
+    template <typename SendType, typename RecvType>
+    request i_scatter(const SendType* send_buffer, const int send_count,
+                      RecvType* recv_buffer, const int recv_count,
+                      int root_rank) const
+    {
+        request req;
+        GKO_ASSERT_NO_MPI_ERRORS(MPI_Iscatter(
+            send_buffer, send_count, type_impl<SendType>::get_type(),
+            recv_buffer, recv_count, type_impl<RecvType>::get_type(), root_rank,
+            this->get(), req.get()));
+        return req;
     }
 
     /**
@@ -694,11 +817,38 @@ class communicator {
         GKO_ASSERT_NO_MPI_ERRORS(MPI_Scatterv(
             send_buffer, send_counts, displacements,
             type_impl<SendType>::get_type(), recv_buffer, recv_count,
-            type_impl<RecvType>::get_type(), root_rank, get()));
+            type_impl<RecvType>::get_type(), root_rank, this->get()));
     }
 
     /**
-     * Communicate data from all ranks to all other ranks in place
+     * (Non-blocking) Scatter data from root rank to all ranks in the
+     * communicator with offsets.
+     *
+     * @param send_buffer  the buffer to gather from
+     * @param send_count  the number of elements to send
+     * @param recv_buffer  the buffer to gather into
+     * @param recv_count  the number of elements to receive
+     * @param displacements  the offsets for the buffer
+     * @param comm  the communicator
+     *
+     * @return  the request handle for the call
+     */
+    template <typename SendType, typename RecvType>
+    request i_scatter_v(const SendType* send_buffer, const int* send_counts,
+                        const int* displacements, RecvType* recv_buffer,
+                        const int recv_count, int root_rank) const
+    {
+        request req;
+        GKO_ASSERT_NO_MPI_ERRORS(
+            MPI_Iscatterv(send_buffer, send_counts, displacements,
+                          type_impl<SendType>::get_type(), recv_buffer,
+                          recv_count, type_impl<RecvType>::get_type(),
+                          root_rank, this->get(), req.get()));
+        return req;
+    }
+
+    /**
+     * (In-place) Communicate data from all ranks to all other ranks in place
      * (MPI_Alltoall). See MPI documentation for more details.
      *
      * @param buffer  the buffer to send and the buffer receive
@@ -713,12 +863,13 @@ class communicator {
     {
         GKO_ASSERT_NO_MPI_ERRORS(MPI_Alltoall(
             in_place<RecvType>(), recv_count, type_impl<RecvType>::get_type(),
-            recv_buffer, recv_count, type_impl<RecvType>::get_type(), get()));
+            recv_buffer, recv_count, type_impl<RecvType>::get_type(),
+            this->get()));
     }
 
     /**
-     * Communicate data from all ranks to all other ranks in place
-     * (MPI_Alltoall). See MPI documentation for more details.
+     * (In-place, Non-blocking) Communicate data from all ranks to all other
+     * ranks in place (MPI_Alltoall). See MPI documentation for more details.
      *
      * @param buffer  the buffer to send and the buffer receive
      * @param recv_count  the number of elements to receive
@@ -735,8 +886,8 @@ class communicator {
         request req;
         GKO_ASSERT_NO_MPI_ERRORS(MPI_Ialltoall(
             in_place<RecvType>(), recv_count, type_impl<RecvType>::get_type(),
-            recv_buffer, recv_count, type_impl<RecvType>::get_type(), get(),
-            req.get()));
+            recv_buffer, recv_count, type_impl<RecvType>::get_type(),
+            this->get(), req.get()));
         return req;
     }
 
@@ -755,12 +906,13 @@ class communicator {
     {
         GKO_ASSERT_NO_MPI_ERRORS(MPI_Alltoall(
             send_buffer, send_count, type_impl<SendType>::get_type(),
-            recv_buffer, recv_count, type_impl<RecvType>::get_type(), get()));
+            recv_buffer, recv_count, type_impl<RecvType>::get_type(),
+            this->get()));
     }
 
     /**
-     * Communicate data from all ranks to all other ranks (MPI_Alltoall).
-     * See MPI documentation for more details.
+     * (Non-blocking) Communicate data from all ranks to all other ranks
+     * (MPI_Alltoall). See MPI documentation for more details.
      *
      * @param send_buffer  the buffer to send
      * @param send_count  the number of elements to send
@@ -776,8 +928,8 @@ class communicator {
         request req;
         GKO_ASSERT_NO_MPI_ERRORS(MPI_Ialltoall(
             send_buffer, send_count, type_impl<SendType>::get_type(),
-            recv_buffer, recv_count, type_impl<RecvType>::get_type(), get(),
-            req.get()));
+            recv_buffer, recv_count, type_impl<RecvType>::get_type(),
+            this->get(), req.get()));
         return req;
     }
 
@@ -801,7 +953,7 @@ class communicator {
         GKO_ASSERT_NO_MPI_ERRORS(MPI_Alltoallv(
             send_buffer, send_counts, send_offsets,
             type_impl<SendType>::get_type(), recv_buffer, recv_counts,
-            recv_offsets, type_impl<RecvType>::get_type(), get()));
+            recv_offsets, type_impl<RecvType>::get_type(), this->get()));
     }
 
     /**
@@ -827,7 +979,8 @@ class communicator {
         GKO_ASSERT_NO_MPI_ERRORS(MPI_Ialltoallv(
             send_buffer, send_counts, send_offsets,
             type_impl<SendType>::get_type(), recv_buffer, recv_counts,
-            recv_offsets, type_impl<RecvType>::get_type(), get(), req.get()));
+            recv_offsets, type_impl<RecvType>::get_type(), this->get(),
+            req.get()));
         return req;
     }
 
@@ -846,7 +999,7 @@ class communicator {
     {
         GKO_ASSERT_NO_MPI_ERRORS(MPI_Scan(send_buffer, recv_buffer, count,
                                           type_impl<ScanType>::get_type(),
-                                          operation, get()));
+                                          operation, this->get()));
     }
 
     /**
@@ -867,7 +1020,7 @@ class communicator {
         request req;
         GKO_ASSERT_NO_MPI_ERRORS(MPI_Iscan(send_buffer, recv_buffer, count,
                                            type_impl<ScanType>::get_type(),
-                                           operation, get(), req.get()));
+                                           operation, this->get(), req.get()));
         return req;
     }
 
@@ -889,7 +1042,7 @@ class communicator {
         MPI_Comm local_comm;
         int rank;
         GKO_ASSERT_NO_MPI_ERRORS(MPI_Comm_split_type(
-            get(), MPI_COMM_TYPE_SHARED, 0, MPI_INFO_NULL, &local_comm));
+            this->get(), MPI_COMM_TYPE_SHARED, 0, MPI_INFO_NULL, &local_comm));
         GKO_ASSERT_NO_MPI_ERRORS(MPI_Comm_rank(local_comm, &rank));
         MPI_Comm_free(&local_comm);
         return rank;
@@ -898,7 +1051,7 @@ class communicator {
     int get_num_ranks() const
     {
         int size = 1;
-        GKO_ASSERT_NO_MPI_ERRORS(MPI_Comm_size(get(), &size));
+        GKO_ASSERT_NO_MPI_ERRORS(MPI_Comm_size(this->get(), &size));
         return size;
     }
 
@@ -1025,9 +1178,10 @@ class window {
      * object.
      *
      * @param rank  the target rank.
+     * @param lock_t  the type of the lock: shared or exclusive
      * @param assert  the optimization level. 0 is always valid.
      */
-    void lock(int rank, int assert = 0, lock_type lock_t = lock_type::shared)
+    void lock(int rank, lock_type lock_t = lock_type::shared, int assert = 0)
     {
         if (lock_t == lock_type::shared) {
             GKO_ASSERT_NO_MPI_ERRORS(
@@ -1143,7 +1297,7 @@ class window {
         GKO_ASSERT_NO_MPI_ERRORS(
             MPI_Put(origin_buffer, origin_count, type_impl<PutType>::get_type(),
                     target_rank, target_disp, target_count,
-                    type_impl<PutType>::get_type(), get_window()));
+                    type_impl<PutType>::get_type(), this->get_window()));
     }
 
     /**
@@ -1166,7 +1320,54 @@ class window {
         GKO_ASSERT_NO_MPI_ERRORS(MPI_Rput(
             origin_buffer, origin_count, type_impl<PutType>::get_type(),
             target_rank, target_disp, target_count,
-            type_impl<PutType>::get_type(), get_window(), req.get()));
+            type_impl<PutType>::get_type(), this->get_window(), req.get()));
+        return req;
+    }
+
+    /**
+     * Accumulate data into the target window.
+     *
+     * @param origin_buffer  the buffer to send
+     * @param origin_count  the number of elements to put
+     * @param target_rank  the rank to put the data to
+     * @param target_disp  the displacement at the target window
+     * @param target_count  the request handle for the send call
+     * @param operation  the reduce operation. See @MPI_Op
+     */
+    template <typename PutType>
+    void accumulate(const PutType* origin_buffer, const int origin_count,
+                    const int target_rank, const unsigned int target_disp,
+                    const int target_count, MPI_Op operation) const
+    {
+        GKO_ASSERT_NO_MPI_ERRORS(MPI_Accumulate(
+            origin_buffer, origin_count, type_impl<PutType>::get_type(),
+            target_rank, target_disp, target_count,
+            type_impl<PutType>::get_type(), operation, this->get_window()));
+    }
+
+    /**
+     * (Non-blocking) Accumulate data into the target window.
+     *
+     * @param origin_buffer  the buffer to send
+     * @param origin_count  the number of elements to put
+     * @param target_rank  the rank to put the data to
+     * @param target_disp  the displacement at the target window
+     * @param target_count  the request handle for the send call
+     * @param operation  the reduce operation. See @MPI_Op
+     *
+     * @return  the request handle for the send call
+     */
+    template <typename PutType>
+    request r_accumulate(const PutType* origin_buffer, const int origin_count,
+                         const int target_rank, const unsigned int target_disp,
+                         const int target_count, MPI_Op operation) const
+    {
+        request req;
+        GKO_ASSERT_NO_MPI_ERRORS(MPI_Raccumulate(
+            origin_buffer, origin_count, type_impl<PutType>::get_type(),
+            target_rank, target_disp, target_count,
+            type_impl<PutType>::get_type(), operation, this->get_window(),
+            req.get()));
         return req;
     }
 
@@ -1187,7 +1388,7 @@ class window {
         GKO_ASSERT_NO_MPI_ERRORS(
             MPI_Get(origin_buffer, origin_count, type_impl<GetType>::get_type(),
                     target_rank, target_disp, target_count,
-                    type_impl<GetType>::get_type(), get_window()));
+                    type_impl<GetType>::get_type(), this->get_window()));
     }
 
     /**
@@ -1210,10 +1411,85 @@ class window {
         GKO_ASSERT_NO_MPI_ERRORS(MPI_Rget(
             origin_buffer, origin_count, type_impl<GetType>::get_type(),
             target_rank, target_disp, target_count,
-            type_impl<GetType>::get_type(), get_window(), req.get()));
+            type_impl<GetType>::get_type(), this->get_window(), req.get()));
+        return req;
+    }
+
+    /**
+     * Get Accumulate data from the target window.
+     *
+     * @param origin_buffer  the buffer to send
+     * @param origin_count  the number of elements to get
+     * @param result_buffer  the buffer to receive the target data
+     * @param result_count  the number of elements to get
+     * @param target_rank  the rank to get the data from
+     * @param target_disp  the displacement at the target window
+     * @param target_count  the request handle for the send call
+     * @param operation  the reduce operation. See @MPI_Op
+     */
+    template <typename GetType>
+    void get_accumulate(GetType* origin_buffer, const int origin_count,
+                        GetType* result_buffer, const int result_count,
+                        const int target_rank, const unsigned int target_disp,
+                        const int target_count, MPI_Op operation) const
+    {
+        GKO_ASSERT_NO_MPI_ERRORS(MPI_Get_accumulate(
+            origin_buffer, origin_count, type_impl<GetType>::get_type(),
+            result_buffer, result_count, type_impl<GetType>::get_type(),
+            target_rank, target_disp, target_count,
+            type_impl<GetType>::get_type(), operation, this->get_window()));
+    }
+
+    /**
+     * (Non-blocking) Get Accumulate data (with handle) from the target window.
+     *
+     * @param origin_buffer  the buffer to send
+     * @param origin_count  the number of elements to get
+     * @param result_buffer  the buffer to receive the target data
+     * @param result_count  the number of elements to get
+     * @param target_rank  the rank to get the data from
+     * @param target_disp  the displacement at the target window
+     * @param target_count  the request handle for the send call
+     * @param operation  the reduce operation. See @MPI_Op
+     *
+     * @return  the request handle for the send call
+     */
+    template <typename GetType>
+    request r_get_accumulate(GetType* origin_buffer, const int origin_count,
+                             GetType* result_buffer, const int result_count,
+                             const int target_rank,
+                             const unsigned int target_disp,
+                             const int target_count, MPI_Op operation) const
+    {
+        request req;
+        GKO_ASSERT_NO_MPI_ERRORS(MPI_Rget_accumulate(
+            origin_buffer, origin_count, type_impl<GetType>::get_type(),
+            result_buffer, result_count, type_impl<GetType>::get_type(),
+            target_rank, target_disp, target_count,
+            type_impl<GetType>::get_type(), operation, this->get_window(),
+            req.get()));
         return req;
     }
 
+    /**
+     * Fetch and operate on data from the target window (An optimized version of
+     * Get_accumulate).
+     *
+     * @param origin_buffer  the buffer to send
+     * @param target_rank  the rank to get the data from
+     * @param target_disp  the displacement at the target window
+     * @param operation  the reduce operation. See @MPI_Op
+     */
+    template <typename GetType>
+    void fetch_and_op(GetType* origin_buffer, GetType* result_buffer,
+                      const int target_rank, const unsigned int target_disp,
+                      MPI_Op operation) const
+    {
+        GKO_ASSERT_NO_MPI_ERRORS(MPI_Fetch_and_op(
+            origin_buffer, result_buffer, type_impl<GetType>::get_type(),
+            target_rank, target_disp, operation, this->get_window()));
+    }
+
 private:
     MPI_Win window_;
 };
diff --git a/third_party/gtest/CMakeLists.txt b/third_party/gtest/CMakeLists.txt
index 52b49207596..99ac3be5dfc 100644
--- a/third_party/gtest/CMakeLists.txt
+++ b/third_party/gtest/CMakeLists.txt
@@ -23,12 +23,3 @@ set_target_properties(gtest gtest_main PROPERTIES
 # by default, the outdated targets are not being exported
 add_library(GTest::Main ALIAS gtest_main)
 add_library(GTest::GTest ALIAS gtest)
-if(GINKGO_BUILD_MPI)
-    add_library(gtest_mpi_main "")
-    target_sources(gtest_mpi_main
-        PRIVATE
-        gtest_mpi_listener.cpp)
-    find_package(MPI REQUIRED)
-    target_link_libraries(gtest_mpi_main PRIVATE GTest::GTest MPI::MPI_CXX)
-    add_library(GTest::MPI_main ALIAS gtest_mpi_main)
-endif()