diff --git a/core/test/utils/assertions.hpp b/core/test/utils/assertions.hpp
index 0a97a0d8e07..15e1d553bbf 100644
--- a/core/test/utils/assertions.hpp
+++ b/core/test/utils/assertions.hpp
@@ -260,7 +260,7 @@ ::testing::AssertionResult matrices_near_impl(
              << second_expression << " is " << err << "\n"
              << "\twhich is larger than " << tolerance_expression
              << " (which is " << tolerance << ")\n";
-        if (num_rows * num_cols <= 25) {
+        if (num_rows <= 10 && num_cols <= 10) {
             fail << first_expression << " is:\n";
             detail::print_matrix(fail, first);
             fail << second_expression << " is:\n";
diff --git a/core/test/utils/matrix_utils_test.cpp b/core/test/utils/matrix_utils_test.cpp
index bf32675247c..7d650e7955a 100644
--- a/core/test/utils/matrix_utils_test.cpp
+++ b/core/test/utils/matrix_utils_test.cpp
@@ -104,11 +104,16 @@ TYPED_TEST(MatrixUtils, MakeHpdMatrixThrowsError)
 
 TYPED_TEST(MatrixUtils, MakeLowerTriangularCorrectly)
 {
+    auto orig_mtx = TestFixture::mtx_type::create(this->exec);
+    orig_mtx->read(this->data);
     gko::test::make_lower_triangular(this->data);
 
     auto mtx = TestFixture::mtx_type::create(this->exec);
     mtx->read(this->data);
     for (gko::size_type i = 0; i < mtx->get_size()[0]; i++) {
+        for (gko::size_type j = 0; j <= i; j++) {
+            ASSERT_EQ(mtx->at(i, j), orig_mtx->at(i, j));
+        }
         for (gko::size_type j = i + 1; j < mtx->get_size()[1]; j++) {
             ASSERT_EQ(mtx->at(i, j), gko::zero<TypeParam>());
         }
@@ -118,6 +123,8 @@ TYPED_TEST(MatrixUtils, MakeLowerTriangularCorrectly)
 
 TYPED_TEST(MatrixUtils, MakeUpperTriangularCorrectly)
 {
+    auto orig_mtx = TestFixture::mtx_type::create(this->exec);
+    orig_mtx->read(this->data);
     gko::test::make_upper_triangular(this->data);
 
     auto mtx = TestFixture::mtx_type::create(this->exec);
@@ -126,6 +133,9 @@ TYPED_TEST(MatrixUtils, MakeUpperTriangularCorrectly)
         for (gko::size_type j = 0; j < i; j++) {
             ASSERT_EQ(mtx->at(i, j), gko::zero<TypeParam>());
         }
+        for (gko::size_type j = i; j < mtx->get_size()[1]; j++) {
+            ASSERT_EQ(mtx->at(i, j), orig_mtx->at(i, j));
+        }
     }
 }
 
@@ -231,7 +241,7 @@ TYPED_TEST(MatrixUtils, MakeHpdMatrixCorrectly)
     using T = typename TestFixture::value_type;
     auto cpy_data = this->data;
 
-    gko::test::make_hpd(this->data);
+    gko::test::make_hpd(this->data, 1.001);
     gko::test::make_hermitian(cpy_data);
     gko::test::make_diag_dominant(cpy_data, 1.001);
 
@@ -266,7 +276,7 @@ TYPED_TEST(MatrixUtils, MakeSpdMatrixCorrectly)
     using T = typename TestFixture::value_type;
     auto cpy_data = this->data;
 
-    gko::test::make_spd(this->data);
+    gko::test::make_spd(this->data, 1.001);
     gko::test::make_symmetric(cpy_data);
     gko::test::make_diag_dominant(cpy_data, 1.001);
 
diff --git a/test/solver/solver.cpp b/test/solver/solver.cpp
index a43f8b79af9..e217473912a 100644
--- a/test/solver/solver.cpp
+++ b/test/solver/solver.cpp
@@ -595,6 +595,8 @@ using SolverTypes =
 TYPED_TEST_SUITE(Solver, SolverTypes, TypenameNameGenerator);
 
 
+// The tolerances we set above don't provide any useful information for float
+#if !(GINKGO_DPCPP_SINGLE_MODE)
 TYPED_TEST(Solver, ApplyIsEquivalentToRef)
 {
     this->forall_matrix_scenarios([&](auto mtx) {
@@ -618,9 +620,9 @@ TYPED_TEST(Solver, AdvancedApplyIsEquivalentToRef)
                 auto alpha = this->gen_scalar();
                 auto beta = this->gen_scalar();
 
-                solver.ref->apply(alpha.ref.get(), b.ref.get(), alpha.ref.get(),
+                solver.ref->apply(alpha.ref.get(), b.ref.get(), beta.ref.get(),
                                   x.ref.get());
-                solver.dev->apply(alpha.dev.get(), b.dev.get(), alpha.dev.get(),
+                solver.dev->apply(alpha.dev.get(), b.dev.get(), beta.dev.get(),
                                   x.dev.get());
 
                 GKO_ASSERT_MTX_NEAR(x.ref, x.dev, this->tol(x));
@@ -630,7 +632,6 @@ TYPED_TEST(Solver, AdvancedApplyIsEquivalentToRef)
 }
 
 
-#if !(GINKGO_DPCPP_SINGLE_MODE)
 TYPED_TEST(Solver, MixedApplyIsEquivalentToRef)
 {
     using MixedVec = typename TestFixture::MixedVec;
@@ -659,9 +660,9 @@ TYPED_TEST(Solver, MixedAdvancedApplyIsEquivalentToRef)
                     auto beta = this->template gen_scalar<MixedVec>();
 
                     solver.ref->apply(alpha.ref.get(), b.ref.get(),
-                                      alpha.ref.get(), x.ref.get());
+                                      beta.ref.get(), x.ref.get());
                     solver.dev->apply(alpha.dev.get(), b.dev.get(),
-                                      alpha.dev.get(), x.dev.get());
+                                      beta.dev.get(), x.dev.get());
 
                     GKO_ASSERT_MTX_NEAR(x.ref, x.dev, this->mixed_tol(x));
                 });