diff --git a/benchmark/conversions/conversions.cpp b/benchmark/conversions/conversions.cpp
index 91e7e02c568..403f416d375 100644
--- a/benchmark/conversions/conversions.cpp
+++ b/benchmark/conversions/conversions.cpp
@@ -150,6 +150,11 @@ int main(int argc, char* argv[])
         } catch (std::exception& e) {
             std::cerr << "Error setting up matrix data, what(): " << e.what()
                       << std::endl;
+            if (FLAGS_keep_errors) {
+                rapidjson::Value msg_value;
+                msg_value.SetString(e.what(), allocator);
+                add_or_set_member(test_case, "error", msg_value, allocator);
+            }
             continue;
         }
         std::clog << "Matrix is of size (" << data.size[0] << ", "
diff --git a/benchmark/solver/solver_common.hpp b/benchmark/solver/solver_common.hpp
index 9f55f66e327..d890ece9022 100644
--- a/benchmark/solver/solver_common.hpp
+++ b/benchmark/solver/solver_common.hpp
@@ -676,6 +676,11 @@ void run_solver_benchmarks(std::shared_ptr<gko::Executor> exec,
         } catch (const std::exception& e) {
             std::cerr << "Error setting up solver, what(): " << e.what()
                       << std::endl;
+            if (FLAGS_keep_errors) {
+                rapidjson::Value msg_value;
+                msg_value.SetString(e.what(), allocator);
+                add_or_set_member(test_case, "error", msg_value, allocator);
+            }
         }
     }
 }
diff --git a/benchmark/sparse_blas/sparse_blas.cpp b/benchmark/sparse_blas/sparse_blas.cpp
index a67f6c1e665..5f45aaec2e4 100644
--- a/benchmark/sparse_blas/sparse_blas.cpp
+++ b/benchmark/sparse_blas/sparse_blas.cpp
@@ -207,6 +207,11 @@ int main(int argc, char* argv[])
         } catch (const std::exception& e) {
             std::cerr << "Error setting up matrix data, what(): " << e.what()
                       << std::endl;
+            if (FLAGS_keep_errors) {
+                rapidjson::Value msg_value;
+                msg_value.SetString(e.what(), allocator);
+                add_or_set_member(test_case, "error", msg_value, allocator);
+            }
         }
     }
 
diff --git a/benchmark/spmv/spmv_common.hpp b/benchmark/spmv/spmv_common.hpp
index cd64b61a405..7fe1aad436d 100644
--- a/benchmark/spmv/spmv_common.hpp
+++ b/benchmark/spmv/spmv_common.hpp
@@ -239,6 +239,11 @@ void run_spmv_benchmark(std::shared_ptr<gko::Executor> exec,
         } catch (const std::exception& e) {
             std::cerr << "Error setting up matrix data, what(): " << e.what()
                       << std::endl;
+            if (FLAGS_keep_errors) {
+                rapidjson::Value msg_value;
+                msg_value.SetString(e.what(), allocator);
+                add_or_set_member(test_case, "error", msg_value, allocator);
+            }
         }
     }
 }