diff --git a/benchmark/blas/blas.cpp b/benchmark/blas/blas.cpp
index 66c71c21ae8..8b97a42fdd3 100644
--- a/benchmark/blas/blas.cpp
+++ b/benchmark/blas/blas.cpp
@@ -472,6 +472,12 @@ void apply_blas(const char *operation_name, std::shared_ptr<gko::Executor> exec,
     } catch (const std::exception &e) {
         add_or_set_member(test_case["blas"][operation_name], "completed", false,
                           allocator);
+        if (FLAGS_keep_errors) {
+            rapidjson::Value msg_value;
+            msg_value.SetString(e.what(), allocator);
+            add_or_set_member(test_case["blas"][operation_name], "error",
+                              msg_value, allocator);
+        }
         std::cerr << "Error when processing test case " << test_case << "\n"
                   << "what(): " << e.what() << std::endl;
     }
diff --git a/benchmark/conversions/conversions.cpp b/benchmark/conversions/conversions.cpp
index 9e48f6ca327..18313bf26db 100644
--- a/benchmark/conversions/conversions.cpp
+++ b/benchmark/conversions/conversions.cpp
@@ -98,6 +98,12 @@ void convert_matrix(const gko::LinOp *matrix_from, const char *format_to,
     } catch (const std::exception &e) {
         add_or_set_member(test_case["conversions"][conversion_name],
                           "completed", false, allocator);
+        if (FLAGS_keep_errors) {
+            rapidjson::Value msg_value;
+            msg_value.SetString(e.what(), allocator);
+            add_or_set_member(test_case["conversions"][conversion_name],
+                              "error", msg_value, allocator);
+        }
         std::cerr << "Error when processing test case " << test_case << "\n"
                   << "what(): " << e.what() << std::endl;
     }
diff --git a/benchmark/preconditioner/preconditioner.cpp b/benchmark/preconditioner/preconditioner.cpp
index 695c1e78314..002faeeddff 100644
--- a/benchmark/preconditioner/preconditioner.cpp
+++ b/benchmark/preconditioner/preconditioner.cpp
@@ -229,6 +229,12 @@ void run_preconditioner(const char *precond_name,
                           rapidjson::Value(rapidjson::kObjectType), allocator);
         add_or_set_member(test_case["preconditioner"][encoded_name.c_str()],
                           "completed", false, allocator);
+        if (FLAGS_keep_errors) {
+            rapidjson::Value msg_value;
+            msg_value.SetString(e.what(), allocator);
+            add_or_set_member(test_case["preconditioner"][encoded_name.c_str()],
+                              "error", msg_value, allocator);
+        }
         std::cerr << "Error when processing test case " << test_case << "\n"
                   << "what(): " << e.what() << std::endl;
     }
diff --git a/benchmark/solver/solver.cpp b/benchmark/solver/solver.cpp
index d709cf442c5..e4df2948cb9 100644
--- a/benchmark/solver/solver.cpp
+++ b/benchmark/solver/solver.cpp
@@ -509,6 +509,12 @@ void solve_system(const std::string &solver_name,
     } catch (const std::exception &e) {
         add_or_set_member(test_case["solver"][precond_solver_name], "completed",
                           false, allocator);
+        if (FLAGS_keep_errors) {
+            rapidjson::Value msg_value;
+            msg_value.SetString(e.what(), allocator);
+            add_or_set_member(test_case["solver"][precond_solver_name], "error",
+                              msg_value, allocator);
+        }
         std::cerr << "Error when processing test case " << test_case << "\n"
                   << "what(): " << e.what() << std::endl;
     }
diff --git a/benchmark/spmv/spmv.cpp b/benchmark/spmv/spmv.cpp
index 4617bb4d7e6..6d21ddd88e6 100644
--- a/benchmark/spmv/spmv.cpp
+++ b/benchmark/spmv/spmv.cpp
@@ -154,6 +154,12 @@ void apply_spmv(const char *format_name, std::shared_ptr<gko::Executor> exec,
     } catch (const std::exception &e) {
         add_or_set_member(test_case["spmv"][format_name], "completed", false,
                           allocator);
+        if (FLAGS_keep_errors) {
+            rapidjson::Value msg_value;
+            msg_value.SetString(e.what(), allocator);
+            add_or_set_member(test_case["spmv"][format_name], "error",
+                              msg_value, allocator);
+        }
         std::cerr << "Error when processing test case " << test_case << "\n"
                   << "what(): " << e.what() << std::endl;
     }
diff --git a/benchmark/utils/general.hpp b/benchmark/utils/general.hpp
index 84fb5926af4..dca22e7fe27 100644
--- a/benchmark/utils/general.hpp
+++ b/benchmark/utils/general.hpp
@@ -84,6 +84,10 @@ DEFINE_string(double_buffer, "",
 DEFINE_bool(detailed, true,
             "If set, performs several runs to obtain more detailed results");
 
+DEFINE_bool(keep_errors, false,
+            "If set, writes exception messages during the execution into the "
+            "JSON output");
+
 DEFINE_bool(nested_names, false, "If set, separately logs nested operations");
 
 DEFINE_uint32(seed, 42, "Seed used for the random number generator");