diff --git a/core/CMakeLists.txt b/core/CMakeLists.txt
index 4a19ca18962..a81493d2666 100644
--- a/core/CMakeLists.txt
+++ b/core/CMakeLists.txt
@@ -16,6 +16,7 @@ target_sources(ginkgo
     base/mpi.cpp
     base/mtx_io.cpp
     base/perturbation.cpp
+    base/segmented_array.cpp
     base/timer.cpp
     base/version.cpp
     config/property_tree.cpp
diff --git a/core/base/segmented_array.cpp b/core/base/segmented_array.cpp
new file mode 100644
index 00000000000..4fba0851bfc
--- /dev/null
+++ b/core/base/segmented_array.cpp
@@ -0,0 +1,144 @@
+// SPDX-FileCopyrightText: 2017 - 2024 The Ginkgo authors
+//
+// SPDX-License-Identifier: BSD-3-Clause
+
+#include <ginkgo/core/base/segmented_array.hpp>
+
+
+#include "core/base/array_access.hpp"
+#include "core/components/prefix_sum_kernels.hpp"
+
+
+namespace gko {
+namespace {
+
+
+GKO_REGISTER_OPERATION(prefix_sum, components::prefix_sum_nonnegative);
+
+
+}
+
+template <typename T>
+segmented_array<T>::segmented_array(std::shared_ptr<const Executor> exec)
+    : buffer_(exec), offsets_(exec, 1)
+{
+    offsets_.fill(0);
+}
+
+
+array<int64> sizes_to_offsets(const gko::array<int64>& sizes)
+{
+    auto exec = sizes.get_executor();
+    array<int64> offsets(exec, sizes.get_size() + 1);
+    exec->copy(sizes.get_size(), sizes.get_const_data(), offsets.get_data());
+    exec->run(make_prefix_sum(offsets.get_data(), offsets.get_size()));
+    return offsets;
+}
+
+
+template <typename T>
+segmented_array<T> segmented_array<T>::create_from_sizes(
+    const gko::array<int64>& sizes)
+{
+    return create_from_offsets(sizes_to_offsets(sizes));
+}
+
+
+template <typename T>
+segmented_array<T> segmented_array<T>::create_from_sizes(
+    gko::array<T> buffer, const gko::array<int64>& sizes)
+{
+    return create_from_offsets(std::move(buffer), sizes_to_offsets(sizes));
+}
+
+
+template <typename T>
+segmented_array<T> segmented_array<T>::create_from_offsets(
+    gko::array<int64> offsets)
+{
+    GKO_THROW_IF_INVALID(offsets.get_size() > 0,
+                         "The offsets for segmented_arrays require at least "
+                         "one element.");
+    auto size =
+        static_cast<size_type>(get_element(offsets, offsets.get_size() - 1));
+    return create_from_offsets(array<T>{offsets.get_executor(), size},
+                               std::move(offsets));
+}
+
+
+template <typename T>
+segmented_array<T> segmented_array<T>::create_from_offsets(
+    gko::array<T> buffer, gko::array<int64> offsets)
+{
+    GKO_ASSERT_EQ(buffer.get_size(),
+                  get_element(offsets, offsets.get_size() - 1));
+    segmented_array<T> result(buffer.get_executor());
+    result.offsets_ = std::move(offsets);
+    result.buffer_ = std::move(buffer);
+    return result;
+}
+
+
+template <typename T>
+segmented_array<T>::segmented_array(std::shared_ptr<const Executor> exec,
+                                    segmented_array&& other)
+    : segmented_array(exec)
+{
+    *this = std::move(other);
+}
+
+
+template <typename T>
+segmented_array<T>::segmented_array(std::shared_ptr<const Executor> exec,
+                                    const segmented_array& other)
+    : segmented_array(exec)
+{
+    *this = other;
+}
+
+
+template <typename T>
+segmented_array<T>::segmented_array(const segmented_array& other)
+    : segmented_array(other.get_executor())
+{
+    *this = other;
+}
+
+
+template <typename T>
+segmented_array<T>::segmented_array(segmented_array&& other)
+    : segmented_array(other.get_executor())
+{
+    *this = std::move(other);
+}
+
+
+template <typename T>
+segmented_array<T>& segmented_array<T>::operator=(const segmented_array& other)
+{
+    if (this != &other) {
+        buffer_ = other.buffer_;
+        offsets_ = other.offsets_;
+    }
+    return *this;
+}
+
+
+template <typename T>
+segmented_array<T>& segmented_array<T>::operator=(segmented_array&& other)
+{
+    if (this != &other) {
+        buffer_ = std::move(other.buffer_);
+        offsets_ = std::exchange(other.offsets_,
+                                 array<int64>{other.get_executor(), {0}});
+    }
+    return *this;
+}
+
+
+#define GKO_DECLARE_SEGMENTED_ARRAY(_type) class segmented_array<_type>
+
+GKO_INSTANTIATE_FOR_EACH_POD_TYPE(GKO_DECLARE_SEGMENTED_ARRAY);
+
+
+}  // namespace gko
diff --git a/core/base/segmented_array.hpp b/core/base/segmented_array.hpp
new file mode 100644
index 00000000000..a3d8c5dc337
--- /dev/null
+++ b/core/base/segmented_array.hpp
@@ -0,0 +1,57 @@
+// SPDX-FileCopyrightText: 2017 - 2024 The Ginkgo authors
+//
+// SPDX-License-Identifier: BSD-3-Clause
+
+#ifndef GINKGO_SEGMENTED_ARRAY_HPP
+#define GINKGO_SEGMENTED_ARRAY_HPP
+
+
+#include <ginkgo/core/base/segmented_array.hpp>
+
+
+namespace gko {
+
+
+/**
+ * Helper struct storing an array segment
+ *
+ * @tparam T  The value type of the array
+ */
+template <typename T>
+struct array_segment {
+    T* begin;
+    T* end;
+};
+
+
+/**
+ * Helper function to create a device-compatible view of an array segment.
+ */
+template <typename T>
+constexpr array_segment<T> get_array_segment(segmented_array<T>& sarr,
+                                             size_type segment_id)
+{
+    assert(segment_id < sarr.get_segment_count());
+    auto offsets = sarr.get_offsets().get_const_data();
+    auto data = sarr.get_flat_data();
+    return {data + offsets[segment_id], data + offsets[segment_id + 1]};
+}
+
+
+/**
+ * Helper function to create a device-compatible view of a const array segment.
+ */
+template <typename T>
+constexpr array_segment<const T> get_array_segment(
+    const segmented_array<T>& sarr, size_type segment_id)
+{
+    assert(segment_id < sarr.get_segment_count());
+    auto offsets = sarr.get_offsets().get_const_data();
+    auto data = sarr.get_const_flat_data();
+    return {data + offsets[segment_id], data + offsets[segment_id + 1]};
+}
+
+
+}  // namespace gko
+
+#endif  // GINKGO_SEGMENTED_ARRAY_HPP
diff --git a/include/ginkgo/core/base/segmented_array.hpp b/include/ginkgo/core/base/segmented_array.hpp
new file mode 100644
index 00000000000..40b4691a580
--- /dev/null
+++ b/include/ginkgo/core/base/segmented_array.hpp
@@ -0,0 +1,225 @@
+// SPDX-FileCopyrightText: 2017 - 2024 The Ginkgo authors
+//
+// SPDX-License-Identifier: BSD-3-Clause
+
+#pragma once
+#include <numeric>
+
+
+#include <ginkgo/config.hpp>
+#include <ginkgo/core/base/array.hpp>
+
+
+namespace gko {
+
+/**
+ * \brief A minimal interface for a segmented array.
+ *
+ * The segmented array is stored as a flat buffer with an offsets array.
+ * The segment `i` contains the index range `[offset[i], offset[i + 1])` of the
+ * flat buffer.
+ *
+ * \tparam T value type stored in the arrays
+ */
+template <typename T>
+struct segmented_array {
+    /**
+     * Create an empty segmented array
+     *
+     * @param exec  executor for storage arrays
+     */
+    explicit segmented_array(std::shared_ptr<const Executor> exec);
+
+    /**
+     * Creates an uninitialized segmented array with predefined segment sizes.
+     *
+     * @param exec  executor for storage arrays
+     * @param sizes  the sizes of each segment
+     */
+    static segmented_array create_from_sizes(const gko::array<int64>& sizes);
+
+    /**
+     * Creates a segmented array from a flat buffer and segment sizes.
+     *
+     * @param buffer  the flat buffer whose size has to match the sum of sizes
+     * @param sizes  the sizes of each segment
+     */
+    static segmented_array create_from_sizes(gko::array<T> buffer,
+                                             const gko::array<int64>& sizes);
+
+    /**
+     * Creates an uninitialized segmented array from offsets.
+     *
+     * @param offsets  the index offsets for each segment, and the total size of
+     *                 the buffer as last element
+     */
+    static segmented_array create_from_offsets(gko::array<int64> offsets);
+
+    /**
+     * Creates a segmented array from a flat buffer and offsets.
+     *
+     * @param buffer  the flat buffer whose size has to match the last element
+     *                of offsets
+     * @param offsets  the index offsets for each segment, and the total size of
+     *                 the buffer as last element
+     */
+    static segmented_array create_from_offsets(gko::array<T> buffer,
+                                               gko::array<int64> offsets);
+
+    /**
+     * Copies a segmented array to a different executor.
+     *
+     * @param exec  the executor to copy to
+     * @param other  the segmented array to copy from
+     */
+    segmented_array(std::shared_ptr<const Executor> exec,
+                    const segmented_array& other);
+
+    /**
+     * Moves a segmented array to a different executor.
+     *
+     * @param exec  the executor to move to
+     * @param other  the segmented array to move from
+     */
+    segmented_array(std::shared_ptr<const Executor> exec,
+                    segmented_array&& other);
+
+    segmented_array(const segmented_array& other);
+
+    segmented_array(segmented_array&& other) noexcept(false);
+
+    segmented_array& operator=(const segmented_array& other);
+
+    segmented_array& operator=(segmented_array&&) noexcept(false);
+
+    /**
+     * Get the total size of the stored buffer.
+     *
+     * @return  the total size of the stored buffer.
+     */
+    size_type get_size() const;
+
+    /**
+     * Get the number of segments.
+     *
+     * @return  the number of segments
+     */
+    size_type get_segment_count() const;
+
+    /**
+     * Access to the flat buffer.
+     *
+     * @return  the flat buffer
+     */
+    T* get_flat_data();
+
+    /**
+     * Const-access to the flat buffer
+     *
+     * @return  the flat buffer
+     */
+    const T* get_const_flat_data() const;
+
+    /**
+     * Access to the segment offsets.
+     *
+     * @return  the segment offsets
+     */
+    const gko::array<int64>& get_offsets() const;
+
+    /**
+     * Access the executor.
+     *
+     * @return  the executor
+     */
+    std::shared_ptr<const Executor> get_executor() const;
+
+private:
+    gko::array<T> buffer_;
+    gko::array<int64> offsets_;
+};
+
+template <typename T>
+size_type segmented_array<T>::get_size() const
+{
+    return buffer_.get_size();
+}
+
+
+template <typename T>
+size_type segmented_array<T>::get_segment_count() const
+{
+    return offsets_.get_size() ? offsets_.get_size() - 1 : 0;
+}
+
+
+template <typename T>
+T* segmented_array<T>::get_flat_data()
+{
+    return buffer_.get_data();
+}
+
+
+template <typename T>
+const T* segmented_array<T>::get_const_flat_data() const
+{
+    return buffer_.get_const_data();
+}
+
+
+template <typename T>
+const gko::array<int64>& segmented_array<T>::get_offsets() const
+{
+    return offsets_;
+}
+
+
+template <typename T>
+std::shared_ptr<const Executor> segmented_array<T>::get_executor() const
+{
+    return buffer_.get_executor();
+}
+
+
+namespace detail {
+template <typename T>
+struct temporary_clone_helper<segmented_array<T>> {
+    static std::unique_ptr<segmented_array<T>> create(
+        std::shared_ptr<const Executor> exec, segmented_array<T>* ptr,
+        bool copy_data)
+    {
+        if (copy_data) {
+            return std::make_unique<segmented_array<T>>(
+                make_array_view(exec, ptr->get_size(), ptr->get_flat_data()),
+                ptr->get_offsets());
+        } else {
+            return std::make_unique<segmented_array<T>>(std::move(exec),
+                                                        ptr->get_offsets());
+        }
+    }
+};
+
+template <typename T>
+struct temporary_clone_helper<const segmented_array<T>> {
+    static std::unique_ptr<const segmented_array<T>> create(
+        std::shared_ptr<const Executor> exec, const segmented_array<T>* ptr,
+        bool)
+    {
+        return std::make_unique<segmented_array<T>>(
+            make_array_view(exec, ptr->get_size(), ptr->get_const_flat_data()),
+            ptr->get_offsets());
+    }
+};
+
+
+template <typename T>
+class copy_back_deleter<segmented_array<T>>
+    : public copy_back_deleter_from_assignment<segmented_array<T>> {
+public:
+    using copy_back_deleter_from_assignment<
+        segmented_array<T>>::copy_back_deleter_from_assignment;
+};
+
+
+}  // namespace detail
+}  // namespace gko
diff --git a/include/ginkgo/ginkgo.hpp b/include/ginkgo/ginkgo.hpp
index 8a42757c9d4..9052565c9f3 100644
--- a/include/ginkgo/ginkgo.hpp
+++ b/include/ginkgo/ginkgo.hpp
@@ -42,6 +42,7 @@
 #include <ginkgo/core/base/range.hpp>
 #include <ginkgo/core/base/range_accessors.hpp>
 #include <ginkgo/core/base/scoped_device_id_guard.hpp>
+#include <ginkgo/core/base/segmented_array.hpp>
 #include <ginkgo/core/base/std_extensions.hpp>
 #include <ginkgo/core/base/stream.hpp>
 #include <ginkgo/core/base/temporary_clone.hpp>