Benchmarks for SQLite write performance (#1)

.gitignore

@@ -0,0 +1,6 @@
+.idea/
+.ipynb_checkpoints/
+cmake-build-debug/
+cmake-build-release/
+build/
+venv/

NOTICE

@@ -0,0 +1,5 @@
+rosbag2
+
+Copyright Holders:
+Copyright (c) 2018, Open Source Robotics Foundation, Inc.
+Copyright (c) 2018, Bosch Software Innovations GmbH

rosbag2_storage_evaluation/CMakeLists.txt

@@ -1,35 +1,81 @@
 cmake_minimum_required(VERSION 3.5)
-project(rosbag2_storage_evaluation)
-
-# Default to C99
-if(NOT CMAKE_C_STANDARD)
-  set(CMAKE_C_STANDARD 99)
-endif()
-
-# Default to C++14
-if(NOT CMAKE_CXX_STANDARD)
-  set(CMAKE_CXX_STANDARD 14)
-endif()
-
-if(CMAKE_COMPILER_IS_GNUCXX OR CMAKE_CXX_COMPILER_ID MATCHES "Clang")
-  add_compile_options(-Wall -Wextra -Wpedantic)
-endif()
-
-# find dependencies
-find_package(ament_cmake REQUIRED)
-# uncomment the following section in order to fill in
-# further dependencies manually.
-# find_package(<dependency> REQUIRED)
-
-if(BUILD_TESTING)
-  find_package(ament_lint_auto REQUIRED)
-  # the following line skips the linter which checks for copyrights
-  # remove the line when a copyright and license is present in all source files
-  set(ament_cmake_copyright_FOUND TRUE)
-  # the following line skips cpplint (only works in a git repo)
-  # remove the line when this package is a git repo
-  set(ament_cmake_cpplint_FOUND TRUE)
-  ament_lint_auto_find_test_dependencies()
-endif()
-
-ament_package()
+
+project(ros2_rosbag_evaluation)
+
+set(CMAKE_CXX_STANDARD 14)
+set(CMAKE_CXX_FLAGS "-O3")
+set(BUILD_SHARED_LIBS ON)
+
+set(EXECUTABLE_OUTPUT_PATH ${CMAKE_BINARY_DIR}/bin)
+
+set(common_sources
+  src/common/strings.cpp)
+
+set(profiler_sources
+  src/profiler/profiler.cpp)
+
+set(sqlite_sources
+  src/writer/sqlite/sqlite.cpp
+  src/writer/sqlite/sqlite_writer.cpp
+  src/writer/sqlite/one_table_sqlite_writer.cpp
+  src/writer/sqlite/separate_topic_table_sqlite_writer.cpp)
+
+set(trivial_writer_benchmark_sources
+  src/benchmark/writer/trivial/trivial_writer_benchmark.cpp
+  src/benchmark/benchmark.cpp
+  src/writer/stream/message_stream_writer.cpp
+  src/generators/message_generator.cpp)
+
+set(sqlite_writer_benchmark_cmd_sources
+  src/benchmark/writer/sqlite/sqlite_writer_benchmark_cmd.cpp
+  src/benchmark/writer/sqlite/sqlite_writer_benchmark.cpp
+  src/benchmark/benchmark.cpp
+  src/generators/message_generator.cpp)
+
+set(small_messages_benchmark_sources
+  src/benchmark/small_messages_benchmark.cpp
+  src/benchmark/writer/sqlite/sqlite_writer_benchmark.cpp
+  src/benchmark/benchmark.cpp
+  src/generators/message_generator.cpp)
+
+set(big_messages_benchmark_sources
+  src/benchmark/big_messages_benchmark.cpp
+  src/benchmark/writer/sqlite/sqlite_writer_benchmark.cpp
+  src/benchmark/benchmark.cpp
+  src/generators/message_generator.cpp)
+
+set(mixed_messages_benchmark_sources
+  src/benchmark/mixed_messages_benchmark.cpp
+  src/benchmark/writer/sqlite/sqlite_writer_benchmark.cpp
+  src/benchmark/benchmark.cpp
+  src/generators/message_generator.cpp)
+
+add_library(common ${common_sources})
+target_include_directories(common PRIVATE src)
+
+add_library(profiler ${profiler_sources})
+target_include_directories(profiler PRIVATE src)
+
+add_library(sqlite ${sqlite_sources})
+target_include_directories(sqlite PRIVATE src)
+target_link_libraries(sqlite sqlite3 common)
+
+add_executable(trivial_writer_benchmark ${trivial_writer_benchmark_sources})
+target_link_libraries(trivial_writer_benchmark profiler sqlite)
+target_include_directories(trivial_writer_benchmark PRIVATE src)
+
+add_executable(sqlite_writer_benchmark_cmd ${sqlite_writer_benchmark_cmd_sources})
+target_link_libraries(sqlite_writer_benchmark_cmd profiler sqlite)
+target_include_directories(sqlite_writer_benchmark_cmd PRIVATE src)
+
+add_executable(small_messages_benchmark ${small_messages_benchmark_sources})
+target_link_libraries(small_messages_benchmark profiler sqlite)
+target_include_directories(small_messages_benchmark PRIVATE src)
+
+add_executable(big_messages_benchmark ${big_messages_benchmark_sources})
+target_link_libraries(big_messages_benchmark profiler sqlite)
+target_include_directories(big_messages_benchmark PRIVATE src)
+
+add_executable(mixed_messages_benchmark ${mixed_messages_benchmark_sources})
+target_link_libraries(mixed_messages_benchmark profiler sqlite)
+target_include_directories(mixed_messages_benchmark PRIVATE src)

rosbag2_storage_evaluation/README.md

@@ -0,0 +1,84 @@
+# ROS 2.0 Rosbag Evaluation
+
+## Benchmarks
+
+This folder currently contains benchmarks which measure the write speed and disk usage of SQLite files.
+
+The single table schema
+```
+  sqlite::create_table(db, "MESSAGES", {
+    "TIMESTAMP INTEGER NOT NULL",
+    "TOPIC TEXT NOT NULL",
+    "DATA BLOB NOT NULL"
+  });
+```
+is compared with a foreign key schema storing topics in a separate table
+```
+  sqlite::create_table(db, "TOPICS", {
+    "ID INTEGER PRIMARY KEY",
+    "TOPIC TEXT NOT NULL"
+  });
+
+  sqlite::create_table(db, "MESSAGES", {
+    "TIMESTAMP INTEGER NOT NULL",
+    "TOPIC_ID INTEGER NOT NULL",
+    "DATA BLOB NOT NULL"
+  }, {sqlite::ForeignKeyDef{"TOPIC_ID", "TOPICS", "ID"}});
+```
+
+It should be **easy to add additional bag file formats**, e.g. for writing directly to disk or writing the RosBag 2.0 format.
+
+### Build from command line
+
+The project is using cmake. The script `./build.sh` can be used to build it.
+
+This will generate benchmark binaries in `./build/bin/`.
+
+### Run
+
+Individual benchmarks in `./build/bin` can be run by hand. To run the complete suite the script `./run_all_benchmarks.sh` can be used.
+
+Each benchmark will generate a CSV file in `./build/bin` containing the measured data for further plotting with the Jupyter Notebook.
+
+## Jupyter Notebook
+
+It is used for data analysis and visualization.
+
+### Setup
+
+Prerequisites: Python 3.5+, [pip](https://pip.pypa.io/en/stable/), [virtualenv](https://virtualenv.pypa.io/en/stable/)
+
+```
+virtualenv -p python3 venv
+. venv/bin/activate
+pip install -r requirements.txt
+```
+
+### Usage
+
+```
+. venv/bin/activate
+jupyter notebook data_analysis_and_visualization.ipynb
+```
+
+A browser window should open. Click `Cell -> Run All`.
+
+## Extending the benchmarks
+
+### Read Tests
+
+To measure retrieval time of the first message, extend the `MessageWriter` interface like so
+```
+  virtual MessageStream::Ptr selectAll() = 0;
+  virtual MessageStream::Ptr selectTopic(std::string const & topic) = 0;
+  virtual MessageStream::Ptr selectFromTo(
+    Message::Timestamp const & fromInclusive, Message::Timestamp const & toExclusive) = 0;
+```
+where `MessageStream` is a lazy data structure
+```
+  virtual bool has_next() const = 0;
+  virtual MessagePtr next() = 0;
+```
+implemented with a streaming SQLite `SELECT` statement.
+
+The desired timings can then be measured with the `Profiler`.

rosbag2_storage_evaluation/build.sh

@@ -0,0 +1,24 @@
+#!/usr/bin/env sh
+#  Copyright (c) 2018,  Bosch Software Innovations GmbH.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+mkdir -p ./build
+
+cd build
+
+cmake ..
+make
+
+cd ..
+