Accept obsolete column families with a warning (#2325)

* Accept obsolete column families with a warning * Add forward --------- Co-authored-by: kamilsa <kamilsa16@gmail.com>
qdrvm · Dec 30, 2024 · 041e1b0 · 041e1b0
1 parent 1f04f5d
commit 041e1b0
Show file tree

Hide file tree

Showing 4 changed files with 68 additions and 50 deletions.
diff --git a/cmake/Hunter/config.cmake b/cmake/Hunter/config.cmake
@@ -62,6 +62,7 @@ if ("${WASM_COMPILER}" STREQUAL "WasmEdge")
       CMAKE_ARGS
         WASMEDGE_BUILD_STATIC_LIB=ON
         WASMEDGE_BUILD_SHARED_LIB=OFF
+        CMAKE_CXX_FLAGS=-Wno-error=maybe-uninitialized
       KEEP_PACKAGE_SOURCES
   )
 endif ()

diff --git a/core/storage/rocksdb/rocksdb.cpp b/core/storage/rocksdb/rocksdb.cpp
@@ -7,6 +7,9 @@
 #include "storage/rocksdb/rocksdb.hpp"
 #include <rocksdb/filter_policy.h>
 #include <rocksdb/table.h>
+#include <algorithm>
+#include <iterator>
+#include <ranges>
 
 #include "filesystem/common.hpp"
 
@@ -20,6 +23,38 @@
 namespace kagome::storage {
   namespace fs = std::filesystem;
 
+  rocksdb::ColumnFamilyOptions configureColumn(uint32_t memory_budget) {
+    rocksdb::ColumnFamilyOptions options;
+    options.OptimizeLevelStyleCompaction(memory_budget);
+    auto table_options = RocksDb::tableOptionsConfiguration();
+    options.table_factory.reset(NewBlockBasedTableFactory(table_options));
+    return options;
+  }
+
+  template <std::ranges::range ColumnFamilyNames>
+  void configureColumnFamilies(
+      std::vector<rocksdb::ColumnFamilyDescriptor> &column_family_descriptors,
+      std::vector<int32_t> &ttls,
+      ColumnFamilyNames &&cf_names,
+      const std::unordered_map<std::string, int32_t> &column_ttl,
+      uint32_t trie_space_cache_size,
+      uint32_t other_spaces_cache_size,
+      log::Logger &log) {
+    for (auto &space_name : std::forward<ColumnFamilyNames>(cf_names)) {
+      auto ttl = 0;
+      if (const auto it = column_ttl.find(space_name); it != column_ttl.end()) {
+        ttl = it->second;
+      }
+      column_family_descriptors.emplace_back(
+          space_name,
+          configureColumn(space_name != spaceName(Space::kTrieNode)
+                              ? other_spaces_cache_size
+                              : trie_space_cache_size));
+      ttls.push_back(ttl);
+      SL_DEBUG(log, "Column family {} configured with TTL {}", space_name, ttl);
+    }
+  }
+
   RocksDb::RocksDb() : logger_(log::createLogger("RocksDB", "storage")) {
     ro_.fill_cache = false;
   }
@@ -30,6 +65,7 @@ namespace kagome::storage {
     }
     delete db_;
   }
+
   outcome::result<std::shared_ptr<RocksDb>> RocksDb::create(
       const filesystem::path &path,
       rocksdb::Options options,
@@ -51,15 +87,6 @@ namespace kagome::storage {
     const uint32_t other_spaces_cache_size =
         (memory_budget - trie_space_cache_size) / (storage::Space::kTotal - 1);
 
-    std::vector<rocksdb::ColumnFamilyDescriptor> column_family_descriptors;
-    std::vector<int32_t> ttls;
-    configureColumnFamilies(column_family_descriptors,
-                            ttls,
-                            column_ttl,
-                            trie_space_cache_size,
-                            other_spaces_cache_size,
-                            log);
-
     std::vector<std::string> existing_families;
     auto res = rocksdb::DB::ListColumnFamilies(
         options, path.native(), &existing_families);
@@ -71,6 +98,35 @@ namespace kagome::storage {
       return status_as_error(res);
     }
 
+    std::unordered_set<std::string> all_families;
+
+    auto required_families =
+        std::views::iota(0, Space::kTotal) | std::views::transform([](int i) {
+          return spaceName(static_cast<Space>(i));
+        });
+    std::ranges::copy(required_families,
+                      std::inserter(all_families, all_families.end()));
+
+    for (auto &existing_family : existing_families) {
+      auto [_, was_inserted] = all_families.insert(existing_family);
+      if (was_inserted) {
+        SL_WARN(log,
+                "Column family '{}' present in database but not used by "
+                "KAGOME, probably obsolete.",
+                existing_family);
+      }
+    }
+
+    std::vector<rocksdb::ColumnFamilyDescriptor> column_family_descriptors;
+    std::vector<int32_t> ttls;
+    configureColumnFamilies(column_family_descriptors,
+                            ttls,
+                            all_families,
+                            column_ttl,
+                            trie_space_cache_size,
+                            other_spaces_cache_size,
+                            log);
+
     options.create_missing_column_families = true;
     auto rocks_db = std::shared_ptr<RocksDb>(new RocksDb);
     const auto ttl_migrated_path = path.parent_path() / "ttl_migrated";
@@ -123,28 +179,6 @@ namespace kagome::storage {
     return outcome::success();
   }
 
-  void RocksDb::configureColumnFamilies(
-      std::vector<rocksdb::ColumnFamilyDescriptor> &column_family_descriptors,
-      std::vector<int32_t> &ttls,
-      const std::unordered_map<std::string, int32_t> &column_ttl,
-      uint32_t trie_space_cache_size,
-      uint32_t other_spaces_cache_size,
-      log::Logger &log) {
-    for (auto i = 0; i < Space::kTotal; ++i) {
-      const auto space_name = spaceName(static_cast<Space>(i));
-      auto ttl = 0;
-      if (const auto it = column_ttl.find(space_name); it != column_ttl.end()) {
-        ttl = it->second;
-      }
-      column_family_descriptors.emplace_back(
-          space_name,
-          configureColumn(i != Space::kTrieNode ? other_spaces_cache_size
-                                                : trie_space_cache_size));
-      ttls.push_back(ttl);
-      SL_DEBUG(log, "Column family {} configured with TTL {}", space_name, ttl);
-    }
-  }
-
   outcome::result<void> RocksDb::openDatabaseWithTTL(
       const rocksdb::Options &options,
       const filesystem::path &path,
@@ -356,15 +390,6 @@ namespace kagome::storage {
     return table_options;
   }
 
-  rocksdb::ColumnFamilyOptions RocksDb::configureColumn(
-      uint32_t memory_budget) {
-    rocksdb::ColumnFamilyOptions options;
-    options.OptimizeLevelStyleCompaction(memory_budget);
-    auto table_options = tableOptionsConfiguration();
-    options.table_factory.reset(NewBlockBasedTableFactory(table_options));
-    return options;
-  }
-
   RocksDb::DatabaseGuard::DatabaseGuard(
       std::shared_ptr<rocksdb::DB> db,
       std::vector<rocksdb::ColumnFamilyHandle *> column_family_handles,

diff --git a/core/storage/rocksdb/rocksdb.hpp b/core/storage/rocksdb/rocksdb.hpp
@@ -97,18 +97,10 @@ namespace kagome::storage {
     };
 
     RocksDb();
-    static rocksdb::ColumnFamilyOptions configureColumn(uint32_t memory_budget);
+
     static outcome::result<void> createDirectory(
         const std::filesystem::path &absolute_path, log::Logger &log);
 
-    static void configureColumnFamilies(
-        std::vector<rocksdb::ColumnFamilyDescriptor> &column_family_descriptors,
-        std::vector<int32_t> &ttls,
-        const std::unordered_map<std::string, int32_t> &column_ttl,
-        uint32_t trie_space_cache_size,
-        uint32_t other_spaces_cache_size,
-        log::Logger &log);
-
     static outcome::result<void> openDatabaseWithTTL(
         const rocksdb::Options &options,
         const filesystem::path &path,

diff --git a/core/utils/storage_explorer.cpp b/core/utils/storage_explorer.cpp
@@ -616,7 +616,7 @@ int storage_explorer_main(int argc, const char **argv) {
   auto configuration =
       std::make_shared<kagome::application::AppConfigurationImpl>();
 
-  size_t kagome_args_start;  // NOLINT(cppcoreguidelines-init-variables)
+  size_t kagome_args_start{};
   bool is_found = false;
   for (size_t i = 1; i < args.size(); i++) {
     if (strcmp(args[i], "--") == 0) {