Add OptionsFormatter class and update to allow auto-logging of Options

CMakeLists.txt

@@ -842,6 +842,7 @@ set(SOURCES
         options/customizable.cc
         options/db_options.cc
         options/options.cc
+        options/options_formatter.cc
         options/options_helper.cc
         options/options_parser.cc
         port/mmap.cc

HISTORY.md

@@ -20,6 +20,9 @@ Based on RocksDB 8.1.1
 
 ### New Features 
 * Non-Blocking Manual Compaction (CompactRange()) - Support non-blocking manual compactions by setting a new CompactRangeOptions option (async_completion_cb). When set, the CompactRange() call will return control to the caller immediately. The manual compaction iteslf will be performed in an internally created thread. The manual compaction will ALWAYS call the specified callback upon completion and provide the completion status (#597).
+* Change the internal Configurable API SerializeOptions to return UserProperties (instead of the final string representation).  Added ToString methods to the ConfigurableOptions class to complete the serialization of Options properties.
+* Added ConfigOptions::compare_to.  When set, this value causes only values that have been changed to be part of the serialized output.
+* Add OptionsFormatter class.  This class allows options to be serialized and configured in different formats.
 * Change the internal Configurable API SerializeOptions to return UserProperties (instead of the final string representation).  Added ToString methods to the ConfigurableOptions class to complete the serialization of Options properties (#619).
 
 ### Enhancements

TARGETS

@@ -167,6 +167,7 @@ cpp_library_wrapper(name="rocksdb_lib", srcs=[
         "options/customizable.cc",
         "options/db_options.cc",
         "options/options.cc",
+        "options/options_formatter.cc",
         "options/options_helper.cc",
         "options/options_parser.cc",
         "port/mmap.cc",
@@ -519,6 +520,7 @@ cpp_library_wrapper(name="rocksdb_whole_archive_lib", srcs=[
         "options/customizable.cc",
         "options/db_options.cc",
         "options/options.cc",
+        "options/options_formatter.cc",
         "options/options_helper.cc",
         "options/options_parser.cc",
         "port/mmap.cc",

db/db_options_test.cc

@@ -35,6 +35,7 @@
 #include "rocksdb/convenience.h"
 #include "rocksdb/rate_limiter.h"
 #include "rocksdb/stats_history.h"
+#include "rocksdb/utilities/options_type.h"
 #include "test_util/sync_point.h"
 #include "test_util/testutil.h"
 #include "util/random.h"
@@ -50,35 +51,30 @@ class DBOptionsTest : public DBTestBase {
     SyncPoint::GetInstance()->ClearAllCallBacks();
   }
 
-  std::unordered_map<std::string, std::string> GetMutableDBOptionsMap(
-      const DBOptions& options) {
+  OptionProperties GetMutableDBOptionsMap(const DBOptions& options) {
     std::string options_str;
-    std::unordered_map<std::string, std::string> mutable_map;
+    OptionProperties mutable_map;
     ConfigOptions config_options(options);
-    config_options.delimiter = "; ";
 
     EXPECT_OK(GetStringFromMutableDBOptions(
         config_options, MutableDBOptions(options), &options_str));
-    EXPECT_OK(StringToMap(options_str, &mutable_map));
+    EXPECT_OK(config_options.ToProps(options_str, &mutable_map));
 
     return mutable_map;
   }
 
-  std::unordered_map<std::string, std::string> GetMutableCFOptionsMap(
-      const ColumnFamilyOptions& options) {
+  OptionProperties GetMutableCFOptionsMap(const ColumnFamilyOptions& options) {
     std::string options_str;
     ConfigOptions config_options;
-    config_options.delimiter = "; ";
 
-    std::unordered_map<std::string, std::string> mutable_map;
+    OptionProperties mutable_map;
     EXPECT_OK(GetStringFromMutableCFOptions(
         config_options, MutableCFOptions(options), &options_str));
-    EXPECT_OK(StringToMap(options_str, &mutable_map));
+    EXPECT_OK(config_options.ToProps(options_str, &mutable_map));
     return mutable_map;
   }
 
-  std::unordered_map<std::string, std::string> GetRandomizedMutableCFOptionsMap(
-      Random* rnd) {
+  OptionProperties GetRandomizedMutableCFOptionsMap(Random* rnd) {
     Options options = CurrentOptions();
     options.env = env_;
     ImmutableDBOptions db_options(options);
@@ -89,8 +85,7 @@ class DBOptionsTest : public DBTestBase {
     return opt_map;
   }
 
-  std::unordered_map<std::string, std::string> GetRandomizedMutableDBOptionsMap(
-      Random* rnd) {
+  OptionProperties GetRandomizedMutableDBOptionsMap(Random* rnd) {
     DBOptions db_options;
     test::RandomInitDBOptions(&db_options, rnd);
     auto sanitized_options = SanitizeOptions(dbname_, db_options);

env/env.cc

@@ -676,10 +676,10 @@ Status Env::CreateFromString(const ConfigOptions& config_options,
 
   Env* env = *result;
   std::string id;
-  std::unordered_map<std::string, std::string> opt_map;
+  OptionProperties props;
 
   Status status =
-      Customizable::GetOptionsMap(config_options, env, value, &id, &opt_map);
+      Customizable::GetOptionsMap(config_options, env, value, &id, &props);
   if (!status.ok()) {  // GetOptionsMap failed
     return status;
   }
@@ -695,7 +695,7 @@ Status Env::CreateFromString(const ConfigOptions& config_options,
   if (config_options.ignore_unsupported_options && status.IsNotSupported()) {
     status = Status::OK();
   } else if (status.ok()) {
-    status = Customizable::ConfigureNewObject(config_options, env, opt_map);
+    status = Customizable::ConfigureNewObject(config_options, env, props);
   }
   if (status.ok()) {
     guard->reset(uniq.release());

include/rocksdb/configurable.h

@@ -285,9 +285,10 @@ class Configurable {
   // found.
   // @return InvalidArgument if the value could not be converted to a map or
   // there was or there is no id property in the map.
-  static Status GetOptionsMap(
-      const std::string& opt_value, const std::string& default_id,
-      std::string* id, std::unordered_map<std::string, std::string>* options);
+  static Status GetOptionsMap(const ConfigOptions& config_options,
+                              const std::string& opt_value,
+                              const std::string& default_id, std::string* id,
+                              OptionProperties* options);
 
  protected:
   // Returns the raw pointer for the associated named option.

include/rocksdb/convenience.h

@@ -32,7 +32,9 @@ namespace ROCKSDB_NAMESPACE {
 class Env;
 class Logger;
 class ObjectRegistry;
+class OptionsFormatter;
 class OptionProperties;
+
 struct ColumnFamilyOptions;
 struct DBOptions;
 struct Options;
@@ -90,6 +92,7 @@ struct ConfigOptions {
   bool mutable_options_only = false;
 
   // The separator between options when converting to a string
+  // This option is now deprecated and replaced by the formatter field
   std::string delimiter = ";";
 
   // Controls how to traverse options during print/match stages
@@ -107,6 +110,9 @@ struct ConfigOptions {
   // The object registry to use for this options
   std::shared_ptr<ObjectRegistry> registry;
 
+  // Helper class for printing and parsing options to/from strings.
+  std::shared_ptr<OptionsFormatter> formatter;
+
   // If set, only changes from this reference version will be serialized.
   Configurable* compare_to = nullptr;
 
@@ -125,9 +131,17 @@ struct ConfigOptions {
   std::string ToString(const std::string& prefix,
                        const OptionProperties& props) const;
 
+  // Converts the string representation into name/value properties
+  Status ToProps(const std::string& opts_str, OptionProperties* props) const;
+
   // Converts the vector options to a single string representation
-  std::string ToString(char separator,
+  std::string ToString(const std::string& prefix, char separator,
                        const std::vector<std::string>& elems) const;
+
+  // Converts the string representation into vector of elements based on the
+  // separator
+  Status ToVector(const std::string& opts_str, char separator,
+                  std::vector<std::string>* elems) const;
 };
 
 
@@ -420,15 +434,13 @@ Status GetStringFromDBOptions(const ConfigOptions& config_options,
                               std::string* opts_str);
 
 Status GetStringFromDBOptions(std::string* opts_str,
-                              const DBOptions& db_options,
-                              const std::string& delimiter = ";  ");
+                              const DBOptions& db_options);
 
 Status GetStringFromColumnFamilyOptions(const ConfigOptions& config_options,
                                         const ColumnFamilyOptions& cf_options,
                                         std::string* opts_str);
 Status GetStringFromColumnFamilyOptions(std::string* opts_str,
-                                        const ColumnFamilyOptions& cf_options,
-                                        const std::string& delimiter = ";  ");
+                                        const ColumnFamilyOptions& cf_options);
 Status GetStringFromCompressionType(std::string* compression_str,
                                     CompressionType compression_type);
 
@@ -454,9 +466,6 @@ Status GetOptionsFromString(const ConfigOptions& config_options,
                             const Options& base_options,
                             const std::string& opts_str, Options* new_options);
 
-Status StringToMap(const std::string& opts_str,
-                   std::unordered_map<std::string, std::string>* opts_map);
-
 // Request stopping background work, if wait is true wait until it's done
 void CancelAllBackgroundWork(DB* db, bool wait = false);
 

include/rocksdb/customizable.h

@@ -181,7 +181,7 @@ class Customizable : public Configurable {
   // options for use in potentially creating a new Customizable object (this
   // method is primarily a support method for LoadSharedObject et al for new
   // Customizable objects). The opt_value may be either name-value pairs
-  // separated by ";" (a=b; c=d), or a simple name (a). In order to create a new
+  // or a simple name (a). In order to create a new
   // Customizable, the ID is determined by:
   // - If the value is a simple name (e.g. "BlockBasedTable"), the id is this
   // name;
@@ -196,10 +196,10 @@ class Customizable : public Configurable {
   //
   // This method returns non-OK if the ID could not be found, or if the
   // opt_value could not be parsed into name-value pairs.
-  static Status GetOptionsMap(
-      const ConfigOptions& config_options, const Customizable* custom,
-      const std::string& opt_value, std::string* id,
-      std::unordered_map<std::string, std::string>* options);
+  static Status GetOptionsMap(const ConfigOptions& config_options,
+                              const Customizable* custom,
+                              const std::string& opt_value, std::string* id,
+                              OptionProperties* options);
 
   // Helper method to configure a new object with the supplied options.
   // If the object is not null and invoke_prepare_options=true, the object