BucketListDB enabled by default

docs/stellar-core_example.cfg

@@ -207,35 +207,38 @@ FLOOD_DEMAND_BACKOFF_DELAY_MS = 500
 #   against each other.
 MAX_DEX_TX_OPERATIONS_IN_TX_SET = 0
 
-# EXPERIMENTAL_BUCKETLIST_DB (bool) default false
-# Determines whether BucketListDB is used as the primary key-value store
-# for Ledger Entry lookups. Currently experimental, should be set to false.
-EXPERIMENTAL_BUCKETLIST_DB = false
-
-# EXPERIMENTAL_BUCKETLIST_DB_INDEX_PAGE_SIZE_EXPONENT (Integer) default 14
+# DEPRECATED_SQL_LEDGER_STATE (bool) default false
+# When set to true, SQL is used to store all ledger state instead of
+# BucketListDB. This is not recommended and may cause performance degregradation.
+# This is deprecated and will be removed in the future. Note that offers table
+# is still maintained in SQL when this is set to false, but all other ledger
+# state tables are dropped.
+DEPRECATED_SQL_LEDGER_STATE = false
+
+# BUCKETLIST_DB_INDEX_PAGE_SIZE_EXPONENT (Integer) default 14
 # Determines page size used by BucketListDB for range indexes, where
-# pageSize == 2^EXPERIMENTAL_BUCKETLIST_DB_INDEX_PAGE_SIZE_EXPONENT. If set to
+# pageSize == 2^BUCKETLIST_DB_INDEX_PAGE_SIZE_EXPONENT. If set to
 # 0, indiviudal index is always used. Default page size 16 kb.
-EXPERIMENTAL_BUCKETLIST_DB_INDEX_PAGE_SIZE_EXPONENT = 14
+BUCKETLIST_DB_INDEX_PAGE_SIZE_EXPONENT = 14
 
-# EXPERIMENTAL_BUCKETLIST_DB_INDEX_CUTOFF (Integer) default 20
+# BUCKETLIST_DB_INDEX_CUTOFF (Integer) default 20
 # Size, in MB, determining whether a bucket should have an individual
 # key index or a key range index. If bucket size is below this value, range
 # based index will be used. If set to 0, all buckets are range indexed. If
-# EXPERIMENTAL_BUCKETLIST_DB_INDEX_PAGE_SIZE_EXPONENT == 0, value ingnored and all
+# BUCKETLIST_DB_INDEX_PAGE_SIZE_EXPONENT == 0, value ingnored and all
 # buckets have individual key index.
-EXPERIMENTAL_BUCKETLIST_DB_INDEX_CUTOFF = 20
+BUCKETLIST_DB_INDEX_CUTOFF = 20
 
-# EXPERIMENTAL_BUCKETLIST_DB_PERSIST_INDEX (bool) default true
+# BUCKETLIST_DB_PERSIST_INDEX (bool) default true
 # Determines whether BucketListDB indexes are saved to disk for faster
 # startup. Should only be set to false for testing.
 # Validators do not currently support persisted indexes. If NODE_IS_VALIDATOR=true,
 # this value is ingnored and indexes are never persisted.
-EXPERIMENTAL_BUCKETLIST_DB_PERSIST_INDEX = true
+BUCKETLIST_DB_PERSIST_INDEX = true
 
-# EXPERIMENTAL_BUCKETLIST_DB (bool) default false
+# EXPERIMENTAL_BACKGROUND_EVICTION_SCAN (bool) default false
 # Determines whether eviction scans occur in the background thread. Requires
-# that EXPERIMENTAL_BUCKETLIST_DB is set to true.
+# that EXPERIMENTAL_BACKGROUND_EVICTION_SCAN is set to true.
 EXPERIMENTAL_BACKGROUND_EVICTION_SCAN = false
 
 # PREFERRED_PEERS (list of strings) default is empty

docs/stellar-core_example_validators.cfg

@@ -4,6 +4,7 @@ PUBLIC_HTTP_PORT=false
 NETWORK_PASSPHRASE="Example configuration"
 
 DATABASE="sqlite3://example.db"
+DEPRECATED_SQL_LEDGER_STATE = false
 
 NODE_SEED="SA7FGJMMUIHNE3ZPI2UO5I632A7O5FBAZTXFAIEVFA4DSSGLHXACLAIT a3"
 NODE_HOME_DOMAIN="domainA"

docs/stellar-core_standalone.cfg

@@ -12,6 +12,7 @@ NODE_IS_VALIDATOR=true
 
 #DATABASE="postgresql://dbname=stellar user=postgres password=password host=localhost"
 DATABASE="sqlite3://stellar.db"
+DEPRECATED_SQL_LEDGER_STATE = false
 
 COMMANDS=["ll?level=debug"]
 

docs/stellar-core_testnet.cfg

@@ -4,6 +4,7 @@ PUBLIC_HTTP_PORT=false
 NETWORK_PASSPHRASE="Test SDF Network ; September 2015"
 
 DATABASE="sqlite3://stellar.db"
+DEPRECATED_SQL_LEDGER_STATE = false
 
 # Stellar Testnet validators
 [[HOME_DOMAINS]]

docs/stellar-core_testnet_legacy.cfg

@@ -9,6 +9,7 @@ KNOWN_PEERS=[
 "core-testnet3.stellar.org"]
 
 DATABASE="sqlite3://stellar.db"
+DEPRECATED_SQL_LEDGER_STATE = false
 UNSAFE_QUORUM=true
 FAILURE_SAFETY=1
 

docs/stellar-core_testnet_validator.cfg

@@ -4,6 +4,7 @@ PUBLIC_HTTP_PORT=false
 NETWORK_PASSPHRASE="Test SDF Network ; September 2015"
 
 DATABASE="sqlite3://stellar.db"
+DEPRECATED_SQL_LEDGER_STATE = false
 
 # Configuring the node as a validator
 # note that this is an unsafe configuration in this particular setup:

src/bucket/BucketIndexImpl.cpp

@@ -46,13 +46,13 @@ static inline std::streamoff
 effectivePageSize(Config const& cfg, size_t bucketSize)
 {
     // Convert cfg param from MB to bytes
-    if (auto cutoff = cfg.EXPERIMENTAL_BUCKETLIST_DB_INDEX_CUTOFF * 1000000;
+    if (auto cutoff = cfg.BUCKETLIST_DB_INDEX_CUTOFF * 1000000;
         bucketSize < cutoff)
     {
         return 0;
     }
 
-    auto pageSizeExp = cfg.EXPERIMENTAL_BUCKETLIST_DB_INDEX_PAGE_SIZE_EXPONENT;
+    auto pageSizeExp = cfg.BUCKETLIST_DB_INDEX_PAGE_SIZE_EXPONENT;
     releaseAssertOrThrow(pageSizeExp < 32);
     return pageSizeExp == 0 ? 0 : 1UL << pageSizeExp;
 }

src/bucket/readme.md

@@ -28,9 +28,9 @@ The individual buckets that compose each level are checkpointed to history
 storage by the [history module](../history). The difference from the current bucket list (a subset
 of the buckets) is retrieved from history and applied in order to perform "fast" catchup.
 
-# BucketList Index [experimental]
+# BucketListDB Index
 
-Currently, the state of the ledger is redundantly copied in two places: the local SQL database
+Previously, the state of the ledger is redundantly copied in two places: the local SQL database
 and the BucketList. The BucketList is used to quickly compute the hash for a ledger, while the
 database is used for key-value lookup. This disjoint setup leads to write and disk amplification.
 Additionally, SQL is not the most efficient database for our read/write patterns. The database
@@ -59,7 +59,8 @@ corresponding `BucketIndex` kept in memory allowing minimal disk IO for each loa
 
 Due to the large number of `LedgerKey`'s, it is not possible to map each entry in a
 `Bucket` individually. Because of this, there are two types of indexes, the
-`IndividualIndex` and the `RangeIndex`. The `IndividualIndex` maps individual `LedgerKey`'s to a file offset as previously described. The `RangeIndex` maps a range of
+`IndividualIndex` and the `RangeIndex`. The `IndividualIndex` maps individual `LedgerKey`'s to a
+file offset as previously described. The `RangeIndex` maps a range of
 `LedgerKey`'s to a given page in the file. Because the `LedgerEntry`'s in each
 bucket file on disk are sorted, the `RangeIndex` can provide the page where a given
 entry may exist. However, the `RangeIndex` cannot determine if the given entry exists
@@ -69,7 +70,7 @@ must be iterated through in order to determine if the target entry exists in the
 
 To avoid additional disk reads from these "false positives", the `RangeIndex` also uses a
 bloom filter to determine if a given entry exists in the `Bucket` before doing a disk read
-and iterating through the corresponding page. The bloom filter is probabalistic and still
+and iterating through the corresponding page. The bloom filter is probabilistic and still
 has a false positive rate, but it is low (1/1000).
 
 Even with the bloom filter, the `RangeIndex` must still iterate through each entry in a
@@ -82,20 +83,20 @@ for smaller memory overhead.
 Because the `BucketIndex`'s must be in memory, there is a tradeoff between BucketList
 lookup speed and memory overhead. The following configuration flags control these options:
 
-- `EXPERIMENTAL_BUCKETLIST_DB`
-  - When set to true, the `BucketList` is indexed and used for ledger entry lookup
-- `EXPERIMENTAL_BUCKETLIST_DB_INDEX_PAGE_SIZE_EXPONENT`
+- `DEPRECATED_SQL_LEDGER_STATE`
+  - When set to false, the `BucketList` is indexed and used for ledger entry lookup
+- `BUCKETLIST_DB_INDEX_PAGE_SIZE_EXPONENT`
   - Page size used for `RangeIndex`, where `pageSize ==
-    2^EXPERIMENTAL_BUCKETLIST_DB_INDEX_PAGE_SIZE_EXPONENT`.
+    2^BUCKETLIST_DB_INDEX_PAGE_SIZE_EXPONENT`.
     Larger values slow down lookup speed but
     decrease memory usage.
-- `EXPERIMENTAL_BUCKETLIST_DB_INDEX_CUTOFF`
-  - Bucket file size, in MB, tyhat determines wether the `IndividualIndex` or
+- `BUCKETLIST_DB_INDEX_CUTOFF`
+  - Bucket file size, in MB, that determines wether the `IndividualIndex` or
    `RangeIndex` is used.
     Default value is 20 MB, which indexes the first ~3 levels with the `IndividualIndex`.
     Larger values speed up lookups but increase memory usage.
-- `EXPERIMENTAL_BUCKETLIST_DB_PERSIST_INDEX`
+- `BUCKETLIST_DB_PERSIST_INDEX`
   - When set to true, BucketListDB indexes are saved to disk to avoid reindexing
     on startup. Defaults to true, should only be set to false for testing purposes.
     Validators do not currently support persisted indexes. If NODE_IS_VALIDATOR=true,
-    this value is ingnored and indexes are never persisted.
+    this value is ignored and indexes are never persisted.

src/bucket/test/BucketIndexTests.cpp

@@ -455,26 +455,26 @@ testAllIndexTypes(std::function<void(Config&)> f)
     SECTION("individual index only")
     {
         Config cfg(getTestConfig());
-        cfg.EXPERIMENTAL_BUCKETLIST_DB = true;
-        cfg.EXPERIMENTAL_BUCKETLIST_DB_INDEX_PAGE_SIZE_EXPONENT = 0;
+        cfg.DEPRECATED_SQL_LEDGER_STATE = false;
+        cfg.BUCKETLIST_DB_INDEX_PAGE_SIZE_EXPONENT = 0;
         f(cfg);
     }
 
     SECTION("individual and range index")
     {
         Config cfg(getTestConfig());
-        cfg.EXPERIMENTAL_BUCKETLIST_DB = true;
+        cfg.DEPRECATED_SQL_LEDGER_STATE = false;
 
         // First 3 levels individual, last 3 range index
-        cfg.EXPERIMENTAL_BUCKETLIST_DB_INDEX_CUTOFF = 1;
+        cfg.BUCKETLIST_DB_INDEX_CUTOFF = 1;
         f(cfg);
     }
 
     SECTION("range index only")
     {
         Config cfg(getTestConfig());
-        cfg.EXPERIMENTAL_BUCKETLIST_DB = true;
-        cfg.EXPERIMENTAL_BUCKETLIST_DB_INDEX_CUTOFF = 0;
+        cfg.DEPRECATED_SQL_LEDGER_STATE = false;
+        cfg.BUCKETLIST_DB_INDEX_CUTOFF = 0;
         f(cfg);
     }
 }
@@ -543,9 +543,9 @@ TEST_CASE("serialize bucket indexes", "[bucket][bucketindex][!hide]")
     Config cfg(getTestConfig(0, Config::TESTDB_ON_DISK_SQLITE));
 
     // All levels use range config
-    cfg.EXPERIMENTAL_BUCKETLIST_DB_INDEX_CUTOFF = 0;
-    cfg.EXPERIMENTAL_BUCKETLIST_DB = true;
-    cfg.EXPERIMENTAL_BUCKETLIST_DB_PERSIST_INDEX = true;
+    cfg.BUCKETLIST_DB_INDEX_CUTOFF = 0;
+    cfg.DEPRECATED_SQL_LEDGER_STATE = false;
+    cfg.BUCKETLIST_DB_PERSIST_INDEX = true;
 
     // Node is not a validator, so indexes will persist
     cfg.NODE_IS_VALIDATOR = false;
@@ -580,8 +580,8 @@ TEST_CASE("serialize bucket indexes", "[bucket][bucketindex][!hide]")
     // Restart app with different config to test that indexes created with
     // different config settings are not loaded from disk. These params will
     // invalidate every index in BL
-    cfg.EXPERIMENTAL_BUCKETLIST_DB_INDEX_CUTOFF = 0;
-    cfg.EXPERIMENTAL_BUCKETLIST_DB_INDEX_PAGE_SIZE_EXPONENT = 10;
+    cfg.BUCKETLIST_DB_INDEX_CUTOFF = 0;
+    cfg.BUCKETLIST_DB_INDEX_PAGE_SIZE_EXPONENT = 10;
     test.restartWithConfig(cfg);
 
     for (auto const& bucketHash : buckets)

src/bucket/test/BucketListTests.cpp

@@ -760,7 +760,7 @@ TEST_CASE_VERSIONS("eviction scan", "[bucketlist]")
     auto test = [&](bool backgroundScan) {
         // BucketTestApplication writes directly to BL and circumvents LedgerTxn
         // interface, so we have to use BucketListDB for lookups
-        cfg.EXPERIMENTAL_BUCKETLIST_DB = true;
+        cfg.DEPRECATED_SQL_LEDGER_STATE = false;
         cfg.EXPERIMENTAL_BACKGROUND_EVICTION_SCAN = backgroundScan;
 
         auto app = createTestApplication<BucketTestApplication>(clock, cfg);
@@ -1230,7 +1230,7 @@ TEST_CASE_VERSIONS("Searchable BucketListDB snapshots", "[bucketlist]")
 {
     VirtualClock clock;
     Config cfg(getTestConfig(0, Config::TESTDB_IN_MEMORY_SQLITE));
-    cfg.EXPERIMENTAL_BUCKETLIST_DB = true;
+    cfg.DEPRECATED_SQL_LEDGER_STATE = false;
 
     auto app = createTestApplication<BucketTestApplication>(clock, cfg);
     LedgerManagerForBucketTests& lm = app->getLedgerManager();

src/bucket/test/BucketManagerTests.cpp

@@ -188,13 +188,13 @@ TEST_CASE_VERSIONS("bucketmanager ownership", "[bucket][bucketmanager]")
         Config cfg = getTestConfig();
 
         // Make sure all Buckets serialize indexes to disk for test
-        cfg.EXPERIMENTAL_BUCKETLIST_DB_INDEX_CUTOFF = 0;
+        cfg.BUCKETLIST_DB_INDEX_CUTOFF = 0;
         cfg.MANUAL_CLOSE = false;
 
         if (bucketListDB)
         {
             // Enable BucketListDB with persistent indexes
-            cfg.EXPERIMENTAL_BUCKETLIST_DB = true;
+            cfg.DEPRECATED_SQL_LEDGER_STATE = false;
             cfg.NODE_IS_VALIDATOR = false;
             cfg.FORCE_SCP = false;
         }

src/history/test/HistoryTestsUtils.cpp

@@ -721,7 +721,7 @@ CatchupSimulation::createCatchupApplication(
     mCfgs.back().CATCHUP_COMPLETE =
         count == std::numeric_limits<uint32_t>::max();
     mCfgs.back().CATCHUP_RECENT = count;
-    mCfgs.back().EXPERIMENTAL_BUCKETLIST_DB = useBucketListDB;
+    mCfgs.back().DEPRECATED_SQL_LEDGER_STATE = !useBucketListDB;
     if (ledgerVersion)
     {
         mCfgs.back().TESTING_UPGRADE_LEDGER_PROTOCOL_VERSION = *ledgerVersion;

src/main/ApplicationImpl.cpp

@@ -731,14 +731,36 @@ ApplicationImpl::validateAndLogConfig()
             "requires --in-memory");
     }
 
-    if (mConfig.EXPERIMENTAL_BUCKETLIST_DB)
+    if (mConfig.isInMemoryMode())
+    {
+        CLOG_WARNING(
+            Bucket,
+            "in-memory mode is enabled. This feature is deprecated! Node "
+            "may see performance degredation and lose sync with the network.");
+    }
+
+    if (mConfig.DEPRECATED_SQL_LEDGER_STATE)
+    {
+        if (mPersistentState->getState(PersistentState::kDBBackend) ==
+            BucketIndex::DB_BACKEND_STATE)
+        {
+            throw std::invalid_argument(
+                "To downgrade to DEPRECATED_SQL_LEDGER_STATE, run "
+                "stellar-core new-db.");
+        }
+
+        CLOG_WARNING(
+            Bucket,
+            "SQL for ledger state is enabled. This feature is deprecated! Node "
+            "may see performance degredation and lose sync with the network.");
+    }
+    else
     {
         if (mConfig.isUsingBucketListDB())
         {
             mPersistentState->setState(PersistentState::kDBBackend,
                                        BucketIndex::DB_BACKEND_STATE);
-            auto pageSizeExp =
-                mConfig.EXPERIMENTAL_BUCKETLIST_DB_INDEX_PAGE_SIZE_EXPONENT;
+            auto pageSizeExp = mConfig.BUCKETLIST_DB_INDEX_PAGE_SIZE_EXPONENT;
             if (pageSizeExp != 0)
             {
                 // If the page size is less than 256 bytes, it is essentially
@@ -747,7 +769,7 @@ ApplicationImpl::validateAndLogConfig()
                 if (pageSizeExp < 8)
                 {
                     throw std::invalid_argument(
-                        "EXPERIMENTAL_BUCKETLIST_DB_INDEX_PAGE_SIZE_EXPONENT "
+                        "BUCKETLIST_DB_INDEX_PAGE_SIZE_EXPONENT "
                         "must be at least 8 or set to 0 for individual entry "
                         "indexing");
                 }
@@ -756,42 +778,35 @@ ApplicationImpl::validateAndLogConfig()
                 if (pageSizeExp > 31)
                 {
                     throw std::invalid_argument(
-                        "EXPERIMENTAL_BUCKETLIST_DB_INDEX_PAGE_SIZE_EXPONENT "
+                        "BUCKETLIST_DB_INDEX_PAGE_SIZE_EXPONENT "
                         "must be less than 32");
                 }
             }
 
             CLOG_INFO(Bucket,
                       "BucketListDB enabled: pageSizeExponent: {} indexCutOff: "
                       "{}MB, persist indexes: {}",
-                      pageSizeExp,
-                      mConfig.EXPERIMENTAL_BUCKETLIST_DB_INDEX_CUTOFF,
+                      pageSizeExp, mConfig.BUCKETLIST_DB_INDEX_CUTOFF,
                       mConfig.isPersistingBucketListDBIndexes());
         }
         else
         {
             CLOG_WARNING(
                 Bucket,
-                "EXPERIMENTAL_BUCKETLIST_DB flag set but "
-                "BucketListDB not enabled. To enable BucketListDB, "
+                "DEPRECATED_SQL_LEDGER_STATE set to false but "
+                "deprecated SQL ledger state is active. To disable deprecated "
+                "SQL ledger state, "
                 "MODE_ENABLES_BUCKETLIST must be set and --in-memory flag "
                 "must not be used.");
         }
     }
-    else if (mPersistentState->getState(PersistentState::kDBBackend) ==
-             BucketIndex::DB_BACKEND_STATE)
-    {
-        throw std::invalid_argument(
-            "To downgrade from EXPERIMENTAL_BUCKETLIST_DB, run "
-            "stellar-core new-db.");
-    }
 
     if (mConfig.EXPERIMENTAL_BACKGROUND_EVICTION_SCAN)
     {
         if (!mConfig.isUsingBucketListDB())
         {
             throw std::invalid_argument(
-                "EXPERIMENTAL_BUCKETLIST_DB must be enabled to use "
+                "DEPRECATED_SQL_LEDGER_STATE must be false to use "
                 "EXPERIMENTAL_BACKGROUND_EVICTION_SCAN");
         }