From c1760c075b37d03a248cdfe724428ce7b2e01946 Mon Sep 17 00:00:00 2001 From: Sagar Vemuri Date: Mon, 28 Oct 2019 16:29:45 -0700 Subject: [PATCH] Change periodic_compaction_seconds to UINT64_MAX --- db/column_family.cc | 9 +++++++-- db/db_compaction_test.cc | 2 +- db/version_set.cc | 13 +++++++++++-- include/rocksdb/advanced_options.h | 12 ++++++++++-- utilities/blob_db/blob_db_impl.cc | 3 +++ 5 files changed, 32 insertions(+), 7 deletions(-) diff --git a/db/column_family.cc b/db/column_family.cc index adefd29663a..f0360eefe96 100644 --- a/db/column_family.cc +++ b/db/column_family.cc @@ -30,6 +30,7 @@ #include "memtable/hash_skiplist_rep.h" #include "monitoring/thread_status_util.h" #include "options/options_helper.h" +#include "port/port.h" #include "table/block_based/block_based_table_factory.h" #include "table/merging_iterator.h" #include "util/autovector.h" @@ -342,10 +343,13 @@ ColumnFamilyOptions SanitizeOptions(const ImmutableDBOptions& db_options, result.max_compaction_bytes = result.target_file_size_base * 25; } + // Turn on periodic compactions and set them to occur once every 30 days if + // compaction filters are used and periodic_compaction_seconds is set to the + // default value. if (result.compaction_style == kCompactionStyleLevel && (result.compaction_filter != nullptr || result.compaction_filter_factory != nullptr) && - result.periodic_compaction_seconds == 0) { + result.periodic_compaction_seconds == port::kMaxUint64) { result.periodic_compaction_seconds = 30 * 24 * 60 * 60; } @@ -1187,7 +1191,8 @@ Status ColumnFamilyData::ValidateOptions( } } - if (cf_options.periodic_compaction_seconds > 0) { + if (cf_options.periodic_compaction_seconds > 0 && + cf_options.periodic_compaction_seconds < port::kMaxUint64) { if (cf_options.table_factory->Name() != BlockBasedTableFactory().Name()) { return Status::NotSupported( "Periodic Compaction is only supported in " diff --git a/db/db_compaction_test.cc b/db/db_compaction_test.cc index ca39a883514..45c140ab572 100644 --- a/db/db_compaction_test.cc +++ b/db/db_compaction_test.cc @@ -3780,7 +3780,7 @@ TEST_F(DBCompactionTest, LevelPeriodicCompactionWithCompactionFilters) { for (COMPACTION_FILTER_TYPE comp_filter_type : {USE_COMPACTION_FILTER, USE_COMPCTION_FILTER_FACTORY}) { // Assert that periodic compactions are not enabled. - ASSERT_EQ(0, options.periodic_compaction_seconds); + ASSERT_EQ(port::kMaxUint64, options.periodic_compaction_seconds); if (comp_filter_type == USE_COMPACTION_FILTER) { options.compaction_filter = &test_compaction_filter; diff --git a/db/version_set.cc b/db/version_set.cc index 11264205a29..61d140a6fb6 100644 --- a/db/version_set.cc +++ b/db/version_set.cc @@ -2445,7 +2445,8 @@ void VersionStorageInfo::ComputeCompactionScore( if (mutable_cf_options.ttl > 0) { ComputeExpiredTtlFiles(immutable_cf_options, mutable_cf_options.ttl); } - if (mutable_cf_options.periodic_compaction_seconds > 0) { + if (mutable_cf_options.periodic_compaction_seconds > 0 && + mutable_cf_options.periodic_compaction_seconds < port::kMaxUint64) { ComputeFilesMarkedForPeriodicCompaction( immutable_cf_options, mutable_cf_options.periodic_compaction_seconds); } @@ -2505,7 +2506,8 @@ void VersionStorageInfo::ComputeExpiredTtlFiles( void VersionStorageInfo::ComputeFilesMarkedForPeriodicCompaction( const ImmutableCFOptions& ioptions, const uint64_t periodic_compaction_seconds) { - assert(periodic_compaction_seconds > 0); + assert(periodic_compaction_seconds > 0 && + periodic_compaction_seconds < port::kMaxUint64); files_marked_for_periodic_compaction_.clear(); @@ -2515,6 +2517,13 @@ void VersionStorageInfo::ComputeFilesMarkedForPeriodicCompaction( return; } const uint64_t current_time = static_cast(temp_current_time); + + assert(periodic_compaction_seconds <= current_time); + // Disable periodic compaction if periodic_compaction_seconds > current_time. + // This also help handle the underflow case. + if (periodic_compaction_seconds > current_time) { + return; + } const uint64_t allowed_time_limit = current_time - periodic_compaction_seconds; diff --git a/include/rocksdb/advanced_options.h b/include/rocksdb/advanced_options.h index 2964491f7ee..77c55d977ef 100644 --- a/include/rocksdb/advanced_options.h +++ b/include/rocksdb/advanced_options.h @@ -670,10 +670,18 @@ struct AdvancedColumnFamilyOptions { // Only supported in Level compaction. // Pre-req: max_open_file == -1. // unit: seconds. Ex: 7 days = 7 * 24 * 60 * 60 - // Default: 0 (disabled) + // + // Values: + // 0: Turn off Periodic compactions. + // UINT64_MAX (i.e 0xffffffffffffffff): Let RocksDB control this feature + // as needed. For now, RocksDB will change this value to 30 days + // (i.e 30 * 24 * 60 * 60) so that every file goes through the compaction + // process at least once every 30 days if not compacted sooner. + // + // Default: UINT64_MAX (allow RocksDB to auto-tune) // // Dynamically changeable through SetOptions() API - uint64_t periodic_compaction_seconds = 0; + uint64_t periodic_compaction_seconds = 0xffffffffffffffff; // If this option is set then 1 in N blocks are compressed // using a fast (lz4) and slow (zstd) compression algorithm. diff --git a/utilities/blob_db/blob_db_impl.cc b/utilities/blob_db/blob_db_impl.cc index b7338f2305f..d3a4d8e4aec 100644 --- a/utilities/blob_db/blob_db_impl.cc +++ b/utilities/blob_db/blob_db_impl.cc @@ -137,6 +137,9 @@ Status BlobDBImpl::Open(std::vector* handles) { cf_options_.compaction_filter_factory != nullptr) { return Status::NotSupported("Blob DB doesn't support compaction filter."); } + // BlobDB does not support Periodic Compactions. So disable periodic + // compactions irrespective of the user set value. + cf_options_.periodic_compaction_seconds = 0; Status s;