Add TableBuilderOptions::level and relevant changes (#1335)

facebook · Sep 18, 2016 · 4c3f449 · 4c3f449
1 parent 3edb946
commit 4c3f449
Show file tree

Hide file tree

Showing 10 changed files with 31 additions and 15 deletions.
diff --git a/db/builder.cc b/db/builder.cc
@@ -45,6 +45,7 @@ TableBuilder* NewTableBuilder(
     uint32_t column_family_id, const std::string& column_family_name,
     WritableFileWriter* file, const CompressionType compression_type,
     const CompressionOptions& compression_opts,
+    int level,
     const std::string* compression_dict, const bool skip_filters) {
   assert((column_family_id ==
           TablePropertiesCollectorFactory::Context::kUnknownColumnFamily) ==
@@ -53,7 +54,7 @@ TableBuilder* NewTableBuilder(
       TableBuilderOptions(ioptions, internal_comparator,
                           int_tbl_prop_collector_factories, compression_type,
                           compression_opts, compression_dict, skip_filters,
-                          column_family_name),
+                          column_family_name, level),
       column_family_id, file);
 }
 
@@ -108,7 +109,7 @@ Status BuildTable(
       builder = NewTableBuilder(
           ioptions, internal_comparator, int_tbl_prop_collector_factories,
           column_family_id, column_family_name, file_writer.get(), compression,
-          compression_opts);
+          compression_opts, level);
     }
 
     MergeHelper merge(env, internal_comparator.user_comparator(),

diff --git a/db/builder.h b/db/builder.h
@@ -48,6 +48,7 @@ TableBuilder* NewTableBuilder(
     uint32_t column_family_id, const std::string& column_family_name,
     WritableFileWriter* file, const CompressionType compression_type,
     const CompressionOptions& compression_opts,
+    int level,
     const std::string* compression_dict = nullptr,
     const bool skip_filters = false);
 

diff --git a/db/compaction_job.cc b/db/compaction_job.cc
@@ -1136,7 +1136,9 @@ Status CompactionJob::OpenCompactionOutputFile(
       *cfd->ioptions(), cfd->internal_comparator(),
       cfd->int_tbl_prop_collector_factories(), cfd->GetID(), cfd->GetName(),
       sub_compact->outfile.get(), sub_compact->compaction->output_compression(),
-      cfd->ioptions()->compression_opts, &sub_compact->compression_dict,
+      cfd->ioptions()->compression_opts,
+      sub_compact->compaction->output_level(),
+      &sub_compact->compression_dict,
       skip_filters));
   LogFlush(db_options_.info_log);
   return s;

diff --git a/db/table_properties_collector_test.cc b/db/table_properties_collector_test.cc
@@ -46,11 +46,12 @@ void MakeBuilder(const Options& options, const ImmutableCFOptions& ioptions,
                  std::unique_ptr<TableBuilder>* builder) {
   unique_ptr<WritableFile> wf(new test::StringSink);
   writable->reset(new WritableFileWriter(std::move(wf), EnvOptions()));
-
+  int unknown_level = -1;
   builder->reset(NewTableBuilder(
       ioptions, internal_comparator, int_tbl_prop_collector_factories,
       kTestColumnFamilyId, kTestColumnFamilyName,
-      writable->get(), options.compression, options.compression_opts));
+      writable->get(), options.compression, options.compression_opts,
+      unknown_level));
 }
 }  // namespace
 

diff --git a/table/sst_file_writer.cc b/table/sst_file_writer.cc
@@ -130,12 +130,12 @@ Status SstFileWriter::Open(const std::string& file_path) {
         new UserKeyTablePropertiesCollectorFactory(
             user_collector_factories[i]));
   }
-
+  int unknown_level = -1;
   TableBuilderOptions table_builder_options(
       r->ioptions, r->internal_comparator, &int_tbl_prop_collector_factories,
       compression_type, r->ioptions.compression_opts,
       nullptr /* compression_dict */, false /* skip_filters */,
-      r->column_family_name);
+      r->column_family_name, unknown_level);
   r->file_writer.reset(
       new WritableFileWriter(std::move(sst_file), r->env_options));
   r->builder.reset(r->ioptions.table_factory->NewTableBuilder(

diff --git a/table/table_builder.h b/table/table_builder.h
@@ -54,15 +54,16 @@ struct TableBuilderOptions {
       CompressionType _compression_type,
       const CompressionOptions& _compression_opts,
       const std::string* _compression_dict, bool _skip_filters,
-      const std::string& _column_family_name)
+      const std::string& _column_family_name, int _level)
       : ioptions(_ioptions),
         internal_comparator(_internal_comparator),
         int_tbl_prop_collector_factories(_int_tbl_prop_collector_factories),
         compression_type(_compression_type),
         compression_opts(_compression_opts),
         compression_dict(_compression_dict),
         skip_filters(_skip_filters),
-        column_family_name(_column_family_name) {}
+        column_family_name(_column_family_name),
+        level(_level) {}
   const ImmutableCFOptions& ioptions;
   const InternalKeyComparator& internal_comparator;
   const std::vector<std::unique_ptr<IntTblPropCollectorFactory>>*
@@ -73,6 +74,7 @@ struct TableBuilderOptions {
   const std::string* compression_dict;
   bool skip_filters;  // only used by BlockBasedTableBuilder
   const std::string& column_family_name;
+  int level; // what level this table/file is on, -1 for "not set, don't know"
 };
 
 // TableBuilder provides the interface used to build a Table

diff --git a/table/table_reader_bench.cc b/table/table_reader_bench.cc
@@ -94,13 +94,14 @@ void TableReaderBenchmark(Options& opts, EnvOptions& env_options,
         int_tbl_prop_collector_factories;
 
     file_writer.reset(new WritableFileWriter(std::move(file), env_options));
-
+    int unknown_level = -1;
     tb = opts.table_factory->NewTableBuilder(
         TableBuilderOptions(ioptions, ikc, &int_tbl_prop_collector_factories,
                             CompressionType::kNoCompression,
                             CompressionOptions(),
                             nullptr /* compression_dict */,
-                            false /* skip_filters */, kDefaultColumnFamilyName),
+                            false /* skip_filters */, kDefaultColumnFamilyName,
+                            unknown_level),
         0 /* column_family_id */, file_writer.get());
   } else {
     s = DB::Open(opts, dbname, &db);

diff --git a/table/table_test.cc b/table/table_test.cc
@@ -307,12 +307,14 @@ class TableConstructor: public Constructor {
     std::vector<std::unique_ptr<IntTblPropCollectorFactory>>
         int_tbl_prop_collector_factories;
     std::string column_family_name;
+    int unknown_level = -1;
     builder.reset(ioptions.table_factory->NewTableBuilder(
         TableBuilderOptions(ioptions, internal_comparator,
                             &int_tbl_prop_collector_factories,
                             options.compression, CompressionOptions(),
                             nullptr /* compression_dict */,
-                            false /* skip_filters */, column_family_name),
+                            false /* skip_filters */, column_family_name,
+                            unknown_level),
         TablePropertiesCollectorFactory::Context::kUnknownColumnFamily,
         file_writer_.get()));
 
@@ -2210,11 +2212,13 @@ TEST_F(PlainTableTest, BasicPlainTableProperties) {
   std::vector<std::unique_ptr<IntTblPropCollectorFactory>>
       int_tbl_prop_collector_factories;
   std::string column_family_name;
+  int unknown_level = -1;
   std::unique_ptr<TableBuilder> builder(factory.NewTableBuilder(
       TableBuilderOptions(ioptions, ikc, &int_tbl_prop_collector_factories,
                           kNoCompression, CompressionOptions(),
                           nullptr /* compression_dict */,
-                          false /* skip_filters */, column_family_name),
+                          false /* skip_filters */, column_family_name,
+                          unknown_level),
       TablePropertiesCollectorFactory::Context::kUnknownColumnFamily,
       file_writer.get()));
 

diff --git a/tools/sst_dump_test.cc b/tools/sst_dump_test.cc
@@ -59,11 +59,13 @@ void createSST(const std::string& file_name,
   unique_ptr<WritableFileWriter> file_writer(
       new WritableFileWriter(std::move(file), EnvOptions()));
   std::string column_family_name;
+  int unknown_level = -1;
   tb.reset(opts.table_factory->NewTableBuilder(
       TableBuilderOptions(imoptions, ikc, &int_tbl_prop_collector_factories,
                           CompressionType::kNoCompression, CompressionOptions(),
                           nullptr /* compression_dict */,
-                          false /* skip_filters */, column_family_name),
+                          false /* skip_filters */, column_family_name,
+                          unknown_level),
       TablePropertiesCollectorFactory::Context::kUnknownColumnFamily,
       file_writer.get()));
 

diff --git a/tools/sst_dump_tool.cc b/tools/sst_dump_tool.cc
@@ -199,10 +199,12 @@ int SstFileReader::ShowAllCompressionSizes(size_t block_size) {
     if (CompressionTypeSupported(i.first)) {
       CompressionOptions compress_opt;
       std::string column_family_name;
+      int unknown_level = -1;
       TableBuilderOptions tb_opts(imoptions, ikc, &block_based_table_factories,
                                   i.first, compress_opt,
                                   nullptr /* compression_dict */,
-                                  false /* skip_filters */, column_family_name);
+                                  false /* skip_filters */, column_family_name,
+                                  unknown_level);
       uint64_t file_size = CalculateCompressedTableSize(tb_opts, block_size);
       fprintf(stdout, "Compression: %s", i.second);
       fprintf(stdout, " Size: %" PRIu64 "\n", file_size);