Strictly check inner space compaction file size (apache#11919)

Caideyipi · Jan 17, 2024 · 22f915f · 22f915f
1 parent e9e01f3
commit 22f915f
Showing 1 changed file with 28 additions and 13 deletions.
diff --git a/...db/db/storageengine/dataregion/compaction/selector/impl/SizeTieredCompactionSelector.java b/...db/db/storageengine/dataregion/compaction/selector/impl/SizeTieredCompactionSelector.java
@@ -118,26 +118,41 @@ private List<List<TsFileResource>> selectTsFileResourcesByLevel(int level) throw
         selectedFileSize = 0L;
         continue;
       }
-      LOGGER.debug("Current File is {}, size is {}", currentFile, currentFile.getTsFileSize());
-      selectedFileList.add(currentFile);
-      selectedFileSize += currentFile.getTsFileSize();
-      LOGGER.debug(
-          "Add tsfile {}, current select file num is {}, size is {}",
-          currentFile,
-          selectedFileList.size(),
-          selectedFileSize);
-      // if the file size or file num reach threshold
-      if (selectedFileSize >= targetCompactionFileSize
-          || selectedFileList.size() >= config.getFileLimitPerInnerTask()) {
-        // submit the task
+
+      long totalSizeIfSelectCurrentFile = selectedFileSize + currentFile.getTsFileSize();
+      boolean canNotAddCurrentFileIntoCurrentTask =
+          totalSizeIfSelectCurrentFile > targetCompactionFileSize
+              || selectedFileList.size() >= config.getFileLimitPerInnerTask();
+      if (canNotAddCurrentFileIntoCurrentTask) {
+        // total file size or num will beyond the threshold if select current file, stop the
+        // selection of current task
         if (selectedFileList.size() > 1) {
+          // submit the task
           taskList.add(new ArrayList<>(selectedFileList));
         }
+        // add current file in a new selected file list
         selectedFileList = new ArrayList<>();
-        selectedFileSize = 0L;
+        selectedFileList.add(currentFile);
+        selectedFileSize = currentFile.getTsFileSize();
+      } else {
+        LOGGER.debug("Current File is {}, size is {}", currentFile, currentFile.getTsFileSize());
+        selectedFileList.add(currentFile);
+        selectedFileSize += currentFile.getTsFileSize();
+        LOGGER.debug(
+            "Add tsfile {}, current select file num is {}, size is {}",
+            currentFile,
+            selectedFileList.size(),
+            selectedFileSize);
       }
     }
 
+    // if the selected file size reach the condition to submit
+    if (selectedFileList.size() == config.getFileLimitPerInnerTask()) {
+      taskList.add(new ArrayList<>(selectedFileList));
+      selectedFileList.clear();
+      selectedFileSize = 0;
+    }
+
     // if next time partition exists
     // submit a merge task even it does not meet the requirement for file num or file size
     if (hasNextTimePartition && selectedFileList.size() > 1) {