Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Limit force-full and cold compaction size. #10536

Merged
merged 1 commit into from
Dec 17, 2018
Merged

Limit force-full and cold compaction size. #10536

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 25 additions & 3 deletions tsdb/engine/tsm1/compact.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,10 @@ const (
TSMFileExtension = "tsm"
)

// ColdCompactionSplitSize is the maximum number of files to compact together
// when performing a compaction on a cold shard.
const ColdCompactionSplitSize = 8

var (
errMaxFileExceeded = fmt.Errorf("max file exceeded")
errSnapshotsDisabled = fmt.Errorf("snapshots disabled")
Expand Down Expand Up @@ -453,11 +457,16 @@ func (c *DefaultPlanner) Plan(lastWrite time.Time) []CompactionGroup {
return nil
}

group := []CompactionGroup{tsmFiles}
if !c.acquire(group) {
var groups []CompactionGroup
if forceFull {
groups = []CompactionGroup{tsmFiles}
} else {
groups = splitCompactionGroups(tsmFiles, ColdCompactionSplitSize)
}
if !c.acquire(groups) {
return nil
}
return group
return groups
}

// don't plan if nothing has changed in the filestore
Expand Down Expand Up @@ -2098,3 +2107,16 @@ func (l *latencies) avg() time.Duration {
}
return time.Duration(0)
}

// splitCompactionGroups returns groups with a maximum size of sz.
func splitCompactionGroups(tsmFiles []string, sz int) []CompactionGroup {
var groups []CompactionGroup
for i := 0; i < len(tsmFiles); i += sz {
var group CompactionGroup
for j := 0; j < sz && i+j < len(tsmFiles); j++ {
group = append(group, tsmFiles[i+j])
}
groups = append(groups, group)
}
return groups
}
60 changes: 52 additions & 8 deletions tsdb/engine/tsm1/compact_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2325,7 +2325,7 @@ func TestDefaultPlanner_PlanOptimize_Tombstones(t *testing.T) {

}

// Ensure that the planner will compact all files if no writes
// Ensure that the planner will compact files in groups if no writes
// have happened in some interval
func TestDefaultPlanner_Plan_FullOnCold(t *testing.T) {
data := []tsm1.FileStat{
Expand Down Expand Up @@ -2353,6 +2353,26 @@ func TestDefaultPlanner_Plan_FullOnCold(t *testing.T) {
Path: "06-01.tsm1",
Size: 2 * 1024 * 1024,
},
{
Path: "07-01.tsm1",
Size: 2 * 1024 * 1024,
},
{
Path: "08-01.tsm1",
Size: 2 * 1024 * 1024,
},
{
Path: "09-01.tsm1",
Size: 2 * 1024 * 1024,
},
{
Path: "10-01.tsm1",
Size: 2 * 1024 * 1024,
},
{
Path: "11-01.tsm1",
Size: 2 * 1024 * 1024,
},
}

cp := tsm1.NewDefaultPlanner(
Expand All @@ -2365,14 +2385,38 @@ func TestDefaultPlanner_Plan_FullOnCold(t *testing.T) {
)

tsm := cp.Plan(time.Now().Add(-time.Second))
if exp, got := len(data), len(tsm[0]); got != exp {
t.Fatalf("tsm file length mismatch: got %v, exp %v", got, exp)
if exp, got := 2, len(tsm); got != exp {
t.Fatalf("tsm groups length mismatch: got %v, exp %v", got, exp)
}

for i, p := range data {
if got, exp := tsm[0][i], p.Path; got != exp {
t.Fatalf("tsm file mismatch: got %v, exp %v", got, exp)
}
if exp, got := 8, len(tsm[0]); got != exp {
t.Fatalf("tsm file length mismatch: got %v, exp %v", got, exp)
} else if got, exp := tsm[0][0], data[0].Path; got != exp {
t.Fatalf("tsm file mismatch: got %v, exp %v", got, exp)
} else if got, exp := tsm[0][1], data[1].Path; got != exp {
t.Fatalf("tsm file mismatch: got %v, exp %v", got, exp)
} else if got, exp := tsm[0][2], data[2].Path; got != exp {
t.Fatalf("tsm file mismatch: got %v, exp %v", got, exp)
} else if got, exp := tsm[0][3], data[3].Path; got != exp {
t.Fatalf("tsm file mismatch: got %v, exp %v", got, exp)
} else if got, exp := tsm[0][4], data[4].Path; got != exp {
t.Fatalf("tsm file mismatch: got %v, exp %v", got, exp)
} else if got, exp := tsm[0][5], data[5].Path; got != exp {
t.Fatalf("tsm file mismatch: got %v, exp %v", got, exp)
} else if got, exp := tsm[0][6], data[6].Path; got != exp {
t.Fatalf("tsm file mismatch: got %v, exp %v", got, exp)
} else if got, exp := tsm[0][7], data[7].Path; got != exp {
t.Fatalf("tsm file mismatch: got %v, exp %v", got, exp)
}

if exp, got := 3, len(tsm[1]); got != exp {
t.Fatalf("tsm file length mismatch: got %v, exp %v", got, exp)
} else if got, exp := tsm[1][0], data[8].Path; got != exp {
t.Fatalf("tsm file mismatch: got %v, exp %v", got, exp)
} else if got, exp := tsm[1][1], data[9].Path; got != exp {
t.Fatalf("tsm file mismatch: got %v, exp %v", got, exp)
} else if got, exp := tsm[1][2], data[10].Path; got != exp {
t.Fatalf("tsm file mismatch: got %v, exp %v", got, exp)
}
}

Expand Down Expand Up @@ -2552,7 +2596,7 @@ func TestDefaultPlanner_Plan_TwoGenLevel3(t *testing.T) {
time.Hour)

tsm := cp.Plan(time.Now().Add(-24 * time.Hour))
if exp, got := 1, len(tsm); got != exp {
if exp, got := 2, len(tsm); got != exp {
t.Fatalf("tsm file length mismatch: got %v, exp %v", got, exp)
}
}
Expand Down