diff --git a/CHANGELOG.md b/CHANGELOG.md index 7ec7a02889..88a9e98fb0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,8 +24,10 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re - [#3378](https://github.com/thanos-io/thanos/pull/3378) Ruler: added the ability to send queries via the HTTP method POST. Helps when alerting/recording rules are extra long because it encodes the actual parameters inside of the body instead of the URI. Thanos Ruler now uses POST by default unless `--query.http-method` is set `GET`. - [#3381](https://github.com/thanos-io/thanos/pull/3381) Querier UI: Add ability to enable or disable metric autocomplete functionality. - [#2979](https://github.com/thanos-io/thanos/pull/2979) Replicator: Add the ability to replicate blocks within a time frame by passing --min-time and --max-time +- [#3409](https://github.com/thanos-io/thanos/pull/3409) Compactor: Added support for no-compact-mark.json which excludes the block from compaction. ### Fixed + - [#3257](https://github.com/thanos-io/thanos/pull/3257) Ruler: Prevent Ruler from crashing when using default DNS to lookup hosts that results in "No such hosts" errors. - [#3331](https://github.com/thanos-io/thanos/pull/3331) Disable Azure blob exception logging - [#3341](https://github.com/thanos-io/thanos/pull/3341) Disable Azure blob syslog exception logging @@ -33,6 +35,9 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re ### Changed +- [#3410](https://github.com/thanos-io/thanos/pull/3410) Compactor: Changed metric `thanos_compactor_blocks_marked_for_deletion_total` to `thanos_compactor_blocks_marked_total` with `marker` label. + Compactor will now automatically disable compaction for blocks with large index that would output blocks after compaction larger than specified value (by default: 64GB). This automatically + handles the Promethus [format limit](https://github.com/thanos-io/thanos/issues/1424). - [#2906](https://github.com/thanos-io/thanos/pull/2906) Tools: Refactor Bucket replicate execution. Removed all `thanos_replicate_origin_.*` metrics. - `thanos_replicate_origin_meta_loads_total` can be replaced by `blocks_meta_synced{state="loaded"}`. - `thanos_replicate_origin_partial_meta_reads_total` can be replaced by `blocks_meta_synced{state="failed"}`. diff --git a/cmd/thanos/compact.go b/cmd/thanos/compact.go index 9b7ab67086..5d0ab85794 100644 --- a/cmd/thanos/compact.go +++ b/cmd/thanos/compact.go @@ -125,10 +125,13 @@ func runCompact( Name: "thanos_compactor_block_cleanup_failures_total", Help: "Failures encountered while deleting blocks in compactor.", }) - blocksMarkedForDeletion := promauto.With(reg).NewCounter(prometheus.CounterOpts{ - Name: "thanos_compactor_blocks_marked_for_deletion_total", - Help: "Total number of blocks marked for deletion in compactor.", - }) + blocksMarked := promauto.With(reg).NewCounterVec(prometheus.CounterOpts{ + Name: "thanos_compactor_blocks_marked_total", + Help: "Total number of blocks marked in compactor.", + }, []string{"marker"}) + blocksMarked.WithLabelValues(metadata.NoCompactMarkFilename) + blocksMarked.WithLabelValues(metadata.DeletionMarkFilename) + garbageCollectedBlocks := promauto.With(reg).NewCounter(prometheus.CounterOpts{ Name: "thanos_compact_garbage_collected_blocks_total", Help: "Total number of blocks marked for deletion by compactor.", @@ -245,7 +248,7 @@ func runCompact( cf, duplicateBlocksFilter, ignoreDeletionMarkFilter, - blocksMarkedForDeletion, + blocksMarked.WithLabelValues(metadata.DeletionMarkFilename), garbageCollectedBlocks, conf.blockSyncConcurrency) if err != nil { @@ -281,9 +284,31 @@ func runCompact( return errors.Wrap(err, "clean working downsample directory") } - grouper := compact.NewDefaultGrouper(logger, bkt, conf.acceptMalformedIndex, enableVerticalCompaction, reg, blocksMarkedForDeletion, garbageCollectedBlocks) + grouper := compact.NewDefaultGrouper( + logger, + bkt, + conf.acceptMalformedIndex, + enableVerticalCompaction, + reg, + blocksMarked.WithLabelValues(metadata.DeletionMarkFilename), + garbageCollectedBlocks, + ) blocksCleaner := compact.NewBlocksCleaner(logger, bkt, ignoreDeletionMarkFilter, deleteDelay, blocksCleaned, blockCleanupFailures) - compactor, err := compact.NewBucketCompactor(logger, sy, grouper, compact.NewPlanner(logger, levels, noCompactMarkerFilter), comp, compactDir, bkt, conf.compactionConcurrency) + compactor, err := compact.NewBucketCompactor( + logger, + sy, + grouper, + compact.WithLargeTotalIndexSizeFilter( + compact.NewPlanner(logger, levels, noCompactMarkerFilter), + bkt, + int64(conf.maxBlockIndexSize), + blocksMarked.WithLabelValues(metadata.NoCompactMarkFilename), + ), + comp, + compactDir, + bkt, + conf.compactionConcurrency, + ) if err != nil { cancel() return errors.Wrap(err, "create bucket compactor") @@ -374,7 +399,7 @@ func runCompact( return errors.Wrap(err, "sync before first pass of downsampling") } - if err := compact.ApplyRetentionPolicyByResolution(ctx, logger, bkt, sy.Metas(), retentionByResolution, blocksMarkedForDeletion); err != nil { + if err := compact.ApplyRetentionPolicyByResolution(ctx, logger, bkt, sy.Metas(), retentionByResolution, blocksMarked.WithLabelValues(metadata.DeletionMarkFilename)); err != nil { return errors.Wrap(err, "retention failed") }