Skip to content

Commit

Permalink
Add seriesLimitSupported to handler
Browse files Browse the repository at this point in the history
Signed-off-by: Saswata Mukherjee <saswataminsta@yahoo.com>
  • Loading branch information
saswatamcode committed Aug 1, 2022
1 parent 076164b commit d101a61
Show file tree
Hide file tree
Showing 2 changed files with 16 additions and 20 deletions.
14 changes: 6 additions & 8 deletions cmd/thanos/receive.go
Original file line number Diff line number Diff line change
Expand Up @@ -186,6 +186,9 @@ func runReceive(
return errors.Wrap(err, "parse relabel configuration")
}

// Impose active series limit only if Receiver is in Router or RouterIngestor mode, and config is provided.
seriesLimitSupported := (receiveMode == receive.RouterOnly || receiveMode == receive.RouterIngestor) && conf.maxPerTenantLimit != 0

dbs := receive.NewMultiTSDB(
conf.dataDir,
logger,
Expand Down Expand Up @@ -215,6 +218,7 @@ func runReceive(
DialOpts: dialOpts,
ForwardTimeout: time.Duration(*conf.forwardTimeout),
TSDBStats: dbs,
SeriesLimitSupported: seriesLimitSupported,
MaxPerTenantLimit: conf.maxPerTenantLimit,
MetaMonitoringUrl: conf.metaMonitoringUrl,
MetaMonitoringHttpClient: conf.metaMonitoringHttpClient,
Expand Down Expand Up @@ -298,8 +302,7 @@ func runReceive(
)
}

seriesLimitSupported := receiveMode == receive.RouterOnly || receiveMode == receive.RouterIngestor
if seriesLimitSupported && conf.maxPerTenantLimit != 0 {
if seriesLimitSupported {
level.Info(logger).Log("msg", "setting up periodic (every 15s) meta-monitoring query for limiting cache")
{
ctx, cancel := context.WithCancel(context.Background())
Expand Down Expand Up @@ -855,12 +858,7 @@ func (rc *receiveConfig) registerFlag(cmd extkingpin.FlagClause) {

cmd.Flag("receive.tenant-limits.meta-monitoring-query", "PromQL Query to execute against meta-monitoring, to get the current number of active series for each tenant, across Receive replicas.").Default("sum(prometheus_tsdb_head_series) by (tenant)").Hidden().StringVar(&rc.metaMonitoringLimitQuery)

rc.metaMonitoringHttpClient = extflag.RegisterPathOrContent(
cmd,
"receive.tenant-limits.meta-monitoring-client",
"YAML file or string with http client configs for meta-monitoring.",
extflag.WithHidden(),
)
rc.metaMonitoringHttpClient = extflag.RegisterPathOrContent(cmd, "receive.tenant-limits.meta-monitoring-client", "YAML file or string with http client configs for meta-monitoring.", extflag.WithHidden())

rc.forwardTimeout = extkingpin.ModelDuration(cmd.Flag("receive-forward-timeout", "Timeout for each forward request.").Default("5s").Hidden())

Expand Down
22 changes: 10 additions & 12 deletions pkg/receive/handler.go
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,7 @@ type Options struct {
ForwardTimeout time.Duration
RelabelConfigs []*relabel.Config
TSDBStats TSDBStats
SeriesLimitSupported bool
MaxPerTenantLimit uint64
MetaMonitoringUrl *url.URL
MetaMonitoringHttpClient *extflag.PathOrContent
Expand Down Expand Up @@ -212,7 +213,7 @@ func NewHandler(logger log.Logger, o *Options) *Handler {
}

h.ActiveSeriesLimit = NewNopSeriesLimit()
if (h.receiverMode == RouterOnly || h.receiverMode == RouterIngestor) && o.MaxPerTenantLimit != 0 {
if h.options.SeriesLimitSupported {
h.ActiveSeriesLimit = NewActiveSeriesLimit(h.options, registerer, h.receiverMode, logger)
}

Expand Down Expand Up @@ -417,18 +418,15 @@ func (h *Handler) receiveHTTP(w http.ResponseWriter, r *http.Request) {

tLogger := log.With(h.logger, "tenant", tenant)

// Impose limits only if Receive is in Router or RouterIngestor mode.
if h.receiverMode == RouterOnly || h.receiverMode == RouterIngestor {
under, err := h.ActiveSeriesLimit.isUnderLimit(tenant, tLogger)
if err != nil {
level.Error(tLogger).Log("msg", "error while limiting", "err", err.Error())
}
under, err := h.ActiveSeriesLimit.isUnderLimit(tenant, tLogger)
if err != nil {
level.Error(tLogger).Log("msg", "error while limiting", "err", err.Error())
}

// Fail request fully if tenant has exceeded set limit.
if !under {
http.Error(w, "tenant is above active series limit", http.StatusTooManyRequests)
return
}
// Fail request fully if tenant has exceeded set limit.
if !under {
http.Error(w, "tenant is above active series limit", http.StatusTooManyRequests)
return
}

// ioutil.ReadAll dynamically adjust the byte slice for read data, starting from 512B.
Expand Down

0 comments on commit d101a61

Please sign in to comment.