From c558d06288bf5cc82a0daab5bdabccdfb8f7ffe8 Mon Sep 17 00:00:00 2001 From: Hans Hasselberg Date: Mon, 3 Feb 2020 12:48:06 +0100 Subject: [PATCH 1/2] Increase watchLimit to 8192. The previous value was too conservative and users with many instances were having problems because of it. This change increases the limit to 8192 which reportedly fixed most of the issues with that. Related: #4984, #4986, #5050. --- agent/consul/state/state_store.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/agent/consul/state/state_store.go b/agent/consul/state/state_store.go index e71c002f283f..6a8d9d8a8fc6 100644 --- a/agent/consul/state/state_store.go +++ b/agent/consul/state/state_store.go @@ -84,10 +84,7 @@ const ( // higher-level watch that's less fine-grained. This isn't as bad as it // seems since we have made the main culprits (nodes and services) more // efficient by diffing before we update via register requests. - // - // Given the current size of aFew == 32 in memdb's watch_few.go, this - // will allow for up to ~64 goroutines per blocking query. - watchLimit = 2048 + watchLimit = 8192 ) // Store is where we store all of Consul's state, including From 200a26839b8699d255c2a6b46bc7c59ccff70212 Mon Sep 17 00:00:00 2001 From: Hans Hasselberg Date: Tue, 4 Feb 2020 10:41:48 +0100 Subject: [PATCH 2/2] update comment --- agent/consul/state/state_store.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/agent/consul/state/state_store.go b/agent/consul/state/state_store.go index 6a8d9d8a8fc6..c07fcf945d11 100644 --- a/agent/consul/state/state_store.go +++ b/agent/consul/state/state_store.go @@ -81,9 +81,13 @@ var ( const ( // watchLimit is used as a soft limit to cap how many watches we allow // for a given blocking query. If this is exceeded, then we will use a - // higher-level watch that's less fine-grained. This isn't as bad as it - // seems since we have made the main culprits (nodes and services) more - // efficient by diffing before we update via register requests. + // higher-level watch that's less fine-grained. Choosing the perfect + // value is impossible given how different deployments and workload + // are. This value was recommended by customers with many servers. We + // expect streaming to arrive soon and that should help a lot with + // blocking queries. Please see + // https://github.com/hashicorp/consul/pull/7200 and linked issues/prs + // for more context watchLimit = 8192 )