extend_replication_when_instance_heartbeat_timeout

Signed-off-by: lanmengran1 <lanmengran1@jd.com>

CHANGELOG.md

@@ -56,6 +56,7 @@
 * [BUGFIX] Querier: fixed panic when querying exemplars and using `-distributor.shard-by-all-labels=false`. #4473
 * [BUGFIX] Querier: honor querier minT,maxT if `nil` SelectHints are passed to Select(). #4413
 * [BUGFIX] Compactor: fixed panic while collecting Prometheus metrics. #4483
+* [BUGFIX] Ring: extend replication when instance heartbeat timeoutgst. #4493
 
 
 ## 1.10.0 / 2021-08-03

pkg/ring/ring.go

@@ -375,10 +375,15 @@ func (r *Ring) Get(key uint32, op Operation, bufDescs []InstanceDesc, bufHosts,
 
 		distinctHosts = append(distinctHosts, info.InstanceID)
 		instance := r.ringDesc.Ingesters[info.InstanceID]
-
+		state := instance.State
+		// Heartbeat unhealthy instance also needs to extend replication.
+		// Since the ring will not update instance state automatically, changing the state to LEFT instead.
+		if !instance.IsHeartbeatHealthy(r.cfg.HeartbeatTimeout, time.Now()) {
+			state = LEFT
+		}
 		// Check whether the replica set should be extended given we're including
 		// this instance.
-		if op.ShouldExtendReplicaSetOnState(instance.State) {
+		if op.ShouldExtendReplicaSetOnState(state) {
 			n++
 		} else if r.cfg.ZoneAwarenessEnabled && info.Zone != "" {
 			// We should only add the zone if we are not going to extend,