base: use separate lease duration for liveness range

This patch adds a separate lease duration for the liveness range. When
the liveness range leaseholder is lost, other nodes are unable to
heartbeat and extend their leases, which can lead to the loss of most
leases in the system. Shortening the liveness lease duration shortens
the period of unavailability for other leases.

Release note (ops change): The duration of the liveness range lease has
been reduced to 2.5s, half of the regular lease duration. When the
liveness range leaseholder is lost (e.g. due to an infrastructure
outage), other nodes are unable to heartbeat and thus extend their own
leases, which can lead to the loss of many of the leases throughtout the
cluster. Reducing the liveness lease duration reduces the period of
unavailability.

pkg/base/config.go

@@ -114,6 +114,18 @@ const (
 	// eagerly renewed 8 seconds into each lease.
 	defaultRangeLeaseRenewalFraction = 0.5
 
+	// defaultLivenessRangeMultiplier specifies the liveness range lease active
+	// duration as a multiple of the standard range lease active duration. When
+	// the liveness leaseholder is lost, other nodes will be unable to heartbeat
+	// and extend their own leases, which can cause a loss of all leases in the
+	// cluster. It is set to half of the regular lease active duration to
+	// counteract this.
+	defaultLivenessRangeLeaseActiveMultiplier = 0.5
+
+	// defaultLivenessRangeLeaseRenewalFraction is like
+	// defaultRangeLeaseRenewalFraction, but for the liveness range lease.
+	defaultLivenessRangeLeaseRenewalFraction = 0.6
+
 	// livenessRenewalFraction specifies what fraction the node liveness
 	// renewal duration should be of the node liveness duration. For example,
 	// with a value of 0.2 and a liveness duration of 10 seconds, each node's
@@ -412,6 +424,17 @@ type RaftConfig struct {
 	// and a value of -1 means never preemptively renew the lease. A value of 1
 	// means always renew.
 	RangeLeaseRenewalFraction float64
+	// LivenessRangeLeaseActiveMultiplier specifies what multiple the liveness
+	// range active duration should be of the regular lease active duration. When
+	// the liveness leaseholder is lost, other nodes are unable to heartbeat and
+	// extend their epoch-based leases, which can cascade into a loss of all
+	// leases in the cluster, so this should be set lower than that.
+	LivenessRangeLeaseActiveMultiplier float64
+	// LivenessRangeLeaseRenewalFraction is like RangeLeaseRenewalFraction but for
+	// the liveness range lease. It is set to 0.6 such that, in the default
+	// configuration, a 2.5s liveness lease is renewed every 1.0s, giving it 1.0s
+	// to succeed before entering the 0.5s stasis period.
+	LivenessRangeLeaseRenewalFraction float64
 
 	// RaftLogTruncationThreshold controls how large a single Range's Raft log
 	// can grow. When a Range's Raft log grows above this size, the Range will
@@ -486,6 +509,12 @@ func (cfg *RaftConfig) SetDefaults() {
 	if cfg.RangeLeaseRenewalFraction == 0 {
 		cfg.RangeLeaseRenewalFraction = defaultRangeLeaseRenewalFraction
 	}
+	if cfg.LivenessRangeLeaseActiveMultiplier == 0 {
+		cfg.LivenessRangeLeaseActiveMultiplier = defaultLivenessRangeLeaseActiveMultiplier
+	}
+	if cfg.LivenessRangeLeaseRenewalFraction == 0 {
+		cfg.LivenessRangeLeaseRenewalFraction = defaultLivenessRangeLeaseRenewalFraction
+	}
 	// TODO(andrei): -1 is a special value for RangeLeaseRenewalFraction which
 	// really means "0" (never renew), except that the zero value means "use
 	// default". We can't turn the -1 into 0 here because, unfortunately,
@@ -581,6 +610,19 @@ func (cfg RaftConfig) RangeLeaseAcquireTimeout() time.Duration {
 	return 2 * cfg.RaftElectionTimeout()
 }
 
+// LivenessRangeLeaseDurations computes durations for range lease expiration and
+// renewal based for the liveness range lease.
+func (cfg RaftConfig) LivenessRangeLeaseDurations() (active, renewal time.Duration) {
+	active = time.Duration(cfg.LivenessRangeLeaseActiveMultiplier *
+		float64(cfg.RangeLeaseActiveDuration()))
+	renewalFraction := cfg.LivenessRangeLeaseRenewalFraction
+	if renewalFraction == -1 {
+		renewalFraction = 0
+	}
+	renewal = time.Duration(float64(active) * renewalFraction)
+	return
+}
+
 // NodeLivenessDurations computes durations for node liveness expiration and
 // renewal based on a default multiple of Raft election timeout.
 func (cfg RaftConfig) NodeLivenessDurations() (livenessActive, livenessRenewal time.Duration) {

pkg/base/config_test.go

@@ -30,6 +30,7 @@ func TestDefaultRaftConfig(t *testing.T) {
 
 	// Assert the config and various derived values.
 	leaseActive, leaseRenewal := cfg.RangeLeaseDurations()
+	livenessActive, livenessRenewal := cfg.LivenessRangeLeaseDurations()
 	nodeActive, nodeRenewal := cfg.NodeLivenessDurations()
 	raftElectionTimeout := cfg.RaftElectionTimeout()
 	raftHeartbeatInterval := cfg.RaftTickInterval * time.Duration(cfg.RaftHeartbeatIntervalTicks)
@@ -41,6 +42,8 @@ func TestDefaultRaftConfig(t *testing.T) {
 		s += fmt.Sprintf("RaftElectionTimeout: %s\n", raftElectionTimeout)
 		s += fmt.Sprintf("RangeLeaseDurations: active=%s renewal=%s\n", leaseActive, leaseRenewal)
 		s += fmt.Sprintf("RangeLeaseAcquireTimeout: %s\n", cfg.RangeLeaseAcquireTimeout())
+		s += fmt.Sprintf("LivenessRangeLeaseDurations: active=%s renewal=%s\n",
+			livenessActive, livenessRenewal)
 		s += fmt.Sprintf("NodeLivenessDurations: active=%s renewal=%s\n", nodeActive, nodeRenewal)
 		s += fmt.Sprintf("SentinelGossipTTL: %s\n", cfg.SentinelGossipTTL())
 		echotest.Require(t, s, testutils.TestDataPath(t, "raft_config"))

pkg/base/testdata/raft_config

@@ -6,6 +6,8 @@ echo
  RaftHeartbeatIntervalTicks: (int) 5,
  RangeLeaseRaftElectionTimeoutMultiplier: (float64) 2.5,
  RangeLeaseRenewalFraction: (float64) 0.5,
+ LivenessRangeLeaseActiveMultiplier: (float64) 0.5,
+ LivenessRangeLeaseRenewalFraction: (float64) 0.6,
  RaftLogTruncationThreshold: (int64) 16777216,
  RaftProposalQuota: (int64) 8388608,
  RaftMaxUncommittedEntriesSize: (uint64) 16777216,
@@ -18,5 +20,6 @@ RaftHeartbeatInterval: 1s
 RaftElectionTimeout: 2s
 RangeLeaseDurations: active=5s renewal=2.5s
 RangeLeaseAcquireTimeout: 4s
+LivenessRangeLeaseDurations: active=2.5s renewal=1.5s
 NodeLivenessDurations: active=5s renewal=2.5s
 SentinelGossipTTL: 5s

pkg/kv/kvserver/replica_range_lease.go

@@ -273,8 +273,14 @@ func (p *pendingLeaseRequest) InitOrJoinRequest(
 		// based leases, it's possible for the new leaseholder that's delayed
 		// in applying the lease transfer to maintain its lease (assuming the
 		// node it's on is able to heartbeat its liveness record).
+		var active time.Duration
+		if p.repl.isLivenessRangeRLocked() {
+			active, _ = p.repl.store.cfg.LivenessRangeLeaseDurations()
+		} else {
+			active = p.repl.store.cfg.RangeLeaseActiveDuration()
+		}
 		reqLease.Expiration = &hlc.Timestamp{}
-		*reqLease.Expiration = status.Now.ToTimestamp().Add(int64(p.repl.store.cfg.RangeLeaseActiveDuration()), 0)
+		*reqLease.Expiration = status.Now.ToTimestamp().Add(active.Nanoseconds(), 0)
 	} else {
 		// Get the liveness for the next lease holder and set the epoch in the lease request.
 		l, ok := p.repl.store.cfg.NodeLiveness.GetLiveness(nextLeaseHolder.NodeID)
@@ -803,6 +809,13 @@ func (r *Replica) requiresExpiringLeaseRLocked() bool {
 		r.mu.state.Desc.StartKey.Less(roachpb.RKey(keys.NodeLivenessKeyMax))
 }
 
+// isLivenessRange returns true if this range contains the node liveness data.
+func (r *Replica) isLivenessRangeRLocked() bool {
+	return r.store.cfg.NodeLiveness != nil &&
+		r.mu.state.Desc.StartKey.Compare(roachpb.RKey(keys.NodeLivenessKeyMax)) < 0 &&
+		r.mu.state.Desc.EndKey.Compare(roachpb.RKey(keys.NodeLivenessPrefix)) > 0
+}
+
 // requestLeaseLocked executes a request to obtain or extend a lease
 // asynchronously and returns a channel on which the result will be posted. If
 // there's already a request in progress, we join in waiting for the results of
@@ -1032,10 +1045,12 @@ func NewLeaseTransferRejectedBecauseTargetMayNeedSnapshotError(
 // lease's expiration (and stasis period).
 func (r *Replica) checkRequestTimeRLocked(now hlc.ClockTimestamp, reqTS hlc.Timestamp) error {
 	var leaseRenewal time.Duration
-	if r.requiresExpiringLeaseRLocked() {
-		_, leaseRenewal = r.store.cfg.RangeLeaseDurations()
-	} else {
+	if !r.requiresExpiringLeaseRLocked() {
 		_, leaseRenewal = r.store.cfg.NodeLivenessDurations()
+	} else if r.isLivenessRangeRLocked() {
+		_, leaseRenewal = r.store.cfg.LivenessRangeLeaseDurations()
+	} else {
+		_, leaseRenewal = r.store.cfg.RangeLeaseDurations()
 	}
 	leaseRenewalMinusStasis := leaseRenewal - r.store.Clock().MaxOffset()
 	if leaseRenewalMinusStasis < 0 {
@@ -1419,7 +1434,13 @@ func (r *Replica) shouldExtendLeaseRLocked(st kvserverpb.LeaseStatus) bool {
 	if _, ok := r.mu.pendingLeaseRequest.RequestPending(); ok {
 		return false
 	}
-	renewal := st.Lease.Expiration.Add(-r.store.cfg.RangeLeaseRenewalDuration().Nanoseconds(), 0)
+	var renewalDuration time.Duration
+	if r.isLivenessRangeRLocked() {
+		_, renewalDuration = r.store.cfg.LivenessRangeLeaseDurations()
+	} else {
+		renewalDuration = r.store.cfg.RangeLeaseRenewalDuration()
+	}
+	renewal := st.Lease.Expiration.Add(-renewalDuration.Nanoseconds(), 0)
 	return renewal.LessEq(st.Now.ToTimestamp())
 }