Add ability to configure allowlist limits

limit.go

@@ -29,6 +29,8 @@ type Limit interface {
 type Limiter interface {
 	GetSystemLimits() Limit
 	GetTransientLimits() Limit
+	GetAllowlistedSystemLimits() Limit
+	GetAllowlistedTransientLimits() Limit
 	GetServiceLimits(svc string) Limit
 	GetServicePeerLimits(svc string) Limit
 	GetProtocolLimits(proto protocol.ID) Limit
@@ -196,6 +198,14 @@ func (l *fixedLimiter) GetTransientLimits() Limit {
 	return &l.Transient
 }
 
+func (l *fixedLimiter) GetAllowlistedSystemLimits() Limit {
+	return &l.AllowlistedSystem
+}
+
+func (l *fixedLimiter) GetAllowlistedTransientLimits() Limit {
+	return &l.AllowlistedTransient
+}
+
 func (l *fixedLimiter) GetServiceLimits(svc string) Limit {
 	sl, ok := l.Service[svc]
 	if !ok {

limit_defaults.go

@@ -24,6 +24,12 @@ type ScalingLimitConfig struct {
 	TransientBaseLimit     BaseLimit
 	TransientLimitIncrease BaseLimitIncrease
 
+	AllowlistedSystemBaseLimit     BaseLimit
+	AllowlistedSystemLimitIncrease BaseLimitIncrease
+
+	AllowlistedTransientBaseLimit     BaseLimit
+	AllowlistedTransientLimitIncrease BaseLimitIncrease
+
 	ServiceBaseLimit     BaseLimit
 	ServiceLimitIncrease BaseLimitIncrease
 	ServiceLimits        map[string]baseLimitConfig // use AddServiceLimit to modify
@@ -105,6 +111,12 @@ type LimitConfig struct {
 	System    BaseLimit `json:",omitempty"`
 	Transient BaseLimit `json:",omitempty"`
 
+	// Limits that are applied to resources with an allowlisted multiaddr.
+	// These will only be used if the normal System & Transient limits are
+	// reached.
+	AllowlistedSystem    BaseLimit `json:",omitempty"`
+	AllowlistedTransient BaseLimit `json:",omitempty"`
+
 	ServiceDefault BaseLimit            `json:",omitempty"`
 	Service        map[string]BaseLimit `json:",omitempty"`
 
@@ -127,6 +139,8 @@ type LimitConfig struct {
 func (cfg *LimitConfig) Apply(c LimitConfig) {
 	cfg.System.Apply(c.System)
 	cfg.Transient.Apply(c.Transient)
+	cfg.AllowlistedSystem.Apply(c.AllowlistedSystem)
+	cfg.AllowlistedTransient.Apply(c.AllowlistedTransient)
 	cfg.ServiceDefault.Apply(c.ServiceDefault)
 	cfg.ProtocolDefault.Apply(c.ProtocolDefault)
 	cfg.ProtocolPeerDefault.Apply(c.ProtocolPeerDefault)
@@ -232,15 +246,17 @@ func (cfg *ScalingLimitConfig) Scale(memory int64, numFD int) LimitConfig {
 		scaleFactor = int((memory - 128<<20) >> 20)
 	}
 	lc := LimitConfig{
-		System:              scale(cfg.SystemBaseLimit, cfg.SystemLimitIncrease, scaleFactor, numFD),
-		Transient:           scale(cfg.TransientBaseLimit, cfg.TransientLimitIncrease, scaleFactor, numFD),
-		ServiceDefault:      scale(cfg.ServiceBaseLimit, cfg.ServiceLimitIncrease, scaleFactor, numFD),
-		ServicePeerDefault:  scale(cfg.ServicePeerBaseLimit, cfg.ServicePeerLimitIncrease, scaleFactor, numFD),
-		ProtocolDefault:     scale(cfg.ProtocolBaseLimit, cfg.ProtocolLimitIncrease, scaleFactor, numFD),
-		ProtocolPeerDefault: scale(cfg.ProtocolPeerBaseLimit, cfg.ProtocolPeerLimitIncrease, scaleFactor, numFD),
-		PeerDefault:         scale(cfg.PeerBaseLimit, cfg.PeerLimitIncrease, scaleFactor, numFD),
-		Conn:                scale(cfg.ConnBaseLimit, cfg.ConnLimitIncrease, scaleFactor, numFD),
-		Stream:              scale(cfg.StreamBaseLimit, cfg.ConnLimitIncrease, scaleFactor, numFD),
+		System:               scale(cfg.SystemBaseLimit, cfg.SystemLimitIncrease, scaleFactor, numFD),
+		Transient:            scale(cfg.TransientBaseLimit, cfg.TransientLimitIncrease, scaleFactor, numFD),
+		AllowlistedSystem:    scale(cfg.AllowlistedSystemBaseLimit, cfg.AllowlistedSystemLimitIncrease, scaleFactor, numFD),
+		AllowlistedTransient: scale(cfg.AllowlistedTransientBaseLimit, cfg.AllowlistedTransientLimitIncrease, scaleFactor, numFD),
+		ServiceDefault:       scale(cfg.ServiceBaseLimit, cfg.ServiceLimitIncrease, scaleFactor, numFD),
+		ServicePeerDefault:   scale(cfg.ServicePeerBaseLimit, cfg.ServicePeerLimitIncrease, scaleFactor, numFD),
+		ProtocolDefault:      scale(cfg.ProtocolBaseLimit, cfg.ProtocolLimitIncrease, scaleFactor, numFD),
+		ProtocolPeerDefault:  scale(cfg.ProtocolPeerBaseLimit, cfg.ProtocolPeerLimitIncrease, scaleFactor, numFD),
+		PeerDefault:          scale(cfg.PeerBaseLimit, cfg.PeerLimitIncrease, scaleFactor, numFD),
+		Conn:                 scale(cfg.ConnBaseLimit, cfg.ConnLimitIncrease, scaleFactor, numFD),
+		Stream:               scale(cfg.StreamBaseLimit, cfg.ConnLimitIncrease, scaleFactor, numFD),
 	}
 	if cfg.ServiceLimits != nil {
 		lc.Service = make(map[string]BaseLimit)
@@ -346,6 +362,55 @@ var DefaultLimits = ScalingLimitConfig{
 		FDFraction:      0.25,
 	},
 
+	// Setting the allowlisted limits to be the same as the normal limits. The
+	// allowlist only activates when you reach your normal system/transient
+	// limits. So it's okay if these limits err on the side of being too big,
+	// since most of the time you won't even use any of these. Tune these down
+	// if you want to manager your resources against an allowlisted endpoint.
+	AllowlistedSystemBaseLimit: BaseLimit{
+		ConnsInbound:    64,
+		ConnsOutbound:   128,
+		Conns:           128,
+		StreamsInbound:  64 * 16,
+		StreamsOutbound: 128 * 16,
+		Streams:         128 * 16,
+		Memory:          128 << 20,
+		FD:              256,
+	},
+
+	AllowlistedSystemLimitIncrease: BaseLimitIncrease{
+		ConnsInbound:    64,
+		ConnsOutbound:   128,
+		Conns:           128,
+		StreamsInbound:  64 * 16,
+		StreamsOutbound: 128 * 16,
+		Streams:         128 * 16,
+		Memory:          1 << 30,
+		FDFraction:      1,
+	},
+
+	AllowlistedTransientBaseLimit: BaseLimit{
+		ConnsInbound:    32,
+		ConnsOutbound:   64,
+		Conns:           64,
+		StreamsInbound:  128,
+		StreamsOutbound: 256,
+		Streams:         256,
+		Memory:          32 << 20,
+		FD:              64,
+	},
+
+	AllowlistedTransientLimitIncrease: BaseLimitIncrease{
+		ConnsInbound:    16,
+		ConnsOutbound:   32,
+		Conns:           32,
+		StreamsInbound:  128,
+		StreamsOutbound: 256,
+		Streams:         256,
+		Memory:          128 << 20,
+		FDFraction:      0.25,
+	},
+
 	ServiceBaseLimit: BaseLimit{
 		StreamsInbound:  1024,
 		StreamsOutbound: 4096,
@@ -451,13 +516,15 @@ var infiniteBaseLimit = BaseLimit{
 // InfiniteLimits are a limiter configuration that uses infinite limits, thus effectively not limiting anything.
 // Keep in mind that the operating system limits the number of file descriptors that an application can use.
 var InfiniteLimits = LimitConfig{
-	System:              infiniteBaseLimit,
-	Transient:           infiniteBaseLimit,
-	ServiceDefault:      infiniteBaseLimit,
-	ServicePeerDefault:  infiniteBaseLimit,
-	ProtocolDefault:     infiniteBaseLimit,
-	ProtocolPeerDefault: infiniteBaseLimit,
-	PeerDefault:         infiniteBaseLimit,
-	Conn:                infiniteBaseLimit,
-	Stream:              infiniteBaseLimit,
+	System:               infiniteBaseLimit,
+	Transient:            infiniteBaseLimit,
+	AllowlistedSystem:    infiniteBaseLimit,
+	AllowlistedTransient: infiniteBaseLimit,
+	ServiceDefault:       infiniteBaseLimit,
+	ServicePeerDefault:   infiniteBaseLimit,
+	ProtocolDefault:      infiniteBaseLimit,
+	ProtocolPeerDefault:  infiniteBaseLimit,
+	PeerDefault:          infiniteBaseLimit,
+	Conn:                 infiniteBaseLimit,
+	Stream:               infiniteBaseLimit,
 }

limit_test.go

@@ -13,7 +13,7 @@ func TestFileDescriptorCounting(t *testing.T) {
 	}
 	n := getNumFDs()
 	require.NotZero(t, n)
-	require.Less(t, n, int(1e6))
+	require.Less(t, n, int(1e7))
 }
 
 func TestScaling(t *testing.T) {

rcmgr.go

@@ -150,9 +150,9 @@ func NewResourceManager(limits Limiter, opts ...Option) (network.ResourceManager
 	r.transient = newTransientScope(limits.GetTransientLimits(), r, "transient", r.system.resourceScope)
 	r.transient.IncRef()
 
-	r.allowlistedSystem = newSystemScope(limits.GetSystemLimits(), r, "allowlistedSystem")
+	r.allowlistedSystem = newSystemScope(limits.GetAllowlistedSystemLimits(), r, "allowlistedSystem")
 	r.allowlistedSystem.IncRef()
-	r.allowlistedTransient = newTransientScope(limits.GetTransientLimits(), r, "allowlistedTransient", r.allowlistedSystem.resourceScope)
+	r.allowlistedTransient = newTransientScope(limits.GetAllowlistedTransientLimits(), r, "allowlistedTransient", r.allowlistedSystem.resourceScope)
 	r.allowlistedTransient.IncRef()
 
 	r.cancelCtx, r.cancel = context.WithCancel(context.Background())

rcmgr_test.go

@@ -977,13 +977,26 @@ func TestResourceManager(t *testing.T) {
 func TestResourceManagerWithAllowlist(t *testing.T) {
 	peerA := test.RandPeerIDFatal(t)
 
-	limits := DefaultLimits.Scale(1<<30, 100)
+	limits := DefaultLimits.AutoScale()
 	limits.System.Conns = 0
-	limits.System.ConnsInbound = 0
-	limits.System.ConnsOutbound = 0
 	limits.Transient.Conns = 0
-	limits.Transient.ConnsInbound = 0
-	limits.Transient.ConnsOutbound = 0
+
+	baseLimit := BaseLimit{
+		Conns:         2,
+		ConnsInbound:  2,
+		ConnsOutbound: 1,
+	}
+	baseLimit.Apply(limits.AllowlistedSystem)
+	limits.AllowlistedSystem = baseLimit
+
+	baseLimit = BaseLimit{
+		Conns:         1,
+		ConnsInbound:  1,
+		ConnsOutbound: 1,
+	}
+	baseLimit.Apply(limits.AllowlistedTransient)
+	limits.AllowlistedTransient = baseLimit
+
 	rcmgr, err := NewResourceManager(NewFixedLimiter(limits), WithAllowlistedMultiaddrs([]multiaddr.Multiaddr{
 		multiaddr.StringCast("/ip4/1.2.3.4"),
 		multiaddr.StringCast("/ip4/4.3.2.1/p2p/" + peerA.String()),
@@ -992,25 +1005,6 @@ func TestResourceManagerWithAllowlist(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	{
-		// Setup allowlist. TODO, replace this with a config once config changes are in
-		r := rcmgr.(*resourceManager)
-
-		sysLimit := limits.System
-		sysLimit.Conns = 2
-		sysLimit.ConnsInbound = 2
-		sysLimit.ConnsOutbound = 1
-		r.allowlistedSystem = newSystemScope(&sysLimit, r, "allowlistedSystem")
-		r.allowlistedSystem.IncRef()
-
-		transLimit := limits.Transient
-		transLimit.Conns = 1
-		transLimit.ConnsInbound = 1
-		transLimit.ConnsOutbound = 1
-		r.allowlistedTransient = newTransientScope(&transLimit, r, "allowlistedTransient", r.allowlistedSystem.resourceScope)
-		r.allowlistedTransient.IncRef()
-	}
-
 	// A connection comes in from a non-allowlisted ip address
 	_, err = rcmgr.OpenConnection(network.DirInbound, true, multiaddr.StringCast("/ip4/1.2.3.5"))
 	if err == nil {