From ee3cc3b8110793e03e204f45bfbd6806937b5ddd Mon Sep 17 00:00:00 2001 From: Marco Munizaga Date: Wed, 8 Jun 2022 15:55:11 -0700 Subject: [PATCH 01/45] Add design doc --- docs/allowlist.md | 51 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 docs/allowlist.md diff --git a/docs/allowlist.md b/docs/allowlist.md new file mode 100644 index 0000000..1819039 --- /dev/null +++ b/docs/allowlist.md @@ -0,0 +1,51 @@ +# Allowlist + +Imagine you have a node that is getting overloaded by possibly malicious +incoming connections. This node won't be able to accept incoming connections +from peers it _knows_ to be good. This node would effectively be _eclipsed_ from +the network since no other nodes will be able to connect to it. + +This is the problem that the Allowlist is designed to solve. + +## Design Goals + +- We should not fail to allocate a resource for an allowlisted peer because the + normal transient and system scopes are at their limits. This is the minimum + bar to avoid eclipse attacks. +- Minimal changes to resource manager and existing code (e.g. go-libp2p). +- The allowlist scope itself is limited to avoid giving an allowlisted peer the + ability to DoS a node. +- PeerIDs can optionally be fed into the allowlist. This will give an extra + step of verification before continuing to allow the peer to open streams. + - A peer may be able to open a connection, but after the handshake, if it's + not an expected peer id we move it to the normal system scope. +- We can have multiple PeerIDs for a given IP addr. + +## Proposed change + +Add a change to `ResourceManager.OpenConnection` so that it accepts a multiaddr +parameter of the endpoint the connection is for. + +Add a change to `ResourceManager` to initialize it with a set of allowlisted +multiaddrs. This set can be modified at runtime as well for dynamic updating. + +For example, an allowlist set could look like: +``` +/ip4/1.1.1.1 +/ip6/2345:0425:2CA1::0567:5673:23b5 +/ip4/192.168.1.1/p2p/qmFoo +/ip4/192.168.1.1/p2p/qmBar +/ip4/1.2.3.0/ipcidr/24 +``` + +When a new connection is opened, the resource manager checks if the multiaddr +matches an item in the set of allowlisted multiaddrs. If so, it creates the +connection resource scope using the allowlisted specific system and transient +resource scopes. Otherwise, it uses the normal system and transient resource +scopes as it does today. + +When an allowlisted connection is tied to a peer id and transfered with +`ConnManagementScope.SetPeer`, we check if that peer id matches the expected +value in the allowlist (if it exists). If it does not match, we attempt to +transfer this resource to the normal system and peer scope. If that transfer +fails we close the connection. From dd0cde1cfd8d2ceddd3db7075bac915c1c7b21bf Mon Sep 17 00:00:00 2001 From: Marco Munizaga Date: Thu, 9 Jun 2022 22:26:24 -0700 Subject: [PATCH 02/45] Add allowlist impl Accepts multiaddrs that define who is allowed. --- allowlist.go | 230 ++++++++++++++++++++++++++++++++++++++++++++++ allowlist_test.go | 220 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 450 insertions(+) create mode 100644 allowlist.go create mode 100644 allowlist_test.go diff --git a/allowlist.go b/allowlist.go new file mode 100644 index 0000000..8f38806 --- /dev/null +++ b/allowlist.go @@ -0,0 +1,230 @@ +package rcmgr + +import ( + "bytes" + "errors" + "net" + + "github.com/libp2p/go-libp2p-core/peer" + "github.com/multiformats/go-multiaddr" + manet "github.com/multiformats/go-multiaddr/net" +) + +type allowlist struct { + // TODO do we want to make this lookup faster? + + // Any peer with these IPs are allowed + allowedIPs []net.IP + allowedNetworks []*net.IPNet + + // Only the specified peers can use these IPs + allowedPeerByIP map[peer.ID][]net.IP + allowedPeerByNetwork map[peer.ID][]*net.IPNet +} + +func newAllowList() allowlist { + return allowlist{ + allowedPeerByIP: make(map[peer.ID][]net.IP), + allowedPeerByNetwork: make(map[peer.ID][]*net.IPNet), + } +} + +func toIPOrIPNet(ma multiaddr.Multiaddr) (net.IP, *net.IPNet, string, error) { + var ipString string + var mask string + var allowedPeer string + + multiaddr.ForEach(ma, func(c multiaddr.Component) bool { + if c.Protocol().Code == multiaddr.P_IP4 || c.Protocol().Code == multiaddr.P_IP6 { + ipString = c.Value() + } + if c.Protocol().Code == multiaddr.P_IPCIDR { + mask = c.Value() + } + if c.Protocol().Code == multiaddr.P_P2P { + allowedPeer = c.Value() + } + return ipString == "" || mask == "" || allowedPeer == "" + }) + + if ipString == "" { + return nil, nil, allowedPeer, errors.New("missing ip address") + } + + if mask == "" { + ip := net.ParseIP(ipString) + if ip == nil { + return nil, nil, allowedPeer, errors.New("invalid ip address") + } + return ip, nil, allowedPeer, nil + } + + _, ipnet, err := net.ParseCIDR(ipString + "/" + mask) + return nil, ipnet, allowedPeer, err + +} + +func (al *allowlist) Add(ma multiaddr.Multiaddr) error { + ip, ipnet, allowedPeerStr, err := toIPOrIPNet(ma) + if err != nil { + return err + } + + if allowedPeerStr != "" { + // We have a peerID constraint + allowedPeer, err := peer.Decode(allowedPeerStr) + if err != nil { + return err + } + if ip != nil { + al.allowedPeerByIP[allowedPeer] = append(al.allowedPeerByIP[allowedPeer], ip) + } else if ipnet != nil { + al.allowedPeerByNetwork[allowedPeer] = append(al.allowedPeerByNetwork[allowedPeer], ipnet) + } + } else { + if ip != nil { + al.allowedIPs = append(al.allowedIPs, ip) + } else if ipnet != nil { + al.allowedNetworks = append(al.allowedNetworks, ipnet) + } + } + return nil +} + +func (al *allowlist) Remove(ma multiaddr.Multiaddr) error { + ip, ipnet, allowedPeerStr, err := toIPOrIPNet(ma) + if err != nil { + return err + } + ipList := al.allowedIPs + ipNetList := al.allowedNetworks + + var allowedPeer peer.ID + if allowedPeerStr != "" { + // We have a peerID constraint + allowedPeer, err = peer.Decode(allowedPeerStr) + if err != nil { + return err + } + ipList = al.allowedPeerByIP[allowedPeer] + ipNetList = al.allowedPeerByNetwork[allowedPeer] + } + + if ip != nil { + i := len(ipList) + for i > 0 { + i-- + if ipList[i].Equal(ip) { + if i == len(ipList)-1 { + // Trim this element from the end + ipList = ipList[:i] + } else { + // swap remove + ipList[i] = ipList[len(ipList)-1] + ipList = ipList[:len(ipList)-1] + } + } + } + } else if ipnet != nil { + i := len(ipNetList) + for i > 0 { + i-- + if ipNetList[i].IP.Equal(ipnet.IP) && bytes.Equal(ipNetList[i].Mask, ipnet.Mask) { + if i == len(ipNetList)-1 { + // Trim this element from the end + ipNetList = ipNetList[:i] + } else { + // swap remove + ipNetList[i] = ipNetList[len(ipNetList)-1] + ipNetList = ipNetList[:len(ipNetList)-1] + } + } + } + } + + if allowedPeer != "" { + al.allowedPeerByIP[allowedPeer] = ipList + al.allowedPeerByNetwork[allowedPeer] = ipNetList + } else { + al.allowedIPs = ipList + al.allowedNetworks = ipNetList + } + + return nil +} + +func (al *allowlist) Allowed(ma multiaddr.Multiaddr) bool { + ip, err := manet.ToIP(ma) + if err != nil { + return false + } + + for _, allowedIP := range al.allowedIPs { + if allowedIP.Equal(ip) { + return true + } + } + + for _, network := range al.allowedNetworks { + if network.Contains(ip) { + return true + } + } + + for _, allowedIPs := range al.allowedPeerByIP { + for _, allowedIP := range allowedIPs { + if allowedIP.Equal(ip) { + return true + } + } + } + + for _, allowedNetworks := range al.allowedPeerByNetwork { + for _, network := range allowedNetworks { + if network.Contains(ip) { + return true + } + } + } + + return false +} + +func (al *allowlist) AllowedPeerAndMultiaddr(peerID peer.ID, ma multiaddr.Multiaddr) bool { + ip, err := manet.ToIP(ma) + if err != nil { + return false + } + + for _, allowedIP := range al.allowedIPs { + if allowedIP.Equal(ip) { + // We found a match that isn't constrained by a peerID + return true + } + } + + for _, network := range al.allowedNetworks { + if network.Contains(ip) { + // We found a match that isn't constrained by a peerID + return true + } + } + + if expectedIPs, ok := al.allowedPeerByIP[peerID]; ok { + for _, expectedIP := range expectedIPs { + if expectedIP.Equal(ip) { + return true + } + } + } + + if expectedNetworks, ok := al.allowedPeerByNetwork[peerID]; ok { + for _, expectedNetwork := range expectedNetworks { + if expectedNetwork.Contains(ip) { + return true + } + } + } + + return false +} diff --git a/allowlist_test.go b/allowlist_test.go new file mode 100644 index 0000000..2974644 --- /dev/null +++ b/allowlist_test.go @@ -0,0 +1,220 @@ +package rcmgr + +import ( + "fmt" + "testing" + + "github.com/libp2p/go-libp2p-core/peer" + "github.com/libp2p/go-libp2p-core/test" + "github.com/multiformats/go-multiaddr" +) + +func TestAllowed(t *testing.T) { + allowlist := newAllowList() + ma, _ := multiaddr.NewMultiaddr("/ip4/1.2.3.4/tcp/1234") + err := allowlist.Add(ma) + if err != nil { + t.Fatalf("failed to add ip4: %s", err) + } + + if !allowlist.Allowed(ma) { + t.Fatalf("addr should be allowed") + } +} + +func TestAllowedNetwork(t *testing.T) { + allowlist := newAllowList() + ma, _ := multiaddr.NewMultiaddr("/ip4/1.2.3.0/ipcidr/24") + err := allowlist.Add(ma) + if err != nil { + t.Fatalf("failed to add ip4: %s", err) + } + + ma2, _ := multiaddr.NewMultiaddr("/ip4/1.2.3.20/tcp/1234") + if !allowlist.Allowed(ma2) { + t.Fatalf("addr should be allowed") + } +} + +func TestAllowedPeerOnIP(t *testing.T) { + allowlist := newAllowList() + p, err := test.RandPeerID() + if err != nil { + t.Fatalf("failed to gen peer ip4: %s", err) + } + + ma, _ := multiaddr.NewMultiaddr("/ip4/1.2.3.4/p2p/" + peer.Encode(p)) + err = allowlist.Add(ma) + if err != nil { + t.Fatalf("failed to add ip4: %s", err) + } + + ma2, _ := multiaddr.NewMultiaddr("/ip4/1.2.3.4") + if !allowlist.AllowedPeerAndMultiaddr(p, ma2) { + t.Fatalf("addr should be allowed") + } +} + +func TestAllowedPeerOnNetwork(t *testing.T) { + allowlist := newAllowList() + p, err := test.RandPeerID() + if err != nil { + t.Fatalf("failed to gen peer ip4: %s", err) + } + + ma, _ := multiaddr.NewMultiaddr("/ip4/1.2.3.0/ipcidr/24/p2p/" + peer.Encode(p)) + err = allowlist.Add(ma) + if err != nil { + t.Fatalf("failed to add ip4: %s", err) + } + + ma2, _ := multiaddr.NewMultiaddr("/ip4/1.2.3.4") + if !allowlist.AllowedPeerAndMultiaddr(p, ma2) { + t.Fatalf("addr should be allowed") + } +} + +func TestAllowedWithPeer(t *testing.T) { + type testcase struct { + name string + allowlist []string + endpoint multiaddr.Multiaddr + peer peer.ID + isAllowed bool + } + + peerA := test.RandPeerIDFatal(t) + peerB := test.RandPeerIDFatal(t) + multiaddrA, _ := multiaddr.NewMultiaddr("/ip4/1.2.3.4/tcp/1234") + multiaddrB, _ := multiaddr.NewMultiaddr("/ip4/2.2.3.4/tcp/1234") + + testcases := []testcase{ + { + name: "Blocked", + isAllowed: false, + allowlist: []string{"/ip4/1.2.3.1"}, + endpoint: multiaddrA, + peer: peerA, + }, + { + name: "Blocked wrong peer", + isAllowed: false, + allowlist: []string{"/ip4/1.2.3.4" + "/p2p/" + peer.Encode(peerB)}, + endpoint: multiaddrA, + peer: peerA, + }, + { + name: "allowed on network", + isAllowed: true, + allowlist: []string{"/ip4/1.2.3.0/ipcidr/24"}, + endpoint: multiaddrA, + peer: peerA, + }, + { + name: "Blocked peer not on network", + isAllowed: true, + allowlist: []string{"/ip4/1.2.3.0/ipcidr/24"}, + endpoint: multiaddrA, + peer: peerA, + }, { + name: "allowed. right network, right peer", + isAllowed: true, + allowlist: []string{"/ip4/1.2.3.0/ipcidr/24" + "/p2p/" + peer.Encode(peerA)}, + endpoint: multiaddrA, + peer: peerA, + }, { + name: "allowed. right network, no peer", + isAllowed: true, + allowlist: []string{"/ip4/1.2.3.0/ipcidr/24"}, + endpoint: multiaddrA, + peer: peerA, + }, + { + name: "Blocked. right network, wrong peer", + isAllowed: false, + allowlist: []string{"/ip4/1.2.3.0/ipcidr/24" + "/p2p/" + peer.Encode(peerB)}, + endpoint: multiaddrA, + peer: peerA, + }, + { + name: "allowed peer any ip", + isAllowed: true, + allowlist: []string{"/ip4/0.0.0.0/ipcidr/0"}, + endpoint: multiaddrA, + peer: peerA, + }, + { + name: "allowed peer multiple ips in allowlist", + isAllowed: true, + allowlist: []string{"/ip4/1.2.3.4/p2p/" + peer.Encode(peerA), "/ip4/2.2.3.4/p2p/" + peer.Encode(peerA)}, + endpoint: multiaddrA, + peer: peerA, + }, + { + name: "allowed peer multiple ips in allowlist", + isAllowed: true, + allowlist: []string{"/ip4/1.2.3.4/p2p/" + peer.Encode(peerA), "/ip4/2.2.3.4/p2p/" + peer.Encode(peerA)}, + endpoint: multiaddrB, + peer: peerA, + }, + } + + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + allowlist := newAllowList() + for _, maStr := range tc.allowlist { + ma, err := multiaddr.NewMultiaddr(maStr) + if err != nil { + fmt.Printf("failed to parse multiaddr: %s", err) + } + allowlist.Add(ma) + } + + if allowlist.AllowedPeerAndMultiaddr(tc.peer, tc.endpoint) != tc.isAllowed { + t.Fatalf("%v: expected %v", !tc.isAllowed, tc.isAllowed) + } + }) + } + +} + +func TestRemoved(t *testing.T) { + type testCase struct { + name string + allowedMA string + } + peerA := test.RandPeerIDFatal(t) + maA, _ := multiaddr.NewMultiaddr("/ip4/1.2.3.4") + + testCases := []testCase{ + {name: "ip4", allowedMA: "/ip4/1.2.3.4"}, + {name: "ip4 with peer", allowedMA: "/ip4/1.2.3.4/p2p/" + peer.Encode(peerA)}, + {name: "ip4 network", allowedMA: "/ip4/0.0.0.0/ipcidr/0"}, + {name: "ip4 network with peer", allowedMA: "/ip4/0.0.0.0/ipcidr/0/p2p/" + peer.Encode(peerA)}, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + allowlist := newAllowList() + ma, err := multiaddr.NewMultiaddr(tc.allowedMA) + if err != nil { + t.Fatalf("failed to parse ma: %s", err) + } + + err = allowlist.Add(ma) + if err != nil { + t.Fatalf("failed to add ip4: %s", err) + } + + if !allowlist.AllowedPeerAndMultiaddr(peerA, maA) { + t.Fatalf("addr should be allowed") + } + + allowlist.Remove((ma)) + + if allowlist.AllowedPeerAndMultiaddr(peerA, maA) { + t.Fatalf("addr should not be allowed") + } + }) + } +} From 1fcd41c41828c3af762789b32b20e0dc773b874e Mon Sep 17 00:00:00 2001 From: Marco Munizaga Date: Thu, 9 Jun 2022 22:33:18 -0700 Subject: [PATCH 03/45] Add allowlist to rcmgr --- rcmgr.go | 119 ++++++++++++++++++++++++++++++++++++++++++++------ rcmgr_test.go | 15 ++++--- 2 files changed, 115 insertions(+), 19 deletions(-) diff --git a/rcmgr.go b/rcmgr.go index bebae9b..c262a8d 100644 --- a/rcmgr.go +++ b/rcmgr.go @@ -9,6 +9,7 @@ import ( "github.com/libp2p/go-libp2p-core/network" "github.com/libp2p/go-libp2p-core/peer" "github.com/libp2p/go-libp2p-core/protocol" + "github.com/multiformats/go-multiaddr" logging "github.com/ipfs/go-log/v2" ) @@ -21,9 +22,14 @@ type resourceManager struct { trace *trace metrics *metrics + allowlist *allowlist + system *systemScope transient *transientScope + allowlistedSystem *systemScope + allowlistedTransient *transientScope + cancelCtx context.Context cancel func() wg sync.WaitGroup @@ -89,10 +95,12 @@ var _ network.PeerScope = (*peerScope)(nil) type connectionScope struct { *resourceScope - dir network.Direction - usefd bool - rcmgr *resourceManager - peer *peerScope + dir network.Direction + usefd bool + isAllowlisted bool + rcmgr *resourceManager + peer *peerScope + endpoint multiaddr.Multiaddr } var _ network.ConnScope = (*connectionScope)(nil) @@ -117,11 +125,13 @@ var _ network.StreamManagementScope = (*streamScope)(nil) type Option func(*resourceManager) error func NewResourceManager(limits Limiter, opts ...Option) (network.ResourceManager, error) { + allowlist := newAllowList() r := &resourceManager{ - limits: limits, - svc: make(map[string]*serviceScope), - proto: make(map[protocol.ID]*protocolScope), - peer: make(map[peer.ID]*peerScope), + limits: limits, + allowlist: &allowlist, + svc: make(map[string]*serviceScope), + proto: make(map[protocol.ID]*protocolScope), + peer: make(map[peer.ID]*peerScope), } for _, opt := range opts { @@ -255,8 +265,14 @@ func (r *resourceManager) nextStreamId() int64 { return r.streamId } -func (r *resourceManager) OpenConnection(dir network.Direction, usefd bool) (network.ConnManagementScope, error) { - conn := newConnectionScope(dir, usefd, r.limits.GetConnLimits(), r) +func (r *resourceManager) OpenConnection(dir network.Direction, usefd bool, endpoint multiaddr.Multiaddr) (network.ConnManagementScope, error) { + allowed := r.allowlist.Allowed(endpoint) + var conn *connectionScope + if allowed { + conn = newAllowListedConnectionScope(dir, usefd, r.limits.GetConnLimits(), r) + } else { + conn = newConnectionScope(dir, usefd, r.limits.GetConnLimits(), r) + } if err := conn.AddConn(dir, usefd); err != nil { conn.Done() @@ -419,6 +435,17 @@ func newConnectionScope(dir network.Direction, usefd bool, limit Limit, rcmgr *r } } +func newAllowListedConnectionScope(dir network.Direction, usefd bool, limit Limit, rcmgr *resourceManager) *connectionScope { + return &connectionScope{ + resourceScope: newResourceScope(limit, + []*resourceScope{rcmgr.allowlistedTransient.resourceScope, rcmgr.allowlistedSystem.resourceScope}, + fmt.Sprintf("conn-%d", rcmgr.nextConnId()), rcmgr.trace, rcmgr.metrics), + dir: dir, + usefd: usefd, + rcmgr: rcmgr, + } +} + func newStreamScope(dir network.Direction, limit Limit, peer *peerScope, rcmgr *resourceManager) *streamScope { return &streamScope{ resourceScope: newResourceScope(limit, @@ -500,13 +527,79 @@ func (s *connectionScope) PeerScope() network.PeerScope { return s.peer } +// transferAllowedToStandard Transfers this connection scope from being part of +// the allowlist set of scopes to being part of the standard set of scopes. +// Happens when we first allowlisted this connection due to its IP, but later +// discovered that the peer id not what we expected. +func (s *connectionScope) transferAllowedToStandard() (err error) { + + systemScope := s.rcmgr.system.resourceScope + transientScope := s.rcmgr.system.resourceScope + + stat := s.resourceScope.rc.stat() + + for _, scope := range s.edges { + scope.ReleaseForChild(stat) + scope.DecRef() // removed from edges + } + s.edges = nil + + if err := systemScope.ReserveForChild(stat); err != nil { + return err + } + systemScope.IncRef() + + // Undo this if we fail later + defer func() { + if err != nil { + systemScope.ReleaseForChild(stat) + systemScope.DecRef() + } + }() + + if err := transientScope.ReserveForChild(stat); err != nil { + return err + } + transientScope.IncRef() + + // Update edges + s.edges = []*resourceScope{ + systemScope, + transientScope, + } + return nil +} + func (s *connectionScope) SetPeer(p peer.ID) error { + // TODO check if this connectionscope is part of the allowlist and do the peer checking if so. s.Lock() defer s.Unlock() if s.peer != nil { return fmt.Errorf("connection scope already attached to a peer") } + + system := s.rcmgr.system + transient := s.rcmgr.transient + + if s.isAllowlisted { + system = s.rcmgr.allowlistedSystem + transient = s.rcmgr.transient + } + + if s.isAllowlisted && !s.rcmgr.allowlist.AllowedPeerAndMultiaddr(p, s.endpoint) { + // This is not a allowed peer + multiaddr combination. We need to + // transfer this connection to the general scope. We'll do this first by + // transferring the connection to the system and transient scopes, then + // continue on with this function. The idea is that a connection + // shouldn't get the benefit of evading the transient scope because it + // was _almost_ an allowlisted connection. + if err := s.transferAllowedToStandard(); err != nil { + // Failed to transfer this connection to the standard scopes + return err + } + } + s.peer = s.rcmgr.getPeerScope(p) // juggle resources from transient scope to peer scope @@ -518,13 +611,13 @@ func (s *connectionScope) SetPeer(p peer.ID) error { return err } - s.rcmgr.transient.ReleaseForChild(stat) - s.rcmgr.transient.DecRef() // removed from edges + transient.ReleaseForChild(stat) + transient.DecRef() // removed from edges // update edges edges := []*resourceScope{ s.peer.resourceScope, - s.rcmgr.system.resourceScope, + system.resourceScope, } s.resourceScope.edges = edges diff --git a/rcmgr_test.go b/rcmgr_test.go index a18815c..4bb8106 100644 --- a/rcmgr_test.go +++ b/rcmgr_test.go @@ -6,8 +6,11 @@ import ( "github.com/libp2p/go-libp2p-core/network" "github.com/libp2p/go-libp2p-core/peer" "github.com/libp2p/go-libp2p-core/protocol" + "github.com/multiformats/go-multiaddr" ) +var dummyMA, _ = multiaddr.NewMultiaddr("/ip4/1.2.3.4/tcp/1234") + func TestResourceManager(t *testing.T) { peerA := peer.ID("A") peerB := peer.ID("B") @@ -242,7 +245,7 @@ func TestResourceManager(t *testing.T) { }) // open an inbound connection, using an fd - conn, err := mgr.OpenConnection(network.DirInbound, true) + conn, err := mgr.OpenConnection(network.DirInbound, true, dummyMA) if err != nil { t.Fatal(err) } @@ -257,10 +260,10 @@ func TestResourceManager(t *testing.T) { }) // the connection is transient, we shouldn't be able to open a second one - if _, err := mgr.OpenConnection(network.DirInbound, true); err == nil { + if _, err := mgr.OpenConnection(network.DirInbound, true, dummyMA); err == nil { t.Fatal("expected OpenConnection to fail") } - if _, err := mgr.OpenConnection(network.DirInbound, false); err == nil { + if _, err := mgr.OpenConnection(network.DirInbound, false, dummyMA); err == nil { t.Fatal("expected OpenConnection to fail") } @@ -277,7 +280,7 @@ func TestResourceManager(t *testing.T) { }) // open another inbound connection, using an fd - conn1, err := mgr.OpenConnection(network.DirInbound, true) + conn1, err := mgr.OpenConnection(network.DirInbound, true, dummyMA) if err != nil { t.Fatal(err) } @@ -315,7 +318,7 @@ func TestResourceManager(t *testing.T) { }) // we should be able to open a second transient connection now - conn2, err := mgr.OpenConnection(network.DirInbound, true) + conn2, err := mgr.OpenConnection(network.DirInbound, true, dummyMA) if err != nil { t.Fatal(err) } @@ -350,7 +353,7 @@ func TestResourceManager(t *testing.T) { // close it and reopen without using an FD -- we should be able to attach now conn2.Done() - conn2, err = mgr.OpenConnection(network.DirInbound, false) + conn2, err = mgr.OpenConnection(network.DirInbound, false, dummyMA) if err != nil { t.Fatal(err) } From 23ec39d74e4d5482253f1e58095400b35a27c190 Mon Sep 17 00:00:00 2001 From: Marco Munizaga Date: Mon, 13 Jun 2022 16:26:59 -0700 Subject: [PATCH 04/45] Simplify by removing ip addr, only networks. Adds a benchmark --- allowlist.go | 87 ++++------------ allowlist_test.go | 251 ++++++++++++++++++++++++++-------------------- 2 files changed, 167 insertions(+), 171 deletions(-) diff --git a/allowlist.go b/allowlist.go index 8f38806..9d4e7ab 100644 --- a/allowlist.go +++ b/allowlist.go @@ -14,28 +14,27 @@ type allowlist struct { // TODO do we want to make this lookup faster? // Any peer with these IPs are allowed - allowedIPs []net.IP allowedNetworks []*net.IPNet // Only the specified peers can use these IPs - allowedPeerByIP map[peer.ID][]net.IP allowedPeerByNetwork map[peer.ID][]*net.IPNet } func newAllowList() allowlist { return allowlist{ - allowedPeerByIP: make(map[peer.ID][]net.IP), allowedPeerByNetwork: make(map[peer.ID][]*net.IPNet), } } -func toIPOrIPNet(ma multiaddr.Multiaddr) (net.IP, *net.IPNet, string, error) { +func toIPNet(ma multiaddr.Multiaddr) (*net.IPNet, string, error) { var ipString string var mask string var allowedPeer string + var isIPV4 bool multiaddr.ForEach(ma, func(c multiaddr.Component) bool { if c.Protocol().Code == multiaddr.P_IP4 || c.Protocol().Code == multiaddr.P_IP6 { + isIPV4 = c.Protocol().Code == multiaddr.P_IP4 ipString = c.Value() } if c.Protocol().Code == multiaddr.P_IPCIDR { @@ -48,24 +47,32 @@ func toIPOrIPNet(ma multiaddr.Multiaddr) (net.IP, *net.IPNet, string, error) { }) if ipString == "" { - return nil, nil, allowedPeer, errors.New("missing ip address") + return nil, allowedPeer, errors.New("missing ip address") } if mask == "" { ip := net.ParseIP(ipString) if ip == nil { - return nil, nil, allowedPeer, errors.New("invalid ip address") + return nil, allowedPeer, errors.New("invalid ip address") } - return ip, nil, allowedPeer, nil + var mask net.IPMask + if isIPV4 { + mask = net.CIDRMask(32, 32) + } else { + mask = net.CIDRMask(128, 128) + } + + net := &net.IPNet{IP: ip, Mask: mask} + return net, allowedPeer, nil } _, ipnet, err := net.ParseCIDR(ipString + "/" + mask) - return nil, ipnet, allowedPeer, err + return ipnet, allowedPeer, err } func (al *allowlist) Add(ma multiaddr.Multiaddr) error { - ip, ipnet, allowedPeerStr, err := toIPOrIPNet(ma) + ipnet, allowedPeerStr, err := toIPNet(ma) if err != nil { return err } @@ -76,15 +83,11 @@ func (al *allowlist) Add(ma multiaddr.Multiaddr) error { if err != nil { return err } - if ip != nil { - al.allowedPeerByIP[allowedPeer] = append(al.allowedPeerByIP[allowedPeer], ip) - } else if ipnet != nil { + if ipnet != nil { al.allowedPeerByNetwork[allowedPeer] = append(al.allowedPeerByNetwork[allowedPeer], ipnet) } } else { - if ip != nil { - al.allowedIPs = append(al.allowedIPs, ip) - } else if ipnet != nil { + if ipnet != nil { al.allowedNetworks = append(al.allowedNetworks, ipnet) } } @@ -92,11 +95,10 @@ func (al *allowlist) Add(ma multiaddr.Multiaddr) error { } func (al *allowlist) Remove(ma multiaddr.Multiaddr) error { - ip, ipnet, allowedPeerStr, err := toIPOrIPNet(ma) + ipnet, allowedPeerStr, err := toIPNet(ma) if err != nil { return err } - ipList := al.allowedIPs ipNetList := al.allowedNetworks var allowedPeer peer.ID @@ -106,26 +108,10 @@ func (al *allowlist) Remove(ma multiaddr.Multiaddr) error { if err != nil { return err } - ipList = al.allowedPeerByIP[allowedPeer] ipNetList = al.allowedPeerByNetwork[allowedPeer] } - if ip != nil { - i := len(ipList) - for i > 0 { - i-- - if ipList[i].Equal(ip) { - if i == len(ipList)-1 { - // Trim this element from the end - ipList = ipList[:i] - } else { - // swap remove - ipList[i] = ipList[len(ipList)-1] - ipList = ipList[:len(ipList)-1] - } - } - } - } else if ipnet != nil { + if ipnet != nil { i := len(ipNetList) for i > 0 { i-- @@ -143,10 +129,8 @@ func (al *allowlist) Remove(ma multiaddr.Multiaddr) error { } if allowedPeer != "" { - al.allowedPeerByIP[allowedPeer] = ipList al.allowedPeerByNetwork[allowedPeer] = ipNetList } else { - al.allowedIPs = ipList al.allowedNetworks = ipNetList } @@ -159,26 +143,14 @@ func (al *allowlist) Allowed(ma multiaddr.Multiaddr) bool { return false } - for _, allowedIP := range al.allowedIPs { - if allowedIP.Equal(ip) { - return true - } - } - + _ = ip for _, network := range al.allowedNetworks { + _ = network if network.Contains(ip) { return true } } - for _, allowedIPs := range al.allowedPeerByIP { - for _, allowedIP := range allowedIPs { - if allowedIP.Equal(ip) { - return true - } - } - } - for _, allowedNetworks := range al.allowedPeerByNetwork { for _, network := range allowedNetworks { if network.Contains(ip) { @@ -196,13 +168,6 @@ func (al *allowlist) AllowedPeerAndMultiaddr(peerID peer.ID, ma multiaddr.Multia return false } - for _, allowedIP := range al.allowedIPs { - if allowedIP.Equal(ip) { - // We found a match that isn't constrained by a peerID - return true - } - } - for _, network := range al.allowedNetworks { if network.Contains(ip) { // We found a match that isn't constrained by a peerID @@ -210,14 +175,6 @@ func (al *allowlist) AllowedPeerAndMultiaddr(peerID peer.ID, ma multiaddr.Multia } } - if expectedIPs, ok := al.allowedPeerByIP[peerID]; ok { - for _, expectedIP := range expectedIPs { - if expectedIP.Equal(ip) { - return true - } - } - } - if expectedNetworks, ok := al.allowedPeerByNetwork[peerID]; ok { for _, expectedNetwork := range expectedNetworks { if expectedNetwork.Contains(ip) { diff --git a/allowlist_test.go b/allowlist_test.go index 2974644..d9245b2 100644 --- a/allowlist_test.go +++ b/allowlist_test.go @@ -1,7 +1,9 @@ package rcmgr import ( + "crypto/rand" "fmt" + "net" "testing" "github.com/libp2p/go-libp2p-core/peer" @@ -9,7 +11,7 @@ import ( "github.com/multiformats/go-multiaddr" ) -func TestAllowed(t *testing.T) { +func TestAllowedSimple(t *testing.T) { allowlist := newAllowList() ma, _ := multiaddr.NewMultiaddr("/ip4/1.2.3.4/tcp/1234") err := allowlist.Add(ma) @@ -22,65 +24,16 @@ func TestAllowed(t *testing.T) { } } -func TestAllowedNetwork(t *testing.T) { - allowlist := newAllowList() - ma, _ := multiaddr.NewMultiaddr("/ip4/1.2.3.0/ipcidr/24") - err := allowlist.Add(ma) - if err != nil { - t.Fatalf("failed to add ip4: %s", err) - } - - ma2, _ := multiaddr.NewMultiaddr("/ip4/1.2.3.20/tcp/1234") - if !allowlist.Allowed(ma2) { - t.Fatalf("addr should be allowed") - } -} - -func TestAllowedPeerOnIP(t *testing.T) { - allowlist := newAllowList() - p, err := test.RandPeerID() - if err != nil { - t.Fatalf("failed to gen peer ip4: %s", err) - } - - ma, _ := multiaddr.NewMultiaddr("/ip4/1.2.3.4/p2p/" + peer.Encode(p)) - err = allowlist.Add(ma) - if err != nil { - t.Fatalf("failed to add ip4: %s", err) - } - - ma2, _ := multiaddr.NewMultiaddr("/ip4/1.2.3.4") - if !allowlist.AllowedPeerAndMultiaddr(p, ma2) { - t.Fatalf("addr should be allowed") - } -} - -func TestAllowedPeerOnNetwork(t *testing.T) { - allowlist := newAllowList() - p, err := test.RandPeerID() - if err != nil { - t.Fatalf("failed to gen peer ip4: %s", err) - } - - ma, _ := multiaddr.NewMultiaddr("/ip4/1.2.3.0/ipcidr/24/p2p/" + peer.Encode(p)) - err = allowlist.Add(ma) - if err != nil { - t.Fatalf("failed to add ip4: %s", err) - } - - ma2, _ := multiaddr.NewMultiaddr("/ip4/1.2.3.4") - if !allowlist.AllowedPeerAndMultiaddr(p, ma2) { - t.Fatalf("addr should be allowed") - } -} - func TestAllowedWithPeer(t *testing.T) { type testcase struct { name string allowlist []string endpoint multiaddr.Multiaddr peer peer.ID - isAllowed bool + // Is this endpoint allowed? (We don't have peer info yet) + isConnAllowed bool + // Is this peer + endpoint allowed? + isAllowedWithPeer bool } peerA := test.RandPeerIDFatal(t) @@ -90,72 +43,82 @@ func TestAllowedWithPeer(t *testing.T) { testcases := []testcase{ { - name: "Blocked", - isAllowed: false, - allowlist: []string{"/ip4/1.2.3.1"}, - endpoint: multiaddrA, - peer: peerA, + name: "Blocked", + isConnAllowed: false, + isAllowedWithPeer: false, + allowlist: []string{"/ip4/1.2.3.1"}, + endpoint: multiaddrA, + peer: peerA, }, { - name: "Blocked wrong peer", - isAllowed: false, - allowlist: []string{"/ip4/1.2.3.4" + "/p2p/" + peer.Encode(peerB)}, - endpoint: multiaddrA, - peer: peerA, + name: "Blocked wrong peer", + isConnAllowed: true, + isAllowedWithPeer: false, + allowlist: []string{"/ip4/1.2.3.4" + "/p2p/" + peer.Encode(peerB)}, + endpoint: multiaddrA, + peer: peerA, }, { - name: "allowed on network", - isAllowed: true, - allowlist: []string{"/ip4/1.2.3.0/ipcidr/24"}, - endpoint: multiaddrA, - peer: peerA, + name: "allowed on network", + isConnAllowed: true, + isAllowedWithPeer: true, + allowlist: []string{"/ip4/1.2.3.0/ipcidr/24"}, + endpoint: multiaddrA, + peer: peerA, }, { - name: "Blocked peer not on network", - isAllowed: true, - allowlist: []string{"/ip4/1.2.3.0/ipcidr/24"}, - endpoint: multiaddrA, - peer: peerA, + name: "Blocked peer not on network", + isConnAllowed: true, + isAllowedWithPeer: true, + allowlist: []string{"/ip4/1.2.3.0/ipcidr/24"}, + endpoint: multiaddrA, + peer: peerA, }, { - name: "allowed. right network, right peer", - isAllowed: true, - allowlist: []string{"/ip4/1.2.3.0/ipcidr/24" + "/p2p/" + peer.Encode(peerA)}, - endpoint: multiaddrA, - peer: peerA, + name: "allowed. right network, right peer", + isConnAllowed: true, + isAllowedWithPeer: true, + allowlist: []string{"/ip4/1.2.3.0/ipcidr/24" + "/p2p/" + peer.Encode(peerA)}, + endpoint: multiaddrA, + peer: peerA, }, { - name: "allowed. right network, no peer", - isAllowed: true, - allowlist: []string{"/ip4/1.2.3.0/ipcidr/24"}, - endpoint: multiaddrA, - peer: peerA, + name: "allowed. right network, no peer", + isConnAllowed: true, + isAllowedWithPeer: true, + allowlist: []string{"/ip4/1.2.3.0/ipcidr/24"}, + endpoint: multiaddrA, + peer: peerA, }, { - name: "Blocked. right network, wrong peer", - isAllowed: false, - allowlist: []string{"/ip4/1.2.3.0/ipcidr/24" + "/p2p/" + peer.Encode(peerB)}, - endpoint: multiaddrA, - peer: peerA, + name: "Blocked. right network, wrong peer", + isConnAllowed: true, + isAllowedWithPeer: false, + allowlist: []string{"/ip4/1.2.3.0/ipcidr/24" + "/p2p/" + peer.Encode(peerB)}, + endpoint: multiaddrA, + peer: peerA, }, { - name: "allowed peer any ip", - isAllowed: true, - allowlist: []string{"/ip4/0.0.0.0/ipcidr/0"}, - endpoint: multiaddrA, - peer: peerA, + name: "allowed peer any ip", + isConnAllowed: true, + isAllowedWithPeer: true, + allowlist: []string{"/ip4/0.0.0.0/ipcidr/0"}, + endpoint: multiaddrA, + peer: peerA, }, { - name: "allowed peer multiple ips in allowlist", - isAllowed: true, - allowlist: []string{"/ip4/1.2.3.4/p2p/" + peer.Encode(peerA), "/ip4/2.2.3.4/p2p/" + peer.Encode(peerA)}, - endpoint: multiaddrA, - peer: peerA, + name: "allowed peer multiple ips in allowlist", + isConnAllowed: true, + isAllowedWithPeer: true, + allowlist: []string{"/ip4/1.2.3.4/p2p/" + peer.Encode(peerA), "/ip4/2.2.3.4/p2p/" + peer.Encode(peerA)}, + endpoint: multiaddrA, + peer: peerA, }, { - name: "allowed peer multiple ips in allowlist", - isAllowed: true, - allowlist: []string{"/ip4/1.2.3.4/p2p/" + peer.Encode(peerA), "/ip4/2.2.3.4/p2p/" + peer.Encode(peerA)}, - endpoint: multiaddrB, - peer: peerA, + name: "allowed peer multiple ips in allowlist", + isConnAllowed: true, + isAllowedWithPeer: true, + allowlist: []string{"/ip4/1.2.3.4/p2p/" + peer.Encode(peerA), "/ip4/2.2.3.4/p2p/" + peer.Encode(peerA)}, + endpoint: multiaddrB, + peer: peerA, }, } @@ -170,8 +133,8 @@ func TestAllowedWithPeer(t *testing.T) { allowlist.Add(ma) } - if allowlist.AllowedPeerAndMultiaddr(tc.peer, tc.endpoint) != tc.isAllowed { - t.Fatalf("%v: expected %v", !tc.isAllowed, tc.isAllowed) + if allowlist.AllowedPeerAndMultiaddr(tc.peer, tc.endpoint) != tc.isAllowedWithPeer { + t.Fatalf("%v: expected %v", !tc.isAllowedWithPeer, tc.isAllowedWithPeer) } }) } @@ -218,3 +181,79 @@ func TestRemoved(t *testing.T) { }) } } + +// BenchmarkAllowlistCheck benchmarks the allowlist with plausible conditions. +func BenchmarkAllowlistCheck(b *testing.B) { + allowlist := newAllowList() + + // How often do we expect a peer to be specified? 1 in N + ratioOfSpecifiedPeers := 10 + + // How often do we expect an allowlist hit? 1 in N + ratioOfAllowlistHit := 100 + + // How many multiaddrs in our allowlist? + howManyMultiaddrsInAllowList := 1_000 + + // How often is the IP addr an IPV6? 1 in N + ratioOfIPV6 := 20 + + countOfTotalPeersForTest := 100_000 + + peers := make([]peer.ID, countOfTotalPeersForTest) + mas := make([]multiaddr.Multiaddr, countOfTotalPeersForTest) + for i := 0; i < countOfTotalPeersForTest; i++ { + peers[i] = test.RandPeerIDFatal(b) + + ip := make([]byte, 16) + n, err := rand.Reader.Read(ip) + if err != nil || n != 16 { + b.Fatalf("Failed to generate IPv4 address") + } + + var ipString string + + if i%ratioOfIPV6 == 0 { + // IPv4 + ip6 := net.IP(ip) + peers[i] = test.RandPeerIDFatal(b) + ipString = "/ip6/" + ip6.String() + } else { + // IPv4 + ip4 := net.IPv4(ip[0], ip[1], ip[2], ip[3]) + peers[i] = test.RandPeerIDFatal(b) + ipString = "/ip4/" + ip4.String() + } + + var ma multiaddr.Multiaddr + if i%ratioOfSpecifiedPeers == 0 { + ma, err = multiaddr.NewMultiaddr(ipString + "/p2p/" + peer.Encode(peers[i])) + } else { + ma, err = multiaddr.NewMultiaddr(ipString) + } + if err != nil { + b.Fatalf("Failed to generate multiaddr: %v", ipString) + } + + mas[i] = ma + } + + for _, ma := range mas[:howManyMultiaddrsInAllowList] { + err := allowlist.Add(ma) + if err != nil { + b.Fatalf("Failed to add multiaddr") + } + } + + masInAllowList := mas[:howManyMultiaddrsInAllowList] + masNotInAllowList := mas[howManyMultiaddrsInAllowList:] + + b.ResetTimer() + for n := 0; n < b.N; n++ { + if n%ratioOfAllowlistHit == 0 { + allowlist.Allowed(masInAllowList[n%len(masInAllowList)]) + } else { + allowlist.Allowed(masNotInAllowList[n%len(masNotInAllowList)]) + } + } +} From 78498fbd88198ae809e7dc21e65cab021e39e2bc Mon Sep 17 00:00:00 2001 From: Marco Munizaga Date: Mon, 13 Jun 2022 16:29:28 -0700 Subject: [PATCH 05/45] Add comment --- allowlist.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/allowlist.go b/allowlist.go index 9d4e7ab..325b7fe 100644 --- a/allowlist.go +++ b/allowlist.go @@ -11,7 +11,10 @@ import ( ) type allowlist struct { - // TODO do we want to make this lookup faster? + // a simple structure of lists of networks. There is probably a faster way + // to check if an IP address is in this network than iterating over this + // list, but this is good enough for small numbers of networks (<1_000). + // Analyze the benchmark before trying to optimize this. // Any peer with these IPs are allowed allowedNetworks []*net.IPNet From 7ecf0fc714a317d9c9dc1a21f443481cae84d6de Mon Sep 17 00:00:00 2001 From: Marco Munizaga Date: Mon, 13 Jun 2022 16:33:37 -0700 Subject: [PATCH 06/45] Cleanup --- allowlist_test.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/allowlist_test.go b/allowlist_test.go index d9245b2..79f5744 100644 --- a/allowlist_test.go +++ b/allowlist_test.go @@ -133,6 +133,10 @@ func TestAllowedWithPeer(t *testing.T) { allowlist.Add(ma) } + if allowlist.Allowed(tc.endpoint) != tc.isConnAllowed { + t.Fatalf("%v: expected %v", !tc.isConnAllowed, tc.isConnAllowed) + } + if allowlist.AllowedPeerAndMultiaddr(tc.peer, tc.endpoint) != tc.isAllowedWithPeer { t.Fatalf("%v: expected %v", !tc.isAllowedWithPeer, tc.isAllowedWithPeer) } @@ -200,34 +204,30 @@ func BenchmarkAllowlistCheck(b *testing.B) { countOfTotalPeersForTest := 100_000 - peers := make([]peer.ID, countOfTotalPeersForTest) mas := make([]multiaddr.Multiaddr, countOfTotalPeersForTest) for i := 0; i < countOfTotalPeersForTest; i++ { - peers[i] = test.RandPeerIDFatal(b) ip := make([]byte, 16) n, err := rand.Reader.Read(ip) if err != nil || n != 16 { - b.Fatalf("Failed to generate IPv4 address") + b.Fatalf("Failed to generate IP address") } var ipString string if i%ratioOfIPV6 == 0 { - // IPv4 + // IPv6 ip6 := net.IP(ip) - peers[i] = test.RandPeerIDFatal(b) ipString = "/ip6/" + ip6.String() } else { // IPv4 ip4 := net.IPv4(ip[0], ip[1], ip[2], ip[3]) - peers[i] = test.RandPeerIDFatal(b) ipString = "/ip4/" + ip4.String() } var ma multiaddr.Multiaddr if i%ratioOfSpecifiedPeers == 0 { - ma, err = multiaddr.NewMultiaddr(ipString + "/p2p/" + peer.Encode(peers[i])) + ma, err = multiaddr.NewMultiaddr(ipString + "/p2p/" + peer.Encode(test.RandPeerIDFatal(b))) } else { ma, err = multiaddr.NewMultiaddr(ipString) } From 297cd00321b29b0faea1e8ebee06889f6a316aff Mon Sep 17 00:00:00 2001 From: Marco Munizaga Date: Mon, 13 Jun 2022 16:39:00 -0700 Subject: [PATCH 07/45] Fix comment --- rcmgr.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/rcmgr.go b/rcmgr.go index c262a8d..2315d8b 100644 --- a/rcmgr.go +++ b/rcmgr.go @@ -571,7 +571,6 @@ func (s *connectionScope) transferAllowedToStandard() (err error) { } func (s *connectionScope) SetPeer(p peer.ID) error { - // TODO check if this connectionscope is part of the allowlist and do the peer checking if so. s.Lock() defer s.Unlock() @@ -588,10 +587,10 @@ func (s *connectionScope) SetPeer(p peer.ID) error { } if s.isAllowlisted && !s.rcmgr.allowlist.AllowedPeerAndMultiaddr(p, s.endpoint) { - // This is not a allowed peer + multiaddr combination. We need to + // This is not an allowed peer + multiaddr combination. We need to // transfer this connection to the general scope. We'll do this first by // transferring the connection to the system and transient scopes, then - // continue on with this function. The idea is that a connection + // continue on with this function. The idea is that a connection // shouldn't get the benefit of evading the transient scope because it // was _almost_ an allowlisted connection. if err := s.transferAllowedToStandard(); err != nil { From 84ca9d337a7da7b33d36d1236f6a80215d15e989 Mon Sep 17 00:00:00 2001 From: Marco Munizaga Date: Tue, 14 Jun 2022 11:46:26 -0700 Subject: [PATCH 08/45] PR comments --- allowlist.go | 75 +++++++++++++++++++++++------------------------ allowlist_test.go | 23 +++++++-------- rcmgr.go | 2 +- 3 files changed, 47 insertions(+), 53 deletions(-) diff --git a/allowlist.go b/allowlist.go index 325b7fe..0bb73fe 100644 --- a/allowlist.go +++ b/allowlist.go @@ -3,6 +3,7 @@ package rcmgr import ( "bytes" "errors" + "fmt" "net" "github.com/libp2p/go-libp2p-core/peer" @@ -23,16 +24,17 @@ type allowlist struct { allowedPeerByNetwork map[peer.ID][]*net.IPNet } -func newAllowList() allowlist { +func newAllowlist() allowlist { return allowlist{ allowedPeerByNetwork: make(map[peer.ID][]*net.IPNet), } } -func toIPNet(ma multiaddr.Multiaddr) (*net.IPNet, string, error) { +func toIPNet(ma multiaddr.Multiaddr) (*net.IPNet, peer.ID, error) { var ipString string var mask string - var allowedPeer string + var allowedPeerStr string + var allowedPeer peer.ID var isIPV4 bool multiaddr.ForEach(ma, func(c multiaddr.Component) bool { @@ -44,15 +46,23 @@ func toIPNet(ma multiaddr.Multiaddr) (*net.IPNet, string, error) { mask = c.Value() } if c.Protocol().Code == multiaddr.P_P2P { - allowedPeer = c.Value() + allowedPeerStr = c.Value() } - return ipString == "" || mask == "" || allowedPeer == "" + return ipString == "" || mask == "" || allowedPeerStr == "" }) if ipString == "" { return nil, allowedPeer, errors.New("missing ip address") } + if allowedPeerStr != "" { + var err error + allowedPeer, err = peer.Decode(allowedPeerStr) + if err != nil { + return nil, allowedPeer, fmt.Errorf("failed to decode allowed peer: %w", err) + } + } + if mask == "" { ip := net.ParseIP(ipString) if ip == nil { @@ -74,59 +84,48 @@ func toIPNet(ma multiaddr.Multiaddr) (*net.IPNet, string, error) { } +// Add takes a multiaddr and adds it to the allowlist. The multiaddr should be +// an ip address of the peer with or without a `/p2p` protocol. +// e.g. /ip4/1.2.3.4/p2p/QmFoo and /ip4/1.2.3.4 are valid. +// /p2p/QmFoo is not valid. func (al *allowlist) Add(ma multiaddr.Multiaddr) error { - ipnet, allowedPeerStr, err := toIPNet(ma) + ipnet, allowedPeer, err := toIPNet(ma) if err != nil { return err } - if allowedPeerStr != "" { + if allowedPeer != peer.ID("") { // We have a peerID constraint - allowedPeer, err := peer.Decode(allowedPeerStr) - if err != nil { - return err - } - if ipnet != nil { - al.allowedPeerByNetwork[allowedPeer] = append(al.allowedPeerByNetwork[allowedPeer], ipnet) - } + al.allowedPeerByNetwork[allowedPeer] = append(al.allowedPeerByNetwork[allowedPeer], ipnet) } else { - if ipnet != nil { - al.allowedNetworks = append(al.allowedNetworks, ipnet) - } + al.allowedNetworks = append(al.allowedNetworks, ipnet) } return nil } func (al *allowlist) Remove(ma multiaddr.Multiaddr) error { - ipnet, allowedPeerStr, err := toIPNet(ma) + ipnet, allowedPeer, err := toIPNet(ma) if err != nil { return err } ipNetList := al.allowedNetworks - var allowedPeer peer.ID - if allowedPeerStr != "" { + if allowedPeer != peer.ID("") { // We have a peerID constraint - allowedPeer, err = peer.Decode(allowedPeerStr) - if err != nil { - return err - } ipNetList = al.allowedPeerByNetwork[allowedPeer] } - if ipnet != nil { - i := len(ipNetList) - for i > 0 { - i-- - if ipNetList[i].IP.Equal(ipnet.IP) && bytes.Equal(ipNetList[i].Mask, ipnet.Mask) { - if i == len(ipNetList)-1 { - // Trim this element from the end - ipNetList = ipNetList[:i] - } else { - // swap remove - ipNetList[i] = ipNetList[len(ipNetList)-1] - ipNetList = ipNetList[:len(ipNetList)-1] - } + i := len(ipNetList) + for i > 0 { + i-- + if ipNetList[i].IP.Equal(ipnet.IP) && bytes.Equal(ipNetList[i].Mask, ipnet.Mask) { + if i == len(ipNetList)-1 { + // Trim this element from the end + ipNetList = ipNetList[:i] + } else { + // swap remove + ipNetList[i] = ipNetList[len(ipNetList)-1] + ipNetList = ipNetList[:len(ipNetList)-1] } } } @@ -146,9 +145,7 @@ func (al *allowlist) Allowed(ma multiaddr.Multiaddr) bool { return false } - _ = ip for _, network := range al.allowedNetworks { - _ = network if network.Contains(ip) { return true } diff --git a/allowlist_test.go b/allowlist_test.go index 79f5744..3a8e9f9 100644 --- a/allowlist_test.go +++ b/allowlist_test.go @@ -12,8 +12,8 @@ import ( ) func TestAllowedSimple(t *testing.T) { - allowlist := newAllowList() - ma, _ := multiaddr.NewMultiaddr("/ip4/1.2.3.4/tcp/1234") + allowlist := newAllowlist() + ma := multiaddr.StringCast("/ip4/1.2.3.4/tcp/1234") err := allowlist.Add(ma) if err != nil { t.Fatalf("failed to add ip4: %s", err) @@ -124,7 +124,7 @@ func TestAllowedWithPeer(t *testing.T) { for _, tc := range testcases { t.Run(tc.name, func(t *testing.T) { - allowlist := newAllowList() + allowlist := newAllowlist() for _, maStr := range tc.allowlist { ma, err := multiaddr.NewMultiaddr(maStr) if err != nil { @@ -151,7 +151,7 @@ func TestRemoved(t *testing.T) { allowedMA string } peerA := test.RandPeerIDFatal(t) - maA, _ := multiaddr.NewMultiaddr("/ip4/1.2.3.4") + maA := multiaddr.StringCast("/ip4/1.2.3.4") testCases := []testCase{ {name: "ip4", allowedMA: "/ip4/1.2.3.4"}, @@ -162,13 +162,10 @@ func TestRemoved(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - allowlist := newAllowList() - ma, err := multiaddr.NewMultiaddr(tc.allowedMA) - if err != nil { - t.Fatalf("failed to parse ma: %s", err) - } + allowlist := newAllowlist() + ma := multiaddr.StringCast(tc.allowedMA) - err = allowlist.Add(ma) + err := allowlist.Add(ma) if err != nil { t.Fatalf("failed to add ip4: %s", err) } @@ -188,7 +185,7 @@ func TestRemoved(t *testing.T) { // BenchmarkAllowlistCheck benchmarks the allowlist with plausible conditions. func BenchmarkAllowlistCheck(b *testing.B) { - allowlist := newAllowList() + allowlist := newAllowlist() // How often do we expect a peer to be specified? 1 in N ratioOfSpecifiedPeers := 10 @@ -227,9 +224,9 @@ func BenchmarkAllowlistCheck(b *testing.B) { var ma multiaddr.Multiaddr if i%ratioOfSpecifiedPeers == 0 { - ma, err = multiaddr.NewMultiaddr(ipString + "/p2p/" + peer.Encode(test.RandPeerIDFatal(b))) + ma = multiaddr.StringCast(ipString + "/p2p/" + peer.Encode(test.RandPeerIDFatal(b))) } else { - ma, err = multiaddr.NewMultiaddr(ipString) + ma = multiaddr.StringCast(ipString) } if err != nil { b.Fatalf("Failed to generate multiaddr: %v", ipString) diff --git a/rcmgr.go b/rcmgr.go index 2315d8b..8a91700 100644 --- a/rcmgr.go +++ b/rcmgr.go @@ -125,7 +125,7 @@ var _ network.StreamManagementScope = (*streamScope)(nil) type Option func(*resourceManager) error func NewResourceManager(limits Limiter, opts ...Option) (network.ResourceManager, error) { - allowlist := newAllowList() + allowlist := newAllowlist() r := &resourceManager{ limits: limits, allowlist: &allowlist, From 539b1476fcac1ba185c049d520867f9af7a59f15 Mon Sep 17 00:00:00 2001 From: Marco Munizaga Date: Tue, 14 Jun 2022 11:48:56 -0700 Subject: [PATCH 09/45] Use the same standard limits for the allowlistedSystem and allowlistedTransient --- rcmgr.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/rcmgr.go b/rcmgr.go index 8a91700..b5c28c2 100644 --- a/rcmgr.go +++ b/rcmgr.go @@ -149,6 +149,11 @@ func NewResourceManager(limits Limiter, opts ...Option) (network.ResourceManager r.transient = newTransientScope(limits.GetTransientLimits(), r) r.transient.IncRef() + r.allowlistedSystem = newSystemScope(limits.GetSystemLimits(), r) + r.allowlistedSystem.IncRef() + r.allowlistedTransient = newTransientScope(limits.GetTransientLimits(), r) + r.allowlistedTransient.IncRef() + r.cancelCtx, r.cancel = context.WithCancel(context.Background()) r.wg.Add(1) From dc09a97d9b9aef8cf0c7c0bac71cdc7df04cff19 Mon Sep 17 00:00:00 2001 From: Marco Munizaga Date: Wed, 15 Jun 2022 15:21:39 -0700 Subject: [PATCH 10/45] Fix allowlist system instantiation --- rcmgr.go | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/rcmgr.go b/rcmgr.go index b5c28c2..a0f75e5 100644 --- a/rcmgr.go +++ b/rcmgr.go @@ -144,14 +144,14 @@ func NewResourceManager(limits Limiter, opts ...Option) (network.ResourceManager return nil, err } - r.system = newSystemScope(limits.GetSystemLimits(), r) + r.system = newSystemScope(limits.GetSystemLimits(), r, "system") r.system.IncRef() - r.transient = newTransientScope(limits.GetTransientLimits(), r) + r.transient = newTransientScope(limits.GetTransientLimits(), r, "transient", r.system.resourceScope) r.transient.IncRef() - r.allowlistedSystem = newSystemScope(limits.GetSystemLimits(), r) + r.allowlistedSystem = newSystemScope(limits.GetSystemLimits(), r, "allowlistedSystem") r.allowlistedSystem.IncRef() - r.allowlistedTransient = newTransientScope(limits.GetTransientLimits(), r) + r.allowlistedTransient = newTransientScope(limits.GetTransientLimits(), r, "allowlistedTransient", r.allowlistedSystem.resourceScope) r.allowlistedTransient.IncRef() r.cancelCtx, r.cancel = context.WithCancel(context.Background()) @@ -384,17 +384,17 @@ func (r *resourceManager) gc() { } } -func newSystemScope(limit Limit, rcmgr *resourceManager) *systemScope { +func newSystemScope(limit Limit, rcmgr *resourceManager, name string) *systemScope { return &systemScope{ - resourceScope: newResourceScope(limit, nil, "system", rcmgr.trace, rcmgr.metrics), + resourceScope: newResourceScope(limit, nil, name, rcmgr.trace, rcmgr.metrics), } } -func newTransientScope(limit Limit, rcmgr *resourceManager) *transientScope { +func newTransientScope(limit Limit, rcmgr *resourceManager, name string, systemScope *resourceScope) *transientScope { return &transientScope{ resourceScope: newResourceScope(limit, - []*resourceScope{rcmgr.system.resourceScope}, - "transient", rcmgr.trace, rcmgr.metrics), + []*resourceScope{systemScope}, + name, rcmgr.trace, rcmgr.metrics), system: rcmgr.system, } } From a41033230f25f225a227948478af481ca7c48f41 Mon Sep 17 00:00:00 2001 From: Marco Munizaga Date: Wed, 15 Jun 2022 15:25:40 -0700 Subject: [PATCH 11/45] Persist endpoint --- rcmgr.go | 1 + 1 file changed, 1 insertion(+) diff --git a/rcmgr.go b/rcmgr.go index a0f75e5..7256f13 100644 --- a/rcmgr.go +++ b/rcmgr.go @@ -278,6 +278,7 @@ func (r *resourceManager) OpenConnection(dir network.Direction, usefd bool, endp } else { conn = newConnectionScope(dir, usefd, r.limits.GetConnLimits(), r) } + conn.endpoint = endpoint if err := conn.AddConn(dir, usefd); err != nil { conn.Done() From 1bf5e930ee94f59fb1b3c0f15abaa721bf7c0ba4 Mon Sep 17 00:00:00 2001 From: Marco Munizaga Date: Mon, 27 Jun 2022 13:59:26 -0700 Subject: [PATCH 12/45] Fuse with if above --- rcmgr.go | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/rcmgr.go b/rcmgr.go index 7256f13..63ce9cf 100644 --- a/rcmgr.go +++ b/rcmgr.go @@ -590,18 +590,18 @@ func (s *connectionScope) SetPeer(p peer.ID) error { if s.isAllowlisted { system = s.rcmgr.allowlistedSystem transient = s.rcmgr.transient - } - if s.isAllowlisted && !s.rcmgr.allowlist.AllowedPeerAndMultiaddr(p, s.endpoint) { - // This is not an allowed peer + multiaddr combination. We need to - // transfer this connection to the general scope. We'll do this first by - // transferring the connection to the system and transient scopes, then - // continue on with this function. The idea is that a connection - // shouldn't get the benefit of evading the transient scope because it - // was _almost_ an allowlisted connection. - if err := s.transferAllowedToStandard(); err != nil { - // Failed to transfer this connection to the standard scopes - return err + if !s.rcmgr.allowlist.AllowedPeerAndMultiaddr(p, s.endpoint) { + // This is not an allowed peer + multiaddr combination. We need to + // transfer this connection to the general scope. We'll do this first by + // transferring the connection to the system and transient scopes, then + // continue on with this function. The idea is that a connection + // shouldn't get the benefit of evading the transient scope because it + // was _almost_ an allowlisted connection. + if err := s.transferAllowedToStandard(); err != nil { + // Failed to transfer this connection to the standard scopes + return err + } } } From fd82a9fe0140f3a2ada8f768467ef630e335c387 Mon Sep 17 00:00:00 2001 From: Marco Munizaga Date: Mon, 27 Jun 2022 14:40:07 -0700 Subject: [PATCH 13/45] Protect inner data with RWMutex --- allowlist.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/allowlist.go b/allowlist.go index 0bb73fe..efe37a3 100644 --- a/allowlist.go +++ b/allowlist.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "net" + "sync" "github.com/libp2p/go-libp2p-core/peer" "github.com/multiformats/go-multiaddr" @@ -12,6 +13,7 @@ import ( ) type allowlist struct { + mu sync.RWMutex // a simple structure of lists of networks. There is probably a faster way // to check if an IP address is in this network than iterating over this // list, but this is good enough for small numbers of networks (<1_000). @@ -89,6 +91,8 @@ func toIPNet(ma multiaddr.Multiaddr) (*net.IPNet, peer.ID, error) { // e.g. /ip4/1.2.3.4/p2p/QmFoo and /ip4/1.2.3.4 are valid. // /p2p/QmFoo is not valid. func (al *allowlist) Add(ma multiaddr.Multiaddr) error { + al.mu.Lock() + defer al.mu.Unlock() ipnet, allowedPeer, err := toIPNet(ma) if err != nil { return err @@ -104,6 +108,9 @@ func (al *allowlist) Add(ma multiaddr.Multiaddr) error { } func (al *allowlist) Remove(ma multiaddr.Multiaddr) error { + al.mu.Lock() + defer al.mu.Unlock() + ipnet, allowedPeer, err := toIPNet(ma) if err != nil { return err @@ -140,6 +147,8 @@ func (al *allowlist) Remove(ma multiaddr.Multiaddr) error { } func (al *allowlist) Allowed(ma multiaddr.Multiaddr) bool { + al.mu.RLock() + defer al.mu.RUnlock() ip, err := manet.ToIP(ma) if err != nil { return false @@ -163,6 +172,8 @@ func (al *allowlist) Allowed(ma multiaddr.Multiaddr) bool { } func (al *allowlist) AllowedPeerAndMultiaddr(peerID peer.ID, ma multiaddr.Multiaddr) bool { + al.mu.RLock() + defer al.mu.RUnlock() ip, err := manet.ToIP(ma) if err != nil { return false From 61008b316cd64938dc41634e347c5d381618b2d9 Mon Sep 17 00:00:00 2001 From: Marco Munizaga Date: Mon, 27 Jun 2022 15:04:27 -0700 Subject: [PATCH 14/45] Only use allowlist if the normal resource scopes would have failed --- rcmgr.go | 51 +++++++++++++++++++++++++++++++++------------------ 1 file changed, 33 insertions(+), 18 deletions(-) diff --git a/rcmgr.go b/rcmgr.go index 63ce9cf..ee1a488 100644 --- a/rcmgr.go +++ b/rcmgr.go @@ -271,16 +271,22 @@ func (r *resourceManager) nextStreamId() int64 { } func (r *resourceManager) OpenConnection(dir network.Direction, usefd bool, endpoint multiaddr.Multiaddr) (network.ConnManagementScope, error) { - allowed := r.allowlist.Allowed(endpoint) var conn *connectionScope - if allowed { - conn = newAllowListedConnectionScope(dir, usefd, r.limits.GetConnLimits(), r) - } else { - conn = newConnectionScope(dir, usefd, r.limits.GetConnLimits(), r) + conn = newConnectionScope(dir, usefd, r.limits.GetConnLimits(), r, endpoint) + + err := conn.AddConn(dir, usefd) + if err != nil { + // Try again if this is an allowlisted connection + // Failed to open connection, let's see if this was allowlisted and try again + allowed := r.allowlist.Allowed(endpoint) + if allowed { + conn.Done() + conn = newAllowListedConnectionScope(dir, usefd, r.limits.GetConnLimits(), r, endpoint) + err = conn.AddConn(dir, usefd) + } } - conn.endpoint = endpoint - if err := conn.AddConn(dir, usefd); err != nil { + if err != nil { conn.Done() r.metrics.BlockConn(dir, usefd) return nil, err @@ -430,25 +436,28 @@ func newPeerScope(p peer.ID, limit Limit, rcmgr *resourceManager) *peerScope { } } -func newConnectionScope(dir network.Direction, usefd bool, limit Limit, rcmgr *resourceManager) *connectionScope { +func newConnectionScope(dir network.Direction, usefd bool, limit Limit, rcmgr *resourceManager, endpoint multiaddr.Multiaddr) *connectionScope { return &connectionScope{ resourceScope: newResourceScope(limit, []*resourceScope{rcmgr.transient.resourceScope, rcmgr.system.resourceScope}, fmt.Sprintf("conn-%d", rcmgr.nextConnId()), rcmgr.trace, rcmgr.metrics), - dir: dir, - usefd: usefd, - rcmgr: rcmgr, + dir: dir, + usefd: usefd, + rcmgr: rcmgr, + endpoint: endpoint, } } -func newAllowListedConnectionScope(dir network.Direction, usefd bool, limit Limit, rcmgr *resourceManager) *connectionScope { +func newAllowListedConnectionScope(dir network.Direction, usefd bool, limit Limit, rcmgr *resourceManager, endpoint multiaddr.Multiaddr) *connectionScope { return &connectionScope{ resourceScope: newResourceScope(limit, []*resourceScope{rcmgr.allowlistedTransient.resourceScope, rcmgr.allowlistedSystem.resourceScope}, fmt.Sprintf("conn-%d", rcmgr.nextConnId()), rcmgr.trace, rcmgr.metrics), - dir: dir, - usefd: usefd, - rcmgr: rcmgr, + dir: dir, + usefd: usefd, + rcmgr: rcmgr, + endpoint: endpoint, + isAllowlisted: true, } } @@ -533,14 +542,14 @@ func (s *connectionScope) PeerScope() network.PeerScope { return s.peer } -// transferAllowedToStandard Transfers this connection scope from being part of +// transferAllowedToStandard transfers this connection scope from being part of // the allowlist set of scopes to being part of the standard set of scopes. // Happens when we first allowlisted this connection due to its IP, but later // discovered that the peer id not what we expected. func (s *connectionScope) transferAllowedToStandard() (err error) { systemScope := s.rcmgr.system.resourceScope - transientScope := s.rcmgr.system.resourceScope + transientScope := s.rcmgr.transient.resourceScope stat := s.resourceScope.rc.stat() @@ -589,9 +598,11 @@ func (s *connectionScope) SetPeer(p peer.ID) error { if s.isAllowlisted { system = s.rcmgr.allowlistedSystem - transient = s.rcmgr.transient + transient = s.rcmgr.allowlistedTransient if !s.rcmgr.allowlist.AllowedPeerAndMultiaddr(p, s.endpoint) { + s.isAllowlisted = false + // This is not an allowed peer + multiaddr combination. We need to // transfer this connection to the general scope. We'll do this first by // transferring the connection to the system and transient scopes, then @@ -602,6 +613,10 @@ func (s *connectionScope) SetPeer(p peer.ID) error { // Failed to transfer this connection to the standard scopes return err } + + // set the system and transient scopes to the non-allowlisted ones + system = s.rcmgr.system + transient = s.rcmgr.transient } } From 5f5fdb3b6c0f790f706e2199b5ce0bf64150052d Mon Sep 17 00:00:00 2001 From: Marco Munizaga Date: Mon, 27 Jun 2022 15:30:03 -0700 Subject: [PATCH 15/45] Update comment --- allowlist.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/allowlist.go b/allowlist.go index efe37a3..df0e90c 100644 --- a/allowlist.go +++ b/allowlist.go @@ -88,7 +88,7 @@ func toIPNet(ma multiaddr.Multiaddr) (*net.IPNet, peer.ID, error) { // Add takes a multiaddr and adds it to the allowlist. The multiaddr should be // an ip address of the peer with or without a `/p2p` protocol. -// e.g. /ip4/1.2.3.4/p2p/QmFoo and /ip4/1.2.3.4 are valid. +// e.g. /ip4/1.2.3.4/p2p/QmFoo, /ip4/1.2.3.4, and /ip4/1.2.3.0/ipcidr/24 are valid. // /p2p/QmFoo is not valid. func (al *allowlist) Add(ma multiaddr.Multiaddr) error { al.mu.Lock() From a5e8e04ea97bd49fb9fd9ec154a56d436970d754 Mon Sep 17 00:00:00 2001 From: Marco Munizaga Date: Mon, 27 Jun 2022 15:30:21 -0700 Subject: [PATCH 16/45] Allowlist test with rcmgr --- rcmgr_test.go | 67 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/rcmgr_test.go b/rcmgr_test.go index 4bb8106..1822817 100644 --- a/rcmgr_test.go +++ b/rcmgr_test.go @@ -6,6 +6,7 @@ import ( "github.com/libp2p/go-libp2p-core/network" "github.com/libp2p/go-libp2p-core/peer" "github.com/libp2p/go-libp2p-core/protocol" + "github.com/libp2p/go-libp2p-core/test" "github.com/multiformats/go-multiaddr" ) @@ -1001,3 +1002,69 @@ func TestResourceManager(t *testing.T) { } } + +func TestResourceManagerWithAllowlist(t *testing.T) { + limits := NewDefaultLimiter() + limits.SystemLimits = limits.SystemLimits.WithConnLimit(0, 0, 0) + limits.TransientLimits = limits.SystemLimits.WithConnLimit(0, 0, 0) + rcmgr, err := NewResourceManager(limits) + if err != nil { + t.Fatal(err) + } + + peerA := test.RandPeerIDFatal(t) + + { + // Setup allowlist. TODO, replace this with a config once config changes are in + r := rcmgr.(*resourceManager) + + r.allowlistedSystem = newSystemScope(limits.GetSystemLimits().WithConnLimit(2, 1, 2), r, "allowlistedSystem") + r.allowlistedSystem.IncRef() + r.allowlistedTransient = newTransientScope(limits.GetTransientLimits().WithConnLimit(1, 1, 1), r, "allowlistedTransient", r.allowlistedSystem.resourceScope) + r.allowlistedTransient.IncRef() + allowlist := r.allowlist + + allowlist.Add(multiaddr.StringCast("/ip4/1.2.3.4")) + allowlist.Add(multiaddr.StringCast("/ip4/4.3.2.1/p2p/" + peerA.String())) + } + + // A connection comes in from a non-allowlisted ip address + _, err = rcmgr.OpenConnection(network.DirInbound, true, multiaddr.StringCast("/ip4/1.2.3.5")) + if err == nil { + t.Fatalf("Expected this to fail. err=%v", err) + } + + // A connection comes in from an allowlisted ip address + connScope, err := rcmgr.OpenConnection(network.DirInbound, true, multiaddr.StringCast("/ip4/1.2.3.4")) + if err != nil { + t.Fatal(err) + } + + err = connScope.SetPeer(test.RandPeerIDFatal(t)) + if err != nil { + t.Fatal(err) + } + + // A connection comes in that looks like it should be allowlisted, but then has the wrong peer id. + connScope, err = rcmgr.OpenConnection(network.DirInbound, true, multiaddr.StringCast("/ip4/4.3.2.1")) + if err != nil { + t.Fatal(err) + } + + err = connScope.SetPeer(test.RandPeerIDFatal(t)) + if err == nil { + t.Fatalf("Expected this to fail. err=%v", err) + } + + // A connection comes in that looks like it should be allowlisted, and it has the allowlisted peer id + connScope, err = rcmgr.OpenConnection(network.DirInbound, true, multiaddr.StringCast("/ip4/4.3.2.1")) + if err != nil { + t.Fatal(err) + } + + err = connScope.SetPeer(peerA) + if err != nil { + t.Fatal(err) + } + +} From 2bb066a2ce17122ef411bb50fe8f2173399c1899 Mon Sep 17 00:00:00 2001 From: Marco Munizaga Date: Mon, 27 Jun 2022 15:39:26 -0700 Subject: [PATCH 17/45] Export Allowlist type --- allowlist.go | 21 ++++++++++++++------- rcmgr.go | 2 +- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/allowlist.go b/allowlist.go index df0e90c..722e576 100644 --- a/allowlist.go +++ b/allowlist.go @@ -12,7 +12,7 @@ import ( manet "github.com/multiformats/go-multiaddr/net" ) -type allowlist struct { +type Allowlist struct { mu sync.RWMutex // a simple structure of lists of networks. There is probably a faster way // to check if an IP address is in this network than iterating over this @@ -26,8 +26,8 @@ type allowlist struct { allowedPeerByNetwork map[peer.ID][]*net.IPNet } -func newAllowlist() allowlist { - return allowlist{ +func newAllowlist() Allowlist { + return Allowlist{ allowedPeerByNetwork: make(map[peer.ID][]*net.IPNet), } } @@ -90,7 +90,7 @@ func toIPNet(ma multiaddr.Multiaddr) (*net.IPNet, peer.ID, error) { // an ip address of the peer with or without a `/p2p` protocol. // e.g. /ip4/1.2.3.4/p2p/QmFoo, /ip4/1.2.3.4, and /ip4/1.2.3.0/ipcidr/24 are valid. // /p2p/QmFoo is not valid. -func (al *allowlist) Add(ma multiaddr.Multiaddr) error { +func (al *Allowlist) Add(ma multiaddr.Multiaddr) error { al.mu.Lock() defer al.mu.Unlock() ipnet, allowedPeer, err := toIPNet(ma) @@ -100,6 +100,9 @@ func (al *allowlist) Add(ma multiaddr.Multiaddr) error { if allowedPeer != peer.ID("") { // We have a peerID constraint + if al.allowedPeerByNetwork == nil { + al.allowedPeerByNetwork = make(map[peer.ID][]*net.IPNet) + } al.allowedPeerByNetwork[allowedPeer] = append(al.allowedPeerByNetwork[allowedPeer], ipnet) } else { al.allowedNetworks = append(al.allowedNetworks, ipnet) @@ -107,7 +110,7 @@ func (al *allowlist) Add(ma multiaddr.Multiaddr) error { return nil } -func (al *allowlist) Remove(ma multiaddr.Multiaddr) error { +func (al *Allowlist) Remove(ma multiaddr.Multiaddr) error { al.mu.Lock() defer al.mu.Unlock() @@ -122,6 +125,10 @@ func (al *allowlist) Remove(ma multiaddr.Multiaddr) error { ipNetList = al.allowedPeerByNetwork[allowedPeer] } + if ipNetList == nil { + return nil + } + i := len(ipNetList) for i > 0 { i-- @@ -146,7 +153,7 @@ func (al *allowlist) Remove(ma multiaddr.Multiaddr) error { return nil } -func (al *allowlist) Allowed(ma multiaddr.Multiaddr) bool { +func (al *Allowlist) Allowed(ma multiaddr.Multiaddr) bool { al.mu.RLock() defer al.mu.RUnlock() ip, err := manet.ToIP(ma) @@ -171,7 +178,7 @@ func (al *allowlist) Allowed(ma multiaddr.Multiaddr) bool { return false } -func (al *allowlist) AllowedPeerAndMultiaddr(peerID peer.ID, ma multiaddr.Multiaddr) bool { +func (al *Allowlist) AllowedPeerAndMultiaddr(peerID peer.ID, ma multiaddr.Multiaddr) bool { al.mu.RLock() defer al.mu.RUnlock() ip, err := manet.ToIP(ma) diff --git a/rcmgr.go b/rcmgr.go index ee1a488..663f0e3 100644 --- a/rcmgr.go +++ b/rcmgr.go @@ -22,7 +22,7 @@ type resourceManager struct { trace *trace metrics *metrics - allowlist *allowlist + allowlist *Allowlist system *systemScope transient *transientScope From 8c5643aa2f6f828235c6c946e15a4c806f6def0a Mon Sep 17 00:00:00 2001 From: Marco Munizaga Date: Mon, 27 Jun 2022 15:42:00 -0700 Subject: [PATCH 18/45] Add GetAllowlist() --- rcmgr.go | 4 ++++ rcmgr_test.go | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/rcmgr.go b/rcmgr.go index 663f0e3..2be9113 100644 --- a/rcmgr.go +++ b/rcmgr.go @@ -162,6 +162,10 @@ func NewResourceManager(limits Limiter, opts ...Option) (network.ResourceManager return r, nil } +func (r *resourceManager) GetAllowlist() *Allowlist { + return r.allowlist +} + func (r *resourceManager) ViewSystem(f func(network.ResourceScope) error) error { return f(r.system) } diff --git a/rcmgr_test.go b/rcmgr_test.go index 1822817..bf530c0 100644 --- a/rcmgr_test.go +++ b/rcmgr_test.go @@ -1022,7 +1022,7 @@ func TestResourceManagerWithAllowlist(t *testing.T) { r.allowlistedSystem.IncRef() r.allowlistedTransient = newTransientScope(limits.GetTransientLimits().WithConnLimit(1, 1, 1), r, "allowlistedTransient", r.allowlistedSystem.resourceScope) r.allowlistedTransient.IncRef() - allowlist := r.allowlist + allowlist := r.GetAllowlist() allowlist.Add(multiaddr.StringCast("/ip4/1.2.3.4")) allowlist.Add(multiaddr.StringCast("/ip4/4.3.2.1/p2p/" + peerA.String())) From 374b297cee17bc0b73480e738e65137795570bdd Mon Sep 17 00:00:00 2001 From: Marco Munizaga Date: Mon, 27 Jun 2022 15:47:51 -0700 Subject: [PATCH 19/45] Add WithAllowlistedMultiaddr option --- allowlist.go | 13 +++++++++++++ rcmgr_test.go | 14 ++++++-------- 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/allowlist.go b/allowlist.go index 722e576..9e919c5 100644 --- a/allowlist.go +++ b/allowlist.go @@ -26,6 +26,19 @@ type Allowlist struct { allowedPeerByNetwork map[peer.ID][]*net.IPNet } +// WithAllowlistedMultiaddrs sets the multiaddrs to be in the allowlist +func WithAllowlistedMultiaddrs(mas []multiaddr.Multiaddr) Option { + return func(rm *resourceManager) error { + for _, ma := range mas { + err := rm.allowlist.Add(ma) + if err != nil { + return err + } + } + return nil + } +} + func newAllowlist() Allowlist { return Allowlist{ allowedPeerByNetwork: make(map[peer.ID][]*net.IPNet), diff --git a/rcmgr_test.go b/rcmgr_test.go index bf530c0..1656a0d 100644 --- a/rcmgr_test.go +++ b/rcmgr_test.go @@ -1004,16 +1004,19 @@ func TestResourceManager(t *testing.T) { } func TestResourceManagerWithAllowlist(t *testing.T) { + peerA := test.RandPeerIDFatal(t) + limits := NewDefaultLimiter() limits.SystemLimits = limits.SystemLimits.WithConnLimit(0, 0, 0) limits.TransientLimits = limits.SystemLimits.WithConnLimit(0, 0, 0) - rcmgr, err := NewResourceManager(limits) + rcmgr, err := NewResourceManager(limits, WithAllowlistedMultiaddrs([]multiaddr.Multiaddr{ + multiaddr.StringCast("/ip4/1.2.3.4"), + multiaddr.StringCast("/ip4/4.3.2.1/p2p/" + peerA.String()), + })) if err != nil { t.Fatal(err) } - peerA := test.RandPeerIDFatal(t) - { // Setup allowlist. TODO, replace this with a config once config changes are in r := rcmgr.(*resourceManager) @@ -1022,10 +1025,6 @@ func TestResourceManagerWithAllowlist(t *testing.T) { r.allowlistedSystem.IncRef() r.allowlistedTransient = newTransientScope(limits.GetTransientLimits().WithConnLimit(1, 1, 1), r, "allowlistedTransient", r.allowlistedSystem.resourceScope) r.allowlistedTransient.IncRef() - allowlist := r.GetAllowlist() - - allowlist.Add(multiaddr.StringCast("/ip4/1.2.3.4")) - allowlist.Add(multiaddr.StringCast("/ip4/4.3.2.1/p2p/" + peerA.String())) } // A connection comes in from a non-allowlisted ip address @@ -1066,5 +1065,4 @@ func TestResourceManagerWithAllowlist(t *testing.T) { if err != nil { t.Fatal(err) } - } From 600191a10c392829607a2c1eb2cf4f3ae761521b Mon Sep 17 00:00:00 2001 From: Marco Munizaga Date: Mon, 27 Jun 2022 15:54:42 -0700 Subject: [PATCH 20/45] Update allowlist design doc with no extra cost for happy path --- allowlist_test.go | 8 ++++++++ docs/allowlist.md | 13 ++++++++----- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/allowlist_test.go b/allowlist_test.go index 3a8e9f9..713e5b8 100644 --- a/allowlist_test.go +++ b/allowlist_test.go @@ -112,6 +112,14 @@ func TestAllowedWithPeer(t *testing.T) { endpoint: multiaddrA, peer: peerA, }, + { + name: "allowed peer multiple ips in allowlist", + isConnAllowed: true, + isAllowedWithPeer: true, + allowlist: []string{"/ip4/1.2.3.4/p2p/" + peer.Encode(peerA), "/ip4/1.2.3.4/p2p/" + peer.Encode(peerA)}, + endpoint: multiaddrA, + peer: peerA, + }, { name: "allowed peer multiple ips in allowlist", isConnAllowed: true, diff --git a/docs/allowlist.md b/docs/allowlist.md index 1819039..4ada09c 100644 --- a/docs/allowlist.md +++ b/docs/allowlist.md @@ -20,6 +20,8 @@ This is the problem that the Allowlist is designed to solve. - A peer may be able to open a connection, but after the handshake, if it's not an expected peer id we move it to the normal system scope. - We can have multiple PeerIDs for a given IP addr. +- No extra cost for the happy path when we are still below system and transient + limits. ## Proposed change @@ -38,11 +40,12 @@ For example, an allowlist set could look like: /ip4/1.2.3.0/ipcidr/24 ``` -When a new connection is opened, the resource manager checks if the multiaddr -matches an item in the set of allowlisted multiaddrs. If so, it creates the -connection resource scope using the allowlisted specific system and transient -resource scopes. Otherwise, it uses the normal system and transient resource -scopes as it does today. +When a new connection is opened, the resource manager tries to allocate with the +normal system and transient resource scopes. If that fails, it checks if the +multiaddr matches an item in the set of allowlisted multiaddrs. If so, it +creates the connection resource scope using the allowlisted specific system and +transient resource scopes. If it wasn't an allowlisted multiaddr it fails as +before. When an allowlisted connection is tied to a peer id and transfered with `ConnManagementScope.SetPeer`, we check if that peer id matches the expected From 7aecdc148d16e3f53cc6c3e36ecdeceec0b6ebe5 Mon Sep 17 00:00:00 2001 From: Marco Munizaga Date: Wed, 29 Jun 2022 21:18:16 -0700 Subject: [PATCH 21/45] PR nits --- allowlist.go | 24 ++++++++++++++---------- allowlist_test.go | 4 ++-- rcmgr_test.go | 2 +- 3 files changed, 17 insertions(+), 13 deletions(-) diff --git a/allowlist.go b/allowlist.go index 9e919c5..6a4fb1c 100644 --- a/allowlist.go +++ b/allowlist.go @@ -104,12 +104,12 @@ func toIPNet(ma multiaddr.Multiaddr) (*net.IPNet, peer.ID, error) { // e.g. /ip4/1.2.3.4/p2p/QmFoo, /ip4/1.2.3.4, and /ip4/1.2.3.0/ipcidr/24 are valid. // /p2p/QmFoo is not valid. func (al *Allowlist) Add(ma multiaddr.Multiaddr) error { - al.mu.Lock() - defer al.mu.Unlock() ipnet, allowedPeer, err := toIPNet(ma) if err != nil { return err } + al.mu.Lock() + defer al.mu.Unlock() if allowedPeer != peer.ID("") { // We have a peerID constraint @@ -124,16 +124,16 @@ func (al *Allowlist) Add(ma multiaddr.Multiaddr) error { } func (al *Allowlist) Remove(ma multiaddr.Multiaddr) error { - al.mu.Lock() - defer al.mu.Unlock() - ipnet, allowedPeer, err := toIPNet(ma) if err != nil { return err } + al.mu.Lock() + defer al.mu.Unlock() + ipNetList := al.allowedNetworks - if allowedPeer != peer.ID("") { + if allowedPeer != "" { // We have a peerID constraint ipNetList = al.allowedPeerByNetwork[allowedPeer] } @@ -149,10 +149,14 @@ func (al *Allowlist) Remove(ma multiaddr.Multiaddr) error { if i == len(ipNetList)-1 { // Trim this element from the end ipNetList = ipNetList[:i] + // We only remove one thing + break } else { // swap remove ipNetList[i] = ipNetList[len(ipNetList)-1] ipNetList = ipNetList[:len(ipNetList)-1] + // We only remove one thing + break } } } @@ -167,12 +171,12 @@ func (al *Allowlist) Remove(ma multiaddr.Multiaddr) error { } func (al *Allowlist) Allowed(ma multiaddr.Multiaddr) bool { - al.mu.RLock() - defer al.mu.RUnlock() ip, err := manet.ToIP(ma) if err != nil { return false } + al.mu.RLock() + defer al.mu.RUnlock() for _, network := range al.allowedNetworks { if network.Contains(ip) { @@ -192,12 +196,12 @@ func (al *Allowlist) Allowed(ma multiaddr.Multiaddr) bool { } func (al *Allowlist) AllowedPeerAndMultiaddr(peerID peer.ID, ma multiaddr.Multiaddr) bool { - al.mu.RLock() - defer al.mu.RUnlock() ip, err := manet.ToIP(ma) if err != nil { return false } + al.mu.RLock() + defer al.mu.RUnlock() for _, network := range al.allowedNetworks { if network.Contains(ip) { diff --git a/allowlist_test.go b/allowlist_test.go index 713e5b8..7f5267e 100644 --- a/allowlist_test.go +++ b/allowlist_test.go @@ -38,8 +38,8 @@ func TestAllowedWithPeer(t *testing.T) { peerA := test.RandPeerIDFatal(t) peerB := test.RandPeerIDFatal(t) - multiaddrA, _ := multiaddr.NewMultiaddr("/ip4/1.2.3.4/tcp/1234") - multiaddrB, _ := multiaddr.NewMultiaddr("/ip4/2.2.3.4/tcp/1234") + multiaddrA := multiaddr.StringCast("/ip4/1.2.3.4/tcp/1234") + multiaddrB := multiaddr.StringCast("/ip4/2.2.3.4/tcp/1234") testcases := []testcase{ { diff --git a/rcmgr_test.go b/rcmgr_test.go index 1656a0d..fdf8dbf 100644 --- a/rcmgr_test.go +++ b/rcmgr_test.go @@ -10,7 +10,7 @@ import ( "github.com/multiformats/go-multiaddr" ) -var dummyMA, _ = multiaddr.NewMultiaddr("/ip4/1.2.3.4/tcp/1234") +var dummyMA = multiaddr.StringCast("/ip4/1.2.3.4/tcp/1234") func TestResourceManager(t *testing.T) { peerA := peer.ID("A") From 6edfbe4b5c297fd1895f35d9f97de4ac9c351912 Mon Sep 17 00:00:00 2001 From: Marco Munizaga Date: Thu, 30 Jun 2022 09:55:46 -0700 Subject: [PATCH 22/45] Remove branch --- allowlist.go | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/allowlist.go b/allowlist.go index 6a4fb1c..fd72af0 100644 --- a/allowlist.go +++ b/allowlist.go @@ -146,18 +146,11 @@ func (al *Allowlist) Remove(ma multiaddr.Multiaddr) error { for i > 0 { i-- if ipNetList[i].IP.Equal(ipnet.IP) && bytes.Equal(ipNetList[i].Mask, ipnet.Mask) { - if i == len(ipNetList)-1 { - // Trim this element from the end - ipNetList = ipNetList[:i] - // We only remove one thing - break - } else { - // swap remove - ipNetList[i] = ipNetList[len(ipNetList)-1] - ipNetList = ipNetList[:len(ipNetList)-1] - // We only remove one thing - break - } + // swap remove + ipNetList[i] = ipNetList[len(ipNetList)-1] + ipNetList = ipNetList[:len(ipNetList)-1] + // We only remove one thing + break } } From 8eac037a9ddd203431ba9ea32f2c41535985ce51 Mon Sep 17 00:00:00 2001 From: Marco Munizaga Date: Thu, 30 Jun 2022 15:34:57 -0700 Subject: [PATCH 23/45] Update deps --- go.mod | 22 +++++++++++++--------- go.sum | 54 +++++++++++++++++++++++++++++++++++++++++++----------- 2 files changed, 56 insertions(+), 20 deletions(-) diff --git a/go.mod b/go.mod index 39e437a..3c2ea2d 100644 --- a/go.mod +++ b/go.mod @@ -3,36 +3,40 @@ module github.com/libp2p/go-libp2p-resource-manager go 1.17 require ( - github.com/ipfs/go-log/v2 v2.5.0 - github.com/libp2p/go-libp2p-core v0.14.0 + github.com/ipfs/go-log/v2 v2.5.1 + github.com/libp2p/go-libp2p-core v0.19.0 + github.com/multiformats/go-multiaddr v0.6.0 github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 github.com/stretchr/testify v1.7.0 ) require ( - github.com/btcsuite/btcd v0.20.1-beta // indirect + github.com/btcsuite/btcd v0.22.1 // indirect + github.com/btcsuite/btcd/btcec/v2 v2.1.3 // indirect + github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1 // indirect github.com/davecgh/go-spew v1.1.1 // indirect + github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 // indirect github.com/gogo/protobuf v1.3.1 // indirect - github.com/ipfs/go-cid v0.0.7 // indirect + github.com/ipfs/go-cid v0.2.0 // indirect + github.com/klauspost/cpuid/v2 v2.0.4 // indirect github.com/libp2p/go-buffer-pool v0.0.2 // indirect github.com/libp2p/go-openssl v0.0.7 // indirect github.com/mattn/go-isatty v0.0.14 // indirect github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 // indirect - github.com/minio/sha256-simd v0.1.1 // indirect + github.com/minio/sha256-simd v1.0.0 // indirect github.com/mr-tron/base58 v1.2.0 // indirect github.com/multiformats/go-base32 v0.0.3 // indirect github.com/multiformats/go-base36 v0.1.0 // indirect - github.com/multiformats/go-multiaddr v0.4.1 // indirect github.com/multiformats/go-multibase v0.0.3 // indirect - github.com/multiformats/go-multihash v0.0.14 // indirect + github.com/multiformats/go-multicodec v0.4.1 // indirect + github.com/multiformats/go-multihash v0.0.15 // indirect github.com/multiformats/go-varint v0.0.6 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572 // indirect - github.com/spaolacci/murmur3 v1.1.0 // indirect go.uber.org/atomic v1.7.0 // indirect go.uber.org/multierr v1.6.0 // indirect go.uber.org/zap v1.19.1 // indirect - golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 // indirect + golang.org/x/crypto v0.0.0-20210506145944-38f3c27a63bf // indirect golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c // indirect gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect ) diff --git a/go.sum b/go.sum index b78292d..9ecb0a1 100644 --- a/go.sum +++ b/go.sum @@ -1,13 +1,22 @@ github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= -github.com/btcsuite/btcd v0.20.1-beta h1:Ik4hyJqN8Jfyv3S4AGBOmyouMsYE3EdYODkMbQjwPGw= github.com/btcsuite/btcd v0.20.1-beta/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ= +github.com/btcsuite/btcd v0.22.1 h1:CnwP9LM/M9xuRrGSCGeMVs9iv09uMqwsVX7EeIpgV2c= +github.com/btcsuite/btcd v0.22.1/go.mod h1:wqgTSL29+50LRkmOVknEdmt8ZojIzhuWvgu/iptuN7Y= +github.com/btcsuite/btcd/btcec/v2 v2.1.3 h1:xM/n3yIhHAhHy04z4i43C8p4ehixJZMsnrVJkgl+MTE= +github.com/btcsuite/btcd/btcec/v2 v2.1.3/go.mod h1:ctjw4H1kknNJmRN4iP1R7bTQ+v3GJkZBd6mui8ZsAZE= +github.com/btcsuite/btcd/chaincfg/chainhash v1.0.0/go.mod h1:7SFka0XMvUgj3hfZtydOrQY2mwhPclbT2snogU7SQQc= +github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1 h1:q0rUy8C/TYNBQS1+CGKw68tLOFYSNEs0TFnxxnS9+4U= +github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1/go.mod h1:7SFka0XMvUgj3hfZtydOrQY2mwhPclbT2snogU7SQQc= github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA= github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg= +github.com/btcsuite/btcutil v1.0.3-0.20201208143702-a53e38424cce/go.mod h1:0DVlHczLPewLcPGEIeUEzfOJhqGPQ0mJJRDBtD307+o= github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd/go.mod h1:HHNXQzUsZCxOoE+CPiyCTO6x34Zs86zZUiwtpXoGdtg= github.com/btcsuite/goleveldb v0.0.0-20160330041536-7834afc9e8cd/go.mod h1:F+uVaaLLH7j4eDXPRvw78tMflu7Ie2bzYOH4Y8rRKBY= +github.com/btcsuite/goleveldb v1.0.0/go.mod h1:QiK9vBlgftBg6rWQIj6wFzbPfRjiykIEhBH4obrXJ/I= github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc= +github.com/btcsuite/snappy-go v1.0.0/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc= github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY= github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= @@ -15,20 +24,29 @@ github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/decred/dcrd/crypto/blake256 v1.0.0 h1:/8DMNYp9SGi5f0w7uCm6d6M4OU2rGFK09Y2A4Xv7EE0= +github.com/decred/dcrd/crypto/blake256 v1.0.0/go.mod h1:sQl2p6Y26YV+ZOcSTP6thNdn47hh8kt6rqSlvmrXFAc= +github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 h1:YLtO71vCjJRCBcrPMtQ9nqBsqpA1m5sE92cU+pd5Mcc= +github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1/go.mod h1:hyedUtir6IdtD/7lIxGeCxkaw7y45JueMRL4DIyJDKs= +github.com/decred/dcrd/lru v1.0.0/go.mod h1:mxKOwFd7lFjN2GZYsiz/ecgqR6kkYAl+0pz0tEMk218= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/ipfs/go-cid v0.0.7 h1:ysQJVJA3fNDF1qigJbsSQOdjhVLsOEoPdh0+R97k3jY= github.com/ipfs/go-cid v0.0.7/go.mod h1:6Ux9z5e+HpkQdckYoX1PG/6xqKspzlEIR5SDmgqgC/I= -github.com/ipfs/go-log/v2 v2.5.0 h1:+MhAooFd9XZNvR0i9FriKW6HB0ql7HNXUuflWtc0dd4= -github.com/ipfs/go-log/v2 v2.5.0/go.mod h1:prSpmC1Gpllc9UYWxDiZDreBYw7zp4Iqp1kOLU9U5UI= +github.com/ipfs/go-cid v0.2.0 h1:01JTiihFq9en9Vz0lc0VDWvZe/uBonGpzo4THP0vcQ0= +github.com/ipfs/go-cid v0.2.0/go.mod h1:P+HXFDF4CVhaVayiEb4wkAy7zBHxBwsJyt0Y5U6MLro= +github.com/ipfs/go-log/v2 v2.5.1 h1:1XdUzF7048prq4aBjDQQ4SL5RxftpRGdXhNRwKSAlcY= +github.com/ipfs/go-log/v2 v2.5.1/go.mod h1:prSpmC1Gpllc9UYWxDiZDreBYw7zp4Iqp1kOLU9U5UI= github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= +github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= +github.com/klauspost/cpuid/v2 v2.0.4 h1:g0I61F2K2DjRHz1cnxlkNSBIaePVoJIjjnHui8QHbiw= +github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= @@ -37,8 +55,8 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/libp2p/go-buffer-pool v0.0.2 h1:QNK2iAFa8gjAe1SPz6mHSMuCcjs+X1wlHzeOSqcmlfs= github.com/libp2p/go-buffer-pool v0.0.2/go.mod h1:MvaB6xw5vOrDl8rYZGLFdKAuk/hRoRZd1Vi32+RXyFM= github.com/libp2p/go-flow-metrics v0.0.3/go.mod h1:HeoSNUrOJVK1jEpDqVEiUOIXqhbnS27omG0uWU5slZs= -github.com/libp2p/go-libp2p-core v0.14.0 h1:0kYSgiK/D7Eo28GTuRXo5YHsWwAisVpFCqCVPUd/vJs= -github.com/libp2p/go-libp2p-core v0.14.0/go.mod h1:tLasfcVdTXnixsLB0QYaT1syJOhsbrhG7q6pGrHtBg8= +github.com/libp2p/go-libp2p-core v0.19.0 h1:KDw7hanmh0EuVdZqsHCAzmkdiYMk5uR5h0UGSCVTxSU= +github.com/libp2p/go-libp2p-core v0.19.0/go.mod h1:AkA+FUKQfYt1FLNef5fOPlo/naAWjKy/RCjkcPjqzYg= github.com/libp2p/go-msgio v0.0.6/go.mod h1:4ecVB6d9f4BDSL5fqvPiC4A3KivjWn+Venn/1ALLMWA= github.com/libp2p/go-openssl v0.0.7 h1:eCAzdLejcNVBzP/iZM9vqHnQm+XyCEbSSIheIPRGNsw= github.com/libp2p/go-openssl v0.0.7/go.mod h1:unDrJpgy3oFr+rqXsarWifmJuNnJR4chtO1HmaZjggc= @@ -47,8 +65,8 @@ github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27k github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 h1:lYpkrQH5ajf0OXOcUbGjvZxxijuBwbbmlSxLiuofa+g= github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1/go.mod h1:pD8RvIylQ358TN4wwqatJ8rNavkEINozVn9DtGI3dfQ= github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= -github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU= -github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= +github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g= +github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM= github.com/mr-tron/base58 v1.1.0/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8= github.com/mr-tron/base58 v1.1.3/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o= @@ -59,16 +77,22 @@ github.com/multiformats/go-base36 v0.1.0 h1:JR6TyF7JjGd3m6FbLU2cOxhC0Li8z8dLNGQ8 github.com/multiformats/go-base36 v0.1.0/go.mod h1:kFGE83c6s80PklsHO9sRn2NCoffoRdUUOENyW/Vv6sM= github.com/multiformats/go-multiaddr v0.4.1 h1:Pq37uLx3hsyNlTDir7FZyU8+cFCTqd5y1KiM2IzOutI= github.com/multiformats/go-multiaddr v0.4.1/go.mod h1:3afI9HfVW8csiF8UZqtpYRiDyew8pRX7qLIGHu9FLuM= +github.com/multiformats/go-multiaddr v0.6.0 h1:qMnoOPj2s8xxPU5kZ57Cqdr0hHhARz7mFsPMIiYNqzg= +github.com/multiformats/go-multiaddr v0.6.0/go.mod h1:F4IpaKZuPP360tOMn2Tpyu0At8w23aRyVqeK0DbFeGM= github.com/multiformats/go-multibase v0.0.3 h1:l/B6bJDQjvQ5G52jw4QGSYeOTZoAwIO77RblWplfIqk= github.com/multiformats/go-multibase v0.0.3/go.mod h1:5+1R4eQrT3PkYZ24C3W2Ue2tPwIdYQD509ZjSb5y9Oc= +github.com/multiformats/go-multicodec v0.4.1 h1:BSJbf+zpghcZMZrwTYBGwy0CPcVZGWiC72Cp8bBd4R4= +github.com/multiformats/go-multicodec v0.4.1/go.mod h1:1Hj/eHRaVWSXiSNNfcEPcwZleTmdNP81xlxDLnWU9GQ= github.com/multiformats/go-multihash v0.0.13/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc= -github.com/multiformats/go-multihash v0.0.14 h1:QoBceQYQQtNUuf6s7wHxnE2c8bhbMqhfGzNI032se/I= github.com/multiformats/go-multihash v0.0.14/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc= +github.com/multiformats/go-multihash v0.0.15 h1:hWOPdrNqDjwHDx82vsYGSDZNyktOJJ2dzZJzFkOV1jM= +github.com/multiformats/go-multihash v0.0.15/go.mod h1:D6aZrWNLFTV/ynMpKsNtB40mJzmCl4jb1alC0OvHiHg= github.com/multiformats/go-varint v0.0.5/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= github.com/multiformats/go-varint v0.0.6 h1:gk85QWKxh3TazbLxED/NlDVv8+q+ReFJk7Y2W/KhfNY= github.com/multiformats/go-varint v0.0.6/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/gomega v1.4.1/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 h1:onHthvaw9LFnH4t2DcNVpwGmV9E1BkGknEliJkfwQj0= github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhMYhSNPKjeNKa5WY9YCIEBRbNzFFPJbWO6Y= @@ -78,7 +102,6 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572 h1:RC6RW7j+1+HkWaX/Yh71Ee5ZHaHYt7ZP4sQgUrm6cDU= github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572/go.mod h1:w0SWMsp6j9O/dk4/ZpIhL+3CkG8ofA2vuv7k+ltqUMc= -github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= @@ -96,14 +119,20 @@ go.uber.org/zap v1.19.1/go.mod h1:j3DNczoxDZroyBnOT1L/Q79cfUMGZxlv/9dzN7SM1rI= golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200115085410-6d4e4cb37c7d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/crypto v0.0.0-20210506145944-38f3c27a63bf h1:B2n+Zi5QeYRDAEodEu72OS36gmTWjgpXr2+cWcBW90o= +golang.org/x/crypto v0.0.0-20210506145944-38f3c27a63bf/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20180719180050-a680a1efc54d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -112,11 +141,14 @@ golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210309074719-68d13333faf2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c h1:F1jZWGFhYfh0Ci55sIpILtKKK8p3i2/krTr0H1rg74I= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= From 0487d3e13bec1b0641a24332c553d81121e5930a Mon Sep 17 00:00:00 2001 From: Marco Munizaga Date: Thu, 30 Jun 2022 16:53:48 -0700 Subject: [PATCH 24/45] Mod tidy --- go.sum | 1 - 1 file changed, 1 deletion(-) diff --git a/go.sum b/go.sum index 9ecb0a1..f4ec102 100644 --- a/go.sum +++ b/go.sum @@ -75,7 +75,6 @@ github.com/multiformats/go-base32 v0.0.3 h1:tw5+NhuwaOjJCC5Pp82QuXbrmLzWg7uxlMFp github.com/multiformats/go-base32 v0.0.3/go.mod h1:pLiuGC8y0QR3Ue4Zug5UzK9LjgbkL8NSQj0zQ5Nz/AA= github.com/multiformats/go-base36 v0.1.0 h1:JR6TyF7JjGd3m6FbLU2cOxhC0Li8z8dLNGQ89tUg4F4= github.com/multiformats/go-base36 v0.1.0/go.mod h1:kFGE83c6s80PklsHO9sRn2NCoffoRdUUOENyW/Vv6sM= -github.com/multiformats/go-multiaddr v0.4.1 h1:Pq37uLx3hsyNlTDir7FZyU8+cFCTqd5y1KiM2IzOutI= github.com/multiformats/go-multiaddr v0.4.1/go.mod h1:3afI9HfVW8csiF8UZqtpYRiDyew8pRX7qLIGHu9FLuM= github.com/multiformats/go-multiaddr v0.6.0 h1:qMnoOPj2s8xxPU5kZ57Cqdr0hHhARz7mFsPMIiYNqzg= github.com/multiformats/go-multiaddr v0.6.0/go.mod h1:F4IpaKZuPP360tOMn2Tpyu0At8w23aRyVqeK0DbFeGM= From 1c12460e6523caed85c9b5ea555ab1baf0923cf9 Mon Sep 17 00:00:00 2001 From: Marco Munizaga Date: Wed, 15 Jun 2022 14:12:25 -0700 Subject: [PATCH 25/45] Add multimetrics reporter --- metrics.go | 117 ++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 115 insertions(+), 2 deletions(-) diff --git a/metrics.go b/metrics.go index a875415..c71ed90 100644 --- a/metrics.go +++ b/metrics.go @@ -47,10 +47,20 @@ type metrics struct { reporter MetricsReporter } -// WithMetrics is a resource manager option to enable metrics collection +// WithMetrics is a resource manager option to enable metrics collection. Can be +// called multiple times to add multiple reporters. func WithMetrics(reporter MetricsReporter) Option { return func(r *resourceManager) error { - r.metrics = &metrics{reporter: reporter} + if r.metrics == nil { + r.metrics = &metrics{reporter: reporter} + } else if multimetrics, ok := r.metrics.reporter.(*MultiMetricsReporter); ok { + multimetrics.reporters = append(multimetrics.reporters, reporter) + } else { + // This was a single reporter. Lets convert it to a multimetrics reporter + r.metrics = &metrics{ + reporter: &MultiMetricsReporter{reporters: []MetricsReporter{r.metrics.reporter, reporter}}, + } + } return nil } } @@ -166,3 +176,106 @@ func (m *metrics) BlockMemory(size int) { m.reporter.BlockMemory(size) } + +// MultiMetricsReporter is a helper that allows you to report to multiple metrics reporters. +type MultiMetricsReporter struct { + reporters []MetricsReporter +} + +// AllowConn is invoked when opening a connection is allowed +func (mmr *MultiMetricsReporter) AllowConn(dir network.Direction, usefd bool) { + for _, r := range mmr.reporters { + r.AllowConn(dir, usefd) + } +} + +// BlockConn is invoked when opening a connection is blocked +func (mmr *MultiMetricsReporter) BlockConn(dir network.Direction, usefd bool) { + for _, r := range mmr.reporters { + r.BlockConn(dir, usefd) + } +} + +// AllowStream is invoked when opening a stream is allowed +func (mmr *MultiMetricsReporter) AllowStream(p peer.ID, dir network.Direction) { + for _, r := range mmr.reporters { + r.AllowStream(p, dir) + } +} + +// BlockStream is invoked when opening a stream is blocked +func (mmr *MultiMetricsReporter) BlockStream(p peer.ID, dir network.Direction) { + for _, r := range mmr.reporters { + r.BlockStream(p, dir) + } +} + +// AllowPeer is invoked when attaching ac onnection to a peer is allowed +func (mmr *MultiMetricsReporter) AllowPeer(p peer.ID) { + for _, r := range mmr.reporters { + r.AllowPeer(p) + } +} + +// BlockPeer is invoked when attaching ac onnection to a peer is blocked +func (mmr *MultiMetricsReporter) BlockPeer(p peer.ID) { + for _, r := range mmr.reporters { + r.BlockPeer(p) + } +} + +// AllowProtocol is invoked when setting the protocol for a stream is allowed +func (mmr *MultiMetricsReporter) AllowProtocol(proto protocol.ID) { + for _, r := range mmr.reporters { + r.AllowProtocol(proto) + } +} + +// BlockProtocol is invoked when setting the protocol for a stream is blocked +func (mmr *MultiMetricsReporter) BlockProtocol(proto protocol.ID) { + for _, r := range mmr.reporters { + r.BlockProtocol(proto) + } +} + +// BlockedProtocolPeer is invoekd when setting the protocol for a stream is blocked at the per protocol peer scope +func (mmr *MultiMetricsReporter) BlockProtocolPeer(proto protocol.ID, p peer.ID) { + for _, r := range mmr.reporters { + r.BlockProtocolPeer(proto, p) + } +} + +// AllowPService is invoked when setting the protocol for a stream is allowed +func (mmr *MultiMetricsReporter) AllowService(svc string) { + for _, r := range mmr.reporters { + r.AllowService(svc) + } +} + +// BlockPService is invoked when setting the protocol for a stream is blocked +func (mmr *MultiMetricsReporter) BlockService(svc string) { + for _, r := range mmr.reporters { + r.BlockService(svc) + } +} + +// BlockedServicePeer is invoked when setting the service for a stream is blocked at the per service peer scope +func (mmr *MultiMetricsReporter) BlockServicePeer(svc string, p peer.ID) { + for _, r := range mmr.reporters { + r.BlockServicePeer(svc, p) + } +} + +// AllowMemory is invoked when a memory reservation is allowed +func (mmr *MultiMetricsReporter) AllowMemory(size int) { + for _, r := range mmr.reporters { + r.AllowMemory(size) + } +} + +// BlockMemory is invoked when a memory reservation is blocked +func (mmr *MultiMetricsReporter) BlockMemory(size int) { + for _, r := range mmr.reporters { + r.BlockMemory(size) + } +} From 35abda3b138ba5297e18ca22ef8645c88b70182a Mon Sep 17 00:00:00 2001 From: Marco Munizaga Date: Fri, 17 Jun 2022 11:37:53 -0700 Subject: [PATCH 26/45] Add trace reporter to expose traces to users --- trace.go | 119 +++++++++++++++++++++++++++++++++++-------------------- 1 file changed, 75 insertions(+), 44 deletions(-) diff --git a/trace.go b/trace.go index 1018c2c..fb4103f 100644 --- a/trace.go +++ b/trace.go @@ -21,31 +21,54 @@ type trace struct { cancel func() closed chan struct{} - mx sync.Mutex - done bool - pend []interface{} + mx sync.Mutex + done bool + pendingWrites []interface{} + reporters []TraceReporter +} + +type TraceReporter interface { + // ConsumeEvent consumes a trace event. This is called synchronously, + // implementations should process the event quickly. + ConsumeEvent(TraceEvt) } func WithTrace(path string) Option { return func(r *resourceManager) error { - r.trace = &trace{path: path} + if r.trace == nil { + r.trace = &trace{path: path} + } else { + r.trace.path = path + } + return nil + } +} + +func WithTraceReporter(reporter TraceReporter) Option { + return func(r *resourceManager) error { + if r.trace == nil { + r.trace = &trace{reporters: []TraceReporter{reporter}} + } + r.trace.reporters = append(r.trace.reporters, reporter) return nil } } +type TraceEvtTyp string + const ( - traceStartEvt = "start" - traceCreateScopeEvt = "create_scope" - traceDestroyScopeEvt = "destroy_scope" - traceReserveMemoryEvt = "reserve_memory" - traceBlockReserveMemoryEvt = "block_reserve_memory" - traceReleaseMemoryEvt = "release_memory" - traceAddStreamEvt = "add_stream" - traceBlockAddStreamEvt = "block_add_stream" - traceRemoveStreamEvt = "remove_stream" - traceAddConnEvt = "add_conn" - traceBlockAddConnEvt = "block_add_conn" - traceRemoveConnEvt = "remove_conn" + TraceStartEvt TraceEvtTyp = "start" + TraceCreateScopeEvt TraceEvtTyp = "create_scope" + TraceDestroyScopeEvt TraceEvtTyp = "destroy_scope" + TraceReserveMemoryEvt TraceEvtTyp = "reserve_memory" + TraceBlockReserveMemoryEvt TraceEvtTyp = "block_reserve_memory" + TraceReleaseMemoryEvt TraceEvtTyp = "release_memory" + TraceAddStreamEvt TraceEvtTyp = "add_stream" + TraceBlockAddStreamEvt TraceEvtTyp = "block_add_stream" + TraceRemoveStreamEvt TraceEvtTyp = "remove_stream" + TraceAddConnEvt TraceEvtTyp = "add_conn" + TraceBlockAddConnEvt TraceEvtTyp = "block_add_conn" + TraceRemoveConnEvt TraceEvtTyp = "remove_conn" ) type scopeClass struct { @@ -163,7 +186,7 @@ func (s scopeClass) MarshalJSON() ([]byte, error) { type TraceEvt struct { Time string - Type string + Type TraceEvtTyp Scope *scopeClass `json:",omitempty"` Name string `json:",omitempty"` @@ -199,10 +222,16 @@ func (t *trace) push(evt TraceEvt) { evt.Scope = &scopeClass{name: evt.Name} } - t.pend = append(t.pend, evt) + for _, reporter := range t.reporters { + reporter.ConsumeEvent(evt) + } + + if t.path != "" { + t.pendingWrites = append(t.pendingWrites, evt) + } } -func (t *trace) background(out io.WriteCloser) { +func (t *trace) backgroundWriter(out io.WriteCloser) { defer close(t.closed) defer out.Close() @@ -218,8 +247,8 @@ func (t *trace) background(out io.WriteCloser) { getEvents := func() { t.mx.Lock() - tmp := t.pend - t.pend = pend[:0] + tmp := t.pendingWrites + t.pendingWrites = pend[:0] pend = tmp t.mx.Unlock() } @@ -288,15 +317,17 @@ func (t *trace) Start(limits Limiter) error { t.ctx, t.cancel = context.WithCancel(context.Background()) t.closed = make(chan struct{}) - out, err := os.OpenFile(t.path, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644) - if err != nil { - return nil - } + if t.path != "" { + out, err := os.OpenFile(t.path, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644) + if err != nil { + return nil + } - go t.background(out) + go t.backgroundWriter(out) + } t.push(TraceEvt{ - Type: traceStartEvt, + Type: TraceStartEvt, Limit: limits, }) @@ -329,7 +360,7 @@ func (t *trace) CreateScope(scope string, limit Limit) { } t.push(TraceEvt{ - Type: traceCreateScopeEvt, + Type: TraceCreateScopeEvt, Name: scope, Limit: limit, }) @@ -341,7 +372,7 @@ func (t *trace) DestroyScope(scope string) { } t.push(TraceEvt{ - Type: traceDestroyScopeEvt, + Type: TraceDestroyScopeEvt, Name: scope, }) } @@ -356,7 +387,7 @@ func (t *trace) ReserveMemory(scope string, prio uint8, size, mem int64) { } t.push(TraceEvt{ - Type: traceReserveMemoryEvt, + Type: TraceReserveMemoryEvt, Name: scope, Priority: prio, Delta: size, @@ -374,7 +405,7 @@ func (t *trace) BlockReserveMemory(scope string, prio uint8, size, mem int64) { } t.push(TraceEvt{ - Type: traceBlockReserveMemoryEvt, + Type: TraceBlockReserveMemoryEvt, Name: scope, Priority: prio, Delta: size, @@ -392,7 +423,7 @@ func (t *trace) ReleaseMemory(scope string, size, mem int64) { } t.push(TraceEvt{ - Type: traceReleaseMemoryEvt, + Type: TraceReleaseMemoryEvt, Name: scope, Delta: size, Memory: mem, @@ -412,7 +443,7 @@ func (t *trace) AddStream(scope string, dir network.Direction, nstreamsIn, nstre } t.push(TraceEvt{ - Type: traceAddStreamEvt, + Type: TraceAddStreamEvt, Name: scope, DeltaIn: deltaIn, DeltaOut: deltaOut, @@ -434,7 +465,7 @@ func (t *trace) BlockAddStream(scope string, dir network.Direction, nstreamsIn, } t.push(TraceEvt{ - Type: traceBlockAddStreamEvt, + Type: TraceBlockAddStreamEvt, Name: scope, DeltaIn: deltaIn, DeltaOut: deltaOut, @@ -456,7 +487,7 @@ func (t *trace) RemoveStream(scope string, dir network.Direction, nstreamsIn, ns } t.push(TraceEvt{ - Type: traceRemoveStreamEvt, + Type: TraceRemoveStreamEvt, Name: scope, DeltaIn: deltaIn, DeltaOut: deltaOut, @@ -475,7 +506,7 @@ func (t *trace) AddStreams(scope string, deltaIn, deltaOut, nstreamsIn, nstreams } t.push(TraceEvt{ - Type: traceAddStreamEvt, + Type: TraceAddStreamEvt, Name: scope, DeltaIn: deltaIn, DeltaOut: deltaOut, @@ -494,7 +525,7 @@ func (t *trace) BlockAddStreams(scope string, deltaIn, deltaOut, nstreamsIn, nst } t.push(TraceEvt{ - Type: traceBlockAddStreamEvt, + Type: TraceBlockAddStreamEvt, Name: scope, DeltaIn: deltaIn, DeltaOut: deltaOut, @@ -513,7 +544,7 @@ func (t *trace) RemoveStreams(scope string, deltaIn, deltaOut, nstreamsIn, nstre } t.push(TraceEvt{ - Type: traceRemoveStreamEvt, + Type: TraceRemoveStreamEvt, Name: scope, DeltaIn: -deltaIn, DeltaOut: -deltaOut, @@ -538,7 +569,7 @@ func (t *trace) AddConn(scope string, dir network.Direction, usefd bool, nconnsI } t.push(TraceEvt{ - Type: traceAddConnEvt, + Type: TraceAddConnEvt, Name: scope, DeltaIn: deltaIn, DeltaOut: deltaOut, @@ -565,7 +596,7 @@ func (t *trace) BlockAddConn(scope string, dir network.Direction, usefd bool, nc } t.push(TraceEvt{ - Type: traceBlockAddConnEvt, + Type: TraceBlockAddConnEvt, Name: scope, DeltaIn: deltaIn, DeltaOut: deltaOut, @@ -592,7 +623,7 @@ func (t *trace) RemoveConn(scope string, dir network.Direction, usefd bool, ncon } t.push(TraceEvt{ - Type: traceRemoveConnEvt, + Type: TraceRemoveConnEvt, Name: scope, DeltaIn: deltaIn, DeltaOut: deltaOut, @@ -613,7 +644,7 @@ func (t *trace) AddConns(scope string, deltaIn, deltaOut, deltafd, nconnsIn, nco } t.push(TraceEvt{ - Type: traceAddConnEvt, + Type: TraceAddConnEvt, Name: scope, DeltaIn: deltaIn, DeltaOut: deltaOut, @@ -634,7 +665,7 @@ func (t *trace) BlockAddConns(scope string, deltaIn, deltaOut, deltafd, nconnsIn } t.push(TraceEvt{ - Type: traceBlockAddConnEvt, + Type: TraceBlockAddConnEvt, Name: scope, DeltaIn: deltaIn, DeltaOut: deltaOut, @@ -655,7 +686,7 @@ func (t *trace) RemoveConns(scope string, deltaIn, deltaOut, deltafd, nconnsIn, } t.push(TraceEvt{ - Type: traceRemoveConnEvt, + Type: TraceRemoveConnEvt, Name: scope, DeltaIn: -deltaIn, DeltaOut: -deltaOut, From 9ce3760d70828ca4b0eb9f12803f89b37472582a Mon Sep 17 00:00:00 2001 From: Marco Munizaga Date: Mon, 20 Jun 2022 16:52:34 -0700 Subject: [PATCH 27/45] Flip sign of delta --- trace.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trace.go b/trace.go index fb4103f..464e8cf 100644 --- a/trace.go +++ b/trace.go @@ -425,7 +425,7 @@ func (t *trace) ReleaseMemory(scope string, size, mem int64) { t.push(TraceEvt{ Type: TraceReleaseMemoryEvt, Name: scope, - Delta: size, + Delta: -size, Memory: mem, }) } From 7efd81c617a4d3f46162ad871bf63822d5551148 Mon Sep 17 00:00:00 2001 From: Marco Munizaga Date: Mon, 20 Jun 2022 16:52:56 -0700 Subject: [PATCH 28/45] Add scope name helpers --- rcmgr.go | 89 +++++++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 85 insertions(+), 4 deletions(-) diff --git a/rcmgr.go b/rcmgr.go index 65f330e..5d64f7d 100644 --- a/rcmgr.go +++ b/rcmgr.go @@ -3,6 +3,7 @@ package rcmgr import ( "context" "fmt" + "strings" "sync" "time" @@ -434,7 +435,7 @@ func newPeerScope(p peer.ID, limit Limit, rcmgr *resourceManager) *peerScope { return &peerScope{ resourceScope: newResourceScope(limit, []*resourceScope{rcmgr.system.resourceScope}, - fmt.Sprintf("peer:%s", p), rcmgr.trace, rcmgr.metrics), + peerScopeName(p), rcmgr.trace, rcmgr.metrics), peer: p, rcmgr: rcmgr, } @@ -444,7 +445,7 @@ func newConnectionScope(dir network.Direction, usefd bool, limit Limit, rcmgr *r return &connectionScope{ resourceScope: newResourceScope(limit, []*resourceScope{rcmgr.transient.resourceScope, rcmgr.system.resourceScope}, - fmt.Sprintf("conn-%d", rcmgr.nextConnId()), rcmgr.trace, rcmgr.metrics), + connScopeName(rcmgr.nextConnId()), rcmgr.trace, rcmgr.metrics), dir: dir, usefd: usefd, rcmgr: rcmgr, @@ -456,7 +457,7 @@ func newAllowListedConnectionScope(dir network.Direction, usefd bool, limit Limi return &connectionScope{ resourceScope: newResourceScope(limit, []*resourceScope{rcmgr.allowlistedTransient.resourceScope, rcmgr.allowlistedSystem.resourceScope}, - fmt.Sprintf("conn-%d", rcmgr.nextConnId()), rcmgr.trace, rcmgr.metrics), + connScopeName(rcmgr.nextConnId()), rcmgr.trace, rcmgr.metrics), dir: dir, usefd: usefd, rcmgr: rcmgr, @@ -469,13 +470,93 @@ func newStreamScope(dir network.Direction, limit Limit, peer *peerScope, rcmgr * return &streamScope{ resourceScope: newResourceScope(limit, []*resourceScope{peer.resourceScope, rcmgr.transient.resourceScope, rcmgr.system.resourceScope}, - fmt.Sprintf("stream-%d", rcmgr.nextStreamId()), rcmgr.trace, rcmgr.metrics), + streamScopeName(rcmgr.nextStreamId()), rcmgr.trace, rcmgr.metrics), dir: dir, rcmgr: peer.rcmgr, peer: peer, } } +func IsSystemScope(name string) bool { + return strings.HasPrefix(name, "system") +} + +func IsTransientScope(name string) bool { + return strings.HasPrefix(name, "transient") +} + +func streamScopeName(streamId int64) string { + return fmt.Sprintf("stream-%d", streamId) +} + +func IsStreamScope(name string) bool { + return strings.HasPrefix(name, "stream-") +} + +func connScopeName(streamId int64) string { + return fmt.Sprintf("conn-%d", streamId) +} + +func IsConnScope(name string) bool { + return strings.HasPrefix(name, "conn-") +} + +func peerScopeName(p peer.ID) string { + return fmt.Sprintf("peer:%s", p) +} + +// ParsePeerScopeName returns "" if name is not a peerScopeName +func ParsePeerScopeName(name string) peer.ID { + if !strings.HasPrefix(name, "peer:") { + return peer.ID("") + } + parts := strings.SplitN(name, "peer:", 2) + if len(parts) != 2 { + return peer.ID("") + } + p, err := peer.Decode(parts[1]) + if err != nil { + return peer.ID("") + } + return p +} + +// ParseServiceScopeName returns the service name if name is a serviceScopeName. +// Otherwise returns "" +func ParseServiceScopeName(name string) string { + if strings.HasPrefix(name, "service:") { + if strings.Contains(name, "peer:") { + // This is a service peer scope + return "" + } + parts := strings.SplitN(name, ":", 2) + if len(parts) != 2 { + return ("") + } + + return parts[1] + } + return "" +} + +// ParseProtocolScopeName returns the service name if name is a serviceScopeName. +// Otherwise returns "" +func ParseProtocolScopeName(name string) string { + if strings.HasPrefix(name, "protocol:") { + if strings.Contains(name, "peer:") { + // This is a protocol peer scope + return "" + } + parts := strings.SplitN(name, ":", 2) + if len(parts) != 2 { + return ("") + } + + return parts[1] + } + return "" +} + func (s *serviceScope) Name() string { return s.service } From 19b6d584b2b65773fbd6f56bf7293f228b2736c1 Mon Sep 17 00:00:00 2001 From: Marco Munizaga Date: Mon, 20 Jun 2022 16:56:01 -0700 Subject: [PATCH 29/45] Add obs to resource manager --- obs/stats.go | 341 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 341 insertions(+) create mode 100644 obs/stats.go diff --git a/obs/stats.go b/obs/stats.go new file mode 100644 index 0000000..cafc885 --- /dev/null +++ b/obs/stats.go @@ -0,0 +1,341 @@ +package obs + +import ( + "context" + "strings" + + logging "github.com/ipfs/go-log/v2" + rcmgr "github.com/libp2p/go-libp2p-resource-manager" + "go.opencensus.io/stats" + "go.opencensus.io/stats/view" + "go.opencensus.io/tag" +) + +var log = logging.Logger("rcmgrObs") + +var ( + systemOutboundConns = stats.Int64("system/outbound/conn", "Number of outbound Connections", stats.UnitDimensionless) + systemInboundConns = stats.Int64("system/inbound/conn", "Number of inbound Connections", stats.UnitDimensionless) + + conns = stats.Int64("connections", "Number of Connections", stats.UnitDimensionless) + + peerConns = stats.Int64("peer/connections", "Number of connections this peer has", stats.UnitDimensionless) + peerConnsNegative = stats.Int64("peer/connections_negative", "Number of connections this peer had", stats.UnitDimensionless) + + streams = stats.Int64("streams", "Number of Streams", stats.UnitDimensionless) + + peerStreams = stats.Int64("peer/streams", "Number of streams this peer has", stats.UnitDimensionless) + peerStreamsNegative = stats.Int64("peer/streams_negative", "Number of streams this peer had", stats.UnitDimensionless) + + memory = stats.Int64("memory", "Amount of memory reserved as reported to the Resource Manager", stats.UnitDimensionless) + peerMemory = stats.Int64("peer/memory", "Amount of memory currently reseved for peer", stats.UnitDimensionless) + peerMemoryNegative = stats.Int64("peer/memory_negative", "Amount of memory previously reseved for peer", stats.UnitDimensionless) + + connMemory = stats.Int64("conn/memory", "Amount of memory currently reseved for the connection", stats.UnitDimensionless) + connMemoryNegative = stats.Int64("conn/memory_negative", "Amount of memory previously reseved for the connection", stats.UnitDimensionless) + + fds = stats.Int64("fds", "Number of fds as reported to the Resource Manager", stats.UnitDimensionless) + + blockedResources = stats.Int64("blocked_resources", "Number of resource requests blocked", stats.UnitDimensionless) +) + +var ( + LessThanEq, _ = tag.NewKey("le") + Direction, _ = tag.NewKey("dir") + Scope, _ = tag.NewKey("scope") + Service, _ = tag.NewKey("service") + Protocol, _ = tag.NewKey("protocol") + Resource, _ = tag.NewKey("resource") +) + +var ( + SystemOutboundConnsView = &view.View{Measure: systemOutboundConns, Aggregation: view.Sum()} + SystemInboundConnsView = &view.View{Measure: systemInboundConns, Aggregation: view.Sum()} + + ConnView = &view.View{Measure: conns, Aggregation: view.Sum(), TagKeys: []tag.Key{Direction, Scope}} + + fibLikeDistribution = []float64{ + 1.1, 2.1, 3.1, 5.1, 8.1, 13.1, 21.1, 34.1, 55.1, 100.1, 200.1, + } + + PeerConnsView = &view.View{ + Measure: peerConns, + Aggregation: view.Distribution(fibLikeDistribution...), + TagKeys: []tag.Key{Direction}, + } + PeerConnsNegativeView = &view.View{ + Measure: peerConnsNegative, + Aggregation: view.Distribution(fibLikeDistribution...), + TagKeys: []tag.Key{Direction}, + } + + StreamView = &view.View{Measure: streams, Aggregation: view.Sum(), TagKeys: []tag.Key{Direction, Scope, Service, Protocol}} + PeerStreamsView = &view.View{Measure: peerStreams, Aggregation: view.Distribution(fibLikeDistribution...), TagKeys: []tag.Key{Direction}} + PeerStreamNegativeView = &view.View{Measure: peerStreamsNegative, Aggregation: view.Distribution(fibLikeDistribution...), TagKeys: []tag.Key{Direction}} + + MemoryView = &view.View{Measure: memory, Aggregation: view.Sum(), TagKeys: []tag.Key{Scope, Service, Protocol}} + + memDistribution = []float64{ + 1 << 10, // 1KB + 1 << 12, // 4KB + 1 << 15, // 32KB + 1 << 20, // 1MB + 1 << 25, // 32MB + 1 << 28, // 256MB + 1 << 29, // 512MB + 1 << 30, // 1GB + 1 << 31, // 2GB + 1 << 32, // 4GB + } + PeerMemoryView = &view.View{ + Measure: peerMemory, + Aggregation: view.Distribution(memDistribution...), + } + PeerMemoryNegativeView = &view.View{ + Measure: peerMemoryNegative, + Aggregation: view.Distribution(memDistribution...), + } + + // Not setup yet. Memory isn't attached to a given connection. + ConnMemoryView = &view.View{ + Measure: connMemory, + Aggregation: view.Distribution(memDistribution...), + } + ConnMemoryNegativeView = &view.View{ + Measure: connMemoryNegative, + Aggregation: view.Distribution(memDistribution...), + } + + FDsView = &view.View{Measure: fds, Aggregation: view.Sum(), TagKeys: []tag.Key{Scope}} + + BlockedResourcesView = &view.View{ + Measure: blockedResources, + Aggregation: view.Sum(), + TagKeys: []tag.Key{Scope, Resource}, + } +) + +var DefaultViews []*view.View = []*view.View{ + ConnView, + PeerConnsView, + PeerConnsNegativeView, + FDsView, + + StreamView, + PeerStreamsView, + PeerStreamNegativeView, + + MemoryView, + PeerMemoryView, + PeerMemoryNegativeView, + + BlockedResourcesView, +} + +// StatsTraceReporter reports stats on the resource manager using its traces. +type StatsTraceReporter struct{} + +func NewStatsTraceReporter() (StatsTraceReporter, error) { + return StatsTraceReporter{}, nil +} + +func (r StatsTraceReporter) ConsumeEvent(evt rcmgr.TraceEvt) { + ctx := context.Background() + + switch evt.Type { + case rcmgr.TraceAddStreamEvt, rcmgr.TraceRemoveStreamEvt: + if p := rcmgr.ParsePeerScopeName(evt.Scope); p.Validate() == nil { + // Aggregated peer stats. Counts how many peers have N number of streams open. + // Uses two buckets aggregations. One to count how many streams the + // peer has now. The other to count the negative value, or how many + // streams did the peer use to have. When looking at the data you + // take the difference from the two. + + oldStreamsOut := int64(evt.StreamsOut - evt.DeltaOut) + peerStreamsOut := int64(evt.StreamsOut) + if oldStreamsOut != peerStreamsOut { + if oldStreamsOut != 0 { + stats.RecordWithTags(ctx, []tag.Mutator{tag.Upsert(Direction, "outbound")}, peerStreamsNegative.M(oldStreamsOut)) + } + if peerStreamsOut != 0 { + stats.RecordWithTags(ctx, []tag.Mutator{tag.Upsert(Direction, "outbound")}, peerStreams.M(peerStreamsOut)) + } + } + + oldStreamsIn := int64(evt.StreamsIn - evt.DeltaIn) + peerStreamsIn := int64(evt.StreamsIn) + if oldStreamsIn != peerStreamsIn { + if oldStreamsIn != 0 { + stats.RecordWithTags(ctx, []tag.Mutator{tag.Upsert(Direction, "inbound")}, peerStreamsNegative.M(oldStreamsIn)) + } + if peerStreamsIn != 0 { + stats.RecordWithTags(ctx, []tag.Mutator{tag.Upsert(Direction, "inbound")}, peerStreams.M(peerStreamsIn)) + } + } + } else { + var tags []tag.Mutator + if rcmgr.IsSystemScope(evt.Scope) || rcmgr.IsTransientScope(evt.Scope) { + tags = append(tags, tag.Upsert(Scope, evt.Scope)) + } else if svc := rcmgr.ParseServiceScopeName(evt.Scope); svc != "" { + tags = append(tags, tag.Upsert(Scope, "service"), tag.Upsert(Service, svc)) + } else if proto := rcmgr.ParseProtocolScopeName(evt.Scope); proto != "" { + tags = append(tags, tag.Upsert(Scope, "protocol"), tag.Upsert(Protocol, proto)) + } else { + // Not measuring connscope, servicepeer and protocolpeer. Lots of data, and + // you can use aggregated peer stats + service stats to infer + // this. + break + } + + if evt.DeltaOut != 0 { + stats.RecordWithTags( + ctx, + append([]tag.Mutator{tag.Upsert(Direction, "outbound")}, tags...), + streams.M(int64(evt.DeltaOut)), + ) + } + + if evt.DeltaIn != 0 { + stats.RecordWithTags( + ctx, + append([]tag.Mutator{tag.Upsert(Direction, "inbound")}, tags...), + streams.M(int64(evt.DeltaIn)), + ) + } + } + + case rcmgr.TraceAddConnEvt, rcmgr.TraceRemoveConnEvt: + if p := rcmgr.ParsePeerScopeName(evt.Scope); p.Validate() == nil { + // Aggregated peer stats. Counts how many peers have N number of connections. + // Uses two buckets aggregations. One to count how many streams the + // peer has now. The other to count the negative value, or how many + // conns did the peer use to have. When looking at the data you + // take the difference from the two. + + oldConnsOut := int64(evt.ConnsOut - evt.DeltaOut) + connsOut := int64(evt.ConnsOut) + if oldConnsOut != connsOut { + if oldConnsOut != 0 { + stats.RecordWithTags(ctx, []tag.Mutator{tag.Upsert(Direction, "outbound")}, peerConnsNegative.M(oldConnsOut)) + } + if connsOut != 0 { + stats.RecordWithTags(ctx, []tag.Mutator{tag.Upsert(Direction, "outbound")}, peerConns.M(connsOut)) + } + } + + oldConnsIn := int64(evt.ConnsIn - evt.DeltaIn) + connsIn := int64(evt.ConnsIn) + if oldConnsIn != connsIn { + if oldConnsIn != 0 { + stats.RecordWithTags(ctx, []tag.Mutator{tag.Upsert(Direction, "inbound")}, peerConnsNegative.M(oldConnsIn)) + } + if connsIn != 0 { + stats.RecordWithTags(ctx, []tag.Mutator{tag.Upsert(Direction, "inbound")}, peerConns.M(connsIn)) + } + } + } else { + var tags []tag.Mutator + if rcmgr.IsSystemScope(evt.Scope) || rcmgr.IsTransientScope(evt.Scope) { + tags = append(tags, tag.Upsert(Scope, evt.Scope)) + } else if rcmgr.IsConnScope(evt.Scope) { + // Not measuring this. I don't think it's useful. + break + } else { + // There shouldn't be anything here. But we keep going so the metrics will tell us if we're wrong (scope="") + log.Debugf("unexpected event in stats: %s", evt.Scope) + } + + if evt.DeltaOut != 0 { + stats.RecordWithTags( + ctx, + append([]tag.Mutator{tag.Upsert(Direction, "outbound")}, tags...), + conns.M(int64(evt.DeltaOut)), + ) + } + + if evt.DeltaIn != 0 { + stats.RecordWithTags( + ctx, + append([]tag.Mutator{tag.Upsert(Direction, "inbound")}, tags...), + conns.M(int64(evt.DeltaIn)), + ) + } + + // Represents the delta in fds + if evt.Delta != 0 { + stats.RecordWithTags( + ctx, + tags, + fds.M(int64(evt.Delta)), + ) + } + } + case rcmgr.TraceReserveMemoryEvt, rcmgr.TraceReleaseMemoryEvt: + if p := rcmgr.ParsePeerScopeName(evt.Scope); p.Validate() == nil { + oldMem := evt.Memory - evt.Delta + if oldMem != evt.Memory { + if oldMem != 0 { + stats.Record(ctx, peerMemoryNegative.M(oldMem)) + } + if evt.Memory != 0 { + stats.Record(ctx, peerMemory.M(evt.Memory)) + } + } + } else if rcmgr.IsConnScope(evt.Scope) { + oldMem := evt.Memory - evt.Delta + if oldMem != evt.Memory { + if oldMem != 0 { + stats.Record(ctx, connMemoryNegative.M(oldMem)) + } + if evt.Memory != 0 { + stats.Record(ctx, connMemory.M(evt.Memory)) + } + } + } else { + var tags []tag.Mutator + if rcmgr.IsSystemScope(evt.Scope) || rcmgr.IsTransientScope(evt.Scope) { + tags = append(tags, tag.Upsert(Scope, evt.Scope)) + } else if svc := rcmgr.ParseServiceScopeName(evt.Scope); svc != "" { + tags = append(tags, tag.Upsert(Scope, "service"), tag.Upsert(Service, svc)) + } else if proto := rcmgr.ParseProtocolScopeName(evt.Scope); proto != "" { + tags = append(tags, tag.Upsert(Scope, "protocol"), tag.Upsert(Protocol, proto)) + } else { + // Not measuring connscope, servicepeer and protocolpeer. Lots of data, and + // you can use aggregated peer stats + service stats to infer + // this. + break + } + + if evt.Delta != 0 { + stats.RecordWithTags(ctx, tags, memory.M(int64(evt.Delta))) + } + } + + case rcmgr.TraceBlockAddConnEvt, rcmgr.TraceBlockAddStreamEvt, rcmgr.TraceBlockReserveMemoryEvt: + var resource string + if evt.Type == rcmgr.TraceBlockAddConnEvt { + resource = "connection" + } else if evt.Type == rcmgr.TraceBlockAddStreamEvt { + resource = "stream" + } else { + resource = "memory" + } + + // Only the top scope. We don't want to get the peerid here. + scope := strings.SplitN(evt.Scope, ":", 2)[0] + + tags := []tag.Mutator{tag.Upsert(Scope, scope), tag.Upsert(Resource, resource)} + + if evt.DeltaIn != 0 { + stats.RecordWithTags(ctx, tags, blockedResources.M(int64(evt.DeltaIn))) + } + + if evt.DeltaOut != 0 { + stats.RecordWithTags(ctx, tags, blockedResources.M(int64(evt.DeltaOut))) + } + + if evt.Delta != 0 { + stats.RecordWithTags(ctx, tags, blockedResources.M(evt.Delta)) + } + } +} From 3a06e88227f22afc605050a2a4f3738cb278141e Mon Sep 17 00:00:00 2001 From: Marco Munizaga Date: Mon, 20 Jun 2022 18:02:14 -0700 Subject: [PATCH 30/45] Add dep to go.mod/sum --- go.mod | 1 + go.sum | 72 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 73 insertions(+) diff --git a/go.mod b/go.mod index 3c2ea2d..3071287 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( github.com/multiformats/go-multiaddr v0.6.0 github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 github.com/stretchr/testify v1.7.0 + go.opencensus.io v0.23.0 ) require ( diff --git a/go.sum b/go.sum index f4ec102..ccf6dda 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,5 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= @@ -19,6 +21,9 @@ github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku github.com/btcsuite/snappy-go v1.0.0/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc= github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY= github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -29,10 +34,34 @@ github.com/decred/dcrd/crypto/blake256 v1.0.0/go.mod h1:sQl2p6Y26YV+ZOcSTP6thNdn github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 h1:YLtO71vCjJRCBcrPMtQ9nqBsqpA1m5sE92cU+pd5Mcc= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1/go.mod h1:hyedUtir6IdtD/7lIxGeCxkaw7y45JueMRL4DIyJDKs= github.com/decred/dcrd/lru v1.0.0/go.mod h1:mxKOwFd7lFjN2GZYsiz/ecgqR6kkYAl+0pz0tEMk218= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.3 h1:x95R7cp+rSeeqAMI2knLtQ0DKlaBhv2NrtrOvafPHRo= +github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/ipfs/go-cid v0.0.7/go.mod h1:6Ux9z5e+HpkQdckYoX1PG/6xqKspzlEIR5SDmgqgC/I= github.com/ipfs/go-cid v0.2.0 h1:01JTiihFq9en9Vz0lc0VDWvZe/uBonGpzo4THP0vcQ0= @@ -99,14 +128,18 @@ github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572 h1:RC6RW7j+1+HkWaX/Yh71Ee5ZHaHYt7ZP4sQgUrm6cDU= github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572/go.mod h1:w0SWMsp6j9O/dk4/ZpIhL+3CkG8ofA2vuv7k+ltqUMc= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M= +go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/goleak v1.1.11-0.20210813005559-691160354723 h1:sHOAIxRGBp443oHZIPB+HsUGaksVCXVQENPxwTfQdH4= @@ -121,26 +154,39 @@ golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200115085410-6d4e4cb37c7d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210506145944-38f3c27a63bf h1:B2n+Zi5QeYRDAEodEu72OS36gmTWjgpXr2+cWcBW90o= golang.org/x/crypto v0.0.0-20210506145944-38f3c27a63bf/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180719180050-a680a1efc54d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210309074719-68d13333faf2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -153,12 +199,36 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -170,3 +240,5 @@ gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= From 1c172ace77b7e5c13e7a3d25ce1200d8bdbab733 Mon Sep 17 00:00:00 2001 From: Marco Munizaga Date: Mon, 20 Jun 2022 18:05:29 -0700 Subject: [PATCH 31/45] Revert "Add multimetrics reporter" This reverts commit 81fb49d3d68153b610b245d6f730f4d6b34b3706. --- metrics.go | 117 +---------------------------------------------------- 1 file changed, 2 insertions(+), 115 deletions(-) diff --git a/metrics.go b/metrics.go index c71ed90..a875415 100644 --- a/metrics.go +++ b/metrics.go @@ -47,20 +47,10 @@ type metrics struct { reporter MetricsReporter } -// WithMetrics is a resource manager option to enable metrics collection. Can be -// called multiple times to add multiple reporters. +// WithMetrics is a resource manager option to enable metrics collection func WithMetrics(reporter MetricsReporter) Option { return func(r *resourceManager) error { - if r.metrics == nil { - r.metrics = &metrics{reporter: reporter} - } else if multimetrics, ok := r.metrics.reporter.(*MultiMetricsReporter); ok { - multimetrics.reporters = append(multimetrics.reporters, reporter) - } else { - // This was a single reporter. Lets convert it to a multimetrics reporter - r.metrics = &metrics{ - reporter: &MultiMetricsReporter{reporters: []MetricsReporter{r.metrics.reporter, reporter}}, - } - } + r.metrics = &metrics{reporter: reporter} return nil } } @@ -176,106 +166,3 @@ func (m *metrics) BlockMemory(size int) { m.reporter.BlockMemory(size) } - -// MultiMetricsReporter is a helper that allows you to report to multiple metrics reporters. -type MultiMetricsReporter struct { - reporters []MetricsReporter -} - -// AllowConn is invoked when opening a connection is allowed -func (mmr *MultiMetricsReporter) AllowConn(dir network.Direction, usefd bool) { - for _, r := range mmr.reporters { - r.AllowConn(dir, usefd) - } -} - -// BlockConn is invoked when opening a connection is blocked -func (mmr *MultiMetricsReporter) BlockConn(dir network.Direction, usefd bool) { - for _, r := range mmr.reporters { - r.BlockConn(dir, usefd) - } -} - -// AllowStream is invoked when opening a stream is allowed -func (mmr *MultiMetricsReporter) AllowStream(p peer.ID, dir network.Direction) { - for _, r := range mmr.reporters { - r.AllowStream(p, dir) - } -} - -// BlockStream is invoked when opening a stream is blocked -func (mmr *MultiMetricsReporter) BlockStream(p peer.ID, dir network.Direction) { - for _, r := range mmr.reporters { - r.BlockStream(p, dir) - } -} - -// AllowPeer is invoked when attaching ac onnection to a peer is allowed -func (mmr *MultiMetricsReporter) AllowPeer(p peer.ID) { - for _, r := range mmr.reporters { - r.AllowPeer(p) - } -} - -// BlockPeer is invoked when attaching ac onnection to a peer is blocked -func (mmr *MultiMetricsReporter) BlockPeer(p peer.ID) { - for _, r := range mmr.reporters { - r.BlockPeer(p) - } -} - -// AllowProtocol is invoked when setting the protocol for a stream is allowed -func (mmr *MultiMetricsReporter) AllowProtocol(proto protocol.ID) { - for _, r := range mmr.reporters { - r.AllowProtocol(proto) - } -} - -// BlockProtocol is invoked when setting the protocol for a stream is blocked -func (mmr *MultiMetricsReporter) BlockProtocol(proto protocol.ID) { - for _, r := range mmr.reporters { - r.BlockProtocol(proto) - } -} - -// BlockedProtocolPeer is invoekd when setting the protocol for a stream is blocked at the per protocol peer scope -func (mmr *MultiMetricsReporter) BlockProtocolPeer(proto protocol.ID, p peer.ID) { - for _, r := range mmr.reporters { - r.BlockProtocolPeer(proto, p) - } -} - -// AllowPService is invoked when setting the protocol for a stream is allowed -func (mmr *MultiMetricsReporter) AllowService(svc string) { - for _, r := range mmr.reporters { - r.AllowService(svc) - } -} - -// BlockPService is invoked when setting the protocol for a stream is blocked -func (mmr *MultiMetricsReporter) BlockService(svc string) { - for _, r := range mmr.reporters { - r.BlockService(svc) - } -} - -// BlockedServicePeer is invoked when setting the service for a stream is blocked at the per service peer scope -func (mmr *MultiMetricsReporter) BlockServicePeer(svc string, p peer.ID) { - for _, r := range mmr.reporters { - r.BlockServicePeer(svc, p) - } -} - -// AllowMemory is invoked when a memory reservation is allowed -func (mmr *MultiMetricsReporter) AllowMemory(size int) { - for _, r := range mmr.reporters { - r.AllowMemory(size) - } -} - -// BlockMemory is invoked when a memory reservation is blocked -func (mmr *MultiMetricsReporter) BlockMemory(size int) { - for _, r := range mmr.reporters { - r.BlockMemory(size) - } -} From 2385a33f55c8e8352272abbe55700c43e9011473 Mon Sep 17 00:00:00 2001 From: Marco Munizaga Date: Mon, 20 Jun 2022 18:09:58 -0700 Subject: [PATCH 32/45] Remove connid from scopename in blocked conns --- obs/stats.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/obs/stats.go b/obs/stats.go index cafc885..d45284b 100644 --- a/obs/stats.go +++ b/obs/stats.go @@ -323,6 +323,8 @@ func (r StatsTraceReporter) ConsumeEvent(evt rcmgr.TraceEvt) { // Only the top scope. We don't want to get the peerid here. scope := strings.SplitN(evt.Scope, ":", 2)[0] + // Drop the connection or stream id + scope = strings.SplitN(scope, "-", 2)[0] tags := []tag.Mutator{tag.Upsert(Scope, scope), tag.Upsert(Resource, resource)} From c9413d73201da127dbe9e78df4f539f9cf0564f5 Mon Sep 17 00:00:00 2001 From: Marco Munizaga Date: Mon, 20 Jun 2022 18:20:29 -0700 Subject: [PATCH 33/45] Add Grafana dashboard --- obs/grafana-dashboards/README.md | 42 + obs/grafana-dashboards/resource-manager.json | 1702 ++++++++++++++++++ 2 files changed, 1744 insertions(+) create mode 100644 obs/grafana-dashboards/README.md create mode 100644 obs/grafana-dashboards/resource-manager.json diff --git a/obs/grafana-dashboards/README.md b/obs/grafana-dashboards/README.md new file mode 100644 index 0000000..2dad848 --- /dev/null +++ b/obs/grafana-dashboards/README.md @@ -0,0 +1,42 @@ +# Ready to go Grafana Dashboard + +Here are some prebuilt dashboards that you can add to your Grafana instance. To +import follow the Grafana docs [here](https://grafana.com/docs/grafana/latest/dashboards/export-import/#import-dashboard) + +## Setup + +To make sure you're emitting the correct metrics you'll have to hook up the +Opencensus views that `stats.go` exports. For Prometheus this looks like: + +``` go +import ( + // ... + ocprom "contrib.go.opencensus.io/exporter/prometheus" + + rcmgr "github.com/libp2p/go-libp2p-resource-manager" + rcmgrObs "github.com/libp2p/go-libp2p-resource-manager/obs" + + "github.com/prometheus/client_golang/prometheus" + "go.opencensus.io/stats/view" +) + + func SetupResourceManager() (network.ResourceManager, error) { + // Hook up the trace reporter metrics + view.Register(rcmgrObs.DefaultViews...) + ocprom.NewExporter(ocprom.Options{ + Registry: prometheus.DefaultRegisterer.(*prometheus.Registry), + Namespace: "rcmgr_trace_metrics", + }) + + str, err := rcmgrObs.NewStatsTraceReporter() + if err != nil { + return nil, err + } + + return rcmgr.NewResourceManager(limiter, rcmgr.WithTraceReporter(str)) + } +``` + +It should be fairly similar for other exporters. See the [OpenCensus +docs](https://opencensus.io/exporters/supported-exporters/go/) to see how to +export to another exporter. \ No newline at end of file diff --git a/obs/grafana-dashboards/resource-manager.json b/obs/grafana-dashboards/resource-manager.json new file mode 100644 index 0000000..ac22c51 --- /dev/null +++ b/obs/grafana-dashboards/resource-manager.json @@ -0,0 +1,1702 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 2, + "links": [], + "liveNow": false, + "panels": [ + { + "description": "This should be empty. If it's large you are running into your resource manager limits.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 67, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "count(rcmgr_trace_metrics_blocked_resources) by (resource,scope)", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Number of blocked resource requests", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 65, + "panels": [], + "title": "Blocked Resources", + "type": "row" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 9 + }, + "id": 42, + "panels": [], + "title": "Streams", + "type": "row" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 10 + }, + "id": 48, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "rcmgr_trace_metrics_streams{scope=\"system\"}", + "interval": "", + "legendFormat": "{{dir}}", + "refId": "A" + } + ], + "title": "System Streams", + "type": "timeseries" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 10 + }, + "id": 44, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "rcmgr_trace_metrics_streams{scope=\"transient\"}", + "interval": "", + "legendFormat": "{{dir}}", + "refId": "A" + } + ], + "title": "Transient Streams", + "type": "timeseries" + }, + { + "description": "How many streams does each service have open", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 18 + }, + "id": 43, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "rcmgr_trace_metrics_streams{scope=\"service\"}", + "interval": "", + "legendFormat": "{{dir}} {{service}}", + "refId": "A" + } + ], + "title": "Streams by service", + "type": "timeseries" + }, + { + "description": "How many streams does each protocol have open.\n\nA protocol is similiar to a service except there may be many protocols for a single service. A service may have \nmultiple protocols for backwards compatibility. For example, bitswap the service is a single entity, but it supports bitswap protocols v1.1 and v1.2.\n\nA service is attached to a stream manually by the protocol developer. A service may be missing if there is no `StreamScope.SetService` call.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 27 + }, + "id": 52, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "rcmgr_trace_metrics_streams{scope=\"protocol\"}", + "interval": "", + "legendFormat": "{{dir}} {{protocol}}", + "refId": "A" + } + ], + "title": "Streams by protocol", + "type": "timeseries" + }, + { + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 36 + }, + "id": 35, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "histogram_quantile(0.50, (rcmgr_trace_metrics_peer_streams_bucket - rcmgr_trace_metrics_peer_streams_negative_bucket)) - 0.1", + "interval": "", + "legendFormat": "p50 {{dir}} connections per peer", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "histogram_quantile(0.90, (rcmgr_trace_metrics_peer_streams_bucket - rcmgr_trace_metrics_peer_streams_negative_bucket)) - 0.1", + "hide": false, + "interval": "", + "legendFormat": "p90 {{dir}} connections per peer", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "histogram_quantile(1, (rcmgr_trace_metrics_peer_streams_bucket - rcmgr_trace_metrics_peer_streams_negative_bucket)) - 0.1", + "hide": false, + "interval": "", + "legendFormat": "max {{dir}} connections per peer", + "refId": "C" + } + ], + "title": "Streams per peer, aggregated", + "type": "timeseries" + }, + { + "description": "How many peers have N-0.1 streams open", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 46 + }, + "id": 46, + "options": { + "displayMode": "gradient", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "showUnfilled": true + }, + "pluginVersion": "8.4.5", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": false, + "expr": "rcmgr_trace_metrics_peer_streams_bucket{dir=\"inbound\"}-rcmgr_trace_metrics_peer_streams_negative_bucket{dir=\"inbound\"}", + "format": "heatmap", + "hide": false, + "interval": "", + "legendFormat": "{{le}}", + "refId": "A" + } + ], + "title": "Current inbound streams per peer histogram", + "type": "bargauge" + }, + { + "description": "How many peers have N-0.1 streams open", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 46 + }, + "id": 47, + "options": { + "displayMode": "gradient", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "showUnfilled": true + }, + "pluginVersion": "8.4.5", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": false, + "expr": "rcmgr_trace_metrics_peer_streams_bucket{dir=\"outbound\"}-rcmgr_trace_metrics_peer_streams_negative_bucket{dir=\"outbound\"}", + "format": "heatmap", + "hide": false, + "interval": "", + "legendFormat": "{{le}}", + "refId": "A" + } + ], + "title": "Current outbound streams per peer histogram", + "type": "bargauge" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 54 + }, + "id": 29, + "panels": [], + "title": "Connections", + "type": "row" + }, + { + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 55 + }, + "id": 31, + "options": { + "content": "# Libp2p Connections\n\nBroken down by [Resource Scope](https://github.com/libp2p/go-libp2p-resource-manager#resource-scopes). \nScopes represent what is imposing limits on this resource. For connections, we have three main scopes:\n\n1. System. The total number of connections owned by the process. Includes both application usable connections + the number of transient connections.\n2. Transient. The total number of connections that are being upgraded into usable connections in the process.\n3. Peer. The total number of connections associated with this peer. When a connection has this scope it is usable by the application.\n\nAn example of a System connection is a connection you can open a libp2p stream on and send data.\nA transient connection is not yet usable for application data since it may be negotiating \na security handshake or a multiplexer.\n\nConnections start in the transient scope and move over to the System and Peer scopes once they are ready to be used.\n\nIt would be unusual to see a lot of transient connections. It would also be unusal to see a peer with a lot of connections.", + "mode": "markdown" + }, + "pluginVersion": "8.4.5", + "title": "libp2p Connections", + "type": "text" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 64 + }, + "id": 33, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "rcmgr_trace_metrics_connections{scope=\"system\"}", + "interval": "", + "legendFormat": "{{dir}}", + "refId": "A" + } + ], + "title": "System Connections", + "type": "timeseries" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 64 + }, + "id": 36, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "rcmgr_trace_metrics_connections{scope=\"transient\"}", + "interval": "", + "legendFormat": "{{dir}}", + "refId": "A" + } + ], + "title": "Transient Connections", + "type": "timeseries" + }, + { + "gridPos": { + "h": 3, + "w": 24, + "x": 0, + "y": 72 + }, + "id": 38, + "options": { + "content": "These are aggregated stats. They are grouped by buckets. Each bucket represents how many peers have N number of connections.\n\nDue to a quirk in [opencensus](https://github.com/census-instrumentation/opencensus-go/blob/v0.23.0/stats/view/aggregation_data.go#L195) the bucket values have to be a bit bigger than the integer values.\nSo subtract 0.1 from the number to get the true number of connections. e.g. If a peer has 3 connections, it'll be put in the 3.1 bucket. \n", + "mode": "markdown" + }, + "pluginVersion": "8.4.5", + "title": "Connections per Peer", + "type": "text" + }, + { + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 75 + }, + "id": 45, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "histogram_quantile(0.50, (rcmgr_trace_metrics_peer_connections_bucket - rcmgr_trace_metrics_peer_connections_negative_bucket)) - 0.1", + "interval": "", + "legendFormat": "p50 {{dir}} connections per peer", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "histogram_quantile(0.90, (rcmgr_trace_metrics_peer_connections_bucket - rcmgr_trace_metrics_peer_connections_negative_bucket)) - 0.1", + "hide": false, + "interval": "", + "legendFormat": "p90 {{dir}} connections per peer", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "histogram_quantile(1, (rcmgr_trace_metrics_peer_connections_bucket - rcmgr_trace_metrics_peer_connections_negative_bucket)) - 0.1", + "hide": false, + "interval": "", + "legendFormat": "max {{dir}} connections per peer", + "refId": "C" + } + ], + "title": "Connections per peer, aggregated", + "type": "timeseries" + }, + { + "description": "How many peers have N-0.1 connections open", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 85 + }, + "id": 39, + "options": { + "displayMode": "gradient", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "showUnfilled": true + }, + "pluginVersion": "8.4.5", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": false, + "expr": "rcmgr_trace_metrics_peer_connections_bucket{dir=\"inbound\"}-rcmgr_trace_metrics_peer_connections_negative_bucket{dir=\"inbound\"}", + "format": "heatmap", + "hide": false, + "interval": "", + "legendFormat": "{{le}}", + "refId": "A" + } + ], + "title": "Current inbound connections per peer histogram", + "type": "bargauge" + }, + { + "description": "How many peers have N-0.1 connections open", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 85 + }, + "id": 40, + "options": { + "displayMode": "gradient", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "showUnfilled": true + }, + "pluginVersion": "8.4.5", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": false, + "expr": "rcmgr_trace_metrics_peer_connections_bucket{dir=\"outbound\"}-rcmgr_trace_metrics_peer_connections_negative_bucket{dir=\"outbound\"} ", + "format": "heatmap", + "hide": false, + "interval": "", + "legendFormat": "{{le}}", + "refId": "A" + } + ], + "title": "Curent outbound connections per peer histogram", + "type": "bargauge" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 93 + }, + "id": 54, + "panels": [], + "title": "Memory", + "type": "row" + }, + { + "description": "As reported to Resource Manager", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 94 + }, + "id": 56, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "rcmgr_trace_metrics_memory{scope=\"system\"}", + "interval": "", + "legendFormat": "Bytes Reserved", + "refId": "A" + } + ], + "title": "System memory reservations", + "type": "timeseries" + }, + { + "description": "As reported to Resource Manager", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 103 + }, + "id": 57, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "rcmgr_trace_metrics_memory{scope=\"protocol\"}", + "interval": "", + "legendFormat": "{{protocol}}", + "refId": "A" + } + ], + "title": "Memory reservations by protocol", + "type": "timeseries" + }, + { + "description": "As reported to Resource Manager", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 103 + }, + "id": 58, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "rcmgr_trace_metrics_memory{scope=\"service\"}", + "interval": "", + "legendFormat": "{{service}}", + "refId": "A" + } + ], + "title": "Memory reservations by service", + "type": "timeseries" + }, + { + "description": "As reported to the resource manager", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Number of peers aggregated" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-green", + "mode": "fixed" + } + }, + { + "id": "custom.axisPlacement", + "value": "right" + }, + { + "id": "unit" + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 111 + }, + "id": 59, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "histogram_quantile(0.50, (rcmgr_trace_metrics_peer_memory_bucket - rcmgr_trace_metrics_peer_memory_negative_bucket)) - 0.1", + "interval": "", + "legendFormat": "p50 {{dir}} connections per peer", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "histogram_quantile(0.90, (rcmgr_trace_metrics_peer_memory_bucket - rcmgr_trace_metrics_peer_memory_negative_bucket)) - 0.1", + "hide": false, + "interval": "", + "legendFormat": "p90 {{dir}} connections per peer", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "histogram_quantile(1, (rcmgr_trace_metrics_peer_memory_bucket - rcmgr_trace_metrics_peer_memory_negative_bucket)) - 0.1", + "hide": false, + "interval": "", + "legendFormat": "max {{dir}} connections per peer", + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "rcmgr_trace_metrics_peer_memory_count-rcmgr_trace_metrics_peer_memory_negative_count", + "hide": false, + "interval": "", + "legendFormat": "Number of peers aggregated", + "refId": "D" + } + ], + "title": "Memory reservations per peer, aggregated", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 121 + }, + "id": 62, + "panels": [], + "title": "File Descriptors", + "type": "row" + }, + { + "description": "As reported to the resource manager", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 122 + }, + "id": 60, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "rcmgr_trace_metrics_fds", + "interval": "", + "legendFormat": "{{scope}}", + "refId": "A" + } + ], + "title": "FDs in use", + "type": "timeseries" + } + ], + "refresh": false, + "schemaVersion": 35, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-5m", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Resource Manager", + "uid": "MgmGIjjnk", + "version": 13, + "weekStart": "" + } \ No newline at end of file From 04b17931b06d28ed9bb33b9af12026311287660e Mon Sep 17 00:00:00 2001 From: Marco Munizaga Date: Mon, 20 Jun 2022 18:31:09 -0700 Subject: [PATCH 34/45] Fix merge conflicts --- obs/stats.go | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/obs/stats.go b/obs/stats.go index d45284b..7838a66 100644 --- a/obs/stats.go +++ b/obs/stats.go @@ -144,7 +144,7 @@ func (r StatsTraceReporter) ConsumeEvent(evt rcmgr.TraceEvt) { switch evt.Type { case rcmgr.TraceAddStreamEvt, rcmgr.TraceRemoveStreamEvt: - if p := rcmgr.ParsePeerScopeName(evt.Scope); p.Validate() == nil { + if p := rcmgr.ParsePeerScopeName(evt.Name); p.Validate() == nil { // Aggregated peer stats. Counts how many peers have N number of streams open. // Uses two buckets aggregations. One to count how many streams the // peer has now. The other to count the negative value, or how many @@ -174,11 +174,11 @@ func (r StatsTraceReporter) ConsumeEvent(evt rcmgr.TraceEvt) { } } else { var tags []tag.Mutator - if rcmgr.IsSystemScope(evt.Scope) || rcmgr.IsTransientScope(evt.Scope) { - tags = append(tags, tag.Upsert(Scope, evt.Scope)) - } else if svc := rcmgr.ParseServiceScopeName(evt.Scope); svc != "" { + if rcmgr.IsSystemScope(evt.Name) || rcmgr.IsTransientScope(evt.Name) { + tags = append(tags, tag.Upsert(Scope, evt.Name)) + } else if svc := rcmgr.ParseServiceScopeName(evt.Name); svc != "" { tags = append(tags, tag.Upsert(Scope, "service"), tag.Upsert(Service, svc)) - } else if proto := rcmgr.ParseProtocolScopeName(evt.Scope); proto != "" { + } else if proto := rcmgr.ParseProtocolScopeName(evt.Name); proto != "" { tags = append(tags, tag.Upsert(Scope, "protocol"), tag.Upsert(Protocol, proto)) } else { // Not measuring connscope, servicepeer and protocolpeer. Lots of data, and @@ -205,7 +205,7 @@ func (r StatsTraceReporter) ConsumeEvent(evt rcmgr.TraceEvt) { } case rcmgr.TraceAddConnEvt, rcmgr.TraceRemoveConnEvt: - if p := rcmgr.ParsePeerScopeName(evt.Scope); p.Validate() == nil { + if p := rcmgr.ParsePeerScopeName(evt.Name); p.Validate() == nil { // Aggregated peer stats. Counts how many peers have N number of connections. // Uses two buckets aggregations. One to count how many streams the // peer has now. The other to count the negative value, or how many @@ -235,14 +235,14 @@ func (r StatsTraceReporter) ConsumeEvent(evt rcmgr.TraceEvt) { } } else { var tags []tag.Mutator - if rcmgr.IsSystemScope(evt.Scope) || rcmgr.IsTransientScope(evt.Scope) { - tags = append(tags, tag.Upsert(Scope, evt.Scope)) - } else if rcmgr.IsConnScope(evt.Scope) { + if rcmgr.IsSystemScope(evt.Name) || rcmgr.IsTransientScope(evt.Name) { + tags = append(tags, tag.Upsert(Scope, evt.Name)) + } else if rcmgr.IsConnScope(evt.Name) { // Not measuring this. I don't think it's useful. break } else { // There shouldn't be anything here. But we keep going so the metrics will tell us if we're wrong (scope="") - log.Debugf("unexpected event in stats: %s", evt.Scope) + log.Debugf("unexpected event in stats: %s", evt.Name) } if evt.DeltaOut != 0 { @@ -271,7 +271,7 @@ func (r StatsTraceReporter) ConsumeEvent(evt rcmgr.TraceEvt) { } } case rcmgr.TraceReserveMemoryEvt, rcmgr.TraceReleaseMemoryEvt: - if p := rcmgr.ParsePeerScopeName(evt.Scope); p.Validate() == nil { + if p := rcmgr.ParsePeerScopeName(evt.Name); p.Validate() == nil { oldMem := evt.Memory - evt.Delta if oldMem != evt.Memory { if oldMem != 0 { @@ -281,7 +281,7 @@ func (r StatsTraceReporter) ConsumeEvent(evt rcmgr.TraceEvt) { stats.Record(ctx, peerMemory.M(evt.Memory)) } } - } else if rcmgr.IsConnScope(evt.Scope) { + } else if rcmgr.IsConnScope(evt.Name) { oldMem := evt.Memory - evt.Delta if oldMem != evt.Memory { if oldMem != 0 { @@ -293,11 +293,11 @@ func (r StatsTraceReporter) ConsumeEvent(evt rcmgr.TraceEvt) { } } else { var tags []tag.Mutator - if rcmgr.IsSystemScope(evt.Scope) || rcmgr.IsTransientScope(evt.Scope) { - tags = append(tags, tag.Upsert(Scope, evt.Scope)) - } else if svc := rcmgr.ParseServiceScopeName(evt.Scope); svc != "" { + if rcmgr.IsSystemScope(evt.Name) || rcmgr.IsTransientScope(evt.Name) { + tags = append(tags, tag.Upsert(Scope, evt.Name)) + } else if svc := rcmgr.ParseServiceScopeName(evt.Name); svc != "" { tags = append(tags, tag.Upsert(Scope, "service"), tag.Upsert(Service, svc)) - } else if proto := rcmgr.ParseProtocolScopeName(evt.Scope); proto != "" { + } else if proto := rcmgr.ParseProtocolScopeName(evt.Name); proto != "" { tags = append(tags, tag.Upsert(Scope, "protocol"), tag.Upsert(Protocol, proto)) } else { // Not measuring connscope, servicepeer and protocolpeer. Lots of data, and @@ -322,7 +322,7 @@ func (r StatsTraceReporter) ConsumeEvent(evt rcmgr.TraceEvt) { } // Only the top scope. We don't want to get the peerid here. - scope := strings.SplitN(evt.Scope, ":", 2)[0] + scope := strings.SplitN(evt.Name, ":", 2)[0] // Drop the connection or stream id scope = strings.SplitN(scope, "-", 2)[0] From bc012d37ffa41a948c900a0ee536cbb36bd5e769 Mon Sep 17 00:00:00 2001 From: Marco Munizaga Date: Tue, 21 Jun 2022 07:38:25 -0700 Subject: [PATCH 35/45] Update dashboard json --- obs/grafana-dashboards/resource-manager.json | 3106 +++++++++--------- 1 file changed, 1553 insertions(+), 1553 deletions(-) diff --git a/obs/grafana-dashboards/resource-manager.json b/obs/grafana-dashboards/resource-manager.json index ac22c51..e9cae6d 100644 --- a/obs/grafana-dashboards/resource-manager.json +++ b/obs/grafana-dashboards/resource-manager.json @@ -1,1702 +1,1702 @@ { - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": "-- Grafana --", - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "target": { - "limit": 100, - "matchAny": false, - "tags": [], - "type": "dashboard" - }, - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "id": 2, - "links": [], - "liveNow": false, - "panels": [ + "annotations": { + "list": [ { - "description": "This should be empty. If it's large you are running into your resource manager limits.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 2, + "links": [], + "liveNow": false, + "panels": [ + { + "description": "This should be empty. If it's large you are running into your resource manager limits.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" } }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 0 - }, - "id": 67, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "single", - "sort": "none" + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] } }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "exemplar": true, - "expr": "count(rcmgr_trace_metrics_blocked_resources) by (resource,scope)", - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "title": "Number of blocked resource requests", - "type": "timeseries" + "overrides": [] }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 8 - }, - "id": 65, - "panels": [], - "title": "Blocked Resources", - "type": "row" + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 9 + "id": 67, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" }, - "id": 42, - "panels": [], - "title": "Streams", - "type": "row" + "tooltip": { + "mode": "single", + "sort": "none" + } }, - { - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 10 - }, - "id": 48, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" + "exemplar": true, + "expr": "count(rcmgr_trace_metrics_blocked_resources) by (resource,scope)", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Number of blocked resource requests", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 65, + "panels": [], + "title": "Blocked Resources", + "type": "row" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 9 + }, + "id": 42, + "panels": [], + "title": "Streams", + "type": "row" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "exemplar": true, - "expr": "rcmgr_trace_metrics_streams{scope=\"system\"}", - "interval": "", - "legendFormat": "{{dir}}", - "refId": "A" - } - ], - "title": "System Streams", - "type": "timeseries" - }, - { - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] + "thresholdsStyle": { + "mode": "off" } }, - "overrides": [] + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 10 + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 10 + }, + "id": 48, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" }, - "id": 44, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "exemplar": true, + "expr": "rcmgr_trace_metrics_streams{scope=\"system\"}", + "interval": "", + "legendFormat": "{{dir}}", + "refId": "A" + } + ], + "title": "System Streams", + "type": "timeseries" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "exemplar": true, - "expr": "rcmgr_trace_metrics_streams{scope=\"transient\"}", - "interval": "", - "legendFormat": "{{dir}}", - "refId": "A" - } - ], - "title": "Transient Streams", - "type": "timeseries" - }, - { - "description": "How many streams does each service have open", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] + "thresholdsStyle": { + "mode": "off" } }, - "overrides": [] + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } }, - "gridPos": { - "h": 9, - "w": 24, - "x": 0, - "y": 18 + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 10 + }, + "id": 44, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" }, - "id": 43, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "exemplar": true, + "expr": "rcmgr_trace_metrics_streams{scope=\"transient\"}", + "interval": "", + "legendFormat": "{{dir}}", + "refId": "A" + } + ], + "title": "Transient Streams", + "type": "timeseries" + }, + { + "description": "How many streams does each service have open", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "exemplar": true, - "expr": "rcmgr_trace_metrics_streams{scope=\"service\"}", - "interval": "", - "legendFormat": "{{dir}} {{service}}", - "refId": "A" - } - ], - "title": "Streams by service", - "type": "timeseries" - }, - { - "description": "How many streams does each protocol have open.\n\nA protocol is similiar to a service except there may be many protocols for a single service. A service may have \nmultiple protocols for backwards compatibility. For example, bitswap the service is a single entity, but it supports bitswap protocols v1.1 and v1.2.\n\nA service is attached to a stream manually by the protocol developer. A service may be missing if there is no `StreamScope.SetService` call.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] + "thresholdsStyle": { + "mode": "off" } }, - "overrides": [] + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } }, - "gridPos": { - "h": 9, - "w": 24, - "x": 0, - "y": 27 + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 18 + }, + "id": 43, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" }, - "id": 52, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "exemplar": true, + "expr": "rcmgr_trace_metrics_streams{scope=\"service\"}", + "interval": "", + "legendFormat": "{{dir}} {{service}}", + "refId": "A" + } + ], + "title": "Streams by service", + "type": "timeseries" + }, + { + "description": "How many streams does each protocol have open.\n\nA protocol is similiar to a service except there may be many protocols for a single service. A service may have \nmultiple protocols for backwards compatibility. For example, bitswap the service is a single entity, but it supports bitswap protocols v1.1 and v1.2.\n\nA service is attached to a stream manually by the protocol developer. A service may be missing if there is no `StreamScope.SetService` call.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "exemplar": true, - "expr": "rcmgr_trace_metrics_streams{scope=\"protocol\"}", - "interval": "", - "legendFormat": "{{dir}} {{protocol}}", - "refId": "A" - } - ], - "title": "Streams by protocol", - "type": "timeseries" - }, - { - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] + "thresholdsStyle": { + "mode": "off" } }, - "overrides": [] + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } }, - "gridPos": { - "h": 10, - "w": 24, - "x": 0, - "y": 36 + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 27 + }, + "id": 52, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" }, - "id": 35, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "exemplar": true, - "expr": "histogram_quantile(0.50, (rcmgr_trace_metrics_peer_streams_bucket - rcmgr_trace_metrics_peer_streams_negative_bucket)) - 0.1", - "interval": "", - "legendFormat": "p50 {{dir}} connections per peer", - "refId": "A" + "exemplar": true, + "expr": "rcmgr_trace_metrics_streams{scope=\"protocol\"}", + "interval": "", + "legendFormat": "{{dir}} {{protocol}}", + "refId": "A" + } + ], + "title": "Streams by protocol", + "type": "timeseries" + }, + { + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "exemplar": true, - "expr": "histogram_quantile(0.90, (rcmgr_trace_metrics_peer_streams_bucket - rcmgr_trace_metrics_peer_streams_negative_bucket)) - 0.1", - "hide": false, - "interval": "", - "legendFormat": "p90 {{dir}} connections per peer", - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" }, - "exemplar": true, - "expr": "histogram_quantile(1, (rcmgr_trace_metrics_peer_streams_bucket - rcmgr_trace_metrics_peer_streams_negative_bucket)) - 0.1", - "hide": false, - "interval": "", - "legendFormat": "max {{dir}} connections per peer", - "refId": "C" - } - ], - "title": "Streams per peer, aggregated", - "type": "timeseries" - }, - { - "description": "How many peers have N-0.1 streams open", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] + "thresholdsStyle": { + "mode": "off" } }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 46 + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } }, - "id": 46, - "options": { - "displayMode": "gradient", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", - "values": false - }, - "showUnfilled": true + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 36 + }, + "id": 35, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" }, - "pluginVersion": "8.4.5", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "exemplar": false, - "expr": "rcmgr_trace_metrics_peer_streams_bucket{dir=\"inbound\"}-rcmgr_trace_metrics_peer_streams_negative_bucket{dir=\"inbound\"}", - "format": "heatmap", - "hide": false, - "interval": "", - "legendFormat": "{{le}}", - "refId": "A" - } - ], - "title": "Current inbound streams per peer histogram", - "type": "bargauge" + "tooltip": { + "mode": "single", + "sort": "none" + } }, - { - "description": "How many peers have N-0.1 streams open", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 46 + "exemplar": true, + "expr": "histogram_quantile(0.50, (rcmgr_trace_metrics_peer_streams_bucket - rcmgr_trace_metrics_peer_streams_negative_bucket)) - 0.1", + "interval": "", + "legendFormat": "p50 {{dir}} streams per peer", + "refId": "A" }, - "id": 47, - "options": { - "displayMode": "gradient", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", - "values": false + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" }, - "showUnfilled": true + "exemplar": true, + "expr": "histogram_quantile(0.90, (rcmgr_trace_metrics_peer_streams_bucket - rcmgr_trace_metrics_peer_streams_negative_bucket)) - 0.1", + "hide": false, + "interval": "", + "legendFormat": "p90 {{dir}} streams per peer", + "refId": "B" }, - "pluginVersion": "8.4.5", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "exemplar": false, - "expr": "rcmgr_trace_metrics_peer_streams_bucket{dir=\"outbound\"}-rcmgr_trace_metrics_peer_streams_negative_bucket{dir=\"outbound\"}", - "format": "heatmap", - "hide": false, - "interval": "", - "legendFormat": "{{le}}", - "refId": "A" + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "histogram_quantile(1, (rcmgr_trace_metrics_peer_streams_bucket - rcmgr_trace_metrics_peer_streams_negative_bucket)) - 0.1", + "hide": false, + "interval": "", + "legendFormat": "max {{dir}} streams per peer", + "refId": "C" + } + ], + "title": "Streams per peer, aggregated", + "type": "timeseries" + }, + { + "description": "How many peers have N-0.1 streams open", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] } - ], - "title": "Current outbound streams per peer histogram", - "type": "bargauge" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 54 }, - "id": 29, - "panels": [], - "title": "Connections", - "type": "row" + "overrides": [] }, - { - "gridPos": { - "h": 9, - "w": 24, - "x": 0, - "y": 55 - }, - "id": 31, - "options": { - "content": "# Libp2p Connections\n\nBroken down by [Resource Scope](https://github.com/libp2p/go-libp2p-resource-manager#resource-scopes). \nScopes represent what is imposing limits on this resource. For connections, we have three main scopes:\n\n1. System. The total number of connections owned by the process. Includes both application usable connections + the number of transient connections.\n2. Transient. The total number of connections that are being upgraded into usable connections in the process.\n3. Peer. The total number of connections associated with this peer. When a connection has this scope it is usable by the application.\n\nAn example of a System connection is a connection you can open a libp2p stream on and send data.\nA transient connection is not yet usable for application data since it may be negotiating \na security handshake or a multiplexer.\n\nConnections start in the transient scope and move over to the System and Peer scopes once they are ready to be used.\n\nIt would be unusual to see a lot of transient connections. It would also be unusal to see a peer with a lot of connections.", - "mode": "markdown" + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 46 + }, + "id": 46, + "options": { + "displayMode": "gradient", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false }, - "pluginVersion": "8.4.5", - "title": "libp2p Connections", - "type": "text" + "showUnfilled": true }, - { - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" + "pluginVersion": "8.4.5", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": false, + "expr": "rcmgr_trace_metrics_peer_streams_bucket{dir=\"inbound\"}-rcmgr_trace_metrics_peer_streams_negative_bucket{dir=\"inbound\"}", + "format": "heatmap", + "hide": false, + "interval": "", + "legendFormat": "{{le}}", + "refId": "A" + } + ], + "title": "Current inbound streams per peer histogram", + "type": "bargauge" + }, + { + "description": "How many peers have N-0.1 streams open", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null }, - "thresholdsStyle": { - "mode": "off" + { + "color": "red", + "value": 80 } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] + ] + } }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 64 + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 46 + }, + "id": 47, + "options": { + "displayMode": "gradient", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false }, - "id": 33, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" + "showUnfilled": true + }, + "pluginVersion": "8.4.5", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "exemplar": true, - "expr": "rcmgr_trace_metrics_connections{scope=\"system\"}", - "interval": "", - "legendFormat": "{{dir}}", - "refId": "A" - } - ], - "title": "System Connections", - "type": "timeseries" + "exemplar": false, + "expr": "rcmgr_trace_metrics_peer_streams_bucket{dir=\"outbound\"}-rcmgr_trace_metrics_peer_streams_negative_bucket{dir=\"outbound\"}", + "format": "heatmap", + "hide": false, + "interval": "", + "legendFormat": "{{le}}", + "refId": "A" + } + ], + "title": "Current outbound streams per peer histogram", + "type": "bargauge" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 54 }, - { - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" + "id": 29, + "panels": [], + "title": "Connections", + "type": "row" + }, + { + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 55 + }, + "id": 31, + "options": { + "content": "# Libp2p Connections\n\nBroken down by [Resource Scope](https://github.com/libp2p/go-libp2p-resource-manager#resource-scopes). \nScopes represent what is imposing limits on this resource. For connections, we have three main scopes:\n\n1. System. The total number of connections owned by the process. Includes both application usable connections + the number of transient connections.\n2. Transient. The total number of connections that are being upgraded into usable connections in the process.\n3. Peer. The total number of connections associated with this peer. When a connection has this scope it is usable by the application.\n\nAn example of a System connection is a connection you can open a libp2p stream on and send data.\nA transient connection is not yet usable for application data since it may be negotiating \na security handshake or a multiplexer.\n\nConnections start in the transient scope and move over to the System and Peer scopes once they are ready to be used.\n\nIt would be unusual to see a lot of transient connections. It would also be unusal to see a peer with a lot of connections.", + "mode": "markdown" + }, + "pluginVersion": "8.4.5", + "title": "libp2p Connections", + "type": "text" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" } }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 64 - }, - "id": 36, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "single", - "sort": "none" + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] } }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "exemplar": true, - "expr": "rcmgr_trace_metrics_connections{scope=\"transient\"}", - "interval": "", - "legendFormat": "{{dir}}", - "refId": "A" - } - ], - "title": "Transient Connections", - "type": "timeseries" + "overrides": [] }, - { - "gridPos": { - "h": 3, - "w": 24, - "x": 0, - "y": 72 - }, - "id": 38, - "options": { - "content": "These are aggregated stats. They are grouped by buckets. Each bucket represents how many peers have N number of connections.\n\nDue to a quirk in [opencensus](https://github.com/census-instrumentation/opencensus-go/blob/v0.23.0/stats/view/aggregation_data.go#L195) the bucket values have to be a bit bigger than the integer values.\nSo subtract 0.1 from the number to get the true number of connections. e.g. If a peer has 3 connections, it'll be put in the 3.1 bucket. \n", - "mode": "markdown" + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 64 + }, + "id": 33, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" }, - "pluginVersion": "8.4.5", - "title": "Connections per Peer", - "type": "text" + "tooltip": { + "mode": "single", + "sort": "none" + } }, - { - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "rcmgr_trace_metrics_connections{scope=\"system\"}", + "interval": "", + "legendFormat": "{{dir}}", + "refId": "A" + } + ], + "title": "System Connections", + "type": "timeseries" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" } }, - "overrides": [] + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } }, - "gridPos": { - "h": 10, - "w": 24, - "x": 0, - "y": 75 + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 64 + }, + "id": 36, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" }, - "id": 45, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "exemplar": true, - "expr": "histogram_quantile(0.50, (rcmgr_trace_metrics_peer_connections_bucket - rcmgr_trace_metrics_peer_connections_negative_bucket)) - 0.1", - "interval": "", - "legendFormat": "p50 {{dir}} connections per peer", - "refId": "A" + "exemplar": true, + "expr": "rcmgr_trace_metrics_connections{scope=\"transient\"}", + "interval": "", + "legendFormat": "{{dir}}", + "refId": "A" + } + ], + "title": "Transient Connections", + "type": "timeseries" + }, + { + "gridPos": { + "h": 3, + "w": 24, + "x": 0, + "y": 72 + }, + "id": 38, + "options": { + "content": "These are aggregated stats. They are grouped by buckets. Each bucket represents how many peers have N number of connections.\n\nDue to a quirk in [opencensus](https://github.com/census-instrumentation/opencensus-go/blob/v0.23.0/stats/view/aggregation_data.go#L195) the bucket values have to be a bit bigger than the integer values.\nSo subtract 0.1 from the number to get the true number of connections. e.g. If a peer has 3 connections, it'll be put in the 3.1 bucket. \n", + "mode": "markdown" + }, + "pluginVersion": "8.4.5", + "title": "Connections per Peer", + "type": "text" + }, + { + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "exemplar": true, - "expr": "histogram_quantile(0.90, (rcmgr_trace_metrics_peer_connections_bucket - rcmgr_trace_metrics_peer_connections_negative_bucket)) - 0.1", - "hide": false, - "interval": "", - "legendFormat": "p90 {{dir}} connections per peer", - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" }, - "exemplar": true, - "expr": "histogram_quantile(1, (rcmgr_trace_metrics_peer_connections_bucket - rcmgr_trace_metrics_peer_connections_negative_bucket)) - 0.1", - "hide": false, - "interval": "", - "legendFormat": "max {{dir}} connections per peer", - "refId": "C" - } - ], - "title": "Connections per peer, aggregated", - "type": "timeseries" - }, - { - "description": "How many peers have N-0.1 connections open", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] + "thresholdsStyle": { + "mode": "off" } }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 85 + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } }, - "id": 39, - "options": { - "displayMode": "gradient", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", - "values": false - }, - "showUnfilled": true + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 75 + }, + "id": 45, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" }, - "pluginVersion": "8.4.5", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "exemplar": false, - "expr": "rcmgr_trace_metrics_peer_connections_bucket{dir=\"inbound\"}-rcmgr_trace_metrics_peer_connections_negative_bucket{dir=\"inbound\"}", - "format": "heatmap", - "hide": false, - "interval": "", - "legendFormat": "{{le}}", - "refId": "A" - } - ], - "title": "Current inbound connections per peer histogram", - "type": "bargauge" + "tooltip": { + "mode": "single", + "sort": "none" + } }, - { - "description": "How many peers have N-0.1 connections open", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" }, - "overrides": [] + "exemplar": true, + "expr": "histogram_quantile(0.50, (rcmgr_trace_metrics_peer_connections_bucket - rcmgr_trace_metrics_peer_connections_negative_bucket)) - 0.1", + "interval": "", + "legendFormat": "p50 {{dir}} connections per peer", + "refId": "A" }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 85 - }, - "id": 40, - "options": { - "displayMode": "gradient", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", - "values": false + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" }, - "showUnfilled": true + "exemplar": true, + "expr": "histogram_quantile(0.90, (rcmgr_trace_metrics_peer_connections_bucket - rcmgr_trace_metrics_peer_connections_negative_bucket)) - 0.1", + "hide": false, + "interval": "", + "legendFormat": "p90 {{dir}} connections per peer", + "refId": "B" }, - "pluginVersion": "8.4.5", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "exemplar": false, - "expr": "rcmgr_trace_metrics_peer_connections_bucket{dir=\"outbound\"}-rcmgr_trace_metrics_peer_connections_negative_bucket{dir=\"outbound\"} ", - "format": "heatmap", - "hide": false, - "interval": "", - "legendFormat": "{{le}}", - "refId": "A" + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "histogram_quantile(1, (rcmgr_trace_metrics_peer_connections_bucket - rcmgr_trace_metrics_peer_connections_negative_bucket)) - 0.1", + "hide": false, + "interval": "", + "legendFormat": "max {{dir}} connections per peer", + "refId": "C" + } + ], + "title": "Connections per peer, aggregated", + "type": "timeseries" + }, + { + "description": "How many peers have N-0.1 connections open", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] } - ], - "title": "Curent outbound connections per peer histogram", - "type": "bargauge" + }, + "overrides": [] }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 93 + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 85 + }, + "id": 39, + "options": { + "displayMode": "gradient", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false }, - "id": 54, - "panels": [], - "title": "Memory", - "type": "row" + "showUnfilled": true }, - { - "description": "As reported to Resource Manager", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" + "pluginVersion": "8.4.5", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": false, + "expr": "rcmgr_trace_metrics_peer_connections_bucket{dir=\"inbound\"}-rcmgr_trace_metrics_peer_connections_negative_bucket{dir=\"inbound\"}", + "format": "heatmap", + "hide": false, + "interval": "", + "legendFormat": "{{le}}", + "refId": "A" + } + ], + "title": "Current inbound connections per peer histogram", + "type": "bargauge" + }, + { + "description": "How many peers have N-0.1 connections open", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null }, - "thresholdsStyle": { - "mode": "off" + { + "color": "red", + "value": 80 } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "decbytes" - }, - "overrides": [] + ] + } }, - "gridPos": { - "h": 9, - "w": 24, - "x": 0, - "y": 94 + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 85 + }, + "id": 40, + "options": { + "displayMode": "gradient", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false }, - "id": 56, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" + "showUnfilled": true + }, + "pluginVersion": "8.4.5", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "exemplar": true, - "expr": "rcmgr_trace_metrics_memory{scope=\"system\"}", - "interval": "", - "legendFormat": "Bytes Reserved", - "refId": "A" - } - ], - "title": "System memory reservations", - "type": "timeseries" + "exemplar": false, + "expr": "rcmgr_trace_metrics_peer_connections_bucket{dir=\"outbound\"}-rcmgr_trace_metrics_peer_connections_negative_bucket{dir=\"outbound\"} ", + "format": "heatmap", + "hide": false, + "interval": "", + "legendFormat": "{{le}}", + "refId": "A" + } + ], + "title": "Curent outbound connections per peer histogram", + "type": "bargauge" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 93 }, - { - "description": "As reported to Resource Manager", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" + "id": 54, + "panels": [], + "title": "Memory", + "type": "row" + }, + { + "description": "As reported to Resource Manager", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" }, - "unit": "decbytes" + "thresholdsStyle": { + "mode": "off" + } }, - "overrides": [] + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 103 + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 94 + }, + "id": 56, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" }, - "id": 57, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "exemplar": true, + "expr": "rcmgr_trace_metrics_memory{scope=\"system\"}", + "interval": "", + "legendFormat": "Bytes Reserved", + "refId": "A" + } + ], + "title": "System memory reservations", + "type": "timeseries" + }, + { + "description": "As reported to Resource Manager", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "exemplar": true, - "expr": "rcmgr_trace_metrics_memory{scope=\"protocol\"}", - "interval": "", - "legendFormat": "{{protocol}}", - "refId": "A" - } - ], - "title": "Memory reservations by protocol", - "type": "timeseries" - }, - { - "description": "As reported to Resource Manager", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null }, - "thresholdsStyle": { - "mode": "off" + { + "color": "red", + "value": 80 } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "decbytes" + ] }, - "overrides": [] + "unit": "decbytes" }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 103 - }, - "id": 58, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "single", - "sort": "none" - } + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 103 + }, + "id": 57, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "exemplar": true, - "expr": "rcmgr_trace_metrics_memory{scope=\"service\"}", - "interval": "", - "legendFormat": "{{service}}", - "refId": "A" - } - ], - "title": "Memory reservations by service", - "type": "timeseries" + "tooltip": { + "mode": "single", + "sort": "none" + } }, - { - "description": "As reported to the resource manager", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "rcmgr_trace_metrics_memory{scope=\"protocol\"}", + "interval": "", + "legendFormat": "{{protocol}}", + "refId": "A" + } + ], + "title": "Memory reservations by protocol", + "type": "timeseries" + }, + { + "description": "As reported to Resource Manager", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" }, - "unit": "decbytes" + "thresholdsStyle": { + "mode": "off" + } }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Number of peers aggregated" + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "dark-green", - "mode": "fixed" - } - }, - { - "id": "custom.axisPlacement", - "value": "right" - }, - { - "id": "unit" - } - ] - } - ] + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" }, - "gridPos": { - "h": 10, - "w": 24, - "x": 0, - "y": 111 + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 103 + }, + "id": 58, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" }, - "id": 59, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "exemplar": true, - "expr": "histogram_quantile(0.50, (rcmgr_trace_metrics_peer_memory_bucket - rcmgr_trace_metrics_peer_memory_negative_bucket)) - 0.1", - "interval": "", - "legendFormat": "p50 {{dir}} connections per peer", - "refId": "A" + "exemplar": true, + "expr": "rcmgr_trace_metrics_memory{scope=\"service\"}", + "interval": "", + "legendFormat": "{{service}}", + "refId": "A" + } + ], + "title": "Memory reservations by service", + "type": "timeseries" + }, + { + "description": "As reported to the resource manager", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "exemplar": true, - "expr": "histogram_quantile(0.90, (rcmgr_trace_metrics_peer_memory_bucket - rcmgr_trace_metrics_peer_memory_negative_bucket)) - 0.1", - "hide": false, - "interval": "", - "legendFormat": "p90 {{dir}} connections per peer", - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" }, - "exemplar": true, - "expr": "histogram_quantile(1, (rcmgr_trace_metrics_peer_memory_bucket - rcmgr_trace_metrics_peer_memory_negative_bucket)) - 0.1", - "hide": false, - "interval": "", - "legendFormat": "max {{dir}} connections per peer", - "refId": "C" + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] }, + "unit": "decbytes" + }, + "overrides": [ { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "matcher": { + "id": "byName", + "options": "Number of peers aggregated" }, - "exemplar": true, - "expr": "rcmgr_trace_metrics_peer_memory_count-rcmgr_trace_metrics_peer_memory_negative_count", - "hide": false, - "interval": "", - "legendFormat": "Number of peers aggregated", - "refId": "D" + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-purple", + "mode": "fixed" + } + }, + { + "id": "custom.axisPlacement", + "value": "right" + }, + { + "id": "unit" + } + ] } - ], - "title": "Memory reservations per peer, aggregated", - "type": "timeseries" + ] }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 121 + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 111 + }, + "id": 59, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" }, - "id": 62, - "panels": [], - "title": "File Descriptors", - "type": "row" + "tooltip": { + "mode": "single", + "sort": "none" + } }, - { - "description": "As reported to the resource manager", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" }, - "overrides": [] + "exemplar": true, + "expr": "histogram_quantile(0.50, (rcmgr_trace_metrics_peer_memory_bucket - rcmgr_trace_metrics_peer_memory_negative_bucket)) - 0.1", + "interval": "", + "legendFormat": "p50 memory usage per peer", + "refId": "A" }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 122 + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "histogram_quantile(0.90, (rcmgr_trace_metrics_peer_memory_bucket - rcmgr_trace_metrics_peer_memory_negative_bucket)) - 0.1", + "hide": false, + "interval": "", + "legendFormat": "p90 memory usage per peer", + "refId": "B" }, - "id": 60, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" }, - "tooltip": { - "mode": "single", - "sort": "none" - } + "exemplar": true, + "expr": "histogram_quantile(1, (rcmgr_trace_metrics_peer_memory_bucket - rcmgr_trace_metrics_peer_memory_negative_bucket)) - 0.1", + "hide": false, + "interval": "", + "legendFormat": "max memory usage per peer", + "refId": "C" }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "exemplar": true, - "expr": "rcmgr_trace_metrics_fds", - "interval": "", - "legendFormat": "{{scope}}", - "refId": "A" - } - ], - "title": "FDs in use", - "type": "timeseries" - } - ], - "refresh": false, - "schemaVersion": 35, - "style": "dark", - "tags": [], - "templating": { - "list": [] + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "rcmgr_trace_metrics_peer_memory_count-rcmgr_trace_metrics_peer_memory_negative_count", + "hide": false, + "interval": "", + "legendFormat": "Number of peers aggregated", + "refId": "D" + } + ], + "title": "Memory reservations per peer, aggregated", + "type": "timeseries" }, - "time": { - "from": "now-5m", - "to": "now" + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 121 + }, + "id": 62, + "panels": [], + "title": "File Descriptors", + "type": "row" }, - "timepicker": {}, - "timezone": "", - "title": "Resource Manager", - "uid": "MgmGIjjnk", - "version": 13, - "weekStart": "" - } \ No newline at end of file + { + "description": "As reported to the resource manager", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 122 + }, + "id": 60, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "rcmgr_trace_metrics_fds", + "interval": "", + "legendFormat": "{{scope}}", + "refId": "A" + } + ], + "title": "FDs in use", + "type": "timeseries" + } + ], + "refresh": false, + "schemaVersion": 35, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-5m", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Resource Manager", + "uid": "MgmGIjjnk", + "version": 14, + "weekStart": "" +} \ No newline at end of file From 04e10885af3aaf6081c10228eaf539bd62a6accc Mon Sep 17 00:00:00 2001 From: Marco Munizaga Date: Thu, 23 Jun 2022 15:29:53 -0700 Subject: [PATCH 36/45] Update dashboard blocked resources --- obs/grafana-dashboards/resource-manager.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/obs/grafana-dashboards/resource-manager.json b/obs/grafana-dashboards/resource-manager.json index e9cae6d..66d390a 100644 --- a/obs/grafana-dashboards/resource-manager.json +++ b/obs/grafana-dashboards/resource-manager.json @@ -102,7 +102,7 @@ "uid": "PBFA97CFB590B2093" }, "exemplar": true, - "expr": "count(rcmgr_trace_metrics_blocked_resources) by (resource,scope)", + "expr": "rate(rcmgr_trace_metrics_blocked_resources[$__rate_interval])", "interval": "", "legendFormat": "", "refId": "A" From d34b9cba642f5587d58e217506d879c9c520e0fa Mon Sep 17 00:00:00 2001 From: Marco Munizaga Date: Thu, 23 Jun 2022 15:30:13 -0700 Subject: [PATCH 37/45] Update bucket distribution and blocked resources --- obs/stats.go | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/obs/stats.go b/obs/stats.go index 7838a66..89dc5b5 100644 --- a/obs/stats.go +++ b/obs/stats.go @@ -54,24 +54,24 @@ var ( ConnView = &view.View{Measure: conns, Aggregation: view.Sum(), TagKeys: []tag.Key{Direction, Scope}} - fibLikeDistribution = []float64{ - 1.1, 2.1, 3.1, 5.1, 8.1, 13.1, 21.1, 34.1, 55.1, 100.1, 200.1, + oneTenThenExpDistribution = []float64{ + 1.1, 2.1, 3.1, 4.1, 5.1, 6.1, 7.1, 8.1, 9.1, 10.1, 16.1, 32.1, 64.1, 128.1, 256.1, } PeerConnsView = &view.View{ Measure: peerConns, - Aggregation: view.Distribution(fibLikeDistribution...), + Aggregation: view.Distribution(oneTenThenExpDistribution...), TagKeys: []tag.Key{Direction}, } PeerConnsNegativeView = &view.View{ Measure: peerConnsNegative, - Aggregation: view.Distribution(fibLikeDistribution...), + Aggregation: view.Distribution(oneTenThenExpDistribution...), TagKeys: []tag.Key{Direction}, } StreamView = &view.View{Measure: streams, Aggregation: view.Sum(), TagKeys: []tag.Key{Direction, Scope, Service, Protocol}} - PeerStreamsView = &view.View{Measure: peerStreams, Aggregation: view.Distribution(fibLikeDistribution...), TagKeys: []tag.Key{Direction}} - PeerStreamNegativeView = &view.View{Measure: peerStreamsNegative, Aggregation: view.Distribution(fibLikeDistribution...), TagKeys: []tag.Key{Direction}} + PeerStreamsView = &view.View{Measure: peerStreams, Aggregation: view.Distribution(oneTenThenExpDistribution...), TagKeys: []tag.Key{Direction}} + PeerStreamNegativeView = &view.View{Measure: peerStreamsNegative, Aggregation: view.Distribution(oneTenThenExpDistribution...), TagKeys: []tag.Key{Direction}} MemoryView = &view.View{Measure: memory, Aggregation: view.Sum(), TagKeys: []tag.Key{Scope, Service, Protocol}} @@ -136,6 +136,7 @@ var DefaultViews []*view.View = []*view.View{ type StatsTraceReporter struct{} func NewStatsTraceReporter() (StatsTraceReporter, error) { + // TODO tell prometheus the system limits return StatsTraceReporter{}, nil } @@ -328,16 +329,19 @@ func (r StatsTraceReporter) ConsumeEvent(evt rcmgr.TraceEvt) { tags := []tag.Mutator{tag.Upsert(Scope, scope), tag.Upsert(Resource, resource)} + // TODO fix the graph in the dashboard for blocked resources. + // TODO change bytes to iec + if evt.DeltaIn != 0 { - stats.RecordWithTags(ctx, tags, blockedResources.M(int64(evt.DeltaIn))) + stats.RecordWithTags(ctx, tags, blockedResources.M(int64(1))) } if evt.DeltaOut != 0 { - stats.RecordWithTags(ctx, tags, blockedResources.M(int64(evt.DeltaOut))) + stats.RecordWithTags(ctx, tags, blockedResources.M(int64(1))) } if evt.Delta != 0 { - stats.RecordWithTags(ctx, tags, blockedResources.M(evt.Delta)) + stats.RecordWithTags(ctx, tags, blockedResources.M(1)) } } } From 6d5898aa99d6282f6bfb9bef2bbf8ac5dbd28ebd Mon Sep 17 00:00:00 2001 From: Marco Munizaga Date: Thu, 23 Jun 2022 15:33:12 -0700 Subject: [PATCH 38/45] Use bytes rather than decbytes (power of 2) --- obs/grafana-dashboards/resource-manager.json | 8 ++++---- obs/stats.go | 3 --- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/obs/grafana-dashboards/resource-manager.json b/obs/grafana-dashboards/resource-manager.json index 66d390a..f2fabfe 100644 --- a/obs/grafana-dashboards/resource-manager.json +++ b/obs/grafana-dashboards/resource-manager.json @@ -1224,7 +1224,7 @@ } ] }, - "unit": "decbytes" + "unit": "bytes" }, "overrides": [] }, @@ -1311,7 +1311,7 @@ } ] }, - "unit": "decbytes" + "unit": "bytes" }, "overrides": [] }, @@ -1398,7 +1398,7 @@ } ] }, - "unit": "decbytes" + "unit": "bytes" }, "overrides": [] }, @@ -1485,7 +1485,7 @@ } ] }, - "unit": "decbytes" + "unit": "bytes" }, "overrides": [ { diff --git a/obs/stats.go b/obs/stats.go index 89dc5b5..262028b 100644 --- a/obs/stats.go +++ b/obs/stats.go @@ -329,9 +329,6 @@ func (r StatsTraceReporter) ConsumeEvent(evt rcmgr.TraceEvt) { tags := []tag.Mutator{tag.Upsert(Scope, scope), tag.Upsert(Resource, resource)} - // TODO fix the graph in the dashboard for blocked resources. - // TODO change bytes to iec - if evt.DeltaIn != 0 { stats.RecordWithTags(ctx, tags, blockedResources.M(int64(1))) } From e70da09acb61da6c67910f70ae393eb99139f741 Mon Sep 17 00:00:00 2001 From: Marco Munizaga Date: Mon, 27 Jun 2022 15:58:42 -0700 Subject: [PATCH 39/45] Unexport tag names, remove unused var --- obs/stats.go | 81 ++++++++++++++++++++++++---------------------------- 1 file changed, 37 insertions(+), 44 deletions(-) diff --git a/obs/stats.go b/obs/stats.go index 262028b..8fc7ec1 100644 --- a/obs/stats.go +++ b/obs/stats.go @@ -14,9 +14,6 @@ import ( var log = logging.Logger("rcmgrObs") var ( - systemOutboundConns = stats.Int64("system/outbound/conn", "Number of outbound Connections", stats.UnitDimensionless) - systemInboundConns = stats.Int64("system/inbound/conn", "Number of inbound Connections", stats.UnitDimensionless) - conns = stats.Int64("connections", "Number of Connections", stats.UnitDimensionless) peerConns = stats.Int64("peer/connections", "Number of connections this peer has", stats.UnitDimensionless) @@ -40,19 +37,15 @@ var ( ) var ( - LessThanEq, _ = tag.NewKey("le") - Direction, _ = tag.NewKey("dir") - Scope, _ = tag.NewKey("scope") - Service, _ = tag.NewKey("service") - Protocol, _ = tag.NewKey("protocol") - Resource, _ = tag.NewKey("resource") + directionTag, _ = tag.NewKey("dir") + scopeTag, _ = tag.NewKey("scope") + serviceTag, _ = tag.NewKey("service") + protocolTag, _ = tag.NewKey("protocol") + resourceTag, _ = tag.NewKey("resource") ) var ( - SystemOutboundConnsView = &view.View{Measure: systemOutboundConns, Aggregation: view.Sum()} - SystemInboundConnsView = &view.View{Measure: systemInboundConns, Aggregation: view.Sum()} - - ConnView = &view.View{Measure: conns, Aggregation: view.Sum(), TagKeys: []tag.Key{Direction, Scope}} + ConnView = &view.View{Measure: conns, Aggregation: view.Sum(), TagKeys: []tag.Key{directionTag, scopeTag}} oneTenThenExpDistribution = []float64{ 1.1, 2.1, 3.1, 4.1, 5.1, 6.1, 7.1, 8.1, 9.1, 10.1, 16.1, 32.1, 64.1, 128.1, 256.1, @@ -61,19 +54,19 @@ var ( PeerConnsView = &view.View{ Measure: peerConns, Aggregation: view.Distribution(oneTenThenExpDistribution...), - TagKeys: []tag.Key{Direction}, + TagKeys: []tag.Key{directionTag}, } PeerConnsNegativeView = &view.View{ Measure: peerConnsNegative, Aggregation: view.Distribution(oneTenThenExpDistribution...), - TagKeys: []tag.Key{Direction}, + TagKeys: []tag.Key{directionTag}, } - StreamView = &view.View{Measure: streams, Aggregation: view.Sum(), TagKeys: []tag.Key{Direction, Scope, Service, Protocol}} - PeerStreamsView = &view.View{Measure: peerStreams, Aggregation: view.Distribution(oneTenThenExpDistribution...), TagKeys: []tag.Key{Direction}} - PeerStreamNegativeView = &view.View{Measure: peerStreamsNegative, Aggregation: view.Distribution(oneTenThenExpDistribution...), TagKeys: []tag.Key{Direction}} + StreamView = &view.View{Measure: streams, Aggregation: view.Sum(), TagKeys: []tag.Key{directionTag, scopeTag, serviceTag, protocolTag}} + PeerStreamsView = &view.View{Measure: peerStreams, Aggregation: view.Distribution(oneTenThenExpDistribution...), TagKeys: []tag.Key{directionTag}} + PeerStreamNegativeView = &view.View{Measure: peerStreamsNegative, Aggregation: view.Distribution(oneTenThenExpDistribution...), TagKeys: []tag.Key{directionTag}} - MemoryView = &view.View{Measure: memory, Aggregation: view.Sum(), TagKeys: []tag.Key{Scope, Service, Protocol}} + MemoryView = &view.View{Measure: memory, Aggregation: view.Sum(), TagKeys: []tag.Key{scopeTag, serviceTag, protocolTag}} memDistribution = []float64{ 1 << 10, // 1KB @@ -106,12 +99,12 @@ var ( Aggregation: view.Distribution(memDistribution...), } - FDsView = &view.View{Measure: fds, Aggregation: view.Sum(), TagKeys: []tag.Key{Scope}} + FDsView = &view.View{Measure: fds, Aggregation: view.Sum(), TagKeys: []tag.Key{scopeTag}} BlockedResourcesView = &view.View{ Measure: blockedResources, Aggregation: view.Sum(), - TagKeys: []tag.Key{Scope, Resource}, + TagKeys: []tag.Key{scopeTag, resourceTag}, } ) @@ -156,10 +149,10 @@ func (r StatsTraceReporter) ConsumeEvent(evt rcmgr.TraceEvt) { peerStreamsOut := int64(evt.StreamsOut) if oldStreamsOut != peerStreamsOut { if oldStreamsOut != 0 { - stats.RecordWithTags(ctx, []tag.Mutator{tag.Upsert(Direction, "outbound")}, peerStreamsNegative.M(oldStreamsOut)) + stats.RecordWithTags(ctx, []tag.Mutator{tag.Upsert(directionTag, "outbound")}, peerStreamsNegative.M(oldStreamsOut)) } if peerStreamsOut != 0 { - stats.RecordWithTags(ctx, []tag.Mutator{tag.Upsert(Direction, "outbound")}, peerStreams.M(peerStreamsOut)) + stats.RecordWithTags(ctx, []tag.Mutator{tag.Upsert(directionTag, "outbound")}, peerStreams.M(peerStreamsOut)) } } @@ -167,20 +160,20 @@ func (r StatsTraceReporter) ConsumeEvent(evt rcmgr.TraceEvt) { peerStreamsIn := int64(evt.StreamsIn) if oldStreamsIn != peerStreamsIn { if oldStreamsIn != 0 { - stats.RecordWithTags(ctx, []tag.Mutator{tag.Upsert(Direction, "inbound")}, peerStreamsNegative.M(oldStreamsIn)) + stats.RecordWithTags(ctx, []tag.Mutator{tag.Upsert(directionTag, "inbound")}, peerStreamsNegative.M(oldStreamsIn)) } if peerStreamsIn != 0 { - stats.RecordWithTags(ctx, []tag.Mutator{tag.Upsert(Direction, "inbound")}, peerStreams.M(peerStreamsIn)) + stats.RecordWithTags(ctx, []tag.Mutator{tag.Upsert(directionTag, "inbound")}, peerStreams.M(peerStreamsIn)) } } } else { var tags []tag.Mutator if rcmgr.IsSystemScope(evt.Name) || rcmgr.IsTransientScope(evt.Name) { - tags = append(tags, tag.Upsert(Scope, evt.Name)) + tags = append(tags, tag.Upsert(scopeTag, evt.Name)) } else if svc := rcmgr.ParseServiceScopeName(evt.Name); svc != "" { - tags = append(tags, tag.Upsert(Scope, "service"), tag.Upsert(Service, svc)) + tags = append(tags, tag.Upsert(scopeTag, "service"), tag.Upsert(serviceTag, svc)) } else if proto := rcmgr.ParseProtocolScopeName(evt.Name); proto != "" { - tags = append(tags, tag.Upsert(Scope, "protocol"), tag.Upsert(Protocol, proto)) + tags = append(tags, tag.Upsert(scopeTag, "protocol"), tag.Upsert(protocolTag, proto)) } else { // Not measuring connscope, servicepeer and protocolpeer. Lots of data, and // you can use aggregated peer stats + service stats to infer @@ -191,7 +184,7 @@ func (r StatsTraceReporter) ConsumeEvent(evt rcmgr.TraceEvt) { if evt.DeltaOut != 0 { stats.RecordWithTags( ctx, - append([]tag.Mutator{tag.Upsert(Direction, "outbound")}, tags...), + append([]tag.Mutator{tag.Upsert(directionTag, "outbound")}, tags...), streams.M(int64(evt.DeltaOut)), ) } @@ -199,7 +192,7 @@ func (r StatsTraceReporter) ConsumeEvent(evt rcmgr.TraceEvt) { if evt.DeltaIn != 0 { stats.RecordWithTags( ctx, - append([]tag.Mutator{tag.Upsert(Direction, "inbound")}, tags...), + append([]tag.Mutator{tag.Upsert(directionTag, "inbound")}, tags...), streams.M(int64(evt.DeltaIn)), ) } @@ -217,10 +210,10 @@ func (r StatsTraceReporter) ConsumeEvent(evt rcmgr.TraceEvt) { connsOut := int64(evt.ConnsOut) if oldConnsOut != connsOut { if oldConnsOut != 0 { - stats.RecordWithTags(ctx, []tag.Mutator{tag.Upsert(Direction, "outbound")}, peerConnsNegative.M(oldConnsOut)) + stats.RecordWithTags(ctx, []tag.Mutator{tag.Upsert(directionTag, "outbound")}, peerConnsNegative.M(oldConnsOut)) } if connsOut != 0 { - stats.RecordWithTags(ctx, []tag.Mutator{tag.Upsert(Direction, "outbound")}, peerConns.M(connsOut)) + stats.RecordWithTags(ctx, []tag.Mutator{tag.Upsert(directionTag, "outbound")}, peerConns.M(connsOut)) } } @@ -228,16 +221,16 @@ func (r StatsTraceReporter) ConsumeEvent(evt rcmgr.TraceEvt) { connsIn := int64(evt.ConnsIn) if oldConnsIn != connsIn { if oldConnsIn != 0 { - stats.RecordWithTags(ctx, []tag.Mutator{tag.Upsert(Direction, "inbound")}, peerConnsNegative.M(oldConnsIn)) + stats.RecordWithTags(ctx, []tag.Mutator{tag.Upsert(directionTag, "inbound")}, peerConnsNegative.M(oldConnsIn)) } if connsIn != 0 { - stats.RecordWithTags(ctx, []tag.Mutator{tag.Upsert(Direction, "inbound")}, peerConns.M(connsIn)) + stats.RecordWithTags(ctx, []tag.Mutator{tag.Upsert(directionTag, "inbound")}, peerConns.M(connsIn)) } } } else { var tags []tag.Mutator if rcmgr.IsSystemScope(evt.Name) || rcmgr.IsTransientScope(evt.Name) { - tags = append(tags, tag.Upsert(Scope, evt.Name)) + tags = append(tags, tag.Upsert(scopeTag, evt.Name)) } else if rcmgr.IsConnScope(evt.Name) { // Not measuring this. I don't think it's useful. break @@ -249,7 +242,7 @@ func (r StatsTraceReporter) ConsumeEvent(evt rcmgr.TraceEvt) { if evt.DeltaOut != 0 { stats.RecordWithTags( ctx, - append([]tag.Mutator{tag.Upsert(Direction, "outbound")}, tags...), + append([]tag.Mutator{tag.Upsert(directionTag, "outbound")}, tags...), conns.M(int64(evt.DeltaOut)), ) } @@ -257,7 +250,7 @@ func (r StatsTraceReporter) ConsumeEvent(evt rcmgr.TraceEvt) { if evt.DeltaIn != 0 { stats.RecordWithTags( ctx, - append([]tag.Mutator{tag.Upsert(Direction, "inbound")}, tags...), + append([]tag.Mutator{tag.Upsert(directionTag, "inbound")}, tags...), conns.M(int64(evt.DeltaIn)), ) } @@ -295,11 +288,11 @@ func (r StatsTraceReporter) ConsumeEvent(evt rcmgr.TraceEvt) { } else { var tags []tag.Mutator if rcmgr.IsSystemScope(evt.Name) || rcmgr.IsTransientScope(evt.Name) { - tags = append(tags, tag.Upsert(Scope, evt.Name)) + tags = append(tags, tag.Upsert(scopeTag, evt.Name)) } else if svc := rcmgr.ParseServiceScopeName(evt.Name); svc != "" { - tags = append(tags, tag.Upsert(Scope, "service"), tag.Upsert(Service, svc)) + tags = append(tags, tag.Upsert(scopeTag, "service"), tag.Upsert(serviceTag, svc)) } else if proto := rcmgr.ParseProtocolScopeName(evt.Name); proto != "" { - tags = append(tags, tag.Upsert(Scope, "protocol"), tag.Upsert(Protocol, proto)) + tags = append(tags, tag.Upsert(scopeTag, "protocol"), tag.Upsert(protocolTag, proto)) } else { // Not measuring connscope, servicepeer and protocolpeer. Lots of data, and // you can use aggregated peer stats + service stats to infer @@ -322,12 +315,12 @@ func (r StatsTraceReporter) ConsumeEvent(evt rcmgr.TraceEvt) { resource = "memory" } - // Only the top scope. We don't want to get the peerid here. - scope := strings.SplitN(evt.Name, ":", 2)[0] + // Only the top scopeName. We don't want to get the peerid here. + scopeName := strings.SplitN(evt.Name, ":", 2)[0] // Drop the connection or stream id - scope = strings.SplitN(scope, "-", 2)[0] + scopeName = strings.SplitN(scopeName, "-", 2)[0] - tags := []tag.Mutator{tag.Upsert(Scope, scope), tag.Upsert(Resource, resource)} + tags := []tag.Mutator{tag.Upsert(scopeTag, scopeName), tag.Upsert(resourceTag, resource)} if evt.DeltaIn != 0 { stats.RecordWithTags(ctx, tags, blockedResources.M(int64(1))) From 5abae10710dac485368a4bedd43e9b7e7b53cbf4 Mon Sep 17 00:00:00 2001 From: Marco Munizaga Date: Wed, 29 Jun 2022 21:03:22 -0700 Subject: [PATCH 40/45] Filter out spans --- obs/stats.go | 4 ++-- rcmgr.go | 21 +++++++++++---------- scope.go | 7 +++++++ 3 files changed, 20 insertions(+), 12 deletions(-) diff --git a/obs/stats.go b/obs/stats.go index 8fc7ec1..725b588 100644 --- a/obs/stats.go +++ b/obs/stats.go @@ -235,8 +235,8 @@ func (r StatsTraceReporter) ConsumeEvent(evt rcmgr.TraceEvt) { // Not measuring this. I don't think it's useful. break } else { - // There shouldn't be anything here. But we keep going so the metrics will tell us if we're wrong (scope="") - log.Debugf("unexpected event in stats: %s", evt.Name) + // This could be a span + break } if evt.DeltaOut != 0 { diff --git a/rcmgr.go b/rcmgr.go index 5d64f7d..99c339c 100644 --- a/rcmgr.go +++ b/rcmgr.go @@ -478,11 +478,12 @@ func newStreamScope(dir network.Direction, limit Limit, peer *peerScope, rcmgr * } func IsSystemScope(name string) bool { - return strings.HasPrefix(name, "system") + return name == "system" } func IsTransientScope(name string) bool { return strings.HasPrefix(name, "transient") + return name == "transient" } func streamScopeName(streamId int64) string { @@ -490,7 +491,7 @@ func streamScopeName(streamId int64) string { } func IsStreamScope(name string) bool { - return strings.HasPrefix(name, "stream-") + return strings.HasPrefix(name, "stream-") && !IsSpan(name) } func connScopeName(streamId int64) string { @@ -498,7 +499,7 @@ func connScopeName(streamId int64) string { } func IsConnScope(name string) bool { - return strings.HasPrefix(name, "conn-") + return strings.HasPrefix(name, "conn-") && !IsSpan(name) } func peerScopeName(p peer.ID) string { @@ -507,16 +508,16 @@ func peerScopeName(p peer.ID) string { // ParsePeerScopeName returns "" if name is not a peerScopeName func ParsePeerScopeName(name string) peer.ID { - if !strings.HasPrefix(name, "peer:") { - return peer.ID("") + if !strings.HasPrefix(name, "peer:") || IsSpan(name) { + return "" } parts := strings.SplitN(name, "peer:", 2) if len(parts) != 2 { - return peer.ID("") + return "" } p, err := peer.Decode(parts[1]) if err != nil { - return peer.ID("") + return "" } return p } @@ -524,14 +525,14 @@ func ParsePeerScopeName(name string) peer.ID { // ParseServiceScopeName returns the service name if name is a serviceScopeName. // Otherwise returns "" func ParseServiceScopeName(name string) string { - if strings.HasPrefix(name, "service:") { + if strings.HasPrefix(name, "service:") && !IsSpan(name) { if strings.Contains(name, "peer:") { // This is a service peer scope return "" } parts := strings.SplitN(name, ":", 2) if len(parts) != 2 { - return ("") + return "" } return parts[1] @@ -542,7 +543,7 @@ func ParseServiceScopeName(name string) string { // ParseProtocolScopeName returns the service name if name is a serviceScopeName. // Otherwise returns "" func ParseProtocolScopeName(name string) string { - if strings.HasPrefix(name, "protocol:") { + if strings.HasPrefix(name, "protocol:") && !IsSpan(name) { if strings.Contains(name, "peer:") { // This is a protocol peer scope return "" diff --git a/scope.go b/scope.go index 2dd9707..7205b2e 100644 --- a/scope.go +++ b/scope.go @@ -2,6 +2,7 @@ package rcmgr import ( "fmt" + "strings" "sync" "github.com/libp2p/go-libp2p-core/network" @@ -73,6 +74,12 @@ func newResourceScopeSpan(owner *resourceScope, id int) *resourceScope { return r } +// IsSpan will return true if this name was created by newResourceScopeSpan +func IsSpan(name string) bool { + return strings.Contains(name, ".span-") + return name == "transient" +} + // Resources implementation func (rc *resources) checkMemory(rsvp int64, prio uint8) error { // overflow check; this also has the side effect that we cannot reserve negative memory. From ac716c1bf306ffc088d39e15b48f8b6edf12f54e Mon Sep 17 00:00:00 2001 From: Marco Munizaga Date: Wed, 29 Jun 2022 21:06:43 -0700 Subject: [PATCH 41/45] Rewrite constants to be more obvious --- obs/stats.go | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/obs/stats.go b/obs/stats.go index 725b588..a11f0fc 100644 --- a/obs/stats.go +++ b/obs/stats.go @@ -69,16 +69,16 @@ var ( MemoryView = &view.View{Measure: memory, Aggregation: view.Sum(), TagKeys: []tag.Key{scopeTag, serviceTag, protocolTag}} memDistribution = []float64{ - 1 << 10, // 1KB - 1 << 12, // 4KB - 1 << 15, // 32KB - 1 << 20, // 1MB - 1 << 25, // 32MB - 1 << 28, // 256MB - 1 << 29, // 512MB - 1 << 30, // 1GB - 1 << 31, // 2GB - 1 << 32, // 4GB + 1 << 10, // 1KB + 4 << 10, // 4KB + 32 << 10, // 32KB + 1 << 20, // 1MB + 32 << 20, // 32MB + 256 << 20, // 256MB + 512 << 20, // 512MB + 1 << 30, // 1GB + 2 << 30, // 2GB + 4 << 30, // 4GB } PeerMemoryView = &view.View{ Measure: peerMemory, From 662c09420118fe63e298d85b7d38852208b93c68 Mon Sep 17 00:00:00 2001 From: Marco Munizaga Date: Wed, 29 Jun 2022 21:09:48 -0700 Subject: [PATCH 42/45] Don't add reporter twice --- trace.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trace.go b/trace.go index 464e8cf..886f89f 100644 --- a/trace.go +++ b/trace.go @@ -47,7 +47,7 @@ func WithTrace(path string) Option { func WithTraceReporter(reporter TraceReporter) Option { return func(r *resourceManager) error { if r.trace == nil { - r.trace = &trace{reporters: []TraceReporter{reporter}} + r.trace = &trace{} } r.trace.reporters = append(r.trace.reporters, reporter) return nil From ccb0c4dd90aeafd8957edebabfcf174f657ab3b2 Mon Sep 17 00:00:00 2001 From: Marco Munizaga Date: Fri, 1 Jul 2022 08:36:18 -0700 Subject: [PATCH 43/45] Update help text --- obs/stats.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/obs/stats.go b/obs/stats.go index a11f0fc..5030d06 100644 --- a/obs/stats.go +++ b/obs/stats.go @@ -17,19 +17,19 @@ var ( conns = stats.Int64("connections", "Number of Connections", stats.UnitDimensionless) peerConns = stats.Int64("peer/connections", "Number of connections this peer has", stats.UnitDimensionless) - peerConnsNegative = stats.Int64("peer/connections_negative", "Number of connections this peer had", stats.UnitDimensionless) + peerConnsNegative = stats.Int64("peer/connections_negative", "Number of connections this peer had. This is used to get the current connection number per peer histogram by subtracting this from the peer/connections histogram", stats.UnitDimensionless) streams = stats.Int64("streams", "Number of Streams", stats.UnitDimensionless) peerStreams = stats.Int64("peer/streams", "Number of streams this peer has", stats.UnitDimensionless) - peerStreamsNegative = stats.Int64("peer/streams_negative", "Number of streams this peer had", stats.UnitDimensionless) + peerStreamsNegative = stats.Int64("peer/streams_negative", "Number of streams this peer had. This is used to get the current streams number per peer histogram by subtracting this from the peer/streams histogram", stats.UnitDimensionless) memory = stats.Int64("memory", "Amount of memory reserved as reported to the Resource Manager", stats.UnitDimensionless) peerMemory = stats.Int64("peer/memory", "Amount of memory currently reseved for peer", stats.UnitDimensionless) - peerMemoryNegative = stats.Int64("peer/memory_negative", "Amount of memory previously reseved for peer", stats.UnitDimensionless) + peerMemoryNegative = stats.Int64("peer/memory_negative", "Amount of memory previously reseved for peer. This is used to get the current memory per peer histogram by subtracting this from the peer/memory histogram", stats.UnitDimensionless) connMemory = stats.Int64("conn/memory", "Amount of memory currently reseved for the connection", stats.UnitDimensionless) - connMemoryNegative = stats.Int64("conn/memory_negative", "Amount of memory previously reseved for the connection", stats.UnitDimensionless) + connMemoryNegative = stats.Int64("conn/memory_negative", "Amount of memory previously reseved for the connection. This is used to get the current memory per connection histogram by subtracting this from the conn/memory histogram", stats.UnitDimensionless) fds = stats.Int64("fds", "Number of fds as reported to the Resource Manager", stats.UnitDimensionless) From 0f3804229e36e66e459e35ea9bb69d14b017a692 Mon Sep 17 00:00:00 2001 From: Marco Munizaga Date: Fri, 1 Jul 2022 08:53:38 -0700 Subject: [PATCH 44/45] Remove unused var --- obs/stats.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/obs/stats.go b/obs/stats.go index 5030d06..4ba2bf4 100644 --- a/obs/stats.go +++ b/obs/stats.go @@ -4,15 +4,12 @@ import ( "context" "strings" - logging "github.com/ipfs/go-log/v2" rcmgr "github.com/libp2p/go-libp2p-resource-manager" "go.opencensus.io/stats" "go.opencensus.io/stats/view" "go.opencensus.io/tag" ) -var log = logging.Logger("rcmgrObs") - var ( conns = stats.Int64("connections", "Number of Connections", stats.UnitDimensionless) From 836d53a98d66a9c33a9c66df0e89ceb274684c91 Mon Sep 17 00:00:00 2001 From: Marco Munizaga Date: Fri, 1 Jul 2022 09:04:31 -0700 Subject: [PATCH 45/45] Fix go vet issue --- rcmgr.go | 1 - scope.go | 1 - 2 files changed, 2 deletions(-) diff --git a/rcmgr.go b/rcmgr.go index 99c339c..a9fbdae 100644 --- a/rcmgr.go +++ b/rcmgr.go @@ -482,7 +482,6 @@ func IsSystemScope(name string) bool { } func IsTransientScope(name string) bool { - return strings.HasPrefix(name, "transient") return name == "transient" } diff --git a/scope.go b/scope.go index 7205b2e..2190527 100644 --- a/scope.go +++ b/scope.go @@ -77,7 +77,6 @@ func newResourceScopeSpan(owner *resourceScope, id int) *resourceScope { // IsSpan will return true if this name was created by newResourceScopeSpan func IsSpan(name string) bool { return strings.Contains(name, ".span-") - return name == "transient" } // Resources implementation