diff --git a/.github/workflows/link-check.yml b/.github/workflows/link-check.yml new file mode 100644 index 0000000000..87c895d44c --- /dev/null +++ b/.github/workflows/link-check.yml @@ -0,0 +1,17 @@ +name: Markdown Link Checking +on: + pull_request: + push: + branches: + - "master" + +jobs: + check-links: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: gaurav-nelson/github-action-markdown-link-check@v1 + with: + use-quiet-mode: 'yes' # show only broken links + use-verbose-mode: 'yes' + config-file: .github/workflows/markdown-links-config.json # for removing any false positives diff --git a/.github/workflows/markdown-links-config.json b/.github/workflows/markdown-links-config.json new file mode 100644 index 0000000000..505831a198 --- /dev/null +++ b/.github/workflows/markdown-links-config.json @@ -0,0 +1,22 @@ +{ + "ignorePatterns": [ + { + "pattern": "^http://localhost" + }, + { + "pattern": "^https://twitter.com/" + }, + { + "pattern": "^https://opensource.org/" + } + ], + "aliveStatusCodes": [200], + "httpHeaders": [ + { + "urls": ["https://docs.github.com/"], + "headers": { + "Accept-Encoding": "*" + } + } + ] +} \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 2306f0ff89..b6ab14c177 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -195,8 +195,8 @@ Fix some test-utils used by https://github.com/libp2p/go-libp2p-kad-dht ### Metrics We've started instrumenting the entire stack. In this release, we're adding metrics for: -* the swarm: tracking incoming and outgoing connections, transports, security protocols and stream multiplexers in use: (https://github.com/libp2p/go-libp2p/blob/master/p2p/net/swarm/grafana-dashboards/swarm.json) -* the event bus: tracking how different events are propagated through the stack and to external consumers (https://github.com/libp2p/go-libp2p/blob/master/p2p/host/eventbus/grafana-dashboards/eventbus.json) +* the swarm: tracking incoming and outgoing connections, transports, security protocols and stream multiplexers in use: (https://github.com/libp2p/go-libp2p/blob/master/dashboards/swarm/swarm.json) +* the event bus: tracking how different events are propagated through the stack and to external consumers (https://github.com/libp2p/go-libp2p/blob/master/dashboards/eventbus/eventbus.json) Our metrics effort is still ongoing, see https://github.com/libp2p/go-libp2p/issues/1356 for progress. We'll add metrics and dashboards for more libp2p components in a future release. diff --git a/config/config.go b/config/config.go index 6bc80c96d4..b529ec1c51 100644 --- a/config/config.go +++ b/config/config.go @@ -126,6 +126,8 @@ type Config struct { PrometheusRegisterer prometheus.Registerer DialRanker network.DialRanker + + SwarmOpts []swarm.Option } func (cfg *Config) makeSwarm(eventBus event.Bus, enableMetrics bool) (*swarm.Swarm, error) { @@ -160,7 +162,7 @@ func (cfg *Config) makeSwarm(eventBus event.Bus, enableMetrics bool) (*swarm.Swa return nil, err } - opts := make([]swarm.Option, 0, 6) + opts := cfg.SwarmOpts if cfg.Reporter != nil { opts = append(opts, swarm.WithMetrics(cfg.Reporter)) } @@ -176,11 +178,13 @@ func (cfg *Config) makeSwarm(eventBus event.Bus, enableMetrics bool) (*swarm.Swa if cfg.MultiaddrResolver != nil { opts = append(opts, swarm.WithMultiaddrResolver(cfg.MultiaddrResolver)) } + dialRanker := cfg.DialRanker if dialRanker == nil { - dialRanker = swarm.NoDelayDialRanker + dialRanker = swarm.DefaultDialRanker } opts = append(opts, swarm.WithDialRanker(dialRanker)) + if enableMetrics { opts = append(opts, swarm.WithMetricsTracer(swarm.NewMetricsTracer(swarm.WithRegisterer(cfg.PrometheusRegisterer)))) diff --git a/core/network/network.go b/core/network/network.go index 0508b871b3..66b0a1cd34 100644 --- a/core/network/network.go +++ b/core/network/network.go @@ -6,13 +6,10 @@ package network import ( - "bytes" "context" "io" "time" - "golang.org/x/exp/slices" - "github.com/libp2p/go-libp2p/core/peer" "github.com/libp2p/go-libp2p/core/peerstore" @@ -199,24 +196,3 @@ type AddrDelay struct { // DialRanker provides a schedule of dialing the provided addresses type DialRanker func([]ma.Multiaddr) []AddrDelay - -// DedupAddrs deduplicates addresses in place, leave only unique addresses. -// It doesn't allocate. -func DedupAddrs(addrs []ma.Multiaddr) []ma.Multiaddr { - if len(addrs) == 0 { - return addrs - } - // Use the new slices package here, as the sort function doesn't allocate (sort.Slice does). - slices.SortFunc(addrs, func(a, b ma.Multiaddr) bool { return bytes.Compare(a.Bytes(), b.Bytes()) < 0 }) - idx := 1 - for i := 1; i < len(addrs); i++ { - if !addrs[i-1].Equal(addrs[i]) { - addrs[idx] = addrs[i] - idx++ - } - } - for i := idx; i < len(addrs); i++ { - addrs[i] = nil - } - return addrs[:idx] -} diff --git a/core/network/network_test.go b/core/network/network_test.go deleted file mode 100644 index 472acb0341..0000000000 --- a/core/network/network_test.go +++ /dev/null @@ -1,59 +0,0 @@ -package network - -import ( - "fmt" - "math" - "math/rand" - "testing" - - ma "github.com/multiformats/go-multiaddr" - - "github.com/stretchr/testify/require" -) - -func TestDedupAddrs(t *testing.T) { - tcpAddr := ma.StringCast("/ip4/127.0.0.1/tcp/1234") - quicAddr := ma.StringCast("/ip4/127.0.0.1/udp/1234/quic-v1") - wsAddr := ma.StringCast("/ip4/127.0.0.1/tcp/1234/ws") - - type testcase struct { - in, out []ma.Multiaddr - } - - for i, tc := range []testcase{ - {in: nil, out: nil}, - {in: []ma.Multiaddr{tcpAddr}, out: []ma.Multiaddr{tcpAddr}}, - {in: []ma.Multiaddr{tcpAddr, tcpAddr, tcpAddr}, out: []ma.Multiaddr{tcpAddr}}, - {in: []ma.Multiaddr{tcpAddr, quicAddr, tcpAddr}, out: []ma.Multiaddr{tcpAddr, quicAddr}}, - {in: []ma.Multiaddr{tcpAddr, quicAddr, wsAddr}, out: []ma.Multiaddr{tcpAddr, quicAddr, wsAddr}}, - } { - tc := tc - t.Run(fmt.Sprintf("test %d", i), func(t *testing.T) { - deduped := DedupAddrs(tc.in) - for _, a := range tc.out { - require.Contains(t, deduped, a) - } - }) - } -} - -func BenchmarkDedupAddrs(b *testing.B) { - b.ReportAllocs() - var addrs []ma.Multiaddr - r := rand.New(rand.NewSource(1234)) - for i := 0; i < 100; i++ { - tcpAddr := ma.StringCast(fmt.Sprintf("/ip4/127.0.0.1/tcp/%d", r.Intn(math.MaxUint16))) - quicAddr := ma.StringCast(fmt.Sprintf("/ip4/127.0.0.1/udp/%d/quic-v1", r.Intn(math.MaxUint16))) - wsAddr := ma.StringCast(fmt.Sprintf("/ip4/127.0.0.1/tcp/%d/ws", r.Intn(math.MaxUint16))) - addrs = append(addrs, tcpAddr, tcpAddr, quicAddr, quicAddr, wsAddr) - } - for _, sz := range []int{10, 20, 30, 50, 100} { - b.Run(fmt.Sprintf("%d", sz), func(b *testing.B) { - items := make([]ma.Multiaddr, sz) - for i := 0; i < b.N; i++ { - copy(items, addrs[:sz]) - DedupAddrs(items) - } - }) - } -} diff --git a/dashboards/resource-manager/README.md b/dashboards/resource-manager/README.md index 8697769d3a..cded715679 100644 --- a/dashboards/resource-manager/README.md +++ b/dashboards/resource-manager/README.md @@ -5,28 +5,9 @@ import follow the Grafana docs [here](https://grafana.com/docs/grafana/latest/da ## Setup -To make sure you're emitting the metrics you'll have to create the Resource -Manager with a StatsTraceReporter. By default metrics will be sent to -prometheus.DefaultRegisterer. To use a different Registerer use the libp2p -option libp2p.PrometheusRegisterer. For example: - -``` go -import ( - // ... - rcmgr "github.com/libp2p/go-libp2p/p2p/host/resource-manager" - - "github.com/prometheus/client_golang/prometheus" -) - - func SetupResourceManager() (network.ResourceManager, error) { - str, err := rcmgr.NewStatsTraceReporter() - if err != nil { - return nil, err - } - - return rcmgr.NewResourceManager(limiter, rcmgr.WithTraceReporter(str)) - } -``` +Metrics are enabled by default. By default, metrics will be sent to +`prometheus.DefaultRegisterer`. To use a different Registerer use the libp2p +option `libp2p.PrometheusRegisterer`. ## Updating Dashboard json diff --git a/examples/multipro/README.md b/examples/multipro/README.md index db032848fb..6fb0b2e967 100644 --- a/examples/multipro/README.md +++ b/examples/multipro/README.md @@ -1,7 +1,7 @@ # Protocol Multiplexing using rpc-style protobufs with libp2p This example shows how to use protobufs to encode and transmit information between libp2p hosts using libp2p Streams. -This example expects that you are already familiar with the [echo example](https://github.com/libp2p/go-libp2p-examples/tree/master/echo). +This example expects that you are already familiar with the [echo example](https://github.com/libp2p/go-libp2p/tree/master/examples/echo). ## Build diff --git a/go.mod b/go.mod index 898a4e8243..4f67444695 100644 --- a/go.mod +++ b/go.mod @@ -36,7 +36,7 @@ require ( github.com/minio/sha256-simd v1.0.1 github.com/mr-tron/base58 v1.2.0 github.com/multiformats/go-base32 v0.1.0 - github.com/multiformats/go-multiaddr v0.9.0 + github.com/multiformats/go-multiaddr v0.10.1 github.com/multiformats/go-multiaddr-dns v0.3.1 github.com/multiformats/go-multiaddr-fmt v0.1.0 github.com/multiformats/go-multibase v0.2.0 @@ -54,7 +54,7 @@ require ( go.uber.org/fx v1.19.2 go.uber.org/goleak v1.1.12 golang.org/x/crypto v0.7.0 - golang.org/x/exp v0.0.0-20230321023759-10a507213a29 + golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df golang.org/x/sync v0.2.0 golang.org/x/sys v0.8.0 golang.org/x/tools v0.9.1 @@ -110,7 +110,7 @@ require ( go.uber.org/dig v1.17.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.24.0 // indirect - golang.org/x/mod v0.10.0 // indirect + golang.org/x/mod v0.11.0 // indirect golang.org/x/net v0.10.0 // indirect golang.org/x/text v0.9.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index 149299576e..0c7c595c35 100644 --- a/go.sum +++ b/go.sum @@ -353,8 +353,8 @@ github.com/multiformats/go-base36 v0.2.0 h1:lFsAbNOGeKtuKozrtBsAkSVhv1p9D0/qedU9 github.com/multiformats/go-base36 v0.2.0/go.mod h1:qvnKE++v+2MWCfePClUEjE78Z7P2a1UV0xHgWc0hkp4= github.com/multiformats/go-multiaddr v0.1.1/go.mod h1:aMKBKNEYmzmDmxfX88/vz+J5IU55txyt0p4aiWVohjo= github.com/multiformats/go-multiaddr v0.2.0/go.mod h1:0nO36NvPpyV4QzvTLi/lafl2y95ncPj0vFwVF6k6wJ4= -github.com/multiformats/go-multiaddr v0.9.0 h1:3h4V1LHIk5w4hJHekMKWALPXErDfz/sggzwC/NcqbDQ= -github.com/multiformats/go-multiaddr v0.9.0/go.mod h1:mI67Lb1EeTOYb8GQfL/7wpIZwc46ElrvzhYnoJOmTT0= +github.com/multiformats/go-multiaddr v0.10.1 h1:HghtFrWyZEPrpTvgAMFJi6gFdgHfs2cb0pyfDsk+lqU= +github.com/multiformats/go-multiaddr v0.10.1/go.mod h1:jLEZsA61rwWNZQTHHnqq2HNa+4os/Hz54eqiRnsRqYQ= github.com/multiformats/go-multiaddr-dns v0.3.1 h1:QgQgR+LQVt3NPTjbrLLpsaT2ufAA2y0Mkk+QRVJbW3A= github.com/multiformats/go-multiaddr-dns v0.3.1/go.mod h1:G/245BRQ6FJGmryJCrOuTdB37AMA5AMOVuO6NY3JwTk= github.com/multiformats/go-multiaddr-fmt v0.1.0 h1:WLEFClPycPkp4fnIzoFoV9FVd49/eQsuaL3/CWe167E= @@ -563,8 +563,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/exp v0.0.0-20230321023759-10a507213a29 h1:ooxPy7fPvB4kwsA2h+iBNHkAbp/4JxTSwCmvdjEYmug= -golang.org/x/exp v0.0.0-20230321023759-10a507213a29/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= +golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df h1:UA2aFVmmsIlefxMk29Dp2juaUSth8Pyn3Tq5Y5mJGME= +golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -588,8 +588,8 @@ golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.10.0 h1:lFO9qtOdlre5W1jxS3r/4szv2/6iXxScdzjoBMXNhYk= -golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.11.0 h1:bUO06HqtnRcc/7l71XBe4WcqTZ+3AH1J59zWDDwLKgU= +golang.org/x/mod v0.11.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= diff --git a/options.go b/options.go index a124a2e27b..3e072b950c 100644 --- a/options.go +++ b/options.go @@ -23,6 +23,7 @@ import ( "github.com/libp2p/go-libp2p/core/transport" "github.com/libp2p/go-libp2p/p2p/host/autorelay" bhost "github.com/libp2p/go-libp2p/p2p/host/basic" + "github.com/libp2p/go-libp2p/p2p/net/swarm" tptu "github.com/libp2p/go-libp2p/p2p/net/upgrader" relayv2 "github.com/libp2p/go-libp2p/p2p/protocol/circuitv2/relay" "github.com/libp2p/go-libp2p/p2p/protocol/holepunch" @@ -587,3 +588,11 @@ func DialRanker(d network.DialRanker) Option { return nil } } + +// SwarmOpts configures libp2p to use swarm with opts +func SwarmOpts(opts ...swarm.Option) Option { + return func(cfg *Config) error { + cfg.SwarmOpts = opts + return nil + } +} diff --git a/p2p/host/autonat/options.go b/p2p/host/autonat/options.go index 8e653f8163..dec62c5f1d 100644 --- a/p2p/host/autonat/options.go +++ b/p2p/host/autonat/options.go @@ -91,9 +91,9 @@ func UsingAddresses(addrFunc AddrFunc) Option { } } -// WithSchedule configures how agressively probes will be made to verify the +// WithSchedule configures how aggressively probes will be made to verify the // address of the host. retryInterval indicates how often probes should be made -// when the host lacks confident about its address, while refresh interval +// when the host lacks confidence about its address, while refreshInterval // is the schedule of periodic probes when the host believes it knows its // steady-state reachability. func WithSchedule(retryInterval, refreshInterval time.Duration) Option { diff --git a/p2p/host/basic/basic_host.go b/p2p/host/basic/basic_host.go index 998a28dd13..89f5d28db9 100644 --- a/p2p/host/basic/basic_host.go +++ b/p2p/host/basic/basic_host.go @@ -841,7 +841,7 @@ func (h *BasicHost) AllAddrs() []ma.Multiaddr { finalAddrs = append(finalAddrs, resolved...) } - finalAddrs = network.DedupAddrs(finalAddrs) + finalAddrs = ma.Unique(finalAddrs) // use nat mappings if we have them if h.natmgr != nil && h.natmgr.HasDiscoveredNAT() { @@ -910,7 +910,7 @@ func (h *BasicHost) AllAddrs() []ma.Multiaddr { } finalAddrs = append(finalAddrs, observedAddrs...) } - finalAddrs = network.DedupAddrs(finalAddrs) + finalAddrs = ma.Unique(finalAddrs) finalAddrs = inferWebtransportAddrsFromQuic(finalAddrs) return finalAddrs diff --git a/p2p/host/resource-manager/README.md b/p2p/host/resource-manager/README.md index 9533eadb04..9371832c9a 100644 --- a/p2p/host/resource-manager/README.md +++ b/p2p/host/resource-manager/README.md @@ -47,15 +47,10 @@ limits := cfg.Build(scaledDefaultLimits) // The resource manager expects a limiter, se we create one from our limits. limiter := rcmgr.NewFixedLimiter(limits) -// (Optional if you want metrics) -rcmgr.MustRegisterWith(prometheus.DefaultRegisterer) -str, err := rcmgr.NewStatsTraceReporter() -if err != nil { - panic(err) -} - +// Metrics are enabled by default. If you want to disable metrics, use the +// WithMetricsDisabled option // Initialize the resource manager -rm, err := rcmgr.NewResourceManager(limiter, rcmgr.WithTraceReporter(str)) +rm, err := rcmgr.NewResourceManager(limiter, rcmgr.WithMetricsDisabled()) if err != nil { panic(err) } diff --git a/p2p/host/resource-manager/rcmgr.go b/p2p/host/resource-manager/rcmgr.go index 7f15bb7685..188a171f56 100644 --- a/p2p/host/resource-manager/rcmgr.go +++ b/p2p/host/resource-manager/rcmgr.go @@ -20,8 +20,9 @@ var log = logging.Logger("rcmgr") type resourceManager struct { limits Limiter - trace *trace - metrics *metrics + trace *trace + metrics *metrics + disableMetrics bool allowlist *Allowlist @@ -141,6 +142,28 @@ func NewResourceManager(limits Limiter, opts ...Option) (network.ResourceManager } } + if !r.disableMetrics { + var sr TraceReporter + sr, err := NewStatsTraceReporter() + if err != nil { + log.Errorf("failed to initialise StatsTraceReporter %s", err) + } else { + if r.trace == nil { + r.trace = &trace{} + } + found := false + for _, rep := range r.trace.reporters { + if rep == sr { + found = true + break + } + } + if !found { + r.trace.reporters = append(r.trace.reporters, sr) + } + } + } + if err := r.trace.Start(limits); err != nil { return nil, err } diff --git a/p2p/host/resource-manager/stats.go b/p2p/host/resource-manager/stats.go index b9f90b8449..fd0772948a 100644 --- a/p2p/host/resource-manager/stats.go +++ b/p2p/host/resource-manager/stats.go @@ -160,6 +160,13 @@ func MustRegisterWith(reg prometheus.Registerer) { ) } +func WithMetricsDisabled() Option { + return func(r *resourceManager) error { + r.disableMetrics = true + return nil + } +} + // StatsTraceReporter reports stats on the resource manager using its traces. type StatsTraceReporter struct{} diff --git a/p2p/net/mock/interface.go b/p2p/net/mock/interface.go index d89342b009..acb2563500 100644 --- a/p2p/net/mock/interface.go +++ b/p2p/net/mock/interface.go @@ -10,6 +10,7 @@ import ( "io" "time" + "github.com/libp2p/go-libp2p/core/connmgr" ic "github.com/libp2p/go-libp2p/core/crypto" "github.com/libp2p/go-libp2p/core/host" "github.com/libp2p/go-libp2p/core/network" @@ -19,14 +20,24 @@ import ( ma "github.com/multiformats/go-multiaddr" ) +type PeerOptions struct { + // ps is the Peerstore to use when adding peer. If nil, a default peerstore will be created. + ps peerstore.Peerstore + + // gater is the ConnectionGater to use when adding a peer. If nil, no connection gater will be used. + gater connmgr.ConnectionGater +} + type Mocknet interface { // GenPeer generates a peer and its network.Network in the Mocknet GenPeer() (host.Host, error) + GenPeerWithOptions(PeerOptions) (host.Host, error) // AddPeer adds an existing peer. we need both a privkey and addr. // ID is derived from PrivKey AddPeer(ic.PrivKey, ma.Multiaddr) (host.Host, error) AddPeerWithPeerstore(peer.ID, peerstore.Peerstore) (host.Host, error) + AddPeerWithOptions(peer.ID, PeerOptions) (host.Host, error) // retrieve things (with randomized iteration order) Peers() []peer.ID diff --git a/p2p/net/mock/mock_net.go b/p2p/net/mock/mock_net.go index cde4052369..43294d4a54 100644 --- a/p2p/net/mock/mock_net.go +++ b/p2p/net/mock/mock_net.go @@ -64,6 +64,13 @@ func (mn *mocknet) Close() error { } func (mn *mocknet) GenPeer() (host.Host, error) { + return mn.GenPeerWithOptions(PeerOptions{}) +} + +func (mn *mocknet) GenPeerWithOptions(opts PeerOptions) (host.Host, error) { + if err := mn.addDefaults(&opts); err != nil { + return nil, err + } sk, _, err := ic.GenerateECDSAKeyPair(rand.Reader) if err != nil { return nil, err @@ -83,7 +90,20 @@ func (mn *mocknet) GenPeer() (host.Host, error) { return nil, fmt.Errorf("failed to create test multiaddr: %s", err) } - h, err := mn.AddPeer(sk, a) + var ps peerstore.Peerstore + if opts.ps == nil { + ps, err = pstoremem.NewPeerstore() + if err != nil { + return nil, err + } + } else { + ps = opts.ps + } + p, err := mn.updatePeerstore(sk, a, ps) + if err != nil { + return nil, err + } + h, err := mn.AddPeerWithOptions(p, opts) if err != nil { return nil, err } @@ -92,36 +112,39 @@ func (mn *mocknet) GenPeer() (host.Host, error) { } func (mn *mocknet) AddPeer(k ic.PrivKey, a ma.Multiaddr) (host.Host, error) { - p, err := peer.IDFromPublicKey(k.GetPublic()) + ps, err := pstoremem.NewPeerstore() if err != nil { return nil, err } - - ps, err := pstoremem.NewPeerstore() + p, err := mn.updatePeerstore(k, a, ps) if err != nil { return nil, err } - ps.AddAddr(p, a, peerstore.PermanentAddrTTL) - ps.AddPrivKey(p, k) - ps.AddPubKey(p, k.GetPublic()) return mn.AddPeerWithPeerstore(p, ps) } func (mn *mocknet) AddPeerWithPeerstore(p peer.ID, ps peerstore.Peerstore) (host.Host, error) { + return mn.AddPeerWithOptions(p, PeerOptions{ps: ps}) +} + +func (mn *mocknet) AddPeerWithOptions(p peer.ID, opts PeerOptions) (host.Host, error) { bus := eventbus.NewBus() - n, err := newPeernet(mn, p, ps, bus) + if err := mn.addDefaults(&opts); err != nil { + return nil, err + } + n, err := newPeernet(mn, p, opts, bus) if err != nil { return nil, err } - opts := &bhost.HostOpts{ + hostOpts := &bhost.HostOpts{ NegotiationTimeout: -1, DisableSignedPeerRecord: true, EventBus: bus, } - h, err := bhost.NewHost(n, opts) + h, err := bhost.NewHost(n, hostOpts) if err != nil { return nil, err } @@ -134,6 +157,35 @@ func (mn *mocknet) AddPeerWithPeerstore(p peer.ID, ps peerstore.Peerstore) (host return h, nil } +func (mn *mocknet) addDefaults(opts *PeerOptions) error { + if opts.ps == nil { + ps, err := pstoremem.NewPeerstore() + if err != nil { + return err + } + opts.ps = ps + } + return nil +} + +func (mn *mocknet) updatePeerstore(k ic.PrivKey, a ma.Multiaddr, ps peerstore.Peerstore) (peer.ID, error) { + p, err := peer.IDFromPublicKey(k.GetPublic()) + if err != nil { + return "", err + } + + ps.AddAddr(p, a, peerstore.PermanentAddrTTL) + err = ps.AddPrivKey(p, k) + if err != nil { + return "", err + } + err = ps.AddPubKey(p, k.GetPublic()) + if err != nil { + return "", err + } + return p, nil +} + func (mn *mocknet) Peers() []peer.ID { mn.Lock() defer mn.Unlock() diff --git a/p2p/net/mock/mock_peernet.go b/p2p/net/mock/mock_peernet.go index f5f707e0b3..2e56b7f2bb 100644 --- a/p2p/net/mock/mock_peernet.go +++ b/p2p/net/mock/mock_peernet.go @@ -7,6 +7,7 @@ import ( "math/rand" "sync" + "github.com/libp2p/go-libp2p/core/connmgr" "github.com/libp2p/go-libp2p/core/event" "github.com/libp2p/go-libp2p/core/network" "github.com/libp2p/go-libp2p/core/peer" @@ -28,6 +29,9 @@ type peernet struct { connsByPeer map[peer.ID]map[*conn]struct{} connsByLink map[*link]map[*conn]struct{} + // connection gater to check before dialing or accepting connections. May be nil to allow all. + gater connmgr.ConnectionGater + // implement network.Network streamHandler network.StreamHandler @@ -38,7 +42,7 @@ type peernet struct { } // newPeernet constructs a new peernet -func newPeernet(m *mocknet, p peer.ID, ps peerstore.Peerstore, bus event.Bus) (*peernet, error) { +func newPeernet(m *mocknet, p peer.ID, opts PeerOptions, bus event.Bus) (*peernet, error) { emitter, err := bus.Emitter(&event.EvtPeerConnectednessChanged{}) if err != nil { return nil, err @@ -47,7 +51,8 @@ func newPeernet(m *mocknet, p peer.ID, ps peerstore.Peerstore, bus event.Bus) (* n := &peernet{ mocknet: m, peer: p, - ps: ps, + ps: opts.ps, + gater: opts.gater, emitter: emitter, connsByPeer: map[peer.ID]map[*conn]struct{}{}, @@ -124,6 +129,10 @@ func (pn *peernet) connect(p peer.ID) (*conn, error) { } pn.RUnlock() + if pn.gater != nil && !pn.gater.InterceptPeerDial(p) { + log.Debugf("gater disallowed outbound connection to peer %s", p) + return nil, fmt.Errorf("%v connection gater disallowed connection to %v", pn.peer, p) + } log.Debugf("%s (newly) dialing %s", pn.peer, p) // ok, must create a new connection. we need a link @@ -139,18 +148,51 @@ func (pn *peernet) connect(p peer.ID) (*conn, error) { log.Debugf("%s dialing %s openingConn", pn.peer, p) // create a new connection with link - c := pn.openConn(p, l.(*link)) - return c, nil + return pn.openConn(p, l.(*link)) } -func (pn *peernet) openConn(r peer.ID, l *link) *conn { +func (pn *peernet) openConn(r peer.ID, l *link) (*conn, error) { lc, rc := l.newConnPair(pn) - log.Debugf("%s opening connection to %s", pn.LocalPeer(), lc.RemotePeer()) addConnPair(pn, rc.net, lc, rc) + log.Debugf("%s opening connection to %s", pn.LocalPeer(), lc.RemotePeer()) + abort := func() { + _ = lc.Close() + _ = rc.Close() + } + if pn.gater != nil && !pn.gater.InterceptAddrDial(lc.remote, lc.remoteAddr) { + abort() + return nil, fmt.Errorf("%v rejected dial to %v on addr %v", lc.local, lc.remote, lc.remoteAddr) + } + if rc.net.gater != nil && !rc.net.gater.InterceptAccept(rc) { + abort() + return nil, fmt.Errorf("%v rejected connection from %v", rc.local, rc.remote) + } + if err := checkSecureAndUpgrade(network.DirOutbound, pn.gater, lc); err != nil { + abort() + return nil, err + } + if err := checkSecureAndUpgrade(network.DirInbound, rc.net.gater, rc); err != nil { + abort() + return nil, err + } go rc.net.remoteOpenedConn(rc) pn.addConn(lc) - return lc + return lc, nil +} + +func checkSecureAndUpgrade(dir network.Direction, gater connmgr.ConnectionGater, c *conn) error { + if gater == nil { + return nil + } + if !gater.InterceptSecured(dir, c.remote, c) { + return fmt.Errorf("%v rejected secure handshake with %v", c.local, c.remote) + } + allow, _ := gater.InterceptUpgraded(c) + if !allow { + return fmt.Errorf("%v rejected upgrade with %v", c.local, c.remote) + } + return nil } // addConnPair adds connection to both peernets at the same time diff --git a/p2p/net/mock/mock_test.go b/p2p/net/mock/mock_test.go index 2ea1bf18dd..863e54f1c7 100644 --- a/p2p/net/mock/mock_test.go +++ b/p2p/net/mock/mock_test.go @@ -13,9 +13,12 @@ import ( "github.com/libp2p/go-libp2p/core/crypto" "github.com/libp2p/go-libp2p/core/event" + "github.com/libp2p/go-libp2p/core/host" "github.com/libp2p/go-libp2p/core/network" "github.com/libp2p/go-libp2p/core/peer" "github.com/libp2p/go-libp2p/core/protocol" + "github.com/libp2p/go-libp2p/p2p/net/conngater" + manet "github.com/multiformats/go-multiaddr/net" "github.com/libp2p/go-libp2p-testing/ci" tetc "github.com/libp2p/go-libp2p-testing/etc" @@ -681,3 +684,68 @@ func TestEventBus(t *testing.T) { } } } + +func TestBlockByPeerID(t *testing.T) { + m, gater1, host1, _, host2 := WithConnectionGaters(t) + + err := gater1.BlockPeer(host2.ID()) + if err != nil { + t.Fatal(err) + } + + _, err = m.ConnectPeers(host1.ID(), host2.ID()) + if err == nil { + t.Fatal("Should have blocked connection to banned peer") + } + + _, err = m.ConnectPeers(host2.ID(), host1.ID()) + if err == nil { + t.Fatal("Should have blocked connection from banned peer") + } +} + +func TestBlockByIP(t *testing.T) { + m, gater1, host1, _, host2 := WithConnectionGaters(t) + + ip, err := manet.ToIP(host2.Addrs()[0]) + if err != nil { + t.Fatal(err) + } + err = gater1.BlockAddr(ip) + if err != nil { + t.Fatal(err) + } + + _, err = m.ConnectPeers(host1.ID(), host2.ID()) + if err == nil { + t.Fatal("Should have blocked connection to banned IP") + } + + _, err = m.ConnectPeers(host2.ID(), host1.ID()) + if err == nil { + t.Fatal("Should have blocked connection from banned IP") + } +} + +func WithConnectionGaters(t *testing.T) (Mocknet, *conngater.BasicConnectionGater, host.Host, *conngater.BasicConnectionGater, host.Host) { + m := New() + addPeer := func() (*conngater.BasicConnectionGater, host.Host) { + gater, err := conngater.NewBasicConnectionGater(nil) + if err != nil { + t.Fatal(err) + } + h, err := m.GenPeerWithOptions(PeerOptions{gater: gater}) + if err != nil { + t.Fatal(err) + } + return gater, h + } + gater1, host1 := addPeer() + gater2, host2 := addPeer() + + err := m.LinkAll() + if err != nil { + t.Fatal(err) + } + return m, gater1, host1, gater2, host2 +} diff --git a/p2p/net/swarm/black_hole_detector.go b/p2p/net/swarm/black_hole_detector.go index 0c415080e0..078b1126c4 100644 --- a/p2p/net/swarm/black_hole_detector.go +++ b/p2p/net/swarm/black_hole_detector.go @@ -241,17 +241,36 @@ func (d *blackHoleDetector) RecordResult(addr ma.Multiaddr, success bool) { } } -func newBlackHoleDetector(detectUDP, detectIPv6 bool, mt MetricsTracer) *blackHoleDetector { +// blackHoleConfig is the config used for black hole detection +type blackHoleConfig struct { + // Enabled enables black hole detection + Enabled bool + // N is the size of the sliding window used to evaluate black hole state + N int + // MinSuccesses is the minimum number of successes out of N required to not + // block requests + MinSuccesses int +} + +func newBlackHoleDetector(udpConfig, ipv6Config blackHoleConfig, mt MetricsTracer) *blackHoleDetector { d := &blackHoleDetector{} - // A black hole is a binary property. On a network if UDP dials are blocked or there is - // no IPv6 connectivity, all dials will fail. So a low success rate of 5 out 100 dials - // is good enough. - if detectUDP { - d.udp = &blackHoleFilter{n: 100, minSuccesses: 5, name: "UDP", metricsTracer: mt} + if udpConfig.Enabled { + d.udp = &blackHoleFilter{ + n: udpConfig.N, + minSuccesses: udpConfig.MinSuccesses, + name: "UDP", + metricsTracer: mt, + } } - if detectIPv6 { - d.ipv6 = &blackHoleFilter{n: 100, minSuccesses: 5, name: "IPv6", metricsTracer: mt} + + if ipv6Config.Enabled { + d.ipv6 = &blackHoleFilter{ + n: ipv6Config.N, + minSuccesses: ipv6Config.MinSuccesses, + name: "IPv6", + metricsTracer: mt, + } } return d } diff --git a/p2p/net/swarm/black_hole_detector_test.go b/p2p/net/swarm/black_hole_detector_test.go index 564fc07767..7b10fc88a6 100644 --- a/p2p/net/swarm/black_hole_detector_test.go +++ b/p2p/net/swarm/black_hole_detector_test.go @@ -75,7 +75,9 @@ func TestBlackHoleFilterSuccessFraction(t *testing.T) { } func TestBlackHoleDetectorInApplicableAddress(t *testing.T) { - bhd := newBlackHoleDetector(true, true, nil) + udpConfig := blackHoleConfig{Enabled: true, N: 10, MinSuccesses: 5} + ipv6Config := blackHoleConfig{Enabled: true, N: 10, MinSuccesses: 5} + bhd := newBlackHoleDetector(udpConfig, ipv6Config, nil) addrs := []ma.Multiaddr{ ma.StringCast("/ip4/1.2.3.4/tcp/1234"), ma.StringCast("/ip4/1.2.3.4/tcp/1233"), @@ -92,7 +94,8 @@ func TestBlackHoleDetectorInApplicableAddress(t *testing.T) { } func TestBlackHoleDetectorUDPDisabled(t *testing.T) { - bhd := newBlackHoleDetector(false, true, nil) + ipv6Config := blackHoleConfig{Enabled: true, N: 10, MinSuccesses: 5} + bhd := newBlackHoleDetector(blackHoleConfig{Enabled: false}, ipv6Config, nil) publicAddr := ma.StringCast("/ip4/1.2.3.4/udp/1234/quic-v1") privAddr := ma.StringCast("/ip4/192.168.1.5/udp/1234/quic-v1") for i := 0; i < 100; i++ { @@ -103,7 +106,8 @@ func TestBlackHoleDetectorUDPDisabled(t *testing.T) { } func TestBlackHoleDetectorIPv6Disabled(t *testing.T) { - bhd := newBlackHoleDetector(true, false, nil) + udpConfig := blackHoleConfig{Enabled: true, N: 10, MinSuccesses: 5} + bhd := newBlackHoleDetector(udpConfig, blackHoleConfig{Enabled: false}, nil) publicAddr := ma.StringCast("/ip6/1::1/tcp/1234") privAddr := ma.StringCast("/ip6/::1/tcp/1234") addrs := []ma.Multiaddr{publicAddr, privAddr} diff --git a/p2p/net/swarm/dial_ranker.go b/p2p/net/swarm/dial_ranker.go index 8cdcfa69a2..3725884e2e 100644 --- a/p2p/net/swarm/dial_ranker.go +++ b/p2p/net/swarm/dial_ranker.go @@ -21,7 +21,7 @@ const ( PrivateQUICDelay = 30 * time.Millisecond // RelayDelay is the duration by which relay dials are delayed relative to direct addresses - RelayDelay = 250 * time.Millisecond + RelayDelay = 500 * time.Millisecond ) // NoDelayDialRanker ranks addresses with no delay. This is useful for simultaneous connect requests. diff --git a/p2p/net/swarm/swarm.go b/p2p/net/swarm/swarm.go index c0e8f1cf12..9dfab2d9c0 100644 --- a/p2p/net/swarm/swarm.go +++ b/p2p/net/swarm/swarm.go @@ -108,6 +108,26 @@ func WithDialRanker(d network.DialRanker) Option { } } +// WithUDPBlackHoleConfig configures swarm to use c as the config for UDP black hole detection +// n is the size of the sliding window used to evaluate black hole state +// min is the minimum number of successes out of n required to not block requests +func WithUDPBlackHoleConfig(enabled bool, n, min int) Option { + return func(s *Swarm) error { + s.udpBlackHoleConfig = blackHoleConfig{Enabled: enabled, N: n, MinSuccesses: min} + return nil + } +} + +// WithIPv6BlackHoleConfig configures swarm to use c as the config for IPv6 black hole detection +// n is the size of the sliding window used to evaluate black hole state +// min is the minimum number of successes out of n required to not block requests +func WithIPv6BlackHoleConfig(enabled bool, n, min int) Option { + return func(s *Swarm) error { + s.ipv6BlackHoleConfig = blackHoleConfig{Enabled: enabled, N: n, MinSuccesses: min} + return nil + } +} + // Swarm is a connection muxer, allowing connections to other peers to // be opened and closed, while still using the same Chan for all // communication. The Chan sends/receives Messages, which note the @@ -174,7 +194,9 @@ type Swarm struct { dialRanker network.DialRanker - bhd *blackHoleDetector + udpBlackHoleConfig blackHoleConfig + ipv6BlackHoleConfig blackHoleConfig + bhd *blackHoleDetector } // NewSwarm constructs a Swarm. @@ -194,6 +216,12 @@ func NewSwarm(local peer.ID, peers peerstore.Peerstore, eventBus event.Bus, opts dialTimeoutLocal: defaultDialTimeoutLocal, maResolver: madns.DefaultResolver, dialRanker: DefaultDialRanker, + + // A black hole is a binary property. On a network if UDP dials are blocked or there is + // no IPv6 connectivity, all dials will fail. So a low success rate of 5 out 100 dials + // is good enough. + udpBlackHoleConfig: blackHoleConfig{Enabled: true, N: 100, MinSuccesses: 5}, + ipv6BlackHoleConfig: blackHoleConfig{Enabled: true, N: 100, MinSuccesses: 5}, } s.conns.m = make(map[peer.ID][]*Conn) @@ -215,7 +243,7 @@ func NewSwarm(local peer.ID, peers peerstore.Peerstore, eventBus event.Bus, opts s.limiter = newDialLimiter(s.dialAddr) s.backf.init(s.ctx) - s.bhd = newBlackHoleDetector(true, true, s.metricsTracer) + s.bhd = newBlackHoleDetector(s.udpBlackHoleConfig, s.ipv6BlackHoleConfig, s.metricsTracer) return s, nil } diff --git a/p2p/net/swarm/swarm_dial.go b/p2p/net/swarm/swarm_dial.go index b9c47c72a3..f278554a97 100644 --- a/p2p/net/swarm/swarm_dial.go +++ b/p2p/net/swarm/swarm_dial.go @@ -306,7 +306,8 @@ func (s *Swarm) addrsForDial(ctx context.Context, p peer.ID) ([]ma.Multiaddr, er forceDirect, _ := network.GetForceDirectDial(ctx) goodAddrs := s.filterKnownUndialables(p, resolved, forceDirect) - goodAddrs = network.DedupAddrs(goodAddrs) + goodAddrs = ma.Unique(goodAddrs) + if len(goodAddrs) == 0 { return nil, ErrNoGoodAddresses } diff --git a/p2p/protocol/holepunch/holepunch_test.go b/p2p/protocol/holepunch/holepunch_test.go index 9cf57ff65c..29d589cd7a 100644 --- a/p2p/protocol/holepunch/holepunch_test.go +++ b/p2p/protocol/holepunch/holepunch_test.go @@ -102,6 +102,10 @@ func TestNoHolePunchIfDirectConnExists(t *testing.T) { } func TestDirectDialWorks(t *testing.T) { + if race.WithRace() { + t.Skip("modifying manet.Private4 is racy") + } + // mark all addresses as public cpy := manet.Private4 manet.Private4 = []*net.IPNet{} diff --git a/p2p/protocol/identify/id_test.go b/p2p/protocol/identify/id_test.go index 8def30b1dd..b18ba51ca5 100644 --- a/p2p/protocol/identify/id_test.go +++ b/p2p/protocol/identify/id_test.go @@ -29,6 +29,7 @@ import ( "github.com/libp2p/go-libp2p/p2p/protocol/identify/pb" mockClock "github.com/benbjohnson/clock" + "github.com/libp2p/go-libp2p-testing/race" "github.com/libp2p/go-msgio/pbio" ma "github.com/multiformats/go-multiaddr" "github.com/stretchr/testify/assert" @@ -560,6 +561,9 @@ func TestSendPush(t *testing.T) { } func TestLargeIdentifyMessage(t *testing.T) { + if race.WithRace() { + t.Skip("setting peerstore.RecentlyConnectedAddrTTL is racy") + } oldTTL := peerstore.RecentlyConnectedAddrTTL peerstore.RecentlyConnectedAddrTTL = 500 * time.Millisecond t.Cleanup(func() { peerstore.RecentlyConnectedAddrTTL = oldTTL }) diff --git a/p2p/protocol/identify/obsaddr.go b/p2p/protocol/identify/obsaddr.go index 3422a286ff..0412541f5d 100644 --- a/p2p/protocol/identify/obsaddr.go +++ b/p2p/protocol/identify/obsaddr.go @@ -375,6 +375,11 @@ func shouldRecordObservation(host addrsProvider, network listenAddrsProvider, co return false } + // Provided by NAT64 peers, these addresses are specific to the peer and not publicly routable + if manet.IsNAT64IPv4ConvertedIPv6Addr(observed) { + return false + } + // we should only use ObservedAddr when our connection's LocalAddr is one // of our ListenAddrs. If we Dial out using an ephemeral addr, knowing that // address's external mapping is not very useful because the port will not be diff --git a/p2p/protocol/identify/obsaddr_glass_test.go b/p2p/protocol/identify/obsaddr_glass_test.go index 497b08e0bd..f96d3a3576 100644 --- a/p2p/protocol/identify/obsaddr_glass_test.go +++ b/p2p/protocol/identify/obsaddr_glass_test.go @@ -4,6 +4,7 @@ package identify // can access internal types. import ( + "fmt" "testing" ma "github.com/multiformats/go-multiaddr" @@ -103,3 +104,50 @@ func TestShouldRecordObservationWithWebTransport(t *testing.T) { require.True(t, shouldRecordObservation(h, h, c, observedAddr)) } + +func TestShouldRecordObservationWithNAT64Addr(t *testing.T) { + listenAddr1 := ma.StringCast("/ip4/0.0.0.0/tcp/1234") + ifaceAddr1 := ma.StringCast("/ip4/10.0.0.2/tcp/4321") + listenAddr2 := ma.StringCast("/ip6/::/tcp/1234") + ifaceAddr2 := ma.StringCast("/ip6/1::1/tcp/4321") + + h := &mockHost{ + listenAddrs: []ma.Multiaddr{listenAddr1, listenAddr2}, + ifaceListenAddrs: []ma.Multiaddr{ifaceAddr1, ifaceAddr2}, + addrs: []ma.Multiaddr{listenAddr1, listenAddr2}, + } + c := &mockConn{ + local: listenAddr1, + remote: ma.StringCast("/ip4/1.2.3.6/tcp/4321"), + } + + cases := []struct { + addr ma.Multiaddr + want bool + failureReason string + }{ + { + addr: ma.StringCast("/ip4/1.2.3.4/tcp/1234"), + want: true, + failureReason: "IPv4 should be observed", + }, + { + addr: ma.StringCast("/ip6/1::4/tcp/1234"), + want: true, + failureReason: "public IPv6 address should be observed", + }, + { + addr: ma.StringCast("/ip6/64:ff9b::192.0.1.2/tcp/1234"), + want: false, + failureReason: "NAT64 IPv6 address shouldn't be observed", + }, + } + for i, tc := range cases { + t.Run(fmt.Sprintf("%d", i), func(t *testing.T) { + + if shouldRecordObservation(h, h, c, tc.addr) != tc.want { + t.Fatalf("%s %s", tc.addr, tc.failureReason) + } + }) + } +} diff --git a/p2p/test/resource-manager/rcmgr_test.go b/p2p/test/resource-manager/rcmgr_test.go index e4a1227685..4cdd5f2f05 100644 --- a/p2p/test/resource-manager/rcmgr_test.go +++ b/p2p/test/resource-manager/rcmgr_test.go @@ -317,14 +317,10 @@ func TestReadmeExample(t *testing.T) { // The resource manager expects a limiter, se we create one from our limits. limiter := rcmgr.NewFixedLimiter(limits) - // (Optional if you want metrics) Construct the OpenCensus metrics reporter. - str, err := rcmgr.NewStatsTraceReporter() - if err != nil { - panic(err) - } - + // Metrics are enabled by default. If you want to disable metrics, use the + // WithMetricsDisabled option // Initialize the resource manager - rm, err := rcmgr.NewResourceManager(limiter, rcmgr.WithTraceReporter(str)) + rm, err := rcmgr.NewResourceManager(limiter, rcmgr.WithMetricsDisabled()) if err != nil { panic(err) } @@ -334,6 +330,5 @@ func TestReadmeExample(t *testing.T) { if err != nil { panic(err) } - host.Close() } diff --git a/p2p/test/transport/transport_test.go b/p2p/test/transport/transport_test.go index 513d2d9b2b..e960806444 100644 --- a/p2p/test/transport/transport_test.go +++ b/p2p/test/transport/transport_test.go @@ -368,10 +368,15 @@ func TestMoreStreamsThanOurLimits(t *testing.T) { })) var handledStreams atomic.Int32 + var sawFirstErr atomic.Bool + + semaphore := make(chan struct{}, streamCount) + // Start with a single stream at a time. If that works, we'll increase the number of concurrent streams. + semaphore <- struct{}{} + listener.SetStreamHandler("echo", func(s network.Stream) { io.Copy(s, s) s.Close() - handledStreams.Add(1) }) wg := sync.WaitGroup{} @@ -380,14 +385,31 @@ func TestMoreStreamsThanOurLimits(t *testing.T) { var completedStreams atomic.Int32 for i := 0; i < streamCount; i++ { go func() { + <-semaphore + var didErr bool defer wg.Done() defer completedStreams.Add(1) + defer func() { + select { + case semaphore <- struct{}{}: + default: + } + if !didErr && !sawFirstErr.Load() { + // No error! We can add one more stream to our concurrency limit. + select { + case semaphore <- struct{}{}: + default: + } + } + }() var s network.Stream var err error // maxRetries is an arbitrary retry amount if there's any error. maxRetries := streamCount * 4 shouldRetry := func(err error) bool { + didErr = true + sawFirstErr.Store(true) maxRetries-- if maxRetries == 0 || len(errCh) > 0 { select { @@ -426,14 +448,13 @@ func TestMoreStreamsThanOurLimits(t *testing.T) { if !bytes.Equal(b, []byte("hello")) { return errors.New("received data does not match sent data") } + handledStreams.Add(1) return nil }(s) - if err != nil { - if shouldRetry(err) { - time.Sleep(50 * time.Millisecond) - continue - } + if err != nil && shouldRetry(err) { + time.Sleep(50 * time.Millisecond) + continue } return } diff --git a/test-plans/README.md b/test-plans/README.md index 1cc19ab86e..1d180aacd2 100644 --- a/test-plans/README.md +++ b/test-plans/README.md @@ -24,7 +24,7 @@ of these nodes with the other version's interop test. # Running all interop tests locally with Compose To run this test against all released libp2p versions you'll need to have the -(libp2p/test-plans)[https://github.com/libp2p/test-plans] checked out. Then do +[libp2p/test-plans](https://github.com/libp2p/test-plans) checked out. Then do the following (from the root directory of this repository): 1. Build the image: `docker build -t go-libp2p-head -f test-plans/PingDockerfile .`. diff --git a/test-plans/go.mod b/test-plans/go.mod index 89777e1d0e..f06f689f25 100644 --- a/test-plans/go.mod +++ b/test-plans/go.mod @@ -5,7 +5,7 @@ go 1.19 require ( github.com/go-redis/redis/v8 v8.11.5 github.com/libp2p/go-libp2p v0.0.0 - github.com/multiformats/go-multiaddr v0.9.0 + github.com/multiformats/go-multiaddr v0.10.1 ) require ( @@ -85,8 +85,8 @@ require ( go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.24.0 // indirect golang.org/x/crypto v0.7.0 // indirect - golang.org/x/exp v0.0.0-20230321023759-10a507213a29 // indirect - golang.org/x/mod v0.10.0 // indirect + golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df // indirect + golang.org/x/mod v0.11.0 // indirect golang.org/x/net v0.10.0 // indirect golang.org/x/sync v0.2.0 // indirect golang.org/x/sys v0.8.0 // indirect diff --git a/test-plans/go.sum b/test-plans/go.sum index 9e074f6e96..f26383db39 100644 --- a/test-plans/go.sum +++ b/test-plans/go.sum @@ -189,8 +189,8 @@ github.com/multiformats/go-base36 v0.2.0 h1:lFsAbNOGeKtuKozrtBsAkSVhv1p9D0/qedU9 github.com/multiformats/go-base36 v0.2.0/go.mod h1:qvnKE++v+2MWCfePClUEjE78Z7P2a1UV0xHgWc0hkp4= github.com/multiformats/go-multiaddr v0.1.1/go.mod h1:aMKBKNEYmzmDmxfX88/vz+J5IU55txyt0p4aiWVohjo= github.com/multiformats/go-multiaddr v0.2.0/go.mod h1:0nO36NvPpyV4QzvTLi/lafl2y95ncPj0vFwVF6k6wJ4= -github.com/multiformats/go-multiaddr v0.9.0 h1:3h4V1LHIk5w4hJHekMKWALPXErDfz/sggzwC/NcqbDQ= -github.com/multiformats/go-multiaddr v0.9.0/go.mod h1:mI67Lb1EeTOYb8GQfL/7wpIZwc46ElrvzhYnoJOmTT0= +github.com/multiformats/go-multiaddr v0.10.1 h1:HghtFrWyZEPrpTvgAMFJi6gFdgHfs2cb0pyfDsk+lqU= +github.com/multiformats/go-multiaddr v0.10.1/go.mod h1:jLEZsA61rwWNZQTHHnqq2HNa+4os/Hz54eqiRnsRqYQ= github.com/multiformats/go-multiaddr-dns v0.3.1 h1:QgQgR+LQVt3NPTjbrLLpsaT2ufAA2y0Mkk+QRVJbW3A= github.com/multiformats/go-multiaddr-dns v0.3.1/go.mod h1:G/245BRQ6FJGmryJCrOuTdB37AMA5AMOVuO6NY3JwTk= github.com/multiformats/go-multiaddr-fmt v0.1.0 h1:WLEFClPycPkp4fnIzoFoV9FVd49/eQsuaL3/CWe167E= @@ -323,8 +323,8 @@ golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm golang.org/x/crypto v0.7.0 h1:AvwMYaRytfdeVt3u6mLaxYtErKYjxA2OXjJ1HHq6t3A= golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20230321023759-10a507213a29 h1:ooxPy7fPvB4kwsA2h+iBNHkAbp/4JxTSwCmvdjEYmug= -golang.org/x/exp v0.0.0-20230321023759-10a507213a29/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= +golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df h1:UA2aFVmmsIlefxMk29Dp2juaUSth8Pyn3Tq5Y5mJGME= +golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= @@ -334,8 +334,8 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.10.0 h1:lFO9qtOdlre5W1jxS3r/4szv2/6iXxScdzjoBMXNhYk= -golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.11.0 h1:bUO06HqtnRcc/7l71XBe4WcqTZ+3AH1J59zWDDwLKgU= +golang.org/x/mod v0.11.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=