diff --git a/cmd/access/node_builder/access_node_builder.go b/cmd/access/node_builder/access_node_builder.go index c1aba0b8a81..bf7a52047b4 100644 --- a/cmd/access/node_builder/access_node_builder.go +++ b/cmd/access/node_builder/access_node_builder.go @@ -72,7 +72,6 @@ import ( "github.com/onflow/flow-go/network/p2p/middleware" "github.com/onflow/flow-go/network/p2p/p2pbuilder" p2pconfig "github.com/onflow/flow-go/network/p2p/p2pbuilder/config" - "github.com/onflow/flow-go/network/p2p/p2pbuilder/inspector" "github.com/onflow/flow-go/network/p2p/subscription" "github.com/onflow/flow-go/network/p2p/tracer" "github.com/onflow/flow-go/network/p2p/translator" @@ -1199,25 +1198,26 @@ func (builder *FlowAccessNodeBuilder) initPublicLibp2pNode(networkKey crypto.Pri builder.IdentityProvider, builder.FlowConfig.NetworkConfig.GossipSubConfig.LocalMeshLogInterval) - // setup RPC inspectors - rpcInspectorBuilder := inspector.NewGossipSubInspectorBuilder(builder.Logger, builder.SporkID, &builder.FlowConfig.NetworkConfig.GossipSubConfig.GossipSubRPCInspectorsConfig, builder.IdentityProvider, builder.Metrics.Network) - rpcInspectorSuite, err := rpcInspectorBuilder. - SetNetworkType(network.PublicNetwork). - SetMetrics(&p2pconfig.MetricsConfig{ - HeroCacheFactory: builder.HeroCacheMetricsFactory(), - Metrics: builder.Metrics.Network, - }).Build() - if err != nil { - return nil, fmt.Errorf("failed to create gossipsub rpc inspectors for access node: %w", err) - } - libp2pNode, err := p2pbuilder.NewNodeBuilder( builder.Logger, - networkMetrics, + &p2pconfig.MetricsConfig{ + HeroCacheFactory: builder.HeroCacheMetricsFactory(), + Metrics: networkMetrics, + }, + network.PublicNetwork, bindAddress, networkKey, builder.SporkID, + builder.IdentityProvider, &builder.FlowConfig.NetworkConfig.ResourceManagerConfig, + &builder.FlowConfig.NetworkConfig.GossipSubConfig.GossipSubRPCInspectorsConfig, + &p2pconfig.PeerManagerConfig{ + // TODO: eventually, we need pruning enabled even on public network. However, it needs a modified version of + // the peer manager that also operate on the public identities. + ConnectionPruning: connection.PruningDisabled, + UpdateInterval: builder.FlowConfig.NetworkConfig.PeerUpdateInterval, + ConnectorFactory: connection.DefaultLibp2pBackoffConnectorFactory(), + }, &p2p.DisallowListCacheConfig{ MaxSize: builder.FlowConfig.NetworkConfig.DisallowListNotificationCacheSize, Metrics: metrics.DisallowListCacheMetricsFactory(builder.HeroCacheMetricsFactory(), network.PublicNetwork), @@ -1240,11 +1240,9 @@ func (builder *FlowAccessNodeBuilder) initPublicLibp2pNode(networkKey crypto.Pri ) }). // disable connection pruning for the access node which supports the observer - SetPeerManagerOptions(connection.PruningDisabled, builder.FlowConfig.NetworkConfig.PeerUpdateInterval). SetStreamCreationRetryInterval(builder.FlowConfig.NetworkConfig.UnicastCreateStreamRetryDelay). SetGossipSubTracer(meshTracer). SetGossipSubScoreTracerInterval(builder.FlowConfig.NetworkConfig.GossipSubConfig.ScoreTracerInterval). - SetGossipSubRpcInspectorSuite(rpcInspectorSuite). Build() if err != nil { diff --git a/cmd/collection/main.go b/cmd/collection/main.go index 6a742189180..27faa959ca2 100644 --- a/cmd/collection/main.go +++ b/cmd/collection/main.go @@ -47,7 +47,6 @@ import ( "github.com/onflow/flow-go/module/mempool/queue" "github.com/onflow/flow-go/module/metrics" "github.com/onflow/flow-go/network/channels" - "github.com/onflow/flow-go/network/p2p" "github.com/onflow/flow-go/state/protocol" badgerState "github.com/onflow/flow-go/state/protocol/badger" "github.com/onflow/flow-go/state/protocol/blocktimer" @@ -593,13 +592,7 @@ func main() { // register the manager for protocol events node.ProtocolEvents.AddConsumer(manager) - - for _, rpcInspector := range node.GossipSubRpcInspectorSuite.Inspectors() { - if r, ok := rpcInspector.(p2p.GossipSubMsgValidationRpcInspector); ok { - clusterEvents.AddConsumer(r) - } - } - + clusterEvents.AddConsumer(node.LibP2PNode) return manager, err }) diff --git a/cmd/node_builder.go b/cmd/node_builder.go index 1b944db832b..9fb490d3f02 100644 --- a/cmd/node_builder.go +++ b/cmd/node_builder.go @@ -223,9 +223,6 @@ type NodeConfig struct { // UnicastRateLimiterDistributor notifies consumers when a peer's unicast message is rate limited. UnicastRateLimiterDistributor p2p.UnicastRateLimiterDistributor - - // GossipSubRpcInspectorSuite rpc inspector suite. - GossipSubRpcInspectorSuite p2p.GossipSubInspectorSuite } // StateExcerptAtBoot stores information about the root snapshot and latest finalized block for use in bootstrapping. diff --git a/cmd/observer/node_builder/observer_builder.go b/cmd/observer/node_builder/observer_builder.go index 253c5f5e8c7..be518249714 100644 --- a/cmd/observer/node_builder/observer_builder.go +++ b/cmd/observer/node_builder/observer_builder.go @@ -57,7 +57,6 @@ import ( "github.com/onflow/flow-go/network/p2p/middleware" "github.com/onflow/flow-go/network/p2p/p2pbuilder" p2pconfig "github.com/onflow/flow-go/network/p2p/p2pbuilder/config" - "github.com/onflow/flow-go/network/p2p/p2pbuilder/inspector" "github.com/onflow/flow-go/network/p2p/subscription" "github.com/onflow/flow-go/network/p2p/tracer" "github.com/onflow/flow-go/network/p2p/translator" @@ -710,23 +709,20 @@ func (builder *ObserverServiceBuilder) initPublicLibp2pNode(networkKey crypto.Pr builder.IdentityProvider, builder.FlowConfig.NetworkConfig.GossipSubConfig.LocalMeshLogInterval) - rpcInspectorSuite, err := inspector.NewGossipSubInspectorBuilder(builder.Logger, builder.SporkID, &builder.FlowConfig.NetworkConfig.GossipSubConfig.GossipSubRPCInspectorsConfig, builder.IdentityProvider, builder.Metrics.Network). - SetNetworkType(network.PublicNetwork). - SetMetrics(&p2pconfig.MetricsConfig{ - HeroCacheFactory: builder.HeroCacheMetricsFactory(), - Metrics: builder.Metrics.Network, - }).Build() - if err != nil { - return nil, fmt.Errorf("could not initialize gossipsub inspectors for observer node: %w", err) - } - node, err := p2pbuilder.NewNodeBuilder( builder.Logger, - builder.Metrics.Network, + &p2pconfig.MetricsConfig{ + HeroCacheFactory: builder.HeroCacheMetricsFactory(), + Metrics: builder.Metrics.Network, + }, + network.PublicNetwork, builder.BaseConfig.BindAddr, networkKey, builder.SporkID, + builder.IdentityProvider, &builder.FlowConfig.NetworkConfig.ResourceManagerConfig, + &builder.FlowConfig.NetworkConfig.GossipSubConfig.GossipSubRPCInspectorsConfig, + p2pconfig.PeerManagerDisableConfig(), // disable peer manager for observer node. &p2p.DisallowListCacheConfig{ MaxSize: builder.FlowConfig.NetworkConfig.DisallowListNotificationCacheSize, Metrics: metrics.DisallowListCacheMetricsFactory(builder.HeroCacheMetricsFactory(), network.PublicNetwork), @@ -747,7 +743,6 @@ func (builder *ObserverServiceBuilder) initPublicLibp2pNode(networkKey crypto.Pr SetStreamCreationRetryInterval(builder.FlowConfig.NetworkConfig.UnicastCreateStreamRetryDelay). SetGossipSubTracer(meshTracer). SetGossipSubScoreTracerInterval(builder.FlowConfig.NetworkConfig.GossipSubConfig.ScoreTracerInterval). - SetGossipSubRpcInspectorSuite(rpcInspectorSuite). Build() if err != nil { diff --git a/cmd/scaffold.go b/cmd/scaffold.go index 40d4181e8ad..32114adb25e 100644 --- a/cmd/scaffold.go +++ b/cmd/scaffold.go @@ -50,11 +50,11 @@ import ( "github.com/onflow/flow-go/network/p2p" "github.com/onflow/flow-go/network/p2p/cache" "github.com/onflow/flow-go/network/p2p/conduit" + "github.com/onflow/flow-go/network/p2p/connection" "github.com/onflow/flow-go/network/p2p/dns" "github.com/onflow/flow-go/network/p2p/middleware" "github.com/onflow/flow-go/network/p2p/p2pbuilder" p2pconfig "github.com/onflow/flow-go/network/p2p/p2pbuilder/config" - "github.com/onflow/flow-go/network/p2p/p2pbuilder/inspector" "github.com/onflow/flow-go/network/p2p/ping" "github.com/onflow/flow-go/network/p2p/subscription" "github.com/onflow/flow-go/network/p2p/unicast/protocols" @@ -336,6 +336,7 @@ func (fnb *FlowNodeBuilder) EnqueueNetworkInit() { peerManagerCfg := &p2pconfig.PeerManagerConfig{ ConnectionPruning: fnb.FlowConfig.NetworkConfig.NetworkConnectionPruning, UpdateInterval: fnb.FlowConfig.NetworkConfig.PeerUpdateInterval, + ConnectorFactory: connection.DefaultLibp2pBackoffConnectorFactory(), } fnb.Component(LibP2PNodeComponent, func(node *NodeConfig) (module.ReadyDoneAware, error) { @@ -344,34 +345,23 @@ func (fnb *FlowNodeBuilder) EnqueueNetworkInit() { myAddr = fnb.BaseConfig.BindAddr } - metricsCfg := &p2pconfig.MetricsConfig{ - Metrics: fnb.Metrics.Network, - HeroCacheFactory: fnb.HeroCacheMetricsFactory(), - } - - rpcInspectorSuite, err := inspector.NewGossipSubInspectorBuilder(fnb.Logger, fnb.SporkID, &fnb.FlowConfig.NetworkConfig.GossipSubConfig.GossipSubRPCInspectorsConfig, fnb.IdentityProvider, fnb.Metrics.Network). - SetNetworkType(network.PrivateNetwork). - SetMetrics(metricsCfg). - Build() - if err != nil { - return nil, fmt.Errorf("failed to create gossipsub rpc inspectors for default libp2p node: %w", err) - } - - fnb.GossipSubRpcInspectorSuite = rpcInspectorSuite - builder, err := p2pbuilder.DefaultNodeBuilder( fnb.Logger, myAddr, + network.PrivateNetwork, fnb.NetworkKey, fnb.SporkID, fnb.IdentityProvider, - metricsCfg, + &p2pconfig.MetricsConfig{ + Metrics: fnb.Metrics.Network, + HeroCacheFactory: fnb.HeroCacheMetricsFactory(), + }, fnb.Resolver, fnb.BaseConfig.NodeRole, connGaterCfg, peerManagerCfg, &fnb.FlowConfig.NetworkConfig.GossipSubConfig, - fnb.GossipSubRpcInspectorSuite, + &fnb.FlowConfig.NetworkConfig.GossipSubRPCInspectorsConfig, &fnb.FlowConfig.NetworkConfig.ResourceManagerConfig, uniCfg, &fnb.FlowConfig.NetworkConfig.ConnectionManagerConfig, diff --git a/cmd/util/cmd/execution-state-extract/export_report.json b/cmd/util/cmd/execution-state-extract/export_report.json new file mode 100644 index 00000000000..2cbadb698d0 --- /dev/null +++ b/cmd/util/cmd/execution-state-extract/export_report.json @@ -0,0 +1,6 @@ +{ + "EpochCounter": 0, + "PreviousStateCommitment": "18eb0e8beef7ce851e552ecd29c813fde0a9e6f0c5614d7615642076602a48cf", + "CurrentStateCommitment": "18eb0e8beef7ce851e552ecd29c813fde0a9e6f0c5614d7615642076602a48cf", + "ReportSucceeded": true +} \ No newline at end of file diff --git a/config/base_flags.go b/config/base_flags.go index 5cc22422b40..360c4af89b6 100644 --- a/config/base_flags.go +++ b/config/base_flags.go @@ -3,7 +3,7 @@ package config import ( "github.com/spf13/pflag" - netconf "github.com/onflow/flow-go/config/network" + "github.com/onflow/flow-go/network/netconf" ) const ( diff --git a/config/config.go b/config/config.go index ca9da641a2c..bbb71ad769d 100644 --- a/config/config.go +++ b/config/config.go @@ -14,7 +14,7 @@ import ( "github.com/spf13/pflag" "github.com/spf13/viper" - "github.com/onflow/flow-go/config/network" + "github.com/onflow/flow-go/network/netconf" ) var ( @@ -28,7 +28,7 @@ var ( type FlowConfig struct { // ConfigFile used to set a path to a config.yml file used to override the default-config.yml file. ConfigFile string `validate:"filepath" mapstructure:"config-file"` - NetworkConfig *network.Config `mapstructure:"network-config"` + NetworkConfig *netconf.Config `mapstructure:"network-config"` } // Validate checks validity of the Flow config. Errors indicate that either the configuration is broken, @@ -184,12 +184,45 @@ func LogConfig(logger *zerolog.Event, flags *pflag.FlagSet) map[string]struct{} // keys do not match the CLI flags 1:1. ie: networking-connection-pruning -> network-config.networking-connection-pruning. After aliases // are set the conf store will override values with any CLI flag values that are set as expected. func setAliases() { - err := network.SetAliases(conf) + err := SetAliases(conf) if err != nil { panic(fmt.Errorf("failed to set network aliases: %w", err)) } } +// SetAliases this func sets an aliases for each CLI flag defined for network config overrides to it's corresponding +// full key in the viper config store. This is required because in our config.yml file all configuration values for the +// Flow network are stored one level down on the network-config property. When the default config is bootstrapped viper will +// store these values with the "network-config." prefix on the config key, because we do not want to use CLI flags like --network-config.networking-connection-pruning +// to override default values we instead use cleans flags like --networking-connection-pruning and create an alias from networking-connection-pruning -> network-config.networking-connection-pruning +// to ensure overrides happen as expected. +// Args: +// *viper.Viper: instance of the viper store to register network config aliases on. +// Returns: +// error: if a flag does not have a corresponding key in the viper store. +func SetAliases(conf *viper.Viper) error { + m := make(map[string]string) + // create map of key -> full pathkey + // ie: "networking-connection-pruning" -> "network-config.networking-connection-pruning" + for _, key := range conf.AllKeys() { + s := strings.Split(key, ".") + // check len of s, we expect all network keys to have a single prefix "network-config" + // s should always contain only 2 elements + if len(s) == 2 { + m[s[1]] = key + } + } + // each flag name should correspond to exactly one key in our config store after it is loaded with the default config + for _, flagName := range netconf.AllFlagNames() { + fullKey, ok := m[flagName] + if !ok { + return fmt.Errorf("invalid network configuration missing configuration key flag name %s check config file and cli flags", flagName) + } + conf.RegisterAlias(fullKey, flagName) + } + return nil +} + // overrideConfigFile overrides the default config file by reading in the config file at the path set // by the --config-file flag in our viper config store. // diff --git a/engine/access/rest/middleware/metrics.go b/engine/access/rest/middleware/metrics.go index 54dd5dd2c6a..25f82bf4277 100644 --- a/engine/access/rest/middleware/metrics.go +++ b/engine/access/rest/middleware/metrics.go @@ -11,12 +11,19 @@ import ( "github.com/onflow/flow-go/module" ) -func MetricsMiddleware(restCollector module.RestMetrics) mux.MiddlewareFunc { +func MetricsMiddleware(restCollector module.RestMetrics, urlToRoute func(string) (string, error)) mux.MiddlewareFunc { metricsMiddleware := middleware.New(middleware.Config{Recorder: restCollector}) return func(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + //urlToRoute transforms specific URL to generic url pattern + routeName, err := urlToRoute(req.URL.Path) + if err != nil { + // In case of an error, an empty route name filled with "unknown" + routeName = "unknown" + } + // This is a custom metric being called on every http request - restCollector.AddTotalRequests(req.Context(), req.Method, req.URL.Path) + restCollector.AddTotalRequests(req.Context(), req.Method, routeName) // Modify the writer respWriter := &responseWriter{w, http.StatusOK} diff --git a/engine/access/rest/router.go b/engine/access/rest/router.go index da39912eff9..f51f1c65f3e 100644 --- a/engine/access/rest/router.go +++ b/engine/access/rest/router.go @@ -1,7 +1,10 @@ package rest import ( + "fmt" "net/http" + "regexp" + "strings" "github.com/gorilla/mux" "github.com/rs/zerolog" @@ -21,7 +24,7 @@ func newRouter(backend access.API, logger zerolog.Logger, chain flow.Chain, rest v1SubRouter.Use(middleware.LoggingMiddleware(logger)) v1SubRouter.Use(middleware.QueryExpandable()) v1SubRouter.Use(middleware.QuerySelect()) - v1SubRouter.Use(middleware.MetricsMiddleware(restCollector)) + v1SubRouter.Use(middleware.MetricsMiddleware(restCollector, URLToRoute)) linkGenerator := models.NewLinkGeneratorImpl(v1SubRouter) @@ -114,3 +117,59 @@ var Routes = []route{{ Name: "getNodeVersionInfo", Handler: GetNodeVersionInfo, }} + +var routeUrlMap = map[string]string{} +var routeRE = regexp.MustCompile(`(?i)/v1/(\w+)(/(\w+)(/(\w+))?)?`) + +func init() { + for _, r := range Routes { + routeUrlMap[r.Pattern] = r.Name + } +} + +func URLToRoute(url string) (string, error) { + normalized, err := normalizeURL(url) + if err != nil { + return "", err + } + + name, ok := routeUrlMap[normalized] + if !ok { + return "", fmt.Errorf("invalid url") + } + return name, nil +} + +func normalizeURL(url string) (string, error) { + matches := routeRE.FindAllStringSubmatch(url, -1) + if len(matches) != 1 || len(matches[0]) != 6 { + return "", fmt.Errorf("invalid url") + } + + // given a URL like + // /v1/blocks/1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef/payload + // groups [ 1 ] [ 3 ] [ 5 ] + // normalized form like /v1/blocks/{id}/payload + + parts := []string{matches[0][1]} + + switch len(matches[0][3]) { + case 0: + // top level resource. e.g. /v1/blocks + case 64: + // id based resource. e.g. /v1/blocks/1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef + parts = append(parts, "{id}") + case 16: + // address based resource. e.g. /v1/accounts/1234567890abcdef + parts = append(parts, "{address}") + default: + // named resource. e.g. /v1/network/parameters + parts = append(parts, matches[0][3]) + } + + if matches[0][5] != "" { + parts = append(parts, matches[0][5]) + } + + return "/" + strings.Join(parts, "/"), nil +} diff --git a/engine/access/rest/router_test.go b/engine/access/rest/router_test.go new file mode 100644 index 00000000000..5b6578be8a1 --- /dev/null +++ b/engine/access/rest/router_test.go @@ -0,0 +1,185 @@ +package rest + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestParseURL(t *testing.T) { + tests := []struct { + name string + url string + expected string + }{ + { + name: "/v1/transactions", + url: "/v1/transactions", + expected: "createTransaction", + }, + { + name: "/v1/transactions/{id}", + url: "/v1/transactions/53730d3f3d2d2f46cb910b16db817d3a62adaaa72fdb3a92ee373c37c5b55a76", + expected: "getTransactionByID", + }, + { + name: "/v1/transaction_results/{id}", + url: "/v1/transaction_results/53730d3f3d2d2f46cb910b16db817d3a62adaaa72fdb3a92ee373c37c5b55a76", + expected: "getTransactionResultByID", + }, + { + name: "/v1/blocks", + url: "/v1/blocks", + expected: "getBlocksByHeight", + }, + { + name: "/v1/blocks/{id}", + url: "/v1/blocks/53730d3f3d2d2f46cb910b16db817d3a62adaaa72fdb3a92ee373c37c5b55a76", + expected: "getBlocksByIDs", + }, + { + name: "/v1/blocks/{id}/payload", + url: "/v1/blocks/53730d3f3d2d2f46cb910b16db817d3a62adaaa72fdb3a92ee373c37c5b55a76/payload", + expected: "getBlockPayloadByID", + }, + { + name: "/v1/execution_results/{id}", + url: "/v1/execution_results/53730d3f3d2d2f46cb910b16db817d3a62adaaa72fdb3a92ee373c37c5b55a76", + expected: "getExecutionResultByID", + }, + { + name: "/v1/execution_results", + url: "/v1/execution_results", + expected: "getExecutionResultByBlockID", + }, + { + name: "/v1/collections/{id}", + url: "/v1/collections/53730d3f3d2d2f46cb910b16db817d3a62adaaa72fdb3a92ee373c37c5b55a76", + expected: "getCollectionByID", + }, + { + name: "/v1/scripts", + url: "/v1/scripts", + expected: "executeScript", + }, + { + name: "/v1/accounts/{address}", + url: "/v1/accounts/6a587be304c1224c", + expected: "getAccount", + }, + { + name: "/v1/events", + url: "/v1/events", + expected: "getEvents", + }, + { + name: "/v1/network/parameters", + url: "/v1/network/parameters", + expected: "getNetworkParameters", + }, + { + name: "/v1/node_version_info", + url: "/v1/node_version_info", + expected: "getNodeVersionInfo", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := URLToRoute(tt.url) + require.NoError(t, err) + assert.Equal(t, tt.expected, got) + }) + } +} + +func TestBenchmarkParseURL(t *testing.T) { + tests := []struct { + name string + url string + expected string + }{ + { + name: "/v1/transactions", + url: "/v1/transactions", + expected: "createTransaction", + }, + { + name: "/v1/transactions/{id}", + url: "/v1/transactions/53730d3f3d2d2f46cb910b16db817d3a62adaaa72fdb3a92ee373c37c5b55a76", + expected: "getTransactionByID", + }, + { + name: "/v1/transaction_results/{id}", + url: "/v1/transaction_results/53730d3f3d2d2f46cb910b16db817d3a62adaaa72fdb3a92ee373c37c5b55a76", + expected: "getTransactionResultByID", + }, + { + name: "/v1/blocks", + url: "/v1/blocks", + expected: "getBlocksByHeight", + }, + { + name: "/v1/blocks/{id}", + url: "/v1/blocks/53730d3f3d2d2f46cb910b16db817d3a62adaaa72fdb3a92ee373c37c5b55a76", + expected: "getBlocksByIDs", + }, + { + name: "/v1/blocks/{id}/payload", + url: "/v1/blocks/53730d3f3d2d2f46cb910b16db817d3a62adaaa72fdb3a92ee373c37c5b55a76/payload", + expected: "getBlockPayloadByID", + }, + { + name: "/v1/execution_results/{id}", + url: "/v1/execution_results/53730d3f3d2d2f46cb910b16db817d3a62adaaa72fdb3a92ee373c37c5b55a76", + expected: "getExecutionResultByID", + }, + { + name: "/v1/execution_results", + url: "/v1/execution_results", + expected: "getExecutionResultByBlockID", + }, + { + name: "/v1/collections/{id}", + url: "/v1/collections/53730d3f3d2d2f46cb910b16db817d3a62adaaa72fdb3a92ee373c37c5b55a76", + expected: "getCollectionByID", + }, + { + name: "/v1/scripts", + url: "/v1/scripts", + expected: "executeScript", + }, + { + name: "/v1/accounts/{address}", + url: "/v1/accounts/6a587be304c1224c", + expected: "getAccount", + }, + { + name: "/v1/events", + url: "/v1/events", + expected: "getEvents", + }, + { + name: "/v1/network/parameters", + url: "/v1/network/parameters", + expected: "getNetworkParameters", + }, + { + name: "/v1/node_version_info", + url: "/v1/node_version_info", + expected: "getNodeVersionInfo", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + start := time.Now() + for i := 0; i < 100_000; i++ { + _, _ = URLToRoute(tt.url) + } + t.Logf("%s: %v", tt.name, time.Since(start)/100_000) + }) + } +} diff --git a/follower/follower_builder.go b/follower/follower_builder.go index ee98a2bcb3a..36486907d1c 100644 --- a/follower/follower_builder.go +++ b/follower/follower_builder.go @@ -51,7 +51,6 @@ import ( "github.com/onflow/flow-go/network/p2p/middleware" "github.com/onflow/flow-go/network/p2p/p2pbuilder" p2pconfig "github.com/onflow/flow-go/network/p2p/p2pbuilder/config" - "github.com/onflow/flow-go/network/p2p/p2pbuilder/inspector" "github.com/onflow/flow-go/network/p2p/subscription" "github.com/onflow/flow-go/network/p2p/tracer" "github.com/onflow/flow-go/network/p2p/translator" @@ -611,23 +610,21 @@ func (builder *FollowerServiceBuilder) initPublicLibp2pNode(networkKey crypto.Pr builder.Metrics.Network, builder.IdentityProvider, builder.FlowConfig.NetworkConfig.GossipSubConfig.LocalMeshLogInterval) - rpcInspectorSuite, err := inspector.NewGossipSubInspectorBuilder(builder.Logger, builder.SporkID, &builder.FlowConfig.NetworkConfig.GossipSubConfig.GossipSubRPCInspectorsConfig, builder.IdentityProvider, builder.Metrics.Network). - SetNetworkType(network.PublicNetwork). - SetMetrics(&p2pconfig.MetricsConfig{ - HeroCacheFactory: builder.HeroCacheMetricsFactory(), - Metrics: builder.Metrics.Network, - }).Build() - if err != nil { - return nil, fmt.Errorf("failed to create gossipsub rpc inspectors for public libp2p node: %w", err) - } node, err := p2pbuilder.NewNodeBuilder( builder.Logger, - builder.Metrics.Network, + &p2pconfig.MetricsConfig{ + HeroCacheFactory: builder.HeroCacheMetricsFactory(), + Metrics: builder.Metrics.Network, + }, + network.PublicNetwork, builder.BaseConfig.BindAddr, networkKey, builder.SporkID, + builder.IdentityProvider, &builder.FlowConfig.NetworkConfig.ResourceManagerConfig, + &builder.FlowConfig.NetworkConfig.GossipSubConfig.GossipSubRPCInspectorsConfig, + p2pconfig.PeerManagerDisableConfig(), // disable peer manager for follower &p2p.DisallowListCacheConfig{ MaxSize: builder.FlowConfig.NetworkConfig.DisallowListNotificationCacheSize, Metrics: metrics.DisallowListCacheMetricsFactory(builder.HeroCacheMetricsFactory(), network.PublicNetwork), @@ -648,9 +645,7 @@ func (builder *FollowerServiceBuilder) initPublicLibp2pNode(networkKey crypto.Pr SetStreamCreationRetryInterval(builder.FlowConfig.NetworkConfig.UnicastCreateStreamRetryDelay). SetGossipSubTracer(meshTracer). SetGossipSubScoreTracerInterval(builder.FlowConfig.NetworkConfig.GossipSubConfig.ScoreTracerInterval). - SetGossipSubRpcInspectorSuite(rpcInspectorSuite). Build() - if err != nil { return nil, fmt.Errorf("could not build public libp2p node: %w", err) } diff --git a/insecure/cmd/corrupted_builder.go b/insecure/cmd/corrupted_builder.go index bd85b83768b..34de857fda5 100644 --- a/insecure/cmd/corrupted_builder.go +++ b/insecure/cmd/corrupted_builder.go @@ -14,6 +14,7 @@ import ( "github.com/onflow/flow-go/module/metrics" "github.com/onflow/flow-go/network" "github.com/onflow/flow-go/network/p2p" + "github.com/onflow/flow-go/network/p2p/connection" p2pconfig "github.com/onflow/flow-go/network/p2p/p2pbuilder/config" "github.com/onflow/flow-go/network/p2p/unicast/ratelimit" "github.com/onflow/flow-go/utils/logging" @@ -85,6 +86,7 @@ func (cnb *CorruptedNodeBuilder) enqueueNetworkingLayer() { peerManagerCfg := &p2pconfig.PeerManagerConfig{ ConnectionPruning: cnb.FlowConfig.NetworkConfig.NetworkConnectionPruning, UpdateInterval: cnb.FlowConfig.NetworkConfig.PeerUpdateInterval, + ConnectorFactory: connection.DefaultLibp2pBackoffConnectorFactory(), } // create default libp2p factory if corrupt node should enable the topic validator diff --git a/insecure/cmd/mods_override.sh b/insecure/cmd/mods_override.sh index 6f6b4d4a6a7..d1877ac7b5c 100755 --- a/insecure/cmd/mods_override.sh +++ b/insecure/cmd/mods_override.sh @@ -6,7 +6,7 @@ cp ./go.mod ./go2.mod cp ./go.sum ./go2.sum # inject forked libp2p-pubsub into main module to allow building corrupt Docker images -echo "require github.com/yhassanzadeh13/go-libp2p-pubsub v0.6.2-0.20221208234712-b44d9133e4ee" >> ./go.mod +echo "require github.com/yhassanzadeh13/go-libp2p-pubsub v0.6.11-flow-expose-msg" >> ./go.mod # update go.sum since added new dependency go mod tidy diff --git a/insecure/corruptlibp2p/gossipsub_spammer.go b/insecure/corruptlibp2p/gossipsub_spammer.go index 11a651c5d1a..2ec81a89e53 100644 --- a/insecure/corruptlibp2p/gossipsub_spammer.go +++ b/insecure/corruptlibp2p/gossipsub_spammer.go @@ -8,6 +8,7 @@ import ( pb "github.com/libp2p/go-libp2p-pubsub/pb" "github.com/libp2p/go-libp2p/core/peer" "github.com/stretchr/testify/require" + corrupt "github.com/yhassanzadeh13/go-libp2p-pubsub" "github.com/onflow/flow-go/insecure/internal" @@ -26,8 +27,8 @@ type GossipSubRouterSpammer struct { } // NewGossipSubRouterSpammer is the main method tests call for spamming attacks. -func NewGossipSubRouterSpammer(t *testing.T, sporkId flow.Identifier, role flow.Role, provider module.IdentityProvider) *GossipSubRouterSpammer { - spammerNode, spammerId, router := createSpammerNode(t, sporkId, role, provider) +func NewGossipSubRouterSpammer(t *testing.T, sporkId flow.Identifier, role flow.Role, provider module.IdentityProvider, opts ...p2ptest.NodeFixtureParameterOption) *GossipSubRouterSpammer { + spammerNode, spammerId, router := createSpammerNode(t, sporkId, role, provider, opts...) return &GossipSubRouterSpammer{ router: router, SpammerNode: spammerNode, @@ -37,9 +38,9 @@ func NewGossipSubRouterSpammer(t *testing.T, sporkId flow.Identifier, role flow. // SpamControlMessage spams the victim with junk control messages. // ctlMessages is the list of spam messages to send to the victim node. -func (s *GossipSubRouterSpammer) SpamControlMessage(t *testing.T, victim p2p.LibP2PNode, ctlMessages []pb.ControlMessage) { +func (s *GossipSubRouterSpammer) SpamControlMessage(t *testing.T, victim p2p.LibP2PNode, ctlMessages []pb.ControlMessage, msgs ...*pb.Message) { for _, ctlMessage := range ctlMessages { - require.True(t, s.router.Get().SendControl(victim.Host().ID(), &ctlMessage)) + require.True(t, s.router.Get().SendControl(victim.Host().ID(), &ctlMessage, msgs...)) } } @@ -64,14 +65,9 @@ func (s *GossipSubRouterSpammer) Start(t *testing.T) { s.router.set(s.router.Get()) } -func createSpammerNode(t *testing.T, sporkId flow.Identifier, role flow.Role, provider module.IdentityProvider) (p2p.LibP2PNode, flow.Identity, *atomicRouter) { +func createSpammerNode(t *testing.T, sporkId flow.Identifier, role flow.Role, provider module.IdentityProvider, opts ...p2ptest.NodeFixtureParameterOption) (p2p.LibP2PNode, flow.Identity, *atomicRouter) { router := newAtomicRouter() - spammerNode, spammerId := p2ptest.NodeFixture( - t, - sporkId, - t.Name(), - provider, - p2ptest.WithRole(role), + opts = append(opts, p2ptest.WithRole(role), internal.WithCorruptGossipSub(CorruptGossipSubFactory(func(r *corrupt.GossipSubRouter) { require.NotNil(t, r) router.set(r) @@ -79,7 +75,13 @@ func createSpammerNode(t *testing.T, sporkId flow.Identifier, role flow.Role, pr CorruptGossipSubConfigFactoryWithInspector(func(id peer.ID, rpc *corrupt.RPC) error { // here we can inspect the incoming RPC message to the spammer node return nil - })), + }))) + spammerNode, spammerId := p2ptest.NodeFixture( + t, + sporkId, + t.Name(), + provider, + opts..., ) return spammerNode, spammerId, router } diff --git a/insecure/corruptlibp2p/libp2p_node_factory.go b/insecure/corruptlibp2p/libp2p_node_factory.go index 1cc64f3a46b..43096e01f98 100644 --- a/insecure/corruptlibp2p/libp2p_node_factory.go +++ b/insecure/corruptlibp2p/libp2p_node_factory.go @@ -10,16 +10,15 @@ import ( "github.com/rs/zerolog" corrupt "github.com/yhassanzadeh13/go-libp2p-pubsub" - netconf "github.com/onflow/flow-go/config/network" fcrypto "github.com/onflow/flow-go/crypto" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module" "github.com/onflow/flow-go/module/metrics" "github.com/onflow/flow-go/network" + "github.com/onflow/flow-go/network/netconf" "github.com/onflow/flow-go/network/p2p" "github.com/onflow/flow-go/network/p2p/p2pbuilder" p2pconfig "github.com/onflow/flow-go/network/p2p/p2pbuilder/config" - "github.com/onflow/flow-go/network/p2p/p2pbuilder/inspector" ) // InitCorruptLibp2pNode initializes and returns a corrupt libp2p node that should only be used for BFT testing in @@ -74,17 +73,10 @@ func InitCorruptLibp2pNode( Metrics: metricsCfg, } - rpcInspectorSuite, err := inspector.NewGossipSubInspectorBuilder(log, sporkId, &netConfig.GossipSubConfig.GossipSubRPCInspectorsConfig, idProvider, metricsCfg). - SetNetworkType(network.PrivateNetwork). - SetMetrics(metCfg). - Build() - if err != nil { - return nil, fmt.Errorf("failed to create gossipsub rpc inspectors for default libp2p node: %w", err) - } - builder, err := p2pbuilder.DefaultNodeBuilder( log, address, + network.PrivateNetwork, flowKey, sporkId, idProvider, @@ -94,7 +86,7 @@ func InitCorruptLibp2pNode( connGaterCfg, peerManagerCfg, &netConfig.GossipSubConfig, - rpcInspectorSuite, + &netConfig.GossipSubRPCInspectorsConfig, &netConfig.ResourceManagerConfig, uniCfg, &netConfig.ConnectionManagerConfig, @@ -114,8 +106,8 @@ func InitCorruptLibp2pNode( // CorruptGossipSubFactory returns a factory function that creates a new instance of the forked gossipsub module from // github.com/yhassanzadeh13/go-libp2p-pubsub for the purpose of BFT testing and attack vector implementation. func CorruptGossipSubFactory(routerOpts ...func(*corrupt.GossipSubRouter)) p2p.GossipSubFactoryFunc { - factory := func(ctx context.Context, logger zerolog.Logger, host host.Host, cfg p2p.PubSubAdapterConfig) (p2p.PubSubAdapter, error) { - adapter, router, err := NewCorruptGossipSubAdapter(ctx, logger, host, cfg) + factory := func(ctx context.Context, logger zerolog.Logger, host host.Host, cfg p2p.PubSubAdapterConfig, clusterChangeConsumer p2p.CollectionClusterChangesConsumer) (p2p.PubSubAdapter, error) { + adapter, router, err := NewCorruptGossipSubAdapter(ctx, logger, host, cfg, clusterChangeConsumer) for _, opt := range routerOpts { opt(router) } diff --git a/insecure/corruptlibp2p/pubsub_adapter.go b/insecure/corruptlibp2p/pubsub_adapter.go index c059bb0e3f1..f9d08eed50e 100644 --- a/insecure/corruptlibp2p/pubsub_adapter.go +++ b/insecure/corruptlibp2p/pubsub_adapter.go @@ -11,6 +11,7 @@ import ( corrupt "github.com/yhassanzadeh13/go-libp2p-pubsub" "github.com/onflow/flow-go/insecure/internal" + "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module/component" "github.com/onflow/flow-go/module/irrecoverable" "github.com/onflow/flow-go/network/p2p" @@ -28,9 +29,11 @@ import ( // totally separated from the rest of the codebase. type CorruptGossipSubAdapter struct { component.Component - gossipSub *corrupt.PubSub - router *corrupt.GossipSubRouter - logger zerolog.Logger + gossipSub *corrupt.PubSub + router *corrupt.GossipSubRouter + logger zerolog.Logger + clusterChangeConsumer p2p.CollectionClusterChangesConsumer + peerScoreExposer p2p.PeerScoreExposer } var _ p2p.PubSubAdapter = (*CorruptGossipSubAdapter)(nil) @@ -104,7 +107,26 @@ func (c *CorruptGossipSubAdapter) ListPeers(topic string) []peer.ID { return c.gossipSub.ListPeers(topic) } -func NewCorruptGossipSubAdapter(ctx context.Context, logger zerolog.Logger, h host.Host, cfg p2p.PubSubAdapterConfig) (p2p.PubSubAdapter, *corrupt.GossipSubRouter, error) { +func (c *CorruptGossipSubAdapter) ActiveClustersChanged(lst flow.ChainIDList) { + c.clusterChangeConsumer.ActiveClustersChanged(lst) +} + +// PeerScoreExposer returns the peer score exposer for the gossipsub adapter. The exposer is a read-only interface +// for querying peer scores and returns the local scoring table of the underlying gossipsub node. +// The exposer is only available if the gossipsub adapter was configured with a score tracer. +// If the gossipsub adapter was not configured with a score tracer, the exposer will be nil. +// Args: +// +// None. +// +// Returns: +// +// The peer score exposer for the gossipsub adapter. +func (c *CorruptGossipSubAdapter) PeerScoreExposer() p2p.PeerScoreExposer { + return c.peerScoreExposer +} + +func NewCorruptGossipSubAdapter(ctx context.Context, logger zerolog.Logger, h host.Host, cfg p2p.PubSubAdapterConfig, clusterChangeConsumer p2p.CollectionClusterChangesConsumer) (p2p.PubSubAdapter, *corrupt.GossipSubRouter, error) { gossipSubConfig, ok := cfg.(*CorruptPubSubAdapterConfig) if !ok { return nil, nil, fmt.Errorf("invalid gossipsub config type: %T", cfg) @@ -119,18 +141,34 @@ func NewCorruptGossipSubAdapter(ctx context.Context, logger zerolog.Logger, h ho return nil, nil, fmt.Errorf("failed to create corrupt gossipsub: %w", err) } - builder := component.NewComponentManagerBuilder(). - AddWorker(func(ctx irrecoverable.SignalerContext, ready component.ReadyFunc) { - ready() - <-ctx.Done() - }).Build() - + builder := component.NewComponentManagerBuilder() adapter := &CorruptGossipSubAdapter{ - Component: builder, - gossipSub: gossipSub, - router: router, - logger: logger, + gossipSub: gossipSub, + router: router, + logger: logger, + clusterChangeConsumer: clusterChangeConsumer, + } + + if scoreTracer := gossipSubConfig.ScoreTracer(); scoreTracer != nil { + builder.AddWorker(func(ctx irrecoverable.SignalerContext, ready component.ReadyFunc) { + ready() + logger.Debug().Str("component", "corrupt-gossipsub_score_tracer").Msg("starting score tracer") + scoreTracer.Start(ctx) + logger.Debug().Str("component", "corrupt-gossipsub_score_tracer").Msg("score tracer started") + + <-scoreTracer.Done() + logger.Debug().Str("component", "corrupt-gossipsub_score_tracer").Msg("score tracer stopped") + }) + adapter.peerScoreExposer = scoreTracer } + builder.AddWorker(func(ctx irrecoverable.SignalerContext, ready component.ReadyFunc) { + ready() + // it is likely that this adapter is configured without a score tracer, so we need to + // wait for the context to be done in order to prevent immature shutdown. + <-ctx.Done() + }) + + adapter.Component = builder.Build() return adapter, router, nil } diff --git a/insecure/corruptlibp2p/pubsub_adapter_config.go b/insecure/corruptlibp2p/pubsub_adapter_config.go index 002e18608e0..1bae78dd872 100644 --- a/insecure/corruptlibp2p/pubsub_adapter_config.go +++ b/insecure/corruptlibp2p/pubsub_adapter_config.go @@ -24,6 +24,7 @@ type CorruptPubSubAdapterConfig struct { inspector func(peer.ID, *corrupt.RPC) error withMessageSigning bool withStrictSignatureVerification bool + scoreTracer p2p.PeerScoreTracer } type CorruptPubSubAdapterConfigOption func(config *CorruptPubSubAdapterConfig) @@ -78,8 +79,56 @@ func (c *CorruptPubSubAdapterConfig) WithSubscriptionFilter(filter p2p.Subscript c.options = append(c.options, corrupt.WithSubscriptionFilter(filter)) } -func (c *CorruptPubSubAdapterConfig) WithScoreOption(_ p2p.ScoreOptionBuilder) { - // CorruptPubSub does not support score options. This is a no-op. +func (c *CorruptPubSubAdapterConfig) WithScoreOption(option p2p.ScoreOptionBuilder) { + params, thresholds := option.BuildFlowPubSubScoreOption() + // convert flow pubsub score option to corrupt pubsub score option + corruptParams := &corrupt.PeerScoreParams{ + SkipAtomicValidation: params.SkipAtomicValidation, + TopicScoreCap: params.TopicScoreCap, + AppSpecificScore: params.AppSpecificScore, + AppSpecificWeight: params.AppSpecificWeight, + IPColocationFactorWeight: params.IPColocationFactorWeight, + IPColocationFactorThreshold: params.IPColocationFactorThreshold, + IPColocationFactorWhitelist: params.IPColocationFactorWhitelist, + BehaviourPenaltyWeight: params.BehaviourPenaltyWeight, + BehaviourPenaltyThreshold: params.BehaviourPenaltyThreshold, + BehaviourPenaltyDecay: params.BehaviourPenaltyDecay, + DecayInterval: params.DecayInterval, + DecayToZero: params.DecayToZero, + RetainScore: params.RetainScore, + SeenMsgTTL: params.SeenMsgTTL, + } + corruptThresholds := &corrupt.PeerScoreThresholds{ + SkipAtomicValidation: thresholds.SkipAtomicValidation, + GossipThreshold: thresholds.GossipThreshold, + PublishThreshold: thresholds.PublishThreshold, + GraylistThreshold: thresholds.GraylistThreshold, + AcceptPXThreshold: thresholds.AcceptPXThreshold, + OpportunisticGraftThreshold: thresholds.OpportunisticGraftThreshold, + } + for topic, topicParams := range params.Topics { + corruptParams.Topics[topic] = &corrupt.TopicScoreParams{ + SkipAtomicValidation: topicParams.SkipAtomicValidation, + TopicWeight: topicParams.TopicWeight, + TimeInMeshWeight: topicParams.TimeInMeshWeight, + TimeInMeshQuantum: topicParams.TimeInMeshQuantum, + TimeInMeshCap: topicParams.TimeInMeshCap, + FirstMessageDeliveriesWeight: topicParams.FirstMessageDeliveriesWeight, + FirstMessageDeliveriesDecay: topicParams.FirstMessageDeliveriesDecay, + FirstMessageDeliveriesCap: topicParams.FirstMessageDeliveriesCap, + MeshMessageDeliveriesWeight: topicParams.MeshMessageDeliveriesWeight, + MeshMessageDeliveriesDecay: topicParams.MeshMessageDeliveriesDecay, + MeshMessageDeliveriesCap: topicParams.MeshMessageDeliveriesCap, + MeshMessageDeliveriesThreshold: topicParams.MeshMessageDeliveriesThreshold, + MeshMessageDeliveriesWindow: topicParams.MeshMessageDeliveriesWindow, + MeshMessageDeliveriesActivation: topicParams.MeshMessageDeliveriesActivation, + MeshFailurePenaltyWeight: topicParams.MeshFailurePenaltyWeight, + MeshFailurePenaltyDecay: topicParams.MeshFailurePenaltyDecay, + InvalidMessageDeliveriesWeight: topicParams.InvalidMessageDeliveriesWeight, + InvalidMessageDeliveriesDecay: topicParams.InvalidMessageDeliveriesDecay, + } + } + c.options = append(c.options, corrupt.WithPeerScore(corruptParams, corruptThresholds)) } func (c *CorruptPubSubAdapterConfig) WithTracer(_ p2p.PubSubTracer) { @@ -93,8 +142,15 @@ func (c *CorruptPubSubAdapterConfig) WithMessageIdFunction(f func([]byte) string })) } -func (c *CorruptPubSubAdapterConfig) WithScoreTracer(_ p2p.PeerScoreTracer) { - // CorruptPubSub does not support score tracer. This is a no-op. +func (c *CorruptPubSubAdapterConfig) WithScoreTracer(tracer p2p.PeerScoreTracer) { + c.scoreTracer = tracer + c.options = append(c.options, corrupt.WithPeerScoreInspect(func(snapshot map[peer.ID]*corrupt.PeerScoreSnapshot) { + tracer.UpdatePeerScoreSnapshots(convertPeerScoreSnapshots(snapshot)) + }, tracer.UpdateInterval())) +} + +func (c *CorruptPubSubAdapterConfig) ScoreTracer() p2p.PeerScoreTracer { + return c.scoreTracer } func (c *CorruptPubSubAdapterConfig) WithInspectorSuite(_ p2p.GossipSubInspectorSuite) { @@ -112,3 +168,43 @@ func defaultCorruptPubsubOptions(base *p2p.BasePubSubAdapterConfig, withMessageS corrupt.WithMaxMessageSize(base.MaxMessageSize), } } + +// convertPeerScoreSnapshots converts a libp2p pubsub peer score snapshot to a Flow peer score snapshot. +// Args: +// - snapshot: the libp2p pubsub peer score snapshot. +// +// Returns: +// - map[peer.ID]*p2p.PeerScoreSnapshot: the Flow peer score snapshot. +func convertPeerScoreSnapshots(snapshot map[peer.ID]*corrupt.PeerScoreSnapshot) map[peer.ID]*p2p.PeerScoreSnapshot { + newSnapshot := make(map[peer.ID]*p2p.PeerScoreSnapshot) + for id, snap := range snapshot { + newSnapshot[id] = &p2p.PeerScoreSnapshot{ + Topics: convertTopicScoreSnapshot(snap.Topics), + Score: snap.Score, + AppSpecificScore: snap.AppSpecificScore, + BehaviourPenalty: snap.BehaviourPenalty, + IPColocationFactor: snap.IPColocationFactor, + } + } + return newSnapshot +} + +// convertTopicScoreSnapshot converts a libp2p pubsub topic score snapshot to a Flow topic score snapshot. +// Args: +// - snapshot: the libp2p pubsub topic score snapshot. +// +// Returns: +// - map[string]*p2p.TopicScoreSnapshot: the Flow topic score snapshot. +func convertTopicScoreSnapshot(snapshot map[string]*corrupt.TopicScoreSnapshot) map[string]*p2p.TopicScoreSnapshot { + newSnapshot := make(map[string]*p2p.TopicScoreSnapshot) + for topic, snap := range snapshot { + newSnapshot[topic] = &p2p.TopicScoreSnapshot{ + TimeInMesh: snap.TimeInMesh, + FirstMessageDeliveries: snap.FirstMessageDeliveries, + MeshMessageDeliveries: snap.MeshMessageDeliveries, + InvalidMessageDeliveries: snap.InvalidMessageDeliveries, + } + } + + return newSnapshot +} diff --git a/insecure/corruptlibp2p/spam_test.go b/insecure/corruptlibp2p/spam_test.go index cd759fd52e9..06f6183c03e 100644 --- a/insecure/corruptlibp2p/spam_test.go +++ b/insecure/corruptlibp2p/spam_test.go @@ -19,7 +19,6 @@ import ( "github.com/onflow/flow-go/insecure/corruptlibp2p" "github.com/onflow/flow-go/insecure/internal" "github.com/onflow/flow-go/module/irrecoverable" - mockmodule "github.com/onflow/flow-go/module/mock" "github.com/onflow/flow-go/network/p2p" p2ptest "github.com/onflow/flow-go/network/p2p/test" "github.com/onflow/flow-go/utils/unittest" @@ -32,8 +31,8 @@ func TestSpam_IHave(t *testing.T) { const messagesToSpam = 3 sporkId := unittest.IdentifierFixture() role := flow.RoleConsensus - idProvider := mockmodule.NewIdentityProvider(t) - gsrSpammer := corruptlibp2p.NewGossipSubRouterSpammer(t, sporkId, role, nil) + idProvider := unittest.NewUpdatableIDProvider(flow.IdentityList{}) + gsrSpammer := corruptlibp2p.NewGossipSubRouterSpammer(t, sporkId, role, idProvider) allSpamIHavesReceived := sync.WaitGroup{} allSpamIHavesReceived.Add(messagesToSpam) @@ -57,7 +56,7 @@ func TestSpam_IHave(t *testing.T) { return nil })), ) - idProvider.On("ByPeerID", victimNode.Host().ID()).Return(&victimIdentity, true).Maybe() + idProvider.SetIdentities(flow.IdentityList{&victimIdentity, &gsrSpammer.SpammerId}) // starts nodes ctx, cancel := context.WithCancel(context.Background()) signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) diff --git a/insecure/go.mod b/insecure/go.mod index e3748bc6c3d..121b45694a5 100644 --- a/insecure/go.mod +++ b/insecure/go.mod @@ -14,7 +14,7 @@ require ( github.com/rs/zerolog v1.29.0 github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.8.2 - github.com/yhassanzadeh13/go-libp2p-pubsub v0.6.2-0.20221208234712-b44d9133e4ee + github.com/yhassanzadeh13/go-libp2p-pubsub v0.6.11-flow-expose-msg go.uber.org/atomic v1.10.0 google.golang.org/grpc v1.53.0 google.golang.org/protobuf v1.30.0 diff --git a/insecure/go.sum b/insecure/go.sum index 614d1513e79..dceda2bb27e 100644 --- a/insecure/go.sum +++ b/insecure/go.sum @@ -1481,8 +1481,8 @@ github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= -github.com/yhassanzadeh13/go-libp2p-pubsub v0.6.2-0.20221208234712-b44d9133e4ee h1:yFB2xjfswpuRh8FHagdBMKcBMltjr5u/XKzX6fkJO5E= -github.com/yhassanzadeh13/go-libp2p-pubsub v0.6.2-0.20221208234712-b44d9133e4ee/go.mod h1:Tylw4k1H86gbJx84i3r7qahN/mBaeMpUBvHY0Igshfw= +github.com/yhassanzadeh13/go-libp2p-pubsub v0.6.11-flow-expose-msg h1:CtmNcsXmPNYnpzogVjrIy99if5Hvh79kd+ecoQgA7Vg= +github.com/yhassanzadeh13/go-libp2p-pubsub v0.6.11-flow-expose-msg/go.mod h1:Tylw4k1H86gbJx84i3r7qahN/mBaeMpUBvHY0Igshfw= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= diff --git a/insecure/rpc_inspector/metrics_inspector_test.go b/insecure/integration/functional/test/gossipsub/rpc_inspector/metrics_inspector_test.go similarity index 94% rename from insecure/rpc_inspector/metrics_inspector_test.go rename to insecure/integration/functional/test/gossipsub/rpc_inspector/metrics_inspector_test.go index 38078dd6993..c62cfba2f8b 100644 --- a/insecure/rpc_inspector/metrics_inspector_test.go +++ b/insecure/integration/functional/test/gossipsub/rpc_inspector/metrics_inspector_test.go @@ -15,7 +15,6 @@ import ( "github.com/onflow/flow-go/insecure/internal" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module/irrecoverable" - mockmodule "github.com/onflow/flow-go/module/mock" "github.com/onflow/flow-go/network/channels" "github.com/onflow/flow-go/network/p2p" "github.com/onflow/flow-go/network/p2p/inspector" @@ -29,7 +28,8 @@ func TestMetricsInspector_ObserveRPC(t *testing.T) { t.Parallel() role := flow.RoleConsensus sporkID := unittest.IdentifierFixture() - spammer := corruptlibp2p.NewGossipSubRouterSpammer(t, sporkID, role, nil) + idProvider := unittest.NewUpdatableIDProvider(flow.IdentityList{}) + spammer := corruptlibp2p.NewGossipSubRouterSpammer(t, sporkID, role, idProvider) ctx, cancel := context.WithCancel(context.Background()) signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) @@ -57,7 +57,6 @@ func TestMetricsInspector_ObserveRPC(t *testing.T) { }) metricsInspector := inspector.NewControlMsgMetricsInspector(unittest.Logger(), mockMetricsObserver, 2) corruptInspectorFunc := corruptlibp2p.CorruptInspectorFunc(metricsInspector) - idProvider := mockmodule.NewIdentityProvider(t) victimNode, victimIdentity := p2ptest.NodeFixture( t, sporkID, @@ -67,7 +66,7 @@ func TestMetricsInspector_ObserveRPC(t *testing.T) { internal.WithCorruptGossipSub(corruptlibp2p.CorruptGossipSubFactory(), corruptlibp2p.CorruptGossipSubConfigFactoryWithInspector(corruptInspectorFunc)), ) - idProvider.On("ByPeerID", victimNode.Host().ID()).Return(&victimIdentity, true).Maybe() + idProvider.SetIdentities(flow.IdentityList{&victimIdentity, &spammer.SpammerId}) metricsInspector.Start(signalerCtx) nodes := []p2p.LibP2PNode{victimNode, spammer.SpammerNode} startNodesAndEnsureConnected(t, signalerCtx, nodes, sporkID) diff --git a/insecure/rpc_inspector/utils.go b/insecure/integration/functional/test/gossipsub/rpc_inspector/utils.go similarity index 100% rename from insecure/rpc_inspector/utils.go rename to insecure/integration/functional/test/gossipsub/rpc_inspector/utils.go diff --git a/insecure/rpc_inspector/validation_inspector_test.go b/insecure/integration/functional/test/gossipsub/rpc_inspector/validation_inspector_test.go similarity index 76% rename from insecure/rpc_inspector/validation_inspector_test.go rename to insecure/integration/functional/test/gossipsub/rpc_inspector/validation_inspector_test.go index e033d713071..5917ceee31e 100644 --- a/insecure/rpc_inspector/validation_inspector_test.go +++ b/insecure/integration/functional/test/gossipsub/rpc_inspector/validation_inspector_test.go @@ -16,7 +16,6 @@ import ( "go.uber.org/atomic" "github.com/onflow/flow-go/config" - netconf "github.com/onflow/flow-go/config/network" "github.com/onflow/flow-go/insecure/corruptlibp2p" "github.com/onflow/flow-go/insecure/internal" "github.com/onflow/flow-go/model/flow" @@ -26,7 +25,9 @@ import ( "github.com/onflow/flow-go/network/channels" "github.com/onflow/flow-go/network/p2p" "github.com/onflow/flow-go/network/p2p/inspector/validation" + p2pmsg "github.com/onflow/flow-go/network/p2p/message" mockp2p "github.com/onflow/flow-go/network/p2p/mock" + "github.com/onflow/flow-go/network/p2p/p2pconf" p2ptest "github.com/onflow/flow-go/network/p2p/test" "github.com/onflow/flow-go/utils/unittest" ) @@ -48,9 +49,9 @@ func TestValidationInspector_SafetyThreshold(t *testing.T) { inspectorConfig.PruneLimits.SafetyThreshold = safetyThreshold // expected log message logged when valid number GRAFT control messages spammed under safety threshold - graftExpectedMessageStr := fmt.Sprintf("control message %s inspection passed 5 is below configured safety threshold", p2p.CtrlMsgGraft) + graftExpectedMessageStr := fmt.Sprintf("control message %s inspection passed 5 is below configured safety threshold", p2pmsg.CtrlMsgGraft) // expected log message logged when valid number PRUNE control messages spammed under safety threshold - pruneExpectedMessageStr := fmt.Sprintf("control message %s inspection passed 5 is below configured safety threshold", p2p.CtrlMsgGraft) + pruneExpectedMessageStr := fmt.Sprintf("control message %s inspection passed 5 is below configured safety threshold", p2pmsg.CtrlMsgGraft) graftInfoLogsReceived := atomic.NewInt64(0) pruneInfoLogsReceived := atomic.NewInt64(0) // setup logger hook, we expect info log validation is skipped @@ -67,7 +68,36 @@ func TestValidationInspector_SafetyThreshold(t *testing.T) { }) logger := zerolog.New(os.Stdout).Hook(hook) - signalerCtx, cancelFunc, spammer, victimNode, _, distributor, validationInspector, _ := setupTest(t, logger, role, sporkID, &inspectorConfig) + idProvider := mock.NewIdentityProvider(t) + spammer := corruptlibp2p.NewGossipSubRouterSpammer(t, sporkID, role, idProvider) + + ctx, cancel := context.WithCancel(context.Background()) + signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) + + distributor := mockp2p.NewGossipSubInspectorNotificationDistributor(t) + mockDistributorReadyDoneAware(distributor) + validationInspector, err := validation.NewControlMsgValidationInspector( + logger, + sporkID, + &inspectorConfig, + distributor, + metrics.NewNoopCollector(), + metrics.NewNoopCollector(), + idProvider, + metrics.NewNoopCollector()) + require.NoError(t, err) + corruptInspectorFunc := corruptlibp2p.CorruptInspectorFunc(validationInspector) + victimNode, victimIdentity := p2ptest.NodeFixture( + t, + sporkID, + t.Name(), + idProvider, + p2ptest.WithRole(role), + internal.WithCorruptGossipSub(corruptlibp2p.CorruptGossipSubFactory(), + corruptlibp2p.CorruptGossipSubConfigFactoryWithInspector(corruptInspectorFunc)), + ) + idProvider.On("ByPeerID", victimNode.Host().ID()).Return(&victimIdentity, true).Maybe() + idProvider.On("ByPeerID", spammer.SpammerNode.Host().ID()).Return(&spammer.SpammerId, true).Maybe() messageCount := 5 controlMessageCount := int64(2) @@ -79,7 +109,7 @@ func TestValidationInspector_SafetyThreshold(t *testing.T) { nodes := []p2p.LibP2PNode{victimNode, spammer.SpammerNode} startNodesAndEnsureConnected(t, signalerCtx, nodes, sporkID) spammer.Start(t) - defer stopNodesAndInspector(t, cancelFunc, nodes, validationInspector) + defer stopNodesAndInspector(t, cancel, nodes, validationInspector) // prepare to spam - generate control messages ctlMsgs := spammer.GenerateCtlMessages(int(controlMessageCount), corruptlibp2p.WithGraft(messageCount, channels.PushBlocks.String()), @@ -126,9 +156,9 @@ func TestValidationInspector_HardThreshold_Detection(t *testing.T) { require.Equal(t, spammer.SpammerNode.Host().ID(), notification.PeerID) require.Equal(t, uint64(messageCount), notification.Count) switch notification.MsgType { - case p2p.CtrlMsgGraft: + case p2pmsg.CtrlMsgGraft: invGraftNotifCount.Inc() - case p2p.CtrlMsgPrune: + case p2pmsg.CtrlMsgPrune: invPruneNotifCount.Inc() default: require.Fail(t, "unexpected control message type") @@ -139,13 +169,43 @@ func TestValidationInspector_HardThreshold_Detection(t *testing.T) { } } - signalerCtx, cancelFunc, spammer, victimNode, _, _, validationInspector, _ := setupTest(t, unittest.Logger(), role, sporkID, &inspectorConfig, withExpectedNotificationDissemination(2, inspectDisseminatedNotif)) + idProvider := mock.NewIdentityProvider(t) + spammer := corruptlibp2p.NewGossipSubRouterSpammer(t, sporkID, role, idProvider) + + ctx, cancel := context.WithCancel(context.Background()) + signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) + + distributor := mockp2p.NewGossipSubInspectorNotificationDistributor(t) + mockDistributorReadyDoneAware(distributor) + withExpectedNotificationDissemination(2, inspectDisseminatedNotif)(distributor, spammer) + validationInspector, err := validation.NewControlMsgValidationInspector( + unittest.Logger(), + sporkID, + &inspectorConfig, + distributor, + metrics.NewNoopCollector(), + metrics.NewNoopCollector(), + idProvider, + metrics.NewNoopCollector()) + require.NoError(t, err) + corruptInspectorFunc := corruptlibp2p.CorruptInspectorFunc(validationInspector) + victimNode, victimIdentity := p2ptest.NodeFixture( + t, + sporkID, + t.Name(), + idProvider, + p2ptest.WithRole(role), + internal.WithCorruptGossipSub(corruptlibp2p.CorruptGossipSubFactory(), + corruptlibp2p.CorruptGossipSubConfigFactoryWithInspector(corruptInspectorFunc)), + ) + idProvider.On("ByPeerID", victimNode.Host().ID()).Return(&victimIdentity, true).Maybe() + idProvider.On("ByPeerID", spammer.SpammerNode.Host().ID()).Return(&spammer.SpammerId, true).Maybe() validationInspector.Start(signalerCtx) nodes := []p2p.LibP2PNode{victimNode, spammer.SpammerNode} startNodesAndEnsureConnected(t, signalerCtx, nodes, sporkID) spammer.Start(t) - defer stopNodesAndInspector(t, cancelFunc, nodes, validationInspector) + defer stopNodesAndInspector(t, cancel, nodes, validationInspector) // prepare to spam - generate control messages graftCtlMsgs := spammer.GenerateCtlMessages(int(controlMessageCount), corruptlibp2p.WithGraft(messageCount, channels.PushBlocks.String())) @@ -194,7 +254,7 @@ func TestValidationInspector_HardThresholdIHave_Detection(t *testing.T) { require.Equal(t, uint64(messageCount), notification.Count) require.True(t, channels.IsInvalidTopicErr(notification.Err)) switch notification.MsgType { - case p2p.CtrlMsgIHave: + case p2pmsg.CtrlMsgIHave: invIhaveNotifCount.Inc() default: require.Fail(t, "unexpected control message type") @@ -205,13 +265,43 @@ func TestValidationInspector_HardThresholdIHave_Detection(t *testing.T) { } } - signalerCtx, cancelFunc, spammer, victimNode, _, _, validationInspector, _ := setupTest(t, unittest.Logger(), role, sporkID, &inspectorConfig, withExpectedNotificationDissemination(1, inspectDisseminatedNotif)) + idProvider := mock.NewIdentityProvider(t) + spammer := corruptlibp2p.NewGossipSubRouterSpammer(t, sporkID, role, idProvider) + + ctx, cancel := context.WithCancel(context.Background()) + signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) + + distributor := mockp2p.NewGossipSubInspectorNotificationDistributor(t) + mockDistributorReadyDoneAware(distributor) + withExpectedNotificationDissemination(1, inspectDisseminatedNotif)(distributor, spammer) + validationInspector, err := validation.NewControlMsgValidationInspector( + unittest.Logger(), + sporkID, + &inspectorConfig, + distributor, + metrics.NewNoopCollector(), + metrics.NewNoopCollector(), + idProvider, + metrics.NewNoopCollector()) + require.NoError(t, err) + corruptInspectorFunc := corruptlibp2p.CorruptInspectorFunc(validationInspector) + victimNode, victimIdentity := p2ptest.NodeFixture( + t, + sporkID, + t.Name(), + idProvider, + p2ptest.WithRole(role), + internal.WithCorruptGossipSub(corruptlibp2p.CorruptGossipSubFactory(), + corruptlibp2p.CorruptGossipSubConfigFactoryWithInspector(corruptInspectorFunc)), + ) + idProvider.On("ByPeerID", victimNode.Host().ID()).Return(&victimIdentity, true).Maybe() + idProvider.On("ByPeerID", spammer.SpammerNode.Host().ID()).Return(&spammer.SpammerId, true).Maybe() validationInspector.Start(signalerCtx) nodes := []p2p.LibP2PNode{victimNode, spammer.SpammerNode} startNodesAndEnsureConnected(t, signalerCtx, nodes, sporkID) spammer.Start(t) - defer stopNodesAndInspector(t, cancelFunc, nodes, validationInspector) + defer stopNodesAndInspector(t, cancel, nodes, validationInspector) // add an unknown topic to each of our ihave control messages, this will ensure // that whatever random sample of topic ids that are inspected cause validation @@ -257,9 +347,9 @@ func TestValidationInspector_RateLimitedPeer_Detection(t *testing.T) { require.True(t, validation.IsErrRateLimitedControlMsg(notification.Err)) require.Equal(t, uint64(messageCount), notification.Count) switch notification.MsgType { - case p2p.CtrlMsgGraft: + case p2pmsg.CtrlMsgGraft: invGraftNotifCount.Inc() - case p2p.CtrlMsgPrune: + case p2pmsg.CtrlMsgPrune: invPruneNotifCount.Inc() default: require.Fail(t, "unexpected control message type") @@ -270,13 +360,43 @@ func TestValidationInspector_RateLimitedPeer_Detection(t *testing.T) { } } - signalerCtx, cancelFunc, spammer, victimNode, _, _, validationInspector, _ := setupTest(t, unittest.Logger(), role, sporkID, &inspectorConfig, withExpectedNotificationDissemination(4, inspectDisseminatedNotif)) + idProvider := mock.NewIdentityProvider(t) + spammer := corruptlibp2p.NewGossipSubRouterSpammer(t, sporkID, role, idProvider) + + ctx, cancel := context.WithCancel(context.Background()) + signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) + + distributor := mockp2p.NewGossipSubInspectorNotificationDistributor(t) + mockDistributorReadyDoneAware(distributor) + withExpectedNotificationDissemination(4, inspectDisseminatedNotif)(distributor, spammer) + validationInspector, err := validation.NewControlMsgValidationInspector( + unittest.Logger(), + sporkID, + &inspectorConfig, + distributor, + metrics.NewNoopCollector(), + metrics.NewNoopCollector(), + idProvider, + metrics.NewNoopCollector()) + require.NoError(t, err) + corruptInspectorFunc := corruptlibp2p.CorruptInspectorFunc(validationInspector) + victimNode, victimIdentity := p2ptest.NodeFixture( + t, + sporkID, + t.Name(), + idProvider, + p2ptest.WithRole(role), + internal.WithCorruptGossipSub(corruptlibp2p.CorruptGossipSubFactory(), + corruptlibp2p.CorruptGossipSubConfigFactoryWithInspector(corruptInspectorFunc)), + ) + idProvider.On("ByPeerID", victimNode.Host().ID()).Return(&victimIdentity, true).Maybe() + idProvider.On("ByPeerID", spammer.SpammerNode.Host().ID()).Return(&spammer.SpammerId, true).Maybe() validationInspector.Start(signalerCtx) nodes := []p2p.LibP2PNode{victimNode, spammer.SpammerNode} startNodesAndEnsureConnected(t, signalerCtx, nodes, sporkID) spammer.Start(t) - defer stopNodesAndInspector(t, cancelFunc, nodes, validationInspector) + defer stopNodesAndInspector(t, cancel, nodes, validationInspector) // the first time we spam this message it will be processed completely so we need to ensure // all topics are valid and no duplicates exists. @@ -353,13 +473,13 @@ func TestValidationInspector_InvalidTopicId_Detection(t *testing.T) { require.Equal(t, spammer.SpammerNode.Host().ID(), notification.PeerID) require.True(t, channels.IsInvalidTopicErr(notification.Err)) switch notification.MsgType { - case p2p.CtrlMsgGraft: + case p2pmsg.CtrlMsgGraft: invGraftNotifCount.Inc() require.Equal(t, messageCount, notification.Count) - case p2p.CtrlMsgPrune: + case p2pmsg.CtrlMsgPrune: invPruneNotifCount.Inc() require.Equal(t, messageCount, notification.Count) - case p2p.CtrlMsgIHave: + case p2pmsg.CtrlMsgIHave: require.Equal(t, uint64(ihaveMessageCount), notification.Count) invIHaveNotifCount.Inc() default: @@ -371,7 +491,37 @@ func TestValidationInspector_InvalidTopicId_Detection(t *testing.T) { } } - signalerCtx, cancelFunc, spammer, victimNode, _, _, validationInspector, _ := setupTest(t, unittest.Logger(), role, sporkID, &inspectorConfig, withExpectedNotificationDissemination(expectedNumOfTotalNotif, inspectDisseminatedNotif)) + idProvider := mock.NewIdentityProvider(t) + spammer := corruptlibp2p.NewGossipSubRouterSpammer(t, sporkID, role, idProvider) + + ctx, cancel := context.WithCancel(context.Background()) + signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) + + distributor := mockp2p.NewGossipSubInspectorNotificationDistributor(t) + mockDistributorReadyDoneAware(distributor) + withExpectedNotificationDissemination(expectedNumOfTotalNotif, inspectDisseminatedNotif)(distributor, spammer) + validationInspector, err := validation.NewControlMsgValidationInspector( + unittest.Logger(), + sporkID, + &inspectorConfig, + distributor, + metrics.NewNoopCollector(), + metrics.NewNoopCollector(), + idProvider, + metrics.NewNoopCollector()) + require.NoError(t, err) + corruptInspectorFunc := corruptlibp2p.CorruptInspectorFunc(validationInspector) + victimNode, victimIdentity := p2ptest.NodeFixture( + t, + sporkID, + t.Name(), + idProvider, + p2ptest.WithRole(role), + internal.WithCorruptGossipSub(corruptlibp2p.CorruptGossipSubFactory(), + corruptlibp2p.CorruptGossipSubConfigFactoryWithInspector(corruptInspectorFunc)), + ) + idProvider.On("ByPeerID", victimNode.Host().ID()).Return(&victimIdentity, true).Maybe() + idProvider.On("ByPeerID", spammer.SpammerNode.Host().ID()).Return(&spammer.SpammerId, true).Maybe() // create unknown topic unknownTopic := channels.Topic(fmt.Sprintf("%s/%s", corruptlibp2p.GossipSubTopicIdFixture(), sporkID)) @@ -384,7 +534,7 @@ func TestValidationInspector_InvalidTopicId_Detection(t *testing.T) { nodes := []p2p.LibP2PNode{victimNode, spammer.SpammerNode} startNodesAndEnsureConnected(t, signalerCtx, nodes, sporkID) spammer.Start(t) - defer stopNodesAndInspector(t, cancelFunc, nodes, validationInspector) + defer stopNodesAndInspector(t, cancel, nodes, validationInspector) // prepare to spam - generate control messages graftCtlMsgsWithUnknownTopic := spammer.GenerateCtlMessages(int(controlMessageCount), corruptlibp2p.WithGraft(int(messageCount), unknownTopic.String())) @@ -461,9 +611,9 @@ func TestValidationInspector_DuplicateTopicId_Detection(t *testing.T) { require.True(t, validation.IsErrDuplicateTopic(notification.Err)) require.Equal(t, messageCount, notification.Count) switch notification.MsgType { - case p2p.CtrlMsgGraft: + case p2pmsg.CtrlMsgGraft: invGraftNotifCount.Inc() - case p2p.CtrlMsgPrune: + case p2pmsg.CtrlMsgPrune: invPruneNotifCount.Inc() default: require.Fail(t, "unexpected control message type") @@ -474,7 +624,37 @@ func TestValidationInspector_DuplicateTopicId_Detection(t *testing.T) { } } - signalerCtx, cancelFunc, spammer, victimNode, _, _, validationInspector, _ := setupTest(t, unittest.Logger(), role, sporkID, &inspectorConfig, withExpectedNotificationDissemination(expectedNumOfTotalNotif, inspectDisseminatedNotif)) + idProvider := mock.NewIdentityProvider(t) + spammer := corruptlibp2p.NewGossipSubRouterSpammer(t, sporkID, role, idProvider) + + ctx, cancel := context.WithCancel(context.Background()) + signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) + + distributor := mockp2p.NewGossipSubInspectorNotificationDistributor(t) + mockDistributorReadyDoneAware(distributor) + withExpectedNotificationDissemination(expectedNumOfTotalNotif, inspectDisseminatedNotif)(distributor, spammer) + validationInspector, err := validation.NewControlMsgValidationInspector( + unittest.Logger(), + sporkID, + &inspectorConfig, + distributor, + metrics.NewNoopCollector(), + metrics.NewNoopCollector(), + idProvider, + metrics.NewNoopCollector()) + require.NoError(t, err) + corruptInspectorFunc := corruptlibp2p.CorruptInspectorFunc(validationInspector) + victimNode, victimIdentity := p2ptest.NodeFixture( + t, + sporkID, + t.Name(), + idProvider, + p2ptest.WithRole(role), + internal.WithCorruptGossipSub(corruptlibp2p.CorruptGossipSubFactory(), + corruptlibp2p.CorruptGossipSubConfigFactoryWithInspector(corruptInspectorFunc)), + ) + idProvider.On("ByPeerID", victimNode.Host().ID()).Return(&victimIdentity, true).Maybe() + idProvider.On("ByPeerID", spammer.SpammerNode.Host().ID()).Return(&spammer.SpammerId, true).Maybe() // a topics spork ID is considered invalid if it does not match the current spork ID duplicateTopic := channels.Topic(fmt.Sprintf("%s/%s", channels.PushBlocks, sporkID)) @@ -483,7 +663,7 @@ func TestValidationInspector_DuplicateTopicId_Detection(t *testing.T) { nodes := []p2p.LibP2PNode{victimNode, spammer.SpammerNode} startNodesAndEnsureConnected(t, signalerCtx, nodes, sporkID) spammer.Start(t) - defer stopNodesAndInspector(t, cancelFunc, nodes, validationInspector) + defer stopNodesAndInspector(t, cancel, nodes, validationInspector) // prepare to spam - generate control messages graftCtlMsgsDuplicateTopic := spammer.GenerateCtlMessages(int(controlMessageCount), corruptlibp2p.WithGraft(int(messageCount), duplicateTopic.String())) @@ -538,9 +718,9 @@ func TestValidationInspector_UnknownClusterId_Detection(t *testing.T) { require.True(t, channels.IsUnknownClusterIDErr(notification.Err)) require.Equal(t, messageCount, notification.Count) switch notification.MsgType { - case p2p.CtrlMsgGraft: + case p2pmsg.CtrlMsgGraft: invGraftNotifCount.Inc() - case p2p.CtrlMsgPrune: + case p2pmsg.CtrlMsgPrune: invPruneNotifCount.Inc() default: require.Fail(t, "unexpected control message type") @@ -551,8 +731,36 @@ func TestValidationInspector_UnknownClusterId_Detection(t *testing.T) { } } - signalerCtx, cancelFunc, spammer, victimNode, _, _, validationInspector, idProvider := setupTest(t, unittest.Logger(), role, sporkID, &inspectorConfig, withExpectedNotificationDissemination(expectedNumOfTotalNotif, inspectDisseminatedNotif)) - idProvider.On("ByPeerID", spammer.SpammerNode.Host().ID()).Return(&spammer.SpammerId, true).Twice() + idProvider := mock.NewIdentityProvider(t) + spammer := corruptlibp2p.NewGossipSubRouterSpammer(t, sporkID, role, idProvider) + ctx, cancel := context.WithCancel(context.Background()) + signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) + + distributor := mockp2p.NewGossipSubInspectorNotificationDistributor(t) + mockDistributorReadyDoneAware(distributor) + withExpectedNotificationDissemination(expectedNumOfTotalNotif, inspectDisseminatedNotif)(distributor, spammer) + validationInspector, err := validation.NewControlMsgValidationInspector( + unittest.Logger(), + sporkID, + &inspectorConfig, + distributor, + metrics.NewNoopCollector(), + metrics.NewNoopCollector(), + idProvider, + metrics.NewNoopCollector()) + require.NoError(t, err) + corruptInspectorFunc := corruptlibp2p.CorruptInspectorFunc(validationInspector) + victimNode, victimIdentity := p2ptest.NodeFixture( + t, + sporkID, + t.Name(), + idProvider, + p2ptest.WithRole(role), + internal.WithCorruptGossipSub(corruptlibp2p.CorruptGossipSubFactory(), + corruptlibp2p.CorruptGossipSubConfigFactoryWithInspector(corruptInspectorFunc)), + ) + idProvider.On("ByPeerID", victimNode.Host().ID()).Return(&victimIdentity, true).Maybe() + idProvider.On("ByPeerID", spammer.SpammerNode.Host().ID()).Return(&spammer.SpammerId, true).Times(3) // setup cluster prefixed topic with an invalid cluster ID unknownClusterID := channels.Topic(channels.SyncCluster("unknown-cluster-ID")) @@ -563,7 +771,7 @@ func TestValidationInspector_UnknownClusterId_Detection(t *testing.T) { nodes := []p2p.LibP2PNode{victimNode, spammer.SpammerNode} startNodesAndEnsureConnected(t, signalerCtx, nodes, sporkID) spammer.Start(t) - defer stopNodesAndInspector(t, cancelFunc, nodes, validationInspector) + defer stopNodesAndInspector(t, cancel, nodes, validationInspector) // prepare to spam - generate control messages graftCtlMsgsDuplicateTopic := spammer.GenerateCtlMessages(int(controlMessageCount), corruptlibp2p.WithGraft(int(messageCount), unknownClusterID.String())) @@ -608,7 +816,7 @@ func TestValidationInspector_ActiveClusterIdsNotSet_Graft_Detection(t *testing.T require.True(t, validation.IsErrActiveClusterIDsNotSet(notification.Err)) require.Equal(t, spammer.SpammerNode.Host().ID(), notification.PeerID) switch notification.MsgType { - case p2p.CtrlMsgGraft: + case p2pmsg.CtrlMsgGraft: invGraftNotifCount.Inc() default: require.Fail(t, "unexpected control message type") @@ -620,8 +828,36 @@ func TestValidationInspector_ActiveClusterIdsNotSet_Graft_Detection(t *testing.T } } - signalerCtx, cancelFunc, spammer, victimNode, _, _, validationInspector, idProvider := setupTest(t, unittest.Logger(), role, sporkID, &inspectorConfig, withExpectedNotificationDissemination(expectedNumOfTotalNotif, inspectDisseminatedNotif)) - idProvider.On("ByPeerID", spammer.SpammerNode.Host().ID()).Return(&spammer.SpammerId, true).Times(int(controlMessageCount)) + idProvider := mock.NewIdentityProvider(t) + spammer := corruptlibp2p.NewGossipSubRouterSpammer(t, sporkID, role, idProvider) + ctx, cancel := context.WithCancel(context.Background()) + signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) + + distributor := mockp2p.NewGossipSubInspectorNotificationDistributor(t) + mockDistributorReadyDoneAware(distributor) + withExpectedNotificationDissemination(expectedNumOfTotalNotif, inspectDisseminatedNotif)(distributor, spammer) + validationInspector, err := validation.NewControlMsgValidationInspector( + unittest.Logger(), + sporkID, + &inspectorConfig, + distributor, + metrics.NewNoopCollector(), + metrics.NewNoopCollector(), + idProvider, + metrics.NewNoopCollector()) + require.NoError(t, err) + corruptInspectorFunc := corruptlibp2p.CorruptInspectorFunc(validationInspector) + victimNode, victimIdentity := p2ptest.NodeFixture( + t, + sporkID, + t.Name(), + idProvider, + p2ptest.WithRole(role), + internal.WithCorruptGossipSub(corruptlibp2p.CorruptGossipSubFactory(), + corruptlibp2p.CorruptGossipSubConfigFactoryWithInspector(corruptInspectorFunc)), + ) + idProvider.On("ByPeerID", victimNode.Host().ID()).Return(&victimIdentity, true).Maybe() + idProvider.On("ByPeerID", spammer.SpammerNode.Host().ID()).Return(&spammer.SpammerId, true).Times(int(controlMessageCount + 1)) // we deliberately avoid setting the cluster IDs so that we eventually receive errors after we have exceeded the allowed cluster // prefixed hard threshold @@ -629,7 +865,7 @@ func TestValidationInspector_ActiveClusterIdsNotSet_Graft_Detection(t *testing.T nodes := []p2p.LibP2PNode{victimNode, spammer.SpammerNode} startNodesAndEnsureConnected(t, signalerCtx, nodes, sporkID) spammer.Start(t) - defer stopNodesAndInspector(t, cancelFunc, nodes, validationInspector) + defer stopNodesAndInspector(t, cancel, nodes, validationInspector) // generate multiple control messages with GRAFT's for randomly generated // cluster prefixed channels, this ensures we do not encounter duplicate topic ID errors ctlMsgs := spammer.GenerateCtlMessages(int(controlMessageCount), @@ -672,7 +908,7 @@ func TestValidationInspector_ActiveClusterIdsNotSet_Prune_Detection(t *testing.T require.True(t, validation.IsErrActiveClusterIDsNotSet(notification.Err)) require.Equal(t, spammer.SpammerNode.Host().ID(), notification.PeerID) switch notification.MsgType { - case p2p.CtrlMsgPrune: + case p2pmsg.CtrlMsgPrune: invPruneNotifCount.Inc() default: require.Fail(t, "unexpected control message type") @@ -684,8 +920,36 @@ func TestValidationInspector_ActiveClusterIdsNotSet_Prune_Detection(t *testing.T } } - signalerCtx, cancelFunc, spammer, victimNode, _, _, validationInspector, idProvider := setupTest(t, unittest.Logger(), role, sporkID, &inspectorConfig, withExpectedNotificationDissemination(expectedNumOfTotalNotif, inspectDisseminatedNotif)) - idProvider.On("ByPeerID", spammer.SpammerNode.Host().ID()).Return(&spammer.SpammerId, true).Times(int(controlMessageCount)) + idProvider := mock.NewIdentityProvider(t) + spammer := corruptlibp2p.NewGossipSubRouterSpammer(t, sporkID, role, idProvider) + ctx, cancel := context.WithCancel(context.Background()) + signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) + + distributor := mockp2p.NewGossipSubInspectorNotificationDistributor(t) + mockDistributorReadyDoneAware(distributor) + withExpectedNotificationDissemination(expectedNumOfTotalNotif, inspectDisseminatedNotif)(distributor, spammer) + validationInspector, err := validation.NewControlMsgValidationInspector( + unittest.Logger(), + sporkID, + &inspectorConfig, + distributor, + metrics.NewNoopCollector(), + metrics.NewNoopCollector(), + idProvider, + metrics.NewNoopCollector()) + require.NoError(t, err) + corruptInspectorFunc := corruptlibp2p.CorruptInspectorFunc(validationInspector) + victimNode, victimIdentity := p2ptest.NodeFixture( + t, + sporkID, + t.Name(), + idProvider, + p2ptest.WithRole(role), + internal.WithCorruptGossipSub(corruptlibp2p.CorruptGossipSubFactory(), + corruptlibp2p.CorruptGossipSubConfigFactoryWithInspector(corruptInspectorFunc)), + ) + idProvider.On("ByPeerID", victimNode.Host().ID()).Return(&victimIdentity, true).Maybe() + idProvider.On("ByPeerID", spammer.SpammerNode.Host().ID()).Return(&spammer.SpammerId, true).Times(int(controlMessageCount + 1)) // we deliberately avoid setting the cluster IDs so that we eventually receive errors after we have exceeded the allowed cluster // prefixed hard threshold @@ -693,7 +957,7 @@ func TestValidationInspector_ActiveClusterIdsNotSet_Prune_Detection(t *testing.T nodes := []p2p.LibP2PNode{victimNode, spammer.SpammerNode} startNodesAndEnsureConnected(t, signalerCtx, nodes, sporkID) spammer.Start(t) - defer stopNodesAndInspector(t, cancelFunc, nodes, validationInspector) + defer stopNodesAndInspector(t, cancel, nodes, validationInspector) // generate multiple control messages with GRAFT's for randomly generated // cluster prefixed channels, this ensures we do not encounter duplicate topic ID errors ctlMsgs := spammer.GenerateCtlMessages(int(controlMessageCount), @@ -745,9 +1009,9 @@ func TestValidationInspector_UnstakedNode_Detection(t *testing.T) { require.True(t, validation.IsErrUnstakedPeer(notification.Err)) require.Equal(t, messageCount, notification.Count) switch notification.MsgType { - case p2p.CtrlMsgGraft: + case p2pmsg.CtrlMsgGraft: invGraftNotifCount.Inc() - case p2p.CtrlMsgPrune: + case p2pmsg.CtrlMsgPrune: invPruneNotifCount.Inc() default: require.Fail(t, "unexpected control message type") @@ -758,8 +1022,36 @@ func TestValidationInspector_UnstakedNode_Detection(t *testing.T) { } } - signalerCtx, cancelFunc, spammer, victimNode, _, _, validationInspector, idProvider := setupTest(t, unittest.Logger(), role, sporkID, &inspectorConfig, withExpectedNotificationDissemination(expectedNumOfTotalNotif, inspectDisseminatedNotif)) - idProvider.On("ByPeerID", spammer.SpammerNode.Host().ID()).Return(nil, false).Twice() + idProvider := mock.NewIdentityProvider(t) + spammer := corruptlibp2p.NewGossipSubRouterSpammer(t, sporkID, role, idProvider) + ctx, cancel := context.WithCancel(context.Background()) + signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) + + distributor := mockp2p.NewGossipSubInspectorNotificationDistributor(t) + mockDistributorReadyDoneAware(distributor) + withExpectedNotificationDissemination(expectedNumOfTotalNotif, inspectDisseminatedNotif)(distributor, spammer) + validationInspector, err := validation.NewControlMsgValidationInspector( + unittest.Logger(), + sporkID, + &inspectorConfig, + distributor, + metrics.NewNoopCollector(), + metrics.NewNoopCollector(), + idProvider, + metrics.NewNoopCollector()) + require.NoError(t, err) + corruptInspectorFunc := corruptlibp2p.CorruptInspectorFunc(validationInspector) + victimNode, victimIdentity := p2ptest.NodeFixture( + t, + sporkID, + t.Name(), + idProvider, + p2ptest.WithRole(role), + internal.WithCorruptGossipSub(corruptlibp2p.CorruptGossipSubFactory(), + corruptlibp2p.CorruptGossipSubConfigFactoryWithInspector(corruptInspectorFunc)), + ) + idProvider.On("ByPeerID", victimNode.Host().ID()).Return(&victimIdentity, true).Maybe() + idProvider.On("ByPeerID", spammer.SpammerNode.Host().ID()).Return(nil, false).Times(3) // setup cluster prefixed topic with an invalid cluster ID clusterID := flow.ChainID("known-cluster-id") @@ -771,7 +1063,7 @@ func TestValidationInspector_UnstakedNode_Detection(t *testing.T) { nodes := []p2p.LibP2PNode{victimNode, spammer.SpammerNode} startNodesAndEnsureConnected(t, signalerCtx, nodes, sporkID) spammer.Start(t) - defer stopNodesAndInspector(t, cancelFunc, nodes, validationInspector) + defer stopNodesAndInspector(t, cancel, nodes, validationInspector) // prepare to spam - generate control messages graftCtlMsgsDuplicateTopic := spammer.GenerateCtlMessages(int(controlMessageCount), corruptlibp2p.WithGraft(int(messageCount), clusterIDTopic.String())) @@ -805,7 +1097,7 @@ func withExpectedNotificationDissemination(expectedNumOfTotalNotif int, f onNoti } // setupTest sets up common components of RPC inspector test. -func setupTest(t *testing.T, logger zerolog.Logger, role flow.Role, sporkID flow.Identifier, inspectorConfig *netconf.GossipSubRPCValidationInspectorConfigs, mockDistributorOpts ...mockDistributorOption) (*irrecoverable.MockSignalerContext, context.CancelFunc, *corruptlibp2p.GossipSubRouterSpammer, p2p.LibP2PNode, flow.Identity, *mockp2p.GossipSubInspectorNotificationDistributor, *validation.ControlMsgValidationInspector, *mock.IdentityProvider) { +func setupTest(t *testing.T, logger zerolog.Logger, role flow.Role, sporkID flow.Identifier, inspectorConfig *p2pconf.GossipSubRPCValidationInspectorConfigs, mockDistributorOpts ...mockDistributorOption) (*irrecoverable.MockSignalerContext, context.CancelFunc, *corruptlibp2p.GossipSubRouterSpammer, p2p.LibP2PNode, flow.Identity, *mockp2p.GossipSubInspectorNotificationDistributor, *validation.ControlMsgValidationInspector, *mock.IdentityProvider) { idProvider := mock.NewIdentityProvider(t) spammer := corruptlibp2p.NewGossipSubRouterSpammer(t, sporkID, role, idProvider) ctx, cancel := context.WithCancel(context.Background()) @@ -843,7 +1135,7 @@ func TestGossipSubSpamMitigationIntegration(t *testing.T) { t.Parallel() idProvider := mock.NewIdentityProvider(t) sporkID := unittest.IdentifierFixture() - spammer := corruptlibp2p.NewGossipSubRouterSpammer(t, sporkID, flow.RoleConsensus, nil) + spammer := corruptlibp2p.NewGossipSubRouterSpammer(t, sporkID, flow.RoleConsensus, idProvider) ctx, cancel := context.WithCancel(context.Background()) signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) diff --git a/insecure/integration/functional/test/gossipsub/scoring/scoring_test.go b/insecure/integration/functional/test/gossipsub/scoring/scoring_test.go new file mode 100644 index 00000000000..ec024775cf0 --- /dev/null +++ b/insecure/integration/functional/test/gossipsub/scoring/scoring_test.go @@ -0,0 +1,173 @@ +package scoring + +import ( + "context" + "testing" + "time" + + pubsub_pb "github.com/libp2p/go-libp2p-pubsub/pb" + "github.com/libp2p/go-libp2p/core/peer" + "github.com/stretchr/testify/require" + + "github.com/onflow/flow-go/insecure/corruptlibp2p" + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module/irrecoverable" + "github.com/onflow/flow-go/module/mock" + "github.com/onflow/flow-go/network/channels" + "github.com/onflow/flow-go/network/p2p" + "github.com/onflow/flow-go/network/p2p/scoring" + p2ptest "github.com/onflow/flow-go/network/p2p/test" + "github.com/onflow/flow-go/utils/unittest" +) + +// TestGossipSubInvalidMessageDelivery_Integration tests that when a victim peer is spammed with invalid messages from +// a spammer peer, the victim will eventually penalize the spammer and stop receiving messages from them. +// Note: the term integration is used here because it requires integrating all components of the libp2p stack. +func TestGossipSubInvalidMessageDelivery_Integration(t *testing.T) { + tt := []struct { + name string + spamMsgFactory func(spammerId peer.ID, victimId peer.ID, topic channels.Topic) *pubsub_pb.Message + }{ + { + name: "unknown peer, invalid signature", + spamMsgFactory: func(spammerId peer.ID, _ peer.ID, topic channels.Topic) *pubsub_pb.Message { + return p2ptest.PubsubMessageFixture(t, p2ptest.WithTopic(topic.String())) + }, + }, + { + name: "unknown peer, missing signature", + spamMsgFactory: func(spammerId peer.ID, _ peer.ID, topic channels.Topic) *pubsub_pb.Message { + return p2ptest.PubsubMessageFixture(t, p2ptest.WithTopic(topic.String()), p2ptest.WithoutSignature()) + }, + }, + { + name: "known peer, invalid signature", + spamMsgFactory: func(spammerId peer.ID, _ peer.ID, topic channels.Topic) *pubsub_pb.Message { + return p2ptest.PubsubMessageFixture(t, p2ptest.WithFrom(spammerId), p2ptest.WithTopic(topic.String())) + }, + }, + { + name: "known peer, missing signature", + spamMsgFactory: func(spammerId peer.ID, _ peer.ID, topic channels.Topic) *pubsub_pb.Message { + return p2ptest.PubsubMessageFixture(t, p2ptest.WithFrom(spammerId), p2ptest.WithTopic(topic.String()), p2ptest.WithoutSignature()) + }, + }, + { + name: "self-origin, invalid signature", // bounce back our own messages + spamMsgFactory: func(_ peer.ID, victimId peer.ID, topic channels.Topic) *pubsub_pb.Message { + return p2ptest.PubsubMessageFixture(t, p2ptest.WithFrom(victimId), p2ptest.WithTopic(topic.String())) + }, + }, + { + name: "self-origin, no signature", // bounce back our own messages + spamMsgFactory: func(_ peer.ID, victimId peer.ID, topic channels.Topic) *pubsub_pb.Message { + return p2ptest.PubsubMessageFixture(t, p2ptest.WithFrom(victimId), p2ptest.WithTopic(topic.String()), p2ptest.WithoutSignature()) + }, + }, + { + name: "no sender", + spamMsgFactory: func(_ peer.ID, victimId peer.ID, topic channels.Topic) *pubsub_pb.Message { + return p2ptest.PubsubMessageFixture(t, p2ptest.WithoutSignerId(), p2ptest.WithTopic(topic.String())) + }, + }, + { + name: "no sender, missing signature", + spamMsgFactory: func(_ peer.ID, victimId peer.ID, topic channels.Topic) *pubsub_pb.Message { + return p2ptest.PubsubMessageFixture(t, p2ptest.WithoutSignerId(), p2ptest.WithTopic(topic.String()), p2ptest.WithoutSignature()) + }, + }, + } + + for _, tc := range tt { + t.Run(tc.name, func(t *testing.T) { + testGossipSubInvalidMessageDeliveryScoring(t, tc.spamMsgFactory) + }) + } +} + +// testGossipSubInvalidMessageDeliveryScoring tests that when a victim peer is spammed with invalid messages from +// a spammer peer, the victim will eventually penalize the spammer and stop receiving messages from them. +// Args: +// - t: the test instance. +// - spamMsgFactory: a function that creates unique invalid messages to spam the victim with. +func testGossipSubInvalidMessageDeliveryScoring(t *testing.T, spamMsgFactory func(peer.ID, peer.ID, channels.Topic) *pubsub_pb.Message) { + role := flow.RoleConsensus + sporkId := unittest.IdentifierFixture() + blockTopic := channels.TopicFromChannel(channels.PushBlocks, sporkId) + + idProvider := mock.NewIdentityProvider(t) + spammer := corruptlibp2p.NewGossipSubRouterSpammer(t, sporkId, role, idProvider) + ctx, cancel := context.WithCancel(context.Background()) + signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) + + victimNode, victimIdentity := p2ptest.NodeFixture( + t, + sporkId, + t.Name(), + idProvider, + p2ptest.WithRole(role), + p2ptest.WithPeerScoreTracerInterval(1*time.Second), + p2ptest.WithPeerScoringEnabled(idProvider), + ) + + idProvider.On("ByPeerID", victimNode.Host().ID()).Return(&victimIdentity, true).Maybe() + idProvider.On("ByPeerID", spammer.SpammerNode.Host().ID()).Return(&spammer.SpammerId, true).Maybe() + ids := flow.IdentityList{&spammer.SpammerId, &victimIdentity} + nodes := []p2p.LibP2PNode{spammer.SpammerNode, victimNode} + + p2ptest.StartNodes(t, signalerCtx, nodes, 100*time.Millisecond) + defer p2ptest.StopNodes(t, nodes, cancel, 2*time.Second) + + p2ptest.LetNodesDiscoverEachOther(t, ctx, nodes, ids) + p2ptest.TryConnectionAndEnsureConnected(t, ctx, nodes) + + // checks end-to-end message delivery works on GossipSub + p2ptest.EnsurePubsubMessageExchange(t, ctx, nodes, func() (interface{}, channels.Topic) { + return unittest.ProposalFixture(), blockTopic + }) + + totalSpamMessages := 20 + for i := 0; i <= totalSpamMessages; i++ { + spammer.SpamControlMessage(t, victimNode, + spammer.GenerateCtlMessages(1), + spamMsgFactory(spammer.SpammerNode.Host().ID(), victimNode.Host().ID(), blockTopic)) + } + + // wait for at most 3 seconds for the victim node to penalize the spammer node. + // Each heartbeat is 1 second, so 3 heartbeats should be enough to penalize the spammer node. + // Ideally, we should wait for 1 heartbeat, but the score may not be updated immediately after the heartbeat. + require.Eventually(t, func() bool { + spammerScore, ok := victimNode.PeerScoreExposer().GetScore(spammer.SpammerNode.Host().ID()) + if !ok { + return false + } + if spammerScore >= scoring.DefaultGossipThreshold { + // ensure the score is low enough so that no gossip is routed by victim node to spammer node. + return false + } + if spammerScore >= scoring.DefaultPublishThreshold { + // ensure the score is low enough so that non of the published messages of the victim node are routed to the spammer node. + return false + } + if spammerScore >= scoring.DefaultGraylistThreshold { + // ensure the score is low enough so that the victim node does not accept RPC messages from the spammer node. + return false + } + + return true + }, 3*time.Second, 100*time.Millisecond) + + topicsSnapshot, ok := victimNode.PeerScoreExposer().GetTopicScores(spammer.SpammerNode.Host().ID()) + require.True(t, ok) + require.NotNil(t, topicsSnapshot, "topic scores must not be nil") + require.NotEmpty(t, topicsSnapshot, "topic scores must not be empty") + blkTopicSnapshot, ok := topicsSnapshot[blockTopic.String()] + require.True(t, ok) + + // ensure that the topic snapshot of the spammer contains a record of at least (60%) of the spam messages sent. The 60% is to account for the messages that were delivered before the score was updated, after the spammer is PRUNED, as well as to account for decay. + require.True(t, blkTopicSnapshot.InvalidMessageDeliveries > 0.6*float64(totalSpamMessages), "invalid message deliveries must be greater than %f. invalid message deliveries: %f", 0.9*float64(totalSpamMessages), blkTopicSnapshot.InvalidMessageDeliveries) + + p2ptest.EnsureNoPubsubExchangeBetweenGroups(t, ctx, []p2p.LibP2PNode{victimNode}, []p2p.LibP2PNode{spammer.SpammerNode}, func() (interface{}, channels.Topic) { + return unittest.ProposalFixture(), blockTopic + }) +} diff --git a/insecure/integration/test/composability_test.go b/insecure/integration/tests/composability_test.go similarity index 99% rename from insecure/integration/test/composability_test.go rename to insecure/integration/tests/composability_test.go index c7ba04d3b7a..4bac2aeb0c5 100644 --- a/insecure/integration/test/composability_test.go +++ b/insecure/integration/tests/composability_test.go @@ -1,4 +1,4 @@ -package test +package tests import ( "context" diff --git a/insecure/integration/test/mock_orchestrator.go b/insecure/integration/tests/mock_orchestrator.go similarity index 99% rename from insecure/integration/test/mock_orchestrator.go rename to insecure/integration/tests/mock_orchestrator.go index b992e1d6c20..5d11879d6df 100644 --- a/insecure/integration/test/mock_orchestrator.go +++ b/insecure/integration/tests/mock_orchestrator.go @@ -1,4 +1,4 @@ -package test +package tests import ( "github.com/onflow/flow-go/insecure" diff --git a/integration/go.mod b/integration/go.mod index 629b5e458ab..4be06e9a9cf 100644 --- a/integration/go.mod +++ b/integration/go.mod @@ -287,7 +287,7 @@ require ( github.com/whyrusleeping/timecache v0.0.0-20160911033111-cfcb2f1abfee // indirect github.com/x448/float16 v0.8.4 // indirect github.com/xanzy/ssh-agent v0.3.3 // indirect - github.com/yhassanzadeh13/go-libp2p-pubsub v0.6.2-0.20221208234712-b44d9133e4ee // indirect + github.com/yhassanzadeh13/go-libp2p-pubsub v0.6.11-flow-expose-msg // indirect github.com/yusufpapurcu/wmi v1.2.2 // indirect github.com/zeebo/blake3 v0.2.3 // indirect github.com/zeebo/xxh3 v1.0.2 // indirect diff --git a/integration/go.sum b/integration/go.sum index 374df68f7cf..3545b49110c 100644 --- a/integration/go.sum +++ b/integration/go.sum @@ -1733,8 +1733,8 @@ github.com/xeipuuv/gojsonschema v0.0.0-20180618132009-1d523034197f/go.mod h1:5yf github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xlab/treeprint v0.0.0-20180616005107-d6fb6747feb6/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= -github.com/yhassanzadeh13/go-libp2p-pubsub v0.6.2-0.20221208234712-b44d9133e4ee h1:yFB2xjfswpuRh8FHagdBMKcBMltjr5u/XKzX6fkJO5E= -github.com/yhassanzadeh13/go-libp2p-pubsub v0.6.2-0.20221208234712-b44d9133e4ee/go.mod h1:Tylw4k1H86gbJx84i3r7qahN/mBaeMpUBvHY0Igshfw= +github.com/yhassanzadeh13/go-libp2p-pubsub v0.6.11-flow-expose-msg h1:CtmNcsXmPNYnpzogVjrIy99if5Hvh79kd+ecoQgA7Vg= +github.com/yhassanzadeh13/go-libp2p-pubsub v0.6.11-flow-expose-msg/go.mod h1:Tylw4k1H86gbJx84i3r7qahN/mBaeMpUBvHY0Igshfw= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= diff --git a/module/metrics.go b/module/metrics.go index 9ca9eb3d992..338f87c1ecc 100644 --- a/module/metrics.go +++ b/module/metrics.go @@ -610,7 +610,7 @@ type RestMetrics interface { // Example recorder taken from: // https://github.com/slok/go-http-metrics/blob/master/metrics/prometheus/prometheus.go httpmetrics.Recorder - AddTotalRequests(ctx context.Context, service string, id string) + AddTotalRequests(ctx context.Context, method string, routeName string) } type GRPCConnectionPoolMetrics interface { diff --git a/module/metrics/rest_api.go b/module/metrics/rest_api.go index efa12688a81..0ab08d0a3ca 100644 --- a/module/metrics/rest_api.go +++ b/module/metrics/rest_api.go @@ -81,7 +81,7 @@ func NewRestCollector(cfg metricsProm.Config) module.RestMetrics { Subsystem: "http", Name: "requests_total", Help: "The number of requests handled over time.", - }, []string{cfg.ServiceLabel, cfg.HandlerIDLabel}), + }, []string{cfg.MethodLabel, cfg.HandlerIDLabel}), } cfg.Registry.MustRegister( @@ -108,6 +108,6 @@ func (r *RestCollector) AddInflightRequests(_ context.Context, p httpmetrics.HTT } // New custom method to track all requests made for every REST API request -func (r *RestCollector) AddTotalRequests(_ context.Context, method string, id string) { - r.httpRequestsTotal.WithLabelValues(method, id).Inc() +func (r *RestCollector) AddTotalRequests(_ context.Context, method string, routeName string) { + r.httpRequestsTotal.WithLabelValues(method, routeName).Inc() } diff --git a/module/mock/access_metrics.go b/module/mock/access_metrics.go index f5a76f391ae..83690a36625 100644 --- a/module/mock/access_metrics.go +++ b/module/mock/access_metrics.go @@ -23,9 +23,9 @@ func (_m *AccessMetrics) AddInflightRequests(ctx context.Context, props metrics. _m.Called(ctx, props, quantity) } -// AddTotalRequests provides a mock function with given fields: ctx, service, id -func (_m *AccessMetrics) AddTotalRequests(ctx context.Context, service string, id string) { - _m.Called(ctx, service, id) +// AddTotalRequests provides a mock function with given fields: ctx, method, routeName +func (_m *AccessMetrics) AddTotalRequests(ctx context.Context, method string, routeName string) { + _m.Called(ctx, method, routeName) } // ConnectionAddedToPool provides a mock function with given fields: diff --git a/module/mock/rest_metrics.go b/module/mock/rest_metrics.go index f1544ca5823..b5fbd8bc50a 100644 --- a/module/mock/rest_metrics.go +++ b/module/mock/rest_metrics.go @@ -21,9 +21,9 @@ func (_m *RestMetrics) AddInflightRequests(ctx context.Context, props metrics.HT _m.Called(ctx, props, quantity) } -// AddTotalRequests provides a mock function with given fields: ctx, service, id -func (_m *RestMetrics) AddTotalRequests(ctx context.Context, service string, id string) { - _m.Called(ctx, service, id) +// AddTotalRequests provides a mock function with given fields: ctx, method, routeName +func (_m *RestMetrics) AddTotalRequests(ctx context.Context, method string, routeName string) { + _m.Called(ctx, method, routeName) } // ObserveHTTPRequestDuration provides a mock function with given fields: ctx, props, duration diff --git a/network/alsp/manager/manager_test.go b/network/alsp/manager/manager_test.go index dbe5207e431..03f012bb206 100644 --- a/network/alsp/manager/manager_test.go +++ b/network/alsp/manager/manager_test.go @@ -53,16 +53,10 @@ func TestNetworkPassesReportedMisbehavior(t *testing.T) { misbehaviorReportManger.On("Ready").Return(readyDoneChan).Once() misbehaviorReportManger.On("Done").Return(readyDoneChan).Once() + ids, nodes, _ := testutils.LibP2PNodeForMiddlewareFixture(t, 1) + mws, _ := testutils.MiddlewareFixtures(t, ids, nodes, testutils.MiddlewareConfigFixture(t)) - ids, nodes, mws, _, _ := testutils.GenerateIDsAndMiddlewares( - t, - 1, - unittest.Logger(), - unittest.NetworkCodec(), - unittest.NetworkSlashingViolationsConsumer(unittest.Logger(), metrics.NewNoopCollector())) - sms := testutils.GenerateSubscriptionManagers(t, mws) - - networkCfg := testutils.NetworkConfigFixture(t, unittest.Logger(), *ids[0], ids, mws[0], sms[0]) + networkCfg := testutils.NetworkConfigFixture(t, *ids[0], ids, mws[0]) net, err := p2p.NewNetwork(networkCfg, p2p.WithAlspManager(misbehaviorReportManger)) require.NoError(t, err) @@ -116,15 +110,9 @@ func TestHandleReportedMisbehavior_Cache_Integration(t *testing.T) { return cache }), } - - ids, nodes, mws, _, _ := testutils.GenerateIDsAndMiddlewares( - t, - 1, - unittest.Logger(), - unittest.NetworkCodec(), - unittest.NetworkSlashingViolationsConsumer(unittest.Logger(), metrics.NewNoopCollector())) - sms := testutils.GenerateSubscriptionManagers(t, mws) - networkCfg := testutils.NetworkConfigFixture(t, unittest.Logger(), *ids[0], ids, mws[0], sms[0], p2p.WithAlspConfig(cfg)) + ids, nodes, _ := testutils.LibP2PNodeForMiddlewareFixture(t, 1) + mws, _ := testutils.MiddlewareFixtures(t, ids, nodes, testutils.MiddlewareConfigFixture(t)) + networkCfg := testutils.NetworkConfigFixture(t, *ids[0], ids, mws[0], p2p.WithAlspConfig(cfg)) net, err := p2p.NewNetwork(networkCfg) require.NoError(t, err) @@ -216,14 +204,10 @@ func TestHandleReportedMisbehavior_And_DisallowListing_Integration(t *testing.T) }), } - ids, nodes, mws, _, _ := testutils.GenerateIDsAndMiddlewares( - t, - 3, - unittest.Logger(), - unittest.NetworkCodec(), - unittest.NetworkSlashingViolationsConsumer(unittest.Logger(), metrics.NewNoopCollector())) - sms := testutils.GenerateSubscriptionManagers(t, mws) - networkCfg := testutils.NetworkConfigFixture(t, unittest.Logger(), *ids[0], ids, mws[0], sms[0], p2p.WithAlspConfig(cfg)) + ids, nodes, _ := testutils.LibP2PNodeForMiddlewareFixture(t, 3, + p2ptest.WithPeerManagerEnabled(p2ptest.PeerManagerConfigFixture(), nil)) + mws, _ := testutils.MiddlewareFixtures(t, ids, nodes, testutils.MiddlewareConfigFixture(t)) + networkCfg := testutils.NetworkConfigFixture(t, *ids[0], ids, mws[0], p2p.WithAlspConfig(cfg)) victimNetwork, err := p2p.NewNetwork(networkCfg) require.NoError(t, err) @@ -294,15 +278,9 @@ func TestMisbehaviorReportMetrics(t *testing.T) { alspMetrics := mockmodule.NewAlspMetrics(t) cfg.AlspMetrics = alspMetrics - ids, nodes, mws, _, _ := testutils.GenerateIDsAndMiddlewares( - t, - 1, - unittest.Logger(), - unittest.NetworkCodec(), - unittest.NetworkSlashingViolationsConsumer(unittest.Logger(), metrics.NewNoopCollector())) - sms := testutils.GenerateSubscriptionManagers(t, mws) - - networkCfg := testutils.NetworkConfigFixture(t, unittest.Logger(), *ids[0], ids, mws[0], sms[0], p2p.WithAlspConfig(cfg)) + ids, nodes, _ := testutils.LibP2PNodeForMiddlewareFixture(t, 1) + mws, _ := testutils.MiddlewareFixtures(t, ids, nodes, testutils.MiddlewareConfigFixture(t)) + networkCfg := testutils.NetworkConfigFixture(t, *ids[0], ids, mws[0], p2p.WithAlspConfig(cfg)) net, err := p2p.NewNetwork(networkCfg) require.NoError(t, err) diff --git a/network/internal/p2pfixtures/fixtures.go b/network/internal/p2pfixtures/fixtures.go index c98c261c33d..40229337dfa 100644 --- a/network/internal/p2pfixtures/fixtures.go +++ b/network/internal/p2pfixtures/fixtures.go @@ -26,15 +26,15 @@ import ( "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module/id" "github.com/onflow/flow-go/module/metrics" + flownet "github.com/onflow/flow-go/network" "github.com/onflow/flow-go/network/channels" "github.com/onflow/flow-go/network/internal/p2putils" - "github.com/onflow/flow-go/network/internal/testutils" "github.com/onflow/flow-go/network/message" "github.com/onflow/flow-go/network/p2p" p2pdht "github.com/onflow/flow-go/network/p2p/dht" "github.com/onflow/flow-go/network/p2p/keyutils" "github.com/onflow/flow-go/network/p2p/p2pbuilder" - inspectorbuilder "github.com/onflow/flow-go/network/p2p/p2pbuilder/inspector" + p2pconfig "github.com/onflow/flow-go/network/p2p/p2pbuilder/config" "github.com/onflow/flow-go/network/p2p/tracer" "github.com/onflow/flow-go/network/p2p/unicast" "github.com/onflow/flow-go/network/p2p/unicast/protocols" @@ -97,6 +97,7 @@ func WithSubscriptionFilter(filter pubsub.SubscriptionFilter) nodeOpt { } } +// TODO: this should be replaced by node fixture: https://github.com/onflow/flow-go/blob/master/network/p2p/test/fixtures.go func CreateNode(t *testing.T, networkKey crypto.PrivateKey, sporkID flow.Identifier, logger zerolog.Logger, nodeIds flow.IdentityList, opts ...nodeOpt) p2p.LibP2PNode { idProvider := id.NewFixedIdentityProvider(nodeIds) defaultFlowConfig, err := config.DefaultConfig() @@ -107,17 +108,20 @@ func CreateNode(t *testing.T, networkKey crypto.PrivateKey, sporkID flow.Identif idProvider, defaultFlowConfig.NetworkConfig.GossipSubConfig.LocalMeshLogInterval) - met := metrics.NewNoopCollector() - rpcInspectorSuite, err := inspectorbuilder.NewGossipSubInspectorBuilder(logger, sporkID, &defaultFlowConfig.NetworkConfig.GossipSubConfig.GossipSubRPCInspectorsConfig, idProvider, met).Build() - require.NoError(t, err) - builder := p2pbuilder.NewNodeBuilder( logger, - met, + &p2pconfig.MetricsConfig{ + HeroCacheFactory: metrics.NewNoopHeroCacheMetricsFactory(), + Metrics: metrics.NewNoopCollector(), + }, + flownet.PrivateNetwork, unittest.DefaultAddress, networkKey, sporkID, + idProvider, &defaultFlowConfig.NetworkConfig.ResourceManagerConfig, + &defaultFlowConfig.NetworkConfig.GossipSubRPCInspectorsConfig, + p2pconfig.PeerManagerDisableConfig(), &p2p.DisallowListCacheConfig{ MaxSize: uint32(1000), Metrics: metrics.NewNoopCollector(), @@ -125,11 +129,10 @@ func CreateNode(t *testing.T, networkKey crypto.PrivateKey, sporkID flow.Identif SetRoutingSystem(func(c context.Context, h host.Host) (routing.Routing, error) { return p2pdht.NewDHT(c, h, protocols.FlowDHTProtocolID(sporkID), zerolog.Nop(), metrics.NewNoopCollector()) }). - SetResourceManager(testutils.NewResourceManager(t)). + SetResourceManager(&network.NullResourceManager{}). SetStreamCreationRetryInterval(unicast.DefaultRetryDelay). SetGossipSubTracer(meshTracer). - SetGossipSubScoreTracerInterval(defaultFlowConfig.NetworkConfig.GossipSubConfig.ScoreTracerInterval). - SetGossipSubRpcInspectorSuite(rpcInspectorSuite) + SetGossipSubScoreTracerInterval(defaultFlowConfig.NetworkConfig.GossipSubConfig.ScoreTracerInterval) for _, opt := range opts { opt(builder) diff --git a/network/internal/testutils/testUtil.go b/network/internal/testutils/testUtil.go index 47533cb2677..2457c7c6af7 100644 --- a/network/internal/testutils/testUtil.go +++ b/network/internal/testutils/testUtil.go @@ -1,7 +1,6 @@ package testutils import ( - "context" "fmt" "reflect" "runtime" @@ -10,18 +9,11 @@ import ( "testing" "time" - dht "github.com/libp2p/go-libp2p-kad-dht" - "github.com/libp2p/go-libp2p/core/host" - p2pNetwork "github.com/libp2p/go-libp2p/core/network" "github.com/libp2p/go-libp2p/core/peer" - pc "github.com/libp2p/go-libp2p/core/protocol" - "github.com/libp2p/go-libp2p/core/routing" "github.com/rs/zerolog" "github.com/stretchr/testify/require" "github.com/onflow/flow-go/config" - netconf "github.com/onflow/flow-go/config/network" - "github.com/onflow/flow-go/crypto" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/model/flow/filter" libp2pmessage "github.com/onflow/flow-go/model/libp2p/message" @@ -36,19 +28,15 @@ import ( netcache "github.com/onflow/flow-go/network/cache" "github.com/onflow/flow-go/network/channels" "github.com/onflow/flow-go/network/codec/cbor" + "github.com/onflow/flow-go/network/mocknetwork" + "github.com/onflow/flow-go/network/netconf" "github.com/onflow/flow-go/network/p2p" "github.com/onflow/flow-go/network/p2p/conduit" "github.com/onflow/flow-go/network/p2p/connection" - p2pdht "github.com/onflow/flow-go/network/p2p/dht" "github.com/onflow/flow-go/network/p2p/middleware" - "github.com/onflow/flow-go/network/p2p/p2pbuilder" - inspectorbuilder "github.com/onflow/flow-go/network/p2p/p2pbuilder/inspector" "github.com/onflow/flow-go/network/p2p/subscription" + p2ptest "github.com/onflow/flow-go/network/p2p/test" "github.com/onflow/flow-go/network/p2p/translator" - "github.com/onflow/flow-go/network/p2p/unicast" - "github.com/onflow/flow-go/network/p2p/unicast/protocols" - "github.com/onflow/flow-go/network/p2p/unicast/ratelimit" - "github.com/onflow/flow-go/network/slashing" "github.com/onflow/flow-go/utils/unittest" ) @@ -126,121 +114,108 @@ func NewTagWatchingConnManager(log zerolog.Logger, metrics module.LibP2PConnecti }, nil } -// GenerateIDs is a test helper that generate flow identities with a valid port and libp2p nodes. -func GenerateIDs(t *testing.T, logger zerolog.Logger, n int, opts ...func(*optsConfig)) (flow.IdentityList, - []p2p.LibP2PNode, - []observable.Observable) { - libP2PNodes := make([]p2p.LibP2PNode, n) - tagObservables := make([]observable.Observable, n) - - identities := unittest.IdentityListFixture(n, unittest.WithAllRoles()) - idProvider := NewUpdatableIDProvider(identities) - o := &optsConfig{ - peerUpdateInterval: connection.DefaultPeerUpdateInterval, - unicastRateLimiterDistributor: ratelimit.NewUnicastRateLimiterDistributor(), - connectionGaterFactory: func() p2p.ConnectionGater { - return NewConnectionGater(idProvider, func(p peer.ID) error { - return nil - }) - }, - createStreamRetryInterval: unicast.DefaultRetryDelay, - } - for _, opt := range opts { - opt(o) - } - - for _, identity := range identities { - for _, idOpt := range o.idOpts { - idOpt(identity) - } - } - - // generates keys and address for the node - for i, identity := range identities { - // generate key - key, err := generateNetworkingKey(identity.NodeID) - require.NoError(t, err) - - var opts []nodeBuilderOption - - opts = append(opts, withDHT(o.dhtPrefix, o.dhtOpts...)) - opts = append(opts, withPeerManagerOptions(connection.PruningEnabled, o.peerUpdateInterval)) - opts = append(opts, withRateLimiterDistributor(o.unicastRateLimiterDistributor)) - opts = append(opts, withConnectionGater(o.connectionGaterFactory())) - opts = append(opts, withUnicastManagerOpts(o.createStreamRetryInterval)) +// LibP2PNodeForMiddlewareFixture is a test helper that generate flow identities with a valid port and libp2p nodes. +// Note that the LibP2PNode created by this fixture is meant to used with a middleware component. +// If you want to create a standalone LibP2PNode without network and middleware components, please use p2ptest.NodeFixture. +// Args: +// +// t: testing.T- the test object +// +// n: int - number of nodes to create +// opts: []p2ptest.NodeFixtureParameterOption - options to configure the nodes +// Returns: +// +// flow.IdentityList - list of identities created for the nodes, one for each node. +// +// []p2p.LibP2PNode - list of libp2p nodes created. +// []observable.Observable - list of observables created for each node. +func LibP2PNodeForMiddlewareFixture(t *testing.T, n int, opts ...p2ptest.NodeFixtureParameterOption) (flow.IdentityList, []p2p.LibP2PNode, []observable.Observable) { + + libP2PNodes := make([]p2p.LibP2PNode, 0) + identities := make(flow.IdentityList, 0) + tagObservables := make([]observable.Observable, 0) + idProvider := unittest.NewUpdatableIDProvider(flow.IdentityList{}) + defaultFlowConfig, err := config.DefaultConfig() + require.NoError(t, err) - libP2PNodes[i], tagObservables[i] = generateLibP2PNode(t, logger, key, idProvider, opts...) + opts = append(opts, p2ptest.WithUnicastHandlerFunc(nil)) - _, port, err := libP2PNodes[i].GetIPPort() + for i := 0; i < n; i++ { + // TODO: generating a tag watching connection manager can be moved to a separate function, as only a few tests need this. + // For the rest of tests, the node can run on the default connection manager without setting and option. + connManager, err := NewTagWatchingConnManager(unittest.Logger(), metrics.NewNoopCollector(), &defaultFlowConfig.NetworkConfig.ConnectionManagerConfig) require.NoError(t, err) - identities[i].Address = unittest.IPPort(port) - identities[i].NetworkPubKey = key.PublicKey() - } - + opts = append(opts, p2ptest.WithConnectionManager(connManager)) + node, nodeId := p2ptest.NodeFixture(t, + sporkID, + t.Name(), + idProvider, + opts...) + libP2PNodes = append(libP2PNodes, node) + identities = append(identities, &nodeId) + tagObservables = append(tagObservables, connManager) + } + idProvider.SetIdentities(identities) return identities, libP2PNodes, tagObservables } -// GenerateMiddlewares creates and initializes middleware instances for all the identities -func GenerateMiddlewares(t *testing.T, - logger zerolog.Logger, - identities flow.IdentityList, - libP2PNodes []p2p.LibP2PNode, - codec network.Codec, - consumer slashing.ViolationsConsumer, - opts ...func(*optsConfig)) ([]network.Middleware, []*UpdatableIDProvider) { +// MiddlewareConfigFixture is a test helper that generates a middleware config for testing. +// Args: +// - t: the test instance. +// Returns: +// - a middleware config. +func MiddlewareConfigFixture(t *testing.T) *middleware.Config { + return &middleware.Config{ + Logger: unittest.Logger(), + BitSwapMetrics: metrics.NewNoopCollector(), + RootBlockID: sporkID, + UnicastMessageTimeout: middleware.DefaultUnicastTimeout, + Codec: unittest.NetworkCodec(), + SlashingViolationsConsumer: mocknetwork.NewViolationsConsumer(t), + } +} + +// MiddlewareFixtures is a test helper that generates middlewares with the given identities and libp2p nodes. +// It also generates a list of UpdatableIDProvider that can be used to update the identities of the middlewares. +// The number of identities and libp2p nodes must be the same. +// Args: +// - identities: a list of flow identities that correspond to the libp2p nodes. +// - libP2PNodes: a list of libp2p nodes that correspond to the identities. +// - cfg: the middleware config. +// - opts: a list of middleware option functions. +// Returns: +// - a list of middlewares - one for each identity. +// - a list of UpdatableIDProvider - one for each identity. +func MiddlewareFixtures(t *testing.T, identities flow.IdentityList, libP2PNodes []p2p.LibP2PNode, cfg *middleware.Config, opts ...middleware.OptionFn) ([]network.Middleware, []*unittest.UpdatableIDProvider) { + require.Equal(t, len(identities), len(libP2PNodes)) + mws := make([]network.Middleware, len(identities)) - idProviders := make([]*UpdatableIDProvider, len(identities)) - bitswapmet := metrics.NewNoopCollector() - o := &optsConfig{ - peerUpdateInterval: connection.DefaultPeerUpdateInterval, - unicastRateLimiters: ratelimit.NoopRateLimiters(), - networkMetrics: metrics.NewNoopCollector(), - peerManagerFilters: []p2p.PeerFilter{}, - } + idProviders := make([]*unittest.UpdatableIDProvider, len(identities)) - for _, opt := range opts { - opt(o) - } + for i := 0; i < len(identities); i++ { + i := i + cfg.Libp2pNode = libP2PNodes[i] + cfg.FlowId = identities[i].NodeID + idProviders[i] = unittest.NewUpdatableIDProvider(identities) + cfg.IdTranslator = translator.NewIdentityProviderIDTranslator(idProviders[i]) - total := len(identities) - for i := 0; i < total; i++ { - // casts libP2PNode instance to a local variable to avoid closure - node := libP2PNodes[i] - nodeId := identities[i].NodeID - - idProviders[i] = NewUpdatableIDProvider(identities) - - // creating middleware of nodes - mws[i] = middleware.NewMiddleware(&middleware.Config{ - Logger: logger, - Libp2pNode: node, - FlowId: nodeId, - BitSwapMetrics: bitswapmet, - RootBlockID: sporkID, - UnicastMessageTimeout: middleware.DefaultUnicastTimeout, - IdTranslator: translator.NewIdentityProviderIDTranslator(idProviders[i]), - Codec: codec, - SlashingViolationsConsumer: consumer, - }, - middleware.WithUnicastRateLimiters(o.unicastRateLimiters), - middleware.WithPeerManagerFilters(o.peerManagerFilters)) + mws[i] = middleware.NewMiddleware(cfg, opts...) } return mws, idProviders } // NetworksFixture generates the network for the given middlewares func NetworksFixture(t *testing.T, - log zerolog.Logger, ids flow.IdentityList, - mws []network.Middleware, - sms []network.SubscriptionManager) []network.Network { + mws []network.Middleware) []network.Network { count := len(ids) nets := make([]network.Network, 0) for i := 0; i < count; i++ { - params := NetworkConfigFixture(t, log, *ids[i], ids, mws[i], sms[i]) + + params := NetworkConfigFixture(t, *ids[i], ids, mws[i]) net, err := p2p.NewNetwork(params) require.NoError(t, err) @@ -252,11 +227,10 @@ func NetworksFixture(t *testing.T, func NetworkConfigFixture( t *testing.T, - logger zerolog.Logger, myId flow.Identity, allIds flow.IdentityList, mw network.Middleware, - subMgr network.SubscriptionManager, opts ...p2p.NetworkConfigOption) *p2p.NetworkConfig { + opts ...p2p.NetworkConfigOption) *p2p.NetworkConfig { me := mock.NewLocal(t) me.On("NodeID").Return(myId.NodeID).Maybe() @@ -266,10 +240,14 @@ func NetworkConfigFixture( defaultFlowConfig, err := config.DefaultConfig() require.NoError(t, err) - receiveCache := netcache.NewHeroReceiveCache(defaultFlowConfig.NetworkConfig.NetworkReceivedMessageCacheSize, logger, metrics.NewNoopCollector()) + receiveCache := netcache.NewHeroReceiveCache( + defaultFlowConfig.NetworkConfig.NetworkReceivedMessageCacheSize, + unittest.Logger(), + metrics.NewNoopCollector()) + subMgr := subscription.NewChannelSubscriptionManager(mw) params := &p2p.NetworkConfig{ - Logger: logger, - Codec: cbor.NewCodec(), + Logger: unittest.Logger(), + Codec: unittest.NetworkCodec(), Me: me, MiddlewareFactory: func() (network.Middleware, error) { return mw, nil }, Topology: unittest.NetworkTopology(), @@ -295,100 +273,6 @@ func NetworkConfigFixture( return params } -// GenerateIDsAndMiddlewares returns nodeIDs, libp2pNodes, middlewares, and observables which can be subscirbed to in order to witness protect events from pubsub -func GenerateIDsAndMiddlewares(t *testing.T, - n int, - logger zerolog.Logger, - codec network.Codec, - consumer slashing.ViolationsConsumer, - opts ...func(*optsConfig)) (flow.IdentityList, []p2p.LibP2PNode, []network.Middleware, []observable.Observable, []*UpdatableIDProvider) { - - ids, libP2PNodes, protectObservables := GenerateIDs(t, logger, n, opts...) - mws, providers := GenerateMiddlewares(t, logger, ids, libP2PNodes, codec, consumer, opts...) - return ids, libP2PNodes, mws, protectObservables, providers -} - -type optsConfig struct { - idOpts []func(*flow.Identity) - dhtPrefix string - dhtOpts []dht.Option - unicastRateLimiters *ratelimit.RateLimiters - peerUpdateInterval time.Duration - networkMetrics module.NetworkMetrics - peerManagerFilters []p2p.PeerFilter - unicastRateLimiterDistributor p2p.UnicastRateLimiterDistributor - connectionGaterFactory func() p2p.ConnectionGater - createStreamRetryInterval time.Duration -} - -func WithCreateStreamRetryInterval(delay time.Duration) func(*optsConfig) { - return func(o *optsConfig) { - o.createStreamRetryInterval = delay - } -} - -func WithUnicastRateLimiterDistributor(distributor p2p.UnicastRateLimiterDistributor) func(*optsConfig) { - return func(o *optsConfig) { - o.unicastRateLimiterDistributor = distributor - } -} - -func WithIdentityOpts(idOpts ...func(*flow.Identity)) func(*optsConfig) { - return func(o *optsConfig) { - o.idOpts = idOpts - } -} - -func WithDHT(prefix string, dhtOpts ...dht.Option) func(*optsConfig) { - return func(o *optsConfig) { - o.dhtPrefix = prefix - o.dhtOpts = dhtOpts - } -} - -func WithPeerUpdateInterval(interval time.Duration) func(*optsConfig) { - return func(o *optsConfig) { - o.peerUpdateInterval = interval - } -} - -func WithPeerManagerFilters(filters ...p2p.PeerFilter) func(*optsConfig) { - return func(o *optsConfig) { - o.peerManagerFilters = filters - } -} - -func WithUnicastRateLimiters(limiters *ratelimit.RateLimiters) func(*optsConfig) { - return func(o *optsConfig) { - o.unicastRateLimiters = limiters - } -} - -func WithConnectionGaterFactory(connectionGaterFactory func() p2p.ConnectionGater) func(*optsConfig) { - return func(o *optsConfig) { - o.connectionGaterFactory = connectionGaterFactory - } -} - -func WithNetworkMetrics(m module.NetworkMetrics) func(*optsConfig) { - return func(o *optsConfig) { - o.networkMetrics = m - } -} - -func GenerateIDsMiddlewaresNetworks(t *testing.T, - n int, - log zerolog.Logger, - codec network.Codec, - consumer slashing.ViolationsConsumer, - opts ...func(*optsConfig)) (flow.IdentityList, []p2p.LibP2PNode, []network.Middleware, []network.Network, []observable.Observable) { - ids, libp2pNodes, mws, observables, _ := GenerateIDsAndMiddlewares(t, n, log, codec, consumer, opts...) - sms := GenerateSubscriptionManagers(t, mws) - networks := NetworksFixture(t, log, ids, mws, sms) - - return ids, libp2pNodes, mws, networks, observables -} - // GenerateEngines generates MeshEngines for the given networks func GenerateEngines(t *testing.T, nets []network.Network) []*MeshEngine { count := len(nets) @@ -455,81 +339,6 @@ func StopComponents[R module.ReadyDoneAware](t *testing.T, rda []R, duration tim unittest.RequireComponentsDoneBefore(t, duration, comps...) } -type nodeBuilderOption func(p2p.NodeBuilder) - -func withDHT(prefix string, dhtOpts ...dht.Option) nodeBuilderOption { - return func(nb p2p.NodeBuilder) { - nb.SetRoutingSystem(func(c context.Context, h host.Host) (routing.Routing, error) { - return p2pdht.NewDHT(c, h, pc.ID(protocols.FlowDHTProtocolIDPrefix+prefix), zerolog.Nop(), metrics.NewNoopCollector(), dhtOpts...) - }) - } -} - -func withPeerManagerOptions(connectionPruning bool, updateInterval time.Duration) nodeBuilderOption { - return func(nb p2p.NodeBuilder) { - nb.SetPeerManagerOptions(connectionPruning, updateInterval) - } -} - -func withRateLimiterDistributor(distributor p2p.UnicastRateLimiterDistributor) nodeBuilderOption { - return func(nb p2p.NodeBuilder) { - nb.SetRateLimiterDistributor(distributor) - } -} - -func withConnectionGater(connectionGater p2p.ConnectionGater) nodeBuilderOption { - return func(nb p2p.NodeBuilder) { - nb.SetConnectionGater(connectionGater) - } -} - -func withUnicastManagerOpts(delay time.Duration) nodeBuilderOption { - return func(nb p2p.NodeBuilder) { - nb.SetStreamCreationRetryInterval(delay) - } -} - -// generateLibP2PNode generates a `LibP2PNode` on localhost using a port assigned by the OS -func generateLibP2PNode(t *testing.T, logger zerolog.Logger, key crypto.PrivateKey, provider *UpdatableIDProvider, opts ...nodeBuilderOption) (p2p.LibP2PNode, observable.Observable) { - - noopMetrics := metrics.NewNoopCollector() - defaultFlowConfig, err := config.DefaultConfig() - require.NoError(t, err) - - // Inject some logic to be able to observe connections of this node - connManager, err := NewTagWatchingConnManager(logger, noopMetrics, &defaultFlowConfig.NetworkConfig.ConnectionManagerConfig) - require.NoError(t, err) - met := metrics.NewNoopCollector() - - rpcInspectorSuite, err := inspectorbuilder.NewGossipSubInspectorBuilder(logger, sporkID, &defaultFlowConfig.NetworkConfig.GossipSubConfig.GossipSubRPCInspectorsConfig, provider, met).Build() - require.NoError(t, err) - - builder := p2pbuilder.NewNodeBuilder( - logger, - met, - unittest.DefaultAddress, - key, - sporkID, - &defaultFlowConfig.NetworkConfig.ResourceManagerConfig, - &p2p.DisallowListCacheConfig{ - MaxSize: uint32(1000), - Metrics: metrics.NewNoopCollector(), - }). - SetConnectionManager(connManager). - SetResourceManager(NewResourceManager(t)). - SetStreamCreationRetryInterval(unicast.DefaultRetryDelay). - SetGossipSubRpcInspectorSuite(rpcInspectorSuite) - - for _, opt := range opts { - opt(builder) - } - - libP2PNode, err := builder.Build() - require.NoError(t, err) - - return libP2PNode, connManager -} - // OptionalSleep introduces a sleep to allow nodes to heartbeat and discover each other (only needed when using PubSub) func OptionalSleep(send ConduitSendWrapperFunc) { sendFuncName := runtime.FuncForPC(reflect.ValueOf(send).Pointer()).Name() @@ -538,24 +347,6 @@ func OptionalSleep(send ConduitSendWrapperFunc) { } } -// generateNetworkingKey generates a Flow ECDSA key using the given seed -func generateNetworkingKey(s flow.Identifier) (crypto.PrivateKey, error) { - seed := make([]byte, crypto.KeyGenSeedMinLen) - copy(seed, s[:]) - return crypto.GeneratePrivateKey(crypto.ECDSASecp256k1, seed) -} - -// GenerateSubscriptionManagers creates and returns a ChannelSubscriptionManager for each middleware object. -func GenerateSubscriptionManagers(t *testing.T, mws []network.Middleware) []network.SubscriptionManager { - require.NotEmpty(t, mws) - - sms := make([]network.SubscriptionManager, len(mws)) - for i, mw := range mws { - sms[i] = subscription.NewChannelSubscriptionManager(mw) - } - return sms -} - // NetworkPayloadFixture creates a blob of random bytes with the given size (in bytes) and returns it. // The primary goal of utilizing this helper function is to apply stress tests on the network layer by // sending large messages to transmit. @@ -593,20 +384,6 @@ func NetworkPayloadFixture(t *testing.T, size uint) []byte { return payload } -// NewResourceManager creates a new resource manager for testing with no limits. -func NewResourceManager(t *testing.T) p2pNetwork.ResourceManager { - return &p2pNetwork.NullResourceManager{} -} - -// NewConnectionGater creates a new connection gater for testing with given allow listing filter. -func NewConnectionGater(idProvider module.IdentityProvider, allowListFilter p2p.PeerFilter) p2p.ConnectionGater { - filters := []p2p.PeerFilter{allowListFilter} - return connection.NewConnGater(unittest.Logger(), - idProvider, - connection.WithOnInterceptPeerDialFilters(filters), - connection.WithOnInterceptSecuredFilters(filters)) -} - // IsRateLimitedPeerFilter returns a p2p.PeerFilter that will return an error if the peer is rate limited. func IsRateLimitedPeerFilter(rateLimiter p2p.RateLimiter) p2p.PeerFilter { return func(p peer.ID) error { diff --git a/network/message/gossipsub.go b/network/message/gossipsub.go new file mode 100644 index 00000000000..ede1b09878e --- /dev/null +++ b/network/message/gossipsub.go @@ -0,0 +1 @@ +package message diff --git a/network/mocknetwork/connector.go b/network/mocknetwork/connector.go index 7f6a50e317c..deedbd4f815 100644 --- a/network/mocknetwork/connector.go +++ b/network/mocknetwork/connector.go @@ -15,9 +15,9 @@ type Connector struct { mock.Mock } -// UpdatePeers provides a mock function with given fields: ctx, peerIDs -func (_m *Connector) UpdatePeers(ctx context.Context, peerIDs peer.IDSlice) { - _m.Called(ctx, peerIDs) +// Connect provides a mock function with given fields: ctx, peerChan +func (_m *Connector) Connect(ctx context.Context, peerChan <-chan peer.AddrInfo) { + _m.Called(ctx, peerChan) } type mockConstructorTestingTNewConnector interface { diff --git a/network/mocknetwork/connector_factory.go b/network/mocknetwork/connector_factory.go new file mode 100644 index 00000000000..b1baeb4f749 --- /dev/null +++ b/network/mocknetwork/connector_factory.go @@ -0,0 +1,56 @@ +// Code generated by mockery v2.21.4. DO NOT EDIT. + +package mocknetwork + +import ( + host "github.com/libp2p/go-libp2p/core/host" + mock "github.com/stretchr/testify/mock" + + p2p "github.com/onflow/flow-go/network/p2p" +) + +// ConnectorFactory is an autogenerated mock type for the ConnectorFactory type +type ConnectorFactory struct { + mock.Mock +} + +// Execute provides a mock function with given fields: _a0 +func (_m *ConnectorFactory) Execute(_a0 host.Host) (p2p.Connector, error) { + ret := _m.Called(_a0) + + var r0 p2p.Connector + var r1 error + if rf, ok := ret.Get(0).(func(host.Host) (p2p.Connector, error)); ok { + return rf(_a0) + } + if rf, ok := ret.Get(0).(func(host.Host) p2p.Connector); ok { + r0 = rf(_a0) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(p2p.Connector) + } + } + + if rf, ok := ret.Get(1).(func(host.Host) error); ok { + r1 = rf(_a0) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +type mockConstructorTestingTNewConnectorFactory interface { + mock.TestingT + Cleanup(func()) +} + +// NewConnectorFactory creates a new instance of ConnectorFactory. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewConnectorFactory(t mockConstructorTestingTNewConnectorFactory) *ConnectorFactory { + mock := &ConnectorFactory{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/config/network/config.go b/network/netconf/config.go similarity index 66% rename from config/network/config.go rename to network/netconf/config.go index 5d015191d20..f3bcfed1f93 100644 --- a/config/network/config.go +++ b/network/netconf/config.go @@ -1,22 +1,20 @@ -package network +package netconf import ( - "fmt" - "strings" "time" - "github.com/spf13/viper" + "github.com/onflow/flow-go/network/p2p/p2pconf" ) // Config encapsulation of configuration structs for all components related to the Flow network. type Config struct { // UnicastRateLimitersConfig configuration for all unicast rate limiters. - UnicastRateLimitersConfig `mapstructure:",squash"` - ResourceManagerConfig `mapstructure:",squash"` - ConnectionManagerConfig `mapstructure:",squash"` + UnicastRateLimitersConfig `mapstructure:",squash"` + p2pconf.ResourceManagerConfig `mapstructure:",squash"` + ConnectionManagerConfig `mapstructure:",squash"` // GossipSubConfig core gossipsub configuration. - GossipSubConfig `mapstructure:",squash"` - AlspConfig `mapstructure:",squash"` + p2pconf.GossipSubConfig `mapstructure:",squash"` + AlspConfig `mapstructure:",squash"` // NetworkConnectionPruning determines whether connections to nodes // that are not part of protocol state should be trimmed @@ -70,36 +68,3 @@ type AlspConfig struct { // events that are used to perform critical ALSP tasks, such as updating the spam records cache. HearBeatInterval time.Duration `mapstructure:"alsp-heart-beat-interval"` } - -// SetAliases this func sets an aliases for each CLI flag defined for network config overrides to it's corresponding -// full key in the viper config store. This is required because in our config.yml file all configuration values for the -// Flow network are stored one level down on the network-config property. When the default config is bootstrapped viper will -// store these values with the "network-config." prefix on the config key, because we do not want to use CLI flags like --network-config.networking-connection-pruning -// to override default values we instead use cleans flags like --networking-connection-pruning and create an alias from networking-connection-pruning -> network-config.networking-connection-pruning -// to ensure overrides happen as expected. -// Args: -// *viper.Viper: instance of the viper store to register network config aliases on. -// Returns: -// error: if a flag does not have a corresponding key in the viper store. -func SetAliases(conf *viper.Viper) error { - m := make(map[string]string) - // create map of key -> full pathkey - // ie: "networking-connection-pruning" -> "network-config.networking-connection-pruning" - for _, key := range conf.AllKeys() { - s := strings.Split(key, ".") - // check len of s, we expect all network keys to have a single prefix "network-config" - // s should always contain only 2 elements - if len(s) == 2 { - m[s[1]] = key - } - } - // each flag name should correspond to exactly one key in our config store after it is loaded with the default config - for _, flagName := range AllFlagNames() { - fullKey, ok := m[flagName] - if !ok { - return fmt.Errorf("invalid network configuration missing configuration key flag name %s check config file and cli flags", flagName) - } - conf.RegisterAlias(fullKey, flagName) - } - return nil -} diff --git a/config/network/connection_manager.go b/network/netconf/connection_manager.go similarity index 99% rename from config/network/connection_manager.go rename to network/netconf/connection_manager.go index 92503f4f7d7..333a3f8c6e5 100644 --- a/config/network/connection_manager.go +++ b/network/netconf/connection_manager.go @@ -1,4 +1,4 @@ -package network +package netconf import "time" diff --git a/config/network/flags.go b/network/netconf/flags.go similarity index 98% rename from config/network/flags.go rename to network/netconf/flags.go index 8beb0e30dc4..2b76042e6c8 100644 --- a/config/network/flags.go +++ b/network/netconf/flags.go @@ -1,4 +1,4 @@ -package network +package netconf import ( "fmt" @@ -6,7 +6,7 @@ import ( "github.com/spf13/pflag" - "github.com/onflow/flow-go/network/p2p" + p2pmsg "github.com/onflow/flow-go/network/p2p/message" ) const ( @@ -137,7 +137,7 @@ func initRpcInspectorValidationLimitsFlags(flags *pflag.FlagSet, defaultNetConfi for _, ctrlMsgValidationConfig := range validationInspectorConfig.AllCtrlMsgValidationConfig() { ctrlMsgType := ctrlMsgValidationConfig.ControlMsg - if ctrlMsgValidationConfig.ControlMsg == p2p.CtrlMsgIWant { + if ctrlMsgValidationConfig.ControlMsg == p2pmsg.CtrlMsgIWant { continue } s := strings.ToLower(ctrlMsgType.String()) diff --git a/network/p2p/builder.go b/network/p2p/builder.go index a8a89367013..3bd8e278716 100644 --- a/network/p2p/builder.go +++ b/network/p2p/builder.go @@ -13,12 +13,16 @@ import ( madns "github.com/multiformats/go-multiaddr-dns" "github.com/rs/zerolog" + "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module" "github.com/onflow/flow-go/module/irrecoverable" + "github.com/onflow/flow-go/module/metrics" + flownet "github.com/onflow/flow-go/network" "github.com/onflow/flow-go/network/channels" + "github.com/onflow/flow-go/network/p2p/p2pconf" ) -type GossipSubFactoryFunc func(context.Context, zerolog.Logger, host.Host, PubSubAdapterConfig) (PubSubAdapter, error) +type GossipSubFactoryFunc func(context.Context, zerolog.Logger, host.Host, PubSubAdapterConfig, CollectionClusterChangesConsumer) (PubSubAdapter, error) type CreateNodeFunc func(zerolog.Logger, host.Host, ProtocolPeerCache, PeerManager, *DisallowListCacheConfig) LibP2PNode type GossipSubAdapterConfigFunc func(*BasePubSubAdapterConfig) PubSubAdapterConfig @@ -53,18 +57,16 @@ type GossipSubBuilder interface { // If the gossipsub tracer has already been set, a fatal error is logged. SetGossipSubTracer(PubSubTracer) - // SetIDProvider sets the identity provider of the builder. - // If the identity provider has already been set, a fatal error is logged. - SetIDProvider(module.IdentityProvider) - // SetRoutingSystem sets the routing system of the builder. // If the routing system has already been set, a fatal error is logged. SetRoutingSystem(routing.Routing) - // SetGossipSubRPCInspectorSuite sets the gossipsub rpc inspector suite of the builder. It contains the - // inspector function that is injected into the gossipsub rpc layer, as well as the notification distributors that - // are used to notify the app specific scoring mechanism of misbehaving peers. - SetGossipSubRPCInspectorSuite(GossipSubInspectorSuite) + // OverrideDefaultRpcInspectorSuiteFactory overrides the default RPC inspector suite factory of the builder. + // A default RPC inspector suite factory is provided by the node. This function overrides the default factory. + // The purpose of override is to allow the node to provide a custom RPC inspector suite factory for sake of testing + // or experimentation. + // It is NOT recommended to override the default RPC inspector suite factory in production unless you know what you are doing. + OverrideDefaultRpcInspectorSuiteFactory(GossipSubRpcInspectorSuiteFactoryFunc) // Build creates a new GossipSub pubsub system. // It returns the newly created GossipSub pubsub system and any errors encountered during its creation. @@ -74,10 +76,9 @@ type GossipSubBuilder interface { // // Returns: // - PubSubAdapter: a GossipSub pubsub system for the libp2p node. - // - PeerScoreTracer: a peer score tracer for the GossipSub pubsub system (if enabled, otherwise nil). // - error: if an error occurs during the creation of the GossipSub pubsub system, it is returned. Otherwise, nil is returned. // Note that on happy path, the returned error is nil. Any error returned is unexpected and should be handled as irrecoverable. - Build(irrecoverable.SignalerContext) (PubSubAdapter, PeerScoreTracer, error) + Build(irrecoverable.SignalerContext) (PubSubAdapter, error) } type PeerScoringBuilder interface { @@ -90,6 +91,29 @@ type PeerScoringBuilder interface { SetAppSpecificScoreParams(func(peer.ID) float64) } +// GossipSubRpcInspectorSuiteFactoryFunc is a function that creates a new RPC inspector suite. It is used to create +// RPC inspectors for the gossipsub protocol. The RPC inspectors are used to inspect and validate +// incoming RPC messages before they are processed by the gossipsub protocol. +// Args: +// - logger: logger to use +// - sporkID: spork ID of the node +// - cfg: configuration for the RPC inspectors +// - metrics: metrics to use for the RPC inspectors +// - heroCacheMetricsFactory: metrics factory for the hero cache +// - networkingType: networking type of the node, i.e., public or private +// - identityProvider: identity provider of the node +// Returns: +// - p2p.GossipSubInspectorSuite: new RPC inspector suite +// - error: error if any, any returned error is irrecoverable. +type GossipSubRpcInspectorSuiteFactoryFunc func( + zerolog.Logger, + flow.Identifier, + *p2pconf.GossipSubRPCInspectorsConfig, + module.GossipSubMetrics, + metrics.HeroCacheMetricsFactory, + flownet.NetworkingType, + module.IdentityProvider) (GossipSubInspectorSuite, error) + // NodeBuilder is a builder pattern for creating a libp2p Node instance. type NodeBuilder interface { SetBasicResolver(madns.BasicResolver) NodeBuilder @@ -98,20 +122,19 @@ type NodeBuilder interface { SetConnectionManager(connmgr.ConnManager) NodeBuilder SetConnectionGater(ConnectionGater) NodeBuilder SetRoutingSystem(func(context.Context, host.Host) (routing.Routing, error)) NodeBuilder - SetPeerManagerOptions(bool, time.Duration) NodeBuilder // EnableGossipSubPeerScoring enables peer scoring for the GossipSub pubsub system. // Arguments: // - module.IdentityProvider: the identity provider for the node (must be set before calling this method). // - *PeerScoringConfig: the peer scoring configuration for the GossipSub pubsub system. If nil, the default configuration is used. - EnableGossipSubPeerScoring(module.IdentityProvider, *PeerScoringConfig) NodeBuilder + EnableGossipSubPeerScoring(*PeerScoringConfig) NodeBuilder SetCreateNode(CreateNodeFunc) NodeBuilder SetGossipSubFactory(GossipSubFactoryFunc, GossipSubAdapterConfigFunc) NodeBuilder SetStreamCreationRetryInterval(time.Duration) NodeBuilder SetRateLimiterDistributor(UnicastRateLimiterDistributor) NodeBuilder SetGossipSubTracer(PubSubTracer) NodeBuilder SetGossipSubScoreTracerInterval(time.Duration) NodeBuilder - SetGossipSubRpcInspectorSuite(GossipSubInspectorSuite) NodeBuilder + OverrideDefaultRpcInspectorSuiteFactory(GossipSubRpcInspectorSuiteFactoryFunc) NodeBuilder Build() (LibP2PNode, error) } diff --git a/network/p2p/connection/connManager.go b/network/p2p/connection/connManager.go index c840201e253..54d85175cce 100644 --- a/network/p2p/connection/connManager.go +++ b/network/p2p/connection/connManager.go @@ -10,8 +10,8 @@ import ( libp2pconnmgr "github.com/libp2p/go-libp2p/p2p/net/connmgr" "github.com/rs/zerolog" - netconf "github.com/onflow/flow-go/config/network" "github.com/onflow/flow-go/module" + "github.com/onflow/flow-go/network/netconf" "github.com/onflow/flow-go/network/p2p/connection/internal" ) diff --git a/network/p2p/connection/connManager_test.go b/network/p2p/connection/connManager_test.go index 0a17f6e9b59..39016a09fc3 100644 --- a/network/p2p/connection/connManager_test.go +++ b/network/p2p/connection/connManager_test.go @@ -12,11 +12,11 @@ import ( "github.com/stretchr/testify/require" "github.com/onflow/flow-go/config" - netconf "github.com/onflow/flow-go/config/network" + "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module/irrecoverable" "github.com/onflow/flow-go/module/metrics" - mockmodule "github.com/onflow/flow-go/module/mock" "github.com/onflow/flow-go/network/internal/p2pfixtures" + "github.com/onflow/flow-go/network/netconf" "github.com/onflow/flow-go/network/p2p/connection" p2ptest "github.com/onflow/flow-go/network/p2p/test" "github.com/onflow/flow-go/network/p2p/utils" @@ -117,14 +117,14 @@ func TestConnectionManager_Watermarking(t *testing.T) { metrics.NewNoopCollector(), cfg) require.NoError(t, err) - idProvider := mockmodule.NewIdentityProvider(t) + idProvider := unittest.NewUpdatableIDProvider(flow.IdentityList{}) thisNode, identity := p2ptest.NodeFixture( t, sporkId, t.Name(), idProvider, p2ptest.WithConnectionManager(thisConnMgr)) - idProvider.On("ByPeerID", thisNode.Host().ID()).Return(&identity, true).Maybe() + idProvider.SetIdentities(flow.IdentityList{&identity}) otherNodes, _ := p2ptest.NodesFixture(t, sporkId, t.Name(), 5, idProvider) diff --git a/network/p2p/connection/connection_gater_test.go b/network/p2p/connection/connection_gater_test.go index 07c3f0e2115..0277fc6b632 100644 --- a/network/p2p/connection/connection_gater_test.go +++ b/network/p2p/connection/connection_gater_test.go @@ -17,9 +17,10 @@ import ( mockmodule "github.com/onflow/flow-go/module/mock" "github.com/onflow/flow-go/network/channels" "github.com/onflow/flow-go/network/internal/p2pfixtures" - "github.com/onflow/flow-go/network/internal/testutils" "github.com/onflow/flow-go/network/p2p" + "github.com/onflow/flow-go/network/p2p/connection" mockp2p "github.com/onflow/flow-go/network/p2p/mock" + p2pconfig "github.com/onflow/flow-go/network/p2p/p2pbuilder/config" p2ptest "github.com/onflow/flow-go/network/p2p/test" "github.com/onflow/flow-go/network/p2p/unicast/stream" "github.com/onflow/flow-go/utils/unittest" @@ -39,7 +40,7 @@ func TestConnectionGating(t *testing.T) { sporkID, t.Name(), idProvider, - p2ptest.WithConnectionGater(testutils.NewConnectionGater(idProvider, func(p peer.ID) error { + p2ptest.WithConnectionGater(p2ptest.NewConnectionGater(idProvider, func(p peer.ID) error { if !node1Peers.Has(p) { return fmt.Errorf("id not found: %s", p.String()) } @@ -53,7 +54,7 @@ func TestConnectionGating(t *testing.T) { sporkID, t.Name(), idProvider, - p2ptest.WithConnectionGater(testutils.NewConnectionGater(idProvider, func(p peer.ID) error { + p2ptest.WithConnectionGater(p2ptest.NewConnectionGater(idProvider, func(p peer.ID) error { if !node2Peers.Has(p) { return fmt.Errorf("id not found: %s", p.String()) } @@ -152,7 +153,7 @@ func TestConnectionGating_ResourceAllocation_AllowListing(t *testing.T) { p2ptest.WithMetricsCollector(node2Metrics), // we use default resource manager rather than the test resource manager to ensure that the metrics are called. p2ptest.WithDefaultResourceManager(), - p2ptest.WithConnectionGater(testutils.NewConnectionGater(idProvider, func(p peer.ID) error { + p2ptest.WithConnectionGater(p2ptest.NewConnectionGater(idProvider, func(p peer.ID) error { return nil // allow all connections. }))) idProvider.On("ByPeerID", node1.Host().ID()).Return(&node1Id, true).Maybe() @@ -197,7 +198,7 @@ func TestConnectionGating_ResourceAllocation_DisAllowListing(t *testing.T) { p2ptest.WithMetricsCollector(node2Metrics), // we use default resource manager rather than the test resource manager to ensure that the metrics are called. p2ptest.WithDefaultResourceManager(), - p2ptest.WithConnectionGater(testutils.NewConnectionGater(idProvider, func(p peer.ID) error { + p2ptest.WithConnectionGater(p2ptest.NewConnectionGater(idProvider, func(p peer.ID) error { return fmt.Errorf("disallowed connection") // rejecting all connections. }))) idProvider.On("ByPeerID", node1.Host().ID()).Return(&node1Id, true).Maybe() @@ -249,7 +250,11 @@ func TestConnectionGater_InterceptUpgrade(t *testing.T) { p2ptest.WithRole(flow.RoleConsensus), p2ptest.WithDefaultStreamHandler(handler), // enable peer manager, with a 1-second refresh rate, and connection pruning enabled. - p2ptest.WithPeerManagerEnabled(true, 1*time.Second, func() peer.IDSlice { + p2ptest.WithPeerManagerEnabled(&p2pconfig.PeerManagerConfig{ + ConnectionPruning: true, + UpdateInterval: 1 * time.Second, + ConnectorFactory: connection.DefaultLibp2pBackoffConnectorFactory(), + }, func() peer.IDSlice { list := make(peer.IDSlice, 0) for _, pid := range allPeerIds { if !disallowedPeerIds.Has(pid) { @@ -327,7 +332,11 @@ func TestConnectionGater_Disallow_Integration(t *testing.T) { p2ptest.WithRole(flow.RoleConsensus), p2ptest.WithDefaultStreamHandler(handler), // enable peer manager, with a 1-second refresh rate, and connection pruning enabled. - p2ptest.WithPeerManagerEnabled(true, 1*time.Second, func() peer.IDSlice { + p2ptest.WithPeerManagerEnabled(&p2pconfig.PeerManagerConfig{ + ConnectionPruning: true, + UpdateInterval: 1 * time.Second, + ConnectorFactory: connection.DefaultLibp2pBackoffConnectorFactory(), + }, func() peer.IDSlice { list := make(peer.IDSlice, 0) for _, id := range ids { if disallowedList.Has(id) { @@ -341,7 +350,7 @@ func TestConnectionGater_Disallow_Integration(t *testing.T) { } return list }), - p2ptest.WithConnectionGater(testutils.NewConnectionGater(idProvider, func(pid peer.ID) error { + p2ptest.WithConnectionGater(p2ptest.NewConnectionGater(idProvider, func(pid peer.ID) error { return disallowedList.ForEach(func(id *flow.Identity, _ struct{}) error { bid, err := unittest.PeerIDFromFlowID(id) require.NoError(t, err) diff --git a/network/p2p/connection/connector.go b/network/p2p/connection/connector.go index 0065c395d46..e185d38c69b 100644 --- a/network/p2p/connection/connector.go +++ b/network/p2p/connection/connector.go @@ -2,10 +2,8 @@ package connection import ( "context" - "fmt" "github.com/libp2p/go-libp2p/core/peer" - discoveryBackoff "github.com/libp2p/go-libp2p/p2p/discovery/backoff" "github.com/rs/zerolog" "github.com/onflow/flow-go/network/internal/p2putils" @@ -26,16 +24,16 @@ const ( PruningDisabled = false ) -// Libp2pConnector is a libp2p based Connector implementation to connect and disconnect from peers -type Libp2pConnector struct { - backoffConnector *discoveryBackoff.BackoffConnector +// PeerUpdater is a connector that connects to a list of peers and disconnects from any other connection that the libp2p node might have. +type PeerUpdater struct { + connector p2p.Connector host p2p.ConnectorHost log zerolog.Logger pruneConnections bool } -// ConnectorConfig is the configuration for the libp2p based connector. -type ConnectorConfig struct { +// PeerUpdaterConfig is the configuration for the libp2p based connector. +type PeerUpdaterConfig struct { // PruneConnections is a boolean flag to enable pruning of connections to peers that are not part of the explicit update list. PruneConnections bool @@ -45,32 +43,23 @@ type ConnectorConfig struct { // Host is the libp2p host to be used by the connector. Host p2p.ConnectorHost - // BackoffConnectorFactory is a factory function to create a new BackoffConnector. - BackoffConnectorFactory func() (*discoveryBackoff.BackoffConnector, error) + // ConnectorFactory is a factory function to create a new connector. + Connector p2p.Connector } -var _ p2p.Connector = &Libp2pConnector{} +var _ p2p.PeerUpdater = (*PeerUpdater)(nil) -// NewLibp2pConnector creates a new libp2p based connector +// NewPeerUpdater creates a new libp2p based connector // Args: // - cfg: configuration for the connector // // Returns: -// - *Libp2pConnector: a new libp2p based connector +// - *PeerUpdater: a new libp2p based connector // - error: an error if there is any error while creating the connector. The errors are irrecoverable and unexpected. -func NewLibp2pConnector(cfg *ConnectorConfig) (*Libp2pConnector, error) { - connector, err := cfg.BackoffConnectorFactory() - if err != nil { - return nil, fmt.Errorf("failed to create libP2P connector: %w", err) - } - - if err != nil { - return nil, fmt.Errorf("failed to create peer ID slice shuffler: %w", err) - } - - libP2PConnector := &Libp2pConnector{ +func NewPeerUpdater(cfg *PeerUpdaterConfig) (*PeerUpdater, error) { + libP2PConnector := &PeerUpdater{ log: cfg.Logger, - backoffConnector: connector, + connector: cfg.Connector, host: cfg.Host, pruneConnections: cfg.PruneConnections, } @@ -80,7 +69,7 @@ func NewLibp2pConnector(cfg *ConnectorConfig) (*Libp2pConnector, error) { // UpdatePeers is the implementation of the Connector.UpdatePeers function. It connects to all of the ids and // disconnects from any other connection that the libp2p node might have. -func (l *Libp2pConnector) UpdatePeers(ctx context.Context, peerIDs peer.IDSlice) { +func (l *PeerUpdater) UpdatePeers(ctx context.Context, peerIDs peer.IDSlice) { // connect to each of the peer.AddrInfo in pInfos l.connectToPeers(ctx, peerIDs) @@ -93,7 +82,7 @@ func (l *Libp2pConnector) UpdatePeers(ctx context.Context, peerIDs peer.IDSlice) } // connectToPeers connects each of the peer in pInfos -func (l *Libp2pConnector) connectToPeers(ctx context.Context, peerIDs peer.IDSlice) { +func (l *PeerUpdater) connectToPeers(ctx context.Context, peerIDs peer.IDSlice) { // create a channel of peer.AddrInfo as expected by the connector peerCh := make(chan peer.AddrInfo, len(peerIDs)) @@ -120,13 +109,13 @@ func (l *Libp2pConnector) connectToPeers(ctx context.Context, peerIDs peer.IDSli close(peerCh) // ask the connector to connect to all the peers - l.backoffConnector.Connect(ctx, peerCh) + l.connector.Connect(ctx, peerCh) } // pruneAllConnectionsExcept trims all connections of the node from peers not part of peerIDs. // A node would have created such extra connections earlier when the identity list may have been different, or // it may have been target of such connections from node which have now been excluded. -func (l *Libp2pConnector) pruneAllConnectionsExcept(peerIDs peer.IDSlice) { +func (l *PeerUpdater) pruneAllConnectionsExcept(peerIDs peer.IDSlice) { // convert the peerInfos to a peer.ID -> bool map peersToKeep := make(map[peer.ID]bool, len(peerIDs)) for _, pid := range peerIDs { diff --git a/network/p2p/connection/connector_factory.go b/network/p2p/connection/connector_factory.go index c3ecfaeee9c..aed13a9d168 100644 --- a/network/p2p/connection/connector_factory.go +++ b/network/p2p/connection/connector_factory.go @@ -7,6 +7,8 @@ import ( "github.com/libp2p/go-libp2p/core/host" discoveryBackoff "github.com/libp2p/go-libp2p/p2p/discovery/backoff" + + "github.com/onflow/flow-go/network/p2p" ) const ( @@ -35,8 +37,8 @@ const ( // DefaultLibp2pBackoffConnectorFactory is a factory function to create a new BackoffConnector. It uses the default // values for the backoff connector. // (https://github.com/libp2p/go-libp2p-pubsub/blob/master/discovery.go#L34) -func DefaultLibp2pBackoffConnectorFactory(host host.Host) func() (*discoveryBackoff.BackoffConnector, error) { - return func() (*discoveryBackoff.BackoffConnector, error) { +func DefaultLibp2pBackoffConnectorFactory() p2p.ConnectorFactory { + return func(host host.Host) (p2p.Connector, error) { rngSrc := rand.NewSource(rand.Int63()) cacheSize := 100 diff --git a/network/p2p/connection/peerManager.go b/network/p2p/connection/peerManager.go index 05cc7c47129..83dee63359e 100644 --- a/network/p2p/connection/peerManager.go +++ b/network/p2p/connection/peerManager.go @@ -31,7 +31,7 @@ type PeerManager struct { logger zerolog.Logger peersProvider p2p.PeersProvider // callback to retrieve list of peers to connect to peerRequestQ chan struct{} // a channel to queue a peer update request - connector p2p.Connector // connector to connect or disconnect from peers + connector p2p.PeerUpdater // connector to connect or disconnect from peers peerUpdateInterval time.Duration // interval the peer manager runs on peersProviderMu sync.RWMutex @@ -39,7 +39,7 @@ type PeerManager struct { // NewPeerManager creates a new peer manager which calls the peersProvider callback to get a list of peers to connect to // and it uses the connector to actually connect or disconnect from peers. -func NewPeerManager(logger zerolog.Logger, updateInterval time.Duration, connector p2p.Connector) *PeerManager { +func NewPeerManager(logger zerolog.Logger, updateInterval time.Duration, connector p2p.PeerUpdater) *PeerManager { pm := &PeerManager{ logger: logger, connector: connector, diff --git a/network/p2p/connection/peerManager_integration_test.go b/network/p2p/connection/peerManager_integration_test.go index ca0d3ce513b..684acfbdb8a 100644 --- a/network/p2p/connection/peerManager_integration_test.go +++ b/network/p2p/connection/peerManager_integration_test.go @@ -14,7 +14,6 @@ import ( "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module/irrecoverable" - mockmodule "github.com/onflow/flow-go/module/mock" "github.com/onflow/flow-go/network/p2p" "github.com/onflow/flow-go/network/p2p/connection" p2ptest "github.com/onflow/flow-go/network/p2p/test" @@ -23,7 +22,7 @@ import ( "github.com/onflow/flow-go/utils/unittest" ) -// TestPeerManager_Integration tests the correctness of integration between PeerManager and Libp2pConnector over +// TestPeerManager_Integration tests the correctness of integration between PeerManager and PeerUpdater over // a fully connected topology. // PeerManager should be able to connect to all peers using the connector, and must also tear down the connection to // peers that are excluded from its identity provider. @@ -34,12 +33,9 @@ func TestPeerManager_Integration(t *testing.T) { signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) // create nodes - idProvider := mockmodule.NewIdentityProvider(t) + idProvider := unittest.NewUpdatableIDProvider(flow.IdentityList{}) nodes, identities := p2ptest.NodesFixture(t, unittest.IdentifierFixture(), "test_peer_manager", count, idProvider) - for i, node := range nodes { - idProvider.On("ByPeerID", node.Host().ID()).Return(&identities[i], true).Maybe() - - } + idProvider.SetIdentities(identities) p2ptest.StartNodes(t, signalerCtx, nodes, 100*time.Millisecond) defer p2ptest.StopNodes(t, nodes, cancel, 100*time.Millisecond) @@ -53,19 +49,21 @@ func TestPeerManager_Integration(t *testing.T) { thisNode.Host().Peerstore().SetAddrs(i.ID, i.Addrs, peerstore.PermanentAddrTTL) } + connector, err := connection.DefaultLibp2pBackoffConnectorFactory()(thisNode.Host()) + require.NoError(t, err) // setup - connector, err := connection.NewLibp2pConnector(&connection.ConnectorConfig{ - PruneConnections: connection.PruningEnabled, - Logger: unittest.Logger(), - Host: connection.NewConnectorHost(thisNode.Host()), - BackoffConnectorFactory: connection.DefaultLibp2pBackoffConnectorFactory(thisNode.Host()), + peerUpdater, err := connection.NewPeerUpdater(&connection.PeerUpdaterConfig{ + PruneConnections: connection.PruningEnabled, + Logger: unittest.Logger(), + Host: connection.NewConnectorHost(thisNode.Host()), + Connector: connector, }) require.NoError(t, err) idTranslator, err := translator.NewFixedTableIdentityTranslator(identities) require.NoError(t, err) - peerManager := connection.NewPeerManager(unittest.Logger(), connection.DefaultPeerUpdateInterval, connector) + peerManager := connection.NewPeerManager(unittest.Logger(), connection.DefaultPeerUpdateInterval, peerUpdater) peerManager.SetPeersProvider(func() peer.IDSlice { // peerManager is furnished with a full topology that connects to all nodes // in the topologyPeers. diff --git a/network/p2p/connection/peerManager_test.go b/network/p2p/connection/peerManager_test.go index f2a9305c31b..2b4c8e367e9 100644 --- a/network/p2p/connection/peerManager_test.go +++ b/network/p2p/connection/peerManager_test.go @@ -18,9 +18,9 @@ import ( "github.com/onflow/flow-go/module/irrecoverable" "github.com/onflow/flow-go/network/internal/p2pfixtures" - "github.com/onflow/flow-go/network/mocknetwork" "github.com/onflow/flow-go/network/p2p/connection" "github.com/onflow/flow-go/network/p2p/keyutils" + mockp2p "github.com/onflow/flow-go/network/p2p/mock" "github.com/onflow/flow-go/utils/unittest" ) @@ -60,8 +60,8 @@ func (suite *PeerManagerTestSuite) TestUpdatePeers() { pids := suite.generatePeerIDs(10) // create the connector mock to check ids requested for connect and disconnect - connector := new(mocknetwork.Connector) - connector.On("UpdatePeers", mock.Anything, mock.AnythingOfType("peer.IDSlice")). + peerUpdater := mockp2p.NewPeerUpdater(suite.T()) + peerUpdater.On("UpdatePeers", mock.Anything, mock.AnythingOfType("peer.IDSlice")). Run(func(args mock.Arguments) { idArg := args[1].(peer.IDSlice) assert.ElementsMatch(suite.T(), pids, idArg) @@ -69,7 +69,7 @@ func (suite *PeerManagerTestSuite) TestUpdatePeers() { Return(nil) // create the peer manager (but don't start it) - pm := connection.NewPeerManager(suite.log, connection.DefaultPeerUpdateInterval, connector) + pm := connection.NewPeerManager(suite.log, connection.DefaultPeerUpdateInterval, peerUpdater) pm.SetPeersProvider(func() peer.IDSlice { return pids }) @@ -77,7 +77,7 @@ func (suite *PeerManagerTestSuite) TestUpdatePeers() { // very first call to updatepeer suite.Run("updatePeers only connects to all peers the first time", func() { pm.ForceUpdatePeers(ctx) - connector.AssertNumberOfCalls(suite.T(), "UpdatePeers", 1) + peerUpdater.AssertNumberOfCalls(suite.T(), "UpdatePeers", 1) }) // a subsequent call to updatePeers should request a connector.UpdatePeers to existing ids and new ids @@ -87,7 +87,7 @@ func (suite *PeerManagerTestSuite) TestUpdatePeers() { pids = append(pids, newPIDs...) pm.ForceUpdatePeers(ctx) - connector.AssertNumberOfCalls(suite.T(), "UpdatePeers", 2) + peerUpdater.AssertNumberOfCalls(suite.T(), "UpdatePeers", 2) }) // when ids are only excluded, connector.UpdatePeers should be called @@ -96,7 +96,7 @@ func (suite *PeerManagerTestSuite) TestUpdatePeers() { pids = removeRandomElement(pids) pm.ForceUpdatePeers(ctx) - connector.AssertNumberOfCalls(suite.T(), "UpdatePeers", 3) + peerUpdater.AssertNumberOfCalls(suite.T(), "UpdatePeers", 3) }) // addition and deletion of ids should result in a call to connector.UpdatePeers @@ -111,7 +111,7 @@ func (suite *PeerManagerTestSuite) TestUpdatePeers() { pm.ForceUpdatePeers(ctx) - connector.AssertNumberOfCalls(suite.T(), "UpdatePeers", 4) + peerUpdater.AssertNumberOfCalls(suite.T(), "UpdatePeers", 4) }) } @@ -131,13 +131,13 @@ func (suite *PeerManagerTestSuite) TestPeriodicPeerUpdate() { // create some test ids pids := suite.generatePeerIDs(10) - connector := new(mocknetwork.Connector) + peerUpdater := mockp2p.NewPeerUpdater(suite.T()) wg := &sync.WaitGroup{} // keeps track of number of calls on `ConnectPeers` mu := &sync.Mutex{} // provides mutual exclusion on calls to `ConnectPeers` count := 0 times := 2 // we expect it to be called twice at least wg.Add(times) - connector.On("UpdatePeers", mock.Anything, mock.Anything).Run(func(args mock.Arguments) { + peerUpdater.On("UpdatePeers", mock.Anything, mock.Anything).Run(func(args mock.Arguments) { mu.Lock() defer mu.Unlock() @@ -148,7 +148,7 @@ func (suite *PeerManagerTestSuite) TestPeriodicPeerUpdate() { }).Return(nil) peerUpdateInterval := 10 * time.Millisecond - pm := connection.NewPeerManager(suite.log, peerUpdateInterval, connector) + pm := connection.NewPeerManager(suite.log, peerUpdateInterval, peerUpdater) pm.SetPeersProvider(func() peer.IDSlice { return pids }) @@ -173,15 +173,15 @@ func (suite *PeerManagerTestSuite) TestOnDemandPeerUpdate() { // chooses peer interval rate deliberately long to capture on demand peer update peerUpdateInterval := time.Hour - // creates mock connector + // creates mock peerUpdater wg := &sync.WaitGroup{} // keeps track of number of calls on `ConnectPeers` mu := &sync.Mutex{} // provides mutual exclusion on calls to `ConnectPeers` count := 0 times := 2 // we expect it to be called twice overall wg.Add(1) // this accounts for one invocation, the other invocation is subsequent - connector := new(mocknetwork.Connector) + peerUpdater := mockp2p.NewPeerUpdater(suite.T()) // captures the first periodic update initiated after start to complete - connector.On("UpdatePeers", mock.Anything, mock.Anything).Run(func(args mock.Arguments) { + peerUpdater.On("UpdatePeers", mock.Anything, mock.Anything).Run(func(args mock.Arguments) { mu.Lock() defer mu.Unlock() @@ -191,7 +191,7 @@ func (suite *PeerManagerTestSuite) TestOnDemandPeerUpdate() { } }).Return(nil) - pm := connection.NewPeerManager(suite.log, peerUpdateInterval, connector) + pm := connection.NewPeerManager(suite.log, peerUpdateInterval, peerUpdater) pm.SetPeersProvider(func() peer.IDSlice { return pids }) @@ -220,17 +220,17 @@ func (suite *PeerManagerTestSuite) TestConcurrentOnDemandPeerUpdate() { // create some test ids pids := suite.generatePeerIDs(10) - connector := new(mocknetwork.Connector) - // connectPeerGate channel gates the return of the connector + peerUpdater := mockp2p.NewPeerUpdater(suite.T()) + // connectPeerGate channel gates the return of the peerUpdater connectPeerGate := make(chan time.Time) defer close(connectPeerGate) // choose the periodic interval as a high value so that periodic runs don't interfere with this test peerUpdateInterval := time.Hour - connector.On("UpdatePeers", mock.Anything, mock.Anything).Return(nil). + peerUpdater.On("UpdatePeers", mock.Anything, mock.Anything).Return(nil). WaitUntil(connectPeerGate) // blocks call for connectPeerGate channel - pm := connection.NewPeerManager(suite.log, peerUpdateInterval, connector) + pm := connection.NewPeerManager(suite.log, peerUpdateInterval, peerUpdater) pm.SetPeersProvider(func() peer.IDSlice { return pids }) @@ -243,7 +243,7 @@ func (suite *PeerManagerTestSuite) TestConcurrentOnDemandPeerUpdate() { // assert that the first update started assert.Eventually(suite.T(), func() bool { - return connector.AssertNumberOfCalls(suite.T(), "UpdatePeers", 1) + return peerUpdater.AssertNumberOfCalls(suite.T(), "UpdatePeers", 1) }, 3*time.Second, 100*time.Millisecond) // makes 10 concurrent request for peer update @@ -255,6 +255,6 @@ func (suite *PeerManagerTestSuite) TestConcurrentOnDemandPeerUpdate() { // assert that only two calls to UpdatePeers were made (one by the periodic update and one by the on-demand update) assert.Eventually(suite.T(), func() bool { - return connector.AssertNumberOfCalls(suite.T(), "UpdatePeers", 2) + return peerUpdater.AssertNumberOfCalls(suite.T(), "UpdatePeers", 2) }, 10*time.Second, 100*time.Millisecond) } diff --git a/network/p2p/connector.go b/network/p2p/connector.go index 5ba291d7063..f9c5897352c 100644 --- a/network/p2p/connector.go +++ b/network/p2p/connector.go @@ -3,12 +3,14 @@ package p2p import ( "context" + "github.com/libp2p/go-libp2p/core/host" "github.com/libp2p/go-libp2p/core/network" "github.com/libp2p/go-libp2p/core/peer" ) -// Connector connects to peer and disconnects from peer using the underlying networking library -type Connector interface { +// PeerUpdater connects to the given peer.IDs. It also disconnects from any other peers with which it may have +// previously established connection. +type PeerUpdater interface { // UpdatePeers connects to the given peer.IDs. It also disconnects from any other peers with which it may have // previously established connection. // UpdatePeers implementation should be idempotent such that multiple calls to connect to the same peer should not @@ -16,6 +18,22 @@ type Connector interface { UpdatePeers(ctx context.Context, peerIDs peer.IDSlice) } +// Connector is an interface that allows connecting to a peer.ID. +type Connector interface { + // Connect connects to the given peer.ID. + // Note that connection may be established asynchronously. Any error encountered while connecting to the peer.ID + // is benign and should not be returned. Also, Connect implementation should not cause any blocking or crash. + // Args: + // ctx: context.Context to be used for the connection + // peerChan: channel to which the peer.AddrInfo of the connected peer.ID is sent. + // Returns: + // none. + Connect(ctx context.Context, peerChan <-chan peer.AddrInfo) +} + +// ConnectorFactory is a factory function to create a new Connector. +type ConnectorFactory func(host host.Host) (Connector, error) + type PeerFilter func(peer.ID) error // AllowAllPeerFilter returns a peer filter that does not do any filtering. diff --git a/network/p2p/consumers.go b/network/p2p/consumers.go index ee12521aadb..356bb410d81 100644 --- a/network/p2p/consumers.go +++ b/network/p2p/consumers.go @@ -5,27 +5,9 @@ import ( "github.com/libp2p/go-libp2p/core/peer" "github.com/onflow/flow-go/module/component" + p2pmsg "github.com/onflow/flow-go/network/p2p/message" ) -// ControlMessageType is the type of control message, as defined in the libp2p pubsub spec. -type ControlMessageType string - -func (c ControlMessageType) String() string { - return string(c) -} - -const ( - CtrlMsgIHave ControlMessageType = "IHAVE" - CtrlMsgIWant ControlMessageType = "IWANT" - CtrlMsgGraft ControlMessageType = "GRAFT" - CtrlMsgPrune ControlMessageType = "PRUNE" -) - -// ControlMessageTypes returns list of all libp2p control message types. -func ControlMessageTypes() []ControlMessageType { - return []ControlMessageType{CtrlMsgIHave, CtrlMsgIWant, CtrlMsgGraft, CtrlMsgPrune} -} - // GossipSubInspectorNotifDistributor is the interface for the distributor that distributes gossip sub inspector notifications. // It is used to distribute notifications to the consumers in an asynchronous manner and non-blocking manner. // The implementation should guarantee that all registered consumers are called upon distribution of a new event. @@ -48,7 +30,7 @@ type InvCtrlMsgNotif struct { // PeerID is the ID of the peer that sent the invalid control message. PeerID peer.ID // MsgType is the type of control message that was received. - MsgType ControlMessageType + MsgType p2pmsg.ControlMessageType // Count is the number of invalid control messages received from the peer that is reported in this notification. Count uint64 // Err any error associated with the invalid control message. @@ -56,7 +38,7 @@ type InvCtrlMsgNotif struct { } // NewInvalidControlMessageNotification returns a new *InvCtrlMsgNotif -func NewInvalidControlMessageNotification(peerID peer.ID, msgType ControlMessageType, count uint64, err error) *InvCtrlMsgNotif { +func NewInvalidControlMessageNotification(peerID peer.ID, msgType p2pmsg.ControlMessageType, count uint64, err error) *InvCtrlMsgNotif { return &InvCtrlMsgNotif{ PeerID: peerID, MsgType: msgType, @@ -80,6 +62,7 @@ type GossipSubInvCtrlMsgNotifConsumer interface { // It encapsulates the rpc inspectors and the notification distributors. type GossipSubInspectorSuite interface { component.Component + CollectionClusterChangesConsumer // InspectFunc returns the inspect function that is used to inspect the gossipsub rpc messages. // This function follows a dependency injection pattern, where the inspect function is injected into the gossipsu, and // is called whenever a gossipsub rpc message is received. @@ -89,7 +72,5 @@ type GossipSubInspectorSuite interface { // This consumer is notified when a misbehaving peer regarding gossipsub control messages is detected. This follows a pub/sub // pattern where the consumer is notified when a new notification is published. // A consumer is only notified once for each notification, and only receives notifications that were published after it was added. - AddInvCtrlMsgNotifConsumer(GossipSubInvCtrlMsgNotifConsumer) - // Inspectors returns all inspectors in the inspector suite. - Inspectors() []GossipSubRPCInspector + AddInvalidControlMessageConsumer(GossipSubInvCtrlMsgNotifConsumer) } diff --git a/network/p2p/dht/dht_test.go b/network/p2p/dht/dht_test.go index d26dfc3fe31..5ea0ee70e6a 100644 --- a/network/p2p/dht/dht_test.go +++ b/network/p2p/dht/dht_test.go @@ -11,6 +11,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/onflow/flow-go/model/flow" libp2pmsg "github.com/onflow/flow-go/model/libp2p/message" "github.com/onflow/flow-go/module/irrecoverable" mockmodule "github.com/onflow/flow-go/module/mock" @@ -36,18 +37,14 @@ func TestFindPeerWithDHT(t *testing.T) { golog.SetAllLoggers(golog.LevelFatal) // change this to Debug if libp2p logs are needed sporkId := unittest.IdentifierFixture() - idProvider := mockmodule.NewIdentityProvider(t) + idProvider := unittest.NewUpdatableIDProvider(flow.IdentityList{}) dhtServerNodes, serverIDs := p2ptest.NodesFixture(t, sporkId, "dht_test", 2, idProvider, p2ptest.WithDHTOptions(dht.AsServer())) require.Len(t, dhtServerNodes, 2) dhtClientNodes, clientIDs := p2ptest.NodesFixture(t, sporkId, "dht_test", count-2, idProvider, p2ptest.WithDHTOptions(dht.AsClient())) - ids := append(serverIDs, clientIDs...) nodes := append(dhtServerNodes, dhtClientNodes...) - for i, node := range nodes { - idProvider.On("ByPeerID", node.Host().ID()).Return(&ids[i], true).Maybe() - - } + idProvider.SetIdentities(append(serverIDs, clientIDs...)) p2ptest.StartNodes(t, signalerCtx, nodes, 100*time.Millisecond) defer p2ptest.StopNodes(t, nodes, cancel, 100*time.Millisecond) diff --git a/network/p2p/distributor/gossipsub_inspector_test.go b/network/p2p/distributor/gossipsub_inspector_test.go index e5e94af36ce..23323d7282c 100644 --- a/network/p2p/distributor/gossipsub_inspector_test.go +++ b/network/p2p/distributor/gossipsub_inspector_test.go @@ -14,6 +14,7 @@ import ( "github.com/onflow/flow-go/module/irrecoverable" "github.com/onflow/flow-go/network/p2p" "github.com/onflow/flow-go/network/p2p/distributor" + p2pmsg "github.com/onflow/flow-go/network/p2p/message" mockp2p "github.com/onflow/flow-go/network/p2p/mock" p2ptest "github.com/onflow/flow-go/network/p2p/test" "github.com/onflow/flow-go/utils/unittest" @@ -94,7 +95,7 @@ func invalidControlMessageNotificationListFixture(t *testing.T, n int) []*p2p.In func invalidControlMessageNotificationFixture(t *testing.T) *p2p.InvCtrlMsgNotif { return &p2p.InvCtrlMsgNotif{ PeerID: p2ptest.PeerIdFixture(t), - MsgType: []p2p.ControlMessageType{p2p.CtrlMsgGraft, p2p.CtrlMsgPrune, p2p.CtrlMsgIHave, p2p.CtrlMsgIWant}[rand.Intn(4)], + MsgType: []p2pmsg.ControlMessageType{p2pmsg.CtrlMsgGraft, p2pmsg.CtrlMsgPrune, p2pmsg.CtrlMsgIHave, p2pmsg.CtrlMsgIWant}[rand.Intn(4)], Count: rand.Uint64(), } } diff --git a/network/p2p/inspector/validation/control_message_validation_inspector.go b/network/p2p/inspector/validation/control_message_validation_inspector.go index e2ab265b1ed..7daf3f38599 100644 --- a/network/p2p/inspector/validation/control_message_validation_inspector.go +++ b/network/p2p/inspector/validation/control_message_validation_inspector.go @@ -10,7 +10,6 @@ import ( "github.com/rs/zerolog" "golang.org/x/time/rate" - netconf "github.com/onflow/flow-go/config/network" "github.com/onflow/flow-go/engine/common/worker" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module" @@ -22,6 +21,8 @@ import ( "github.com/onflow/flow-go/network/p2p" "github.com/onflow/flow-go/network/p2p/inspector/internal/cache" "github.com/onflow/flow-go/network/p2p/inspector/internal/ratelimit" + p2pmsg "github.com/onflow/flow-go/network/p2p/message" + "github.com/onflow/flow-go/network/p2p/p2pconf" "github.com/onflow/flow-go/state/protocol" "github.com/onflow/flow-go/state/protocol/events" "github.com/onflow/flow-go/utils/logging" @@ -37,7 +38,7 @@ type ControlMsgValidationInspector struct { sporkID flow.Identifier metrics module.GossipSubRpcValidationInspectorMetrics // config control message validation configurations. - config *netconf.GossipSubRPCValidationInspectorConfigs + config *p2pconf.GossipSubRPCValidationInspectorConfigs // distributor used to disseminate invalid RPC message notifications. distributor p2p.GossipSubInspectorNotifDistributor // workerPool queue that stores *InspectMsgRequest that will be processed by component workers. @@ -52,7 +53,7 @@ type ControlMsgValidationInspector struct { // In such cases, the inspector will allow a configured number of these messages from the corresponding peer. tracker *cache.ClusterPrefixedMessagesReceivedTracker idProvider module.IdentityProvider - rateLimiters map[p2p.ControlMessageType]p2p.BasicRateLimiter + rateLimiters map[p2pmsg.ControlMessageType]p2p.BasicRateLimiter } var _ component.Component = (*ControlMsgValidationInspector)(nil) @@ -74,7 +75,7 @@ var _ protocol.Consumer = (*ControlMsgValidationInspector)(nil) func NewControlMsgValidationInspector( logger zerolog.Logger, sporkID flow.Identifier, - config *netconf.GossipSubRPCValidationInspectorConfigs, + config *p2pconf.GossipSubRPCValidationInspectorConfigs, distributor p2p.GossipSubInspectorNotifDistributor, inspectMsgQueueCacheCollector module.HeroCacheMetrics, clusterPrefixedCacheCollector module.HeroCacheMetrics, @@ -95,7 +96,7 @@ func NewControlMsgValidationInspector( tracker: tracker, idProvider: idProvider, metrics: inspectorMetrics, - rateLimiters: make(map[p2p.ControlMessageType]p2p.BasicRateLimiter), + rateLimiters: make(map[p2pmsg.ControlMessageType]p2p.BasicRateLimiter), } store := queue.NewHeroStore(config.CacheSize, logger, inspectMsgQueueCacheCollector) @@ -147,7 +148,7 @@ func NewControlMsgValidationInspector( // The returned error is returned to the gossipsub node which causes the rejection of rpc (for non-nil errors). func (c *ControlMsgValidationInspector) Inspect(from peer.ID, rpc *pubsub.RPC) error { control := rpc.GetControl() - for _, ctrlMsgType := range p2p.ControlMessageTypes() { + for _, ctrlMsgType := range p2pmsg.ControlMessageTypes() { lg := c.logger.With(). Str("peer_id", from.String()). Str("ctrl_msg_type", string(ctrlMsgType)).Logger() @@ -158,7 +159,7 @@ func (c *ControlMsgValidationInspector) Inspect(from peer.ID, rpc *pubsub.RPC) e } switch ctrlMsgType { - case p2p.CtrlMsgGraft, p2p.CtrlMsgPrune: + case p2pmsg.CtrlMsgGraft, p2pmsg.CtrlMsgPrune: // normal pre-processing err := c.blockingPreprocessingRpc(from, validationConfig, control) if err != nil { @@ -167,7 +168,7 @@ func (c *ControlMsgValidationInspector) Inspect(from peer.ID, rpc *pubsub.RPC) e Msg("could not pre-process rpc, aborting") return fmt.Errorf("could not pre-process rpc, aborting: %w", err) } - case p2p.CtrlMsgIHave: + case p2pmsg.CtrlMsgIHave: // iHave specific pre-processing sampleSize := util.SampleN(len(control.GetIhave()), c.config.IHaveInspectionMaxSampleSize, c.config.IHaveSyncInspectSampleSizePercentage) err := c.blockingIHaveSamplePreprocessing(from, validationConfig, control, sampleSize) @@ -210,9 +211,9 @@ func (c *ControlMsgValidationInspector) ActiveClustersChanged(clusterIDList flow // - ErrDiscardThreshold: if control message count exceeds the configured discard threshold. // // blockingPreprocessingRpc generic pre-processing validation func that ensures the RPC control message count does not exceed the configured hard threshold. -func (c *ControlMsgValidationInspector) blockingPreprocessingRpc(from peer.ID, validationConfig *netconf.CtrlMsgValidationConfig, controlMessage *pubsub_pb.ControlMessage) error { - if validationConfig.ControlMsg != p2p.CtrlMsgGraft && validationConfig.ControlMsg != p2p.CtrlMsgPrune { - return fmt.Errorf("unexpected control message type %s encountered during blocking pre-processing rpc, expected %s or %s", validationConfig.ControlMsg, p2p.CtrlMsgGraft, p2p.CtrlMsgPrune) +func (c *ControlMsgValidationInspector) blockingPreprocessingRpc(from peer.ID, validationConfig *p2pconf.CtrlMsgValidationConfig, controlMessage *pubsub_pb.ControlMessage) error { + if validationConfig.ControlMsg != p2pmsg.CtrlMsgGraft && validationConfig.ControlMsg != p2pmsg.CtrlMsgPrune { + return fmt.Errorf("unexpected control message type %s encountered during blocking pre-processing rpc, expected %s or %s", validationConfig.ControlMsg, p2pmsg.CtrlMsgGraft, p2pmsg.CtrlMsgPrune) } count := c.getCtrlMsgCount(validationConfig.ControlMsg, controlMessage) lg := c.logger.With(). @@ -249,11 +250,11 @@ func (c *ControlMsgValidationInspector) blockingPreprocessingRpc(from peer.ID, v } // blockingPreprocessingSampleRpc blocking pre-processing of a sample of iHave control messages. -func (c *ControlMsgValidationInspector) blockingIHaveSamplePreprocessing(from peer.ID, validationConfig *netconf.CtrlMsgValidationConfig, controlMessage *pubsub_pb.ControlMessage, sampleSize uint) error { - c.metrics.BlockingPreProcessingStarted(p2p.CtrlMsgIHave.String(), sampleSize) +func (c *ControlMsgValidationInspector) blockingIHaveSamplePreprocessing(from peer.ID, validationConfig *p2pconf.CtrlMsgValidationConfig, controlMessage *pubsub_pb.ControlMessage, sampleSize uint) error { + c.metrics.BlockingPreProcessingStarted(p2pmsg.CtrlMsgIHave.String(), sampleSize) start := time.Now() defer func() { - c.metrics.BlockingPreProcessingFinished(p2p.CtrlMsgIHave.String(), sampleSize, time.Since(start)) + c.metrics.BlockingPreProcessingFinished(p2pmsg.CtrlMsgIHave.String(), sampleSize, time.Since(start)) }() err := c.blockingPreprocessingSampleRpc(from, validationConfig, controlMessage, sampleSize) if err != nil { @@ -265,9 +266,9 @@ func (c *ControlMsgValidationInspector) blockingIHaveSamplePreprocessing(from pe // blockingPreprocessingSampleRpc blocking pre-processing validation func that performs some pre-validation of RPC control messages. // If the RPC control message count exceeds the configured hard threshold we perform synchronous topic validation on a subset // of the control messages. This is used for control message types that do not have an upper bound on the amount of messages a node can send. -func (c *ControlMsgValidationInspector) blockingPreprocessingSampleRpc(from peer.ID, validationConfig *netconf.CtrlMsgValidationConfig, controlMessage *pubsub_pb.ControlMessage, sampleSize uint) error { - if validationConfig.ControlMsg != p2p.CtrlMsgIHave && validationConfig.ControlMsg != p2p.CtrlMsgIWant { - return fmt.Errorf("unexpected control message type %s encountered during blocking pre-processing sample rpc, expected %s or %s", validationConfig.ControlMsg, p2p.CtrlMsgIHave, p2p.CtrlMsgIWant) +func (c *ControlMsgValidationInspector) blockingPreprocessingSampleRpc(from peer.ID, validationConfig *p2pconf.CtrlMsgValidationConfig, controlMessage *pubsub_pb.ControlMessage, sampleSize uint) error { + if validationConfig.ControlMsg != p2pmsg.CtrlMsgIHave && validationConfig.ControlMsg != p2pmsg.CtrlMsgIWant { + return fmt.Errorf("unexpected control message type %s encountered during blocking pre-processing sample rpc, expected %s or %s", validationConfig.ControlMsg, p2pmsg.CtrlMsgIHave, p2pmsg.CtrlMsgIWant) } activeClusterIDS := c.tracker.GetActiveClusterIds() count := c.getCtrlMsgCount(validationConfig.ControlMsg, controlMessage) @@ -279,7 +280,7 @@ func (c *ControlMsgValidationInspector) blockingPreprocessingSampleRpc(from peer if count > validationConfig.HardThreshold { // for iHave control message topic validation we only validate a random subset of the messages // shuffle the ihave messages to perform random validation on a subset of size sampleSize - err := c.sampleCtrlMessages(p2p.CtrlMsgIHave, controlMessage, sampleSize) + err := c.sampleCtrlMessages(p2pmsg.CtrlMsgIHave, controlMessage, sampleSize) if err != nil { return fmt.Errorf("failed to sample ihave messages: %w", err) } @@ -305,7 +306,7 @@ func (c *ControlMsgValidationInspector) blockingPreprocessingSampleRpc(from peer // to randomize async validation to avoid data race that can occur when // performing the sampling asynchronously. // for iHave control message topic validation we only validate a random subset of the messages - err := c.sampleCtrlMessages(p2p.CtrlMsgIHave, controlMessage, sampleSize) + err := c.sampleCtrlMessages(p2pmsg.CtrlMsgIHave, controlMessage, sampleSize) if err != nil { return fmt.Errorf("failed to sample ihave messages: %w", err) } @@ -314,9 +315,9 @@ func (c *ControlMsgValidationInspector) blockingPreprocessingSampleRpc(from peer // sampleCtrlMessages performs sampling on the specified control message that will randomize // the items in the control message slice up to index sampleSize-1. -func (c *ControlMsgValidationInspector) sampleCtrlMessages(ctrlMsgType p2p.ControlMessageType, ctrlMsg *pubsub_pb.ControlMessage, sampleSize uint) error { +func (c *ControlMsgValidationInspector) sampleCtrlMessages(ctrlMsgType p2pmsg.ControlMessageType, ctrlMsg *pubsub_pb.ControlMessage, sampleSize uint) error { switch ctrlMsgType { - case p2p.CtrlMsgIHave: + case p2pmsg.CtrlMsgIHave: iHaves := ctrlMsg.GetIhave() swap := func(i, j uint) { iHaves[i], iHaves[j] = iHaves[j], iHaves[i] @@ -375,13 +376,13 @@ func (c *ControlMsgValidationInspector) processInspectMsgReq(req *InspectMsgRequ } // getCtrlMsgCount returns the amount of specified control message type in the rpc ControlMessage. -func (c *ControlMsgValidationInspector) getCtrlMsgCount(ctrlMsgType p2p.ControlMessageType, ctrlMsg *pubsub_pb.ControlMessage) uint64 { +func (c *ControlMsgValidationInspector) getCtrlMsgCount(ctrlMsgType p2pmsg.ControlMessageType, ctrlMsg *pubsub_pb.ControlMessage) uint64 { switch ctrlMsgType { - case p2p.CtrlMsgGraft: + case p2pmsg.CtrlMsgGraft: return uint64(len(ctrlMsg.GetGraft())) - case p2p.CtrlMsgPrune: + case p2pmsg.CtrlMsgPrune: return uint64(len(ctrlMsg.GetPrune())) - case p2p.CtrlMsgIHave: + case p2pmsg.CtrlMsgIHave: return uint64(len(ctrlMsg.GetIhave())) default: return 0 @@ -392,20 +393,20 @@ func (c *ControlMsgValidationInspector) getCtrlMsgCount(ctrlMsgType p2p.ControlM // Expected error returns during normal operations: // - channels.InvalidTopicErr: if topic is invalid. // - ErrDuplicateTopic: if a duplicate topic ID is encountered. -func (c *ControlMsgValidationInspector) validateTopics(from peer.ID, validationConfig *netconf.CtrlMsgValidationConfig, ctrlMsg *pubsub_pb.ControlMessage) error { +func (c *ControlMsgValidationInspector) validateTopics(from peer.ID, validationConfig *p2pconf.CtrlMsgValidationConfig, ctrlMsg *pubsub_pb.ControlMessage) error { activeClusterIDS := c.tracker.GetActiveClusterIds() switch validationConfig.ControlMsg { - case p2p.CtrlMsgGraft: + case p2pmsg.CtrlMsgGraft: return c.validateGrafts(from, ctrlMsg, activeClusterIDS) - case p2p.CtrlMsgPrune: + case p2pmsg.CtrlMsgPrune: return c.validatePrunes(from, ctrlMsg, activeClusterIDS) - case p2p.CtrlMsgIHave: + case p2pmsg.CtrlMsgIHave: return c.validateIhaves(from, validationConfig, ctrlMsg, activeClusterIDS) default: // sanity check // This should never happen validateTopics is only used to validate GRAFT and PRUNE control message types // if any other control message type is encountered here this indicates invalid state irrecoverable error. - c.logger.Fatal().Msg(fmt.Sprintf("encountered invalid control message type in validate topics expected %s, %s or %s got %s", p2p.CtrlMsgGraft, p2p.CtrlMsgPrune, p2p.CtrlMsgIHave, validationConfig.ControlMsg)) + c.logger.Fatal().Msg(fmt.Sprintf("encountered invalid control message type in validate topics expected %s, %s or %s got %s", p2pmsg.CtrlMsgGraft, p2pmsg.CtrlMsgPrune, p2pmsg.CtrlMsgIHave, validationConfig.ControlMsg)) } return nil } @@ -445,7 +446,7 @@ func (c *ControlMsgValidationInspector) validatePrunes(from peer.ID, ctrlMsg *pu } // validateIhaves performs topic validation on all ihaves in the control message using the provided validateTopic func while tracking duplicates. -func (c *ControlMsgValidationInspector) validateIhaves(from peer.ID, validationConfig *netconf.CtrlMsgValidationConfig, ctrlMsg *pubsub_pb.ControlMessage, activeClusterIDS flow.ChainIDList) error { +func (c *ControlMsgValidationInspector) validateIhaves(from peer.ID, validationConfig *p2pconf.CtrlMsgValidationConfig, ctrlMsg *pubsub_pb.ControlMessage, activeClusterIDS flow.ChainIDList) error { sampleSize := util.SampleN(len(ctrlMsg.GetIhave()), c.config.IHaveInspectionMaxSampleSize, c.config.IHaveAsyncInspectSampleSizePercentage) return c.validateTopicsSample(from, validationConfig, ctrlMsg, activeClusterIDS, sampleSize) } @@ -453,10 +454,10 @@ func (c *ControlMsgValidationInspector) validateIhaves(from peer.ID, validationC // validateTopicsSample samples a subset of topics from the specified control message and ensures the sample contains only valid flow topic/channel and no duplicate topics exist. // Sample size ensures liveness of the network when validating messages with no upper bound on the amount of messages that may be received. // All errors returned from this function can be considered benign. -func (c *ControlMsgValidationInspector) validateTopicsSample(from peer.ID, validationConfig *netconf.CtrlMsgValidationConfig, ctrlMsg *pubsub_pb.ControlMessage, activeClusterIDS flow.ChainIDList, sampleSize uint) error { +func (c *ControlMsgValidationInspector) validateTopicsSample(from peer.ID, validationConfig *p2pconf.CtrlMsgValidationConfig, ctrlMsg *pubsub_pb.ControlMessage, activeClusterIDS flow.ChainIDList, sampleSize uint) error { tracker := make(duplicateTopicTracker) switch validationConfig.ControlMsg { - case p2p.CtrlMsgIHave: + case p2pmsg.CtrlMsgIHave: for i := uint(0); i < sampleSize; i++ { topic := channels.Topic(ctrlMsg.Ihave[i].GetTopicID()) if tracker.isDuplicate(topic) { @@ -472,7 +473,7 @@ func (c *ControlMsgValidationInspector) validateTopicsSample(from peer.ID, valid // sanity check // This should never happen validateTopicsSample is only used to validate IHAVE control message types // if any other control message type is encountered here this indicates invalid state irrecoverable error. - c.logger.Fatal().Msg(fmt.Sprintf("encountered invalid control message type in validate topics sample expected %s got %s", p2p.CtrlMsgIHave, validationConfig.ControlMsg)) + c.logger.Fatal().Msg(fmt.Sprintf("encountered invalid control message type in validate topics sample expected %s got %s", p2pmsg.CtrlMsgIHave, validationConfig.ControlMsg)) } return nil } diff --git a/network/p2p/inspector/validation/errors.go b/network/p2p/inspector/validation/errors.go index 9842ac449c8..231fe979498 100644 --- a/network/p2p/inspector/validation/errors.go +++ b/network/p2p/inspector/validation/errors.go @@ -5,13 +5,13 @@ import ( "fmt" "github.com/onflow/flow-go/network/channels" - "github.com/onflow/flow-go/network/p2p" + p2pmsg "github.com/onflow/flow-go/network/p2p/message" ) // ErrHardThreshold indicates that the amount of RPC messages received exceeds hard threshold. type ErrHardThreshold struct { // controlMsg the control message type. - controlMsg p2p.ControlMessageType + controlMsg p2pmsg.ControlMessageType // amount the amount of control messages. amount uint64 // hardThreshold configured hard threshold. @@ -23,7 +23,7 @@ func (e ErrHardThreshold) Error() string { } // NewHardThresholdErr returns a new ErrHardThreshold. -func NewHardThresholdErr(controlMsg p2p.ControlMessageType, amount, hardThreshold uint64) ErrHardThreshold { +func NewHardThresholdErr(controlMsg p2pmsg.ControlMessageType, amount, hardThreshold uint64) ErrHardThreshold { return ErrHardThreshold{controlMsg: controlMsg, amount: amount, hardThreshold: hardThreshold} } @@ -35,7 +35,7 @@ func IsErrHardThreshold(err error) bool { // ErrRateLimitedControlMsg indicates the specified RPC control message is rate limited for the specified peer. type ErrRateLimitedControlMsg struct { - controlMsg p2p.ControlMessageType + controlMsg p2pmsg.ControlMessageType } func (e ErrRateLimitedControlMsg) Error() string { @@ -43,7 +43,7 @@ func (e ErrRateLimitedControlMsg) Error() string { } // NewRateLimitedControlMsgErr returns a new ErrValidationLimit. -func NewRateLimitedControlMsgErr(controlMsg p2p.ControlMessageType) ErrRateLimitedControlMsg { +func NewRateLimitedControlMsgErr(controlMsg p2pmsg.ControlMessageType) ErrRateLimitedControlMsg { return ErrRateLimitedControlMsg{controlMsg: controlMsg} } diff --git a/network/p2p/inspector/validation/errors_test.go b/network/p2p/inspector/validation/errors_test.go index 355b403e908..9bef259fd41 100644 --- a/network/p2p/inspector/validation/errors_test.go +++ b/network/p2p/inspector/validation/errors_test.go @@ -7,7 +7,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/onflow/flow-go/network/channels" - "github.com/onflow/flow-go/network/p2p" + p2pmsg "github.com/onflow/flow-go/network/p2p/message" ) // TestErrActiveClusterIDsNotSetRoundTrip ensures correct error formatting for ErrActiveClusterIdsNotSet. @@ -29,7 +29,7 @@ func TestErrActiveClusterIDsNotSetRoundTrip(t *testing.T) { // TestErrHardThresholdRoundTrip ensures correct error formatting for ErrHardThreshold. func TestErrHardThresholdRoundTrip(t *testing.T) { - controlMsg := p2p.CtrlMsgGraft + controlMsg := p2pmsg.CtrlMsgGraft amount := uint64(100) hardThreshold := uint64(500) err := NewHardThresholdErr(controlMsg, amount, hardThreshold) @@ -48,7 +48,7 @@ func TestErrHardThresholdRoundTrip(t *testing.T) { // TestErrRateLimitedControlMsgRoundTrip ensures correct error formatting for ErrRateLimitedControlMsg. func TestErrRateLimitedControlMsgRoundTrip(t *testing.T) { - controlMsg := p2p.CtrlMsgGraft + controlMsg := p2pmsg.CtrlMsgGraft err := NewRateLimitedControlMsgErr(controlMsg) // tests the error message formatting. diff --git a/network/p2p/inspector/validation/inspect_message_request.go b/network/p2p/inspector/validation/inspect_message_request.go index 08cc3ab336d..bbd68878428 100644 --- a/network/p2p/inspector/validation/inspect_message_request.go +++ b/network/p2p/inspector/validation/inspect_message_request.go @@ -6,8 +6,8 @@ import ( pubsub_pb "github.com/libp2p/go-libp2p-pubsub/pb" "github.com/libp2p/go-libp2p/core/peer" - netconf "github.com/onflow/flow-go/config/network" "github.com/onflow/flow-go/network/p2p/inspector/internal" + "github.com/onflow/flow-go/network/p2p/p2pconf" ) // InspectMsgRequest represents a short digest of an RPC control message. It is used for further message inspection by component workers. @@ -18,11 +18,11 @@ type InspectMsgRequest struct { Peer peer.ID // CtrlMsg the control message that will be inspected. ctrlMsg *pubsub_pb.ControlMessage - validationConfig *netconf.CtrlMsgValidationConfig + validationConfig *p2pconf.CtrlMsgValidationConfig } // NewInspectMsgRequest returns a new *InspectMsgRequest. -func NewInspectMsgRequest(from peer.ID, validationConfig *netconf.CtrlMsgValidationConfig, ctrlMsg *pubsub_pb.ControlMessage) (*InspectMsgRequest, error) { +func NewInspectMsgRequest(from peer.ID, validationConfig *p2pconf.CtrlMsgValidationConfig, ctrlMsg *pubsub_pb.ControlMessage) (*InspectMsgRequest, error) { nonce, err := internal.Nonce() if err != nil { return nil, fmt.Errorf("failed to get inspect message request nonce: %w", err) diff --git a/network/p2p/libp2pNode.go b/network/p2p/libp2pNode.go index a5a92d5cc70..061e45a43ff 100644 --- a/network/p2p/libp2pNode.go +++ b/network/p2p/libp2pNode.go @@ -10,6 +10,7 @@ import ( "github.com/libp2p/go-libp2p/core/protocol" "github.com/libp2p/go-libp2p/core/routing" + "github.com/onflow/flow-go/engine/collection" "github.com/onflow/flow-go/module" "github.com/onflow/flow-go/module/component" "github.com/onflow/flow-go/module/irrecoverable" @@ -35,6 +36,12 @@ type LibP2PNode interface { // DisallowListNotificationConsumer exposes the disallow list notification consumer API for the node so that // it will be notified when a new disallow list update is distributed. DisallowListNotificationConsumer + // CollectionClusterChangesConsumer is the interface for consuming the events of changes in the collection cluster. + // This is used to notify the node of changes in the collection cluster. + // LibP2PNode implements this interface and consumes the events to be notified of changes in the clustering channels. + // The clustering channels are used by the collection nodes of a cluster to communicate with each other. + // As the cluster (and hence their cluster channels) of collection nodes changes over time (per epoch) the node needs to be notified of these changes. + CollectionClusterChangesConsumer // DisallowListOracle exposes the disallow list oracle API for external consumers to query about the disallow list. DisallowListOracle // Start the libp2p node. @@ -93,16 +100,21 @@ type Subscriptions interface { SetUnicastManager(uniMgr UnicastManager) } +// CollectionClusterChangesConsumer is the interface for consuming the events of changes in the collection cluster. +// This is used to notify the node of changes in the collection cluster. +// LibP2PNode implements this interface and consumes the events to be notified of changes in the clustering channels. +// The clustering channels are used by the collection nodes of a cluster to communicate with each other. +// As the cluster (and hence their cluster channels) of collection nodes changes over time (per epoch) the node needs to be notified of these changes. +type CollectionClusterChangesConsumer interface { + collection.ClusterEvents +} + // PeerScore is the interface for the peer score module. It is used to expose the peer score to other // components of the node. It is also used to set the peer score exposer implementation. type PeerScore interface { - // SetPeerScoreExposer sets the node's peer score exposer implementation. - // SetPeerScoreExposer may be called at most once. It is an irrecoverable error to call this - // method if the node's peer score exposer has already been set. - SetPeerScoreExposer(e PeerScoreExposer) // PeerScoreExposer returns the node's peer score exposer implementation. // If the node's peer score exposer has not been set, the second return value will be false. - PeerScoreExposer() (PeerScoreExposer, bool) + PeerScoreExposer() PeerScoreExposer } // PeerConnections subset of funcs related to underlying libp2p host connections. diff --git a/network/p2p/message/types.go b/network/p2p/message/types.go new file mode 100644 index 00000000000..087cbd21455 --- /dev/null +++ b/network/p2p/message/types.go @@ -0,0 +1,20 @@ +package p2pmsg + +// ControlMessageType is the type of control message, as defined in the libp2p pubsub spec. +type ControlMessageType string + +func (c ControlMessageType) String() string { + return string(c) +} + +const ( + CtrlMsgIHave ControlMessageType = "IHAVE" + CtrlMsgIWant ControlMessageType = "IWANT" + CtrlMsgGraft ControlMessageType = "GRAFT" + CtrlMsgPrune ControlMessageType = "PRUNE" +) + +// ControlMessageTypes returns list of all libp2p control message types. +func ControlMessageTypes() []ControlMessageType { + return []ControlMessageType{CtrlMsgIHave, CtrlMsgIWant, CtrlMsgGraft, CtrlMsgPrune} +} diff --git a/network/p2p/mock/collection_cluster_changes_consumer.go b/network/p2p/mock/collection_cluster_changes_consumer.go new file mode 100644 index 00000000000..cb76577f06f --- /dev/null +++ b/network/p2p/mock/collection_cluster_changes_consumer.go @@ -0,0 +1,33 @@ +// Code generated by mockery v2.21.4. DO NOT EDIT. + +package mockp2p + +import ( + flow "github.com/onflow/flow-go/model/flow" + mock "github.com/stretchr/testify/mock" +) + +// CollectionClusterChangesConsumer is an autogenerated mock type for the CollectionClusterChangesConsumer type +type CollectionClusterChangesConsumer struct { + mock.Mock +} + +// ActiveClustersChanged provides a mock function with given fields: _a0 +func (_m *CollectionClusterChangesConsumer) ActiveClustersChanged(_a0 flow.ChainIDList) { + _m.Called(_a0) +} + +type mockConstructorTestingTNewCollectionClusterChangesConsumer interface { + mock.TestingT + Cleanup(func()) +} + +// NewCollectionClusterChangesConsumer creates a new instance of CollectionClusterChangesConsumer. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewCollectionClusterChangesConsumer(t mockConstructorTestingTNewCollectionClusterChangesConsumer) *CollectionClusterChangesConsumer { + mock := &CollectionClusterChangesConsumer{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/network/p2p/mock/connector.go b/network/p2p/mock/connector.go index d1e6733cbab..2c8a49c9070 100644 --- a/network/p2p/mock/connector.go +++ b/network/p2p/mock/connector.go @@ -15,9 +15,9 @@ type Connector struct { mock.Mock } -// UpdatePeers provides a mock function with given fields: ctx, peerIDs -func (_m *Connector) UpdatePeers(ctx context.Context, peerIDs peer.IDSlice) { - _m.Called(ctx, peerIDs) +// Connect provides a mock function with given fields: ctx, peerChan +func (_m *Connector) Connect(ctx context.Context, peerChan <-chan peer.AddrInfo) { + _m.Called(ctx, peerChan) } type mockConstructorTestingTNewConnector interface { diff --git a/network/p2p/mock/connector_factory.go b/network/p2p/mock/connector_factory.go new file mode 100644 index 00000000000..a22788969f8 --- /dev/null +++ b/network/p2p/mock/connector_factory.go @@ -0,0 +1,56 @@ +// Code generated by mockery v2.21.4. DO NOT EDIT. + +package mockp2p + +import ( + host "github.com/libp2p/go-libp2p/core/host" + mock "github.com/stretchr/testify/mock" + + p2p "github.com/onflow/flow-go/network/p2p" +) + +// ConnectorFactory is an autogenerated mock type for the ConnectorFactory type +type ConnectorFactory struct { + mock.Mock +} + +// Execute provides a mock function with given fields: _a0 +func (_m *ConnectorFactory) Execute(_a0 host.Host) (p2p.Connector, error) { + ret := _m.Called(_a0) + + var r0 p2p.Connector + var r1 error + if rf, ok := ret.Get(0).(func(host.Host) (p2p.Connector, error)); ok { + return rf(_a0) + } + if rf, ok := ret.Get(0).(func(host.Host) p2p.Connector); ok { + r0 = rf(_a0) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(p2p.Connector) + } + } + + if rf, ok := ret.Get(1).(func(host.Host) error); ok { + r1 = rf(_a0) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +type mockConstructorTestingTNewConnectorFactory interface { + mock.TestingT + Cleanup(func()) +} + +// NewConnectorFactory creates a new instance of ConnectorFactory. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewConnectorFactory(t mockConstructorTestingTNewConnectorFactory) *ConnectorFactory { + mock := &ConnectorFactory{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/network/p2p/mock/gossip_sub_builder.go b/network/p2p/mock/gossip_sub_builder.go index e01ff021e0d..2146f922c9b 100644 --- a/network/p2p/mock/gossip_sub_builder.go +++ b/network/p2p/mock/gossip_sub_builder.go @@ -10,8 +10,6 @@ import ( mock "github.com/stretchr/testify/mock" - module "github.com/onflow/flow-go/module" - p2p "github.com/onflow/flow-go/network/p2p" peer "github.com/libp2p/go-libp2p/core/peer" @@ -29,13 +27,12 @@ type GossipSubBuilder struct { } // Build provides a mock function with given fields: _a0 -func (_m *GossipSubBuilder) Build(_a0 irrecoverable.SignalerContext) (p2p.PubSubAdapter, p2p.PeerScoreTracer, error) { +func (_m *GossipSubBuilder) Build(_a0 irrecoverable.SignalerContext) (p2p.PubSubAdapter, error) { ret := _m.Called(_a0) var r0 p2p.PubSubAdapter - var r1 p2p.PeerScoreTracer - var r2 error - if rf, ok := ret.Get(0).(func(irrecoverable.SignalerContext) (p2p.PubSubAdapter, p2p.PeerScoreTracer, error)); ok { + var r1 error + if rf, ok := ret.Get(0).(func(irrecoverable.SignalerContext) (p2p.PubSubAdapter, error)); ok { return rf(_a0) } if rf, ok := ret.Get(0).(func(irrecoverable.SignalerContext) p2p.PubSubAdapter); ok { @@ -46,21 +43,18 @@ func (_m *GossipSubBuilder) Build(_a0 irrecoverable.SignalerContext) (p2p.PubSub } } - if rf, ok := ret.Get(1).(func(irrecoverable.SignalerContext) p2p.PeerScoreTracer); ok { + if rf, ok := ret.Get(1).(func(irrecoverable.SignalerContext) error); ok { r1 = rf(_a0) } else { - if ret.Get(1) != nil { - r1 = ret.Get(1).(p2p.PeerScoreTracer) - } + r1 = ret.Error(1) } - if rf, ok := ret.Get(2).(func(irrecoverable.SignalerContext) error); ok { - r2 = rf(_a0) - } else { - r2 = ret.Error(2) - } + return r0, r1 +} - return r0, r1, r2 +// OverrideDefaultRpcInspectorSuiteFactory provides a mock function with given fields: _a0 +func (_m *GossipSubBuilder) OverrideDefaultRpcInspectorSuiteFactory(_a0 p2p.GossipSubRpcInspectorSuiteFactoryFunc) { + _m.Called(_a0) } // SetAppSpecificScoreParams provides a mock function with given fields: _a0 @@ -83,11 +77,6 @@ func (_m *GossipSubBuilder) SetGossipSubPeerScoring(_a0 bool) { _m.Called(_a0) } -// SetGossipSubRPCInspectorSuite provides a mock function with given fields: _a0 -func (_m *GossipSubBuilder) SetGossipSubRPCInspectorSuite(_a0 p2p.GossipSubInspectorSuite) { - _m.Called(_a0) -} - // SetGossipSubScoreTracerInterval provides a mock function with given fields: _a0 func (_m *GossipSubBuilder) SetGossipSubScoreTracerInterval(_a0 time.Duration) { _m.Called(_a0) @@ -103,11 +92,6 @@ func (_m *GossipSubBuilder) SetHost(_a0 host.Host) { _m.Called(_a0) } -// SetIDProvider provides a mock function with given fields: _a0 -func (_m *GossipSubBuilder) SetIDProvider(_a0 module.IdentityProvider) { - _m.Called(_a0) -} - // SetRoutingSystem provides a mock function with given fields: _a0 func (_m *GossipSubBuilder) SetRoutingSystem(_a0 routing.Routing) { _m.Called(_a0) diff --git a/network/p2p/mock/gossip_sub_factory_func.go b/network/p2p/mock/gossip_sub_factory_func.go index 06cd0346c8c..14aa9a7cec4 100644 --- a/network/p2p/mock/gossip_sub_factory_func.go +++ b/network/p2p/mock/gossip_sub_factory_func.go @@ -18,25 +18,25 @@ type GossipSubFactoryFunc struct { mock.Mock } -// Execute provides a mock function with given fields: _a0, _a1, _a2, _a3 -func (_m *GossipSubFactoryFunc) Execute(_a0 context.Context, _a1 zerolog.Logger, _a2 host.Host, _a3 p2p.PubSubAdapterConfig) (p2p.PubSubAdapter, error) { - ret := _m.Called(_a0, _a1, _a2, _a3) +// Execute provides a mock function with given fields: _a0, _a1, _a2, _a3, _a4 +func (_m *GossipSubFactoryFunc) Execute(_a0 context.Context, _a1 zerolog.Logger, _a2 host.Host, _a3 p2p.PubSubAdapterConfig, _a4 p2p.CollectionClusterChangesConsumer) (p2p.PubSubAdapter, error) { + ret := _m.Called(_a0, _a1, _a2, _a3, _a4) var r0 p2p.PubSubAdapter var r1 error - if rf, ok := ret.Get(0).(func(context.Context, zerolog.Logger, host.Host, p2p.PubSubAdapterConfig) (p2p.PubSubAdapter, error)); ok { - return rf(_a0, _a1, _a2, _a3) + if rf, ok := ret.Get(0).(func(context.Context, zerolog.Logger, host.Host, p2p.PubSubAdapterConfig, p2p.CollectionClusterChangesConsumer) (p2p.PubSubAdapter, error)); ok { + return rf(_a0, _a1, _a2, _a3, _a4) } - if rf, ok := ret.Get(0).(func(context.Context, zerolog.Logger, host.Host, p2p.PubSubAdapterConfig) p2p.PubSubAdapter); ok { - r0 = rf(_a0, _a1, _a2, _a3) + if rf, ok := ret.Get(0).(func(context.Context, zerolog.Logger, host.Host, p2p.PubSubAdapterConfig, p2p.CollectionClusterChangesConsumer) p2p.PubSubAdapter); ok { + r0 = rf(_a0, _a1, _a2, _a3, _a4) } else { if ret.Get(0) != nil { r0 = ret.Get(0).(p2p.PubSubAdapter) } } - if rf, ok := ret.Get(1).(func(context.Context, zerolog.Logger, host.Host, p2p.PubSubAdapterConfig) error); ok { - r1 = rf(_a0, _a1, _a2, _a3) + if rf, ok := ret.Get(1).(func(context.Context, zerolog.Logger, host.Host, p2p.PubSubAdapterConfig, p2p.CollectionClusterChangesConsumer) error); ok { + r1 = rf(_a0, _a1, _a2, _a3, _a4) } else { r1 = ret.Error(1) } diff --git a/network/p2p/mock/gossip_sub_inspector_suite.go b/network/p2p/mock/gossip_sub_inspector_suite.go index b9a9c0deb8b..90c7e5b15d7 100644 --- a/network/p2p/mock/gossip_sub_inspector_suite.go +++ b/network/p2p/mock/gossip_sub_inspector_suite.go @@ -3,7 +3,9 @@ package mockp2p import ( + flow "github.com/onflow/flow-go/model/flow" irrecoverable "github.com/onflow/flow-go/module/irrecoverable" + mock "github.com/stretchr/testify/mock" p2p "github.com/onflow/flow-go/network/p2p" @@ -18,8 +20,13 @@ type GossipSubInspectorSuite struct { mock.Mock } -// AddInvCtrlMsgNotifConsumer provides a mock function with given fields: _a0 -func (_m *GossipSubInspectorSuite) AddInvCtrlMsgNotifConsumer(_a0 p2p.GossipSubInvCtrlMsgNotifConsumer) { +// ActiveClustersChanged provides a mock function with given fields: _a0 +func (_m *GossipSubInspectorSuite) ActiveClustersChanged(_a0 flow.ChainIDList) { + _m.Called(_a0) +} + +// AddInvalidControlMessageConsumer provides a mock function with given fields: _a0 +func (_m *GossipSubInspectorSuite) AddInvalidControlMessageConsumer(_a0 p2p.GossipSubInvCtrlMsgNotifConsumer) { _m.Called(_a0) } @@ -55,22 +62,6 @@ func (_m *GossipSubInspectorSuite) InspectFunc() func(peer.ID, *pubsub.RPC) erro return r0 } -// Inspectors provides a mock function with given fields: -func (_m *GossipSubInspectorSuite) Inspectors() []p2p.GossipSubRPCInspector { - ret := _m.Called() - - var r0 []p2p.GossipSubRPCInspector - if rf, ok := ret.Get(0).(func() []p2p.GossipSubRPCInspector); ok { - r0 = rf() - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).([]p2p.GossipSubRPCInspector) - } - } - - return r0 -} - // Ready provides a mock function with given fields: func (_m *GossipSubInspectorSuite) Ready() <-chan struct{} { ret := _m.Called() diff --git a/network/p2p/mock/gossip_sub_rpc_inspector_suite_factory_func.go b/network/p2p/mock/gossip_sub_rpc_inspector_suite_factory_func.go new file mode 100644 index 00000000000..03ea2329d85 --- /dev/null +++ b/network/p2p/mock/gossip_sub_rpc_inspector_suite_factory_func.go @@ -0,0 +1,66 @@ +// Code generated by mockery v2.21.4. DO NOT EDIT. + +package mockp2p + +import ( + flow "github.com/onflow/flow-go/model/flow" + metrics "github.com/onflow/flow-go/module/metrics" + + mock "github.com/stretchr/testify/mock" + + module "github.com/onflow/flow-go/module" + + network "github.com/onflow/flow-go/network" + + p2p "github.com/onflow/flow-go/network/p2p" + + p2pconf "github.com/onflow/flow-go/network/p2p/p2pconf" + + zerolog "github.com/rs/zerolog" +) + +// GossipSubRpcInspectorSuiteFactoryFunc is an autogenerated mock type for the GossipSubRpcInspectorSuiteFactoryFunc type +type GossipSubRpcInspectorSuiteFactoryFunc struct { + mock.Mock +} + +// Execute provides a mock function with given fields: _a0, _a1, _a2, _a3, _a4, _a5, _a6 +func (_m *GossipSubRpcInspectorSuiteFactoryFunc) Execute(_a0 zerolog.Logger, _a1 flow.Identifier, _a2 *p2pconf.GossipSubRPCInspectorsConfig, _a3 module.GossipSubMetrics, _a4 metrics.HeroCacheMetricsFactory, _a5 network.NetworkingType, _a6 module.IdentityProvider) (p2p.GossipSubInspectorSuite, error) { + ret := _m.Called(_a0, _a1, _a2, _a3, _a4, _a5, _a6) + + var r0 p2p.GossipSubInspectorSuite + var r1 error + if rf, ok := ret.Get(0).(func(zerolog.Logger, flow.Identifier, *p2pconf.GossipSubRPCInspectorsConfig, module.GossipSubMetrics, metrics.HeroCacheMetricsFactory, network.NetworkingType, module.IdentityProvider) (p2p.GossipSubInspectorSuite, error)); ok { + return rf(_a0, _a1, _a2, _a3, _a4, _a5, _a6) + } + if rf, ok := ret.Get(0).(func(zerolog.Logger, flow.Identifier, *p2pconf.GossipSubRPCInspectorsConfig, module.GossipSubMetrics, metrics.HeroCacheMetricsFactory, network.NetworkingType, module.IdentityProvider) p2p.GossipSubInspectorSuite); ok { + r0 = rf(_a0, _a1, _a2, _a3, _a4, _a5, _a6) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(p2p.GossipSubInspectorSuite) + } + } + + if rf, ok := ret.Get(1).(func(zerolog.Logger, flow.Identifier, *p2pconf.GossipSubRPCInspectorsConfig, module.GossipSubMetrics, metrics.HeroCacheMetricsFactory, network.NetworkingType, module.IdentityProvider) error); ok { + r1 = rf(_a0, _a1, _a2, _a3, _a4, _a5, _a6) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +type mockConstructorTestingTNewGossipSubRpcInspectorSuiteFactoryFunc interface { + mock.TestingT + Cleanup(func()) +} + +// NewGossipSubRpcInspectorSuiteFactoryFunc creates a new instance of GossipSubRpcInspectorSuiteFactoryFunc. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewGossipSubRpcInspectorSuiteFactoryFunc(t mockConstructorTestingTNewGossipSubRpcInspectorSuiteFactoryFunc) *GossipSubRpcInspectorSuiteFactoryFunc { + mock := &GossipSubRpcInspectorSuiteFactoryFunc{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/network/p2p/mock/lib_p2_p_node.go b/network/p2p/mock/lib_p2_p_node.go index 6665b1a9cb2..5813983110e 100644 --- a/network/p2p/mock/lib_p2_p_node.go +++ b/network/p2p/mock/lib_p2_p_node.go @@ -8,6 +8,8 @@ import ( context "context" + flow "github.com/onflow/flow-go/model/flow" + flow_gonetwork "github.com/onflow/flow-go/network" host "github.com/libp2p/go-libp2p/core/host" @@ -36,6 +38,11 @@ type LibP2PNode struct { mock.Mock } +// ActiveClustersChanged provides a mock function with given fields: _a0 +func (_m *LibP2PNode) ActiveClustersChanged(_a0 flow.ChainIDList) { + _m.Called(_a0) +} + // AddPeer provides a mock function with given fields: ctx, peerInfo func (_m *LibP2PNode) AddPeer(ctx context.Context, peerInfo peer.AddrInfo) error { ret := _m.Called(ctx, peerInfo) @@ -262,14 +269,10 @@ func (_m *LibP2PNode) PeerManagerComponent() component.Component { } // PeerScoreExposer provides a mock function with given fields: -func (_m *LibP2PNode) PeerScoreExposer() (p2p.PeerScoreExposer, bool) { +func (_m *LibP2PNode) PeerScoreExposer() p2p.PeerScoreExposer { ret := _m.Called() var r0 p2p.PeerScoreExposer - var r1 bool - if rf, ok := ret.Get(0).(func() (p2p.PeerScoreExposer, bool)); ok { - return rf() - } if rf, ok := ret.Get(0).(func() p2p.PeerScoreExposer); ok { r0 = rf() } else { @@ -278,13 +281,7 @@ func (_m *LibP2PNode) PeerScoreExposer() (p2p.PeerScoreExposer, bool) { } } - if rf, ok := ret.Get(1).(func() bool); ok { - r1 = rf() - } else { - r1 = ret.Get(1).(bool) - } - - return r0, r1 + return r0 } // Publish provides a mock function with given fields: ctx, topic, data @@ -373,11 +370,6 @@ func (_m *LibP2PNode) SetComponentManager(cm *component.ComponentManager) { _m.Called(cm) } -// SetPeerScoreExposer provides a mock function with given fields: e -func (_m *LibP2PNode) SetPeerScoreExposer(e p2p.PeerScoreExposer) { - _m.Called(e) -} - // SetPubSub provides a mock function with given fields: ps func (_m *LibP2PNode) SetPubSub(ps p2p.PubSubAdapter) { _m.Called(ps) diff --git a/network/p2p/mock/node_builder.go b/network/p2p/mock/node_builder.go index a14e07363ae..97ab398f37a 100644 --- a/network/p2p/mock/node_builder.go +++ b/network/p2p/mock/node_builder.go @@ -13,8 +13,6 @@ import ( mock "github.com/stretchr/testify/mock" - module "github.com/onflow/flow-go/module" - network "github.com/libp2p/go-libp2p/core/network" p2p "github.com/onflow/flow-go/network/p2p" @@ -57,13 +55,29 @@ func (_m *NodeBuilder) Build() (p2p.LibP2PNode, error) { return r0, r1 } -// EnableGossipSubPeerScoring provides a mock function with given fields: _a0, _a1 -func (_m *NodeBuilder) EnableGossipSubPeerScoring(_a0 module.IdentityProvider, _a1 *p2p.PeerScoringConfig) p2p.NodeBuilder { - ret := _m.Called(_a0, _a1) +// EnableGossipSubPeerScoring provides a mock function with given fields: _a0 +func (_m *NodeBuilder) EnableGossipSubPeerScoring(_a0 *p2p.PeerScoringConfig) p2p.NodeBuilder { + ret := _m.Called(_a0) var r0 p2p.NodeBuilder - if rf, ok := ret.Get(0).(func(module.IdentityProvider, *p2p.PeerScoringConfig) p2p.NodeBuilder); ok { - r0 = rf(_a0, _a1) + if rf, ok := ret.Get(0).(func(*p2p.PeerScoringConfig) p2p.NodeBuilder); ok { + r0 = rf(_a0) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(p2p.NodeBuilder) + } + } + + return r0 +} + +// OverrideDefaultRpcInspectorSuiteFactory provides a mock function with given fields: _a0 +func (_m *NodeBuilder) OverrideDefaultRpcInspectorSuiteFactory(_a0 p2p.GossipSubRpcInspectorSuiteFactoryFunc) p2p.NodeBuilder { + ret := _m.Called(_a0) + + var r0 p2p.NodeBuilder + if rf, ok := ret.Get(0).(func(p2p.GossipSubRpcInspectorSuiteFactoryFunc) p2p.NodeBuilder); ok { + r0 = rf(_a0) } else { if ret.Get(0) != nil { r0 = ret.Get(0).(p2p.NodeBuilder) @@ -153,22 +167,6 @@ func (_m *NodeBuilder) SetGossipSubFactory(_a0 p2p.GossipSubFactoryFunc, _a1 p2p return r0 } -// SetGossipSubRpcInspectorSuite provides a mock function with given fields: _a0 -func (_m *NodeBuilder) SetGossipSubRpcInspectorSuite(_a0 p2p.GossipSubInspectorSuite) p2p.NodeBuilder { - ret := _m.Called(_a0) - - var r0 p2p.NodeBuilder - if rf, ok := ret.Get(0).(func(p2p.GossipSubInspectorSuite) p2p.NodeBuilder); ok { - r0 = rf(_a0) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(p2p.NodeBuilder) - } - } - - return r0 -} - // SetGossipSubScoreTracerInterval provides a mock function with given fields: _a0 func (_m *NodeBuilder) SetGossipSubScoreTracerInterval(_a0 time.Duration) p2p.NodeBuilder { ret := _m.Called(_a0) @@ -201,22 +199,6 @@ func (_m *NodeBuilder) SetGossipSubTracer(_a0 p2p.PubSubTracer) p2p.NodeBuilder return r0 } -// SetPeerManagerOptions provides a mock function with given fields: _a0, _a1 -func (_m *NodeBuilder) SetPeerManagerOptions(_a0 bool, _a1 time.Duration) p2p.NodeBuilder { - ret := _m.Called(_a0, _a1) - - var r0 p2p.NodeBuilder - if rf, ok := ret.Get(0).(func(bool, time.Duration) p2p.NodeBuilder); ok { - r0 = rf(_a0, _a1) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(p2p.NodeBuilder) - } - } - - return r0 -} - // SetRateLimiterDistributor provides a mock function with given fields: _a0 func (_m *NodeBuilder) SetRateLimiterDistributor(_a0 p2p.UnicastRateLimiterDistributor) p2p.NodeBuilder { ret := _m.Called(_a0) diff --git a/network/p2p/mock/peer_score.go b/network/p2p/mock/peer_score.go index 374d03d6749..81ea79ec570 100644 --- a/network/p2p/mock/peer_score.go +++ b/network/p2p/mock/peer_score.go @@ -13,14 +13,10 @@ type PeerScore struct { } // PeerScoreExposer provides a mock function with given fields: -func (_m *PeerScore) PeerScoreExposer() (p2p.PeerScoreExposer, bool) { +func (_m *PeerScore) PeerScoreExposer() p2p.PeerScoreExposer { ret := _m.Called() var r0 p2p.PeerScoreExposer - var r1 bool - if rf, ok := ret.Get(0).(func() (p2p.PeerScoreExposer, bool)); ok { - return rf() - } if rf, ok := ret.Get(0).(func() p2p.PeerScoreExposer); ok { r0 = rf() } else { @@ -29,18 +25,7 @@ func (_m *PeerScore) PeerScoreExposer() (p2p.PeerScoreExposer, bool) { } } - if rf, ok := ret.Get(1).(func() bool); ok { - r1 = rf() - } else { - r1 = ret.Get(1).(bool) - } - - return r0, r1 -} - -// SetPeerScoreExposer provides a mock function with given fields: e -func (_m *PeerScore) SetPeerScoreExposer(e p2p.PeerScoreExposer) { - _m.Called(e) + return r0 } type mockConstructorTestingTNewPeerScore interface { diff --git a/network/p2p/mock/peer_updater.go b/network/p2p/mock/peer_updater.go new file mode 100644 index 00000000000..7a708e6c3df --- /dev/null +++ b/network/p2p/mock/peer_updater.go @@ -0,0 +1,36 @@ +// Code generated by mockery v2.21.4. DO NOT EDIT. + +package mockp2p + +import ( + context "context" + + mock "github.com/stretchr/testify/mock" + + peer "github.com/libp2p/go-libp2p/core/peer" +) + +// PeerUpdater is an autogenerated mock type for the PeerUpdater type +type PeerUpdater struct { + mock.Mock +} + +// UpdatePeers provides a mock function with given fields: ctx, peerIDs +func (_m *PeerUpdater) UpdatePeers(ctx context.Context, peerIDs peer.IDSlice) { + _m.Called(ctx, peerIDs) +} + +type mockConstructorTestingTNewPeerUpdater interface { + mock.TestingT + Cleanup(func()) +} + +// NewPeerUpdater creates a new instance of PeerUpdater. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewPeerUpdater(t mockConstructorTestingTNewPeerUpdater) *PeerUpdater { + mock := &PeerUpdater{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/network/p2p/mock/pub_sub_adapter.go b/network/p2p/mock/pub_sub_adapter.go index d8f2cf533a2..ec05980dea1 100644 --- a/network/p2p/mock/pub_sub_adapter.go +++ b/network/p2p/mock/pub_sub_adapter.go @@ -3,7 +3,9 @@ package mockp2p import ( + flow "github.com/onflow/flow-go/model/flow" irrecoverable "github.com/onflow/flow-go/module/irrecoverable" + mock "github.com/stretchr/testify/mock" p2p "github.com/onflow/flow-go/network/p2p" @@ -16,6 +18,11 @@ type PubSubAdapter struct { mock.Mock } +// ActiveClustersChanged provides a mock function with given fields: _a0 +func (_m *PubSubAdapter) ActiveClustersChanged(_a0 flow.ChainIDList) { + _m.Called(_a0) +} + // Done provides a mock function with given fields: func (_m *PubSubAdapter) Done() <-chan struct{} { ret := _m.Called() @@ -90,6 +97,22 @@ func (_m *PubSubAdapter) ListPeers(topic string) []peer.ID { return r0 } +// PeerScoreExposer provides a mock function with given fields: +func (_m *PubSubAdapter) PeerScoreExposer() p2p.PeerScoreExposer { + ret := _m.Called() + + var r0 p2p.PeerScoreExposer + if rf, ok := ret.Get(0).(func() p2p.PeerScoreExposer); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(p2p.PeerScoreExposer) + } + } + + return r0 +} + // Ready provides a mock function with given fields: func (_m *PubSubAdapter) Ready() <-chan struct{} { ret := _m.Called() diff --git a/network/p2p/mock/score_option_builder.go b/network/p2p/mock/score_option_builder.go index eabe096b50a..d0f437bfc12 100644 --- a/network/p2p/mock/score_option_builder.go +++ b/network/p2p/mock/score_option_builder.go @@ -14,15 +14,43 @@ type ScoreOptionBuilder struct { } // BuildFlowPubSubScoreOption provides a mock function with given fields: -func (_m *ScoreOptionBuilder) BuildFlowPubSubScoreOption() pubsub.Option { +func (_m *ScoreOptionBuilder) BuildFlowPubSubScoreOption() (*pubsub.PeerScoreParams, *pubsub.PeerScoreThresholds) { ret := _m.Called() - var r0 pubsub.Option - if rf, ok := ret.Get(0).(func() pubsub.Option); ok { + var r0 *pubsub.PeerScoreParams + var r1 *pubsub.PeerScoreThresholds + if rf, ok := ret.Get(0).(func() (*pubsub.PeerScoreParams, *pubsub.PeerScoreThresholds)); ok { + return rf() + } + if rf, ok := ret.Get(0).(func() *pubsub.PeerScoreParams); ok { r0 = rf() } else { if ret.Get(0) != nil { - r0 = ret.Get(0).(pubsub.Option) + r0 = ret.Get(0).(*pubsub.PeerScoreParams) + } + } + + if rf, ok := ret.Get(1).(func() *pubsub.PeerScoreThresholds); ok { + r1 = rf() + } else { + if ret.Get(1) != nil { + r1 = ret.Get(1).(*pubsub.PeerScoreThresholds) + } + } + + return r0, r1 +} + +// TopicScoreParams provides a mock function with given fields: _a0 +func (_m *ScoreOptionBuilder) TopicScoreParams(_a0 *pubsub.Topic) *pubsub.TopicScoreParams { + ret := _m.Called(_a0) + + var r0 *pubsub.TopicScoreParams + if rf, ok := ret.Get(0).(func(*pubsub.Topic) *pubsub.TopicScoreParams); ok { + r0 = rf(_a0) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*pubsub.TopicScoreParams) } } diff --git a/network/p2p/p2pbuilder/config/config.go b/network/p2p/p2pbuilder/config/config.go index a950a6b2fb1..f784edee0cb 100644 --- a/network/p2p/p2pbuilder/config/config.go +++ b/network/p2p/p2pbuilder/config/config.go @@ -29,4 +29,15 @@ type PeerManagerConfig struct { ConnectionPruning bool // UpdateInterval interval used by the libp2p node peer manager component to periodically request peer updates. UpdateInterval time.Duration + // ConnectorFactory is a factory function to create a new connector. + ConnectorFactory p2p.ConnectorFactory +} + +// PeerManagerDisableConfig returns a configuration that disables the peer manager. +func PeerManagerDisableConfig() *PeerManagerConfig { + return &PeerManagerConfig{ + ConnectionPruning: false, + UpdateInterval: 0, + ConnectorFactory: nil, + } } diff --git a/network/p2p/p2pbuilder/gossipsub/gossipSubBuilder.go b/network/p2p/p2pbuilder/gossipsub/gossipSubBuilder.go index e4422c31c70..89b1351691f 100644 --- a/network/p2p/p2pbuilder/gossipsub/gossipSubBuilder.go +++ b/network/p2p/p2pbuilder/gossipsub/gossipSubBuilder.go @@ -11,10 +11,20 @@ import ( "github.com/libp2p/go-libp2p/core/routing" "github.com/rs/zerolog" + "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module" "github.com/onflow/flow-go/module/irrecoverable" + "github.com/onflow/flow-go/module/mempool/queue" + "github.com/onflow/flow-go/module/metrics" + "github.com/onflow/flow-go/network" "github.com/onflow/flow-go/network/channels" "github.com/onflow/flow-go/network/p2p" + "github.com/onflow/flow-go/network/p2p/distributor" + "github.com/onflow/flow-go/network/p2p/inspector" + "github.com/onflow/flow-go/network/p2p/inspector/validation" + p2pconfig "github.com/onflow/flow-go/network/p2p/p2pbuilder/config" + inspectorbuilder "github.com/onflow/flow-go/network/p2p/p2pbuilder/inspector" + "github.com/onflow/flow-go/network/p2p/p2pconf" "github.com/onflow/flow-go/network/p2p/p2pnode" "github.com/onflow/flow-go/network/p2p/scoring" "github.com/onflow/flow-go/network/p2p/tracer" @@ -23,8 +33,10 @@ import ( // The Builder struct is used to configure and create a new GossipSub pubsub system. type Builder struct { + networkType network.NetworkingType + sporkId flow.Identifier logger zerolog.Logger - metrics module.GossipSubMetrics + metricsCfg *p2pconfig.MetricsConfig h host.Host subscriptionFilter pubsub.SubscriptionFilter gossipSubFactory p2p.GossipSubFactoryFunc @@ -33,11 +45,12 @@ type Builder struct { gossipSubScoreTracerInterval time.Duration // the interval at which the gossipsub score tracer logs the peer scores. // gossipSubTracer is a callback interface that is called by the gossipsub implementation upon // certain events. Currently, we use it to log and observe the local mesh of the node. - gossipSubTracer p2p.PubSubTracer - scoreOptionConfig *scoring.ScoreOptionConfig - idProvider module.IdentityProvider - routingSystem routing.Routing - rpcInspectorSuite p2p.GossipSubInspectorSuite + gossipSubTracer p2p.PubSubTracer + scoreOptionConfig *scoring.ScoreOptionConfig + idProvider module.IdentityProvider + routingSystem routing.Routing + rpcInspectorConfig *p2pconf.GossipSubRPCInspectorsConfig + rpcInspectorSuiteFactory p2p.GossipSubRpcInspectorSuiteFactoryFunc } var _ p2p.GossipSubBuilder = (*Builder)(nil) @@ -109,18 +122,6 @@ func (g *Builder) SetGossipSubTracer(gossipSubTracer p2p.PubSubTracer) { g.gossipSubTracer = gossipSubTracer } -// SetIDProvider sets the identity provider of the builder. -// If the identity provider has already been set, a fatal error is logged. -func (g *Builder) SetIDProvider(idProvider module.IdentityProvider) { - if g.idProvider != nil { - g.logger.Fatal().Msg("id provider has already been set") - return - } - - g.idProvider = idProvider - g.scoreOptionConfig.SetProvider(idProvider) -} - // SetRoutingSystem sets the routing system of the builder. // If the routing system has already been set, a fatal error is logged. func (g *Builder) SetRoutingSystem(routingSystem routing.Routing) { @@ -131,44 +132,130 @@ func (g *Builder) SetRoutingSystem(routingSystem routing.Routing) { g.routingSystem = routingSystem } +// SetTopicScoreParams sets the topic score params of the builder. +// There is a default topic score parameters that is used if this function is not called for a topic. +// However, if this function is called multiple times for a topic, the last topic score params will be used. +// Note: calling this function will override the default topic score params for the topic. Don't call this function +// unless you know what you are doing. func (g *Builder) SetTopicScoreParams(topic channels.Topic, topicScoreParams *pubsub.TopicScoreParams) { - g.scoreOptionConfig.SetTopicScoreParams(topic, topicScoreParams) + g.scoreOptionConfig.OverrideTopicScoreParams(topic, topicScoreParams) } +// SetAppSpecificScoreParams sets the app specific score params of the builder. +// There is no default app specific score function. However, if this function is called multiple times, the last function will be used. func (g *Builder) SetAppSpecificScoreParams(f func(peer.ID) float64) { g.scoreOptionConfig.SetAppSpecificScoreFunction(f) } -// SetGossipSubRPCInspectorSuite sets the gossipsub rpc inspector suite of the builder. It contains the -// inspector function that is injected into the gossipsub rpc layer, as well as the notification distributors that -// are used to notify the app specific scoring mechanism of misbehaving peers.. -func (g *Builder) SetGossipSubRPCInspectorSuite(inspectorSuite p2p.GossipSubInspectorSuite) { - g.rpcInspectorSuite = inspectorSuite +// OverrideDefaultRpcInspectorSuiteFactory overrides the default rpc inspector suite factory. +// Note: this function should only be used for testing purposes. Never override the default rpc inspector suite factory unless you know what you are doing. +func (g *Builder) OverrideDefaultRpcInspectorSuiteFactory(factory p2p.GossipSubRpcInspectorSuiteFactoryFunc) { + g.logger.Warn().Msg("overriding default rpc inspector suite factory") + g.rpcInspectorSuiteFactory = factory } -func NewGossipSubBuilder(logger zerolog.Logger, metrics module.GossipSubMetrics) *Builder { +// NewGossipSubBuilder returns a new gossipsub builder. +// Args: +// - logger: the logger of the node. +// - metricsCfg: the metrics config of the node. +// - networkType: the network type of the node. +// - sporkId: the spork id of the node. +// - idProvider: the identity provider of the node. +// - rpcInspectorConfig: the rpc inspector config of the node. +// Returns: +// - a new gossipsub builder. +// Note: the builder is not thread-safe. It should only be used in the main thread. +func NewGossipSubBuilder( + logger zerolog.Logger, + metricsCfg *p2pconfig.MetricsConfig, + networkType network.NetworkingType, + sporkId flow.Identifier, + idProvider module.IdentityProvider, + rpcInspectorConfig *p2pconf.GossipSubRPCInspectorsConfig, +) *Builder { lg := logger.With().Str("component", "gossipsub").Logger() - return &Builder{ - logger: lg, - metrics: metrics, - gossipSubFactory: defaultGossipSubFactory(), - gossipSubConfigFunc: defaultGossipSubAdapterConfig(), - scoreOptionConfig: scoring.NewScoreOptionConfig(lg), + b := &Builder{ + logger: lg, + metricsCfg: metricsCfg, + sporkId: sporkId, + networkType: networkType, + idProvider: idProvider, + gossipSubFactory: defaultGossipSubFactory(), + gossipSubConfigFunc: defaultGossipSubAdapterConfig(), + scoreOptionConfig: scoring.NewScoreOptionConfig(lg, idProvider), + rpcInspectorConfig: rpcInspectorConfig, + rpcInspectorSuiteFactory: defaultInspectorSuite(), } + return b } +// defaultGossipSubFactory returns the default gossipsub factory function. It is used to create the default gossipsub factory. +// Note: always use the default gossipsub factory function to create the gossipsub factory (unless you know what you are doing). func defaultGossipSubFactory() p2p.GossipSubFactoryFunc { - return func(ctx context.Context, logger zerolog.Logger, h host.Host, cfg p2p.PubSubAdapterConfig) (p2p.PubSubAdapter, error) { - return p2pnode.NewGossipSubAdapter(ctx, logger, h, cfg) + return func( + ctx context.Context, + logger zerolog.Logger, + h host.Host, + cfg p2p.PubSubAdapterConfig, + clusterChangeConsumer p2p.CollectionClusterChangesConsumer) (p2p.PubSubAdapter, error) { + return p2pnode.NewGossipSubAdapter(ctx, logger, h, cfg, clusterChangeConsumer) } } +// defaultGossipSubAdapterConfig returns the default gossipsub config function. It is used to create the default gossipsub config. +// Note: always use the default gossipsub config function to create the gossipsub config (unless you know what you are doing). func defaultGossipSubAdapterConfig() p2p.GossipSubAdapterConfigFunc { return func(cfg *p2p.BasePubSubAdapterConfig) p2p.PubSubAdapterConfig { return p2pnode.NewGossipSubAdapterConfig(cfg) } } +// defaultInspectorSuite returns the default inspector suite factory function. It is used to create the default inspector suite. +// Inspector suite is utilized to inspect the incoming gossipsub rpc messages from different perspectives. +// Note: always use the default inspector suite factory function to create the inspector suite (unless you know what you are doing). +func defaultInspectorSuite() p2p.GossipSubRpcInspectorSuiteFactoryFunc { + return func( + logger zerolog.Logger, + sporkId flow.Identifier, + inspectorCfg *p2pconf.GossipSubRPCInspectorsConfig, + gossipSubMetrics module.GossipSubMetrics, + heroCacheMetricsFactory metrics.HeroCacheMetricsFactory, + networkType network.NetworkingType, + idProvider module.IdentityProvider) (p2p.GossipSubInspectorSuite, error) { + metricsInspector := inspector.NewControlMsgMetricsInspector( + logger, + p2pnode.NewGossipSubControlMessageMetrics(gossipSubMetrics, logger), + inspectorCfg.GossipSubRPCMetricsInspectorConfigs.NumberOfWorkers, + []queue.HeroStoreConfigOption{ + queue.WithHeroStoreSizeLimit(inspectorCfg.GossipSubRPCMetricsInspectorConfigs.CacheSize), + queue.WithHeroStoreCollector(metrics.GossipSubRPCMetricsObserverInspectorQueueMetricFactory(heroCacheMetricsFactory, networkType)), + }...) + notificationDistributor := distributor.DefaultGossipSubInspectorNotificationDistributor( + logger, + []queue.HeroStoreConfigOption{ + queue.WithHeroStoreSizeLimit(inspectorCfg.GossipSubRPCInspectorNotificationCacheSize), + queue.WithHeroStoreCollector(metrics.RpcInspectorNotificationQueueMetricFactory(heroCacheMetricsFactory, networkType))}...) + + inspectMsgQueueCacheCollector := metrics.GossipSubRPCInspectorQueueMetricFactory(heroCacheMetricsFactory, networkType) + clusterPrefixedCacheCollector := metrics.GossipSubRPCInspectorClusterPrefixedCacheMetricFactory(heroCacheMetricsFactory, networkType) + rpcValidationInspector, err := validation.NewControlMsgValidationInspector( + logger, + sporkId, + &inspectorCfg.GossipSubRPCValidationInspectorConfigs, + notificationDistributor, + inspectMsgQueueCacheCollector, + clusterPrefixedCacheCollector, + idProvider, + gossipSubMetrics, + ) + if err != nil { + return nil, fmt.Errorf("failed to create new control message valiadation inspector: %w", err) + } + + return inspectorbuilder.NewGossipSubInspectorSuite([]p2p.GossipSubRPCInspector{metricsInspector, rpcValidationInspector}, notificationDistributor), nil + } +} + // Build creates a new GossipSub pubsub system. // It returns the newly created GossipSub pubsub system and any errors encountered during its creation. // Arguments: @@ -179,14 +266,14 @@ func defaultGossipSubAdapterConfig() p2p.GossipSubAdapterConfigFunc { // - p2p.PeerScoreTracer: a peer score tracer for the GossipSub pubsub system (if enabled, otherwise nil). // - error: if an error occurs during the creation of the GossipSub pubsub system, it is returned. Otherwise, nil is returned. // Note that on happy path, the returned error is nil. Any error returned is unexpected and should be handled as irrecoverable. -func (g *Builder) Build(ctx irrecoverable.SignalerContext) (p2p.PubSubAdapter, p2p.PeerScoreTracer, error) { +func (g *Builder) Build(ctx irrecoverable.SignalerContext) (p2p.PubSubAdapter, error) { gossipSubConfigs := g.gossipSubConfigFunc(&p2p.BasePubSubAdapterConfig{ MaxMessageSize: p2pnode.DefaultMaxPubSubMsgSize, }) gossipSubConfigs.WithMessageIdFunction(utils.MessageID) if g.routingSystem == nil { - return nil, nil, fmt.Errorf("could not create gossipsub: routing system is nil") + return nil, fmt.Errorf("could not create gossipsub: routing system is nil") } gossipSubConfigs.WithRoutingDiscovery(g.routingSystem) @@ -194,17 +281,23 @@ func (g *Builder) Build(ctx irrecoverable.SignalerContext) (p2p.PubSubAdapter, p gossipSubConfigs.WithSubscriptionFilter(g.subscriptionFilter) } - if g.rpcInspectorSuite != nil { - gossipSubConfigs.WithInspectorSuite(g.rpcInspectorSuite) + inspectorSuite, err := g.rpcInspectorSuiteFactory( + g.logger, + g.sporkId, + g.rpcInspectorConfig, + g.metricsCfg.Metrics, + g.metricsCfg.HeroCacheFactory, + g.networkType, + g.idProvider) + if err != nil { + return nil, fmt.Errorf("could not create gossipsub inspector suite: %w", err) } + gossipSubConfigs.WithInspectorSuite(inspectorSuite) var scoreOpt *scoring.ScoreOption var scoreTracer p2p.PeerScoreTracer if g.gossipSubPeerScoring { - if g.rpcInspectorSuite != nil { - g.scoreOptionConfig.SetRegisterNotificationConsumerFunc(g.rpcInspectorSuite.AddInvCtrlMsgNotifConsumer) - } - + g.scoreOptionConfig.SetRegisterNotificationConsumerFunc(inspectorSuite.AddInvalidControlMessageConsumer) scoreOpt = scoring.NewScoreOption(g.scoreOptionConfig) gossipSubConfigs.WithScoreOption(scoreOpt) @@ -212,7 +305,7 @@ func (g *Builder) Build(ctx irrecoverable.SignalerContext) (p2p.PubSubAdapter, p scoreTracer = tracer.NewGossipSubScoreTracer( g.logger, g.idProvider, - g.metrics, + g.metricsCfg.Metrics, g.gossipSubScoreTracerInterval) gossipSubConfigs.WithScoreTracer(scoreTracer) } @@ -224,20 +317,20 @@ func (g *Builder) Build(ctx irrecoverable.SignalerContext) (p2p.PubSubAdapter, p } if g.h == nil { - return nil, nil, fmt.Errorf("could not create gossipsub: host is nil") + return nil, fmt.Errorf("could not create gossipsub: host is nil") } - gossipSub, err := g.gossipSubFactory(ctx, g.logger, g.h, gossipSubConfigs) + gossipSub, err := g.gossipSubFactory(ctx, g.logger, g.h, gossipSubConfigs, inspectorSuite) if err != nil { - return nil, nil, fmt.Errorf("could not create gossipsub: %w", err) + return nil, fmt.Errorf("could not create gossipsub: %w", err) } if scoreOpt != nil { err := scoreOpt.SetSubscriptionProvider(scoring.NewSubscriptionProvider(g.logger, gossipSub)) if err != nil { - return nil, nil, fmt.Errorf("could not set subscription provider: %w", err) + return nil, fmt.Errorf("could not set subscription provider: %w", err) } } - return gossipSub, scoreTracer, nil + return gossipSub, nil } diff --git a/network/p2p/p2pbuilder/inspector/suite/aggregate.go b/network/p2p/p2pbuilder/inspector/aggregate.go similarity index 98% rename from network/p2p/p2pbuilder/inspector/suite/aggregate.go rename to network/p2p/p2pbuilder/inspector/aggregate.go index 9c774f40291..604a888fb45 100644 --- a/network/p2p/p2pbuilder/inspector/suite/aggregate.go +++ b/network/p2p/p2pbuilder/inspector/aggregate.go @@ -1,4 +1,4 @@ -package suite +package inspector import ( "github.com/hashicorp/go-multierror" diff --git a/network/p2p/p2pbuilder/inspector/rpc_inspector_builder.go b/network/p2p/p2pbuilder/inspector/rpc_inspector_builder.go deleted file mode 100644 index 4d67dd51b9b..00000000000 --- a/network/p2p/p2pbuilder/inspector/rpc_inspector_builder.go +++ /dev/null @@ -1,120 +0,0 @@ -package inspector - -import ( - "fmt" - - "github.com/rs/zerolog" - - netconf "github.com/onflow/flow-go/config/network" - "github.com/onflow/flow-go/model/flow" - "github.com/onflow/flow-go/module" - "github.com/onflow/flow-go/module/mempool/queue" - "github.com/onflow/flow-go/module/metrics" - "github.com/onflow/flow-go/network" - "github.com/onflow/flow-go/network/p2p" - "github.com/onflow/flow-go/network/p2p/distributor" - "github.com/onflow/flow-go/network/p2p/inspector" - "github.com/onflow/flow-go/network/p2p/inspector/validation" - p2pconfig "github.com/onflow/flow-go/network/p2p/p2pbuilder/config" - "github.com/onflow/flow-go/network/p2p/p2pbuilder/inspector/suite" - "github.com/onflow/flow-go/network/p2p/p2pnode" -) - -// GossipSubInspectorBuilder builder that constructs all rpc inspectors used by gossip sub. The following -// rpc inspectors are created with this builder. -// - validation inspector: performs validation on all control messages. -// - metrics inspector: observes metrics for each rpc message received. -type GossipSubInspectorBuilder struct { - logger zerolog.Logger - sporkID flow.Identifier - inspectorsConfig *netconf.GossipSubRPCInspectorsConfig - metricsCfg *p2pconfig.MetricsConfig - idProvider module.IdentityProvider - inspectorMetrics module.GossipSubRpcValidationInspectorMetrics - networkType network.NetworkingType -} - -// NewGossipSubInspectorBuilder returns new *GossipSubInspectorBuilder. -func NewGossipSubInspectorBuilder(logger zerolog.Logger, sporkID flow.Identifier, inspectorsConfig *netconf.GossipSubRPCInspectorsConfig, provider module.IdentityProvider, inspectorMetrics module.GossipSubRpcValidationInspectorMetrics) *GossipSubInspectorBuilder { - return &GossipSubInspectorBuilder{ - logger: logger, - sporkID: sporkID, - inspectorsConfig: inspectorsConfig, - metricsCfg: &p2pconfig.MetricsConfig{ - Metrics: metrics.NewNoopCollector(), - HeroCacheFactory: metrics.NewNoopHeroCacheMetricsFactory(), - }, - idProvider: provider, - inspectorMetrics: inspectorMetrics, - networkType: network.PublicNetwork, - } -} - -// SetMetrics sets the network metrics and registry. -func (b *GossipSubInspectorBuilder) SetMetrics(metricsCfg *p2pconfig.MetricsConfig) *GossipSubInspectorBuilder { - b.metricsCfg = metricsCfg - return b -} - -// SetNetworkType sets the network type for the inspector. -// This is used to determine if the node is running on a public or private network. -// Args: -// - networkType: the network type. -// Returns: -// - *GossipSubInspectorBuilder: the builder. -func (b *GossipSubInspectorBuilder) SetNetworkType(networkType network.NetworkingType) *GossipSubInspectorBuilder { - b.networkType = networkType - return b -} - -// buildGossipSubMetricsInspector builds the gossipsub rpc metrics inspector. -func (b *GossipSubInspectorBuilder) buildGossipSubMetricsInspector() p2p.GossipSubRPCInspector { - gossipSubMetrics := p2pnode.NewGossipSubControlMessageMetrics(b.metricsCfg.Metrics, b.logger) - metricsInspector := inspector.NewControlMsgMetricsInspector( - b.logger, - gossipSubMetrics, - b.inspectorsConfig.GossipSubRPCMetricsInspectorConfigs.NumberOfWorkers, - []queue.HeroStoreConfigOption{ - queue.WithHeroStoreSizeLimit(b.inspectorsConfig.GossipSubRPCMetricsInspectorConfigs.CacheSize), - queue.WithHeroStoreCollector(metrics.GossipSubRPCMetricsObserverInspectorQueueMetricFactory(b.metricsCfg.HeroCacheFactory, b.networkType)), - }...) - return metricsInspector -} - -// buildGossipSubValidationInspector builds the gossipsub rpc validation inspector. -func (b *GossipSubInspectorBuilder) buildGossipSubValidationInspector() (p2p.GossipSubRPCInspector, *distributor.GossipSubInspectorNotifDistributor, error) { - notificationDistributor := distributor.DefaultGossipSubInspectorNotificationDistributor( - b.logger, - []queue.HeroStoreConfigOption{ - queue.WithHeroStoreSizeLimit(b.inspectorsConfig.GossipSubRPCInspectorNotificationCacheSize), - queue.WithHeroStoreCollector(metrics.RpcInspectorNotificationQueueMetricFactory(b.metricsCfg.HeroCacheFactory, b.networkType))}...) - - inspectMsgQueueCacheCollector := metrics.GossipSubRPCInspectorQueueMetricFactory(b.metricsCfg.HeroCacheFactory, b.networkType) - clusterPrefixedCacheCollector := metrics.GossipSubRPCInspectorClusterPrefixedCacheMetricFactory(b.metricsCfg.HeroCacheFactory, b.networkType) - rpcValidationInspector, err := validation.NewControlMsgValidationInspector( - b.logger, - b.sporkID, - &b.inspectorsConfig.GossipSubRPCValidationInspectorConfigs, - notificationDistributor, - inspectMsgQueueCacheCollector, - clusterPrefixedCacheCollector, - b.idProvider, - b.inspectorMetrics, - ) - if err != nil { - return nil, nil, fmt.Errorf("failed to create new control message valiadation inspector: %w", err) - } - return rpcValidationInspector, notificationDistributor, nil -} - -// Build builds the rpc inspectors used by gossipsub. -// Any returned error from this func indicates a problem setting up rpc inspectors. -// In libp2p node setup, the returned error should be treated as a fatal error. -func (b *GossipSubInspectorBuilder) Build() (p2p.GossipSubInspectorSuite, error) { - metricsInspector := b.buildGossipSubMetricsInspector() - validationInspector, notificationDistributor, err := b.buildGossipSubValidationInspector() - if err != nil { - return nil, err - } - return suite.NewGossipSubInspectorSuite([]p2p.GossipSubRPCInspector{metricsInspector, validationInspector}, notificationDistributor), nil -} diff --git a/network/p2p/p2pbuilder/inspector/suite/suite.go b/network/p2p/p2pbuilder/inspector/suite.go similarity index 75% rename from network/p2p/p2pbuilder/inspector/suite/suite.go rename to network/p2p/p2pbuilder/inspector/suite.go index 6271d627cf4..eda1d68089b 100644 --- a/network/p2p/p2pbuilder/inspector/suite/suite.go +++ b/network/p2p/p2pbuilder/inspector/suite.go @@ -1,9 +1,10 @@ -package suite +package inspector import ( pubsub "github.com/libp2p/go-libp2p-pubsub" "github.com/libp2p/go-libp2p/core/peer" + "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module/component" "github.com/onflow/flow-go/module/irrecoverable" "github.com/onflow/flow-go/network/p2p" @@ -17,6 +18,8 @@ type GossipSubInspectorSuite struct { ctrlMsgInspectDistributor p2p.GossipSubInspectorNotifDistributor } +var _ p2p.GossipSubInspectorSuite = (*GossipSubInspectorSuite)(nil) + // NewGossipSubInspectorSuite creates a new GossipSubInspectorSuite. // The suite is composed of the aggregated inspector, which is used to inspect the gossipsub rpc messages, and the // control message notification distributor, which is used to notify consumers when a misbehaving peer regarding gossipsub @@ -62,15 +65,21 @@ func (s *GossipSubInspectorSuite) InspectFunc() func(peer.ID, *pubsub.RPC) error return s.aggregatedInspector.Inspect } -// AddInvalidCtrlMsgNotificationConsumer adds a consumer to the invalid control message notification distributor. +// AddInvalidControlMessageConsumer adds a consumer to the invalid control message notification distributor. // This consumer is notified when a misbehaving peer regarding gossipsub control messages is detected. This follows a pub/sub // pattern where the consumer is notified when a new notification is published. // A consumer is only notified once for each notification, and only receives notifications that were published after it was added. -func (s *GossipSubInspectorSuite) AddInvCtrlMsgNotifConsumer(c p2p.GossipSubInvCtrlMsgNotifConsumer) { +func (s *GossipSubInspectorSuite) AddInvalidControlMessageConsumer(c p2p.GossipSubInvCtrlMsgNotifConsumer) { s.ctrlMsgInspectDistributor.AddConsumer(c) } -// Inspectors returns all inspectors in the inspector suite. -func (s *GossipSubInspectorSuite) Inspectors() []p2p.GossipSubRPCInspector { - return s.aggregatedInspector.Inspectors() +// ActiveClustersChanged is called when the list of active collection nodes cluster is changed. +// GossipSubInspectorSuite consumes this event and forwards it to all the respective rpc inspectors, that are +// concerned with this cluster-based topics (i.e., channels), so that they can update their internal state. +func (s *GossipSubInspectorSuite) ActiveClustersChanged(list flow.ChainIDList) { + for _, rpcInspector := range s.aggregatedInspector.Inspectors() { + if r, ok := rpcInspector.(p2p.GossipSubMsgValidationRpcInspector); ok { + r.ActiveClustersChanged(list) + } + } } diff --git a/network/p2p/p2pbuilder/libp2pNodeBuilder.go b/network/p2p/p2pbuilder/libp2pNodeBuilder.go index 376cd71246c..4d30b1fff02 100644 --- a/network/p2p/p2pbuilder/libp2pNodeBuilder.go +++ b/network/p2p/p2pbuilder/libp2pNodeBuilder.go @@ -21,18 +21,20 @@ import ( madns "github.com/multiformats/go-multiaddr-dns" "github.com/rs/zerolog" - netconf "github.com/onflow/flow-go/config/network" fcrypto "github.com/onflow/flow-go/crypto" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module" "github.com/onflow/flow-go/module/component" "github.com/onflow/flow-go/module/irrecoverable" + flownet "github.com/onflow/flow-go/network" + "github.com/onflow/flow-go/network/netconf" "github.com/onflow/flow-go/network/p2p" "github.com/onflow/flow-go/network/p2p/connection" "github.com/onflow/flow-go/network/p2p/dht" "github.com/onflow/flow-go/network/p2p/keyutils" p2pconfig "github.com/onflow/flow-go/network/p2p/p2pbuilder/config" gossipsubbuilder "github.com/onflow/flow-go/network/p2p/p2pbuilder/gossipsub" + "github.com/onflow/flow-go/network/p2p/p2pconf" "github.com/onflow/flow-go/network/p2p/p2pnode" "github.com/onflow/flow-go/network/p2p/subscription" "github.com/onflow/flow-go/network/p2p/tracer" @@ -51,20 +53,19 @@ type GossipSubAdapterConfigFunc func(*p2p.BasePubSubAdapterConfig) p2p.PubSubAda type LibP2PNodeBuilder struct { gossipSubBuilder p2p.GossipSubBuilder - sporkID flow.Identifier - addr string + sporkId flow.Identifier + address string networkKey fcrypto.PrivateKey logger zerolog.Logger metrics module.LibP2PMetrics basicResolver madns.BasicResolver resourceManager network.ResourceManager - resourceManagerCfg *netconf.ResourceManagerConfig + resourceManagerCfg *p2pconf.ResourceManagerConfig connManager connmgr.ConnManager connGater p2p.ConnectionGater routingFactory func(context.Context, host.Host) (routing.Routing, error) - peerManagerEnablePruning bool - peerManagerUpdateInterval time.Duration + peerManagerConfig *p2pconfig.PeerManagerConfig createNode p2p.CreateNodeFunc createStreamRetryInterval time.Duration rateLimiterDistributor p2p.UnicastRateLimiterDistributor @@ -72,26 +73,40 @@ type LibP2PNodeBuilder struct { disallowListCacheCfg *p2p.DisallowListCacheConfig } -func NewNodeBuilder(logger zerolog.Logger, - metrics module.LibP2PMetrics, - addr string, +func NewNodeBuilder( + logger zerolog.Logger, + metricsConfig *p2pconfig.MetricsConfig, + networkingType flownet.NetworkingType, + address string, networkKey fcrypto.PrivateKey, - sporkID flow.Identifier, - rCfg *netconf.ResourceManagerConfig, + sporkId flow.Identifier, + idProvider module.IdentityProvider, + rCfg *p2pconf.ResourceManagerConfig, + rpcInspectorCfg *p2pconf.GossipSubRPCInspectorsConfig, + peerManagerConfig *p2pconfig.PeerManagerConfig, disallowListCacheCfg *p2p.DisallowListCacheConfig) *LibP2PNodeBuilder { return &LibP2PNodeBuilder{ logger: logger, - sporkID: sporkID, - addr: addr, + sporkId: sporkId, + address: address, networkKey: networkKey, createNode: DefaultCreateNodeFunc, - metrics: metrics, + metrics: metricsConfig.Metrics, resourceManagerCfg: rCfg, - gossipSubBuilder: gossipsubbuilder.NewGossipSubBuilder(logger, metrics), disallowListCacheCfg: disallowListCacheCfg, + gossipSubBuilder: gossipsubbuilder.NewGossipSubBuilder( + logger, + metricsConfig, + networkingType, + sporkId, + idProvider, + rpcInspectorCfg), + peerManagerConfig: peerManagerConfig, } } +var _ p2p.NodeBuilder = &LibP2PNodeBuilder{} + // SetBasicResolver sets the DNS resolver for the node. func (builder *LibP2PNodeBuilder) SetBasicResolver(br madns.BasicResolver) p2p.NodeBuilder { builder.basicResolver = br @@ -136,11 +151,9 @@ func (builder *LibP2PNodeBuilder) SetGossipSubFactory(gf p2p.GossipSubFactoryFun // EnableGossipSubPeerScoring enables peer scoring for the GossipSub pubsub system. // Arguments: -// - module.IdentityProvider: the identity provider for the node (must be set before calling this method). // - *PeerScoringConfig: the peer scoring configuration for the GossipSub pubsub system. If nil, the default configuration is used. -func (builder *LibP2PNodeBuilder) EnableGossipSubPeerScoring(provider module.IdentityProvider, config *p2p.PeerScoringConfig) p2p.NodeBuilder { +func (builder *LibP2PNodeBuilder) EnableGossipSubPeerScoring(config *p2p.PeerScoringConfig) p2p.NodeBuilder { builder.gossipSubBuilder.SetGossipSubPeerScoring(true) - builder.gossipSubBuilder.SetIDProvider(provider) if config != nil { if config.AppSpecificScoreParams != nil { builder.gossipSubBuilder.SetAppSpecificScoreParams(config.AppSpecificScoreParams) @@ -155,13 +168,6 @@ func (builder *LibP2PNodeBuilder) EnableGossipSubPeerScoring(provider module.Ide return builder } -// SetPeerManagerOptions sets the peer manager options. -func (builder *LibP2PNodeBuilder) SetPeerManagerOptions(connectionPruning bool, updateInterval time.Duration) p2p.NodeBuilder { - builder.peerManagerEnablePruning = connectionPruning - builder.peerManagerUpdateInterval = updateInterval - return builder -} - func (builder *LibP2PNodeBuilder) SetGossipSubTracer(tracer p2p.PubSubTracer) p2p.NodeBuilder { builder.gossipSubBuilder.SetGossipSubTracer(tracer) builder.gossipSubTracer = tracer @@ -188,8 +194,8 @@ func (builder *LibP2PNodeBuilder) SetGossipSubScoreTracerInterval(interval time. return builder } -func (builder *LibP2PNodeBuilder) SetGossipSubRpcInspectorSuite(inspectorSuite p2p.GossipSubInspectorSuite) p2p.NodeBuilder { - builder.gossipSubBuilder.SetGossipSubRPCInspectorSuite(inspectorSuite) +func (builder *LibP2PNodeBuilder) OverrideDefaultRpcInspectorSuiteFactory(factory p2p.GossipSubRpcInspectorSuiteFactoryFunc) p2p.NodeBuilder { + builder.gossipSubBuilder.OverrideDefaultRpcInspectorSuiteFactory(factory) return builder } @@ -274,7 +280,7 @@ func (builder *LibP2PNodeBuilder) Build() (p2p.LibP2PNode, error) { opts = append(opts, libp2p.ConnectionGater(builder.connGater)) } - h, err := DefaultLibP2PHost(builder.addr, builder.networkKey, opts...) + h, err := DefaultLibP2PHost(builder.address, builder.networkKey, opts...) if err != nil { return nil, err } @@ -286,18 +292,22 @@ func (builder *LibP2PNodeBuilder) Build() (p2p.LibP2PNode, error) { } var peerManager p2p.PeerManager - if builder.peerManagerUpdateInterval > 0 { - connector, err := connection.NewLibp2pConnector(&connection.ConnectorConfig{ - PruneConnections: builder.peerManagerEnablePruning, - Logger: builder.logger, - Host: connection.NewConnectorHost(h), - BackoffConnectorFactory: connection.DefaultLibp2pBackoffConnectorFactory(h), + if builder.peerManagerConfig.UpdateInterval > 0 { + connector, err := builder.peerManagerConfig.ConnectorFactory(h) + if err != nil { + return nil, fmt.Errorf("failed to create libp2p connector: %w", err) + } + peerUpdater, err := connection.NewPeerUpdater(&connection.PeerUpdaterConfig{ + PruneConnections: builder.peerManagerConfig.ConnectionPruning, + Logger: builder.logger, + Host: connection.NewConnectorHost(h), + Connector: connector, }) if err != nil { return nil, fmt.Errorf("failed to create libp2p connector: %w", err) } - peerManager = connection.NewPeerManager(builder.logger, builder.peerManagerUpdateInterval, connector) + peerManager = connection.NewPeerManager(builder.logger, builder.peerManagerConfig.UpdateInterval, peerUpdater) if builder.rateLimiterDistributor != nil { builder.rateLimiterDistributor.AddConsumer(peerManager) @@ -312,7 +322,7 @@ func (builder *LibP2PNodeBuilder) Build() (p2p.LibP2PNode, error) { unicastManager := unicast.NewUnicastManager(builder.logger, stream.NewLibP2PStreamFactory(h), - builder.sporkID, + builder.sporkId, builder.createStreamRetryInterval, node, builder.metrics) @@ -329,13 +339,10 @@ func (builder *LibP2PNodeBuilder) Build() (p2p.LibP2PNode, error) { builder.gossipSubBuilder.SetRoutingSystem(routingSystem) // gossipsub is created here, because it needs to be created during the node startup. - gossipSub, scoreTracer, err := builder.gossipSubBuilder.Build(ctx) + gossipSub, err := builder.gossipSubBuilder.Build(ctx) if err != nil { ctx.Throw(fmt.Errorf("could not create gossipsub: %w", err)) } - if scoreTracer != nil { - node.SetPeerScoreExposer(scoreTracer) - } node.SetPubSub(gossipSub) gossipSub.Start(ctx) ready() @@ -428,8 +435,10 @@ func DefaultCreateNodeFunc(logger zerolog.Logger, } // DefaultNodeBuilder returns a node builder. -func DefaultNodeBuilder(log zerolog.Logger, +func DefaultNodeBuilder( + logger zerolog.Logger, address string, + networkingType flownet.NetworkingType, flowKey fcrypto.PrivateKey, sporkId flow.Identifier, idProvider module.IdentityProvider, @@ -438,14 +447,14 @@ func DefaultNodeBuilder(log zerolog.Logger, role string, connGaterCfg *p2pconfig.ConnectionGaterConfig, peerManagerCfg *p2pconfig.PeerManagerConfig, - gossipCfg *netconf.GossipSubConfig, - rpcInspectorSuite p2p.GossipSubInspectorSuite, - rCfg *netconf.ResourceManagerConfig, + gossipCfg *p2pconf.GossipSubConfig, + rpcInspectorCfg *p2pconf.GossipSubRPCInspectorsConfig, + rCfg *p2pconf.ResourceManagerConfig, uniCfg *p2pconfig.UnicastConfig, connMgrConfig *netconf.ConnectionManagerConfig, disallowListCacheCfg *p2p.DisallowListCacheConfig) (p2p.NodeBuilder, error) { - connManager, err := connection.NewConnManager(log, metricsCfg.Metrics, connMgrConfig) + connManager, err := connection.NewConnManager(logger, metricsCfg.Metrics, connMgrConfig) if err != nil { return nil, fmt.Errorf("could not create connection manager: %w", err) } @@ -454,30 +463,39 @@ func DefaultNodeBuilder(log zerolog.Logger, peerFilter := notEjectedPeerFilter(idProvider) peerFilters := []p2p.PeerFilter{peerFilter} - connGater := connection.NewConnGater(log, + connGater := connection.NewConnGater(logger, idProvider, connection.WithOnInterceptPeerDialFilters(append(peerFilters, connGaterCfg.InterceptPeerDialFilters...)), connection.WithOnInterceptSecuredFilters(append(peerFilters, connGaterCfg.InterceptSecuredFilters...))) - builder := NewNodeBuilder(log, metricsCfg.Metrics, address, flowKey, sporkId, rCfg, disallowListCacheCfg). + builder := NewNodeBuilder( + logger, + metricsCfg, + networkingType, + address, + flowKey, + sporkId, + idProvider, + rCfg, + rpcInspectorCfg, + peerManagerCfg, + disallowListCacheCfg). SetBasicResolver(resolver). SetConnectionManager(connManager). SetConnectionGater(connGater). SetRoutingSystem(func(ctx context.Context, host host.Host) (routing.Routing, error) { - return dht.NewDHT(ctx, host, protocols.FlowDHTProtocolID(sporkId), log, metricsCfg.Metrics, dht.AsServer()) + return dht.NewDHT(ctx, host, protocols.FlowDHTProtocolID(sporkId), logger, metricsCfg.Metrics, dht.AsServer()) }). - SetPeerManagerOptions(peerManagerCfg.ConnectionPruning, peerManagerCfg.UpdateInterval). SetStreamCreationRetryInterval(uniCfg.StreamRetryInterval). SetCreateNode(DefaultCreateNodeFunc). - SetRateLimiterDistributor(uniCfg.RateLimiterDistributor). - SetGossipSubRpcInspectorSuite(rpcInspectorSuite) + SetRateLimiterDistributor(uniCfg.RateLimiterDistributor) if gossipCfg.PeerScoring { // currently, we only enable peer scoring with default parameters. So, we set the score parameters to nil. - builder.EnableGossipSubPeerScoring(idProvider, nil) + builder.EnableGossipSubPeerScoring(nil) } - meshTracer := tracer.NewGossipSubMeshTracer(log, metricsCfg.Metrics, idProvider, gossipCfg.LocalMeshLogInterval) + meshTracer := tracer.NewGossipSubMeshTracer(logger, metricsCfg.Metrics, idProvider, gossipCfg.LocalMeshLogInterval) builder.SetGossipSubTracer(meshTracer) builder.SetGossipSubScoreTracerInterval(gossipCfg.ScoreTracerInterval) diff --git a/config/network/errors.go b/network/p2p/p2pconf/errors.go similarity index 79% rename from config/network/errors.go rename to network/p2p/p2pconf/errors.go index 9fee981e7d5..88417ced914 100644 --- a/config/network/errors.go +++ b/network/p2p/p2pconf/errors.go @@ -1,10 +1,10 @@ -package network +package p2pconf import ( "errors" "fmt" - "github.com/onflow/flow-go/network/p2p" + p2pmsg "github.com/onflow/flow-go/network/p2p/message" ) // InvalidLimitConfigError indicates the validation limit is < 0. @@ -21,7 +21,7 @@ func (e InvalidLimitConfigError) Unwrap() error { } // NewInvalidLimitConfigErr returns a new ErrValidationLimit. -func NewInvalidLimitConfigErr(controlMsg p2p.ControlMessageType, err error) InvalidLimitConfigError { +func NewInvalidLimitConfigErr(controlMsg p2pmsg.ControlMessageType, err error) InvalidLimitConfigError { return InvalidLimitConfigError{fmt.Errorf("invalid rpc control message %s validation limit configuration: %w", controlMsg, err)} } diff --git a/config/network/errors_test.go b/network/p2p/p2pconf/errors_test.go similarity index 90% rename from config/network/errors_test.go rename to network/p2p/p2pconf/errors_test.go index 604314fc83c..681d839a5fa 100644 --- a/config/network/errors_test.go +++ b/network/p2p/p2pconf/errors_test.go @@ -1,4 +1,4 @@ -package network +package p2pconf import ( "fmt" @@ -6,12 +6,12 @@ import ( "github.com/stretchr/testify/assert" - "github.com/onflow/flow-go/network/p2p" + p2pmsg "github.com/onflow/flow-go/network/p2p/message" ) // TestErrInvalidLimitConfigRoundTrip ensures correct error formatting for ErrInvalidLimitConfig. func TestErrInvalidLimitConfigRoundTrip(t *testing.T) { - controlMsg := p2p.CtrlMsgGraft + controlMsg := p2pmsg.CtrlMsgGraft limit := uint64(500) e := fmt.Errorf("invalid rate limit value %d must be greater than 0", limit) diff --git a/config/network/gossipsub.go b/network/p2p/p2pconf/gossipsub.go similarity index 98% rename from config/network/gossipsub.go rename to network/p2p/p2pconf/gossipsub.go index c67b1174750..f9155129efd 100644 --- a/config/network/gossipsub.go +++ b/network/p2p/p2pconf/gossipsub.go @@ -1,4 +1,4 @@ -package network +package p2pconf import ( "time" diff --git a/config/network/gossipsub_rpc_inspectors.go b/network/p2p/p2pconf/gossipsub_rpc_inspectors.go similarity index 94% rename from config/network/gossipsub_rpc_inspectors.go rename to network/p2p/p2pconf/gossipsub_rpc_inspectors.go index 040f8a9560d..43c041eb1a8 100644 --- a/config/network/gossipsub_rpc_inspectors.go +++ b/network/p2p/p2pconf/gossipsub_rpc_inspectors.go @@ -1,9 +1,9 @@ -package network +package p2pconf import ( "fmt" - "github.com/onflow/flow-go/network/p2p" + p2pmsg "github.com/onflow/flow-go/network/p2p/message" ) // GossipSubRPCInspectorsConfig encompasses configuration related to gossipsub RPC message inspectors. @@ -62,25 +62,25 @@ type GossipSubRPCValidationInspectorConfigs struct { } // GetCtrlMsgValidationConfig returns the CtrlMsgValidationConfig for the specified p2p.ControlMessageType. -func (conf *GossipSubRPCValidationInspectorConfigs) GetCtrlMsgValidationConfig(controlMsg p2p.ControlMessageType) (*CtrlMsgValidationConfig, bool) { +func (conf *GossipSubRPCValidationInspectorConfigs) GetCtrlMsgValidationConfig(controlMsg p2pmsg.ControlMessageType) (*CtrlMsgValidationConfig, bool) { switch controlMsg { - case p2p.CtrlMsgGraft: + case p2pmsg.CtrlMsgGraft: return &CtrlMsgValidationConfig{ - ControlMsg: p2p.CtrlMsgGraft, + ControlMsg: p2pmsg.CtrlMsgGraft, HardThreshold: conf.GraftLimits.HardThreshold, SafetyThreshold: conf.GraftLimits.SafetyThreshold, RateLimit: conf.GraftLimits.RateLimit, }, true - case p2p.CtrlMsgPrune: + case p2pmsg.CtrlMsgPrune: return &CtrlMsgValidationConfig{ - ControlMsg: p2p.CtrlMsgPrune, + ControlMsg: p2pmsg.CtrlMsgPrune, HardThreshold: conf.PruneLimits.HardThreshold, SafetyThreshold: conf.PruneLimits.SafetyThreshold, RateLimit: conf.PruneLimits.RateLimit, }, true - case p2p.CtrlMsgIHave: + case p2pmsg.CtrlMsgIHave: return &CtrlMsgValidationConfig{ - ControlMsg: p2p.CtrlMsgIHave, + ControlMsg: p2pmsg.CtrlMsgIHave, HardThreshold: conf.IHaveLimits.HardThreshold, SafetyThreshold: conf.IHaveLimits.SafetyThreshold, RateLimit: conf.IHaveLimits.RateLimit, @@ -93,17 +93,17 @@ func (conf *GossipSubRPCValidationInspectorConfigs) GetCtrlMsgValidationConfig(c // AllCtrlMsgValidationConfig returns all control message validation configs in a list. func (conf *GossipSubRPCValidationInspectorConfigs) AllCtrlMsgValidationConfig() CtrlMsgValidationConfigs { return CtrlMsgValidationConfigs{&CtrlMsgValidationConfig{ - ControlMsg: p2p.CtrlMsgGraft, + ControlMsg: p2pmsg.CtrlMsgGraft, HardThreshold: conf.GraftLimits.HardThreshold, SafetyThreshold: conf.GraftLimits.SafetyThreshold, RateLimit: conf.GraftLimits.RateLimit, }, &CtrlMsgValidationConfig{ - ControlMsg: p2p.CtrlMsgPrune, + ControlMsg: p2pmsg.CtrlMsgPrune, HardThreshold: conf.PruneLimits.HardThreshold, SafetyThreshold: conf.PruneLimits.SafetyThreshold, RateLimit: conf.PruneLimits.RateLimit, }, &CtrlMsgValidationConfig{ - ControlMsg: p2p.CtrlMsgIHave, + ControlMsg: p2pmsg.CtrlMsgIHave, HardThreshold: conf.IHaveLimits.HardThreshold, SafetyThreshold: conf.IHaveLimits.SafetyThreshold, RateLimit: conf.IHaveLimits.RateLimit, @@ -116,7 +116,7 @@ type CtrlMsgValidationConfigs []*CtrlMsgValidationConfig // CtrlMsgValidationConfig configuration values for upper, lower threshold and rate limit. type CtrlMsgValidationConfig struct { // ControlMsg the type of RPC control message. - ControlMsg p2p.ControlMessageType + ControlMsg p2pmsg.ControlMessageType // HardThreshold specifies the hard limit for the size of an RPC control message. // While it is generally expected that RPC messages with a size greater than HardThreshold should be dropped, // there are exceptions. For instance, if the message is an 'iHave', blocking processing is performed diff --git a/network/p2p/p2pnode/disallow_listing_test.go b/network/p2p/p2pnode/disallow_listing_test.go index 0249c3ee91f..566b2d6ec5e 100644 --- a/network/p2p/p2pnode/disallow_listing_test.go +++ b/network/p2p/p2pnode/disallow_listing_test.go @@ -11,9 +11,9 @@ import ( "github.com/onflow/flow-go/module/irrecoverable" mockmodule "github.com/onflow/flow-go/module/mock" "github.com/onflow/flow-go/network" - "github.com/onflow/flow-go/network/internal/testutils" "github.com/onflow/flow-go/network/p2p" "github.com/onflow/flow-go/network/p2p/connection" + p2pconfig "github.com/onflow/flow-go/network/p2p/p2pbuilder/config" p2ptest "github.com/onflow/flow-go/network/p2p/test" "github.com/onflow/flow-go/utils/unittest" ) @@ -35,11 +35,15 @@ func TestDisconnectingFromDisallowListedNode(t *testing.T) { sporkID, t.Name(), idProvider, - p2ptest.WithPeerManagerEnabled(true, connection.DefaultPeerUpdateInterval, + p2ptest.WithPeerManagerEnabled(&p2pconfig.PeerManagerConfig{ + ConnectionPruning: true, + UpdateInterval: connection.DefaultPeerUpdateInterval, + ConnectorFactory: connection.DefaultLibp2pBackoffConnectorFactory(), + }, func() peer.IDSlice { return peerIDSlice }), - p2ptest.WithConnectionGater(testutils.NewConnectionGater(idProvider, func(p peer.ID) error { + p2ptest.WithConnectionGater(p2ptest.NewConnectionGater(idProvider, func(p peer.ID) error { // allow all the connections, except for the ones that are disallow-listed, which are determined when // this connection gater object queries the disallow listing oracle that will be provided to it by // the libp2p node. So, here, we don't need to do anything except just enabling the connection gater. diff --git a/network/p2p/p2pnode/gossipSubAdapter.go b/network/p2p/p2pnode/gossipSubAdapter.go index ab72db379f9..861093993cc 100644 --- a/network/p2p/p2pnode/gossipSubAdapter.go +++ b/network/p2p/p2pnode/gossipSubAdapter.go @@ -9,9 +9,11 @@ import ( "github.com/libp2p/go-libp2p/core/peer" "github.com/rs/zerolog" + "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module/component" "github.com/onflow/flow-go/module/irrecoverable" "github.com/onflow/flow-go/network/p2p" + "github.com/onflow/flow-go/network/p2p/utils" "github.com/onflow/flow-go/utils/logging" ) @@ -20,12 +22,24 @@ import ( type GossipSubAdapter struct { component.Component gossipSub *pubsub.PubSub - logger zerolog.Logger + // topicScoreParamFunc is a function that returns the topic score params for a given topic. + // If no function is provided the node will join the topic with no scoring params. As the + // node will not be able to score other peers in the topic, it may be vulnerable to routing + // attacks on the topic that may also affect the overall function of the node. + // It is not recommended to use this adapter without a topicScoreParamFunc. Also in mature + // implementations of the Flow network, the topicScoreParamFunc must be a required parameter. + topicScoreParamFunc func(topic *pubsub.Topic) *pubsub.TopicScoreParams + logger zerolog.Logger + peerScoreExposer p2p.PeerScoreExposer + // clusterChangeConsumer is a callback that is invoked when the set of active clusters of collection nodes changes. + // This callback is implemented by the rpc inspector suite of the GossipSubAdapter, and consumes the cluster changes + // to update the rpc inspector state of the recent topics (i.e., channels). + clusterChangeConsumer p2p.CollectionClusterChangesConsumer } var _ p2p.PubSubAdapter = (*GossipSubAdapter)(nil) -func NewGossipSubAdapter(ctx context.Context, logger zerolog.Logger, h host.Host, cfg p2p.PubSubAdapterConfig) (p2p.PubSubAdapter, error) { +func NewGossipSubAdapter(ctx context.Context, logger zerolog.Logger, h host.Host, cfg p2p.PubSubAdapterConfig, clusterChangeConsumer p2p.CollectionClusterChangesConsumer) (p2p.PubSubAdapter, error) { gossipSubConfig, ok := cfg.(*GossipSubAdapterConfig) if !ok { return nil, fmt.Errorf("invalid gossipsub config type: %T", cfg) @@ -39,8 +53,16 @@ func NewGossipSubAdapter(ctx context.Context, logger zerolog.Logger, h host.Host builder := component.NewComponentManagerBuilder() a := &GossipSubAdapter{ - gossipSub: gossipSub, - logger: logger, + gossipSub: gossipSub, + logger: logger.With().Str("component", "gossipsub-adapter").Logger(), + clusterChangeConsumer: clusterChangeConsumer, + } + + topicScoreParamFunc, ok := gossipSubConfig.TopicScoreParamFunc() + if ok { + a.topicScoreParamFunc = topicScoreParamFunc + } else { + a.logger.Warn().Msg("no topic score param func provided") } if scoreTracer := gossipSubConfig.ScoreTracer(); scoreTracer != nil { @@ -53,6 +75,7 @@ func NewGossipSubAdapter(ctx context.Context, logger zerolog.Logger, h host.Host <-scoreTracer.Done() a.logger.Debug().Str("component", "gossipsub_score_tracer").Msg("score tracer stopped") }) + a.peerScoreExposer = scoreTracer } if tracer := gossipSubConfig.PubSubTracer(); tracer != nil { @@ -124,6 +147,20 @@ func (g *GossipSubAdapter) Join(topic string) (p2p.Topic, error) { if err != nil { return nil, fmt.Errorf("could not join topic %s: %w", topic, err) } + + if g.topicScoreParamFunc != nil { + topicParams := g.topicScoreParamFunc(t) + err = t.SetScoreParams(topicParams) + if err != nil { + return nil, fmt.Errorf("could not set score params for topic %s: %w", topic, err) + } + topicParamsLogger := utils.TopicScoreParamsLogger(g.logger, topic, topicParams) + topicParamsLogger.Info().Msg("joined topic with score params set") + } else { + g.logger.Warn(). + Str("topic", topic). + Msg("joining topic without score params, this is not recommended from a security perspective") + } return NewGossipSubTopic(t), nil } @@ -134,3 +171,29 @@ func (g *GossipSubAdapter) GetTopics() []string { func (g *GossipSubAdapter) ListPeers(topic string) []peer.ID { return g.gossipSub.ListPeers(topic) } + +// PeerScoreExposer returns the peer score exposer for the gossipsub adapter. The exposer is a read-only interface +// for querying peer scores and returns the local scoring table of the underlying gossipsub node. +// The exposer is only available if the gossipsub adapter was configured with a score tracer. +// If the gossipsub adapter was not configured with a score tracer, the exposer will be nil. +// Args: +// +// None. +// +// Returns: +// +// The peer score exposer for the gossipsub adapter. +func (g *GossipSubAdapter) PeerScoreExposer() p2p.PeerScoreExposer { + return g.peerScoreExposer +} + +// ActiveClustersChanged is called when the active clusters of collection nodes changes. +// GossipSubAdapter implements this method to forward the call to the clusterChangeConsumer (rpc inspector), +// which will then update the cluster state of the rpc inspector. +// Args: +// - lst: the list of active clusters +// Returns: +// - void +func (g *GossipSubAdapter) ActiveClustersChanged(lst flow.ChainIDList) { + g.clusterChangeConsumer.ActiveClustersChanged(lst) +} diff --git a/network/p2p/p2pnode/gossipSubAdapterConfig.go b/network/p2p/p2pnode/gossipSubAdapterConfig.go index c5fafd20dbe..a2dbe59289f 100644 --- a/network/p2p/p2pnode/gossipSubAdapterConfig.go +++ b/network/p2p/p2pnode/gossipSubAdapterConfig.go @@ -16,6 +16,7 @@ import ( type GossipSubAdapterConfig struct { options []pubsub.Option scoreTracer p2p.PeerScoreTracer + scoreOption p2p.ScoreOptionBuilder pubsubTracer p2p.PubSubTracer inspectorSuite p2p.GossipSubInspectorSuite // currently only used to manage the lifecycle. } @@ -60,7 +61,9 @@ func (g *GossipSubAdapterConfig) WithSubscriptionFilter(filter p2p.SubscriptionF // Returns: // -None func (g *GossipSubAdapterConfig) WithScoreOption(option p2p.ScoreOptionBuilder) { - g.options = append(g.options, option.BuildFlowPubSubScoreOption()) + params, thresholds := option.BuildFlowPubSubScoreOption() + g.scoreOption = option + g.options = append(g.options, pubsub.WithPeerScore(params, thresholds)) } // WithMessageIdFunction adds a message ID function option to the config. @@ -177,6 +180,24 @@ func convertTopicScoreSnapshot(snapshot map[string]*pubsub.TopicScoreSnapshot) m return newSnapshot } +// TopicScoreParamFunc returns the topic score param function. This function is used to get the topic score params for a topic. +// The topic score params are used to set the topic parameters in GossipSub at the time of joining the topic. +// Args: +// - None +// +// Returns: +// - func(topic *pubsub.Topic) *pubsub.TopicScoreParams: the topic score param function if set, nil otherwise. +// - bool: true if the topic score param function is set, false otherwise. +func (g *GossipSubAdapterConfig) TopicScoreParamFunc() (func(topic *pubsub.Topic) *pubsub.TopicScoreParams, bool) { + if g.scoreOption != nil { + return func(topic *pubsub.Topic) *pubsub.TopicScoreParams { + return g.scoreOption.TopicScoreParams(topic) + }, true + } + + return nil, false +} + // Build returns the libp2p pubsub options. // Args: // - None diff --git a/network/p2p/p2pnode/libp2pNode.go b/network/p2p/p2pnode/libp2pNode.go index 60a548c694b..38746521430 100644 --- a/network/p2p/p2pnode/libp2pNode.go +++ b/network/p2p/p2pnode/libp2pNode.go @@ -18,6 +18,7 @@ import ( "github.com/libp2p/go-libp2p/core/routing" "github.com/rs/zerolog" + "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module/component" "github.com/onflow/flow-go/module/irrecoverable" flownet "github.com/onflow/flow-go/network" @@ -53,16 +54,15 @@ var _ p2p.LibP2PNode = (*Node)(nil) type Node struct { component.Component sync.RWMutex - uniMgr p2p.UnicastManager - host host.Host // reference to the libp2p host (https://godoc.org/github.com/libp2p/go-libp2p/core/host) - pubSub p2p.PubSubAdapter - logger zerolog.Logger // used to provide logging - topics map[channels.Topic]p2p.Topic // map of a topic string to an actual topic instance - subs map[channels.Topic]p2p.Subscription // map of a topic string to an actual subscription - routing routing.Routing - pCache p2p.ProtocolPeerCache - peerManager p2p.PeerManager - peerScoreExposer p2p.PeerScoreExposer + uniMgr p2p.UnicastManager + host host.Host // reference to the libp2p host (https://godoc.org/github.com/libp2p/go-libp2p/core/host) + pubSub p2p.PubSubAdapter + logger zerolog.Logger // used to provide logging + topics map[channels.Topic]p2p.Topic // map of a topic string to an actual topic instance + subs map[channels.Topic]p2p.Subscription // map of a topic string to an actual subscription + routing routing.Routing + pCache p2p.ProtocolPeerCache + peerManager p2p.PeerManager // Cache of temporary disallow-listed peers, when a peer is disallow-listed, the connections to that peer // are closed and further connections are not allowed till the peer is removed from the disallow-list. disallowListedCache p2p.DisallowListCache @@ -91,7 +91,7 @@ func NewNode( } } -var _ component.Component = (*Node)(nil) +var _ p2p.LibP2PNode = (*Node)(nil) func (n *Node) Start(ctx irrecoverable.SignalerContext) { n.Component.Start(ctx) @@ -428,25 +428,10 @@ func (n *Node) Routing() routing.Routing { return n.routing } -// SetPeerScoreExposer sets the node's peer score exposer implementation. -// SetPeerScoreExposer may be called at most once. It is an irrecoverable error to call this -// method if the node's peer score exposer has already been set. -func (n *Node) SetPeerScoreExposer(e p2p.PeerScoreExposer) { - if n.peerScoreExposer != nil { - n.logger.Fatal().Msg("peer score exposer already set") - } - - n.peerScoreExposer = e -} - // PeerScoreExposer returns the node's peer score exposer implementation. // If the node's peer score exposer has not been set, the second return value will be false. -func (n *Node) PeerScoreExposer() (p2p.PeerScoreExposer, bool) { - if n.peerScoreExposer == nil { - return nil, false - } - - return n.peerScoreExposer, true +func (n *Node) PeerScoreExposer() p2p.PeerScoreExposer { + return n.pubSub.PeerScoreExposer() } // SetPubSub sets the node's pubsub implementation. @@ -535,3 +520,14 @@ func (n *Node) OnAllowListNotification(peerId peer.ID, cause flownet.DisallowLis func (n *Node) IsDisallowListed(peerId peer.ID) ([]flownet.DisallowListedCause, bool) { return n.disallowListedCache.IsDisallowListed(peerId) } + +// ActiveClustersChanged is called when the active clusters list of the collection clusters has changed. +// The LibP2PNode implementation directly calls the ActiveClustersChanged method of the pubsub implementation, as +// the pubsub implementation is responsible for the actual handling of the event. +// Args: +// - list: the new active clusters list. +// Returns: +// - none +func (n *Node) ActiveClustersChanged(list flow.ChainIDList) { + n.pubSub.ActiveClustersChanged(list) +} diff --git a/network/p2p/p2pnode/libp2pNode_test.go b/network/p2p/p2pnode/libp2pNode_test.go index 3644bd3dbf2..519a0579163 100644 --- a/network/p2p/p2pnode/libp2pNode_test.go +++ b/network/p2p/p2pnode/libp2pNode_test.go @@ -24,7 +24,6 @@ import ( "github.com/onflow/flow-go/network/channels" "github.com/onflow/flow-go/network/internal/p2pfixtures" "github.com/onflow/flow-go/network/internal/p2putils" - "github.com/onflow/flow-go/network/internal/testutils" "github.com/onflow/flow-go/network/p2p" "github.com/onflow/flow-go/network/p2p/p2pnode" p2ptest "github.com/onflow/flow-go/network/p2p/test" @@ -114,8 +113,8 @@ func TestAddPeers(t *testing.T) { count := 3 ctx, cancel := context.WithCancel(context.Background()) signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) - idProvider := mockmodule.NewIdentityProvider(t) - // create nodes + idProvider := unittest.NewUpdatableIDProvider(flow.IdentityList{}) + nodes, identities := p2ptest.NodesFixture(t, unittest.IdentifierFixture(), "test_add_peers", count, idProvider) p2ptest.StartNodes(t, signalerCtx, nodes, 100*time.Millisecond) defer p2ptest.StopNodes(t, nodes, cancel, 100*time.Millisecond) @@ -136,7 +135,7 @@ func TestRemovePeers(t *testing.T) { count := 3 ctx, cancel := context.WithCancel(context.Background()) signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) - idProvider := mockmodule.NewIdentityProvider(t) + idProvider := unittest.NewUpdatableIDProvider(flow.IdentityList{}) // create nodes nodes, identities := p2ptest.NodesFixture(t, unittest.IdentifierFixture(), "test_remove_peers", count, idProvider) peerInfos, errs := utils.PeerInfosFromIDs(identities) @@ -173,7 +172,7 @@ func TestConnGater(t *testing.T) { sporkID, t.Name(), idProvider, - p2ptest.WithConnectionGater(testutils.NewConnectionGater(idProvider, func(pid peer.ID) error { + p2ptest.WithConnectionGater(p2ptest.NewConnectionGater(idProvider, func(pid peer.ID) error { if !node1Peers.Has(pid) { return fmt.Errorf("peer id not found: %s", pid.String()) } @@ -192,7 +191,7 @@ func TestConnGater(t *testing.T) { t, sporkID, t.Name(), idProvider, - p2ptest.WithConnectionGater(testutils.NewConnectionGater(idProvider, func(pid peer.ID) error { + p2ptest.WithConnectionGater(p2ptest.NewConnectionGater(idProvider, func(pid peer.ID) error { if !node2Peers.Has(pid) { return fmt.Errorf("id not found: %s", pid.String()) } @@ -230,7 +229,7 @@ func TestConnGater(t *testing.T) { func TestNode_HasSubscription(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) - idProvider := mockmodule.NewIdentityProvider(t) + idProvider := unittest.NewUpdatableIDProvider(flow.IdentityList{}) sporkID := unittest.IdentifierFixture() node, _ := p2ptest.NodeFixture(t, sporkID, "test_has_subscription", idProvider) @@ -263,13 +262,14 @@ func TestCreateStream_SinglePairwiseConnection(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) - idProvider := mockmodule.NewIdentityProvider(t) + idProvider := unittest.NewUpdatableIDProvider(flow.IdentityList{}) nodes, ids := p2ptest.NodesFixture(t, sporkId, "test_create_stream_single_pairwise_connection", nodeCount, idProvider, p2ptest.WithDefaultResourceManager()) + idProvider.SetIdentities(ids) p2ptest.StartNodes(t, signalerCtx, nodes, 100*time.Millisecond) defer p2ptest.StopNodes(t, nodes, cancel, 100*time.Millisecond) @@ -337,7 +337,7 @@ func TestCreateStream_SinglePeerDial(t *testing.T) { sporkID, t.Name(), idProvider, - p2ptest.WithConnectionGater(testutils.NewConnectionGater(idProvider, func(pid peer.ID) error { + p2ptest.WithConnectionGater(p2ptest.NewConnectionGater(idProvider, func(pid peer.ID) error { // avoid connection gating outbound messages on sender return nil })), @@ -353,7 +353,7 @@ func TestCreateStream_SinglePeerDial(t *testing.T) { sporkID, t.Name(), idProvider, - p2ptest.WithConnectionGater(testutils.NewConnectionGater(idProvider, func(pid peer.ID) error { + p2ptest.WithConnectionGater(p2ptest.NewConnectionGater(idProvider, func(pid peer.ID) error { // connection gate all incoming connections forcing the senders unicast manager to perform retries return fmt.Errorf("gate keep") })), diff --git a/network/p2p/p2pnode/libp2pStream_test.go b/network/p2p/p2pnode/libp2pStream_test.go index e3b7bf281b3..551cabca4d0 100644 --- a/network/p2p/p2pnode/libp2pStream_test.go +++ b/network/p2p/p2pnode/libp2pStream_test.go @@ -11,9 +11,6 @@ import ( "testing" "time" - "github.com/onflow/flow-go/network/p2p" - p2ptest "github.com/onflow/flow-go/network/p2p/test" - "github.com/libp2p/go-libp2p/core" "github.com/libp2p/go-libp2p/core/network" "github.com/libp2p/go-libp2p/core/peerstore" @@ -21,6 +18,9 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/onflow/flow-go/network/p2p" + p2ptest "github.com/onflow/flow-go/network/p2p/test" + "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module/irrecoverable" mockmodule "github.com/onflow/flow-go/module/mock" @@ -42,7 +42,7 @@ func TestStreamClosing(t *testing.T) { var msgRegex = regexp.MustCompile("^hello[0-9]") handler, streamCloseWG := mockStreamHandlerForMessages(t, ctx, count, msgRegex) - idProvider := mockmodule.NewIdentityProvider(t) + idProvider := unittest.NewUpdatableIDProvider(flow.IdentityList{}) // Creates nodes nodes, identities := p2ptest.NodesFixture(t, unittest.IdentifierFixture(), @@ -50,10 +50,8 @@ func TestStreamClosing(t *testing.T) { 2, idProvider, p2ptest.WithDefaultStreamHandler(handler)) - for i, node := range nodes { - idProvider.On("ByPeerID", node.Host().ID()).Return(&identities[i], true).Maybe() + idProvider.SetIdentities(identities) - } p2ptest.StartNodes(t, signalerCtx, nodes, 100*time.Millisecond) defer p2ptest.StopNodes(t, nodes, cancel, 100*time.Millisecond) @@ -153,17 +151,14 @@ func testCreateStream(t *testing.T, sporkId flow.Identifier, unicasts []protocol count := 2 ctx, cancel := context.WithCancel(context.Background()) signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) - idProvider := mockmodule.NewIdentityProvider(t) + idProvider := unittest.NewUpdatableIDProvider(flow.IdentityList{}) nodes, identities := p2ptest.NodesFixture(t, sporkId, "test_create_stream", count, idProvider, p2ptest.WithPreferredUnicasts(unicasts)) - for i, node := range nodes { - idProvider.On("ByPeerID", node.Host().ID()).Return(&identities[i], true).Maybe() - - } + idProvider.SetIdentities(identities) p2ptest.StartNodes(t, signalerCtx, nodes, 100*time.Millisecond) defer p2ptest.StopNodes(t, nodes, cancel, 100*time.Millisecond) @@ -285,14 +280,11 @@ func TestCreateStream_FallBack(t *testing.T) { func TestCreateStreamIsConcurrencySafe(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) - idProvider := mockmodule.NewIdentityProvider(t) + idProvider := unittest.NewUpdatableIDProvider(flow.IdentityList{}) // create two nodes nodes, identities := p2ptest.NodesFixture(t, unittest.IdentifierFixture(), "test_create_stream_is_concurrency_safe", 2, idProvider) require.Len(t, identities, 2) - for i, node := range nodes { - idProvider.On("ByPeerID", node.Host().ID()).Return(&identities[i], true).Maybe() - - } + idProvider.SetIdentities(flow.IdentityList{identities[0], identities[1]}) p2ptest.StartNodes(t, signalerCtx, nodes, 100*time.Millisecond) defer p2ptest.StopNodes(t, nodes, cancel, 100*time.Millisecond) @@ -336,7 +328,7 @@ func TestNoBackoffWhenCreatingStream(t *testing.T) { ctx2, cancel2 := context.WithCancel(ctx) signalerCtx2 := irrecoverable.NewMockSignalerContext(t, ctx2) - idProvider := mockmodule.NewIdentityProvider(t) + idProvider := unittest.NewUpdatableIDProvider(flow.IdentityList{}) count := 2 // Creates nodes nodes, identities := p2ptest.NodesFixture(t, @@ -347,10 +339,7 @@ func TestNoBackoffWhenCreatingStream(t *testing.T) { ) node1 := nodes[0] node2 := nodes[1] - for i, node := range nodes { - idProvider.On("ByPeerID", node.Host().ID()).Return(&identities[i], true).Maybe() - - } + idProvider.SetIdentities(flow.IdentityList{identities[0], identities[1]}) p2ptest.StartNode(t, signalerCtx1, node1, 100*time.Millisecond) p2ptest.StartNode(t, signalerCtx2, node2, 100*time.Millisecond) @@ -497,7 +486,7 @@ func TestUnicastOverStream_Fallback(t *testing.T) { func TestCreateStreamTimeoutWithUnresponsiveNode(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) - idProvider := mockmodule.NewIdentityProvider(t) + idProvider := unittest.NewUpdatableIDProvider(flow.IdentityList{}) // creates a regular node nodes, identities := p2ptest.NodesFixture(t, unittest.IdentifierFixture(), @@ -506,10 +495,7 @@ func TestCreateStreamTimeoutWithUnresponsiveNode(t *testing.T) { idProvider, ) require.Len(t, identities, 1) - for i, node := range nodes { - idProvider.On("ByPeerID", node.Host().ID()).Return(&identities[i], true).Maybe() - - } + idProvider.SetIdentities(identities) p2ptest.StartNodes(t, signalerCtx, nodes, 100*time.Millisecond) defer p2ptest.StopNodes(t, nodes, cancel, 100*time.Millisecond) @@ -542,7 +528,7 @@ func TestCreateStreamTimeoutWithUnresponsiveNode(t *testing.T) { func TestCreateStreamIsConcurrent(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) - idProvider := mockmodule.NewIdentityProvider(t) + idProvider := unittest.NewUpdatableIDProvider(flow.IdentityList{}) // create two regular node goodNodes, goodNodeIds := p2ptest.NodesFixture(t, unittest.IdentifierFixture(), @@ -551,10 +537,7 @@ func TestCreateStreamIsConcurrent(t *testing.T) { idProvider, ) require.Len(t, goodNodeIds, 2) - for i, node := range goodNodes { - idProvider.On("ByPeerID", node.Host().ID()).Return(&goodNodeIds[i], true).Maybe() - - } + idProvider.SetIdentities(goodNodeIds) p2ptest.StartNodes(t, signalerCtx, goodNodes, 100*time.Millisecond) defer p2ptest.StopNodes(t, goodNodes, cancel, 100*time.Millisecond) diff --git a/network/p2p/pubsub.go b/network/p2p/pubsub.go index 3f19c892c75..3e8e030ad8c 100644 --- a/network/p2p/pubsub.go +++ b/network/p2p/pubsub.go @@ -27,6 +27,12 @@ type TopicValidatorFunc func(context.Context, peer.ID, *pubsub.Message) Validati // PubSubAdapter is the abstraction of the underlying pubsub logic that is used by the Flow network. type PubSubAdapter interface { component.Component + // CollectionClusterChangesConsumer is the interface for consuming the events of changes in the collection cluster. + // This is used to notify the node of changes in the collection cluster. + // PubSubAdapter implements this interface and consumes the events to be notified of changes in the clustering channels. + // The clustering channels are used by the collection nodes of a cluster to communicate with each other. + // As the cluster (and hence their cluster channels) of collection nodes changes over time (per epoch) the node needs to be notified of these changes. + CollectionClusterChangesConsumer // RegisterTopicValidator registers a validator for topic. RegisterTopicValidator(topic string, topicValidator TopicValidatorFunc) error @@ -47,6 +53,16 @@ type PubSubAdapter interface { // For example, if current peer has subscribed to topics A and B, then ListPeers only return // subscribed peers for topics A and B, and querying for topic C will return an empty list. ListPeers(topic string) []peer.ID + + // PeerScoreExposer returns the peer score exposer for the gossipsub adapter. The exposer is a read-only interface + // for querying peer scores and returns the local scoring table of the underlying gossipsub node. + // The exposer is only available if the gossipsub adapter was configured with a score tracer. + // If the gossipsub adapter was not configured with a score tracer, the exposer will be nil. + // Args: + // None. + // Returns: + // The peer score exposer for the gossipsub adapter. + PeerScoreExposer() PeerScoreExposer } // PubSubAdapterConfig abstracts the configuration for the underlying pubsub implementation. @@ -113,7 +129,10 @@ type Topic interface { // ScoreOptionBuilder abstracts the configuration for the underlying pubsub score implementation. type ScoreOptionBuilder interface { // BuildFlowPubSubScoreOption builds the pubsub score options as pubsub.Option for the Flow network. - BuildFlowPubSubScoreOption() pubsub.Option + BuildFlowPubSubScoreOption() (*pubsub.PeerScoreParams, *pubsub.PeerScoreThresholds) + // TopicScoreParams returns the topic score params for the given topic. + // If the topic score params for the given topic does not exist, it will return the default topic score params. + TopicScoreParams(*pubsub.Topic) *pubsub.TopicScoreParams } // Subscription is the abstraction of the underlying pubsub subscription that is used by the Flow network. diff --git a/network/p2p/scoring/README.md b/network/p2p/scoring/README.md index a965d324052..622ecadd3fe 100644 --- a/network/p2p/scoring/README.md +++ b/network/p2p/scoring/README.md @@ -73,6 +73,26 @@ scoreOption := NewScoreOption(config) 5. `AcceptPXThreshold`: The threshold above which a peer's score will result in accepting PX information with a prune from that peer. PX stands for "Peer Exchange" in the context of libp2p's gossipsub protocol. When a peer sends a PRUNE control message to another peer, it can include a list of other peers as PX information. The purpose of this is to help the pruned peer find new peers to replace the ones that have been pruned from its mesh. When a node receives a PRUNE message containing PX information, it can decide whether to connect to the suggested peers based on its own criteria. In this package, the `DefaultAcceptPXThreshold` is used to determine if the originating peer's penalty score is good enough to accept the PX information. If the originating peer's penalty score exceeds the threshold, the node will consider connecting to the suggested peers. 6. `OpportunisticGraftThreshold`: The threshold below which the median peer score in the mesh may result in selecting more peers with a higher score for opportunistic grafting. +### Flow Specific Scoring Parameters and Thresholds +# GossipSub Scoring Parameters Explained +1. `DefaultAppSpecificScoreWeight = 1`: This is the default weight for application-specific scoring. It basically tells us how important the application-specific score is in comparison to other scores. +2. `MaxAppSpecificPenalty = -100` and `MinAppSpecificPenalty = -1`: These values define the range for application-specific penalties. A peer can have a maximum penalty of -100 and a minimum penalty of -1. +3. `MaxAppSpecificReward = 100`: This is the maximum reward a peer can earn for good behavior. +4. `DefaultStakedIdentityReward = MaxAppSpecificReward`: This reward is given to peers that contribute positively to the network (i.e., no misbehavior). It’s to encourage them and prioritize them in neighbor selection. +5. `DefaultUnknownIdentityPenalty = MaxAppSpecificPenalty`: This penalty is given to a peer if it's not in the identity list. It's to discourage anonymity. +6. `DefaultInvalidSubscriptionPenalty = MaxAppSpecificPenalty`: This penalty is for peers that subscribe to topics they are not authorized to subscribe to. +7. `DefaultGossipThreshold = -99`: If a peer's penalty goes below this threshold, the peer is ignored for gossip. It means no gossip is sent to or received from that peer. +8. `DefaultPublishThreshold = -99`: If a peer's penalty goes below this threshold, self-published messages will not be sent to this peer. +9. `DefaultGraylistThreshold = -99`: If a peer's penalty goes below this threshold, it is graylisted. This means all incoming messages from this peer are ignored. +10. `DefaultAcceptPXThreshold = 99`: This is a threshold for accepting peers. If a peer sends information and its score is above this threshold, the information is accepted. +11. `DefaultOpportunisticGraftThreshold = MaxAppSpecificReward + 1`: This value is used to selectively connect to new peers if the median score of the current peers drops below this threshold. +12. `defaultScoreCacheSize = 1000`: Sets the default size of the cache used to store the application-specific penalty of peers. +13. `defaultDecayInterval = 1 * time.Minute`: Sets the default interval at which the score of a peer will be decayed. +14. `defaultDecayToZero = 0.01`: This is a threshold below which a decayed score is reset to zero. It prevents the score from decaying to a very small value. +15. `defaultTopicTimeInMeshQuantum` is a parameter in the GossipSub scoring system that represents a fixed time interval used to count the amount of time a peer stays in a topic mesh. It is set to 1 hour, meaning that for each hour a peer remains in a topic mesh, its time-in-mesh counter increases by 1, contributing to its availability score. This is to reward peers that stay in the mesh for longer durations and discourage those that frequently join and leave. +16. `defaultTopicInvalidMessageDeliveriesWeight` is set to -1.0 and is used to penalize peers that send invalid messages by applying it to the square of the number of such messages. A message is considered invalid if it is not properly signed. A peer will be disconnected if it sends around 14 invalid messages within a gossipsub heartbeat interval. +17. `defaultTopicInvalidMessageDeliveriesDecay` is a decay factor set to 0.99. It is used to reduce the number of invalid message deliveries counted against a peer by 1% at each heartbeat interval. This prevents the peer from being disconnected if it stops sending invalid messages. The heartbeat interval in the gossipsub scoring system is set to 1 minute by default. + ## Customization The scoring mechanism can be easily customized to suit the needs of the Flow network. This includes changing the scoring parameters, thresholds, and the scoring function itself. You can customize the scoring parameters and thresholds by using the various setter methods provided in the `ScoreOptionConfig` object. Additionally, you can provide a custom app-specific scoring function through the `SetAppSpecificScoreFunction` method. diff --git a/network/p2p/scoring/registry.go b/network/p2p/scoring/registry.go index 15c67d55b33..9009b86f41a 100644 --- a/network/p2p/scoring/registry.go +++ b/network/p2p/scoring/registry.go @@ -11,6 +11,7 @@ import ( "github.com/onflow/flow-go/module" "github.com/onflow/flow-go/network/p2p" netcache "github.com/onflow/flow-go/network/p2p/cache" + p2pmsg "github.com/onflow/flow-go/network/p2p/message" "github.com/onflow/flow-go/utils/logging" ) @@ -253,13 +254,13 @@ func (r *GossipSubAppSpecificScoreRegistry) OnInvalidControlMessageNotification( record, err := r.spamScoreCache.Update(notification.PeerID, func(record p2p.GossipSubSpamRecord) p2p.GossipSubSpamRecord { switch notification.MsgType { - case p2p.CtrlMsgGraft: + case p2pmsg.CtrlMsgGraft: record.Penalty += r.penalty.Graft - case p2p.CtrlMsgPrune: + case p2pmsg.CtrlMsgPrune: record.Penalty += r.penalty.Prune - case p2p.CtrlMsgIHave: + case p2pmsg.CtrlMsgIHave: record.Penalty += r.penalty.IHave - case p2p.CtrlMsgIWant: + case p2pmsg.CtrlMsgIWant: record.Penalty += r.penalty.IWant default: // the error is considered fatal as it means that we have an unsupported misbehaviour type, we should crash the node to prevent routing attack vulnerability. diff --git a/network/p2p/scoring/registry_test.go b/network/p2p/scoring/registry_test.go index 186ce7bf6bc..843ec2d87ae 100644 --- a/network/p2p/scoring/registry_test.go +++ b/network/p2p/scoring/registry_test.go @@ -15,6 +15,7 @@ import ( "github.com/onflow/flow-go/module/mock" "github.com/onflow/flow-go/network/p2p" netcache "github.com/onflow/flow-go/network/p2p/cache" + p2pmsg "github.com/onflow/flow-go/network/p2p/message" mockp2p "github.com/onflow/flow-go/network/p2p/mock" "github.com/onflow/flow-go/network/p2p/scoring" "github.com/onflow/flow-go/utils/unittest" @@ -48,20 +49,20 @@ func TestNoPenaltyRecord(t *testing.T) { // penalty value as the app specific score. func TestPeerWithSpamRecord(t *testing.T) { t.Run("graft", func(t *testing.T) { - testPeerWithSpamRecord(t, p2p.CtrlMsgGraft, penaltyValueFixtures().Graft) + testPeerWithSpamRecord(t, p2pmsg.CtrlMsgGraft, penaltyValueFixtures().Graft) }) t.Run("prune", func(t *testing.T) { - testPeerWithSpamRecord(t, p2p.CtrlMsgPrune, penaltyValueFixtures().Prune) + testPeerWithSpamRecord(t, p2pmsg.CtrlMsgPrune, penaltyValueFixtures().Prune) }) t.Run("ihave", func(t *testing.T) { - testPeerWithSpamRecord(t, p2p.CtrlMsgIHave, penaltyValueFixtures().IHave) + testPeerWithSpamRecord(t, p2pmsg.CtrlMsgIHave, penaltyValueFixtures().IHave) }) t.Run("iwant", func(t *testing.T) { - testPeerWithSpamRecord(t, p2p.CtrlMsgIWant, penaltyValueFixtures().IWant) + testPeerWithSpamRecord(t, p2pmsg.CtrlMsgIWant, penaltyValueFixtures().IWant) }) } -func testPeerWithSpamRecord(t *testing.T, messageType p2p.ControlMessageType, expectedPenalty float64) { +func testPeerWithSpamRecord(t *testing.T, messageType p2pmsg.ControlMessageType, expectedPenalty float64) { peerID := peer.ID("peer-1") reg, spamRecords := newGossipSubAppSpecificScoreRegistry( t, @@ -98,22 +99,22 @@ func testPeerWithSpamRecord(t *testing.T, messageType p2p.ControlMessageType, ex func TestSpamRecord_With_UnknownIdentity(t *testing.T) { t.Run("graft", func(t *testing.T) { - testSpamRecordWithUnknownIdentity(t, p2p.CtrlMsgGraft, penaltyValueFixtures().Graft) + testSpamRecordWithUnknownIdentity(t, p2pmsg.CtrlMsgGraft, penaltyValueFixtures().Graft) }) t.Run("prune", func(t *testing.T) { - testSpamRecordWithUnknownIdentity(t, p2p.CtrlMsgPrune, penaltyValueFixtures().Prune) + testSpamRecordWithUnknownIdentity(t, p2pmsg.CtrlMsgPrune, penaltyValueFixtures().Prune) }) t.Run("ihave", func(t *testing.T) { - testSpamRecordWithUnknownIdentity(t, p2p.CtrlMsgIHave, penaltyValueFixtures().IHave) + testSpamRecordWithUnknownIdentity(t, p2pmsg.CtrlMsgIHave, penaltyValueFixtures().IHave) }) t.Run("iwant", func(t *testing.T) { - testSpamRecordWithUnknownIdentity(t, p2p.CtrlMsgIWant, penaltyValueFixtures().IWant) + testSpamRecordWithUnknownIdentity(t, p2pmsg.CtrlMsgIWant, penaltyValueFixtures().IWant) }) } // testSpamRecordWithUnknownIdentity tests the app specific penalty computation of the node when there is a spam record for the peer id and // the peer id has an unknown identity. -func testSpamRecordWithUnknownIdentity(t *testing.T, messageType p2p.ControlMessageType, expectedPenalty float64) { +func testSpamRecordWithUnknownIdentity(t *testing.T, messageType p2pmsg.ControlMessageType, expectedPenalty float64) { peerID := peer.ID("peer-1") reg, spamRecords := newGossipSubAppSpecificScoreRegistry( t, @@ -149,22 +150,22 @@ func testSpamRecordWithUnknownIdentity(t *testing.T, messageType p2p.ControlMess func TestSpamRecord_With_SubscriptionPenalty(t *testing.T) { t.Run("graft", func(t *testing.T) { - testSpamRecordWithSubscriptionPenalty(t, p2p.CtrlMsgGraft, penaltyValueFixtures().Graft) + testSpamRecordWithSubscriptionPenalty(t, p2pmsg.CtrlMsgGraft, penaltyValueFixtures().Graft) }) t.Run("prune", func(t *testing.T) { - testSpamRecordWithSubscriptionPenalty(t, p2p.CtrlMsgPrune, penaltyValueFixtures().Prune) + testSpamRecordWithSubscriptionPenalty(t, p2pmsg.CtrlMsgPrune, penaltyValueFixtures().Prune) }) t.Run("ihave", func(t *testing.T) { - testSpamRecordWithSubscriptionPenalty(t, p2p.CtrlMsgIHave, penaltyValueFixtures().IHave) + testSpamRecordWithSubscriptionPenalty(t, p2pmsg.CtrlMsgIHave, penaltyValueFixtures().IHave) }) t.Run("iwant", func(t *testing.T) { - testSpamRecordWithSubscriptionPenalty(t, p2p.CtrlMsgIWant, penaltyValueFixtures().IWant) + testSpamRecordWithSubscriptionPenalty(t, p2pmsg.CtrlMsgIWant, penaltyValueFixtures().IWant) }) } // testSpamRecordWithUnknownIdentity tests the app specific penalty computation of the node when there is a spam record for the peer id and // the peer id has an invalid subscription as well. -func testSpamRecordWithSubscriptionPenalty(t *testing.T, messageType p2p.ControlMessageType, expectedPenalty float64) { +func testSpamRecordWithSubscriptionPenalty(t *testing.T, messageType p2pmsg.ControlMessageType, expectedPenalty float64) { peerID := peer.ID("peer-1") reg, spamRecords := newGossipSubAppSpecificScoreRegistry( t, @@ -208,7 +209,7 @@ func TestSpamPenaltyDecaysInCache(t *testing.T) { // report a misbehavior for the peer id. reg.OnInvalidControlMessageNotification(&p2p.InvCtrlMsgNotif{ PeerID: peerID, - MsgType: p2p.CtrlMsgPrune, + MsgType: p2pmsg.CtrlMsgPrune, Count: 1, }) @@ -216,7 +217,7 @@ func TestSpamPenaltyDecaysInCache(t *testing.T) { reg.OnInvalidControlMessageNotification(&p2p.InvCtrlMsgNotif{ PeerID: peerID, - MsgType: p2p.CtrlMsgGraft, + MsgType: p2pmsg.CtrlMsgGraft, Count: 1, }) @@ -224,7 +225,7 @@ func TestSpamPenaltyDecaysInCache(t *testing.T) { reg.OnInvalidControlMessageNotification(&p2p.InvCtrlMsgNotif{ PeerID: peerID, - MsgType: p2p.CtrlMsgIHave, + MsgType: p2pmsg.CtrlMsgIHave, Count: 1, }) @@ -232,7 +233,7 @@ func TestSpamPenaltyDecaysInCache(t *testing.T) { reg.OnInvalidControlMessageNotification(&p2p.InvCtrlMsgNotif{ PeerID: peerID, - MsgType: p2p.CtrlMsgIWant, + MsgType: p2pmsg.CtrlMsgIWant, Count: 1, }) @@ -274,7 +275,7 @@ func TestSpamPenaltyDecayToZero(t *testing.T) { // report a misbehavior for the peer id. reg.OnInvalidControlMessageNotification(&p2p.InvCtrlMsgNotif{ PeerID: peerID, - MsgType: p2p.CtrlMsgGraft, + MsgType: p2pmsg.CtrlMsgGraft, Count: 1, }) @@ -324,7 +325,7 @@ func TestPersistingUnknownIdentityPenalty(t *testing.T) { // report a misbehavior for the peer id. reg.OnInvalidControlMessageNotification(&p2p.InvCtrlMsgNotif{ PeerID: peerID, - MsgType: p2p.CtrlMsgGraft, + MsgType: p2pmsg.CtrlMsgGraft, Count: 1, }) @@ -377,7 +378,7 @@ func TestPersistingInvalidSubscriptionPenalty(t *testing.T) { // report a misbehavior for the peer id. reg.OnInvalidControlMessageNotification(&p2p.InvCtrlMsgNotif{ PeerID: peerID, - MsgType: p2p.CtrlMsgGraft, + MsgType: p2pmsg.CtrlMsgGraft, Count: 1, }) diff --git a/network/p2p/scoring/score_option.go b/network/p2p/scoring/score_option.go index c6bf52a21be..c743b3efa33 100644 --- a/network/p2p/scoring/score_option.go +++ b/network/p2p/scoring/score_option.go @@ -12,6 +12,7 @@ import ( "github.com/onflow/flow-go/network/channels" "github.com/onflow/flow-go/network/p2p" netcache "github.com/onflow/flow-go/network/p2p/cache" + "github.com/onflow/flow-go/network/p2p/utils" "github.com/onflow/flow-go/utils/logging" ) @@ -93,9 +94,9 @@ const ( defaultScoreCacheSize = 1000 // defaultDecayInterval is the default decay interval for the overall score of a peer at the GossipSub scoring - // system. It is the interval over which we decay the effect of past behavior. So that the effect of past behavior - // is not permanent. - defaultDecayInterval = 1 * time.Hour + // system. We set it to 1 minute so that it is not too short so that a malicious node can recover from a penalty + // and is not too long so that a well-behaved node can't recover from a penalty. + defaultDecayInterval = 1 * time.Minute // defaultDecayToZero is the default decay to zero for the overall score of a peer at the GossipSub scoring system. // It defines the maximum value below which a peer scoring counter is reset to zero. @@ -104,6 +105,43 @@ const ( // When a counter hits the DecayToZero threshold, it means that the peer did not exhibit the behavior // for a long time, and we can reset the counter. defaultDecayToZero = 0.01 + + // defaultTopicSkipAtomicValidation is the default value for the skip atomic validation flag for topics. + // We set it to true, which means gossipsub parameter validation will not fail if we leave some of the + // topic parameters at their default values, i.e., zero. This is because we are not setting all + // topic parameters at the current implementation. + defaultTopicSkipAtomicValidation = true + + // defaultTopicInvalidMessageDeliveriesWeight this value is applied to the square of the number of invalid message deliveries on a topic. + // It is used to penalize peers that send invalid messages. By an invalid message, we mean a message that is not signed by the + // publisher, or a message that is not signed by the peer that sent it. We set it to -1.0, which means that with around 14 invalid + // message deliveries within a gossipsub heartbeat interval, the peer will be disconnected. + // The supporting math is as follows: + // - each staked (i.e., authorized) peer is rewarded by the fixed reward of 100 (i.e., DefaultStakedIdentityReward). + // - x invalid message deliveries will result in a penalty of x^2 * DefaultTopicInvalidMessageDeliveriesWeight, i.e., -x^2. + // - the peer will be disconnected when its penalty reaches -100 (i.e., MaxAppSpecificPenalty). + // - so, the maximum number of invalid message deliveries that a peer can have before being disconnected is sqrt(200/DefaultTopicInvalidMessageDeliveriesWeight) ~ 14. + defaultTopicInvalidMessageDeliveriesWeight = -1.0 + + // defaultTopicInvalidMessageDeliveriesDecay decay factor used to decay the number of invalid message deliveries. + // The total number of invalid message deliveries is multiplied by this factor at each heartbeat interval to + // decay the number of invalid message deliveries, and prevent the peer from being disconnected if it stops + // sending invalid messages. We set it to 0.99, which means that the number of invalid message deliveries will + // decay by 1% at each heartbeat interval. + // The decay heartbeats are defined by the heartbeat interval of the gossipsub scoring system, which is 1 Minute (defaultDecayInterval). + defaultTopicInvalidMessageDeliveriesDecay = .99 + + // defaultTopicTimeInMeshQuantum is the default time in mesh quantum for the GossipSub scoring system. It is used to gauge + // a discrete time interval for the time in mesh counter. We set it to 1 hour, which means that every one complete hour a peer is + // in a topic mesh, the time in mesh counter will be incremented by 1 and is counted towards the availability score of the peer in that topic mesh. + // The reason of setting it to 1 hour is that we want to reward peers that are in a topic mesh for a long time, and we want to avoid rewarding peers that + // are churners, i.e., peers that join and leave a topic mesh frequently. + defaultTopicTimeInMesh = time.Hour + + // defaultTopicWeight is the default weight of a topic in the GossipSub scoring system. The overall score of a peer in a topic mesh is + // multiplied by the weight of the topic when calculating the overall score of the peer. + // We set it to 1.0, which means that the overall score of a peer in a topic mesh is not affected by the weight of the topic. + defaultTopicWeight = 1.0 ) // ScoreOption is a functional option for configuring the peer scoring system. @@ -126,23 +164,16 @@ type ScoreOptionConfig struct { registerNotificationConsumerFunc func(p2p.GossipSubInvCtrlMsgNotifConsumer) } -func NewScoreOptionConfig(logger zerolog.Logger) *ScoreOptionConfig { +func NewScoreOptionConfig(logger zerolog.Logger, idProvider module.IdentityProvider) *ScoreOptionConfig { return &ScoreOptionConfig{ logger: logger, + provider: idProvider, cacheSize: defaultScoreCacheSize, cacheMetrics: metrics.NewNoopCollector(), // no metrics by default topicParams: make([]func(map[string]*pubsub.TopicScoreParams), 0), } } -// SetProvider sets the identity provider for the penalty option. -// It is used to retrieve the identity of a peer when calculating the app specific penalty. -// If the provider is not set, the penalty registry will crash. This is a required field. -// It is safe to call this method multiple times, the last call will be used. -func (c *ScoreOptionConfig) SetProvider(provider module.IdentityProvider) { - c.provider = provider -} - // SetCacheSize sets the size of the cache used to store the app specific penalty of peers. // If the cache size is not set, the default value will be used. // It is safe to call this method multiple times, the last call will be used. @@ -167,10 +198,10 @@ func (c *ScoreOptionConfig) SetAppSpecificScoreFunction(appSpecificScoreFunction c.appScoreFunc = appSpecificScoreFunction } -// SetTopicScoreParams adds the topic penalty parameters to the peer penalty parameters. -// It is used to configure the topic penalty parameters for the pubsub system. -// If there is already a topic penalty parameter for the given topic, the last call will be used. -func (c *ScoreOptionConfig) SetTopicScoreParams(topic channels.Topic, topicScoreParams *pubsub.TopicScoreParams) { +// OverrideTopicScoreParams overrides the topic score parameters for the given topic. +// It is used to override the default topic score parameters for a specific topic. +// If the topic score parameters are not set, the default ones will be used. +func (c *ScoreOptionConfig) OverrideTopicScoreParams(topic channels.Topic, topicScoreParams *pubsub.TopicScoreParams) { c.topicParams = append(c.topicParams, func(topics map[string]*pubsub.TopicScoreParams) { topics[topic.String()] = topicScoreParams }) @@ -229,7 +260,6 @@ func NewScoreOption(cfg *ScoreOptionConfig) *ScoreOption { for _, topicParams := range cfg.topicParams { topicParams(s.peerScoreParams.Topics) } - return s } @@ -237,7 +267,7 @@ func (s *ScoreOption) SetSubscriptionProvider(provider *SubscriptionProvider) er return s.validator.RegisterSubscriptionProvider(provider) } -func (s *ScoreOption) BuildFlowPubSubScoreOption() pubsub.Option { +func (s *ScoreOption) BuildFlowPubSubScoreOption() (*pubsub.PeerScoreParams, *pubsub.PeerScoreThresholds) { s.preparePeerScoreThresholds() s.logger.Info(). @@ -246,12 +276,15 @@ func (s *ScoreOption) BuildFlowPubSubScoreOption() pubsub.Option { Float64("graylist_threshold", s.peerThresholdParams.GraylistThreshold). Float64("accept_px_threshold", s.peerThresholdParams.AcceptPXThreshold). Float64("opportunistic_graft_threshold", s.peerThresholdParams.OpportunisticGraftThreshold). - Msg("peer penalty thresholds configured") + Msg("pubsub score thresholds are set") - return pubsub.WithPeerScore( - s.peerScoreParams, - s.peerThresholdParams, - ) + for topic, topicParams := range s.peerScoreParams.Topics { + topicScoreParamLogger := utils.TopicScoreParamsLogger(s.logger, topic, topicParams) + topicScoreParamLogger.Info(). + Msg("pubsub score topic parameters are set for topic") + } + + return s.peerScoreParams, s.peerThresholdParams } func (s *ScoreOption) preparePeerScoreThresholds() { @@ -264,6 +297,22 @@ func (s *ScoreOption) preparePeerScoreThresholds() { } } +// TopicScoreParams returns the topic score parameters for the given topic. If the topic +// score parameters are not set, it returns the default topic score parameters. +// The custom topic parameters are set at the initialization of the score option. +// Args: +// - topic: the topic for which the score parameters are requested. +// Returns: +// - the topic score parameters for the given topic, or the default topic score parameters if +// the topic score parameters are not set. +func (s *ScoreOption) TopicScoreParams(topic *pubsub.Topic) *pubsub.TopicScoreParams { + params, exists := s.peerScoreParams.Topics[topic.String()] + if !exists { + return defaultTopicScoreParams() + } + return params +} + func defaultPeerScoreParams() *pubsub.PeerScoreParams { return &pubsub.PeerScoreParams{ Topics: make(map[string]*pubsub.TopicScoreParams), @@ -283,19 +332,13 @@ func defaultPeerScoreParams() *pubsub.PeerScoreParams { } } -func (s *ScoreOption) BuildGossipSubScoreOption() pubsub.Option { - s.preparePeerScoreThresholds() - - s.logger.Info(). - Float64("gossip_threshold", s.peerThresholdParams.GossipThreshold). - Float64("publish_threshold", s.peerThresholdParams.PublishThreshold). - Float64("graylist_threshold", s.peerThresholdParams.GraylistThreshold). - Float64("accept_px_threshold", s.peerThresholdParams.AcceptPXThreshold). - Float64("opportunistic_graft_threshold", s.peerThresholdParams.OpportunisticGraftThreshold). - Msg("peer penalty thresholds configured") - - return pubsub.WithPeerScore( - s.peerScoreParams, - s.peerThresholdParams, - ) +// defaultTopicScoreParams returns the default score params for topics. +func defaultTopicScoreParams() *pubsub.TopicScoreParams { + return &pubsub.TopicScoreParams{ + TopicWeight: defaultTopicWeight, + SkipAtomicValidation: defaultTopicSkipAtomicValidation, + InvalidMessageDeliveriesWeight: defaultTopicInvalidMessageDeliveriesWeight, + InvalidMessageDeliveriesDecay: defaultTopicInvalidMessageDeliveriesDecay, + TimeInMeshQuantum: defaultTopicTimeInMesh, + } } diff --git a/network/p2p/scoring/scoring_test.go b/network/p2p/scoring/scoring_test.go index de5baf0420a..db43c59a055 100644 --- a/network/p2p/scoring/scoring_test.go +++ b/network/p2p/scoring/scoring_test.go @@ -9,16 +9,22 @@ import ( pubsub "github.com/libp2p/go-libp2p-pubsub" "github.com/libp2p/go-libp2p/core/peer" + "github.com/rs/zerolog" mocktestify "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module" "github.com/onflow/flow-go/module/component" "github.com/onflow/flow-go/module/id" "github.com/onflow/flow-go/module/irrecoverable" + "github.com/onflow/flow-go/module/metrics" "github.com/onflow/flow-go/module/mock" + flownet "github.com/onflow/flow-go/network" "github.com/onflow/flow-go/network/channels" "github.com/onflow/flow-go/network/p2p" + p2pmsg "github.com/onflow/flow-go/network/p2p/message" + "github.com/onflow/flow-go/network/p2p/p2pconf" p2ptest "github.com/onflow/flow-go/network/p2p/test" "github.com/onflow/flow-go/utils/unittest" ) @@ -34,6 +40,14 @@ type mockInspectorSuite struct { // ensures that mockInspectorSuite implements the GossipSubInspectorSuite interface. var _ p2p.GossipSubInspectorSuite = (*mockInspectorSuite)(nil) +func (m *mockInspectorSuite) AddInvalidControlMessageConsumer(consumer p2p.GossipSubInvCtrlMsgNotifConsumer) { + require.Nil(m.t, m.consumer) + m.consumer = consumer +} +func (m *mockInspectorSuite) ActiveClustersChanged(_ flow.ChainIDList) { + // no-op +} + // newMockInspectorSuite creates a new mockInspectorSuite. // Args: // - t: the test object used for assertions. @@ -60,22 +74,6 @@ func (m *mockInspectorSuite) InspectFunc() func(peer.ID, *pubsub.RPC) error { return nil } -// AddInvCtrlMsgNotifConsumer adds a consumer for invalid control message notifications. -// In this mock implementation, the consumer is stored in the mockInspectorSuite, and is used to simulate the reception of invalid control messages. -// Args: -// - c: the consumer to add. -// Returns: -// - nil. -// Note: this function will fail the test if the consumer is already set. -func (m *mockInspectorSuite) AddInvCtrlMsgNotifConsumer(c p2p.GossipSubInvCtrlMsgNotifConsumer) { - require.Nil(m.t, m.consumer) - m.consumer = c -} - -func (m *mockInspectorSuite) Inspectors() []p2p.GossipSubRPCInspector { - return []p2p.GossipSubRPCInspector{} -} - // TestInvalidCtrlMsgScoringIntegration tests the impact of invalid control messages on the scoring and connectivity of nodes in a network. // It creates a network of 2 nodes, and sends a set of control messages with invalid topic IDs to one of the nodes. // It then checks that the node receiving the invalid control messages decreases its score for the peer spamming the invalid messages, and @@ -95,7 +93,16 @@ func TestInvalidCtrlMsgScoringIntegration(t *testing.T) { idProvider, p2ptest.WithRole(flow.RoleConsensus), p2ptest.WithPeerScoringEnabled(idProvider), - p2ptest.WithGossipSubRpcInspectorSuite(inspectorSuite1)) + p2ptest.OverrideGossipSubRpcInspectorSuiteFactory(func(zerolog.Logger, + flow.Identifier, + *p2pconf.GossipSubRPCInspectorsConfig, + module.GossipSubMetrics, + metrics.HeroCacheMetricsFactory, + flownet.NetworkingType, + module.IdentityProvider) (p2p.GossipSubInspectorSuite, error) { + // override the gossipsub rpc inspector suite factory to return the mock inspector suite + return inspectorSuite1, nil + })) node2, id2 := p2ptest.NodeFixture( t, @@ -132,7 +139,7 @@ func TestInvalidCtrlMsgScoringIntegration(t *testing.T) { for i := 0; i < 30; i++ { inspectorSuite1.consumer.OnInvalidControlMessageNotification(&p2p.InvCtrlMsgNotif{ PeerID: node2.Host().ID(), - MsgType: p2p.ControlMessageTypes()[rand.Intn(len(p2p.ControlMessageTypes()))], + MsgType: p2pmsg.ControlMessageTypes()[rand.Intn(len(p2pmsg.ControlMessageTypes()))], Count: 1, Err: fmt.Errorf("invalid control message"), }) diff --git a/network/p2p/test/fixtures.go b/network/p2p/test/fixtures.go index 32ff45a2e6f..5ee82dd4ab5 100644 --- a/network/p2p/test/fixtures.go +++ b/network/p2p/test/fixtures.go @@ -4,6 +4,7 @@ import ( "bufio" "context" "crypto/rand" + crand "math/rand" "testing" "time" @@ -14,6 +15,7 @@ import ( "github.com/libp2p/go-libp2p/core/peer" "github.com/libp2p/go-libp2p/core/protocol" "github.com/libp2p/go-libp2p/core/routing" + discoveryBackoff "github.com/libp2p/go-libp2p/p2p/discovery/backoff" mh "github.com/multiformats/go-multihash" "github.com/rs/zerolog" "github.com/stretchr/testify/require" @@ -24,14 +26,15 @@ import ( "github.com/onflow/flow-go/module" "github.com/onflow/flow-go/module/irrecoverable" "github.com/onflow/flow-go/module/metrics" + flownet "github.com/onflow/flow-go/network" "github.com/onflow/flow-go/network/channels" "github.com/onflow/flow-go/network/internal/p2pfixtures" - "github.com/onflow/flow-go/network/internal/testutils" "github.com/onflow/flow-go/network/p2p" "github.com/onflow/flow-go/network/p2p/connection" p2pdht "github.com/onflow/flow-go/network/p2p/dht" "github.com/onflow/flow-go/network/p2p/p2pbuilder" - inspectorbuilder "github.com/onflow/flow-go/network/p2p/p2pbuilder/inspector" + p2pconfig "github.com/onflow/flow-go/network/p2p/p2pbuilder/config" + "github.com/onflow/flow-go/network/p2p/p2pconf" "github.com/onflow/flow-go/network/p2p/unicast" "github.com/onflow/flow-go/network/p2p/unicast/protocols" "github.com/onflow/flow-go/network/p2p/utils" @@ -61,23 +64,30 @@ func NodeFixture( require.NoError(t, err) logger := unittest.Logger().Level(zerolog.WarnLevel) - - rpcInspectorSuite, err := inspectorbuilder.NewGossipSubInspectorBuilder(logger, sporkID, &defaultFlowConfig.NetworkConfig.GossipSubConfig.GossipSubRPCInspectorsConfig, idProvider, metrics.NewNoopCollector()). - Build() - require.NoError(t, err) - + require.NotNil(t, idProvider) + connectionGater := NewConnectionGater(idProvider, func(p peer.ID) error { + return nil + }) + require.NotNil(t, connectionGater) parameters := &NodeFixtureParameters{ - HandlerFunc: func(network.Stream) {}, - Unicasts: nil, - Key: NetworkingKeyFixtures(t), - Address: unittest.DefaultAddress, - Logger: logger, - Role: flow.RoleCollection, - CreateStreamRetryDelay: unicast.DefaultRetryDelay, - Metrics: metrics.NewNoopCollector(), - ResourceManager: testutils.NewResourceManager(t), + NetworkingType: flownet.PrivateNetwork, + HandlerFunc: func(network.Stream) {}, + Unicasts: nil, + Key: NetworkingKeyFixtures(t), + Address: unittest.DefaultAddress, + Logger: logger, + Role: flow.RoleCollection, + CreateStreamRetryDelay: unicast.DefaultRetryDelay, + IdProvider: idProvider, + MetricsCfg: &p2pconfig.MetricsConfig{ + HeroCacheFactory: metrics.NewNoopHeroCacheMetricsFactory(), + Metrics: metrics.NewNoopCollector(), + }, + ResourceManager: &network.NullResourceManager{}, GossipSubPeerScoreTracerInterval: 0, // disabled by default - GossipSubRPCInspector: rpcInspectorSuite, + ConnGater: connectionGater, + PeerManagerConfig: PeerManagerConfigFixture(), // disabled by default + GossipSubRPCInspectorCfg: &defaultFlowConfig.NetworkConfig.GossipSubRPCInspectorsConfig, } for _, opt := range opts { @@ -91,16 +101,20 @@ func NodeFixture( logger = parameters.Logger.With().Hex("node_id", logging.ID(identity.NodeID)).Logger() - connManager, err := connection.NewConnManager(logger, parameters.Metrics, &defaultFlowConfig.NetworkConfig.ConnectionManagerConfig) + connManager, err := connection.NewConnManager(logger, parameters.MetricsCfg.Metrics, &defaultFlowConfig.NetworkConfig.ConnectionManagerConfig) require.NoError(t, err) builder := p2pbuilder.NewNodeBuilder( logger, - parameters.Metrics, + parameters.MetricsCfg, + parameters.NetworkingType, parameters.Address, parameters.Key, sporkID, + parameters.IdProvider, &defaultFlowConfig.NetworkConfig.ResourceManagerConfig, + parameters.GossipSubRPCInspectorCfg, + parameters.PeerManagerConfig, &p2p.DisallowListCacheConfig{ MaxSize: uint32(1000), Metrics: metrics.NewNoopCollector(), @@ -110,14 +124,17 @@ func NodeFixture( return p2pdht.NewDHT(c, h, protocol.ID(protocols.FlowDHTProtocolIDPrefix+sporkID.String()+"/"+dhtPrefix), logger, - parameters.Metrics, + parameters.MetricsCfg.Metrics, parameters.DhtOptions..., ) }). SetCreateNode(p2pbuilder.DefaultCreateNodeFunc). SetStreamCreationRetryInterval(parameters.CreateStreamRetryDelay). - SetResourceManager(parameters.ResourceManager). - SetGossipSubRpcInspectorSuite(parameters.GossipSubRPCInspector) + SetResourceManager(parameters.ResourceManager) + + if parameters.GossipSubRpcInspectorSuiteFactory != nil { + builder.OverrideDefaultRpcInspectorSuiteFactory(parameters.GossipSubRpcInspectorSuiteFactory) + } if parameters.ResourceManager != nil { builder.SetResourceManager(parameters.ResourceManager) @@ -128,12 +145,7 @@ func NodeFixture( } if parameters.PeerScoringEnabled { - builder.EnableGossipSubPeerScoring(parameters.IdProvider, parameters.PeerScoreConfig) - } - - if parameters.UpdateInterval != 0 { - require.NotNil(t, parameters.PeerProvider) - builder.SetPeerManagerOptions(parameters.ConnectionPruning, parameters.UpdateInterval) + builder.EnableGossipSubPeerScoring(parameters.PeerScoreConfig) } if parameters.GossipSubFactory != nil && parameters.GossipSubConfig != nil { @@ -148,13 +160,19 @@ func NodeFixture( builder.SetGossipSubTracer(parameters.PubSubTracer) } + if parameters.UnicastRateLimitDistributor != nil { + builder.SetRateLimiterDistributor(parameters.UnicastRateLimitDistributor) + } + builder.SetGossipSubScoreTracerInterval(parameters.GossipSubPeerScoreTracerInterval) n, err := builder.Build() require.NoError(t, err) - err = n.WithDefaultUnicastProtocol(parameters.HandlerFunc, parameters.Unicasts) - require.NoError(t, err) + if parameters.HandlerFunc != nil { + err = n.WithDefaultUnicastProtocol(parameters.HandlerFunc, parameters.Unicasts) + require.NoError(t, err) + } // get the actual IP and port that have been assigned by the subsystem ip, port, err := n.GetIPPort() @@ -171,34 +189,48 @@ func NodeFixture( type NodeFixtureParameterOption func(*NodeFixtureParameters) type NodeFixtureParameters struct { - HandlerFunc network.StreamHandler - Unicasts []protocols.ProtocolName - Key crypto.PrivateKey - Address string - DhtOptions []dht.Option - Role flow.Role - Logger zerolog.Logger - PeerScoringEnabled bool - IdProvider module.IdentityProvider - PeerScoreConfig *p2p.PeerScoringConfig - ConnectionPruning bool // peer manager parameter - UpdateInterval time.Duration // peer manager parameter - PeerProvider p2p.PeersProvider // peer manager parameter - ConnGater p2p.ConnectionGater - ConnManager connmgr.ConnManager - GossipSubFactory p2p.GossipSubFactoryFunc - GossipSubConfig p2p.GossipSubAdapterConfigFunc - Metrics module.LibP2PMetrics - ResourceManager network.ResourceManager - PubSubTracer p2p.PubSubTracer - GossipSubPeerScoreTracerInterval time.Duration // intervals at which the peer score is updated and logged. - CreateStreamRetryDelay time.Duration - GossipSubRPCInspector p2p.GossipSubInspectorSuite -} - -func WithGossipSubRpcInspectorSuite(inspectorSuite p2p.GossipSubInspectorSuite) NodeFixtureParameterOption { + HandlerFunc network.StreamHandler + NetworkingType flownet.NetworkingType + Unicasts []protocols.ProtocolName + Key crypto.PrivateKey + Address string + DhtOptions []dht.Option + Role flow.Role + Logger zerolog.Logger + PeerScoringEnabled bool + IdProvider module.IdentityProvider + PeerScoreConfig *p2p.PeerScoringConfig + PeerManagerConfig *p2pconfig.PeerManagerConfig + PeerProvider p2p.PeersProvider // peer manager parameter + ConnGater p2p.ConnectionGater + ConnManager connmgr.ConnManager + GossipSubFactory p2p.GossipSubFactoryFunc + GossipSubConfig p2p.GossipSubAdapterConfigFunc + MetricsCfg *p2pconfig.MetricsConfig + ResourceManager network.ResourceManager + PubSubTracer p2p.PubSubTracer + GossipSubPeerScoreTracerInterval time.Duration // intervals at which the peer score is updated and logged. + CreateStreamRetryDelay time.Duration + UnicastRateLimitDistributor p2p.UnicastRateLimiterDistributor + GossipSubRpcInspectorSuiteFactory p2p.GossipSubRpcInspectorSuiteFactoryFunc + GossipSubRPCInspectorCfg *p2pconf.GossipSubRPCInspectorsConfig +} + +func WithUnicastRateLimitDistributor(distributor p2p.UnicastRateLimiterDistributor) NodeFixtureParameterOption { return func(p *NodeFixtureParameters) { - p.GossipSubRPCInspector = inspectorSuite + p.UnicastRateLimitDistributor = distributor + } +} + +func OverrideGossipSubRpcInspectorSuiteFactory(factory p2p.GossipSubRpcInspectorSuiteFactoryFunc) NodeFixtureParameterOption { + return func(p *NodeFixtureParameters) { + p.GossipSubRpcInspectorSuiteFactory = factory + } +} + +func OverrideGossipSubRpcInspectorConfig(cfg *p2pconf.GossipSubRPCInspectorsConfig) NodeFixtureParameterOption { + return func(p *NodeFixtureParameters) { + p.GossipSubRPCInspectorCfg = cfg } } @@ -227,10 +259,9 @@ func WithDefaultStreamHandler(handler network.StreamHandler) NodeFixtureParamete } } -func WithPeerManagerEnabled(connectionPruning bool, updateInterval time.Duration, peerProvider p2p.PeersProvider) NodeFixtureParameterOption { +func WithPeerManagerEnabled(cfg *p2pconfig.PeerManagerConfig, peerProvider p2p.PeersProvider) NodeFixtureParameterOption { return func(p *NodeFixtureParameters) { - p.ConnectionPruning = connectionPruning - p.UpdateInterval = updateInterval + p.PeerManagerConfig = cfg p.PeerProvider = peerProvider } } @@ -291,7 +322,7 @@ func WithLogger(logger zerolog.Logger) NodeFixtureParameterOption { func WithMetricsCollector(metrics module.NetworkMetrics) NodeFixtureParameterOption { return func(p *NodeFixtureParameters) { - p.Metrics = metrics + p.MetricsCfg.Metrics = metrics } } @@ -309,6 +340,50 @@ func WithDefaultResourceManager() NodeFixtureParameterOption { } } +func WithUnicastHandlerFunc(handler network.StreamHandler) NodeFixtureParameterOption { + return func(p *NodeFixtureParameters) { + p.HandlerFunc = handler + } +} + +// PeerManagerConfigFixture is a test fixture that sets the default config for the peer manager. +func PeerManagerConfigFixture(opts ...func(*p2pconfig.PeerManagerConfig)) *p2pconfig.PeerManagerConfig { + cfg := &p2pconfig.PeerManagerConfig{ + ConnectionPruning: true, + UpdateInterval: 1 * time.Second, + ConnectorFactory: connection.DefaultLibp2pBackoffConnectorFactory(), + } + for _, opt := range opts { + opt(cfg) + } + return cfg +} + +// WithZeroJitterAndZeroBackoff is a test fixture that sets the default config for the peer manager. +// It uses a backoff connector with zero jitter and zero backoff. +func WithZeroJitterAndZeroBackoff(t *testing.T) func(*p2pconfig.PeerManagerConfig) { + return func(cfg *p2pconfig.PeerManagerConfig) { + cfg.ConnectorFactory = func(host host.Host) (p2p.Connector, error) { + cacheSize := 100 + dialTimeout := time.Minute * 2 + backoff := discoveryBackoff.NewExponentialBackoff( + 1*time.Second, + 1*time.Hour, + func(_, _, _ time.Duration, _ *crand.Rand) time.Duration { + return 0 // no jitter + }, + time.Second, + 1, + 0, + crand.NewSource(crand.Int63()), + ) + backoffConnector, err := discoveryBackoff.NewBackoffConnector(host, cacheSize, dialTimeout, backoff) + require.NoError(t, err) + return backoffConnector, nil + } + } +} + // NodesFixture is a test fixture that creates a number of libp2p nodes with the given callback function for stream handling. // It returns the nodes and their identities. func NodesFixture(t *testing.T, sporkID flow.Identifier, dhtPrefix string, count int, idProvider module.IdentityProvider, opts ...NodeFixtureParameterOption) ([]p2p.LibP2PNode, @@ -589,3 +664,12 @@ func PeerIdSliceFixture(t *testing.T, n int) peer.IDSlice { } return ids } + +// NewConnectionGater creates a new connection gater for testing with given allow listing filter. +func NewConnectionGater(idProvider module.IdentityProvider, allowListFilter p2p.PeerFilter) p2p.ConnectionGater { + filters := []p2p.PeerFilter{allowListFilter} + return connection.NewConnGater(unittest.Logger(), + idProvider, + connection.WithOnInterceptPeerDialFilters(filters), + connection.WithOnInterceptSecuredFilters(filters)) +} diff --git a/network/p2p/test/message.go b/network/p2p/test/message.go new file mode 100644 index 00000000000..cb2d762393d --- /dev/null +++ b/network/p2p/test/message.go @@ -0,0 +1,65 @@ +package p2ptest + +import ( + "testing" + + pb "github.com/libp2p/go-libp2p-pubsub/pb" + "github.com/libp2p/go-libp2p/core/peer" + + "github.com/onflow/flow-go/utils/unittest" +) + +// WithFrom is a test helper that returns a function that sets the from field of a pubsub message to the given peer id. +func WithFrom(from peer.ID) func(*pb.Message) { + return func(m *pb.Message) { + m.From = []byte(from) + } +} + +// WithTopic is a test helper that returns a function that sets the topic of a pubsub message to the given topic. +func WithTopic(topic string) func(*pb.Message) { + return func(m *pb.Message) { + m.Topic = &topic + } +} + +// WithoutSignature is a test helper that returns a function that sets the signature of a pubsub message to nil, effectively removing the signature. +func WithoutSignature() func(*pb.Message) { + return func(m *pb.Message) { + m.Signature = nil + } +} + +// WithoutSignerId is a test helper that returns a function that sets the from field of a pubsub message to nil, effectively removing the signer id. +func WithoutSignerId() func(*pb.Message) { + return func(m *pb.Message) { + m.From = nil + } +} + +// PubsubMessageFixture is a test helper that returns a random pubsub message with the given options applied. +// If no options are provided, the message will be random. +// Args: +// +// t: testing.T +// +// opt: variadic list of options to apply to the message +// Returns: +// *pb.Message: pubsub message +func PubsubMessageFixture(t *testing.T, opts ...func(*pb.Message)) *pb.Message { + topic := unittest.RandomStringFixture(t, 10) + + m := &pb.Message{ + Data: unittest.RandomByteSlice(t, 100), + Topic: &topic, + Signature: unittest.RandomByteSlice(t, 100), + From: unittest.RandomByteSlice(t, 100), + Seqno: unittest.RandomByteSlice(t, 100), + } + + for _, opt := range opts { + opt(m) + } + + return m +} diff --git a/network/p2p/test/sporking_test.go b/network/p2p/test/sporking_test.go index bee29c54aed..14571b59e02 100644 --- a/network/p2p/test/sporking_test.go +++ b/network/p2p/test/sporking_test.go @@ -5,23 +5,22 @@ import ( "testing" "time" - "github.com/onflow/flow-go/model/flow" - libp2pmessage "github.com/onflow/flow-go/model/libp2p/message" - "github.com/onflow/flow-go/network" - "github.com/onflow/flow-go/network/message" - "github.com/libp2p/go-libp2p/core/peer" "github.com/libp2p/go-libp2p/core/peerstore" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/onflow/flow-go/model/flow" + libp2pmessage "github.com/onflow/flow-go/model/libp2p/message" + "github.com/onflow/flow-go/network" + "github.com/onflow/flow-go/network/message" + "github.com/onflow/flow-go/network/p2p" p2ptest "github.com/onflow/flow-go/network/p2p/test" "github.com/onflow/flow-go/network/p2p/utils" "github.com/onflow/flow-go/module/irrecoverable" - mockmodule "github.com/onflow/flow-go/module/mock" "github.com/onflow/flow-go/network/channels" "github.com/onflow/flow-go/network/internal/p2pfixtures" flowpubsub "github.com/onflow/flow-go/network/validator/pubsub" @@ -41,7 +40,7 @@ import ( // TestCrosstalkPreventionOnNetworkKeyChange tests that a node from the old chain cannot talk to a node in the new chain // if it's network key is updated while the libp2p protocol ID remains the same func TestCrosstalkPreventionOnNetworkKeyChange(t *testing.T) { - idProvider := mockmodule.NewIdentityProvider(t) + idProvider := unittest.NewUpdatableIDProvider(flow.IdentityList{}) ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -64,7 +63,7 @@ func TestCrosstalkPreventionOnNetworkKeyChange(t *testing.T) { idProvider, p2ptest.WithNetworkingPrivateKey(node1key), ) - idProvider.On("ByPeerID", node1.Host().ID()).Return(&id1, true).Maybe() + idProvider.SetIdentities(flow.IdentityList{&id1}) p2ptest.StartNode(t, signalerCtx1, node1, 100*time.Millisecond) defer p2ptest.StopNode(t, node1, cancel1, 100*time.Millisecond) @@ -80,7 +79,7 @@ func TestCrosstalkPreventionOnNetworkKeyChange(t *testing.T) { idProvider, p2ptest.WithNetworkingPrivateKey(node2key), ) - idProvider.On("ByPeerID", node2.Host().ID()).Return(&id2, true).Maybe() + idProvider.SetIdentities(flow.IdentityList{&id1, &id2}) p2ptest.StartNode(t, signalerCtx2, node2, 100*time.Millisecond) @@ -105,7 +104,7 @@ func TestCrosstalkPreventionOnNetworkKeyChange(t *testing.T) { p2ptest.WithNetworkingPrivateKey(node2keyNew), p2ptest.WithNetworkingAddress(id2.Address), ) - idProvider.On("ByPeerID", node2.Host().ID()).Return(&id2New, true).Maybe() + idProvider.SetIdentities(flow.IdentityList{&id1, &id2New}) p2ptest.StartNode(t, signalerCtx2a, node2, 100*time.Millisecond) defer p2ptest.StopNode(t, node2, cancel2a, 100*time.Millisecond) @@ -122,7 +121,7 @@ func TestCrosstalkPreventionOnNetworkKeyChange(t *testing.T) { // TestOneToOneCrosstalkPrevention tests that a node from the old chain cannot talk directly to a node in the new chain // if the Flow libp2p protocol ID is updated while the network keys are kept the same. func TestOneToOneCrosstalkPrevention(t *testing.T) { - idProvider := mockmodule.NewIdentityProvider(t) + idProvider := unittest.NewUpdatableIDProvider(flow.IdentityList{}) ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -139,7 +138,6 @@ func TestOneToOneCrosstalkPrevention(t *testing.T) { // create and start node 1 on localhost and random port node1, id1 := p2ptest.NodeFixture(t, sporkId1, "test_one_to_one_crosstalk_prevention", idProvider) - idProvider.On("ByPeerID", node1.Host().ID()).Return(&id1, true).Maybe() p2ptest.StartNode(t, signalerCtx1, node1, 100*time.Millisecond) defer p2ptest.StopNode(t, node1, cancel1, 100*time.Millisecond) @@ -149,8 +147,8 @@ func TestOneToOneCrosstalkPrevention(t *testing.T) { // create and start node 2 on localhost and random port node2, id2 := p2ptest.NodeFixture(t, sporkId1, "test_one_to_one_crosstalk_prevention", idProvider) - idProvider.On("ByPeerID", node2.Host().ID()).Return(&id2, true).Maybe() + idProvider.SetIdentities(flow.IdentityList{&id1, &id2}) p2ptest.StartNode(t, signalerCtx2, node2, 100*time.Millisecond) // create stream from node 2 to node 1 @@ -167,7 +165,7 @@ func TestOneToOneCrosstalkPrevention(t *testing.T) { idProvider, p2ptest.WithNetworkingAddress(id2.Address), ) - idProvider.On("ByPeerID", node2.Host().ID()).Return(&id2New, true).Maybe() + idProvider.SetIdentities(flow.IdentityList{&id1, &id2New}) p2ptest.StartNode(t, signalerCtx2a, node2, 100*time.Millisecond) defer p2ptest.StopNode(t, node2, cancel2a, 100*time.Millisecond) @@ -183,7 +181,7 @@ func TestOneToOneCrosstalkPrevention(t *testing.T) { // TestOneToKCrosstalkPrevention tests that a node from the old chain cannot talk to a node in the new chain via PubSub // if the channel is updated while the network keys are kept the same. func TestOneToKCrosstalkPrevention(t *testing.T) { - idProvider := mockmodule.NewIdentityProvider(t) + idProvider := unittest.NewUpdatableIDProvider(flow.IdentityList{}) ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -202,10 +200,10 @@ func TestOneToKCrosstalkPrevention(t *testing.T) { "test_one_to_k_crosstalk_prevention", idProvider, ) - idProvider.On("ByPeerID", node1.Host().ID()).Return(&id1, true).Maybe() + p2ptest.StartNode(t, signalerCtx1, node1, 100*time.Millisecond) defer p2ptest.StopNode(t, node1, cancel1, 100*time.Millisecond) - + idProvider.SetIdentities(flow.IdentityList{&id1}) // create and start node 2 on localhost and random port with the same root block ID node2, id2 := p2ptest.NodeFixture(t, previousSporkId, diff --git a/network/p2p/tracer/gossipSubScoreTracer_test.go b/network/p2p/tracer/gossipSubScoreTracer_test.go index 269e2c1099f..a759cc2b46f 100644 --- a/network/p2p/tracer/gossipSubScoreTracer_test.go +++ b/network/p2p/tracer/gossipSubScoreTracer_test.go @@ -190,9 +190,7 @@ func TestGossipSubScoreTracer(t *testing.T) { // IP score, and an existing mesh score. assert.Eventually(t, func() bool { // we expect the tracerNode to have the consensusNodes and accessNodes with the correct app scores. - exposer, ok := tracerNode.PeerScoreExposer() - require.True(t, ok) - + exposer := tracerNode.PeerScoreExposer() score, ok := exposer.GetAppScore(consensusNode.Host().ID()) if !ok || score != consensusScore { return false diff --git a/network/p2p/unicast/manager.go b/network/p2p/unicast/manager.go index f45c2ce7bcd..d8af81ed49d 100644 --- a/network/p2p/unicast/manager.go +++ b/network/p2p/unicast/manager.go @@ -69,12 +69,12 @@ func NewUnicastManager(logger zerolog.Logger, // as the core handler for other unicast protocols, e.g., compressions. func (m *Manager) WithDefaultHandler(defaultHandler libp2pnet.StreamHandler) { defaultProtocolID := protocols.FlowProtocolID(m.sporkId) - m.defaultHandler = defaultHandler - if len(m.protocols) > 0 { panic("default handler must be set only once before any unicast registration") } + m.defaultHandler = defaultHandler + m.protocols = []protocols.Protocol{ &PlainStream{ protocolId: defaultProtocolID, diff --git a/network/p2p/utils/logger.go b/network/p2p/utils/logger.go new file mode 100644 index 00000000000..b535d567ccd --- /dev/null +++ b/network/p2p/utils/logger.go @@ -0,0 +1,33 @@ +package utils + +import ( + pubsub "github.com/libp2p/go-libp2p-pubsub" + "github.com/rs/zerolog" +) + +// TopicScoreParamsLogger is a helper function that returns a logger with the topic score params added as fields. +// Args: +// logger: zerolog.Logger - logger to add fields to +// topicName: string - name of the topic +// params: pubsub.TopicScoreParams - topic score params +func TopicScoreParamsLogger(logger zerolog.Logger, topicName string, topicParams *pubsub.TopicScoreParams) zerolog.Logger { + return logger.With().Str("topic", topicName). + Bool("atomic_validation", topicParams.SkipAtomicValidation). + Float64("topic_weight", topicParams.TopicWeight). + Float64("time_in_mesh_weight", topicParams.TimeInMeshWeight). + Dur("time_in_mesh_quantum", topicParams.TimeInMeshQuantum). + Float64("time_in_mesh_cap", topicParams.TimeInMeshCap). + Float64("first_message_deliveries_weight", topicParams.FirstMessageDeliveriesWeight). + Float64("first_message_deliveries_decay", topicParams.FirstMessageDeliveriesDecay). + Float64("first_message_deliveries_cap", topicParams.FirstMessageDeliveriesCap). + Float64("mesh_message_deliveries_weight", topicParams.MeshMessageDeliveriesWeight). + Float64("mesh_message_deliveries_decay", topicParams.MeshMessageDeliveriesDecay). + Float64("mesh_message_deliveries_cap", topicParams.MeshMessageDeliveriesCap). + Float64("mesh_message_deliveries_threshold", topicParams.MeshMessageDeliveriesThreshold). + Dur("mesh_message_deliveries_window", topicParams.MeshMessageDeliveriesWindow). + Dur("mesh_message_deliveries_activation", topicParams.MeshMessageDeliveriesActivation). + Float64("mesh_failure_penalty_weight", topicParams.MeshFailurePenaltyWeight). + Float64("mesh_failure_penalty_decay", topicParams.MeshFailurePenaltyDecay). + Float64("invalid_message_deliveries_weight", topicParams.InvalidMessageDeliveriesWeight). + Float64("invalid_message_deliveries_decay", topicParams.InvalidMessageDeliveriesDecay).Logger() +} diff --git a/network/test/blob_service_test.go b/network/test/blob_service_test.go index bcce039fa35..40c052111d7 100644 --- a/network/test/blob_service_test.go +++ b/network/test/blob_service_test.go @@ -3,7 +3,6 @@ package test import ( "context" "fmt" - "os" "testing" "time" @@ -11,12 +10,13 @@ import ( "github.com/ipfs/go-datastore" "github.com/ipfs/go-datastore/sync" blockstore "github.com/ipfs/go-ipfs-blockstore" - "github.com/rs/zerolog" "github.com/stretchr/testify/suite" "go.uber.org/atomic" + "github.com/onflow/flow-go/network/p2p/connection" "github.com/onflow/flow-go/network/p2p/dht" - + p2pconfig "github.com/onflow/flow-go/network/p2p/p2pbuilder/config" + p2ptest "github.com/onflow/flow-go/network/p2p/test" "github.com/onflow/flow-go/utils/unittest" "github.com/onflow/flow-go/model/flow" @@ -26,7 +26,6 @@ import ( "github.com/onflow/flow-go/network" "github.com/onflow/flow-go/network/channels" "github.com/onflow/flow-go/network/internal/testutils" - "github.com/onflow/flow-go/network/mocknetwork" ) // conditionalTopology is a topology that behaves like the underlying topology when the condition is true, @@ -71,8 +70,6 @@ func (suite *BlobServiceTestSuite) putBlob(ds datastore.Batching, blob blobs.Blo func (suite *BlobServiceTestSuite) SetupTest() { suite.numNodes = 3 - logger := zerolog.New(os.Stdout) - // Bitswap listens to connect events but doesn't iterate over existing connections, and fixing this without // race conditions is tricky given the way the code is architected. As a result, libP2P hosts must first listen // on Bitswap before connecting to each other, otherwise their Bitswap requests may never reach each other. @@ -84,22 +81,21 @@ func (suite *BlobServiceTestSuite) SetupTest() { signalerCtx := irrecoverable.NewMockSignalerContext(suite.T(), ctx) - ids, nodes, mws, networks, _ := testutils.GenerateIDsMiddlewaresNetworks( - suite.T(), + ids, nodes, _ := testutils.LibP2PNodeForMiddlewareFixture(suite.T(), suite.numNodes, - logger, - unittest.NetworkCodec(), - mocknetwork.NewViolationsConsumer(suite.T()), - testutils.WithDHT("blob_service_test", dht.AsServer()), - testutils.WithPeerUpdateInterval(time.Second), - ) - suite.networks = networks - - testutils.StartNodesAndNetworks(signalerCtx, suite.T(), nodes, networks, 100*time.Millisecond) + p2ptest.WithDHTOptions(dht.AsServer()), + p2ptest.WithPeerManagerEnabled(&p2pconfig.PeerManagerConfig{ + UpdateInterval: 1 * time.Second, + ConnectionPruning: true, + ConnectorFactory: connection.DefaultLibp2pBackoffConnectorFactory(), + }, nil)) + mws, _ := testutils.MiddlewareFixtures(suite.T(), ids, nodes, testutils.MiddlewareConfigFixture(suite.T())) + suite.networks = testutils.NetworksFixture(suite.T(), ids, mws) + testutils.StartNodesAndNetworks(signalerCtx, suite.T(), nodes, suite.networks, 100*time.Millisecond) blobExchangeChannel := channels.Channel("blob-exchange") - for i, net := range networks { + for i, net := range suite.networks { ds := sync.MutexWrap(datastore.NewMapDatastore()) suite.datastores = append(suite.datastores, ds) blob := blobs.NewBlob([]byte(fmt.Sprintf("foo%v", i))) @@ -107,7 +103,7 @@ func (suite *BlobServiceTestSuite) SetupTest() { suite.putBlob(ds, blob) blobService, err := net.RegisterBlobService(blobExchangeChannel, ds) suite.Require().NoError(err) - <-blobService.Ready() + unittest.RequireCloseBefore(suite.T(), blobService.Ready(), 100*time.Millisecond, "blob service not ready") suite.blobServices = append(suite.blobServices, blobService) } diff --git a/network/test/echoengine_test.go b/network/test/echoengine_test.go index d04c1a6007c..eb170cbf266 100644 --- a/network/test/echoengine_test.go +++ b/network/test/echoengine_test.go @@ -3,16 +3,12 @@ package test import ( "context" "fmt" - "os" "strings" "sync" "testing" "time" - "github.com/onflow/flow-go/network/p2p" - "github.com/ipfs/go-log" - "github.com/rs/zerolog" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" @@ -23,7 +19,7 @@ import ( "github.com/onflow/flow-go/network" "github.com/onflow/flow-go/network/channels" "github.com/onflow/flow-go/network/internal/testutils" - "github.com/onflow/flow-go/network/mocknetwork" + "github.com/onflow/flow-go/network/p2p" "github.com/onflow/flow-go/utils/unittest" ) @@ -48,7 +44,6 @@ func TestStubEngineTestSuite(t *testing.T) { func (suite *EchoEngineTestSuite) SetupTest() { const count = 2 - logger := zerolog.New(os.Stderr).Level(zerolog.ErrorLevel) log.SetAllLoggers(log.LevelError) ctx, cancel := context.WithCancel(context.Background()) @@ -58,14 +53,9 @@ func (suite *EchoEngineTestSuite) SetupTest() { // both nodes should be of the same role to get connected on epidemic dissemination var nodes []p2p.LibP2PNode - suite.ids, nodes, suite.mws, suite.nets, _ = testutils.GenerateIDsMiddlewaresNetworks( - suite.T(), - count, - logger, - unittest.NetworkCodec(), - mocknetwork.NewViolationsConsumer(suite.T()), - ) - + suite.ids, nodes, _ = testutils.LibP2PNodeForMiddlewareFixture(suite.T(), count) + suite.mws, _ = testutils.MiddlewareFixtures(suite.T(), suite.ids, nodes, testutils.MiddlewareConfigFixture(suite.T())) + suite.nets = testutils.NetworksFixture(suite.T(), suite.ids, suite.mws) testutils.StartNodesAndNetworks(signalerCtx, suite.T(), nodes, suite.nets, 100*time.Millisecond) } diff --git a/network/test/epochtransition_test.go b/network/test/epochtransition_test.go index 8b7c0a655bd..e471b1d8f48 100644 --- a/network/test/epochtransition_test.go +++ b/network/test/epochtransition_test.go @@ -24,7 +24,6 @@ import ( "github.com/onflow/flow-go/module/irrecoverable" "github.com/onflow/flow-go/network" "github.com/onflow/flow-go/network/internal/testutils" - "github.com/onflow/flow-go/network/mocknetwork" mockprotocol "github.com/onflow/flow-go/state/protocol/mock" "github.com/onflow/flow-go/utils/unittest" ) @@ -181,13 +180,9 @@ func (suite *MutableIdentityTableSuite) addNodes(count int) { signalerCtx := irrecoverable.NewMockSignalerContext(suite.T(), ctx) // create the ids, middlewares and networks - ids, nodes, mws, nets, _ := testutils.GenerateIDsMiddlewaresNetworks( - suite.T(), - count, - suite.logger, - unittest.NetworkCodec(), - mocknetwork.NewViolationsConsumer(suite.T()), - ) + ids, nodes, _ := testutils.LibP2PNodeForMiddlewareFixture(suite.T(), count) + mws, _ := testutils.MiddlewareFixtures(suite.T(), ids, nodes, testutils.MiddlewareConfigFixture(suite.T())) + nets := testutils.NetworksFixture(suite.T(), ids, mws) suite.cancels = append(suite.cancels, cancel) testutils.StartNodesAndNetworks(signalerCtx, suite.T(), nodes, nets, 100*time.Millisecond) diff --git a/network/test/meshengine_test.go b/network/test/meshengine_test.go index 221bba44bc2..612d7679796 100644 --- a/network/test/meshengine_test.go +++ b/network/test/meshengine_test.go @@ -20,7 +20,6 @@ import ( "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" - "github.com/onflow/flow-go/network/mocknetwork" "github.com/onflow/flow-go/network/p2p/middleware" "github.com/onflow/flow-go/network/p2p/p2pnode" @@ -74,15 +73,9 @@ func (suite *MeshEngineTestSuite) SetupTest() { signalerCtx := irrecoverable.NewMockSignalerContext(suite.T(), ctx) var nodes []p2p.LibP2PNode - suite.ids, nodes, suite.mws, suite.nets, obs = testutils.GenerateIDsMiddlewaresNetworks( - suite.T(), - count, - logger, - unittest.NetworkCodec(), - mocknetwork.NewViolationsConsumer(suite.T()), - testutils.WithIdentityOpts(unittest.WithAllRoles()), - ) - + suite.ids, nodes, obs = testutils.LibP2PNodeForMiddlewareFixture(suite.T(), count) + suite.mws, _ = testutils.MiddlewareFixtures(suite.T(), suite.ids, nodes, testutils.MiddlewareConfigFixture(suite.T())) + suite.nets = testutils.NetworksFixture(suite.T(), suite.ids, suite.mws) testutils.StartNodesAndNetworks(signalerCtx, suite.T(), nodes, suite.nets, 100*time.Millisecond) for _, observableConnMgr := range obs { diff --git a/network/test/middleware_test.go b/network/test/middleware_test.go index 811b100e52f..8c0c1adb4f0 100644 --- a/network/test/middleware_test.go +++ b/network/test/middleware_test.go @@ -38,7 +38,6 @@ import ( p2ptest "github.com/onflow/flow-go/network/p2p/test" "github.com/onflow/flow-go/network/p2p/unicast/ratelimit" "github.com/onflow/flow-go/network/p2p/utils/ratelimiter" - "github.com/onflow/flow-go/network/slashing" "github.com/onflow/flow-go/utils/unittest" ) @@ -80,12 +79,10 @@ type MiddlewareTestSuite struct { ids []*flow.Identity metrics *metrics.NoopCollector // no-op performance monitoring simulation logger zerolog.Logger - providers []*testutils.UpdatableIDProvider + providers []*unittest.UpdatableIDProvider mwCancel context.CancelFunc mwCtx irrecoverable.SignalerContext - - slashingViolationsConsumer slashing.ViolationsConsumer } // TestMiddlewareTestSuit runs all the test methods in this test suit @@ -109,14 +106,8 @@ func (m *MiddlewareTestSuite) SetupTest() { log: m.logger, } - m.slashingViolationsConsumer = mocknetwork.NewViolationsConsumer(m.T()) - - m.ids, m.nodes, m.mws, obs, m.providers = testutils.GenerateIDsAndMiddlewares(m.T(), - m.size, - m.logger, - unittest.NetworkCodec(), - m.slashingViolationsConsumer) - + m.ids, m.nodes, obs = testutils.LibP2PNodeForMiddlewareFixture(m.T(), m.size) + m.mws, m.providers = testutils.MiddlewareFixtures(m.T(), m.ids, m.nodes, testutils.MiddlewareConfigFixture(m.T())) for _, observableConnMgr := range obs { observableConnMgr.Subscribe(&ob) } @@ -166,9 +157,8 @@ func (m *MiddlewareTestSuite) TestUpdateNodeAddresses() { irrecoverableCtx := irrecoverable.NewMockSignalerContext(m.T(), ctx) // create a new staked identity - ids, libP2PNodes, _ := testutils.GenerateIDs(m.T(), m.logger, 1) - - mws, providers := testutils.GenerateMiddlewares(m.T(), m.logger, ids, libP2PNodes, unittest.NetworkCodec(), m.slashingViolationsConsumer) + ids, libP2PNodes, _ := testutils.LibP2PNodeForMiddlewareFixture(m.T(), 1) + mws, providers := testutils.MiddlewareFixtures(m.T(), ids, libP2PNodes, testutils.MiddlewareConfigFixture(m.T())) require.Len(m.T(), ids, 1) require.Len(m.T(), providers, 1) require.Len(m.T(), mws, 1) @@ -248,33 +238,26 @@ func (m *MiddlewareTestSuite) TestUnicastRateLimit_Messages() { opts := []ratelimit.RateLimitersOption{ratelimit.WithMessageRateLimiter(messageRateLimiter), ratelimit.WithNotifier(distributor), ratelimit.WithDisabledRateLimiting(false)} rateLimiters := ratelimit.NewRateLimiters(opts...) - idProvider := testutils.NewUpdatableIDProvider(m.ids) - // create a new staked identity - connGaterFactory := func() p2p.ConnectionGater { - return testutils.NewConnectionGater(idProvider, func(pid peer.ID) error { + idProvider := unittest.NewUpdatableIDProvider(m.ids) + + ids, libP2PNodes, _ := testutils.LibP2PNodeForMiddlewareFixture(m.T(), + 1, + p2ptest.WithUnicastRateLimitDistributor(distributor), + p2ptest.WithConnectionGater(p2ptest.NewConnectionGater(idProvider, func(pid peer.ID) error { if messageRateLimiter.IsRateLimited(pid) { return fmt.Errorf("rate-limited peer") } return nil - }) - } - - ids, libP2PNodes, _ := testutils.GenerateIDs(m.T(), - m.logger, - 1, - testutils.WithUnicastRateLimiterDistributor(distributor), - testutils.WithConnectionGaterFactory(connGaterFactory)) + }))) idProvider.SetIdentities(append(m.ids, ids...)) // create middleware - mws, providers := testutils.GenerateMiddlewares(m.T(), - m.logger, + mws, providers := testutils.MiddlewareFixtures(m.T(), ids, libP2PNodes, - unittest.NetworkCodec(), - m.slashingViolationsConsumer, - testutils.WithUnicastRateLimiters(rateLimiters), - testutils.WithPeerManagerFilters(testutils.IsRateLimitedPeerFilter(messageRateLimiter))) + testutils.MiddlewareConfigFixture(m.T()), + middleware.WithUnicastRateLimiters(rateLimiters), + middleware.WithPeerManagerFilters([]p2p.PeerFilter{testutils.IsRateLimitedPeerFilter(messageRateLimiter)})) require.Len(m.T(), ids, 1) require.Len(m.T(), providers, 1) @@ -406,34 +389,28 @@ func (m *MiddlewareTestSuite) TestUnicastRateLimit_Bandwidth() { opts := []ratelimit.RateLimitersOption{ratelimit.WithBandwidthRateLimiter(bandwidthRateLimiter), ratelimit.WithNotifier(distributor), ratelimit.WithDisabledRateLimiting(false)} rateLimiters := ratelimit.NewRateLimiters(opts...) - idProvider := testutils.NewUpdatableIDProvider(m.ids) - // create connection gater, connection gater will refuse connections from rate limited nodes - connGaterFactory := func() p2p.ConnectionGater { - return testutils.NewConnectionGater(idProvider, func(pid peer.ID) error { + idProvider := unittest.NewUpdatableIDProvider(m.ids) + // create a new staked identity + ids, libP2PNodes, _ := testutils.LibP2PNodeForMiddlewareFixture(m.T(), + 1, + p2ptest.WithUnicastRateLimitDistributor(distributor), + p2ptest.WithConnectionGater(p2ptest.NewConnectionGater(idProvider, func(pid peer.ID) error { + // create connection gater, connection gater will refuse connections from rate limited nodes if bandwidthRateLimiter.IsRateLimited(pid) { return fmt.Errorf("rate-limited peer") } return nil - }) - } - // create a new staked identity - ids, libP2PNodes, _ := testutils.GenerateIDs(m.T(), - m.logger, - 1, - testutils.WithUnicastRateLimiterDistributor(distributor), - testutils.WithConnectionGaterFactory(connGaterFactory)) + }))) idProvider.SetIdentities(append(m.ids, ids...)) // create middleware - mws, providers := testutils.GenerateMiddlewares(m.T(), - m.logger, + mws, providers := testutils.MiddlewareFixtures(m.T(), ids, libP2PNodes, - unittest.NetworkCodec(), - m.slashingViolationsConsumer, - testutils.WithUnicastRateLimiters(rateLimiters), - testutils.WithPeerManagerFilters(testutils.IsRateLimitedPeerFilter(bandwidthRateLimiter))) + testutils.MiddlewareConfigFixture(m.T()), + middleware.WithUnicastRateLimiters(rateLimiters), + middleware.WithPeerManagerFilters([]p2p.PeerFilter{testutils.IsRateLimitedPeerFilter(bandwidthRateLimiter)})) require.Len(m.T(), ids, 1) require.Len(m.T(), providers, 1) require.Len(m.T(), mws, 1) @@ -524,7 +501,7 @@ func (m *MiddlewareTestSuite) TestUnicastRateLimit_Bandwidth() { require.Equal(m.T(), uint64(1), rateLimits.Load()) } -func (m *MiddlewareTestSuite) createOverlay(provider *testutils.UpdatableIDProvider) *mocknetwork.Overlay { +func (m *MiddlewareTestSuite) createOverlay(provider *unittest.UpdatableIDProvider) *mocknetwork.Overlay { overlay := &mocknetwork.Overlay{} overlay.On("Identities").Maybe().Return(func() flow.IdentityList { return provider.Identities(filter.Any) diff --git a/network/test/unicast_authorization_test.go b/network/test/unicast_authorization_test.go index 6fe4d0b8b58..f4a4171944d 100644 --- a/network/test/unicast_authorization_test.go +++ b/network/test/unicast_authorization_test.go @@ -47,7 +47,7 @@ type UnicastAuthorizationTestSuite struct { // receiverID the identity on the mw sending the message receiverID *flow.Identity // providers id providers generated at beginning of a test run - providers []*testutils.UpdatableIDProvider + providers []*unittest.UpdatableIDProvider // cancel is the cancel func from the context that was used to start the middlewares in a test run cancel context.CancelFunc // waitCh is the channel used to wait for the middleware to perform authorization and invoke the slashing @@ -74,8 +74,10 @@ func (u *UnicastAuthorizationTestSuite) TearDownTest() { // setupMiddlewaresAndProviders will setup 2 middlewares that will be used as a sender and receiver in each suite test. func (u *UnicastAuthorizationTestSuite) setupMiddlewaresAndProviders(slashingViolationsConsumer slashing.ViolationsConsumer) { - ids, libP2PNodes, _ := testutils.GenerateIDs(u.T(), u.logger, 2) - mws, providers := testutils.GenerateMiddlewares(u.T(), u.logger, ids, libP2PNodes, unittest.NetworkCodec(), slashingViolationsConsumer) + ids, libP2PNodes, _ := testutils.LibP2PNodeForMiddlewareFixture(u.T(), 2) + cfg := testutils.MiddlewareConfigFixture(u.T()) + cfg.SlashingViolationsConsumer = slashingViolationsConsumer + mws, providers := testutils.MiddlewareFixtures(u.T(), ids, libP2PNodes, cfg) require.Len(u.T(), ids, 2) require.Len(u.T(), providers, 2) require.Len(u.T(), mws, 2) diff --git a/utils/unittest/bytes.go b/utils/unittest/bytes.go new file mode 100644 index 00000000000..96238cc7ae0 --- /dev/null +++ b/utils/unittest/bytes.go @@ -0,0 +1,20 @@ +package unittest + +import ( + "crypto/rand" + "testing" + + "github.com/stretchr/testify/require" +) + +// RandomByteSlice is a test helper that generates a cryptographically secure random byte slice of size n. +func RandomByteSlice(t *testing.T, n int) []byte { + require.Greater(t, n, 0, "size should be positive") + + byteSlice := make([]byte, n) + n, err := rand.Read(byteSlice) + require.NoErrorf(t, err, "failed to generate random byte slice of size %d", n) + require.Equalf(t, n, len(byteSlice), "failed to generate random byte slice of size %d", n) + + return byteSlice +} diff --git a/utils/unittest/strings.go b/utils/unittest/strings.go new file mode 100644 index 00000000000..9c7b39839af --- /dev/null +++ b/utils/unittest/strings.go @@ -0,0 +1,36 @@ +package unittest + +import ( + "crypto/rand" + "encoding/base64" + "testing" + + "github.com/stretchr/testify/require" +) + +// RandomStringFixture is a test helper that generates a cryptographically secure random string of size n. +func RandomStringFixture(t *testing.T, n int) string { + require.Greater(t, n, 0, "size should be positive") + + // The base64 encoding uses 64 different characters to represent data in + // strings, which makes it possible to represent 6 bits of data with each + // character (as 2^6 is 64). This means that every 3 bytes (24 bits) of + // input data will be represented by 4 characters (4 * 6 bits) in the + // base64 encoding. Consequently, base64 encoding increases the size of + // the data by approximately 1/3 compared to the original input data. + // + // 1. (n+3) / 4 - This calculates how many groups of 4 characters are needed + // in the base64 encoded output to represent at least 'n' characters. + // The +3 ensures rounding up, as integer division truncates the result. + // + // 2. ... * 3 - Each group of 4 base64 characters represents 3 bytes + // of input data. This multiplication calculates the number of bytes + // needed to produce the required length of the base64 string. + byteSlice := make([]byte, (n+3)/4*3) + n, err := rand.Read(byteSlice) + require.NoError(t, err) + require.Equal(t, n, len(byteSlice)) + + encodedString := base64.URLEncoding.EncodeToString(byteSlice) + return encodedString[:n] +} diff --git a/network/internal/testutils/updatable_provider.go b/utils/unittest/updatable_provider.go similarity index 98% rename from network/internal/testutils/updatable_provider.go rename to utils/unittest/updatable_provider.go index 014ae696b99..9661f7039a6 100644 --- a/network/internal/testutils/updatable_provider.go +++ b/utils/unittest/updatable_provider.go @@ -1,4 +1,4 @@ -package testutils +package unittest import ( "sync"