Merge branch 'master' into 1898-slashing-violations-consumer-alsp-int…

…egration-test

.github/workflows/ci.yml

@@ -66,8 +66,8 @@ jobs:
           cache: true
       - name: Run tidy
         run: make tidy
-      - name: Emulator no relic check
-        run: make emulator-norelic-check
+      - name: code sanity check
+        run: make code-sanity-check
 
   shell-check:
     name: ShellCheck

Makefile

@@ -19,7 +19,7 @@ ifeq (${IMAGE_TAG},)
 IMAGE_TAG := ${SHORT_COMMIT}
 endif
 
-IMAGE_TAG_NO_NETGO := $(IMAGE_TAG)-without_netgo
+IMAGE_TAG_NO_NETGO := $(IMAGE_TAG)-without-netgo
 
 # Name of the cover profile
 COVER_PROFILE := coverage.txt
@@ -87,6 +87,23 @@ emulator-norelic-check:
 	# test the fvm package compiles with Relic library disabled (required for the emulator build)
 	cd ./fvm && go test ./... -run=NoTestHasThisPrefix
 
+.SILENT: go-math-rand-check
+go-math-rand-check:
+	# check that the insecure math/rand Go package isn't used by production code.
+	# `exclude` should only specify non production code (test, bench..).
+	# If this check fails, try updating your code by using:
+	#   - "crypto/rand" or "flow-go/utils/rand" for non-deterministic randomness
+	#   - "flow-go/crypto/random" for deterministic randomness 
+	grep --include=\*.go \
+	--exclude=*test* --exclude=*helper* --exclude=*example* --exclude=*fixture* --exclude=*benchmark* --exclude=*profiler* \
+    --exclude-dir=*test* --exclude-dir=*helper* --exclude-dir=*example* --exclude-dir=*fixture* --exclude-dir=*benchmark* --exclude-dir=*profiler* -rnw '"math/rand"'; \
+    if [ $$? -ne 1 ]; then \
+       echo "[Error] Go production code should not use math/rand package"; exit 1; \
+    fi
+
+.PHONY: code-sanity-check
+code-sanity-check: go-math-rand-check emulator-norelic-check
+
 .PHONY: fuzz-fvm
 fuzz-fvm:
 	# run fuzz tests in the fvm package

cmd/access/node_builder/access_node_builder.go

@@ -1191,11 +1191,15 @@ func (builder *FlowAccessNodeBuilder) initPublicLibp2pNode(networkKey crypto.Pri
 		return nil, fmt.Errorf("could not create connection manager: %w", err)
 	}
 
-	meshTracer := tracer.NewGossipSubMeshTracer(
-		builder.Logger,
-		networkMetrics,
-		builder.IdentityProvider,
-		builder.FlowConfig.NetworkConfig.GossipSubConfig.LocalMeshLogInterval)
+	meshTracerCfg := &tracer.GossipSubMeshTracerConfig{
+		Logger:                       builder.Logger,
+		Metrics:                      networkMetrics,
+		IDProvider:                   builder.IdentityProvider,
+		LoggerInterval:               builder.FlowConfig.NetworkConfig.GossipSubConfig.LocalMeshLogInterval,
+		RpcSentTrackerCacheCollector: metrics.GossipSubRPCSentTrackerMetricFactory(builder.HeroCacheMetricsFactory(), network.PublicNetwork),
+		RpcSentTrackerCacheSize:      builder.FlowConfig.NetworkConfig.GossipSubConfig.RPCSentTrackerCacheSize,
+	}
+	meshTracer := tracer.NewGossipSubMeshTracer(meshTracerCfg)
 
 	libp2pNode, err := p2pbuilder.NewNodeBuilder(
 		builder.Logger,

cmd/bootstrap/cmd/clusters.go

@@ -10,7 +10,6 @@ import (
 	"github.com/onflow/flow-go/model/flow/assignment"
 	"github.com/onflow/flow-go/model/flow/factory"
 	"github.com/onflow/flow-go/model/flow/filter"
-	"github.com/onflow/flow-go/utils/rand"
 )
 
 // Construct random cluster assignment with internal and partner nodes.
@@ -39,12 +38,11 @@ func constructClusterAssignment(partnerNodes, internalNodes []model.NodeInfo) (f
 	}
 
 	// shuffle both collector lists based on a non-deterministic algorithm
-	var err error
-	err = rand.Shuffle(uint(len(partners)), func(i, j uint) { partners[i], partners[j] = partners[j], partners[i] })
+	partners, err := partners.Shuffle()
 	if err != nil {
 		log.Fatal().Err(err).Msg("could not shuffle partners")
 	}
-	err = rand.Shuffle(uint(len(internals)), func(i, j uint) { internals[i], internals[j] = internals[j], internals[i] })
+	internals, err = internals.Shuffle()
 	if err != nil {
 		log.Fatal().Err(err).Msg("could not shuffle internals")
 	}

cmd/observer/node_builder/observer_builder.go

@@ -702,11 +702,15 @@ func (builder *ObserverServiceBuilder) initPublicLibp2pNode(networkKey crypto.Pr
 		pis = append(pis, pi)
 	}
 
-	meshTracer := tracer.NewGossipSubMeshTracer(
-		builder.Logger,
-		builder.Metrics.Network,
-		builder.IdentityProvider,
-		builder.FlowConfig.NetworkConfig.GossipSubConfig.LocalMeshLogInterval)
+	meshTracerCfg := &tracer.GossipSubMeshTracerConfig{
+		Logger:                       builder.Logger,
+		Metrics:                      builder.Metrics.Network,
+		IDProvider:                   builder.IdentityProvider,
+		LoggerInterval:               builder.FlowConfig.NetworkConfig.GossipSubConfig.LocalMeshLogInterval,
+		RpcSentTrackerCacheCollector: metrics.GossipSubRPCSentTrackerMetricFactory(builder.HeroCacheMetricsFactory(), network.PublicNetwork),
+		RpcSentTrackerCacheSize:      builder.FlowConfig.NetworkConfig.GossipSubConfig.RPCSentTrackerCacheSize,
+	}
+	meshTracer := tracer.NewGossipSubMeshTracer(meshTracerCfg)
 
 	node, err := p2pbuilder.NewNodeBuilder(
 		builder.Logger,

cmd/scaffold.go

@@ -5,7 +5,6 @@ import (
 	"crypto/x509"
 	"errors"
 	"fmt"
-	"math/rand"
 	"os"
 	"runtime"
 	"strings"
@@ -1773,10 +1772,6 @@ func (fnb *FlowNodeBuilder) Build() (Node, error) {
 }
 
 func (fnb *FlowNodeBuilder) onStart() error {
-
-	// seed random generator
-	rand.Seed(time.Now().UnixNano())
-
 	// init nodeinfo by reading the private bootstrap file if not already set
 	if fnb.NodeID == flow.ZeroID {
 		if err := fnb.initNodeInfo(); err != nil {

cmd/util/cmd/execution-state-extract/export_report.json

@@ -1,6 +1,6 @@
 {
  "EpochCounter": 0,
- "PreviousStateCommitment": "18eb0e8beef7ce851e552ecd29c813fde0a9e6f0c5614d7615642076602a48cf",
- "CurrentStateCommitment": "18eb0e8beef7ce851e552ecd29c813fde0a9e6f0c5614d7615642076602a48cf",
+ "PreviousStateCommitment": "1c9f9d343cb8d4610e0b2c1eb74d6ea2f2f8aef2d666281dc22870e3efaa607b",
+ "CurrentStateCommitment": "1c9f9d343cb8d4610e0b2c1eb74d6ea2f2f8aef2d666281dc22870e3efaa607b",
  "ReportSucceeded": true
 }

config/default-config.yml

@@ -63,6 +63,9 @@ network-config:
   # The default interval at which the gossipsub score tracer logs the peer scores. This is used for debugging and forensics purposes.
   #	Note that we purposefully choose this logging interval high enough to avoid spamming the logs.
   gossipsub-score-tracer-interval: 1m
+  # The default RPC sent tracker cache size. The RPC sent tracker is used to track RPC control messages sent from the local node.
+  # Note: this cache size must be large enough to keep a history of sent messages in a reasonable time window of past history.
+  gossipsub-rpc-sent-tracker-cache-size: 1_000_000
   # Peer scoring is the default value for enabling peer scoring
   gossipsub-peer-scoring-enabled: true
   # Gossipsub rpc inspectors configs

consensus/hotstuff/signature/block_signer_decoder_test.go

@@ -140,7 +140,8 @@ func (s *blockSignerDecoderSuite) Test_EpochTransition() {
 	blockView := s.block.Header.View
 	parentView := s.block.Header.ParentView
 	epoch1Committee := s.allConsensus
-	epoch2Committee := s.allConsensus.SamplePct(.8)
+	epoch2Committee, err := s.allConsensus.SamplePct(.8)
+	require.NoError(s.T(), err)
 
 	*s.committee = *hotstuff.NewDynamicCommittee(s.T())
 	s.committee.On("IdentitiesByEpoch", parentView).Return(epoch1Committee, nil).Maybe()

consensus/integration/network_test.go

@@ -160,7 +160,11 @@ func (n *Network) publish(event interface{}, channel channels.Channel, targetIDs
 // Engines attached to the same channel on other nodes. The targeted nodes are selected based on the selector.
 // In this test helper implementation, multicast uses submit method under the hood.
 func (n *Network) multicast(event interface{}, channel channels.Channel, num uint, targetIDs ...flow.Identifier) error {
-	targetIDs = flow.Sample(num, targetIDs...)
+	var err error
+	targetIDs, err = flow.Sample(num, targetIDs...)
+	if err != nil {
+		return fmt.Errorf("sampling failed: %w", err)
+	}
 	return n.submit(event, channel, targetIDs...)
 }
 

engine/Readme.md

@@ -1,5 +1,4 @@
 # Notifier
-
 The Notifier implements the following state machine
 ![Notifier State Machine](/docs/NotifierStateMachine.png)
 

engine/access/rpc/backend/backend.go

@@ -370,7 +370,10 @@ func executionNodesForBlockID(
 	}
 
 	// randomly choose upto maxExecutionNodesCnt identities
-	executionIdentitiesRandom := subsetENs.Sample(maxExecutionNodesCnt)
+	executionIdentitiesRandom, err := subsetENs.Sample(maxExecutionNodesCnt)
+	if err != nil {
+		return nil, fmt.Errorf("sampling failed: %w", err)
+	}
 
 	if len(executionIdentitiesRandom) == 0 {
 		return nil, fmt.Errorf("no matching execution node found for block ID %v", blockID)

engine/access/rpc/backend/backend_transactions.go

@@ -129,7 +129,10 @@ func (b *backendTransactions) chooseCollectionNodes(tx *flow.TransactionBody, sa
 	}
 
 	// select a random subset of collection nodes from the cluster to be tried in order
-	targetNodes := txCluster.Sample(sampleSize)
+	targetNodes, err := txCluster.Sample(sampleSize)
+	if err != nil {
+		return nil, fmt.Errorf("sampling failed: %w", err)
+	}
 
 	// collect the addresses of all the chosen collection nodes
 	var targetAddrs = make([]string, len(targetNodes))

engine/common/requester/engine.go

@@ -367,7 +367,11 @@ func (e *Engine) dispatchRequest() (bool, error) {
 			if len(providers) == 0 {
 				return false, fmt.Errorf("no valid providers available")
 			}
-			providerID = providers.Sample(1)[0].NodeID
+			id, err := providers.Sample(1)
+			if err != nil {
+				return false, fmt.Errorf("sampling failed: %w", err)
+			}
+			providerID = id[0].NodeID
 		}
 
 		// add item to list and set retry parameters

engine/execution/computation/query/executor.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"encoding/hex"
 	"fmt"
-	"math/rand"
 	"strings"
 	"sync"
 	"time"
@@ -18,6 +17,7 @@ import (
 	"github.com/onflow/flow-go/model/flow"
 	"github.com/onflow/flow-go/module"
 	"github.com/onflow/flow-go/utils/debug"
+	"github.com/onflow/flow-go/utils/rand"
 )
 
 const (
@@ -71,7 +71,6 @@ type QueryExecutor struct {
 	vmCtx            fvm.Context
 	derivedChainData *derived.DerivedChainData
 	rngLock          *sync.Mutex
-	rng              *rand.Rand
 }
 
 var _ Executor = &QueryExecutor{}
@@ -92,7 +91,6 @@ func NewQueryExecutor(
 		vmCtx:            vmCtx,
 		derivedChainData: derivedChainData,
 		rngLock:          &sync.Mutex{},
-		rng:              rand.New(rand.NewSource(time.Now().UnixNano())),
 	}
 }
 
@@ -115,8 +113,11 @@ func (e *QueryExecutor) ExecuteScript(
 	// TODO: this is a temporary measure, we could remove this in the future
 	if e.logger.Debug().Enabled() {
 		e.rngLock.Lock()
-		trackerID := e.rng.Uint32()
-		e.rngLock.Unlock()
+		defer e.rngLock.Unlock()
+		trackerID, err := rand.Uint32()
+		if err != nil {
+			return nil, fmt.Errorf("failed to generate trackerID: %w", err)
+		}
 
 		trackedLogger := e.logger.With().Hex("script_hex", script).Uint32("trackerID", trackerID).Logger()
 		trackedLogger.Debug().Msg("script is sent for execution")

engine/execution/ingestion/uploader/model.go

@@ -29,9 +29,10 @@ func ComputationResultToBlockData(computationResult *execution.ComputationResult
 		txResults[i] = &AllResults[i]
 	}
 
-	events := make([]*flow.Event, 0)
-	for _, e := range computationResult.AllEvents() {
-		events = append(events, &e)
+	eventsList := computationResult.AllEvents()
+	events := make([]*flow.Event, len(eventsList))
+	for i := 0; i < len(eventsList); i++ {
+		events[i] = &eventsList[i]
 	}
 
 	trieUpdates := make(

engine/execution/ingestion/uploader/model_test.go

@@ -11,6 +11,7 @@ import (
 	"github.com/onflow/flow-go/ledger"
 	"github.com/onflow/flow-go/ledger/common/pathfinder"
 	"github.com/onflow/flow-go/ledger/complete"
+	"github.com/onflow/flow-go/model/flow"
 	"github.com/onflow/flow-go/utils/unittest"
 )
 
@@ -29,12 +30,23 @@ func Test_ComputationResultToBlockDataConversion(t *testing.T) {
 		assert.Equal(t, result, *blockData.TxResults[i])
 	}
 
-	// ramtin: warning returned events are not preserving orders,
-	// but since we are going to depricate this part of logic,
-	// I'm not going to spend more time fixing this mess
+	// Since returned events are not preserving orders,
+	// use map with event.ID() as key to confirm all events
+	// are included.
 	allEvents := cr.AllEvents()
 	require.Equal(t, len(allEvents), len(blockData.Events))
 
+	eventsInBlockData := make(map[flow.Identifier]flow.Event)
+	for _, e := range blockData.Events {
+		eventsInBlockData[e.ID()] = *e
+	}
+
+	for _, expectedEvent := range allEvents {
+		event, ok := eventsInBlockData[expectedEvent.ID()]
+		require.True(t, ok)
+		require.Equal(t, expectedEvent, event)
+	}
+
 	assert.Equal(t, len(expectedTrieUpdates), len(blockData.TrieUpdates))
 
 	assert.Equal(t, cr.CurrentEndState(), blockData.FinalStateCommitment)

engine/execution/state/bootstrap/bootstrap_test.go

@@ -53,7 +53,7 @@ func TestBootstrapLedger(t *testing.T) {
 }
 
 func TestBootstrapLedger_ZeroTokenSupply(t *testing.T) {
-	expectedStateCommitmentBytes, _ := hex.DecodeString("60a1998dc3c2656758f76520d040e1612b14d80bae263dd0d1118aa7c7d6e4ee")
+	expectedStateCommitmentBytes, _ := hex.DecodeString("986c540657fdb3b4154311069d901223a3268492f678ae706010cd537cc328ad")
 	expectedStateCommitment, err := flow.ToStateCommitment(expectedStateCommitmentBytes)
 	require.NoError(t, err)