Merge rc/v1.6.0 in feat/sync missing trie nodes 3apr

api/groups/nodeGroup.go

@@ -17,15 +17,16 @@ import (
 )
 
 const (
-	pidQueryParam          = "pid"
-	debugPath              = "/debug"
-	heartbeatStatusPath    = "/heartbeatstatus"
-	metricsPath            = "/metrics"
-	p2pStatusPath          = "/p2pstatus"
-	peerInfoPath           = "/peerinfo"
-	statusPath             = "/status"
-	epochStartDataForEpoch = "/epoch-start/:epoch"
-	bootstrapStatusPath    = "/bootstrapstatus"
+	pidQueryParam             = "pid"
+	debugPath                 = "/debug"
+	heartbeatStatusPath       = "/heartbeatstatus"
+	metricsPath               = "/metrics"
+	p2pStatusPath             = "/p2pstatus"
+	peerInfoPath              = "/peerinfo"
+	statusPath                = "/status"
+	epochStartDataForEpoch    = "/epoch-start/:epoch"
+	bootstrapStatusPath       = "/bootstrapstatus"
+	connectedPeersRatingsPath = "/connected-peers-ratings"
 )
 
 // nodeFacadeHandler defines the methods to be implemented by a facade for node requests
@@ -35,6 +36,7 @@ type nodeFacadeHandler interface {
 	GetQueryHandler(name string) (debug.QueryHandler, error)
 	GetEpochStartDataAPI(epoch uint32) (*common.EpochStartDataAPI, error)
 	GetPeerInfo(pid string) ([]core.QueryP2PPeerInfo, error)
+	GetConnectedPeersRatings() string
 	IsInterfaceNil() bool
 }
 
@@ -102,6 +104,11 @@ func NewNodeGroup(facade nodeFacadeHandler) (*nodeGroup, error) {
 			Method:  http.MethodGet,
 			Handler: ng.bootstrapMetrics,
 		},
+		{
+			Path:    connectedPeersRatingsPath,
+			Method:  http.MethodGet,
+			Handler: ng.connectedPeersRatings,
+		},
 	}
 	ng.endpoints = endpoints
 
@@ -318,6 +325,19 @@ func (ng *nodeGroup) bootstrapMetrics(c *gin.Context) {
 	)
 }
 
+// connectedPeersRatings returns the node's connected peers ratings
+func (ng *nodeGroup) connectedPeersRatings(c *gin.Context) {
+	ratings := ng.getFacade().GetConnectedPeersRatings()
+	c.JSON(
+		http.StatusOK,
+		shared.GenericAPIResponse{
+			Data:  gin.H{"ratings": ratings},
+			Error: "",
+			Code:  shared.ReturnCodeSuccess,
+		},
+	)
+}
+
 func (ng *nodeGroup) getFacade() nodeFacadeHandler {
 	ng.mutFacade.RLock()
 	defer ng.mutFacade.RUnlock()

api/groups/nodeGroup_test.go

@@ -252,6 +252,37 @@ func TestBootstrapStatusMetrics_ShouldWork(t *testing.T) {
 	assert.True(t, valuesFound)
 }
 
+func TestBootstrapGetConnectedPeersRatings_ShouldWork(t *testing.T) {
+	providedRatings := map[string]string{
+		"pid1": "100",
+		"pid2": "-50",
+		"pid3": "-5",
+	}
+	buff, _ := json.Marshal(providedRatings)
+	facade := mock.FacadeStub{
+		GetConnectedPeersRatingsCalled: func() string {
+			return string(buff)
+		},
+	}
+
+	nodeGroup, err := groups.NewNodeGroup(&facade)
+	require.NoError(t, err)
+
+	ws := startWebServer(nodeGroup, "node", getNodeRoutesConfig())
+
+	req, _ := http.NewRequest("GET", "/node/connected-peers-ratings", nil)
+	resp := httptest.NewRecorder()
+	ws.ServeHTTP(resp, req)
+
+	response := &shared.GenericAPIResponse{}
+	loadResponse(resp.Body, response)
+	respMap, ok := response.Data.(map[string]interface{})
+	assert.True(t, ok)
+	ratings, ok := respMap["ratings"].(string)
+	assert.True(t, ok)
+	assert.Equal(t, string(buff), ratings)
+}
+
 func TestStatusMetrics_ShouldDisplayNonP2pMetrics(t *testing.T) {
 	statusMetricsProvider := statusHandler.NewStatusMetrics()
 	key := "test-details-key"
@@ -595,6 +626,7 @@ func getNodeRoutesConfig() config.ApiRoutesConfig {
 					{Name: "/peerinfo", Open: true},
 					{Name: "/epoch-start/:epoch", Open: true},
 					{Name: "/bootstrapstatus", Open: true},
+					{Name: "/connected-peers-ratings", Open: true},
 				},
 			},
 		},

api/mock/facadeStub.go

@@ -42,6 +42,7 @@ type FacadeStub struct {
 	GetQueryHandlerCalled                       func(name string) (debug.QueryHandler, error)
 	GetValueForKeyCalled                        func(address string, key string, options api.AccountQueryOptions) (string, api.BlockInfo, error)
 	GetPeerInfoCalled                           func(pid string) ([]core.QueryP2PPeerInfo, error)
+	GetConnectedPeersRatingsCalled              func() string
 	GetEpochStartDataAPICalled                  func(epoch uint32) (*common.EpochStartDataAPI, error)
 	GetThrottlerForEndpointCalled               func(endpoint string) (core.Throttler, bool)
 	GetUsernameCalled                           func(address string, options api.AccountQueryOptions) (string, api.BlockInfo, error)
@@ -371,6 +372,11 @@ func (f *FacadeStub) GetPeerInfo(pid string) ([]core.QueryP2PPeerInfo, error) {
 	return f.GetPeerInfoCalled(pid)
 }
 
+// GetConnectedPeersRatings -
+func (f *FacadeStub) GetConnectedPeersRatings() string {
+	return f.GetConnectedPeersRatingsCalled()
+}
+
 // GetEpochStartDataAPI -
 func (f *FacadeStub) GetEpochStartDataAPI(epoch uint32) (*common.EpochStartDataAPI, error) {
 	return f.GetEpochStartDataAPICalled(epoch)

api/shared/interface.go

@@ -98,6 +98,7 @@ type FacadeHandler interface {
 	GetQueryHandler(name string) (debug.QueryHandler, error)
 	GetEpochStartDataAPI(epoch uint32) (*common.EpochStartDataAPI, error)
 	GetPeerInfo(pid string) ([]core.QueryP2PPeerInfo, error)
+	GetConnectedPeersRatings() string
 	GetProof(rootHash string, address string) (*common.GetProofResponse, error)
 	GetProofDataTrie(rootHash string, address string, key string) (*common.GetProofResponse, *common.GetProofResponse, error)
 	GetProofCurrentRootHash(address string) (*common.GetProofResponse, error)

cmd/keygenerator/converter/errors.go

@@ -2,4 +2,5 @@ package converter
 
 import "errors"
 
+// ErrNotImplemented is returned when a method is not implemented
 var errNotImplemented = errors.New("not implemented")

cmd/keygenerator/converter/pidPubkeyConverter.go

@@ -1,9 +1,6 @@
 package converter
 
 import (
-	"encoding/hex"
-	"runtime/debug"
-
 	crypto "github.com/multiversx/mx-chain-crypto-go"
 	"github.com/multiversx/mx-chain-crypto-go/signing"
 	"github.com/multiversx/mx-chain-crypto-go/signing/secp256k1"
@@ -33,19 +30,13 @@ func (converter *pidPubkeyConverter) Decode(_ string) ([]byte, error) {
 }
 
 // Encode encodes a byte array in its string representation
-func (converter *pidPubkeyConverter) Encode(pkBytes []byte) string {
+func (converter *pidPubkeyConverter) Encode(pkBytes []byte) (string, error) {
 	pidString, err := converter.encode(pkBytes)
 	if err != nil {
-		log.Warn("pidPubkeyConverter.Encode encode",
-			"hex buff", hex.EncodeToString(pkBytes),
-			"error", err,
-			"stack trace", string(debug.Stack()),
-		)
-
-		return ""
+		return "", err
 	}
 
-	return pidString
+	return pidString, nil
 }
 
 func (converter *pidPubkeyConverter) encode(pkBytes []byte) (string, error) {

cmd/keygenerator/converter/pidPubkeyConverter_test.go

@@ -30,7 +30,8 @@ func TestPidPubkeyConverter_Encode(t *testing.T) {
 			}
 		}()
 
-		encoded := converter.Encode([]byte("not a valid PK"))
+		encoded, err := converter.Encode([]byte("not a valid PK"))
+		assert.Contains(t, err.Error(), "parameter is invalid")
 		assert.Empty(t, encoded)
 	})
 	t.Run("valid public key should work", func(t *testing.T) {
@@ -40,7 +41,8 @@ func TestPidPubkeyConverter_Encode(t *testing.T) {
 		pkBytes, err := hex.DecodeString(pkHex)
 		assert.Nil(t, err)
 
-		encoded := converter.Encode(pkBytes)
+		encoded, err := converter.Encode(pkBytes)
+		assert.Nil(t, err)
 		assert.Equal(t, "16Uiu2HAmSHgyTYyawhsZv9opxTHX77vKjoPeGkyCYS5fYVMssHjN", encoded)
 	})
 }

cmd/keygenerator/main.go

@@ -41,6 +41,7 @@ const minedWalletPrefixKeys = "mined-wallet"
 const nopattern = "nopattern"
 const desiredpattern = "[0-f]+"
 const noshard = -1
+const pubkeyHrp = "erd"
 
 type key struct {
 	skBytes []byte
@@ -49,7 +50,7 @@ type key struct {
 
 type pubKeyConverter interface {
 	Decode(humanReadable string) ([]byte, error)
-	Encode(pkBytes []byte) string
+	Encode(pkBytes []byte) (string, error)
 	IsInterfaceNil() bool
 }
 
@@ -132,7 +133,7 @@ VERSION:
 
 	validatorPubKeyConverter, _ = pubkeyConverter.NewHexPubkeyConverter(blsPubkeyLen)
 	pidPubKeyConverter          = converter.NewPidPubkeyConverter()
-	walletPubKeyConverter, _    = pubkeyConverter.NewBech32PubkeyConverter(txSignPubkeyLen, log)
+	walletPubKeyConverter, _    = pubkeyConverter.NewBech32PubkeyConverter(txSignPubkeyLen, pubkeyHrp)
 )
 
 func main() {
@@ -365,7 +366,10 @@ func writeKeyToStream(writer io.Writer, key key, converter pubKeyConverter) erro
 		return fmt.Errorf("nil writer")
 	}
 
-	pkString := converter.Encode(key.pkBytes)
+	pkString, err := converter.Encode(key.pkBytes)
+	if err != nil {
+		return err
+	}
 
 	blk := pem.Block{
 		Type:  "PRIVATE KEY for " + pkString,

cmd/node/config/api.toml

@@ -35,7 +35,10 @@
         { Name = "/epoch-start/:epoch", Open = true },
 
         # /node/bootstrapstatus will return all metrics available during bootstrap
-        { Name = "/bootstrapstatus", Open = true }
+        { Name = "/bootstrapstatus", Open = true },
+
+        # /node/connected-peers-ratings will return the peers ratings
+        { Name = "/connected-peers-ratings", Open = true }
     ]
 
 [APIPackages.address]

cmd/node/config/config.toml

@@ -485,8 +485,8 @@
     Type = "LRU"
 
 [PeersRatingConfig]
-    TopRatedCacheCapacity = 5000
-    BadRatedCacheCapacity = 5000
+    TopRatedCacheCapacity           = 5000
+    BadRatedCacheCapacity           = 5000
 
 [PoolsCleanersConfig]
     MaxRoundsToKeepUnprocessedMiniBlocks = 300   # max number of rounds unprocessed miniblocks are kept in pool
@@ -589,6 +589,8 @@
     TrieOperationsDeadlineMilliseconds = 10000
     # GetAddressesBulkMaxSize represents the maximum number of addresses to be fetched in a bulk per API request. 0 means unlimited
     GetAddressesBulkMaxSize = 100
+    # VmQueryDelayAfterStartInSec represents the number of seconds to wait when starting node before accepting vm query requests
+    VmQueryDelayAfterStartInSec = 120
     # EndpointsThrottlers represents a map for maximum simultaneous go routines for an endpoint
     EndpointsThrottlers = [{ Endpoint = "/transaction/:hash", MaxNumGoRoutines = 10 },
                            { Endpoint = "/transaction/send", MaxNumGoRoutines = 2 },
@@ -599,6 +601,7 @@
     Length = 32
     Type = "bech32"
     SignatureLength = 64
+    Hrp = "erd"
 
 [ValidatorPubkeyConverter]
     Length = 96
@@ -931,6 +934,7 @@
     HardforkTimeBetweenSendsInSec                    = 60   # 1min       # time between hardfork messages
     TimeBetweenConnectionsMetricsUpdateInSec         = 30   # 30sec      # time between consecutive connections metrics updates
     TimeToReadDirectConnectionsInSec                 = 15   # 15sec      # time between consecutive peer shard mapper updates with direct connections
+    PeerAuthenticationTimeBetweenChecksInSec         = 6    # 6sec
     [HeartbeatV2.HeartbeatPool]
         Name = "HeartbeatPool"
         Capacity = 50000

cmd/node/config/prefs.toml

@@ -4,10 +4,12 @@
    # if "disabled" is provided then the node will start in the corresponding shard for its public key or 0 otherwise
    DestinationShardAsObserver = "disabled"
 
-   # NodeDisplayName represents the friendly name a user can pick for his node in the status monitor
+   # NodeDisplayName represents the friendly name a user can pick for his node in the status monitor when the node does not run in multikey mode
+   # In multikey mode, all bls keys not mentioned in NamedIdentity section will use this one as default
    NodeDisplayName = ""
 
-   # Identity represents the keybase's identity
+   # Identity represents the keybase's identity when the node does not run in multikey mode
+   # In multikey mode, all bls keys not mentioned in NamedIdentity section will use this one as default
    Identity = ""
 
    # RedundancyLevel represents the level of redundancy used by the node (-1 = disabled, 0 = main instance (default),
@@ -47,3 +49,16 @@
    #    { File = "config.toml", Path = "MiniBlocksStorage.Cache.Name", Value = "MiniBlocksStorage" },
    #    { File = "external.toml", Path = "ElasticSearchConnector.Enabled", Value = "true" }
    #]
+
+# NamedIdentity represents an identity that runs nodes on the multikey
+# There can be multiple identities set on the same node, each one of them having different bls keys, just by duplicating the NamedIdentity
+[[NamedIdentity]]
+   # Identity represents the keybase identity for the current NamedIdentity
+   Identity = ""
+   # NodeName represents the name that will be given to the names of the current identity
+   NodeName = ""
+   # BLSKeys represents the BLS keys assigned to the current NamedIdentity
+   BLSKeys = [
+      "",
+      ""
+   ]

cmd/node/flags.go

@@ -208,6 +208,13 @@ var (
 		Usage: "The `filepath` for the PEM file which contains the secret keys for the validator key.",
 		Value: "./config/validatorKey.pem",
 	}
+	// allValidatorKeysPemFile defines a flag for the path to the file that hold all validator keys used in block signing
+	// managed by the current node
+	allValidatorKeysPemFile = cli.StringFlag{
+		Name:  "all-validator-keys-pem-file",
+		Usage: "The `filepath` for the PEM file which contains all the secret keys managed by the current node.",
+		Value: "./config/allValidatorsKeys.pem",
+	}
 
 	// logLevel defines the logger level
 	logLevel = cli.StringFlag{
@@ -397,6 +404,7 @@ func getFlags() []cli.Flag {
 		gasScheduleConfigurationDirectory,
 		validatorKeyIndex,
 		validatorKeyPemFile,
+		allValidatorKeysPemFile,
 		port,
 		profileMode,
 		useHealthService,
@@ -475,6 +483,7 @@ func applyFlags(ctx *cli.Context, cfgs *config.Configs, flagsConfig *config.Cont
 	cfgs.ConfigurationPathsHolder.GasScheduleDirectoryName = ctx.GlobalString(gasScheduleConfigurationDirectory.Name)
 	cfgs.ConfigurationPathsHolder.SmartContracts = ctx.GlobalString(smartContractsFile.Name)
 	cfgs.ConfigurationPathsHolder.ValidatorKey = ctx.GlobalString(validatorKeyPemFile.Name)
+	cfgs.ConfigurationPathsHolder.AllValidatorKeys = ctx.GlobalString(allValidatorKeysPemFile.Name)
 	cfgs.ConfigurationPathsHolder.P2pKey = ctx.GlobalString(p2pKeyPemFile.Name)
 
 	if ctx.IsSet(startInEpoch.Name) {

cmd/node/main.go

@@ -63,7 +63,8 @@ func main() {
 	app.Name = "MultiversX Node CLI App"
 	machineID := core.GetAnonymizedMachineID(app.Name)
 
-	app.Version = fmt.Sprintf("%s/%s/%s-%s/%s", appVersion, runtime.Version(), runtime.GOOS, runtime.GOARCH, machineID)
+	baseVersion := fmt.Sprintf("%s/%s/%s-%s", appVersion, runtime.Version(), runtime.GOOS, runtime.GOARCH)
+	app.Version = fmt.Sprintf("%s/%s", baseVersion, machineID)
 	app.Usage = "This is the entry point for starting a new MultiversX node - the app will start after the genesis timestamp"
 	app.Flags = getFlags()
 	app.Authors = []cli.Author{
@@ -74,7 +75,7 @@ func main() {
 	}
 
 	app.Action = func(c *cli.Context) error {
-		return startNodeRunner(c, log, app.Version)
+		return startNodeRunner(c, log, baseVersion, app.Version)
 	}
 
 	err := app.Run(os.Args)
@@ -84,7 +85,7 @@ func main() {
 	}
 }
 
-func startNodeRunner(c *cli.Context, log logger.Logger, version string) error {
+func startNodeRunner(c *cli.Context, log logger.Logger, baseVersion string, version string) error {
 	flagsConfig := getFlagsConfig(c, log)
 
 	fileLogging, errLogger := attachFileLogger(log, flagsConfig)
@@ -125,6 +126,7 @@ func startNodeRunner(c *cli.Context, log logger.Logger, version string) error {
 		log.Debug("initialized memory ballast object", "size", core.ConvertBytes(uint64(len(memoryBallastObject))))
 	}
 
+	cfgs.FlagsConfig.BaseVersion = baseVersion
 	cfgs.FlagsConfig.Version = version
 
 	nodeRunner, errRunner := node.NewNodeRunner(cfgs)

common/constants.go

@@ -301,7 +301,7 @@ const MetricRedundancyIsMainActive = "erd_redundancy_is_main_active"
 // MetricValueNA represents the value to be used when a metric is not available/applicable
 const MetricValueNA = "N/A"
 
-//MetricProcessedProposedBlock is the metric that specify the percent of the block subround used for header and body
+// MetricProcessedProposedBlock is the metric that specify the percent of the block subround used for header and body
 // processing (0 meaning that the block was processed in no-time and 100 meaning that the block processing used all the
 // subround spare duration)
 const MetricProcessedProposedBlock = "erd_consensus_processed_proposed_block"