Merge rc 1.6.0 trie sync optimizations

api/groups/blockGroup.go

@@ -10,8 +10,8 @@ import (
 
 	"github.com/gin-gonic/gin"
 	"github.com/multiversx/mx-chain-core-go/core/check"
+	"github.com/multiversx/mx-chain-core-go/data/alteredAccount"
 	"github.com/multiversx/mx-chain-core-go/data/api"
-	"github.com/multiversx/mx-chain-core-go/data/outport"
 	"github.com/multiversx/mx-chain-go/api/errors"
 	"github.com/multiversx/mx-chain-go/api/shared"
 	"github.com/multiversx/mx-chain-go/api/shared/logging"
@@ -33,7 +33,7 @@ type blockFacadeHandler interface {
 	GetBlockByHash(hash string, options api.BlockQueryOptions) (*api.Block, error)
 	GetBlockByNonce(nonce uint64, options api.BlockQueryOptions) (*api.Block, error)
 	GetBlockByRound(round uint64, options api.BlockQueryOptions) (*api.Block, error)
-	GetAlteredAccountsForBlock(options api.GetAlteredAccountsForBlockOptions) ([]*outport.AlteredAccount, error)
+	GetAlteredAccountsForBlock(options api.GetAlteredAccountsForBlockOptions) ([]*alteredAccount.AlteredAccount, error)
 	IsInterfaceNil() bool
 }
 

api/groups/blockGroup_test.go

@@ -10,8 +10,8 @@ import (
 	"strings"
 	"testing"
 
+	"github.com/multiversx/mx-chain-core-go/data/alteredAccount"
 	"github.com/multiversx/mx-chain-core-go/data/api"
-	"github.com/multiversx/mx-chain-core-go/data/outport"
 	apiErrors "github.com/multiversx/mx-chain-go/api/errors"
 	"github.com/multiversx/mx-chain-go/api/groups"
 	"github.com/multiversx/mx-chain-go/api/mock"
@@ -39,7 +39,7 @@ func TestNewBlockGroup(t *testing.T) {
 
 type alteredAccountsForBlockResponse struct {
 	Data struct {
-		Accounts []*outport.AlteredAccount `json:"accounts"`
+		Accounts []*alteredAccount.AlteredAccount `json:"accounts"`
 	} `json:"data"`
 	Error string `json:"error"`
 	Code  string `json:"code"`
@@ -251,7 +251,7 @@ func TestBlockGroup_getAlteredAccountsByNonce(t *testing.T) {
 		t.Parallel()
 
 		facade := &mock.FacadeStub{
-			GetAlteredAccountsForBlockCalled: func(options api.GetAlteredAccountsForBlockOptions) ([]*outport.AlteredAccount, error) {
+			GetAlteredAccountsForBlockCalled: func(options api.GetAlteredAccountsForBlockOptions) ([]*alteredAccount.AlteredAccount, error) {
 				return nil, expectedErr
 			},
 		}
@@ -275,15 +275,15 @@ func TestBlockGroup_getAlteredAccountsByNonce(t *testing.T) {
 				Nonce:       providedNonce,
 			},
 		}
-		expectedResponse := []*outport.AlteredAccount{
+		expectedResponse := []*alteredAccount.AlteredAccount{
 			{
 				Address: "alice",
 				Balance: "100000",
 			},
 		}
 
 		facade := &mock.FacadeStub{
-			GetAlteredAccountsForBlockCalled: func(options api.GetAlteredAccountsForBlockOptions) ([]*outport.AlteredAccount, error) {
+			GetAlteredAccountsForBlockCalled: func(options api.GetAlteredAccountsForBlockOptions) ([]*alteredAccount.AlteredAccount, error) {
 				require.Equal(t, expectedOptions, options)
 				return expectedResponse, nil
 			},
@@ -320,7 +320,7 @@ func TestBlockGroup_getAlteredAccountsByHash(t *testing.T) {
 
 		providedHash := hex.EncodeToString([]byte("hash"))
 		facade := &mock.FacadeStub{
-			GetAlteredAccountsForBlockCalled: func(options api.GetAlteredAccountsForBlockOptions) ([]*outport.AlteredAccount, error) {
+			GetAlteredAccountsForBlockCalled: func(options api.GetAlteredAccountsForBlockOptions) ([]*alteredAccount.AlteredAccount, error) {
 				return nil, expectedErr
 			},
 		}
@@ -344,15 +344,15 @@ func TestBlockGroup_getAlteredAccountsByHash(t *testing.T) {
 				Hash:        []byte("hash"),
 			},
 		}
-		expectedResponse := []*outport.AlteredAccount{
+		expectedResponse := []*alteredAccount.AlteredAccount{
 			{
 				Address: "alice",
 				Balance: "100000",
 			},
 		}
 
 		facade := &mock.FacadeStub{
-			GetAlteredAccountsForBlockCalled: func(options api.GetAlteredAccountsForBlockOptions) ([]*outport.AlteredAccount, error) {
+			GetAlteredAccountsForBlockCalled: func(options api.GetAlteredAccountsForBlockOptions) ([]*alteredAccount.AlteredAccount, error) {
 				require.Equal(t, providedHash, hex.EncodeToString(options.Hash))
 				require.Equal(t, expectedOptions, options)
 				return expectedResponse, nil

api/middleware/responseLogger.go

@@ -66,7 +66,7 @@ func (rlm *responseLoggerMiddleware) logRequestAndResponse(c *gin.Context, durat
 		reqBody := c.Request.Body
 		reqBodyBytes, err := ioutil.ReadAll(reqBody)
 		if err != nil {
-			log.Error(err.Error())
+			log.Debug(err.Error())
 			return
 		}
 

api/mock/facadeStub.go

@@ -5,9 +5,9 @@ import (
 	"math/big"
 
 	"github.com/multiversx/mx-chain-core-go/core"
+	"github.com/multiversx/mx-chain-core-go/data/alteredAccount"
 	"github.com/multiversx/mx-chain-core-go/data/api"
 	"github.com/multiversx/mx-chain-core-go/data/esdt"
-	outportcore "github.com/multiversx/mx-chain-core-go/data/outport"
 	"github.com/multiversx/mx-chain-core-go/data/transaction"
 	"github.com/multiversx/mx-chain-core-go/data/vm"
 	"github.com/multiversx/mx-chain-go/common"
@@ -56,7 +56,7 @@ type FacadeStub struct {
 	GetNFTTokenIDsRegisteredByAddressCalled     func(address string, options api.AccountQueryOptions) ([]string, api.BlockInfo, error)
 	GetBlockByHashCalled                        func(hash string, options api.BlockQueryOptions) (*api.Block, error)
 	GetBlockByNonceCalled                       func(nonce uint64, options api.BlockQueryOptions) (*api.Block, error)
-	GetAlteredAccountsForBlockCalled            func(options api.GetAlteredAccountsForBlockOptions) ([]*outportcore.AlteredAccount, error)
+	GetAlteredAccountsForBlockCalled            func(options api.GetAlteredAccountsForBlockOptions) ([]*alteredAccount.AlteredAccount, error)
 	GetBlockByRoundCalled                       func(round uint64, options api.BlockQueryOptions) (*api.Block, error)
 	GetInternalShardBlockByNonceCalled          func(format common.ApiOutputFormat, nonce uint64) (interface{}, error)
 	GetInternalShardBlockByHashCalled           func(format common.ApiOutputFormat, hash string) (interface{}, error)
@@ -411,7 +411,7 @@ func (f *FacadeStub) GetBlockByRound(round uint64, options api.BlockQueryOptions
 }
 
 // GetAlteredAccountsForBlock -
-func (f *FacadeStub) GetAlteredAccountsForBlock(options api.GetAlteredAccountsForBlockOptions) ([]*outportcore.AlteredAccount, error) {
+func (f *FacadeStub) GetAlteredAccountsForBlock(options api.GetAlteredAccountsForBlockOptions) ([]*alteredAccount.AlteredAccount, error) {
 	if f.GetAlteredAccountsForBlockCalled != nil {
 		return f.GetAlteredAccountsForBlockCalled(options)
 	}

api/shared/interface.go

@@ -5,9 +5,9 @@ import (
 
 	"github.com/gin-gonic/gin"
 	"github.com/multiversx/mx-chain-core-go/core"
+	"github.com/multiversx/mx-chain-core-go/data/alteredAccount"
 	"github.com/multiversx/mx-chain-core-go/data/api"
 	"github.com/multiversx/mx-chain-core-go/data/esdt"
-	outportcore "github.com/multiversx/mx-chain-core-go/data/outport"
 	"github.com/multiversx/mx-chain-core-go/data/transaction"
 	"github.com/multiversx/mx-chain-core-go/data/vm"
 	"github.com/multiversx/mx-chain-go/common"
@@ -77,7 +77,7 @@ type FacadeHandler interface {
 	GetBlockByHash(hash string, options api.BlockQueryOptions) (*api.Block, error)
 	GetBlockByNonce(nonce uint64, options api.BlockQueryOptions) (*api.Block, error)
 	GetBlockByRound(round uint64, options api.BlockQueryOptions) (*api.Block, error)
-	GetAlteredAccountsForBlock(options api.GetAlteredAccountsForBlockOptions) ([]*outportcore.AlteredAccount, error)
+	GetAlteredAccountsForBlock(options api.GetAlteredAccountsForBlockOptions) ([]*alteredAccount.AlteredAccount, error)
 	GetInternalShardBlockByNonce(format common.ApiOutputFormat, nonce uint64) (interface{}, error)
 	GetInternalShardBlockByHash(format common.ApiOutputFormat, hash string) (interface{}, error)
 	GetInternalShardBlockByRound(format common.ApiOutputFormat, round uint64) (interface{}, error)

cmd/keygenerator/converter/pidPubkeyConverter.go

@@ -6,11 +6,8 @@ import (
 	"github.com/multiversx/mx-chain-crypto-go/signing/secp256k1"
 	"github.com/multiversx/mx-chain-go/p2p"
 	"github.com/multiversx/mx-chain-go/p2p/factory"
-	logger "github.com/multiversx/mx-chain-logger-go"
 )
 
-var log = logger.GetOrCreate("cmd/keygenerator/converter")
-
 type pidPubkeyConverter struct {
 	keyGen          crypto.KeyGenerator
 	p2PKeyConverter p2p.P2PKeyConverter

cmd/node/config/config.toml

@@ -715,6 +715,9 @@
         "erd1najnxxweyw6plhg8efql330nttrj6l5cf87wqsuym85s9ha0hmdqnqgenp", #shard 2
     ]
     MaxNumAddressesInTransferRole = 100
+    DNSV2Addresses =[
+        "erd1qqqqqqqqqqqqqpgqr46jrxr6r2unaqh75ugd308dwx5vgnhwh47qtvepe3",
+    ]
 
 [Hardfork]
     EnableTrigger = true

cmd/node/config/enableEpochs.toml

@@ -248,15 +248,21 @@
     # MultiClaimOnDelegationEnableEpoch represents the epoch when the multi claim on delegation is enabled
     MultiClaimOnDelegationEnableEpoch = 3
 
-    # BLSMultiSignerEnableEpoch represents the activation epoch for different types of BLS multi-signers
-    BLSMultiSignerEnableEpoch = [
-        { EnableEpoch = 0, Type = "no-KOSK"},
-        { EnableEpoch = 1, Type = "KOSK"}
-    ]
+    # ChangeUsernameEnableEpoch represents the epoch when changing username is enabled
+    ChangeUsernameEnableEpoch = 2
 
     # SetGuardianEnableEpoch represents the epoch when the guard account feature is enabled in the protocol
     SetGuardianEnableEpoch = 2
 
+    # ConsistentTokensValuesLengthCheckEnableEpoch represents the epoch when the consistent tokens values length check is enabled
+    ConsistentTokensValuesLengthCheckEnableEpoch = 2
+
+    # BLSMultiSignerEnableEpoch represents the activation epoch for different types of BLS multi-signers
+    BLSMultiSignerEnableEpoch = [
+        { EnableEpoch = 0, Type = "no-KOSK" },
+        { EnableEpoch = 1, Type = "KOSK" }
+    ]
+
     # MaxNodesChangeEnableEpoch holds configuration for changing the maximum number of nodes and the enabling epoch
     MaxNodesChangeEnableEpoch = [
         { EpochEnable = 0, MaxNumNodes = 36, NodesToShufflePerShard = 4 },

cmd/node/config/external.toml

@@ -37,10 +37,23 @@
     # RequestTimeoutSec defines the timeout in seconds for the http client
     RequestTimeoutSec = 60
 
-[WebSocketConnector]
+    # MarshallerType is used to define the marshaller type to be used for inner
+    # marshalled structures in block events data
+    MarshallerType = "json"
+
+[HostDriverConfig]
     # This flag shall only be used for observer nodes
     Enabled = false
-    URL = "localhost:22111"
+    # This flag will start the WebSocket connector as server or client (can be "client" or "server")
+    Mode = "client"
+    # URL for the WebSocket client/server connection
+    # This value represents the IP address and port number that the WebSocket client or server will use to establish a connection.
+    URL = "127.0.0.1:22111"
+    # After a message will be sent it will wait for an ack message if this flag is enabled
     WithAcknowledge = true
     # Currently, only "json" is supported. In the future, "gogo protobuf" could also be supported
     MarshallerType = "json"
+    # The number of seconds when the client will try again to send the data
+    RetryDurationInSec = 5
+    # Sets if, in case of data payload processing error, we should block or not the advancement to the next processing event. Set this to true if you wish the node to stop processing blocks if the client/server encounters errors while processing requests.
+    BlockingAckOnError = false

cmd/node/config/prefs.toml

@@ -8,7 +8,7 @@
    # In multikey mode, all bls keys not mentioned in NamedIdentity section will use this one as default
    NodeDisplayName = ""
 
-   # Identity represents the keybase's identity when the node does not run in multikey mode
+   # Identity represents the keybase/GitHub identity when the node does not run in multikey mode
    # In multikey mode, all bls keys not mentioned in NamedIdentity section will use this one as default
    Identity = ""
 
@@ -50,10 +50,28 @@
    #    { File = "external.toml", Path = "ElasticSearchConnector.Enabled", Value = "true" }
    #]
 
+# BlockProcessingCutoff can be used to stop processing blocks at a certain round, nonce or epoch.
+# This can be useful for snapshotting different stuff and also for debugging purposes.
+[BlockProcessingCutoff]
+   # If set to true, the node will stop at the given coordinate
+   Enabled = false
+
+   # Mode represents the cutoff mode. possible values: "pause" or "process-error".
+   # "pause" mode will halt the processing at the block with the given coordinates. Useful for snapshots/analytics
+   # "process-error" will return an error when processing the block with the given coordinates. Useful for debugging
+   Mode = "pause"
+
+   # CutoffTrigger represents the kind of coordinate to look after when cutting off the processing.
+   # Possible values: "round", "nonce", or "epoch"
+   CutoffTrigger = "round"
+
+   # The minimum value of the cutoff. For example, if CutoffType is set to "round", and Value to 20, then the node will stop processing at round 20+
+   Value = 0
+
 # NamedIdentity represents an identity that runs nodes on the multikey
 # There can be multiple identities set on the same node, each one of them having different bls keys, just by duplicating the NamedIdentity
 [[NamedIdentity]]
-   # Identity represents the keybase identity for the current NamedIdentity
+   # Identity represents the keybase/GitHub identity for the current NamedIdentity
    Identity = ""
    # NodeName represents the name that will be given to the names of the current identity
    NodeName = ""

cmd/node/config/systemSmartContractsConfig.toml

@@ -17,7 +17,7 @@
     OwnerAddress = "erd1fpkcgel4gcmh8zqqdt043yfcn5tyx8373kg6q2qmkxzu4dqamc0swts65c"
 
 [GovernanceSystemSCConfig]
-    ChangeConfigAddress = "erd1vxy22x0fj4zv6hktmydg8vpfh6euv02cz4yg0aaws6rrad5a5awqgqky80" #should use a multisign contract instead of a wallet address
+    OwnerAddress = "erd1vxy22x0fj4zv6hktmydg8vpfh6euv02cz4yg0aaws6rrad5a5awqgqky80" #should use a multisign contract instead of a wallet address
     [GovernanceSystemSCConfig.V1]
         NumNodes           = 500
         ProposalCost       = "5000000000000000000" #5 eGLD
@@ -26,6 +26,7 @@
         MinVetoThreshold   = 50
     [GovernanceSystemSCConfig.Active]
         ProposalCost       = "1000000000000000000000" #1000 eGLD
+        LostProposalFee    = "10000000000000000000"   #10 eGLD
         MinQuorum          = 0.5  #fraction of value 0.5  - 50%
         MinPassThreshold   = 0.5  #fraction of value 0.5  - 50%
         MinVetoThreshold   = 0.33 #fraction of value 0.33 - 33%

cmd/node/flags.go

@@ -384,6 +384,12 @@ var (
 		Usage: "String flag for specifying the desired `operation mode`(s) of the node, resulting in altering some configuration values accordingly. Possible values are: snapshotless-observer, full-archive, db-lookup-extension, historical-balances or `\"\"` (empty). Multiple values can be separated via ,",
 		Value: "",
 	}
+
+	// repopulateTokensSupplies defines a flag that, if set, will repopulate the tokens supplies database by iterating over the trie
+	repopulateTokensSupplies = cli.BoolFlag{
+		Name:  "repopulate-tokens-supplies",
+		Usage: "Boolean flag for repopulating the tokens supplies database. It will delete the current data, iterate over the entire trie and add he new obtained supplies",
+	}
 )
 
 func getFlags() []cli.Flag {
@@ -443,6 +449,7 @@ func getFlags() []cli.Flag {
 		dbDirectory,
 		logsDirectory,
 		operationMode,
+		repopulateTokensSupplies,
 	}
 }
 
@@ -472,6 +479,7 @@ func getFlagsConfig(ctx *cli.Context, log logger.Logger) *config.ContextFlagsCon
 	flagsConfig.NoKeyProvided = ctx.GlobalBool(noKey.Name)
 	flagsConfig.SnapshotsEnabled = ctx.GlobalBool(snapshotsEnabled.Name)
 	flagsConfig.OperationMode = ctx.GlobalString(operationMode.Name)
+	flagsConfig.RepopulateTokensSupplies = ctx.GlobalBool(repopulateTokensSupplies.Name)
 
 	return flagsConfig
 }
@@ -578,6 +586,11 @@ func applyCompatibleConfigs(log logger.Logger, configs *config.Configs) error {
 		return fmt.Errorf("import-db-no-sig-check can only be used with the import-db flag")
 	}
 
+	if configs.PreferencesConfig.BlockProcessingCutoff.Enabled {
+		log.Debug("node is started by using the block processing cut-off - will disable the watchdog")
+		configs.FlagsConfig.DisableConsensusWatchdog = true
+	}
+
 	operationModes, err := operationmodes.ParseOperationModes(configs.FlagsConfig.OperationMode)
 	if err != nil {
 		return err

common/constants.go

@@ -730,10 +730,6 @@ const InvalidMessageBlacklistDuration = time.Second * 3600
 // rating to a minimum threshold due to improper messages
 const PublicKeyBlacklistDuration = time.Second * 7200
 
-// WrongP2PMessageBlacklistDuration represents the time to keep a peer id in the blacklist if it sends a message that
-// do not follow this protocol
-const WrongP2PMessageBlacklistDuration = time.Second * 7200
-
 // InvalidSigningBlacklistDuration defines the time to keep a peer id in blacklist if it signs a message with invalid signature
 const InvalidSigningBlacklistDuration = time.Second * 7200
 
@@ -774,9 +770,6 @@ const HardforkResolversIdentifier = "hardfork resolver"
 // EpochStartInterceptorsIdentifier represents the identifier that is used in the start-in-epoch process
 const EpochStartInterceptorsIdentifier = "epoch start interceptor"
 
-// GetNodeFromDBErrorString represents the string which is returned when a getting node from DB returns an error
-const GetNodeFromDBErrorString = "getNodeFromDB error"
-
 // TimeoutGettingTrieNodes defines the timeout in trie sync operation if no node is received
 const TimeoutGettingTrieNodes = 2 * time.Minute // to consider syncing a very large trie node of 64MB at ~1MB/s
 
@@ -831,6 +824,28 @@ const (
 	ApiOutputFormatProto ApiOutputFormat = 1
 )
 
+// BlockProcessingCutoffMode represents the type to be used to identify the mode of the block processing cutoff
+type BlockProcessingCutoffMode string
+
+const (
+	// BlockProcessingCutoffModePause represents the mode where the node will pause the processing at the given coordinates
+	BlockProcessingCutoffModePause = "pause"
+	// BlockProcessingCutoffModeProcessError represents the mode where the node will reprocess with error the block at the given coordinates
+	BlockProcessingCutoffModeProcessError = "process-error"
+)
+
+// BlockProcessingCutoffTrigger represents the trigger of the cutoff potentially used in block processing
+type BlockProcessingCutoffTrigger string
+
+const (
+	// BlockProcessingCutoffByNonce represents the cutoff by nonce
+	BlockProcessingCutoffByNonce BlockProcessingCutoffTrigger = "nonce"
+	// BlockProcessingCutoffByRound represents the cutoff by round
+	BlockProcessingCutoffByRound BlockProcessingCutoffTrigger = "round"
+	// BlockProcessingCutoffByEpoch represents the cutoff by epoch
+	BlockProcessingCutoffByEpoch BlockProcessingCutoffTrigger = "epoch"
+)
+
 // MaxIndexOfTxInMiniBlock defines the maximum index of a tx inside one mini block
 const MaxIndexOfTxInMiniBlock = int32(29999)
 

common/dtos.go

@@ -1,6 +1,8 @@
 package common
 
-import "github.com/multiversx/mx-chain-core-go/data/outport"
+import (
+	"github.com/multiversx/mx-chain-core-go/data/alteredAccount"
+)
 
 // GetProofResponse is a struct that stores the response of a GetProof API request
 type GetProofResponse struct {
@@ -71,5 +73,5 @@ type EpochStartDataAPI struct {
 
 // AlteredAccountsForBlockAPIResponse holds the altered accounts for a certain block
 type AlteredAccountsForBlockAPIResponse struct {
-	Accounts []*outport.AlteredAccount `json:"accounts"`
+	Accounts []*alteredAccount.AlteredAccount `json:"accounts"`
 }