Merge rc 1.6.0 trie sync optimizations

api/errors/errors.go

@@ -162,3 +162,6 @@ var ErrRegisteredNFTTokenIDs = errors.New("getting registered nft token ids erro
 
 // ErrInvalidRole signals that an invalid role was provided
 var ErrInvalidRole = errors.New("invalid role")
+
+// ErrIsDataTrieMigrated signals that an error occurred while trying to verify the migration status of the data trie
+var ErrIsDataTrieMigrated = errors.New("could not verify the migration status of the data trie")

api/groups/addressGroup.go

@@ -17,26 +17,27 @@ import (
 )
 
 const (
-	getAccountPath            = "/:address"
-	getAccountsPath           = "/bulk"
-	getBalancePath            = "/:address/balance"
-	getUsernamePath           = "/:address/username"
-	getCodeHashPath           = "/:address/code-hash"
-	getKeysPath               = "/:address/keys"
-	getKeyPath                = "/:address/key/:key"
-	getESDTTokensPath         = "/:address/esdt"
-	getESDTBalancePath        = "/:address/esdt/:tokenIdentifier"
-	getESDTTokensWithRolePath = "/:address/esdts-with-role/:role"
-	getESDTsRolesPath         = "/:address/esdts/roles"
-	getRegisteredNFTsPath     = "/:address/registered-nfts"
-	getESDTNFTDataPath        = "/:address/nft/:tokenIdentifier/nonce/:nonce"
-	getGuardianData           = "/:address/guardian-data"
-	urlParamOnFinalBlock      = "onFinalBlock"
-	urlParamOnStartOfEpoch    = "onStartOfEpoch"
-	urlParamBlockNonce        = "blockNonce"
-	urlParamBlockHash         = "blockHash"
-	urlParamBlockRootHash     = "blockRootHash"
-	urlParamHintEpoch         = "hintEpoch"
+	getAccountPath                 = "/:address"
+	getAccountsPath                = "/bulk"
+	getBalancePath                 = "/:address/balance"
+	getUsernamePath                = "/:address/username"
+	getCodeHashPath                = "/:address/code-hash"
+	getKeysPath                    = "/:address/keys"
+	getKeyPath                     = "/:address/key/:key"
+	getDataTrieMigrationStatusPath = "/:address/is-data-trie-migrated"
+	getESDTTokensPath              = "/:address/esdt"
+	getESDTBalancePath             = "/:address/esdt/:tokenIdentifier"
+	getESDTTokensWithRolePath      = "/:address/esdts-with-role/:role"
+	getESDTsRolesPath              = "/:address/esdts/roles"
+	getRegisteredNFTsPath          = "/:address/registered-nfts"
+	getESDTNFTDataPath             = "/:address/nft/:tokenIdentifier/nonce/:nonce"
+	getGuardianData                = "/:address/guardian-data"
+	urlParamOnFinalBlock           = "onFinalBlock"
+	urlParamOnStartOfEpoch         = "onStartOfEpoch"
+	urlParamBlockNonce             = "blockNonce"
+	urlParamBlockHash              = "blockHash"
+	urlParamBlockRootHash          = "blockRootHash"
+	urlParamHintEpoch              = "hintEpoch"
 )
 
 // addressFacadeHandler defines the methods to be implemented by a facade for handling address requests
@@ -54,6 +55,7 @@ type addressFacadeHandler interface {
 	GetAllESDTTokens(address string, options api.AccountQueryOptions) (map[string]*esdt.ESDigitalToken, api.BlockInfo, error)
 	GetKeyValuePairs(address string, options api.AccountQueryOptions) (map[string]string, api.BlockInfo, error)
 	GetGuardianData(address string, options api.AccountQueryOptions) (api.GuardianData, api.BlockInfo, error)
+	IsDataTrieMigrated(address string, options api.AccountQueryOptions) (bool, error)
 	IsInterfaceNil() bool
 }
 
@@ -164,6 +166,11 @@ func NewAddressGroup(facade addressFacadeHandler) (*addressGroup, error) {
 			Method:  http.MethodGet,
 			Handler: ag.getGuardianData,
 		},
+		{
+			Path:    getDataTrieMigrationStatusPath,
+			Method:  http.MethodGet,
+			Handler: ag.isDataTrieMigrated,
+		},
 	}
 	ag.endpoints = endpoints
 
@@ -442,6 +449,29 @@ func (ag *addressGroup) getAllESDTData(c *gin.Context) {
 	shared.RespondWithSuccess(c, gin.H{"esdts": formattedTokens, "blockInfo": blockInfo})
 }
 
+// isDataTrieMigrated returns true if the data trie is migrated for the given address
+func (ag *addressGroup) isDataTrieMigrated(c *gin.Context) {
+	addr := c.Param("address")
+	if addr == "" {
+		shared.RespondWithValidationError(c, errors.ErrIsDataTrieMigrated, errors.ErrEmptyAddress)
+		return
+	}
+
+	options, err := extractAccountQueryOptions(c)
+	if err != nil {
+		shared.RespondWithValidationError(c, errors.ErrIsDataTrieMigrated, err)
+		return
+	}
+
+	isMigrated, err := ag.getFacade().IsDataTrieMigrated(addr, options)
+	if err != nil {
+		shared.RespondWithInternalError(c, errors.ErrIsDataTrieMigrated, err)
+		return
+	}
+
+	shared.RespondWithSuccess(c, gin.H{"isMigrated": isMigrated})
+}
+
 func buildTokenDataApiResponse(tokenIdentifier string, esdtData *esdt.ESDigitalToken) *esdtNFTTokenData {
 	tokenData := &esdtNFTTokenData{
 		TokenIdentifier: tokenIdentifier,

api/groups/addressGroup_test.go

@@ -1150,8 +1150,93 @@ func getAddressRoutesConfig() config.ApiRoutesConfig {
 					{Name: "/:address/nft/:tokenIdentifier/nonce/:nonce", Open: true},
 					{Name: "/:address/esdts-with-role/:role", Open: true},
 					{Name: "/:address/registered-nfts", Open: true},
+					{Name: "/:address/is-data-trie-migrated", Open: true},
 				},
 			},
 		},
 	}
 }
+
+func TestIsDataTrieMigrated(t *testing.T) {
+	t.Parallel()
+
+	testAddress := "address"
+	expectedErr := errors.New("expected error")
+
+	t.Run("should return error if IsDataTrieMigrated returns error", func(t *testing.T) {
+		t.Parallel()
+
+		facade := mock.FacadeStub{
+			IsDataTrieMigratedCalled: func(address string, _ api.AccountQueryOptions) (bool, error) {
+				return false, expectedErr
+			},
+		}
+
+		addrGroup, err := groups.NewAddressGroup(&facade)
+		require.NoError(t, err)
+		ws := startWebServer(addrGroup, "address", getAddressRoutesConfig())
+
+		req, _ := http.NewRequest("GET", fmt.Sprintf("/address/%s/is-data-trie-migrated", testAddress), nil)
+		resp := httptest.NewRecorder()
+		ws.ServeHTTP(resp, req)
+
+		response := shared.GenericAPIResponse{}
+		loadResponse(resp.Body, &response)
+		assert.Equal(t, http.StatusInternalServerError, resp.Code)
+		assert.True(t, strings.Contains(response.Error, expectedErr.Error()))
+	})
+
+	t.Run("should return true if IsDataTrieMigrated returns true", func(t *testing.T) {
+		t.Parallel()
+
+		facade := mock.FacadeStub{
+			IsDataTrieMigratedCalled: func(address string, _ api.AccountQueryOptions) (bool, error) {
+				return true, nil
+			},
+		}
+
+		addrGroup, err := groups.NewAddressGroup(&facade)
+		require.NoError(t, err)
+		ws := startWebServer(addrGroup, "address", getAddressRoutesConfig())
+
+		req, _ := http.NewRequest("GET", fmt.Sprintf("/address/%s/is-data-trie-migrated", testAddress), nil)
+		resp := httptest.NewRecorder()
+		ws.ServeHTTP(resp, req)
+
+		response := shared.GenericAPIResponse{}
+		loadResponse(resp.Body, &response)
+		assert.Equal(t, http.StatusOK, resp.Code)
+		assert.True(t, response.Error == "")
+
+		respData, ok := response.Data.(map[string]interface{})
+		assert.True(t, ok)
+		assert.True(t, respData["isMigrated"].(bool))
+	})
+
+	t.Run("should return false if IsDataTrieMigrated returns false", func(t *testing.T) {
+		t.Parallel()
+
+		facade := mock.FacadeStub{
+			IsDataTrieMigratedCalled: func(address string, _ api.AccountQueryOptions) (bool, error) {
+				return false, nil
+			},
+		}
+
+		addrGroup, err := groups.NewAddressGroup(&facade)
+		require.NoError(t, err)
+		ws := startWebServer(addrGroup, "address", getAddressRoutesConfig())
+
+		req, _ := http.NewRequest("GET", fmt.Sprintf("/address/%s/is-data-trie-migrated", testAddress), nil)
+		resp := httptest.NewRecorder()
+		ws.ServeHTTP(resp, req)
+
+		response := shared.GenericAPIResponse{}
+		loadResponse(resp.Body, &response)
+		assert.Equal(t, http.StatusOK, resp.Code)
+		assert.True(t, response.Error == "")
+
+		respData, ok := response.Data.(map[string]interface{})
+		assert.True(t, ok)
+		assert.False(t, respData["isMigrated"].(bool))
+	})
+}

api/mock/facadeStub.go

@@ -87,6 +87,7 @@ type FacadeStub struct {
 	RestAPIServerDebugModeCalled                func() bool
 	PprofEnabledCalled                          func() bool
 	DecodeAddressPubkeyCalled                   func(pk string) ([]byte, error)
+	IsDataTrieMigratedCalled                    func(address string, options api.AccountQueryOptions) (bool, error)
 }
 
 // GetTokenSupply -
@@ -553,6 +554,15 @@ func (f *FacadeStub) GetInternalStartOfEpochValidatorsInfo(epoch uint32) ([]*sta
 	return nil, nil
 }
 
+// IsDataTrieMigrated -
+func (f *FacadeStub) IsDataTrieMigrated(address string, options api.AccountQueryOptions) (bool, error) {
+	if f.IsDataTrieMigratedCalled != nil {
+		return f.IsDataTrieMigratedCalled(address, options)
+	}
+
+	return false, nil
+}
+
 // Trigger -
 func (f *FacadeStub) Trigger(_ uint32, _ bool) error {
 	return nil

api/shared/interface.go

@@ -126,5 +126,6 @@ type FacadeHandler interface {
 	GetTransactionsPoolForSender(sender, fields string) (*common.TransactionsPoolForSenderApiResponse, error)
 	GetLastPoolNonceForSender(sender string) (uint64, error)
 	GetTransactionsPoolNonceGapsForSender(sender string) (*common.TransactionsPoolNonceGapsForSenderApiResponse, error)
+	IsDataTrieMigrated(address string, options api.AccountQueryOptions) (bool, error)
 	IsInterfaceNil() bool
 }

cmd/node/config/api.toml

@@ -83,7 +83,10 @@
         { Name = "/:address/esdts-with-role/:role", Open = true },
 
         # /address/:address/registered-nfts will return the token identifiers of the tokens registered by the address
-        { Name = "/:address/registered-nfts", Open = true }
+        { Name = "/:address/registered-nfts", Open = true },
+
+        # /address/:address/is-data-trie-migrated will return the status of the data trie migration for the given address
+        { Name = "/:address/is-data-trie-migrated", Open = true }
     ]
 
 [APIPackages.hardfork]

cmd/node/config/enableEpochs.toml

@@ -224,6 +224,9 @@
     # RuntimeMemStoreLimitEnableEpoch represents the epoch when the condition for Runtime MemStore is enabled
     RuntimeMemStoreLimitEnableEpoch = 1
 
+    # AutoBalanceDataTriesEnableEpoch represents the epoch when the data tries are automatically balanced by inserting at the hashed key instead of the normal key
+    AutoBalanceDataTriesEnableEpoch = 5
+
     # SetSenderInEeiOutputTransferEnableEpoch represents the epoch when setting the sender in eei output transfers will be enabled
     SetSenderInEeiOutputTransferEnableEpoch = 1
 

cmd/node/config/gasSchedules/gasScheduleV1.toml

@@ -18,6 +18,8 @@
     SetGuardian              = 250000
     GuardAccount             = 250000
     UnGuardAccount           = 250000
+    TrieLoadPerNode          = 20000
+    TrieStorePerNode         = 50000
 
 [MetaChainSystemSCsCost]
     Stake                 = 5000000

cmd/node/config/gasSchedules/gasScheduleV2.toml

@@ -18,6 +18,8 @@
     SetGuardian              = 250000
     GuardAccount             = 250000
     UnGuardAccount           = 250000
+    TrieLoadPerNode          = 20000
+    TrieStorePerNode         = 50000
 
 [MetaChainSystemSCsCost]
     Stake                 = 5000000

cmd/node/config/gasSchedules/gasScheduleV3.toml

@@ -18,6 +18,8 @@
     SetGuardian              = 250000
     GuardAccount             = 250000
     UnGuardAccount           = 250000
+    TrieLoadPerNode          = 20000
+    TrieStorePerNode         = 50000
 
 [MetaChainSystemSCsCost]
     Stake                 = 5000000

cmd/node/config/gasSchedules/gasScheduleV4.toml

@@ -18,6 +18,8 @@
     SetGuardian              = 250000
     GuardAccount             = 250000
     UnGuardAccount           = 250000
+    TrieLoadPerNode          = 20000
+    TrieStorePerNode         = 50000
 
 [MetaChainSystemSCsCost]
     Stake                 = 5000000

cmd/node/config/gasSchedules/gasScheduleV5.toml

@@ -18,6 +18,8 @@
     SetGuardian              = 250000
     GuardAccount             = 250000
     UnGuardAccount           = 250000
+    TrieLoadPerNode          = 20000
+    TrieStorePerNode         = 50000
 
 [MetaChainSystemSCsCost]
     Stake                 = 5000000

cmd/node/config/gasSchedules/gasScheduleV6.toml

@@ -18,6 +18,8 @@
     SetGuardian              = 250000
     GuardAccount             = 250000
     UnGuardAccount           = 250000
+    TrieLoadPerNode          = 20000
+    TrieStorePerNode         = 50000
 
 [MetaChainSystemSCsCost]
     Stake                 = 5000000

cmd/node/config/gasSchedules/gasScheduleV7.toml

@@ -19,6 +19,8 @@
     SetGuardian              = 250000
     GuardAccount             = 250000
     UnGuardAccount           = 250000
+    TrieLoadPerNode          = 20000
+    TrieStorePerNode         = 50000
 
 [MetaChainSystemSCsCost]
     Stake                 = 5000000

common/converters_test.go

@@ -16,6 +16,7 @@ import (
 	"github.com/multiversx/mx-chain-go/common/mock"
 	"github.com/multiversx/mx-chain-go/testscommon"
 	"github.com/multiversx/mx-chain-go/testscommon/hashingMocks"
+	"github.com/multiversx/mx-chain-go/testscommon/marshallerMock"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -122,7 +123,7 @@ func TestCalculateHash_Good(t *testing.T) {
 	marshaledData := "marshalized random string"
 	hashedData := "hashed marshalized random string"
 	hash, err := core.CalculateHash(
-		&testscommon.MarshalizerStub{
+		&marshallerMock.MarshalizerStub{
 			MarshalCalled: func(obj interface{}) ([]byte, error) {
 				marshalCalled = true
 				assert.Equal(t, initialObject, obj)

common/enablers/enableEpochsHandler.go

@@ -123,6 +123,7 @@ func (handler *enableEpochsHandler) EpochConfirmed(epoch uint32, _ uint64) {
 	handler.setFlagValue(epoch >= handler.enableEpochsConfig.KeepExecOrderOnCreatedSCRsEnableEpoch, handler.keepExecOrderOnCreatedSCRsFlag, "keepExecOrderOnCreatedSCRsFlag", epoch, handler.enableEpochsConfig.KeepExecOrderOnCreatedSCRsEnableEpoch)
 	handler.setFlagValue(epoch >= handler.enableEpochsConfig.ChangeUsernameEnableEpoch, handler.changeUsernameFlag, "changeUsername", epoch, handler.enableEpochsConfig.ChangeUsernameEnableEpoch)
 	handler.setFlagValue(epoch >= handler.enableEpochsConfig.ConsistentTokensValuesLengthCheckEnableEpoch, handler.consistentTokensValuesCheckFlag, "consistentTokensValuesCheckFlag", epoch, handler.enableEpochsConfig.ConsistentTokensValuesLengthCheckEnableEpoch)
+	handler.setFlagValue(epoch >= handler.enableEpochsConfig.AutoBalanceDataTriesEnableEpoch, handler.autoBalanceDataTriesFlag, "autoBalanceDataTriesFlag", epoch, handler.enableEpochsConfig.AutoBalanceDataTriesEnableEpoch)
 }
 
 func (handler *enableEpochsHandler) setFlagValue(value bool, flag *atomic.Flag, flagName string, epoch uint32, flagEpoch uint32) {

common/enablers/enableEpochsHandler_test.go

@@ -94,6 +94,7 @@ func createEnableEpochsConfig() config.EnableEpochs {
 		MultiClaimOnDelegationEnableEpoch:                 78,
 		KeepExecOrderOnCreatedSCRsEnableEpoch:             79,
 		ChangeUsernameEnableEpoch:                         80,
+		AutoBalanceDataTriesEnableEpoch:                   81,
 	}
 }
 
@@ -220,11 +221,12 @@ func TestNewEnableEpochsHandler_EpochConfirmed(t *testing.T) {
 		assert.False(t, handler.IsKeepExecOrderOnCreatedSCRsEnabled())
 		assert.False(t, handler.IsMultiClaimOnDelegationEnabled())
 		assert.False(t, handler.IsChangeUsernameEnabled())
+		assert.False(t, handler.IsAutoBalanceDataTriesEnabled())
 	})
 	t.Run("flags with == condition should be set, along with all >=", func(t *testing.T) {
 		t.Parallel()
 
-		epoch := uint32(80)
+		epoch := uint32(81)
 		cfg := createEnableEpochsConfig()
 		cfg.StakingV2EnableEpoch = epoch
 		cfg.ESDTEnableEpoch = epoch
@@ -322,6 +324,7 @@ func TestNewEnableEpochsHandler_EpochConfirmed(t *testing.T) {
 		assert.True(t, handler.IsRuntimeCodeSizeFixEnabled())
 		assert.True(t, handler.IsKeepExecOrderOnCreatedSCRsEnabled())
 		assert.True(t, handler.IsChangeUsernameEnabled())
+		assert.True(t, handler.IsAutoBalanceDataTriesEnabled())
 	})
 	t.Run("flags with < should be set", func(t *testing.T) {
 		t.Parallel()
@@ -419,5 +422,6 @@ func TestNewEnableEpochsHandler_EpochConfirmed(t *testing.T) {
 		assert.False(t, handler.IsRuntimeCodeSizeFixEnabled())
 		assert.False(t, handler.IsKeepExecOrderOnCreatedSCRsEnabled())
 		assert.False(t, handler.IsChangeUsernameEnabled())
+		assert.False(t, handler.IsAutoBalanceDataTriesEnabled())
 	})
 }