Merge branch 'proposer-optimisation' of github.com:multiversx/mx-chai…

…n-go into proposer-optimisation

cmd/node/config/enableEpochs.toml

@@ -302,6 +302,10 @@
     # StakingV4Step3EnableEpoch represents the epoch in which selected nodes from auction will be distributed to waiting list
     StakingV4Step3EnableEpoch = 3
 
+    # CleanupAuctionOnLowWaitingListEnableEpoch represents the epoch when duplicated data cleanup from auction list is enabled in the condition of a low waiting list
+    # Should have the same value as StakingV4Step1EnableEpoch if the low waiting list has not happened, otherwise should have a greater value
+    CleanupAuctionOnLowWaitingListEnableEpoch = 1
+
     # AlwaysMergeContextsInEEIEnableEpoch represents the epoch in which the EEI will always merge the contexts
     AlwaysMergeContextsInEEIEnableEpoch = 1
 

common/constants.go

@@ -1011,6 +1011,7 @@ const (
 	StakingV4Step1Flag                                 core.EnableEpochFlag = "StakingV4Step1Flag"
 	StakingV4Step2Flag                                 core.EnableEpochFlag = "StakingV4Step2Flag"
 	StakingV4Step3Flag                                 core.EnableEpochFlag = "StakingV4Step3Flag"
+	CleanupAuctionOnLowWaitingListFlag                 core.EnableEpochFlag = "CleanupAuctionOnLowWaitingListFlag"
 	StakingV4StartedFlag                               core.EnableEpochFlag = "StakingV4StartedFlag"
 	AlwaysMergeContextsInEEIFlag                       core.EnableEpochFlag = "AlwaysMergeContextsInEEIFlag"
 	// all new flags must be added to createAllFlagsMap method, as part of enableEpochsHandler allFlagsDefined

common/enablers/enableEpochsHandler.go

@@ -6,10 +6,11 @@ import (
 
 	"github.com/multiversx/mx-chain-core-go/core"
 	"github.com/multiversx/mx-chain-core-go/core/check"
+	logger "github.com/multiversx/mx-chain-logger-go"
+
 	"github.com/multiversx/mx-chain-go/common"
 	"github.com/multiversx/mx-chain-go/config"
 	"github.com/multiversx/mx-chain-go/process"
-	logger "github.com/multiversx/mx-chain-logger-go"
 )
 
 var log = logger.GetOrCreate("common/enablers")
@@ -713,6 +714,12 @@ func (handler *enableEpochsHandler) createAllFlagsMap() {
 			},
 			activationEpoch: handler.enableEpochsConfig.StakingV4Step3EnableEpoch,
 		},
+		common.CleanupAuctionOnLowWaitingListFlag: {
+			isActiveInEpoch: func(epoch uint32) bool {
+				return epoch >= handler.enableEpochsConfig.CleanupAuctionOnLowWaitingListEnableEpoch
+			},
+			activationEpoch: handler.enableEpochsConfig.CleanupAuctionOnLowWaitingListEnableEpoch,
+		},
 		common.StakingV4StartedFlag: {
 			isActiveInEpoch: func(epoch uint32) bool {
 				return epoch >= handler.enableEpochsConfig.StakingV4Step1EnableEpoch

common/enablers/enableEpochsHandler_test.go

@@ -5,13 +5,14 @@ import (
 	"testing"
 
 	"github.com/multiversx/mx-chain-core-go/core/check"
+	vmcommon "github.com/multiversx/mx-chain-vm-common-go"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
 	"github.com/multiversx/mx-chain-go/common"
 	"github.com/multiversx/mx-chain-go/config"
 	"github.com/multiversx/mx-chain-go/process"
 	"github.com/multiversx/mx-chain-go/testscommon/epochNotifier"
-	vmcommon "github.com/multiversx/mx-chain-vm-common-go"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
 )
 
 func createEnableEpochsConfig() config.EnableEpochs {
@@ -113,6 +114,7 @@ func createEnableEpochsConfig() config.EnableEpochs {
 		StakingV4Step1EnableEpoch:                                96,
 		StakingV4Step2EnableEpoch:                                97,
 		StakingV4Step3EnableEpoch:                                98,
+		CleanupAuctionOnLowWaitingListEnableEpoch:                96,
 		AlwaysMergeContextsInEEIEnableEpoch:                      99,
 	}
 }
@@ -426,6 +428,7 @@ func TestEnableEpochsHandler_GetActivationEpoch(t *testing.T) {
 	require.Equal(t, cfg.StakingV4Step1EnableEpoch, handler.GetActivationEpoch(common.StakingV4Step1Flag))
 	require.Equal(t, cfg.StakingV4Step2EnableEpoch, handler.GetActivationEpoch(common.StakingV4Step2Flag))
 	require.Equal(t, cfg.StakingV4Step3EnableEpoch, handler.GetActivationEpoch(common.StakingV4Step3Flag))
+	require.Equal(t, cfg.CleanupAuctionOnLowWaitingListEnableEpoch, handler.GetActivationEpoch(common.CleanupAuctionOnLowWaitingListFlag))
 	require.Equal(t, cfg.StakingV4Step1EnableEpoch, handler.GetActivationEpoch(common.StakingV4StartedFlag))
 	require.Equal(t, cfg.AlwaysMergeContextsInEEIEnableEpoch, handler.GetActivationEpoch(common.AlwaysMergeContextsInEEIFlag))
 }

config/epochConfig.go

@@ -112,6 +112,7 @@ type EnableEpochs struct {
 	StakingV4Step1EnableEpoch                                uint32
 	StakingV4Step2EnableEpoch                                uint32
 	StakingV4Step3EnableEpoch                                uint32
+	CleanupAuctionOnLowWaitingListEnableEpoch                uint32
 	AlwaysMergeContextsInEEIEnableEpoch                      uint32
 	BLSMultiSignerEnableEpoch                                []MultiSignerConfig
 }

config/tomlConfig_test.go

@@ -5,10 +5,11 @@ import (
 	"strconv"
 	"testing"
 
-	p2pConfig "github.com/multiversx/mx-chain-go/p2p/config"
 	"github.com/pelletier/go-toml"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+
+	p2pConfig "github.com/multiversx/mx-chain-go/p2p/config"
 )
 
 func TestTomlParser(t *testing.T) {
@@ -843,6 +844,9 @@ func TestEnableEpochConfig(t *testing.T) {
     # AlwaysMergeContextsInEEIEnableEpoch represents the epoch in which the EEI will always merge the contexts
     AlwaysMergeContextsInEEIEnableEpoch = 94
 
+    # CleanupAuctionOnLowWaitingListEnableEpoch represents the epoch when the cleanup auction on low waiting list is enabled
+    CleanupAuctionOnLowWaitingListEnableEpoch = 95
+
     # MaxNodesChangeEnableEpoch holds configuration for changing the maximum number of nodes and the enabling epoch
     MaxNodesChangeEnableEpoch = [
         { EpochEnable = 44, MaxNumNodes = 2169, NodesToShufflePerShard = 80 },
@@ -955,6 +959,7 @@ func TestEnableEpochConfig(t *testing.T) {
 			MigrateDataTrieEnableEpoch:                               92,
 			CurrentRandomnessOnSortingEnableEpoch:                    93,
 			AlwaysMergeContextsInEEIEnableEpoch:                      94,
+			CleanupAuctionOnLowWaitingListEnableEpoch:                95,
 			MaxNodesChangeEnableEpoch: []MaxNodesChangeConfig{
 				{
 					EpochEnable:            44,

epochStart/bootstrap/disabled/disabledNodesCoordinator.go

@@ -54,6 +54,11 @@ func (n *nodesCoordinator) GetAllShuffledOutValidatorsPublicKeys(_ uint32) (map[
 	return nil, nil
 }
 
+// GetShuffledOutToAuctionValidatorsPublicKeys -
+func (n *nodesCoordinator) GetShuffledOutToAuctionValidatorsPublicKeys(_ uint32) (map[uint32][][]byte, error) {
+	return nil, nil
+}
+
 // GetConsensusValidatorsPublicKeys -
 func (n *nodesCoordinator) GetConsensusValidatorsPublicKeys(_ []byte, _ uint64, _ uint32, _ uint32) ([]string, error) {
 	return nil, nil

epochStart/metachain/stakingDataProvider.go

@@ -580,7 +580,24 @@ func (sdp *stakingDataProvider) GetCurrentEpochValidatorStats() epochStart.Valid
 	sdp.mutStakingData.RLock()
 	defer sdp.mutStakingData.RUnlock()
 
-	return sdp.validatorStatsInEpoch
+	return copyValidatorStatsInEpoch(sdp.validatorStatsInEpoch)
+}
+
+func copyValidatorStatsInEpoch(oldInstance epochStart.ValidatorStatsInEpoch) epochStart.ValidatorStatsInEpoch {
+	return epochStart.ValidatorStatsInEpoch{
+		Eligible: copyMap(oldInstance.Eligible),
+		Waiting:  copyMap(oldInstance.Waiting),
+		Leaving:  copyMap(oldInstance.Leaving),
+	}
+}
+
+func copyMap(oldMap map[uint32]int) map[uint32]int {
+	newMap := make(map[uint32]int, len(oldMap))
+	for key, value := range oldMap {
+		newMap[key] = value
+	}
+
+	return newMap
 }
 
 // IsInterfaceNil return true if underlying object is nil

epochStart/metachain/stakingDataProvider_test.go

@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"math/big"
 	"strings"
+	"sync"
 	"testing"
 
 	"github.com/multiversx/mx-chain-core-go/core"
@@ -465,16 +466,58 @@ func TestStakingDataProvider_PrepareStakingDataForRewards(t *testing.T) {
 func TestStakingDataProvider_FillValidatorInfo(t *testing.T) {
 	t.Parallel()
 
-	owner := []byte("owner")
-	topUpVal := big.NewInt(828743)
-	basePrice := big.NewInt(100000)
-	stakeVal := big.NewInt(0).Add(topUpVal, basePrice)
-	numRunContractCalls := 0
+	t.Run("should work", func(t *testing.T) {
+		t.Parallel()
 
-	sdp := createStakingDataProviderWithMockArgs(t, owner, topUpVal, stakeVal, &numRunContractCalls)
+		owner := []byte("owner")
+		topUpVal := big.NewInt(828743)
+		basePrice := big.NewInt(100000)
+		stakeVal := big.NewInt(0).Add(topUpVal, basePrice)
+		numRunContractCalls := 0
 
-	err := sdp.FillValidatorInfo(&state.ValidatorInfo{PublicKey: []byte("bls key")})
-	require.NoError(t, err)
+		sdp := createStakingDataProviderWithMockArgs(t, owner, topUpVal, stakeVal, &numRunContractCalls)
+
+		err := sdp.FillValidatorInfo(&state.ValidatorInfo{PublicKey: []byte("bls key")})
+		require.NoError(t, err)
+	})
+	t.Run("concurrent calls should work", func(t *testing.T) {
+		t.Parallel()
+
+		owner := []byte("owner")
+		topUpVal := big.NewInt(828743)
+		basePrice := big.NewInt(100000)
+		stakeVal := big.NewInt(0).Add(topUpVal, basePrice)
+		numRunContractCalls := 0
+
+		sdp := createStakingDataProviderWithMockArgs(t, owner, topUpVal, stakeVal, &numRunContractCalls)
+
+		wg := sync.WaitGroup{}
+		numCalls := 100
+		wg.Add(numCalls)
+
+		require.NotPanics(t, func() {
+			for i := 0; i < numCalls; i++ {
+				go func(idx int) {
+					switch idx % 2 {
+					case 0:
+						err := sdp.FillValidatorInfo(&state.ValidatorInfo{
+							PublicKey: []byte("bls key"),
+							List:      string(common.EligibleList),
+							ShardId:   0,
+						})
+						require.NoError(t, err)
+					case 1:
+						stats := sdp.GetCurrentEpochValidatorStats()
+						log.Info(fmt.Sprintf("%d", stats.Eligible[0]))
+					}
+
+					wg.Done()
+				}(i)
+			}
+
+			wg.Wait()
+		})
+	})
 }
 
 func TestCheckAndFillOwnerValidatorAuctionData(t *testing.T) {

integrationTests/chainSimulator/staking/stake/simpleStake_test.go

@@ -9,6 +9,8 @@ import (
 
 	"github.com/multiversx/mx-chain-core-go/core"
 	"github.com/multiversx/mx-chain-core-go/data/transaction"
+	"github.com/stretchr/testify/require"
+
 	"github.com/multiversx/mx-chain-go/common"
 	"github.com/multiversx/mx-chain-go/config"
 	"github.com/multiversx/mx-chain-go/integrationTests/chainSimulator/staking"
@@ -17,7 +19,6 @@ import (
 	"github.com/multiversx/mx-chain-go/node/chainSimulator/configs"
 	"github.com/multiversx/mx-chain-go/node/chainSimulator/process"
 	"github.com/multiversx/mx-chain-go/vm"
-	"github.com/stretchr/testify/require"
 )
 
 // Test scenarios
@@ -261,30 +262,30 @@ func TestChainSimulator_StakingV4Step2APICalls(t *testing.T) {
 		err = cs.GenerateBlocks(2)
 		require.Nil(t, err)
 
-		numQualified, numUnQualified := getNumQualifiedAndUnqualified(t, metachainNode)
-		require.Equal(t, 8, numQualified)
-		require.Equal(t, 1, numUnQualified)
+		qualified, unQualified := getQualifiedAndUnqualifiedNodes(t, metachainNode)
+		require.Equal(t, 8, len(qualified))
+		require.Equal(t, 1, len(unQualified))
 	}
 }
 
-func getNumQualifiedAndUnqualified(t *testing.T, metachainNode process.NodeHandler) (int, int) {
+func getQualifiedAndUnqualifiedNodes(t *testing.T, metachainNode process.NodeHandler) ([]string, []string) {
 	err := metachainNode.GetProcessComponents().ValidatorsProvider().ForceUpdate()
 	require.Nil(t, err)
 	auctionList, err := metachainNode.GetProcessComponents().ValidatorsProvider().GetAuctionList()
 	require.Nil(t, err)
 
-	numQualified := 0
-	numUnQualified := 0
+	qualified := make([]string, 0)
+	unQualified := make([]string, 0)
 
 	for _, auctionOwnerData := range auctionList {
 		for _, auctionNode := range auctionOwnerData.Nodes {
 			if auctionNode.Qualified {
-				numQualified++
+				qualified = append(qualified, auctionNode.BlsKey)
 			} else {
-				numUnQualified++
+				unQualified = append(unQualified, auctionNode.BlsKey)
 			}
 		}
 	}
 
-	return numQualified, numUnQualified
+	return qualified, unQualified
 }