Add incentive for voter (neo-project#1848)

* add reward key

* incentive for voter

* optimize

* optimize

* optimize

* fix CalculateBonus and add ut

* format

* Item2 -> Votes

* Move fields

* Fix GetCommitteeVotes()

* specify variable name

* Remove double ToArray

* fix ut

* remove

* fix

* fix

* remove useless code

* optimize

* fix

* format

* Add using to dispose enumerators

* Update NeoToken.cs

* Update NeoToken.cs

* fix

* fix CalculateBonus

* Usng asending order

* Update NeoToken.cs

* fix

* optimize findrange

* format

* optimize

* Update NeoToken.cs

* fix

* format

* Use GetCommitteeFromCache

* Ensure registered

* fix PostPersist

* Update NeoToken.cs

* cache committee votes

* cache committee

* Update src/neo/Ledger/StorageItem.cs

Co-authored-by: Erik Zhang <erik@neo.org>

* fix PostPersist

* fix ShouldRefreshCommittee & optimize PostPersist

* add CommitteeEpoch

* fix CalculateBonus

* fix PostPersist

* optimize CalculateBonus

* optimize PostPersist

* Update NeoToken.cs

* add comments for trigger github action

* Allow standby can be voted

* optimize

* rename local variable

* Optimize Unregister Candidate

* Revert

* add ut

* fix ut

* format

* Conflicts

* Update src/neo/SmartContract/Native/Tokens/NeoToken.cs

Co-authored-by: Erik Zhang <erik@neo.org>

Co-authored-by: Tommo-L <luchuan@neo.org>
Co-authored-by: Erik Zhang <erik@neo.org>
Co-authored-by: Vitor Nazário Coelho <vncoelho@gmail.com>
Co-authored-by: Shargon <shargon@gmail.com>

src/neo/SmartContract/Native/Tokens/NeoToken.cs

@@ -29,6 +29,7 @@ public sealed class NeoToken : Nep5Token<NeoToken.NeoAccountState>
         private const byte Prefix_Candidate = 33;
         private const byte Prefix_Committee = 14;
         private const byte Prefix_GasPerBlock = 29;
+        private const byte Prefix_VoterRewardPerCommittee = 23;
 
         private const byte NeoHolderRewardRatio = 10;
         private const byte CommitteeRewardRatio = 5;
@@ -53,24 +54,41 @@ protected override void OnBalanceChanging(ApplicationEngine engine, UInt160 acco
             StorageKey key = CreateStorageKey(Prefix_Candidate).Add(state.VoteTo);
             CandidateState candidate = engine.Snapshot.Storages.GetAndChange(key).GetInteroperable<CandidateState>();
             candidate.Votes += amount;
-            CheckCandidate(engine.Snapshot, key, candidate);
+            CheckCandidate(engine.Snapshot, state.VoteTo, candidate);
         }
 
         private void DistributeGas(ApplicationEngine engine, UInt160 account, NeoAccountState state)
         {
             // PersistingBlock is null when running under the debugger
             if (engine.Snapshot.PersistingBlock == null) return;
 
-            BigInteger gas = CalculateBonus(engine.Snapshot, state.Balance, state.BalanceHeight, engine.Snapshot.PersistingBlock.Index);
+            BigInteger gas = CalculateBonus(engine.Snapshot, state.VoteTo, state.Balance, state.BalanceHeight, engine.Snapshot.PersistingBlock.Index);
             state.BalanceHeight = engine.Snapshot.PersistingBlock.Index;
             GAS.Mint(engine, account, gas);
         }
 
-        private BigInteger CalculateBonus(StoreView snapshot, BigInteger value, uint start, uint end)
+        private BigInteger CalculateBonus(StoreView snapshot, ECPoint vote, BigInteger value, uint start, uint end)
         {
             if (value.IsZero || start >= end) return BigInteger.Zero;
             if (value.Sign < 0) throw new ArgumentOutOfRangeException(nameof(value));
 
+            BigInteger neoHolderReward = CalculateNeoHolderReward(snapshot, value, start, end);
+            if (vote is null) return neoHolderReward;
+
+            byte[] border = CreateStorageKey(Prefix_VoterRewardPerCommittee).Add(vote).ToArray();
+            byte[] keyStart = CreateStorageKey(Prefix_VoterRewardPerCommittee).Add(vote).AddBigEndian(start).ToArray();
+            (_, var item) = snapshot.Storages.FindRange(keyStart, border, SeekDirection.Backward).FirstOrDefault();
+            BigInteger startRewardPerNeo = item ?? BigInteger.Zero;
+
+            byte[] keyEnd = CreateStorageKey(Prefix_VoterRewardPerCommittee).Add(vote).AddBigEndian(end).ToArray();
+            (_, item) = snapshot.Storages.FindRange(keyEnd, border, SeekDirection.Backward).FirstOrDefault();
+            BigInteger endRewardPerNeo = item ?? BigInteger.Zero;
+
+            return neoHolderReward + value * (endRewardPerNeo - startRewardPerNeo) / 100000000L;
+        }
+
+        private BigInteger CalculateNeoHolderReward(StoreView snapshot, BigInteger value, uint start, uint end)
+        {
             BigInteger sum = 0;
             foreach (var (index, gasPerBlock) in GetSortedGasRecords(snapshot, end - 1))
             {
@@ -88,17 +106,22 @@ private BigInteger CalculateBonus(StoreView snapshot, BigInteger value, uint sta
             return value * sum * NeoHolderRewardRatio / 100 / TotalAmount;
         }
 
-        private static void CheckCandidate(StoreView snapshot, StorageKey key, CandidateState candidate)
+        private void CheckCandidate(StoreView snapshot, ECPoint pubkey, CandidateState candidate)
         {
             if (!candidate.Registered && candidate.Votes.IsZero)
-                snapshot.Storages.Delete(key);
+            {
+                foreach (var (rewardKey, _) in snapshot.Storages.Find(CreateStorageKey(Prefix_VoterRewardPerCommittee).Add(pubkey).ToArray()).ToArray())
+                    snapshot.Storages.Delete(rewardKey);
+                snapshot.Storages.Delete(CreateStorageKey(Prefix_Candidate).Add(pubkey));
+            }
         }
 
-        public bool ShouldRefreshCommittee(uint height) => height % (ProtocolSettings.Default.CommitteeMembersCount + ProtocolSettings.Default.ValidatorsCount) == 0;
+        public bool ShouldRefreshCommittee(uint height) => height % ProtocolSettings.Default.CommitteeMembersCount == 0;
 
         internal override void Initialize(ApplicationEngine engine)
         {
-            engine.Snapshot.Storages.Add(CreateStorageKey(Prefix_Committee), new StorageItem(Blockchain.StandbyCommittee.ToByteArray()));
+            var cachedCommittee = new CachedCommittee(Blockchain.StandbyCommittee.Select(p => (p, BigInteger.Zero)));
+            engine.Snapshot.Storages.Add(CreateStorageKey(Prefix_Committee), new StorageItem(cachedCommittee));
             engine.Snapshot.Storages.Add(CreateStorageKey(Prefix_VotersCount), new StorageItem(new byte[0]));
 
             // Initialize economic parameters
@@ -112,10 +135,12 @@ protected override void OnPersist(ApplicationEngine engine)
             base.OnPersist(engine);
 
             // Set next committee
-            if (ShouldRefreshCommittee(engine.Snapshot.Height))
+            if (ShouldRefreshCommittee(engine.Snapshot.PersistingBlock.Index))
             {
                 StorageItem storageItem = engine.Snapshot.Storages.GetAndChange(CreateStorageKey(Prefix_Committee));
-                storageItem.Value = ComputeCommitteeMembers(engine.Snapshot).ToArray().ToByteArray();
+                var cachedCommittee = storageItem.GetInteroperable<CachedCommittee>();
+                cachedCommittee.Clear();
+                cachedCommittee.AddRange(ComputeCommitteeMembers(engine.Snapshot));
             }
         }
 
@@ -125,11 +150,35 @@ protected override void PostPersist(ApplicationEngine engine)
 
             // Distribute GAS for committee
 
-            int index = (int)(engine.Snapshot.PersistingBlock.Index % (uint)ProtocolSettings.Default.CommitteeMembersCount);
+            int m = ProtocolSettings.Default.CommitteeMembersCount;
+            int n = ProtocolSettings.Default.ValidatorsCount;
+            int index = (int)(engine.Snapshot.PersistingBlock.Index % (uint)m);
             var gasPerBlock = GetGasPerBlock(engine.Snapshot);
-            var pubkey = GetCommittee(engine.Snapshot)[index];
+            var committee = GetCommitteeFromCache(engine.Snapshot);
+            var pubkey = committee.ElementAt(index).PublicKey;
             var account = Contract.CreateSignatureRedeemScript(pubkey).ToScriptHash();
             GAS.Mint(engine, account, gasPerBlock * CommitteeRewardRatio / 100);
+
+            // Record the cumulative reward of the voters of committee
+
+            if (ShouldRefreshCommittee(engine.Snapshot.PersistingBlock.Index))
+            {
+                BigInteger voterRewardOfEachCommittee = gasPerBlock * VoterRewardRatio * 100000000L * m / (m + n) / 100; // Zoom in 100000000 times, and the final calculation should be divided 100000000L
+                for (index = 0; index < committee.Count; index++)
+                {
+                    var member = committee.ElementAt(index);
+                    var factor = index < n ? 2 : 1; // The `voter` rewards of validator will double than other committee's
+                    if (member.Votes > 0)
+                    {
+                        BigInteger voterSumRewardPerNEO = factor * voterRewardOfEachCommittee / member.Votes;
+                        StorageKey voterRewardKey = CreateStorageKey(Prefix_VoterRewardPerCommittee).Add(member.PublicKey).AddBigEndian(engine.Snapshot.PersistingBlock.Index + 1);
+                        byte[] border = CreateStorageKey(Prefix_VoterRewardPerCommittee).Add(member.PublicKey).ToArray();
+                        (_, var item) = engine.Snapshot.Storages.FindRange(voterRewardKey.ToArray(), border, SeekDirection.Backward).FirstOrDefault();
+                        voterSumRewardPerNEO += (item ?? BigInteger.Zero);
+                        engine.Snapshot.Storages.Add(voterRewardKey, new StorageItem(voterSumRewardPerNEO));
+                    }
+                }
+            }
         }
 
         [ContractMethod(0_05000000, CallFlags.AllowModifyStates)]
@@ -165,7 +214,7 @@ public BigInteger UnclaimedGas(StoreView snapshot, UInt160 account, uint end)
             StorageItem storage = snapshot.Storages.TryGet(CreateStorageKey(Prefix_Account).Add(account));
             if (storage is null) return BigInteger.Zero;
             NeoAccountState state = storage.GetInteroperable<NeoAccountState>();
-            return CalculateBonus(snapshot, state.Balance, state.BalanceHeight, end);
+            return CalculateBonus(snapshot, state.VoteTo, state.Balance, state.BalanceHeight, end);
         }
 
         [ContractMethod(0_05000000, CallFlags.AllowModifyStates)]
@@ -190,7 +239,7 @@ private bool UnregisterCandidate(ApplicationEngine engine, ECPoint pubkey)
             StorageItem item = engine.Snapshot.Storages.GetAndChange(key);
             CandidateState state = item.GetInteroperable<CandidateState>();
             state.Registered = false;
-            CheckCandidate(engine.Snapshot, key, state);
+            CheckCandidate(engine.Snapshot, pubkey, state);
             return true;
         }
 
@@ -215,13 +264,14 @@ private bool Vote(ApplicationEngine engine, UInt160 account, ECPoint voteTo)
                 else
                     item.Add(-state_account.Balance);
             }
+            DistributeGas(engine, account, state_account);
             if (state_account.VoteTo != null)
             {
                 StorageKey key = CreateStorageKey(Prefix_Candidate).Add(state_account.VoteTo);
                 StorageItem storage_validator = engine.Snapshot.Storages.GetAndChange(key);
                 CandidateState state_validator = storage_validator.GetInteroperable<CandidateState>();
                 state_validator.Votes -= state_account.Balance;
-                CheckCandidate(engine.Snapshot, key, state_validator);
+                CheckCandidate(engine.Snapshot, state_account.VoteTo, state_validator);
             }
             state_account.VoteTo = voteTo;
             if (validator_new != null)
@@ -245,7 +295,7 @@ private bool Vote(ApplicationEngine engine, UInt160 account, ECPoint voteTo)
         [ContractMethod(1_00000000, CallFlags.AllowStates)]
         public ECPoint[] GetCommittee(StoreView snapshot)
         {
-            return GetCommitteeFromCache(snapshot).OrderBy(p => p).ToArray();
+            return GetCommitteeFromCache(snapshot).Select(p => p.PublicKey).OrderBy(p => p).ToArray();
         }
 
         public UInt160 GetCommitteeAddress(StoreView snapshot)
@@ -254,33 +304,32 @@ public UInt160 GetCommitteeAddress(StoreView snapshot)
             return Contract.CreateMultiSigRedeemScript(committees.Length - (committees.Length - 1) / 2, committees).ToScriptHash();
         }
 
-        private IEnumerable<ECPoint> GetCommitteeFromCache(StoreView snapshot)
+        private CachedCommittee GetCommitteeFromCache(StoreView snapshot)
         {
-            return snapshot.Storages[CreateStorageKey(Prefix_Committee)].GetSerializableList<ECPoint>();
+            return snapshot.Storages[CreateStorageKey(Prefix_Committee)].GetInteroperable<CachedCommittee>();
         }
 
         internal ECPoint[] ComputeNextBlockValidators(StoreView snapshot)
         {
-            return ComputeCommitteeMembers(snapshot).Take(ProtocolSettings.Default.ValidatorsCount).OrderBy(p => p).ToArray();
+            return ComputeCommitteeMembers(snapshot).Select(p => p.PublicKey).Take(ProtocolSettings.Default.ValidatorsCount).OrderBy(p => p).ToArray();
         }
 
-        private IEnumerable<ECPoint> ComputeCommitteeMembers(StoreView snapshot)
+        private IEnumerable<(ECPoint PublicKey, BigInteger Votes)> ComputeCommitteeMembers(StoreView snapshot)
         {
             decimal votersCount = (decimal)(BigInteger)snapshot.Storages[CreateStorageKey(Prefix_VotersCount)];
-            decimal VoterTurnout = votersCount / (decimal)TotalAmount;
-            if (VoterTurnout < EffectiveVoterTurnout)
-                return Blockchain.StandbyCommittee;
+            decimal voterTurnout = votersCount / (decimal)TotalAmount;
             var candidates = GetCandidates(snapshot);
-            if (candidates.Length < ProtocolSettings.Default.CommitteeMembersCount)
-                return Blockchain.StandbyCommittee;
-            return candidates.OrderByDescending(p => p.Votes).ThenBy(p => p.PublicKey).Select(p => p.PublicKey).Take(ProtocolSettings.Default.CommitteeMembersCount);
+            if (voterTurnout < EffectiveVoterTurnout || candidates.Length < ProtocolSettings.Default.CommitteeMembersCount)
+                return Blockchain.StandbyCommittee.Select(p => (p, candidates.FirstOrDefault(k => k.PublicKey.Equals(p)).Votes));
+            return candidates.OrderByDescending(p => p.Votes).ThenBy(p => p.PublicKey).Take(ProtocolSettings.Default.CommitteeMembersCount);
         }
 
         [ContractMethod(1_00000000, CallFlags.AllowStates)]
         public ECPoint[] GetNextBlockValidators(StoreView snapshot)
         {
             return GetCommitteeFromCache(snapshot)
                 .Take(ProtocolSettings.Default.ValidatorsCount)
+                .Select(p => p.PublicKey)
                 .OrderBy(p => p)
                 .ToArray();
         }
@@ -324,5 +373,30 @@ public StackItem ToStackItem(ReferenceCounter referenceCounter)
                 return new Struct(referenceCounter) { Registered, Votes };
             }
         }
+
+        public class CachedCommittee : List<(ECPoint PublicKey, BigInteger Votes)>, IInteroperable
+        {
+            public CachedCommittee()
+            {
+            }
+
+            public CachedCommittee(IEnumerable<(ECPoint PublicKey, BigInteger Votes)> collection) : base(collection)
+            {
+            }
+
+            public void FromStackItem(StackItem stackItem)
+            {
+                foreach (StackItem item in (VM.Types.Array)stackItem)
+                {
+                    Struct @struct = (Struct)item;
+                    Add((@struct[0].GetSpan().AsSerializable<ECPoint>(), @struct[1].GetInteger()));
+                }
+            }
+
+            public StackItem ToStackItem(ReferenceCounter referenceCounter)
+            {
+                return new VM.Types.Array(referenceCounter, this.Select(p => new Struct(referenceCounter, new StackItem[] { p.PublicKey.ToArray(), p.Votes })));
+            }
+        }
     }
 }

tests/neo.UnitTests/Consensus/UT_Consensus.cs

@@ -17,8 +17,10 @@
 using System;
 using System.Collections.Generic;
 using System.Linq;
+using System.Numerics;
 using System.Reflection;
 using System.Security.Cryptography;
+using static Neo.SmartContract.Native.Tokens.NeoToken;
 using ECPoint = Neo.Cryptography.ECC.ECPoint;
 
 namespace Neo.UnitTests.Consensus
@@ -277,10 +279,11 @@ public void ConsensusService_SingleNodeActors_OnStart_PrepReq_PrepResponses_Comm
             Console.WriteLine($"\nContract updated: {updatedContract.ScriptHash}");
 
             // ===============================================================
+            CachedCommittee cachedCommittee = new CachedCommittee(mockContext.Object.Validators.Select(p => (p, BigInteger.Zero)));
             mockContext.Object.Snapshot.Storages.Delete(CreateStorageKeyForNativeNeo(14));
             mockContext.Object.Snapshot.Storages.Add(CreateStorageKeyForNativeNeo(14), new StorageItem()
             {
-                Value = mockContext.Object.Validators.ToByteArray()
+                Value = BinarySerializer.Serialize(cachedCommittee.ToStackItem(null), 4096)
             });
             mockContext.Object.Snapshot.Commit();
             // ===============================================================
@@ -298,6 +301,8 @@ public void ConsensusService_SingleNodeActors_OnStart_PrepReq_PrepResponses_Comm
             Console.WriteLine("\n\n==========================");
             Console.WriteLine("\nBasic commits Signatures verification");
             // Basic tests for understanding signatures and ensuring signatures of commits are correct on tests
+
+
             var cmPayloadTemp = GetCommitPayloadModifiedAndSignedCopy(commitPayload, 6, kp_array[6], updatedBlockHashData);
             Crypto.VerifySignature(originalBlockHashData, cm.Signature, mockContext.Object.Validators[0]).Should().BeFalse();
             Crypto.VerifySignature(updatedBlockHashData, cm.Signature, mockContext.Object.Validators[0]).Should().BeFalse();
@@ -414,7 +419,11 @@ public void ConsensusService_SingleNodeActors_OnStart_PrepReq_PrepResponses_Comm
             Console.WriteLine("mockContext Reset for returning Blockchain.Singleton snapshot to original state.");
             mockContext.Object.Reset(0);
             mockContext.Object.Snapshot.Storages.Delete(CreateStorageKeyForNativeNeo(14));
-            mockContext.Object.Snapshot.Storages.Add(CreateStorageKeyForNativeNeo(14), new StorageItem(Blockchain.StandbyCommittee.ToByteArray()));
+            cachedCommittee = new CachedCommittee(Blockchain.StandbyCommittee.Select(p => (p, BigInteger.Zero)));
+            mockContext.Object.Snapshot.Storages.Add(CreateStorageKeyForNativeNeo(14), new StorageItem
+            {
+                Value = BinarySerializer.Serialize(cachedCommittee.ToStackItem(null), 4096)
+            });
             mockContext.Object.Snapshot.Commit();
 
             Console.WriteLine("mockContext Reset.");