From be37423d0fed4b35f358aa2e94552c465cdace24 Mon Sep 17 00:00:00 2001 From: jsolman Date: Sat, 16 Feb 2019 00:13:34 -0800 Subject: [PATCH] Recover nodes requesting ChangeView when possible (#579) --- neo.UnitTests/TestBlockchain.cs | 65 +++ neo.UnitTests/TestUtils.cs | 34 +- neo.UnitTests/UT_Consensus.cs | 436 ++++++++++++++++-- neo.UnitTests/UT_MemoryPool.cs | 76 +-- neo/Consensus/ChangeView.cs | 16 +- neo/Consensus/ConsensusContext.cs | 229 +++++---- neo/Consensus/ConsensusMessageType.cs | 4 + neo/Consensus/ConsensusService.cs | 236 +++++++--- neo/Consensus/IConsensusContext.cs | 22 +- neo/Consensus/PrepareRequest.cs | 12 +- ...ecoveryMessage.ChangeViewPayloadCompact.cs | 51 ++ .../RecoveryMessage.CommitPayloadCompact.cs | 46 ++ ...coveryMessage.PreparationPayloadCompact.cs | 40 ++ neo/Consensus/RecoveryMessage.cs | 148 ++++++ neo/Ledger/Blockchain.cs | 2 +- neo/Ledger/TrimmedBlock.cs | 2 +- neo/Network/P2P/Payloads/Block.cs | 4 +- neo/Network/P2P/Payloads/ConsensusPayload.cs | 37 +- 18 files changed, 1180 insertions(+), 280 deletions(-) create mode 100644 neo.UnitTests/TestBlockchain.cs create mode 100644 neo/Consensus/RecoveryMessage.ChangeViewPayloadCompact.cs create mode 100644 neo/Consensus/RecoveryMessage.CommitPayloadCompact.cs create mode 100644 neo/Consensus/RecoveryMessage.PreparationPayloadCompact.cs create mode 100644 neo/Consensus/RecoveryMessage.cs diff --git a/neo.UnitTests/TestBlockchain.cs b/neo.UnitTests/TestBlockchain.cs new file mode 100644 index 0000000000..b456dc820b --- /dev/null +++ b/neo.UnitTests/TestBlockchain.cs @@ -0,0 +1,65 @@ +using Moq; +using Neo.Cryptography.ECC; +using Neo.IO.Wrappers; +using Neo.Ledger; +using Neo.Persistence; +using System; + +namespace Neo.UnitTests +{ + public static class TestBlockchain + { + private static NeoSystem TheNeoSystem; + + public static NeoSystem InitializeMockNeoSystem() + { + if (TheNeoSystem == null) + { + var mockSnapshot = new Mock(); + mockSnapshot.SetupGet(p => p.Blocks).Returns(new TestDataCache()); + mockSnapshot.SetupGet(p => p.Transactions).Returns(new TestDataCache()); + mockSnapshot.SetupGet(p => p.Accounts).Returns(new TestDataCache()); + mockSnapshot.SetupGet(p => p.UnspentCoins).Returns(new TestDataCache()); + mockSnapshot.SetupGet(p => p.SpentCoins).Returns(new TestDataCache()); + mockSnapshot.SetupGet(p => p.Validators).Returns(new TestDataCache()); + mockSnapshot.SetupGet(p => p.Assets).Returns(new TestDataCache()); + mockSnapshot.SetupGet(p => p.Contracts).Returns(new TestDataCache()); + mockSnapshot.SetupGet(p => p.Storages).Returns(new TestDataCache()); + mockSnapshot.SetupGet(p => p.HeaderHashList) + .Returns(new TestDataCache()); + mockSnapshot.SetupGet(p => p.ValidatorsCount).Returns(new TestMetaDataCache()); + mockSnapshot.SetupGet(p => p.BlockHashIndex).Returns(new TestMetaDataCache()); + mockSnapshot.SetupGet(p => p.HeaderHashIndex).Returns(new TestMetaDataCache()); + + var mockStore = new Mock(); + + var defaultTx = TestUtils.CreateRandomHashInvocationMockTransaction().Object; + mockStore.Setup(p => p.GetBlocks()).Returns(new TestDataCache()); + mockStore.Setup(p => p.GetTransactions()).Returns(new TestDataCache( + new TransactionState + { + BlockIndex = 1, + Transaction = defaultTx + })); + + mockStore.Setup(p => p.GetAccounts()).Returns(new TestDataCache()); + mockStore.Setup(p => p.GetUnspentCoins()).Returns(new TestDataCache()); + mockStore.Setup(p => p.GetSpentCoins()).Returns(new TestDataCache()); + mockStore.Setup(p => p.GetValidators()).Returns(new TestDataCache()); + mockStore.Setup(p => p.GetAssets()).Returns(new TestDataCache()); + mockStore.Setup(p => p.GetContracts()).Returns(new TestDataCache()); + mockStore.Setup(p => p.GetStorages()).Returns(new TestDataCache()); + mockStore.Setup(p => p.GetHeaderHashList()).Returns(new TestDataCache()); + mockStore.Setup(p => p.GetValidatorsCount()).Returns(new TestMetaDataCache()); + mockStore.Setup(p => p.GetBlockHashIndex()).Returns(new TestMetaDataCache()); + mockStore.Setup(p => p.GetHeaderHashIndex()).Returns(new TestMetaDataCache()); + mockStore.Setup(p => p.GetSnapshot()).Returns(mockSnapshot.Object); + + Console.WriteLine("initialize NeoSystem"); + TheNeoSystem = new NeoSystem(mockStore.Object); // new Mock(mockStore.Object); + } + + return TheNeoSystem; + } + } +} \ No newline at end of file diff --git a/neo.UnitTests/TestUtils.cs b/neo.UnitTests/TestUtils.cs index d84ba304e9..a7c4798897 100644 --- a/neo.UnitTests/TestUtils.cs +++ b/neo.UnitTests/TestUtils.cs @@ -1,10 +1,12 @@ -using Neo.Cryptography.ECC; +using Moq; +using Neo.Cryptography.ECC; +using Neo.IO; using Neo.Network.P2P.Payloads; +using Neo.Persistence; using Neo.VM; using System; using System.Collections.Generic; -using Moq; -using Neo.Persistence; +using System.IO; namespace Neo.UnitTests { @@ -119,5 +121,31 @@ public static Mock CreateRandomHashInvocationMockTransact return mockTx; } + + public static Mock CreateRandomMockMinerTransaction() + { + var mockTx = new Mock + { + CallBase = true + }; + var tx = mockTx.Object; + tx.Attributes = new TransactionAttribute[0]; + tx.Inputs = new CoinReference[0]; + tx.Outputs = new TransactionOutput[0]; + tx.Witnesses = new Witness[0]; + tx.Nonce = (uint)TestRandom.Next(); + return mockTx; + } + + public static T CopyMsgBySerialization(T serializableObj, T newObj) where T : ISerializable + { + using (MemoryStream ms = new MemoryStream(serializableObj.ToArray(), false)) + using (BinaryReader reader = new BinaryReader(ms)) + { + newObj.Deserialize(reader); + } + + return newObj; + } } } diff --git a/neo.UnitTests/UT_Consensus.cs b/neo.UnitTests/UT_Consensus.cs index 6010621f27..d3faa858d0 100644 --- a/neo.UnitTests/UT_Consensus.cs +++ b/neo.UnitTests/UT_Consensus.cs @@ -4,16 +4,18 @@ using Microsoft.VisualStudio.TestTools.UnitTesting; using Moq; using Neo.Consensus; +using Neo.Cryptography; using Neo.IO; using Neo.Ledger; using Neo.Network.P2P; using Neo.Network.P2P.Payloads; +using Neo.Persistence; +using Neo.SmartContract; using System; -using System.IO; +using System.Collections.Generic; using System.Linq; using System.Numerics; using System.Security.Cryptography; -using Neo.Persistence; using ECPoint = Neo.Cryptography.ECC.ECPoint; namespace Neo.UnitTests @@ -22,6 +24,12 @@ namespace Neo.UnitTests [TestClass] public class ConsensusTests : TestKit { + [TestInitialize] + public void TestSetup() + { + TestBlockchain.InitializeMockNeoSystem(); + } + [TestCleanup] public void Cleanup() { @@ -109,17 +117,13 @@ public void ConsensusService_Primary_Sends_PrepareRequest_After_OnStart() MinerTransaction = minerTx //(MinerTransaction)Transactions[TransactionHashes[0]], }; - ConsensusMessage mprep = prep; - byte[] prepData = mprep.ToArray(); - ConsensusPayload prepPayload = new ConsensusPayload { Version = 0, PrevHash = mockConsensusContext.Object.PrevHash, BlockIndex = mockConsensusContext.Object.BlockIndex, ValidatorIndex = (ushort)mockConsensusContext.Object.MyIndex, - Timestamp = mockConsensusContext.Object.Timestamp, - Data = prepData + ConsensusMessage = prep }; mockConsensusContext.Setup(mr => mr.MakePrepareRequest()).Returns(prepPayload); @@ -198,8 +202,11 @@ public void TestSerializeAndDeserializeConsensusContext() int txCountToInlcude = 256; consensusContext.TransactionHashes = new UInt256[txCountToInlcude]; + Transaction[] txs = new Transaction[txCountToInlcude]; - for (int i = 0; i < txCountToInlcude; i++) + txs[0] = TestUtils.CreateRandomMockMinerTransaction().Object; + consensusContext.TransactionHashes[0] = txs[0].Hash; + for (int i = 1; i < txCountToInlcude; i++) { txs[i] = TestUtils.CreateRandomHashInvocationMockTransaction().Object; consensusContext.TransactionHashes[i] = txs[i].Hash; @@ -207,32 +214,44 @@ public void TestSerializeAndDeserializeConsensusContext() // consensusContext.TransactionHashes = new UInt256[2] {testTx1.Hash, testTx2.Hash}; consensusContext.Transactions = txs.ToDictionary(p => p.Hash); - consensusContext.Preparations = new [] {null, null, null, consensusContext.PrevHash, null, null, null }; - consensusContext.Commits = new byte[consensusContext.Validators.Length][]; + consensusContext.PreparationPayloads = new ConsensusPayload[consensusContext.Validators.Length]; + var prepareRequestMessage = new PrepareRequest + { + Nonce = consensusContext.Nonce, + NextConsensus = consensusContext.NextConsensus, + TransactionHashes = consensusContext.TransactionHashes, + MinerTransaction = (MinerTransaction)consensusContext.Transactions[consensusContext.TransactionHashes[0]], + Timestamp = 23 + }; + consensusContext.PreparationPayloads[6] = MakeSignedPayload(consensusContext, prepareRequestMessage, 6, new[] { (byte)'3', (byte)'!' }); + consensusContext.PreparationPayloads[0] = MakeSignedPayload(consensusContext, new PrepareResponse { PreparationHash = consensusContext.PreparationPayloads[6].Hash }, 0, new[] { (byte)'t', (byte)'e' }); + consensusContext.PreparationPayloads[1] = MakeSignedPayload(consensusContext, new PrepareResponse { PreparationHash = consensusContext.PreparationPayloads[6].Hash }, 1, new[] { (byte)'s', (byte)'t' }); + consensusContext.PreparationPayloads[2] = null; + consensusContext.PreparationPayloads[3] = MakeSignedPayload(consensusContext, new PrepareResponse { PreparationHash = consensusContext.PreparationPayloads[6].Hash }, 3, new[] { (byte)'1', (byte)'2' }); + consensusContext.PreparationPayloads[4] = null; + consensusContext.PreparationPayloads[5] = null; + + consensusContext.CommitPayloads = new ConsensusPayload[consensusContext.Validators.Length]; using (SHA256 sha256 = SHA256.Create()) { - consensusContext.Commits[3] = sha256.ComputeHash(testTx1.Hash.ToArray()); - consensusContext.Commits[6] = sha256.ComputeHash(testTx2.Hash.ToArray()); + consensusContext.CommitPayloads[3] = MakeSignedPayload(consensusContext, new Commit { Signature = sha256.ComputeHash(testTx1.Hash.ToArray()) }, 3, new[] { (byte)'3', (byte)'4' }); + consensusContext.CommitPayloads[6] = MakeSignedPayload(consensusContext, new Commit { Signature = sha256.ComputeHash(testTx2.Hash.ToArray()) }, 3, new[] { (byte)'6', (byte)'7' }); } - consensusContext.ExpectedView = new byte[consensusContext.Validators.Length]; - consensusContext.ExpectedView[0] = 2; - consensusContext.ExpectedView[1] = 2; - consensusContext.ExpectedView[2] = 1; - consensusContext.ExpectedView[3] = 2; - consensusContext.ExpectedView[4] = 1; - consensusContext.ExpectedView[5] = 1; - consensusContext.ExpectedView[6] = 2; + consensusContext.Timestamp = TimeProvider.Current.UtcNow.ToTimestamp(); - byte[] serializedContextData = consensusContext.ToArray(); + consensusContext.ChangeViewPayloads = new ConsensusPayload[consensusContext.Validators.Length]; + consensusContext.ChangeViewPayloads[0] = MakeSignedPayload(consensusContext, new ChangeView { ViewNumber = 1, NewViewNumber = 2, Timestamp = 6 }, 0, new[] { (byte)'A' }); + consensusContext.ChangeViewPayloads[1] = MakeSignedPayload(consensusContext, new ChangeView { ViewNumber = 1, NewViewNumber = 2, Timestamp = 5 }, 1, new[] { (byte)'B' }); + consensusContext.ChangeViewPayloads[2] = null; + consensusContext.ChangeViewPayloads[3] = MakeSignedPayload(consensusContext, new ChangeView { ViewNumber = 1, NewViewNumber = 2, Timestamp = uint.MaxValue }, 3, new[] { (byte)'C' }); + consensusContext.ChangeViewPayloads[4] = null; + consensusContext.ChangeViewPayloads[5] = null; + consensusContext.ChangeViewPayloads[6] = MakeSignedPayload(consensusContext, new ChangeView { ViewNumber = 1, NewViewNumber = 2, Timestamp = 1 }, 6, new[] { (byte)'D' }); - var copiedContext = new ConsensusContext(null); + consensusContext.LastChangeViewPayloads = new ConsensusPayload[consensusContext.Validators.Length]; - using (MemoryStream ms = new MemoryStream(serializedContextData, false)) - using (BinaryReader reader = new BinaryReader(ms)) - { - copiedContext.Deserialize(reader); - } + var copiedContext = TestUtils.CopyMsgBySerialization(consensusContext, new ConsensusContext(null)); copiedContext.State.Should().Be(consensusContext.State); copiedContext.PrevHash.Should().Be(consensusContext.PrevHash); @@ -247,9 +266,366 @@ public void TestSerializeAndDeserializeConsensusContext() copiedContext.TransactionHashes.ShouldAllBeEquivalentTo(consensusContext.TransactionHashes); copiedContext.Transactions.ShouldAllBeEquivalentTo(consensusContext.Transactions); copiedContext.Transactions.Values.ShouldAllBeEquivalentTo(consensusContext.Transactions.Values); - copiedContext.Preparations.ShouldAllBeEquivalentTo(consensusContext.Preparations); - copiedContext.Commits.ShouldAllBeEquivalentTo(consensusContext.Commits); - copiedContext.ExpectedView.ShouldAllBeEquivalentTo(consensusContext.ExpectedView); + copiedContext.PreparationPayloads.ShouldAllBeEquivalentTo(consensusContext.PreparationPayloads); + copiedContext.CommitPayloads.ShouldAllBeEquivalentTo(consensusContext.CommitPayloads); + copiedContext.ChangeViewPayloads.ShouldAllBeEquivalentTo(consensusContext.ChangeViewPayloads); + } + + [TestMethod] + public void TestSerializeAndDeserializeRecoveryMessageWithChangeViewsAndNoPrepareRequest() + { + var msg = new RecoveryMessage + { + ChangeViewMessages = new Dictionary() + { + { + 0, + new RecoveryMessage.ChangeViewPayloadCompact + { + ValidatorIndex = 0, + OriginalViewNumber = 9, + Timestamp = 6, + InvocationScript = new[] { (byte)'A' } + } + }, + { + 1, + new RecoveryMessage.ChangeViewPayloadCompact + { + ValidatorIndex = 1, + OriginalViewNumber = 7, + Timestamp = 5, + InvocationScript = new[] { (byte)'B' } + } + }, + { + 3, + new RecoveryMessage.ChangeViewPayloadCompact + { + ValidatorIndex = 3, + OriginalViewNumber = 5, + Timestamp = 3, + InvocationScript = new[] { (byte)'C' } + } + }, + { + 6, + new RecoveryMessage.ChangeViewPayloadCompact + { + ValidatorIndex = 6, + OriginalViewNumber = 2, + Timestamp = 1, + InvocationScript = new[] { (byte)'D' } + } + } + }, + PreparationHash = new UInt256(Crypto.Default.Hash256(new[] { (byte)'a' })), + PreparationMessages = new Dictionary() + { + { + 0, + new RecoveryMessage.PreparationPayloadCompact + { + ValidatorIndex = 0, + InvocationScript = new[] { (byte)'t', (byte)'e' } + } + }, + { + 3, + new RecoveryMessage.PreparationPayloadCompact + { + ValidatorIndex = 3, + InvocationScript = new[] { (byte)'1', (byte)'2' } + } + }, + { + 6, + new RecoveryMessage.PreparationPayloadCompact + { + ValidatorIndex = 6, + InvocationScript = new[] { (byte)'3', (byte)'!' } + } + } + }, + CommitMessages = new Dictionary() + }; + + // msg.TransactionHashes = null; + // msg.Nonce = 0; + // msg.NextConsensus = null; + // msg.MinerTransaction = (MinerTransaction) null; + msg.PrepareRequestMessage.Should().Be(null); + + var copiedMsg = TestUtils.CopyMsgBySerialization(msg, new RecoveryMessage()); ; + + copiedMsg.ChangeViewMessages.ShouldAllBeEquivalentTo(msg.ChangeViewMessages); + copiedMsg.PreparationHash.Should().Be(msg.PreparationHash); + copiedMsg.PreparationMessages.ShouldAllBeEquivalentTo(msg.PreparationMessages); + copiedMsg.CommitMessages.Count.Should().Be(0); + } + + [TestMethod] + public void TestSerializeAndDeserializeRecoveryMessageWithChangeViewsAndPrepareRequest() + { + Transaction[] txs = new Transaction[5]; + txs[0] = TestUtils.CreateRandomMockMinerTransaction().Object; + for (int i = 1; i < txs.Length; i++) + txs[i] = TestUtils.CreateRandomHashInvocationMockTransaction().Object; + var msg = new RecoveryMessage + { + ChangeViewMessages = new Dictionary() + { + { + 0, + new RecoveryMessage.ChangeViewPayloadCompact + { + ValidatorIndex = 0, + OriginalViewNumber = 9, + Timestamp = 6, + InvocationScript = new[] { (byte)'A' } + } + }, + { + 1, + new RecoveryMessage.ChangeViewPayloadCompact + { + ValidatorIndex = 1, + OriginalViewNumber = 7, + Timestamp = 5, + InvocationScript = new[] { (byte)'B' } + } + }, + { + 3, + new RecoveryMessage.ChangeViewPayloadCompact + { + ValidatorIndex = 3, + OriginalViewNumber = 5, + Timestamp = 3, + InvocationScript = new[] { (byte)'C' } + } + }, + { + 6, + new RecoveryMessage.ChangeViewPayloadCompact + { + ValidatorIndex = 6, + OriginalViewNumber = 2, + Timestamp = 1, + InvocationScript = new[] { (byte)'D' } + } + } + }, + PrepareRequestMessage = new PrepareRequest + { + TransactionHashes = txs.Select(p => p.Hash).ToArray(), + Nonce = ulong.MaxValue, + NextConsensus = UInt160.Parse("5555AAAA5555AAAA5555AAAA5555AAAA5555AAAA"), + MinerTransaction = (MinerTransaction)txs[0] + }, + PreparationHash = new UInt256(Crypto.Default.Hash256(new[] { (byte)'a' })), + PreparationMessages = new Dictionary() + { + { + 0, + new RecoveryMessage.PreparationPayloadCompact + { + ValidatorIndex = 0, + InvocationScript = new[] { (byte)'t', (byte)'e' } + } + }, + { + 1, + new RecoveryMessage.PreparationPayloadCompact + { + ValidatorIndex = 1, + InvocationScript = new[] { (byte)'s', (byte)'t' } + } + }, + { + 3, + new RecoveryMessage.PreparationPayloadCompact + { + ValidatorIndex = 3, + InvocationScript = new[] { (byte)'1', (byte)'2' } + } + } + }, + CommitMessages = new Dictionary() + }; + + var copiedMsg = TestUtils.CopyMsgBySerialization(msg, new RecoveryMessage()); ; + + copiedMsg.ChangeViewMessages.ShouldAllBeEquivalentTo(msg.ChangeViewMessages); + copiedMsg.PrepareRequestMessage.ShouldBeEquivalentTo(msg.PrepareRequestMessage); + copiedMsg.PreparationHash.Should().Be(null); + copiedMsg.PreparationMessages.ShouldAllBeEquivalentTo(msg.PreparationMessages); + copiedMsg.CommitMessages.Count.Should().Be(0); + } + + [TestMethod] + public void TestSerializeAndDeserializeRecoveryMessageWithoutChangeViewsWithoutCommits() + { + Transaction[] txs = new Transaction[5]; + txs[0] = TestUtils.CreateRandomMockMinerTransaction().Object; + for (int i = 1; i < txs.Length; i++) + txs[i] = TestUtils.CreateRandomHashInvocationMockTransaction().Object; + var msg = new RecoveryMessage + { + ChangeViewMessages = new Dictionary(), + PrepareRequestMessage = new PrepareRequest + { + TransactionHashes = txs.Select(p => p.Hash).ToArray(), + Nonce = ulong.MaxValue, + NextConsensus = UInt160.Parse("5555AAAA5555AAAA5555AAAA5555AAAA5555AAAA"), + MinerTransaction = (MinerTransaction)txs[0] + }, + PreparationMessages = new Dictionary() + { + { + 0, + new RecoveryMessage.PreparationPayloadCompact + { + ValidatorIndex = 0, + InvocationScript = new[] { (byte)'t', (byte)'e' } + } + }, + { + 1, + new RecoveryMessage.PreparationPayloadCompact + { + ValidatorIndex = 1, + InvocationScript = new[] { (byte)'s', (byte)'t' } + } + }, + { + 3, + new RecoveryMessage.PreparationPayloadCompact + { + ValidatorIndex = 3, + InvocationScript = new[] { (byte)'1', (byte)'2' } + } + }, + { + 6, + new RecoveryMessage.PreparationPayloadCompact + { + ValidatorIndex = 6, + InvocationScript = new[] { (byte)'3', (byte)'!' } + } + } + }, + CommitMessages = new Dictionary() + }; + + var copiedMsg = TestUtils.CopyMsgBySerialization(msg, new RecoveryMessage()); ; + + copiedMsg.ChangeViewMessages.Count.Should().Be(0); + copiedMsg.PrepareRequestMessage.ShouldBeEquivalentTo(msg.PrepareRequestMessage); + copiedMsg.PreparationHash.Should().Be(null); + copiedMsg.PreparationMessages.ShouldAllBeEquivalentTo(msg.PreparationMessages); + copiedMsg.CommitMessages.Count.Should().Be(0); + } + + [TestMethod] + public void TestSerializeAndDeserializeRecoveryMessageWithoutChangeViewsWithCommits() + { + Transaction[] txs = new Transaction[5]; + txs[0] = TestUtils.CreateRandomMockMinerTransaction().Object; + for (int i = 1; i < txs.Length; i++) + txs[i] = TestUtils.CreateRandomHashInvocationMockTransaction().Object; + var msg = new RecoveryMessage + { + ChangeViewMessages = new Dictionary(), + PrepareRequestMessage = new PrepareRequest + { + TransactionHashes = txs.Select(p => p.Hash).ToArray(), + Nonce = ulong.MaxValue, + NextConsensus = UInt160.Parse("5555AAAA5555AAAA5555AAAA5555AAAA5555AAAA"), + MinerTransaction = (MinerTransaction)txs[0] + }, + PreparationMessages = new Dictionary() + { + { + 0, + new RecoveryMessage.PreparationPayloadCompact + { + ValidatorIndex = 0, + InvocationScript = new[] { (byte)'t', (byte)'e' } + } + }, + { + 1, + new RecoveryMessage.PreparationPayloadCompact + { + ValidatorIndex = 1, + InvocationScript = new[] { (byte)'s', (byte)'t' } + } + }, + { + 3, + new RecoveryMessage.PreparationPayloadCompact + { + ValidatorIndex = 3, + InvocationScript = new[] { (byte)'1', (byte)'2' } + } + }, + { + 6, + new RecoveryMessage.PreparationPayloadCompact + { + ValidatorIndex = 6, + InvocationScript = new[] { (byte)'3', (byte)'!' } + } + } + }, + CommitMessages = new Dictionary + { + { + 1, + new RecoveryMessage.CommitPayloadCompact + { + ValidatorIndex = 1, + Signature = new byte[64] { (byte)'1', (byte)'2', 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, + InvocationScript = new[] { (byte)'1', (byte)'2' } + } + }, + { + 6, + new RecoveryMessage.CommitPayloadCompact + { + ValidatorIndex = 6, + Signature = new byte[64] { (byte)'3', (byte)'D', (byte)'R', (byte)'I', (byte)'N', (byte)'K', 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, + InvocationScript = new[] { (byte)'6', (byte)'7' } + } + } + } + }; + + var copiedMsg = TestUtils.CopyMsgBySerialization(msg, new RecoveryMessage()); ; + + copiedMsg.ChangeViewMessages.Count.Should().Be(0); + copiedMsg.PrepareRequestMessage.ShouldBeEquivalentTo(msg.PrepareRequestMessage); + copiedMsg.PreparationHash.Should().Be(null); + copiedMsg.PreparationMessages.ShouldAllBeEquivalentTo(msg.PreparationMessages); + copiedMsg.CommitMessages.ShouldAllBeEquivalentTo(msg.CommitMessages); + } + + private static ConsensusPayload MakeSignedPayload(IConsensusContext context, ConsensusMessage message, ushort validatorIndex, byte[] witnessInvocationScript) + { + return new ConsensusPayload + { + Version = ConsensusContext.Version, + PrevHash = context.PrevHash, + BlockIndex = context.BlockIndex, + ValidatorIndex = validatorIndex, + ConsensusMessage = message, + Witness = new Witness + { + InvocationScript = witnessInvocationScript, + VerificationScript = Contract.CreateSignatureRedeemScript(context.Validators[validatorIndex]) + } + }; } } } diff --git a/neo.UnitTests/UT_MemoryPool.cs b/neo.UnitTests/UT_MemoryPool.cs index 75accde851..4c6f77ba24 100644 --- a/neo.UnitTests/UT_MemoryPool.cs +++ b/neo.UnitTests/UT_MemoryPool.cs @@ -1,22 +1,16 @@ -using System; -using System.Collections.Generic; -using System.Linq; +using FluentAssertions; using Microsoft.VisualStudio.TestTools.UnitTesting; -using Moq; using Neo.Ledger; -using FluentAssertions; -using Neo.Cryptography.ECC; -using Neo.IO.Wrappers; using Neo.Network.P2P.Payloads; -using Neo.Persistence; +using System; +using System.Collections.Generic; +using System.Linq; namespace Neo.UnitTests { [TestClass] public class UT_MemoryPool { - private static NeoSystem TheNeoSystem; - private MemoryPool _unit; [TestInitialize] @@ -25,59 +19,7 @@ public void TestSetup() // protect against external changes on TimeProvider TimeProvider.ResetToDefault(); - if (TheNeoSystem == null) - { - var mockSnapshot = new Mock(); - mockSnapshot.SetupGet(p => p.Blocks).Returns(new TestDataCache()); - mockSnapshot.SetupGet(p => p.Transactions).Returns(new TestDataCache()); - mockSnapshot.SetupGet(p => p.Accounts).Returns(new TestDataCache()); - mockSnapshot.SetupGet(p => p.UnspentCoins).Returns(new TestDataCache()); - mockSnapshot.SetupGet(p => p.SpentCoins).Returns(new TestDataCache()); - mockSnapshot.SetupGet(p => p.Validators).Returns(new TestDataCache()); - mockSnapshot.SetupGet(p => p.Assets).Returns(new TestDataCache()); - mockSnapshot.SetupGet(p => p.Contracts).Returns(new TestDataCache()); - mockSnapshot.SetupGet(p => p.Storages).Returns(new TestDataCache()); - mockSnapshot.SetupGet(p => p.HeaderHashList) - .Returns(new TestDataCache()); - mockSnapshot.SetupGet(p => p.ValidatorsCount).Returns(new TestMetaDataCache()); - mockSnapshot.SetupGet(p => p.BlockHashIndex).Returns(new TestMetaDataCache()); - mockSnapshot.SetupGet(p => p.HeaderHashIndex).Returns(new TestMetaDataCache()); - - var mockStore = new Mock(); - - var defaultTx = TestUtils.CreateRandomHashInvocationMockTransaction().Object; - defaultTx.Outputs = new TransactionOutput[1]; - defaultTx.Outputs[0] = new TransactionOutput - { - AssetId = Blockchain.UtilityToken.Hash, - Value = new Fixed8(1000000), - ScriptHash = UInt160.Zero // doesn't matter for our purposes. - }; - - mockStore.Setup(p => p.GetBlocks()).Returns(new TestDataCache()); - mockStore.Setup(p => p.GetTransactions()).Returns(new TestDataCache( - new TransactionState - { - BlockIndex = 1, - Transaction = defaultTx - })); - - mockStore.Setup(p => p.GetAccounts()).Returns(new TestDataCache()); - mockStore.Setup(p => p.GetUnspentCoins()).Returns(new TestDataCache()); - mockStore.Setup(p => p.GetSpentCoins()).Returns(new TestDataCache()); - mockStore.Setup(p => p.GetValidators()).Returns(new TestDataCache()); - mockStore.Setup(p => p.GetAssets()).Returns(new TestDataCache()); - mockStore.Setup(p => p.GetContracts()).Returns(new TestDataCache()); - mockStore.Setup(p => p.GetStorages()).Returns(new TestDataCache()); - mockStore.Setup(p => p.GetHeaderHashList()).Returns(new TestDataCache()); - mockStore.Setup(p => p.GetValidatorsCount()).Returns(new TestMetaDataCache()); - mockStore.Setup(p => p.GetBlockHashIndex()).Returns(new TestMetaDataCache()); - mockStore.Setup(p => p.GetHeaderHashIndex()).Returns(new TestMetaDataCache()); - mockStore.Setup(p => p.GetSnapshot()).Returns(mockSnapshot.Object); - - Console.WriteLine("initialize NeoSystem"); - TheNeoSystem = new NeoSystem(mockStore.Object); // new Mock(mockStore.Object); - } + NeoSystem TheNeoSystem = TestBlockchain.InitializeMockNeoSystem(); // Create a MemoryPool with capacity of 100 _unit = new MemoryPool(TheNeoSystem, 100); @@ -95,7 +37,7 @@ public void TestSetup() long LongRandom(long min, long max, Random rand) { // Only returns positive random long values. - long longRand = (long) rand.NextBigInteger(63); + long longRand = (long)rand.NextBigInteger(63); return longRand % (max - min) + min; } @@ -129,11 +71,11 @@ private Transaction CreateMockLowPriorityTransaction() return CreateMockTransactionWithFee(rNetFee); } - private void AddTransactions(int count, bool isHighPriority=false) + private void AddTransactions(int count, bool isHighPriority = false) { for (int i = 0; i < count; i++) { - var txToAdd = isHighPriority ? CreateMockHighPriorityTransaction(): CreateMockLowPriorityTransaction(); + var txToAdd = isHighPriority ? CreateMockHighPriorityTransaction() : CreateMockLowPriorityTransaction(); Console.WriteLine($"created tx: {txToAdd.Hash}"); _unit.TryAdd(txToAdd.Hash, txToAdd); } @@ -332,7 +274,7 @@ public void VerifySortOrderAndThatHighetFeeTransactionsAreReverifiedFirst() verifiedTxs.Length.ShouldBeEquivalentTo(2); verifiedTxs[0].ShouldBeEquivalentTo(maxHighPriorityTransaction); verifiedTxs[1].ShouldBeEquivalentTo(maxLowPriorityTransaction); - var blockWith2Tx = new Block { Transactions = new Transaction[2] { maxHighPriorityTransaction, maxLowPriorityTransaction }}; + var blockWith2Tx = new Block { Transactions = new Transaction[2] { maxHighPriorityTransaction, maxLowPriorityTransaction } }; // verify and remove the 2 transactions from the verified pool _unit.UpdatePoolForBlockPersisted(blockWith2Tx, Blockchain.Singleton.GetSnapshot()); _unit.SortedHighPrioTxCount.ShouldBeEquivalentTo(0); diff --git a/neo/Consensus/ChangeView.cs b/neo/Consensus/ChangeView.cs index 2bf1412152..589e0f79d9 100644 --- a/neo/Consensus/ChangeView.cs +++ b/neo/Consensus/ChangeView.cs @@ -1,13 +1,20 @@ -using System; -using System.IO; +using System.IO; namespace Neo.Consensus { internal class ChangeView : ConsensusMessage { public byte NewViewNumber; + /// + /// Timestamp of when the ChangeView message was created. This allows receiving nodes to ensure + // they only respond once to a specific ChangeView request (it thus prevents replay of the ChangeView + // message from repeatedly broadcasting RecoveryMessages). + /// + public uint Timestamp; - public override int Size => base.Size + sizeof(byte); + public override int Size => base.Size + + sizeof(byte) //NewViewNumber + + sizeof(uint); //Timestamp public ChangeView() : base(ConsensusMessageType.ChangeView) @@ -18,13 +25,14 @@ public override void Deserialize(BinaryReader reader) { base.Deserialize(reader); NewViewNumber = reader.ReadByte(); - if (NewViewNumber == 0) throw new FormatException(); + Timestamp = reader.ReadUInt32(); } public override void Serialize(BinaryWriter writer) { base.Serialize(writer); writer.Write(NewViewNumber); + writer.Write(Timestamp); } } } diff --git a/neo/Consensus/ConsensusContext.cs b/neo/Consensus/ConsensusContext.cs index fa19a8f174..7e1b436d3f 100644 --- a/neo/Consensus/ConsensusContext.cs +++ b/neo/Consensus/ConsensusContext.cs @@ -29,20 +29,19 @@ internal class ConsensusContext : IConsensusContext public UInt160 NextConsensus { get; set; } public UInt256[] TransactionHashes { get; set; } public Dictionary Transactions { get; set; } - public UInt256[] Preparations { get; set; } - public byte[][] Commits { get; set; } - public byte[] ExpectedView { get; set; } - private Snapshot snapshot; + public ConsensusPayload[] PreparationPayloads { get; set; } + public ConsensusPayload[] CommitPayloads { get; set; } + public ConsensusPayload[] ChangeViewPayloads { get; set; } + public ConsensusPayload[] LastChangeViewPayloads { get; set; } + public Snapshot Snapshot { get; private set; } private KeyPair keyPair; private readonly Wallet wallet; - public int M => Validators.Length - (Validators.Length - 1) / 3; - public Header PrevHeader => snapshot.GetHeader(PrevHash); + public int F => (Validators.Length - 1) / 3; + public int M => Validators.Length - F; + public Header PrevHeader => Snapshot.GetHeader(PrevHash); public int Size => throw new NotImplementedException(); - public bool TransactionExists(UInt256 hash) => snapshot.ContainsTransaction(hash); - public bool VerifyTransaction(Transaction tx) => tx.Verify(snapshot, Transactions.Values); - public ConsensusContext(Wallet wallet) { this.wallet = wallet; @@ -55,11 +54,11 @@ public Block CreateBlock() Contract contract = Contract.CreateMultiSigContract(M, Validators); ContractParametersContext sc = new ContractParametersContext(block); for (int i = 0, j = 0; i < Validators.Length && j < M; i++) - if (Commits[i] != null) - { - sc.AddSignature(contract, Validators[i], Commits[i]); - j++; - } + { + if (CommitPayloads[i] == null) continue; + sc.AddSignature(contract, Validators[i], CommitPayloads[i].GetDeserializedMessage().Signature); + j++; + } sc.Verifiable.Witnesses = sc.GetWitnesses(); block.Transactions = TransactionHashes.Select(p => Transactions[p]).ToArray(); return block; @@ -94,47 +93,48 @@ public void Deserialize(BinaryReader reader) transactions[i] = Transaction.DeserializeFrom(reader); Transactions = transactions.ToDictionary(p => p.Hash); } - Preparations = reader.ReadSerializableArray(); - for (int i = 0; i < Preparations.Length; i++) - if (Preparations[i].Equals(UInt256.Zero)) - Preparations[i] = null; - Commits = new byte[reader.ReadVarInt()][]; - for (int i = 0; i < Commits.Length; i++) - { - Commits[i] = reader.ReadVarBytes(); - if (Commits[i].Length == 0) - Commits[i] = null; - } - ExpectedView = reader.ReadVarBytes(); + PreparationPayloads = new ConsensusPayload[reader.ReadVarInt()]; + for (int i = 0; i < PreparationPayloads.Length; i++) + PreparationPayloads[i] = reader.ReadBoolean() ? reader.ReadSerializable() : null; + CommitPayloads = new ConsensusPayload[reader.ReadVarInt()]; + for (int i = 0; i < CommitPayloads.Length; i++) + CommitPayloads[i] = reader.ReadBoolean() ? reader.ReadSerializable() : null; + ChangeViewPayloads = new ConsensusPayload[reader.ReadVarInt()]; + for (int i = 0; i < ChangeViewPayloads.Length; i++) + ChangeViewPayloads[i] = reader.ReadBoolean() ? reader.ReadSerializable() : null; + LastChangeViewPayloads = new ConsensusPayload[reader.ReadVarInt()]; + for (int i = 0; i < LastChangeViewPayloads.Length; i++) + LastChangeViewPayloads[i] = reader.ReadBoolean() ? reader.ReadSerializable() : null; } public void Dispose() { - snapshot?.Dispose(); + Snapshot?.Dispose(); } - public uint GetPrimaryIndex(byte view_number) + public uint GetPrimaryIndex(byte viewNumber) { - int p = ((int)BlockIndex - view_number) % Validators.Length; + int p = ((int)BlockIndex - viewNumber) % Validators.Length; return p >= 0 ? (uint)p : (uint)(p + Validators.Length); } - public ConsensusPayload MakeChangeView() + public ConsensusPayload MakeChangeView(byte newViewNumber) { - return MakeSignedPayload(new ChangeView + return ChangeViewPayloads[MyIndex] = MakeSignedPayload(new ChangeView { - NewViewNumber = ExpectedView[MyIndex] + NewViewNumber = newViewNumber, + Timestamp = TimeProvider.Current.UtcNow.ToTimestamp() }); } public ConsensusPayload MakeCommit() { - if (Commits[MyIndex] == null) - Commits[MyIndex] = MakeHeader()?.Sign(keyPair); - return MakeSignedPayload(new Commit - { - Signature = Commits[MyIndex] - }); + if (CommitPayloads[MyIndex] == null) + CommitPayloads[MyIndex] = MakeSignedPayload(new Commit + { + Signature = MakeHeader()?.Sign(keyPair) + }); + return CommitPayloads[MyIndex]; } private Block _header = null; @@ -167,8 +167,7 @@ private ConsensusPayload MakeSignedPayload(ConsensusMessage message) PrevHash = PrevHash, BlockIndex = BlockIndex, ValidatorIndex = (ushort)MyIndex, - Timestamp = Timestamp, - Data = message.ToArray() + ConsensusMessage = message }; SignPayload(payload); return payload; @@ -193,6 +192,7 @@ public ConsensusPayload MakePrepareRequest() { return MakeSignedPayload(new PrepareRequest { + Timestamp = Timestamp, Nonce = Nonce, NextConsensus = NextConsensus, TransactionHashes = TransactionHashes, @@ -200,45 +200,79 @@ public ConsensusPayload MakePrepareRequest() }); } - public ConsensusPayload MakePrepareResponse(UInt256 preparation) + public ConsensusPayload MakeRecoveryMessage() + { + PrepareRequest prepareRequestMessage = null; + if (TransactionHashes != null) + { + prepareRequestMessage = new PrepareRequest + { + ViewNumber = ViewNumber, + TransactionHashes = TransactionHashes, + Nonce = Nonce, + NextConsensus = NextConsensus, + MinerTransaction = (MinerTransaction)Transactions[TransactionHashes[0]], + Timestamp = Timestamp + }; + } + return MakeSignedPayload(new RecoveryMessage() + { + ChangeViewMessages = LastChangeViewPayloads.Where(p => p != null).Select(p => RecoveryMessage.ChangeViewPayloadCompact.FromPayload(p)).Take(M).ToDictionary(p => (int)p.ValidatorIndex), + PrepareRequestMessage = prepareRequestMessage, + // We only need a PreparationHash set if we don't have the PrepareRequest information. + PreparationHash = TransactionHashes == null ? PreparationPayloads.Where(p => p != null).GroupBy(p => p.GetDeserializedMessage().PreparationHash, (k, g) => new { Hash = k, Count = g.Count() }).OrderByDescending(p => p.Count).Select(p => p.Hash).FirstOrDefault() : null, + PreparationMessages = PreparationPayloads.Where(p => p != null).Select(p => RecoveryMessage.PreparationPayloadCompact.FromPayload(p)).ToDictionary(p => (int)p.ValidatorIndex), + CommitMessages = State.HasFlag(ConsensusState.CommitSent) + ? CommitPayloads.Where(p => p != null).Select(p => RecoveryMessage.CommitPayloadCompact.FromPayload(p)).ToDictionary(p => (int)p.ValidatorIndex) + : new Dictionary() + }); + } + + public ConsensusPayload MakePrepareResponse() { - return MakeSignedPayload(new PrepareResponse + return PreparationPayloads[MyIndex] = MakeSignedPayload(new PrepareResponse { - PreparationHash = preparation + PreparationHash = PreparationPayloads[PrimaryIndex].Hash }); } - public void Reset(byte view_number) + public void Reset(byte viewNumber) { - if (view_number == 0) + if (viewNumber == 0) { - snapshot?.Dispose(); - snapshot = Blockchain.Singleton.GetSnapshot(); - PrevHash = snapshot.CurrentBlockHash; - BlockIndex = snapshot.Height + 1; - Validators = snapshot.GetValidators(); + Snapshot?.Dispose(); + Snapshot = Blockchain.Singleton.GetSnapshot(); + PrevHash = Snapshot.CurrentBlockHash; + BlockIndex = Snapshot.Height + 1; + Validators = Snapshot.GetValidators(); MyIndex = -1; - ExpectedView = new byte[Validators.Length]; + ChangeViewPayloads = new ConsensusPayload[Validators.Length]; + LastChangeViewPayloads = new ConsensusPayload[Validators.Length]; keyPair = null; for (int i = 0; i < Validators.Length; i++) { WalletAccount account = wallet.GetAccount(Validators[i]); - if (account?.HasKey == true) - { - MyIndex = i; - keyPair = account.GetKey(); - break; - } + if (account?.HasKey != true) continue; + MyIndex = i; + keyPair = account.GetKey(); + break; } } + else + { + for (int i = 0; i < LastChangeViewPayloads.Length; i++) + if (ChangeViewPayloads[i]?.GetDeserializedMessage().NewViewNumber == viewNumber) + LastChangeViewPayloads[i] = ChangeViewPayloads[i]; + else + LastChangeViewPayloads[i] = null; + } State = ConsensusState.Initial; - ViewNumber = view_number; - PrimaryIndex = GetPrimaryIndex(view_number); + ViewNumber = viewNumber; + PrimaryIndex = GetPrimaryIndex(viewNumber); + Timestamp = 0; TransactionHashes = null; - Preparations = new UInt256[Validators.Length]; - Commits = new byte[Validators.Length][]; - if (MyIndex >= 0) - ExpectedView[MyIndex] = view_number; + PreparationPayloads = new ConsensusPayload[Validators.Length]; + CommitPayloads = new ConsensusPayload[Validators.Length]; _header = null; } @@ -257,32 +291,51 @@ public void Serialize(BinaryWriter writer) writer.Write(NextConsensus ?? UInt160.Zero); writer.Write(TransactionHashes ?? new UInt256[0]); writer.Write(Transactions?.Values.ToArray() ?? new Transaction[0]); - writer.WriteVarInt(Preparations.Length); - foreach (UInt256 hash in Preparations) - if (hash is null) - writer.Write(UInt256.Zero); - else - writer.Write(hash); - writer.WriteVarInt(Commits.Length); - foreach (byte[] commit in Commits) - if (commit is null) - writer.WriteVarInt(0); - else - writer.WriteVarBytes(commit); - writer.WriteVarBytes(ExpectedView); + writer.WriteVarInt(PreparationPayloads.Length); + foreach (var payload in PreparationPayloads) + { + bool hasPayload = !(payload is null); + writer.Write(hasPayload); + if (!hasPayload) continue; + writer.Write(payload); + } + writer.WriteVarInt(CommitPayloads.Length); + foreach (var payload in CommitPayloads) + { + bool hasPayload = !(payload is null); + writer.Write(hasPayload); + if (!hasPayload) continue; + writer.Write(payload); + } + writer.WriteVarInt(ChangeViewPayloads.Length); + foreach (var payload in ChangeViewPayloads) + { + bool hasPayload = !(payload is null); + writer.Write(hasPayload); + if (!hasPayload) continue; + writer.Write(payload); + } + writer.WriteVarInt(LastChangeViewPayloads.Length); + foreach (var payload in LastChangeViewPayloads) + { + bool hasPayload = !(payload is null); + writer.Write(hasPayload); + if (!hasPayload) continue; + writer.Write(payload); + } } public void Fill() { - IEnumerable mem_pool = Blockchain.Singleton.MemPool.GetSortedVerifiedTransactions(); + IEnumerable memoryPoolTransactions = Blockchain.Singleton.MemPool.GetSortedVerifiedTransactions(); foreach (IPolicyPlugin plugin in Plugin.Policies) - mem_pool = plugin.FilterForBlock(mem_pool); - List transactions = mem_pool.ToList(); - Fixed8 amount_netfee = Block.CalculateNetFee(transactions); - TransactionOutput[] outputs = amount_netfee == Fixed8.Zero ? new TransactionOutput[0] : new[] { new TransactionOutput + memoryPoolTransactions = plugin.FilterForBlock(memoryPoolTransactions); + List transactions = memoryPoolTransactions.ToList(); + Fixed8 amountNetFee = Block.CalculateNetFee(transactions); + TransactionOutput[] outputs = amountNetFee == Fixed8.Zero ? new TransactionOutput[0] : new[] { new TransactionOutput { AssetId = Blockchain.UtilityToken.Hash, - Value = amount_netfee, + Value = amountNetFee, ScriptHash = wallet.GetChangeAddress() } }; while (true) @@ -296,7 +349,7 @@ public void Fill() Outputs = outputs, Witnesses = new Witness[0] }; - if (!snapshot.ContainsTransaction(tx.Hash)) + if (!Snapshot.ContainsTransaction(tx.Hash)) { Nonce = nonce; transactions.Insert(0, tx); @@ -305,7 +358,7 @@ public void Fill() } TransactionHashes = transactions.Select(p => p.Hash).ToArray(); Transactions = transactions.ToDictionary(p => p.Hash); - NextConsensus = Blockchain.GetConsensusAddress(snapshot.GetValidators(transactions).ToArray()); + NextConsensus = Blockchain.GetConsensusAddress(Snapshot.GetValidators(transactions).ToArray()); Timestamp = Math.Max(TimeProvider.Current.UtcNow.ToTimestamp(), PrevHeader.Timestamp + 1); } @@ -321,11 +374,11 @@ public bool VerifyRequest() { if (!State.HasFlag(ConsensusState.RequestReceived)) return false; - if (!Blockchain.GetConsensusAddress(snapshot.GetValidators(Transactions.Values).ToArray()).Equals(NextConsensus)) + if (!Blockchain.GetConsensusAddress(Snapshot.GetValidators(Transactions.Values).ToArray()).Equals(NextConsensus)) return false; - Transaction tx_gen = Transactions.Values.FirstOrDefault(p => p.Type == TransactionType.MinerTransaction); - Fixed8 amount_netfee = Block.CalculateNetFee(Transactions.Values); - if (tx_gen?.Outputs.Sum(p => p.Value) != amount_netfee) return false; + Transaction minerTx = Transactions.Values.FirstOrDefault(p => p.Type == TransactionType.MinerTransaction); + Fixed8 amountNetFee = Block.CalculateNetFee(Transactions.Values); + if (minerTx?.Outputs.Sum(p => p.Value) != amountNetFee) return false; return true; } } diff --git a/neo/Consensus/ConsensusMessageType.cs b/neo/Consensus/ConsensusMessageType.cs index d134f3bfb2..4fee3acdb4 100644 --- a/neo/Consensus/ConsensusMessageType.cs +++ b/neo/Consensus/ConsensusMessageType.cs @@ -6,11 +6,15 @@ internal enum ConsensusMessageType : byte { [ReflectionCache(typeof(ChangeView))] ChangeView = 0x00, + [ReflectionCache(typeof(PrepareRequest))] PrepareRequest = 0x20, [ReflectionCache(typeof(PrepareResponse))] PrepareResponse = 0x21, [ReflectionCache(typeof(Commit))] Commit = 0x30, + + [ReflectionCache(typeof(RecoveryMessage))] + RecoveryMessage = 0x41, } } diff --git a/neo/Consensus/ConsensusService.cs b/neo/Consensus/ConsensusService.cs index 49816a70d0..b0dff4a5a9 100644 --- a/neo/Consensus/ConsensusService.cs +++ b/neo/Consensus/ConsensusService.cs @@ -31,6 +31,11 @@ internal class Timer { public uint Height; public byte ViewNumber; } private ICancelable timer_token; private DateTime block_received_time; private bool started = false; + /// + /// This will be cleared every block (so it will not grow out of control, but is used to prevent repeatedly + /// responding to the same message. + /// + private readonly HashSet knownHashes = new HashSet(); public ConsensusService(IActorRef localNode, IActorRef taskManager, Store store, Wallet wallet) : this(localNode, taskManager, store, new ConsensusContext(wallet)) @@ -45,10 +50,9 @@ public ConsensusService(IActorRef localNode, IActorRef taskManager, Store store, this.context = context; } - private bool AddTransaction(Transaction tx, bool verify) { - if (verify && !context.VerifyTransaction(tx)) + if (verify && !tx.Verify(context.Snapshot, context.Transactions.Values)) { Log($"Invalid transaction: {tx.Hash}{Environment.NewLine}{tx.ToArray().ToHexString()}", LogLevel.Warning); RequestChangeView(); @@ -65,10 +69,13 @@ private bool AddTransaction(Transaction tx, bool verify) { if (context.VerifyRequest()) { + // if we are the primary for this view, but acting as a backup because we recovered our own + // previously sent prepare request, then we don't want to send a prepare response. + if (context.MyIndex == context.PrimaryIndex) return true; + Log($"send prepare response"); context.State |= ConsensusState.ResponseSent; - context.Preparations[context.MyIndex] = context.Preparations[context.PrimaryIndex]; - localNode.Tell(new LocalNode.SendDirectly { Inventory = context.MakePrepareResponse(context.Preparations[context.MyIndex]) }); + localNode.Tell(new LocalNode.SendDirectly { Inventory = context.MakePrepareResponse() }); CheckPreparations(); } else @@ -92,7 +99,7 @@ private void ChangeTimer(TimeSpan delay) private void CheckCommits() { - if (context.Commits.Count(p => p != null) >= context.M && context.TransactionHashes.All(p => context.Transactions.ContainsKey(p))) + if (context.CommitPayloads.Count(p => p != null) >= context.M && context.TransactionHashes.All(p => context.Transactions.ContainsKey(p))) { Block block = context.CreateBlock(); Log($"relay block: {block.Hash}"); @@ -101,35 +108,44 @@ private void CheckCommits() } } - private void CheckExpectedView(byte view_number) + private void CheckExpectedView(byte viewNumber) { - if (context.ViewNumber == view_number) return; - if (context.ExpectedView.Count(p => p == view_number) >= context.M) - { - InitializeConsensus(view_number); - } + if (context.ViewNumber == viewNumber) return; + if (context.ChangeViewPayloads.Count(p => p != null && p.GetDeserializedMessage().NewViewNumber == viewNumber) >= context.M) + InitializeConsensus(viewNumber); } private void CheckPreparations() { - if (context.Preparations.Count(p => p != null) >= context.M && context.TransactionHashes.All(p => context.Transactions.ContainsKey(p))) + if (context.PreparationPayloads.Count(p => p != null) >= context.M && context.TransactionHashes.All(p => context.Transactions.ContainsKey(p))) { ConsensusPayload payload = context.MakeCommit(); Log($"send commit"); context.State |= ConsensusState.CommitSent; store.Put(ContextSerializationPrefix, new byte[0], context.ToArray()); localNode.Tell(new LocalNode.SendDirectly { Inventory = payload }); + // Set timer, so we will resend the commit in case of a networking issue + ChangeTimer(TimeSpan.FromSeconds(Blockchain.SecondsPerBlock)); CheckCommits(); } } - private void InitializeConsensus(byte view_number) + private byte GetLastExpectedView(int validatorIndex) + { + var lastPreparationPayload = context.PreparationPayloads[validatorIndex]; + if (lastPreparationPayload != null) + return lastPreparationPayload.GetDeserializedMessage().ViewNumber; + + return context.ChangeViewPayloads[validatorIndex]?.GetDeserializedMessage().NewViewNumber ?? (byte)0; + } + + private void InitializeConsensus(byte viewNumber) { - context.Reset(view_number); + context.Reset(viewNumber); if (context.MyIndex < 0) return; - if (view_number > 0) - Log($"changeview: view={view_number} primary={context.Validators[context.GetPrimaryIndex((byte)(view_number - 1u))]}", LogLevel.Warning); - Log($"initialize: height={context.BlockIndex} view={view_number} index={context.MyIndex} role={(context.MyIndex == context.PrimaryIndex ? ConsensusState.Primary : ConsensusState.Backup)}"); + if (viewNumber > 0) + Log($"changeview: view={viewNumber} primary={context.Validators[context.GetPrimaryIndex((byte)(viewNumber - 1u))]}", LogLevel.Warning); + Log($"initialize: height={context.BlockIndex} view={viewNumber} index={context.MyIndex} role={(context.MyIndex == context.PrimaryIndex ? ConsensusState.Primary : ConsensusState.Backup)}"); if (context.MyIndex == context.PrimaryIndex) { context.State |= ConsensusState.Primary; @@ -142,7 +158,7 @@ private void InitializeConsensus(byte view_number) else { context.State = ConsensusState.Backup; - ChangeTimer(TimeSpan.FromSeconds(Blockchain.SecondsPerBlock << (view_number + 1))); + ChangeTimer(TimeSpan.FromSeconds(Blockchain.SecondsPerBlock << (viewNumber + 1))); } } @@ -153,27 +169,60 @@ private void Log(string message, LogLevel level = LogLevel.Info) private void OnChangeViewReceived(ConsensusPayload payload, ChangeView message) { - if (context.State.HasFlag(ConsensusState.CommitSent)) + // Node in commit receiving ChangeView should always send the recovery message. + bool shouldSendRecovery = context.State.HasFlag(ConsensusState.CommitSent); + if (shouldSendRecovery || message.NewViewNumber < context.ViewNumber) + { + if (!shouldSendRecovery) + { + // Limit recovery to sending from `f` nodes when the request is from a lower view number. + int allowedRecoveryNodeCount = context.F; + for (int i = 0; i < allowedRecoveryNodeCount; i++) + { + var eligibleResponders = context.Validators.Length - 1; + var chosenIndex = (payload.ValidatorIndex + i + message.NewViewNumber) % eligibleResponders; + if (chosenIndex >= payload.ValidatorIndex) chosenIndex++; + if (chosenIndex != context.MyIndex) continue; + shouldSendRecovery = true; + break; + } + } + + // We keep track of the payload hashes received in this block, and don't respond with recovery + // in response to the same payload that we already responded to previously. + // ChangeView messages include a Timestamp when the change view is sent, thus if a node restarts + // and issues a change view for the same view, it will have a different hash and will correctly respond + // again; however replay attacks of the ChangeView message from arbitrary nodes will not trigger an + // additonal recovery message response. + if (!shouldSendRecovery || knownHashes.Contains(payload.Hash)) return; + knownHashes.Add(payload.Hash); + + Log($"send recovery from view: {message.ViewNumber} to view: {context.ViewNumber}"); + localNode.Tell(new LocalNode.SendDirectly { Inventory = context.MakeRecoveryMessage() }); return; - if (message.NewViewNumber <= context.ExpectedView[payload.ValidatorIndex]) + } + + var expectedView = GetLastExpectedView(payload.ValidatorIndex); + if (message.NewViewNumber <= expectedView) return; + Log($"{nameof(OnChangeViewReceived)}: height={payload.BlockIndex} view={message.ViewNumber} index={payload.ValidatorIndex} nv={message.NewViewNumber}"); - context.ExpectedView[payload.ValidatorIndex] = message.NewViewNumber; + context.ChangeViewPayloads[payload.ValidatorIndex] = payload; CheckExpectedView(message.NewViewNumber); } private void OnCommitReceived(ConsensusPayload payload, Commit commit) { - if (context.Commits[payload.ValidatorIndex] != null) return; + if (context.CommitPayloads[payload.ValidatorIndex] != null) return; Log($"{nameof(OnCommitReceived)}: height={payload.BlockIndex} view={commit.ViewNumber} index={payload.ValidatorIndex}"); byte[] hashData = context.MakeHeader()?.GetHashData(); if (hashData == null) { - context.Commits[payload.ValidatorIndex] = commit.Signature; + context.CommitPayloads[payload.ValidatorIndex] = payload; } else if (Crypto.Default.VerifySignature(hashData, commit.Signature, context.Validators[payload.ValidatorIndex].EncodePoint(false))) { - context.Commits[payload.ValidatorIndex] = commit.Signature; + context.CommitPayloads[payload.ValidatorIndex] = payload; CheckCommits(); } } @@ -181,9 +230,7 @@ private void OnCommitReceived(ConsensusPayload payload, Commit commit) private void OnConsensusPayload(ConsensusPayload payload) { if (context.State.HasFlag(ConsensusState.BlockSent)) return; - if (payload.ValidatorIndex == context.MyIndex) return; - if (payload.Version != ConsensusContext.Version) - return; + if (payload.Version != ConsensusContext.Version) return; if (payload.PrevHash != context.PrevHash || payload.BlockIndex != context.BlockIndex) { if (context.BlockIndex < payload.BlockIndex) @@ -193,16 +240,9 @@ private void OnConsensusPayload(ConsensusPayload payload) return; } if (payload.ValidatorIndex >= context.Validators.Length) return; - ConsensusMessage message; - try - { - message = ConsensusMessage.DeserializeFrom(payload.Data); - } - catch - { - return; - } - if (message.ViewNumber != context.ViewNumber && message.Type != ConsensusMessageType.ChangeView) + ConsensusMessage message = payload.ConsensusMessage; + if (message.ViewNumber != context.ViewNumber && message.Type != ConsensusMessageType.ChangeView && + message.Type != ConsensusMessageType.RecoveryMessage) return; switch (message) { @@ -218,6 +258,9 @@ private void OnConsensusPayload(ConsensusPayload payload) case Commit commit: OnCommitReceived(payload, commit); break; + case RecoveryMessage recovery: + OnRecoveryMessageReceived(payload, recovery); + break; } } @@ -225,41 +268,71 @@ private void OnPersistCompleted(Block block) { Log($"persist block: {block.Hash}"); block_received_time = TimeProvider.Current.UtcNow; + knownHashes.Clear(); InitializeConsensus(0); } + private void OnRecoveryMessageReceived(ConsensusPayload payload, RecoveryMessage message) + { + if (message.ViewNumber < context.ViewNumber) return; + Log($"{nameof(OnRecoveryMessageReceived)}: height={payload.BlockIndex} view={message.ViewNumber} index={payload.ValidatorIndex}"); + if (message.ViewNumber > context.ViewNumber) + { + if (context.State.HasFlag(ConsensusState.CommitSent)) + return; + ConsensusPayload[] changeViewPayloads = message.GetChangeViewPayloads(context, payload); + foreach (ConsensusPayload changeViewPayload in changeViewPayloads) + ReverifyAndProcessPayload(changeViewPayload); + } + if (message.ViewNumber != context.ViewNumber) return; + if (!context.State.HasFlag(ConsensusState.CommitSent)) + { + ConsensusPayload prepareRequestPayload = message.GetPrepareRequestPayload(context, payload); + if (prepareRequestPayload != null && !context.State.HasFlag(ConsensusState.RequestSent) && !context.State.HasFlag(ConsensusState.RequestReceived)) + ReverifyAndProcessPayload(prepareRequestPayload); + ConsensusPayload[] prepareResponsePayloads = message.GetPrepareResponsePayloads(context, payload, prepareRequestPayload); + foreach (ConsensusPayload prepareResponsePayload in prepareResponsePayloads) + ReverifyAndProcessPayload(prepareResponsePayload); + } + ConsensusPayload[] commitPayloads = message.GetCommitPayloadsFromRecoveryMessage(context, payload); + foreach (ConsensusPayload commitPayload in commitPayloads) + ReverifyAndProcessPayload(commitPayload); + } + private void OnPrepareRequestReceived(ConsensusPayload payload, PrepareRequest message) { - if (context.State.HasFlag(ConsensusState.RequestReceived)) return; + if (context.State.HasFlag(ConsensusState.RequestSent) || context.State.HasFlag(ConsensusState.RequestReceived)) + return; if (payload.ValidatorIndex != context.PrimaryIndex) return; Log($"{nameof(OnPrepareRequestReceived)}: height={payload.BlockIndex} view={message.ViewNumber} index={payload.ValidatorIndex} tx={message.TransactionHashes.Length}"); - if (!context.State.HasFlag(ConsensusState.Backup)) return; - if (payload.Timestamp <= context.PrevHeader.Timestamp || payload.Timestamp > TimeProvider.Current.UtcNow.AddMinutes(10).ToTimestamp()) + if (message.Timestamp <= context.PrevHeader.Timestamp || message.Timestamp > TimeProvider.Current.UtcNow.AddMinutes(10).ToTimestamp()) { - Log($"Timestamp incorrect: {payload.Timestamp}", LogLevel.Warning); + Log($"Timestamp incorrect: {message.Timestamp}", LogLevel.Warning); return; } - if (message.TransactionHashes.Any(p => context.TransactionExists(p))) + if (message.TransactionHashes.Any(p => context.Snapshot.ContainsTransaction(p))) { Log($"Invalid request: transaction already exists", LogLevel.Warning); return; } - context.State |= ConsensusState.RequestReceived; - context.Timestamp = payload.Timestamp; + context.State |= context.State.HasFlag(ConsensusState.Primary) + ? ConsensusState.RequestSent + : ConsensusState.RequestReceived; + context.Timestamp = message.Timestamp; context.Nonce = message.Nonce; context.NextConsensus = message.NextConsensus; context.TransactionHashes = message.TransactionHashes; context.Transactions = new Dictionary(); - for (int i = 0; i < context.Preparations.Length; i++) - if (context.Preparations[i] != null) - if (!context.Preparations[i].Equals(payload.Hash)) - context.Preparations[i] = null; - context.Preparations[payload.ValidatorIndex] = payload.Hash; + for (int i = 0; i < context.PreparationPayloads.Length; i++) + if (context.PreparationPayloads[i] != null) + if (!context.PreparationPayloads[i].Hash.Equals(payload.Hash)) + context.PreparationPayloads[i] = null; + context.PreparationPayloads[payload.ValidatorIndex] = payload; byte[] hashData = context.MakeHeader().GetHashData(); - for (int i = 0; i < context.Commits.Length; i++) - if (context.Commits[i] != null) - if (!Crypto.Default.VerifySignature(hashData, context.Commits[i], context.Validators[i].EncodePoint(false))) - context.Commits[i] = null; + for (int i = 0; i < context.CommitPayloads.Length; i++) + if (context.CommitPayloads[i] != null) + if (!Crypto.Default.VerifySignature(hashData, context.CommitPayloads[i].GetDeserializedMessage().Signature, context.Validators[i].EncodePoint(false))) + context.CommitPayloads[i] = null; Dictionary mempoolVerified = Blockchain.Singleton.MemPool.GetVerifiedTransactions().ToDictionary(p => p.Hash); List unverified = new List(); @@ -272,7 +345,6 @@ private void OnPrepareRequestReceived(ConsensusPayload payload, PrepareRequest m } else { - if (Blockchain.Singleton.MemPool.TryGetValue(hash, out tx)) unverified.Add(tx); } @@ -293,12 +365,14 @@ private void OnPrepareRequestReceived(ConsensusPayload payload, PrepareRequest m private void OnPrepareResponseReceived(ConsensusPayload payload, PrepareResponse message) { - if (context.Preparations[payload.ValidatorIndex] != null) return; - if (context.Preparations[context.PrimaryIndex] != null && !message.PreparationHash.Equals(context.Preparations[context.PrimaryIndex])) + if (context.PreparationPayloads[payload.ValidatorIndex] != null) return; + if (context.PreparationPayloads[context.PrimaryIndex] != null && !message.PreparationHash.Equals(context.PreparationPayloads[context.PrimaryIndex].Hash)) return; Log($"{nameof(OnPrepareResponseReceived)}: height={payload.BlockIndex} view={message.ViewNumber} index={payload.ValidatorIndex}"); if (context.State.HasFlag(ConsensusState.CommitSent)) return; - context.Preparations[payload.ValidatorIndex] = message.PreparationHash; + context.PreparationPayloads[payload.ValidatorIndex] = payload; + if (payload.ValidatorIndex == context.MyIndex) + context.State |= ConsensusState.ResponseSent; if (context.State.HasFlag(ConsensusState.RequestSent) || context.State.HasFlag(ConsensusState.RequestReceived)) CheckPreparations(); } @@ -347,10 +421,15 @@ private void OnStart() context.Deserialize(reader); } } - if (context.BlockIndex == Blockchain.Singleton.Height + 1 && context.State.HasFlag(ConsensusState.CommitSent)) + if (context.State.HasFlag(ConsensusState.CommitSent) && context.BlockIndex == Blockchain.Singleton.Height + 1) CheckPreparations(); else + { InitializeConsensus(0); + // Issue a ChangeView with NewViewNumber of 0 to request recovery messages on start-up. + if (context.BlockIndex == Blockchain.Singleton.HeaderHeight + 1) + localNode.Tell(new LocalNode.SendDirectly { Inventory = context.MakeChangeView(0) }); + } } private void OnTimer(Timer timer) @@ -362,10 +441,11 @@ private void OnTimer(Timer timer) { Log($"send prepare request: height={timer.Height} view={timer.ViewNumber}"); context.Fill(); - ConsensusPayload request = context.MakePrepareRequest(); - localNode.Tell(new LocalNode.SendDirectly { Inventory = request }); + ConsensusPayload prepareRequestPayload = context.MakePrepareRequest(); + localNode.Tell(new LocalNode.SendDirectly { Inventory = prepareRequestPayload }); context.State |= ConsensusState.RequestSent; - context.Preparations[context.MyIndex] = request.Hash; + context.PreparationPayloads[context.MyIndex] = prepareRequestPayload; + if (context.TransactionHashes.Length > 1) { foreach (InvPayload payload in InvPayload.CreateGroup(InventoryType.TX, context.TransactionHashes.Skip(1).ToArray())) @@ -375,16 +455,29 @@ private void OnTimer(Timer timer) } else if ((context.State.HasFlag(ConsensusState.Primary) && context.State.HasFlag(ConsensusState.RequestSent)) || context.State.HasFlag(ConsensusState.Backup)) { - if (!context.State.HasFlag(ConsensusState.CommitSent)) + if (context.State.HasFlag(ConsensusState.CommitSent)) + { + // Re-send commit periodically by sending recover message in case of a network issue. + Log($"send recovery to resend commit"); + localNode.Tell(new LocalNode.SendDirectly { Inventory = context.MakeRecoveryMessage() }); + ChangeTimer(TimeSpan.FromSeconds(Blockchain.SecondsPerBlock << 1)); + } + else + { RequestChangeView(); + } } } private void OnTransaction(Transaction transaction) { if (transaction.Type == TransactionType.MinerTransaction) return; - if (!context.State.HasFlag(ConsensusState.Backup) || !context.State.HasFlag(ConsensusState.RequestReceived) || context.State.HasFlag(ConsensusState.ResponseSent) || context.State.HasFlag(ConsensusState.ViewChanging) || context.State.HasFlag(ConsensusState.BlockSent)) + if (!context.State.HasFlag(ConsensusState.Backup) || !context.State.HasFlag(ConsensusState.RequestReceived) || context.State.HasFlag(ConsensusState.ResponseSent) || context.State.HasFlag(ConsensusState.BlockSent)) return; + // If we are changing view but we already have enough preparation payloads to commit in the current view, + // we must keep on accepting transactions in the current view to be able to create the block. + if (context.State.HasFlag(ConsensusState.ViewChanging) && + context.PreparationPayloads.Count(p => p != null) < context.M) return; if (context.Transactions.ContainsKey(transaction.Hash)) return; if (!context.TransactionHashes.Contains(transaction.Hash)) return; AddTransaction(transaction, true); @@ -406,11 +499,18 @@ public static Props Props(IActorRef localNode, IActorRef taskManager, Store stor private void RequestChangeView() { context.State |= ConsensusState.ViewChanging; - context.ExpectedView[context.MyIndex]++; - Log($"request change view: height={context.BlockIndex} view={context.ViewNumber} nv={context.ExpectedView[context.MyIndex]} state={context.State}"); - ChangeTimer(TimeSpan.FromSeconds(Blockchain.SecondsPerBlock << (context.ExpectedView[context.MyIndex] + 1))); - localNode.Tell(new LocalNode.SendDirectly { Inventory = context.MakeChangeView() }); - CheckExpectedView(context.ExpectedView[context.MyIndex]); + byte expectedView = GetLastExpectedView(context.MyIndex); + expectedView++; + Log($"request change view: height={context.BlockIndex} view={context.ViewNumber} nv={expectedView} state={context.State}"); + ChangeTimer(TimeSpan.FromSeconds(Blockchain.SecondsPerBlock << (expectedView + 1))); + localNode.Tell(new LocalNode.SendDirectly { Inventory = context.MakeChangeView(expectedView) }); + CheckExpectedView(expectedView); + } + + private void ReverifyAndProcessPayload(ConsensusPayload payload) + { + if (!payload.Verify(context.Snapshot)) return; + OnConsensusPayload(payload); } } diff --git a/neo/Consensus/IConsensusContext.cs b/neo/Consensus/IConsensusContext.cs index a482db7988..3a99d33d61 100644 --- a/neo/Consensus/IConsensusContext.cs +++ b/neo/Consensus/IConsensusContext.cs @@ -1,6 +1,7 @@ using Neo.Cryptography.ECC; using Neo.IO; using Neo.Network.P2P.Payloads; +using Neo.Persistence; using System; using System.Collections.Generic; @@ -21,24 +22,23 @@ public interface IConsensusContext : IDisposable, ISerializable UInt160 NextConsensus { get; set; } UInt256[] TransactionHashes { get; set; } Dictionary Transactions { get; set; } - UInt256[] Preparations { get; set; } - byte[][] Commits { get; set; } - byte[] ExpectedView { get; set; } + ConsensusPayload[] PreparationPayloads { get; set; } + ConsensusPayload[] CommitPayloads { get; set; } + ConsensusPayload[] ChangeViewPayloads { get; set; } + Snapshot Snapshot { get; } + int F { get; } int M { get; } Header PrevHeader { get; } - bool TransactionExists(UInt256 hash); - bool VerifyTransaction(Transaction tx); - Block CreateBlock(); //void Dispose(); - uint GetPrimaryIndex(byte view_number); + uint GetPrimaryIndex(byte viewNumber); - ConsensusPayload MakeChangeView(); + ConsensusPayload MakeChangeView(byte newViewNumber); ConsensusPayload MakeCommit(); @@ -46,9 +46,11 @@ public interface IConsensusContext : IDisposable, ISerializable ConsensusPayload MakePrepareRequest(); - ConsensusPayload MakePrepareResponse(UInt256 preparation); + ConsensusPayload MakeRecoveryMessage(); + + ConsensusPayload MakePrepareResponse(); - void Reset(byte view_number); + void Reset(byte viewNumber); void Fill(); diff --git a/neo/Consensus/PrepareRequest.cs b/neo/Consensus/PrepareRequest.cs index 9ebbb678b0..7a422da13e 100644 --- a/neo/Consensus/PrepareRequest.cs +++ b/neo/Consensus/PrepareRequest.cs @@ -8,12 +8,18 @@ namespace Neo.Consensus { internal class PrepareRequest : ConsensusMessage { + public uint Timestamp; public ulong Nonce; public UInt160 NextConsensus; public UInt256[] TransactionHashes; public MinerTransaction MinerTransaction; - public override int Size => base.Size + sizeof(ulong) + NextConsensus.Size + TransactionHashes.GetVarSize() + MinerTransaction.Size; + public override int Size => base.Size + + sizeof(uint) //Timestamp + + sizeof(ulong) //Nonce + + NextConsensus.Size //NextConsensus + + TransactionHashes.GetVarSize() //TransactionHashes + + MinerTransaction.Size; //MinerTransaction public PrepareRequest() : base(ConsensusMessageType.PrepareRequest) @@ -23,9 +29,10 @@ public PrepareRequest() public override void Deserialize(BinaryReader reader) { base.Deserialize(reader); + Timestamp = reader.ReadUInt32(); Nonce = reader.ReadUInt64(); NextConsensus = reader.ReadSerializable(); - TransactionHashes = reader.ReadSerializableArray(); + TransactionHashes = reader.ReadSerializableArray(Block.MaxTransactionsPerBlock); if (TransactionHashes.Distinct().Count() != TransactionHashes.Length) throw new FormatException(); MinerTransaction = reader.ReadSerializable(); @@ -36,6 +43,7 @@ public override void Deserialize(BinaryReader reader) public override void Serialize(BinaryWriter writer) { base.Serialize(writer); + writer.Write(Timestamp); writer.Write(Nonce); writer.Write(NextConsensus); writer.Write(TransactionHashes); diff --git a/neo/Consensus/RecoveryMessage.ChangeViewPayloadCompact.cs b/neo/Consensus/RecoveryMessage.ChangeViewPayloadCompact.cs new file mode 100644 index 0000000000..373e7e75a3 --- /dev/null +++ b/neo/Consensus/RecoveryMessage.ChangeViewPayloadCompact.cs @@ -0,0 +1,51 @@ +using Neo.IO; +using Neo.Network.P2P.Payloads; +using System.IO; + +namespace Neo.Consensus +{ + partial class RecoveryMessage + { + public class ChangeViewPayloadCompact : ISerializable + { + public ushort ValidatorIndex; + public byte OriginalViewNumber; + public uint Timestamp; + public byte[] InvocationScript; + + int ISerializable.Size => + sizeof(ushort) + //ValidatorIndex + sizeof(byte) + //OriginalViewNumber + sizeof(uint) + //Timestamp + InvocationScript.GetVarSize(); //InvocationScript + + void ISerializable.Deserialize(BinaryReader reader) + { + ValidatorIndex = reader.ReadUInt16(); + OriginalViewNumber = reader.ReadByte(); + Timestamp = reader.ReadUInt32(); + InvocationScript = reader.ReadVarBytes(1024); + } + + public static ChangeViewPayloadCompact FromPayload(ConsensusPayload payload) + { + ChangeView message = payload.GetDeserializedMessage(); + return new ChangeViewPayloadCompact + { + ValidatorIndex = payload.ValidatorIndex, + OriginalViewNumber = message.ViewNumber, + Timestamp = message.Timestamp, + InvocationScript = payload.Witness.InvocationScript + }; + } + + void ISerializable.Serialize(BinaryWriter writer) + { + writer.Write(ValidatorIndex); + writer.Write(OriginalViewNumber); + writer.Write(Timestamp); + writer.WriteVarBytes(InvocationScript); + } + } + } +} diff --git a/neo/Consensus/RecoveryMessage.CommitPayloadCompact.cs b/neo/Consensus/RecoveryMessage.CommitPayloadCompact.cs new file mode 100644 index 0000000000..58dd0e34e5 --- /dev/null +++ b/neo/Consensus/RecoveryMessage.CommitPayloadCompact.cs @@ -0,0 +1,46 @@ +using Neo.IO; +using Neo.Network.P2P.Payloads; +using System.IO; + +namespace Neo.Consensus +{ + partial class RecoveryMessage + { + public class CommitPayloadCompact : ISerializable + { + public ushort ValidatorIndex; + public byte[] Signature; + public byte[] InvocationScript; + + int ISerializable.Size => + sizeof(ushort) + //ValidatorIndex + Signature.Length + //Signature + InvocationScript.GetVarSize(); //InvocationScript + + void ISerializable.Deserialize(BinaryReader reader) + { + ValidatorIndex = reader.ReadUInt16(); + Signature = reader.ReadBytes(64); + InvocationScript = reader.ReadVarBytes(1024); + } + + public static CommitPayloadCompact FromPayload(ConsensusPayload payload) + { + Commit message = payload.GetDeserializedMessage(); + return new CommitPayloadCompact + { + ValidatorIndex = payload.ValidatorIndex, + Signature = message.Signature, + InvocationScript = payload.Witness.InvocationScript + }; + } + + void ISerializable.Serialize(BinaryWriter writer) + { + writer.Write(ValidatorIndex); + writer.Write(Signature); + writer.WriteVarBytes(InvocationScript); + } + } + } +} diff --git a/neo/Consensus/RecoveryMessage.PreparationPayloadCompact.cs b/neo/Consensus/RecoveryMessage.PreparationPayloadCompact.cs new file mode 100644 index 0000000000..8c0e52f59f --- /dev/null +++ b/neo/Consensus/RecoveryMessage.PreparationPayloadCompact.cs @@ -0,0 +1,40 @@ +using Neo.IO; +using Neo.Network.P2P.Payloads; +using System.IO; + +namespace Neo.Consensus +{ + partial class RecoveryMessage + { + public class PreparationPayloadCompact : ISerializable + { + public ushort ValidatorIndex; + public byte[] InvocationScript; + + int ISerializable.Size => + sizeof(ushort) + //ValidatorIndex + InvocationScript.GetVarSize(); //InvocationScript + + void ISerializable.Deserialize(BinaryReader reader) + { + ValidatorIndex = reader.ReadUInt16(); + InvocationScript = reader.ReadVarBytes(1024); + } + + public static PreparationPayloadCompact FromPayload(ConsensusPayload payload) + { + return new PreparationPayloadCompact + { + ValidatorIndex = payload.ValidatorIndex, + InvocationScript = payload.Witness.InvocationScript + }; + } + + void ISerializable.Serialize(BinaryWriter writer) + { + writer.Write(ValidatorIndex); + writer.WriteVarBytes(InvocationScript); + } + } + } +} diff --git a/neo/Consensus/RecoveryMessage.cs b/neo/Consensus/RecoveryMessage.cs new file mode 100644 index 0000000000..addcac7eb1 --- /dev/null +++ b/neo/Consensus/RecoveryMessage.cs @@ -0,0 +1,148 @@ +using Neo.IO; +using Neo.Ledger; +using Neo.Network.P2P.Payloads; +using Neo.SmartContract; +using System.Collections.Generic; +using System.IO; +using System.Linq; + +namespace Neo.Consensus +{ + internal partial class RecoveryMessage : ConsensusMessage + { + public Dictionary ChangeViewMessages; + public PrepareRequest PrepareRequestMessage; + /// The PreparationHash in case the PrepareRequest hasn't been received yet. + /// This can be null if the PrepareRequest information is present, since it can be derived in that case. + public UInt256 PreparationHash; + public Dictionary PreparationMessages; + public Dictionary CommitMessages; + + public RecoveryMessage() : base(ConsensusMessageType.RecoveryMessage) + { + } + + public override void Deserialize(BinaryReader reader) + { + base.Deserialize(reader); + ChangeViewMessages = reader.ReadSerializableArray(Blockchain.MaxValidators).ToDictionary(p => (int)p.ValidatorIndex); + if (reader.ReadBoolean()) + PrepareRequestMessage = reader.ReadSerializable(); + else + { + int preparationHashSize = UInt256.Zero.Size; + if (preparationHashSize == (int)reader.ReadVarInt((ulong)preparationHashSize)) + PreparationHash = new UInt256(reader.ReadBytes(preparationHashSize)); + } + + PreparationMessages = reader.ReadSerializableArray(Blockchain.MaxValidators).ToDictionary(p => (int)p.ValidatorIndex); + CommitMessages = reader.ReadSerializableArray(Blockchain.MaxValidators).ToDictionary(p => (int)p.ValidatorIndex); + } + + internal ConsensusPayload[] GetChangeViewPayloads(IConsensusContext context, ConsensusPayload payload) + { + return ChangeViewMessages.Values.Select(p => new ConsensusPayload + { + Version = payload.Version, + PrevHash = payload.PrevHash, + BlockIndex = payload.BlockIndex, + ValidatorIndex = p.ValidatorIndex, + ConsensusMessage = new ChangeView + { + ViewNumber = p.OriginalViewNumber, + NewViewNumber = ViewNumber, + Timestamp = p.Timestamp + }, + Witness = new Witness + { + InvocationScript = p.InvocationScript, + VerificationScript = Contract.CreateSignatureRedeemScript(context.Validators[p.ValidatorIndex]) + } + }).ToArray(); + } + + internal ConsensusPayload[] GetCommitPayloadsFromRecoveryMessage(IConsensusContext context, ConsensusPayload payload) + { + return CommitMessages.Values.Select(p => new ConsensusPayload + { + Version = payload.Version, + PrevHash = payload.PrevHash, + BlockIndex = payload.BlockIndex, + ValidatorIndex = p.ValidatorIndex, + ConsensusMessage = new Commit + { + ViewNumber = ViewNumber, + Signature = p.Signature + }, + Witness = new Witness + { + InvocationScript = p.InvocationScript, + VerificationScript = Contract.CreateSignatureRedeemScript(context.Validators[p.ValidatorIndex]) + } + }).ToArray(); + } + + internal ConsensusPayload GetPrepareRequestPayload(IConsensusContext context, ConsensusPayload payload) + { + if (PrepareRequestMessage == null) return null; + if (!PreparationMessages.TryGetValue((int)context.PrimaryIndex, out RecoveryMessage.PreparationPayloadCompact compact)) + return null; + return new ConsensusPayload + { + Version = payload.Version, + PrevHash = payload.PrevHash, + BlockIndex = payload.BlockIndex, + ValidatorIndex = (ushort)context.PrimaryIndex, + ConsensusMessage = PrepareRequestMessage, + Witness = new Witness + { + InvocationScript = compact.InvocationScript, + VerificationScript = Contract.CreateSignatureRedeemScript(context.Validators[context.PrimaryIndex]) + } + }; + } + + internal ConsensusPayload[] GetPrepareResponsePayloads(IConsensusContext context, ConsensusPayload payload, ConsensusPayload prepareRequestPayload = null) + { + UInt256 preparationHash = PreparationHash ?? prepareRequestPayload?.Hash; + if (preparationHash is null) return new ConsensusPayload[0]; + return PreparationMessages.Values.Where(p => p.ValidatorIndex != context.PrimaryIndex).Select(p => new ConsensusPayload + { + Version = payload.Version, + PrevHash = payload.PrevHash, + BlockIndex = payload.BlockIndex, + ValidatorIndex = p.ValidatorIndex, + ConsensusMessage = new PrepareResponse + { + ViewNumber = ViewNumber, + PreparationHash = preparationHash + }, + Witness = new Witness + { + InvocationScript = p.InvocationScript, + VerificationScript = Contract.CreateSignatureRedeemScript(context.Validators[p.ValidatorIndex]) + } + }).ToArray(); + } + + public override void Serialize(BinaryWriter writer) + { + base.Serialize(writer); + writer.Write(ChangeViewMessages.Values.ToArray()); + bool hasPrepareRequestMessage = PrepareRequestMessage != null; + writer.Write(hasPrepareRequestMessage); + if (hasPrepareRequestMessage) + writer.Write(PrepareRequestMessage); + else + { + if (PreparationHash == null) + writer.WriteVarInt(0); + else + writer.WriteVarBytes(PreparationHash.ToArray()); + } + + writer.Write(PreparationMessages.Values.ToArray()); + writer.Write(CommitMessages.Values.ToArray()); + } + } +} diff --git a/neo/Ledger/Blockchain.cs b/neo/Ledger/Blockchain.cs index 82dac883a6..a2cf256e76 100644 --- a/neo/Ledger/Blockchain.cs +++ b/neo/Ledger/Blockchain.cs @@ -27,7 +27,7 @@ public class ImportCompleted { } public static readonly uint SecondsPerBlock = ProtocolSettings.Default.SecondsPerBlock; public const uint DecrementInterval = 2000000; - public const uint MaxValidators = 1024; + public const int MaxValidators = 1024; public static readonly uint[] GenerationAmount = { 8, 7, 6, 5, 4, 3, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 }; public static readonly TimeSpan TimePerBlock = TimeSpan.FromSeconds(SecondsPerBlock); public static readonly ECPoint[] StandbyValidators = ProtocolSettings.Default.StandbyValidators.OfType().Select(p => ECPoint.DecodePoint(p.HexToBytes(), ECCurve.Secp256r1)).ToArray(); diff --git a/neo/Ledger/TrimmedBlock.cs b/neo/Ledger/TrimmedBlock.cs index b5b9eb0db5..1bf5a371d9 100644 --- a/neo/Ledger/TrimmedBlock.cs +++ b/neo/Ledger/TrimmedBlock.cs @@ -57,7 +57,7 @@ public Header Header public override void Deserialize(BinaryReader reader) { base.Deserialize(reader); - Hashes = reader.ReadSerializableArray(); + Hashes = reader.ReadSerializableArray(Block.MaxTransactionsPerBlock); } public override void Serialize(BinaryWriter writer) diff --git a/neo/Network/P2P/Payloads/Block.cs b/neo/Network/P2P/Payloads/Block.cs index bc1f25704f..a73b5bec23 100644 --- a/neo/Network/P2P/Payloads/Block.cs +++ b/neo/Network/P2P/Payloads/Block.cs @@ -11,6 +11,8 @@ namespace Neo.Network.P2P.Payloads { public class Block : BlockBase, IInventory, IEquatable { + public const int MaxTransactionsPerBlock = ushort.MaxValue; + public Transaction[] Transactions; private Header _header = null; @@ -51,7 +53,7 @@ public static Fixed8 CalculateNetFee(IEnumerable transactions) public override void Deserialize(BinaryReader reader) { base.Deserialize(reader); - Transactions = new Transaction[reader.ReadVarInt(0x10000)]; + Transactions = new Transaction[reader.ReadVarInt(MaxTransactionsPerBlock)]; if (Transactions.Length == 0) throw new FormatException(); HashSet hashes = new HashSet(); for (int i = 0; i < Transactions.Length; i++) diff --git a/neo/Network/P2P/Payloads/ConsensusPayload.cs b/neo/Network/P2P/Payloads/ConsensusPayload.cs index 4fc883e8ab..07c9d6399b 100644 --- a/neo/Network/P2P/Payloads/ConsensusPayload.cs +++ b/neo/Network/P2P/Payloads/ConsensusPayload.cs @@ -1,4 +1,5 @@ -using Neo.Cryptography; +using Neo.Consensus; +using Neo.Cryptography; using Neo.Cryptography.ECC; using Neo.IO; using Neo.Persistence; @@ -15,10 +16,26 @@ public class ConsensusPayload : IInventory public UInt256 PrevHash; public uint BlockIndex; public ushort ValidatorIndex; - public uint Timestamp; public byte[] Data; public Witness Witness; + private ConsensusMessage _deserializedMessage = null; + internal ConsensusMessage ConsensusMessage + { + get + { + return _deserializedMessage; + } + set + { + if (!ReferenceEquals(_deserializedMessage, value)) + { + _deserializedMessage = value; + Data = value?.ToArray(); + } + } + } + private UInt256 _hash = null; public UInt256 Hash { @@ -47,7 +64,18 @@ Witness[] IVerifiable.Witnesses } } - public int Size => sizeof(uint) + PrevHash.Size + sizeof(uint) + sizeof(ushort) + sizeof(uint) + Data.GetVarSize() + 1 + Witness.Size; + public int Size => + sizeof(uint) + //Version + PrevHash.Size + //PrevHash + sizeof(uint) + //BlockIndex + sizeof(ushort) + //ValidatorIndex + Data.GetVarSize() + //Data + 1 + Witness.Size; //Witness + + internal T GetDeserializedMessage() where T : ConsensusMessage + { + return (T)ConsensusMessage; + } void ISerializable.Deserialize(BinaryReader reader) { @@ -62,8 +90,8 @@ void IVerifiable.DeserializeUnsigned(BinaryReader reader) PrevHash = reader.ReadSerializable(); BlockIndex = reader.ReadUInt32(); ValidatorIndex = reader.ReadUInt16(); - Timestamp = reader.ReadUInt32(); Data = reader.ReadVarBytes(); + _deserializedMessage = ConsensusMessage.DeserializeFrom(Data); } byte[] IScriptContainer.GetMessage() @@ -91,7 +119,6 @@ void IVerifiable.SerializeUnsigned(BinaryWriter writer) writer.Write(PrevHash); writer.Write(BlockIndex); writer.Write(ValidatorIndex); - writer.Write(Timestamp); writer.WriteVarBytes(Data); }