Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add some initial MemoryPool unit tests. Fix bug when Persisting the GenesisBlock #549

Merged
merged 2 commits into from
Jan 16, 2019
Merged

Add some initial MemoryPool unit tests. Fix bug when Persisting the GenesisBlock #549

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
fcc2c88
Add MemoryPool Unit tests. Fix bug on initital start of Persisting th…
Jan 16, 2019
ffc3651
Only create the NeoSystem once. Add additional unit test.
Jan 16, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 14 additions & 2 deletions neo.UnitTests/TestDataCache.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,17 @@ public class TestDataCache<TKey, TValue> : DataCache<TKey, TValue>
where TKey : IEquatable<TKey>, ISerializable
where TValue : class, ICloneable<TValue>, ISerializable, new()
{
private readonly TValue _defaultValue;

public TestDataCache()
{
_defaultValue = null;
}

public TestDataCache(TValue defaultValue)
{
this._defaultValue = defaultValue;
}
public override void DeleteInternal(TKey key)
{
}
Expand All @@ -25,12 +36,13 @@ protected override IEnumerable<KeyValuePair<TKey, TValue>> FindInternal(byte[] k

protected override TValue GetInternal(TKey key)
{
throw new NotImplementedException();
if (_defaultValue == null) throw new NotImplementedException();
return _defaultValue;
}

protected override TValue TryGetInternal(TKey key)
{
return null;
return _defaultValue;
}

protected override void UpdateInternal(TKey key, TValue value)
Expand Down
214 changes: 214 additions & 0 deletions neo.UnitTests/UT_MemoryPool.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,214 @@
using System;
using System.Linq;
using System.Reflection;
using Microsoft.VisualStudio.TestTools.UnitTesting;
using Moq;
using Neo.Ledger;
using FluentAssertions;
using Neo.Cryptography.ECC;
using Neo.IO.Wrappers;
using Neo.Network.P2P.Payloads;
using Neo.Persistence;

namespace Neo.UnitTests
{
[TestClass]
public class UT_MemoryPool
{
private static NeoSystem TheNeoSystem;

private readonly Random _random = new Random();

private MemoryPool _unit;

[TestInitialize]
public void TestSetup()
{
if (TheNeoSystem == null)
{
var mockSnapshot = new Mock<Snapshot>();
mockSnapshot.SetupGet(p => p.Blocks).Returns(new TestDataCache<UInt256, BlockState>());
mockSnapshot.SetupGet(p => p.Transactions).Returns(new TestDataCache<UInt256, TransactionState>());
mockSnapshot.SetupGet(p => p.Accounts).Returns(new TestDataCache<UInt160, AccountState>());
mockSnapshot.SetupGet(p => p.UnspentCoins).Returns(new TestDataCache<UInt256, UnspentCoinState>());
mockSnapshot.SetupGet(p => p.SpentCoins).Returns(new TestDataCache<UInt256, SpentCoinState>());
mockSnapshot.SetupGet(p => p.Validators).Returns(new TestDataCache<ECPoint, ValidatorState>());
mockSnapshot.SetupGet(p => p.Assets).Returns(new TestDataCache<UInt256, AssetState>());
mockSnapshot.SetupGet(p => p.Contracts).Returns(new TestDataCache<UInt160, ContractState>());
mockSnapshot.SetupGet(p => p.Storages).Returns(new TestDataCache<StorageKey, StorageItem>());
mockSnapshot.SetupGet(p => p.HeaderHashList)
.Returns(new TestDataCache<UInt32Wrapper, HeaderHashList>());
mockSnapshot.SetupGet(p => p.ValidatorsCount).Returns(new TestMetaDataCache<ValidatorsCountState>());
mockSnapshot.SetupGet(p => p.BlockHashIndex).Returns(new TestMetaDataCache<HashIndexState>());
mockSnapshot.SetupGet(p => p.HeaderHashIndex).Returns(new TestMetaDataCache<HashIndexState>());

var mockStore = new Mock<Store>();

var defaultTx = CreateRandomHashInvocationTransaction();
defaultTx.Outputs = new TransactionOutput[1];
defaultTx.Outputs[0] = new TransactionOutput
{
AssetId = Blockchain.UtilityToken.Hash,
Value = new Fixed8(1000000), // 0.001 GAS (enough to be a high priority TX
ScriptHash = UInt160.Zero // doesn't matter for our purposes.
};

mockStore.Setup(p => p.GetBlocks()).Returns(new TestDataCache<UInt256, BlockState>());
mockStore.Setup(p => p.GetTransactions()).Returns(new TestDataCache<UInt256, TransactionState>(
new TransactionState
{
BlockIndex = 1,
Transaction = defaultTx
}));

mockStore.Setup(p => p.GetAccounts()).Returns(new TestDataCache<UInt160, AccountState>());
mockStore.Setup(p => p.GetUnspentCoins()).Returns(new TestDataCache<UInt256, UnspentCoinState>());
mockStore.Setup(p => p.GetSpentCoins()).Returns(new TestDataCache<UInt256, SpentCoinState>());
mockStore.Setup(p => p.GetValidators()).Returns(new TestDataCache<ECPoint, ValidatorState>());
mockStore.Setup(p => p.GetAssets()).Returns(new TestDataCache<UInt256, AssetState>());
mockStore.Setup(p => p.GetContracts()).Returns(new TestDataCache<UInt160, ContractState>());
mockStore.Setup(p => p.GetStorages()).Returns(new TestDataCache<StorageKey, StorageItem>());
mockStore.Setup(p => p.GetHeaderHashList()).Returns(new TestDataCache<UInt32Wrapper, HeaderHashList>());
mockStore.Setup(p => p.GetValidatorsCount()).Returns(new TestMetaDataCache<ValidatorsCountState>());
mockStore.Setup(p => p.GetBlockHashIndex()).Returns(new TestMetaDataCache<HashIndexState>());
mockStore.Setup(p => p.GetHeaderHashIndex()).Returns(new TestMetaDataCache<HashIndexState>());
mockStore.Setup(p => p.GetSnapshot()).Returns(mockSnapshot.Object);

Console.WriteLine("initialize NeoSystem");
TheNeoSystem = new NeoSystem(mockStore.Object); // new Mock<NeoSystem>(mockStore.Object);
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The singletons could be problematic for unit testing when we are creating more of these in other tests, but this is ok for now.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I talked to @rodoufu recently, and he wisely advised me of the existence of Dependency Injection frameworks. As he explained to me, I soon realized this would be a really nice thing to put on Neo! I didn't mention before, because it's not on top stack, but we will get there... I don't know the technique very well, but I'm pretty sure that would resolve all these situations with singletons, for both tests and production.

}

// Create a MemoryPool with capacity of 100
_unit = new MemoryPool(TheNeoSystem, 100);

// Verify capacity equals the amount specified
_unit.Capacity.ShouldBeEquivalentTo(100);

_unit.VerifiedCount.ShouldBeEquivalentTo(0);
_unit.UnVerifiedCount.ShouldBeEquivalentTo(0);
_unit.Count.ShouldBeEquivalentTo(0);
}

private Transaction CreateRandomHashInvocationTransaction()
{
var tx = new InvocationTransaction();
var randomBytes = new byte[16];
_random.NextBytes(randomBytes);
tx.Script = randomBytes;
tx.Attributes = new TransactionAttribute[0];
tx.Inputs = new CoinReference[0];
tx.Outputs = new TransactionOutput[0];
tx.Witnesses = new Witness[0];
// Force getting the references
// Console.WriteLine($"Reference Count: {tx.References.Count}");
return tx;
}

private Transaction CreateMockHighPriorityTransaction()
{
var tx = CreateRandomHashInvocationTransaction();
tx.Inputs = new CoinReference[1];
// Any input will trigger reading the transaction output and get our mocked transaction output.
tx.Inputs[0] = new CoinReference
{
PrevHash = UInt256.Zero,
PrevIndex = 0
};
return tx;
}


private Transaction CreateMockLowPriorityTransaction()
{
return CreateRandomHashInvocationTransaction();
}

private void AddTransactions(int count, bool isHighPriority=false)
{
for (int i = 0; i < count; i++)
{
var lowPrioTx = isHighPriority ? CreateMockHighPriorityTransaction(): CreateMockLowPriorityTransaction();
Console.WriteLine($"created tx: {lowPrioTx.Hash}");
_unit.TryAdd(lowPrioTx.Hash, lowPrioTx);
}
}

private void AddLowPriorityTransactions(int count) => AddTransactions(count);
public void AddHighPriorityTransactions(int count) => AddTransactions(count, true);


[TestMethod]
public void LowPriorityCapacityTest()
{
// Add over the capacity items, verify that the verified count increases each time
AddLowPriorityTransactions(50);
_unit.VerifiedCount.ShouldBeEquivalentTo(50);
AddLowPriorityTransactions(51);
Console.WriteLine($"VerifiedCount: {_unit.VerifiedCount} LowPrioCount {_unit.SortedLowPrioTxCount} HighPrioCount {_unit.SortedHighPrioTxCount}");
_unit.SortedLowPrioTxCount.ShouldBeEquivalentTo(100);
_unit.SortedHighPrioTxCount.ShouldBeEquivalentTo(0);

_unit.VerifiedCount.ShouldBeEquivalentTo(100);
_unit.UnVerifiedCount.ShouldBeEquivalentTo(0);
_unit.Count.ShouldBeEquivalentTo(100);
}

[TestMethod]
public void HighPriorityCapacityTest()
{
// Add over the capacity items, verify that the verified count increases each time
AddHighPriorityTransactions(101);

Console.WriteLine($"VerifiedCount: {_unit.VerifiedCount} LowPrioCount {_unit.SortedLowPrioTxCount} HighPrioCount {_unit.SortedHighPrioTxCount}");
_unit.SortedLowPrioTxCount.ShouldBeEquivalentTo(0);
_unit.SortedHighPrioTxCount.ShouldBeEquivalentTo(100);

_unit.VerifiedCount.ShouldBeEquivalentTo(100);
_unit.UnVerifiedCount.ShouldBeEquivalentTo(0);
_unit.Count.ShouldBeEquivalentTo(100);
}

[TestMethod]
public void HighPriorityPushesOutLowPriority()
{
// Add over the capacity items, verify that the verified count increases each time
AddLowPriorityTransactions(70);
AddHighPriorityTransactions(40);

Console.WriteLine($"VerifiedCount: {_unit.VerifiedCount} LowPrioCount {_unit.SortedLowPrioTxCount} HighPrioCount {_unit.SortedHighPrioTxCount}");
_unit.SortedLowPrioTxCount.ShouldBeEquivalentTo(60);
_unit.SortedHighPrioTxCount.ShouldBeEquivalentTo(40);
_unit.Count.ShouldBeEquivalentTo(100);
}

[TestMethod]
public void LowPriorityDoesNotPushOutHighPrority()
{
AddHighPriorityTransactions(70);
AddLowPriorityTransactions(40);

_unit.SortedLowPrioTxCount.ShouldBeEquivalentTo(30);
_unit.SortedHighPrioTxCount.ShouldBeEquivalentTo(70);
_unit.Count.ShouldBeEquivalentTo(100);
}

[TestMethod]
public void BlockPersistMovesTxToUnverified()
{
AddLowPriorityTransactions(30);
AddHighPriorityTransactions(70);


var block = new Block
{
Transactions = _unit.GetSortedVerifiedTransactions().Take(10)
.Concat(_unit.GetSortedVerifiedTransactions().Where(x => x.IsLowPriority).Take(5)).ToArray()
};
_unit.UpdatePoolForBlockPersisted(block, Blockchain.Singleton.GetSnapshot());
_unit.SortedLowPrioTxCount.ShouldBeEquivalentTo(0);
_unit.SortedHighPrioTxCount.ShouldBeEquivalentTo(0);
_unit.UnverifiedSortedHighPrioTxCount.ShouldBeEquivalentTo(60);
_unit.UnverifiedSortedLowPrioTxCount.ShouldBeEquivalentTo(25);
}
}
}
18 changes: 13 additions & 5 deletions neo/Ledger/MemoryPool.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -73,13 +73,14 @@ public int CompareTo(PoolItem otherItem)
/// </summary>
private readonly Dictionary<UInt256, PoolItem> _unsortedTransactions = new Dictionary<UInt256, PoolItem>();
/// <summary>
/// Stores the verified low priority sorted transactions currently in the pool.
/// </summary>
private readonly SortedSet<PoolItem> _sortedLowPrioTransactions = new SortedSet<PoolItem>();
/// <summary>
/// Stores the verified high priority sorted transactins currently in the pool.
/// </summary>
private readonly SortedSet<PoolItem> _sortedHighPrioTransactions = new SortedSet<PoolItem>();
/// <summary>
/// Stores the verified low priority sorted transactions currently in the pool.
/// </summary>
private readonly SortedSet<PoolItem> _sortedLowPrioTransactions = new SortedSet<PoolItem>();


/// <summary>
/// Store the unverified transactions currently in the pool.
Expand All @@ -92,6 +93,13 @@ public int CompareTo(PoolItem otherItem)
private readonly SortedSet<PoolItem> _unverifiedSortedHighPriorityTransactions = new SortedSet<PoolItem>();
private readonly SortedSet<PoolItem> _unverifiedSortedLowPriorityTransactions = new SortedSet<PoolItem>();

// internal methods to aid in unit testing
internal int SortedHighPrioTxCount => _sortedHighPrioTransactions.Count;
internal int SortedLowPrioTxCount => _sortedLowPrioTransactions.Count;
internal int UnverifiedSortedHighPrioTxCount => _unverifiedSortedHighPriorityTransactions.Count;
internal int UnverifiedSortedLowPrioTxCount => _unverifiedSortedLowPriorityTransactions.Count;


private int _maxTxPerBlock;
private int _maxLowPriorityTxPerBlock;

Expand Down Expand Up @@ -401,7 +409,7 @@ internal void UpdatePoolForBlockPersisted(Block block, Snapshot snapshot)

// If we know about headers of future blocks, no point in verifying transactions from the unverified tx pool
// until we get caught up.
if (block.Index < Blockchain.Singleton.HeaderHeight)
if (block.Index > 0 && block.Index < Blockchain.Singleton.HeaderHeight)
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

When persisting the GenesisBlock, block.Index will be 0, We can't yet access Blockchain.Singleton at that point since the constructor of Blockchain will still be running. This fixes that issue.

return;

if (Plugin.Policies.Count == 0)
Expand Down