From 040f3b8f2614831f7e3b1eaf0b0f2a072c23a06c Mon Sep 17 00:00:00 2001 From: Charis Zhao Date: Thu, 9 Jan 2020 18:44:52 +0800 Subject: [PATCH 1/6] Use HashsetCache instead of FIFOSet for KnownHashes and SentHashes (#1389) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * update * change fifoset to hashsetcache for knownhashes and senthashes * format * optimise HashSetCache * remove FIFOSet * remove FIFOSet and update ut * change List to LinkedList * remove bucket count upper limit * update exception message * Adding some comments * dotnet format * rename maxBucketCount * format * Cache count * Vitor suggestions * Update HashSetCache.cs * format * Fix * fix counting count * optimise * optimse * update List Co-authored-by: Shargon Co-authored-by: Vitor Nazário Coelho --- src/neo/Helper.cs | 2 +- src/neo/IO/Caching/FIFOSet.cs | 67 ----------- src/neo/IO/Caching/HashSetCache.cs | 106 ++++++++++++++++++ src/neo/Network/P2P/ProtocolHandler.cs | 8 +- src/neo/Network/P2P/TaskManager.cs | 4 +- .../{UT_FIFOSet.cs => UT_HashSetCache.cs} | 83 ++++++-------- 6 files changed, 148 insertions(+), 122 deletions(-) delete mode 100644 src/neo/IO/Caching/FIFOSet.cs create mode 100644 src/neo/IO/Caching/HashSetCache.cs rename tests/neo.UnitTests/IO/Caching/{UT_FIFOSet.cs => UT_HashSetCache.cs} (64%) diff --git a/src/neo/Helper.cs b/src/neo/Helper.cs index b8a0a7f79e..be66ce2013 100644 --- a/src/neo/Helper.cs +++ b/src/neo/Helper.cs @@ -83,7 +83,7 @@ internal static void Remove(this HashSet set, ISet other) } } - internal static void Remove(this HashSet set, FIFOSet other) + internal static void Remove(this HashSet set, HashSetCache other) where T : IEquatable { if (set.Count > other.Count) diff --git a/src/neo/IO/Caching/FIFOSet.cs b/src/neo/IO/Caching/FIFOSet.cs deleted file mode 100644 index af65db1b8b..0000000000 --- a/src/neo/IO/Caching/FIFOSet.cs +++ /dev/null @@ -1,67 +0,0 @@ -using System; -using System.Collections; -using System.Collections.Generic; -using System.Collections.Specialized; -using System.Linq; - -namespace Neo.IO.Caching -{ - internal class FIFOSet : IReadOnlyCollection where T : IEquatable - { - private readonly int maxCapacity; - private readonly int removeCount; - private readonly OrderedDictionary dictionary; - - public int Count => dictionary.Count; - - public FIFOSet(int maxCapacity, decimal batchSize = 0.1m) - { - if (maxCapacity <= 0) throw new ArgumentOutOfRangeException(nameof(maxCapacity)); - if (batchSize <= 0 || batchSize > 1) throw new ArgumentOutOfRangeException(nameof(batchSize)); - - this.maxCapacity = maxCapacity; - this.removeCount = Math.Max((int)(maxCapacity * batchSize), 1); - this.dictionary = new OrderedDictionary(maxCapacity); - } - - public bool Add(T item) - { - if (dictionary.Contains(item)) return false; - if (dictionary.Count >= maxCapacity) - { - if (removeCount == maxCapacity) - { - dictionary.Clear(); - } - else - { - for (int i = 0; i < removeCount; i++) - dictionary.RemoveAt(0); - } - } - dictionary.Add(item, null); - return true; - } - - public bool Contains(T item) - { - return dictionary.Contains(item); - } - - public void ExceptWith(IEnumerable entries) - { - foreach (var entry in entries) - { - dictionary.Remove(entry); - } - } - - public IEnumerator GetEnumerator() - { - var entries = dictionary.Keys.Cast().ToArray(); - foreach (var entry in entries) yield return entry; - } - - IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); - } -} diff --git a/src/neo/IO/Caching/HashSetCache.cs b/src/neo/IO/Caching/HashSetCache.cs new file mode 100644 index 0000000000..af34cd76fa --- /dev/null +++ b/src/neo/IO/Caching/HashSetCache.cs @@ -0,0 +1,106 @@ +using System; +using System.Collections; +using System.Collections.Generic; + +namespace Neo.IO.Caching +{ + public class HashSetCache : IReadOnlyCollection where T : IEquatable + { + /// + /// Sets where the Hashes are stored + /// + private readonly LinkedList> sets = new LinkedList>(); + + /// + /// Maximum capacity of each bucket inside each HashSet of . + /// + private readonly int bucketCapacity; + + /// + /// Maximum number of buckets for the LinkedList, meaning its maximum cardinality. + /// + private readonly int maxBucketCount; + + /// + /// Entry count + /// + public int Count { get; private set; } + + public HashSetCache(int bucketCapacity, int maxBucketCount = 10) + { + if (bucketCapacity <= 0) throw new ArgumentOutOfRangeException($"{nameof(bucketCapacity)} should be greater than 0"); + if (maxBucketCount <= 0) throw new ArgumentOutOfRangeException($"{nameof(maxBucketCount)} should be greater than 0"); + + this.Count = 0; + this.bucketCapacity = bucketCapacity; + this.maxBucketCount = maxBucketCount; + sets.AddFirst(new HashSet()); + } + + public bool Add(T item) + { + if (Contains(item)) return false; + Count++; + if (sets.First.Value.Count < bucketCapacity) return sets.First.Value.Add(item); + var newSet = new HashSet + { + item + }; + sets.AddFirst(newSet); + if (sets.Count > maxBucketCount) + { + Count -= sets.Last.Value.Count; + sets.RemoveLast(); + } + return true; + } + + public bool Contains(T item) + { + foreach (var set in sets) + { + if (set.Contains(item)) return true; + } + return false; + } + + public void ExceptWith(IEnumerable items) + { + List> removeList = null; + foreach (var item in items) + { + foreach (var set in sets) + { + if (set.Remove(item)) + { + Count--; + if (set.Count == 0) + { + removeList ??= new List>(); + removeList.Add(set); + } + break; + } + } + } + if (removeList == null) return; + foreach (var set in removeList) + { + sets.Remove(set); + } + } + + public IEnumerator GetEnumerator() + { + foreach (var set in sets) + { + foreach (var item in set) + { + yield return item; + } + } + } + + IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); + } +} diff --git a/src/neo/Network/P2P/ProtocolHandler.cs b/src/neo/Network/P2P/ProtocolHandler.cs index cecbb02796..a97cb46922 100644 --- a/src/neo/Network/P2P/ProtocolHandler.cs +++ b/src/neo/Network/P2P/ProtocolHandler.cs @@ -32,8 +32,8 @@ protected override UInt256 GetKeyForItem((UInt256, DateTime) item) private readonly NeoSystem system; private readonly PendingKnownHashesCollection pendingKnownHashes; - private readonly FIFOSet knownHashes; - private readonly FIFOSet sentHashes; + private readonly HashSetCache knownHashes; + private readonly HashSetCache sentHashes; private VersionPayload version; private bool verack = false; private BloomFilter bloom_filter; @@ -47,8 +47,8 @@ public ProtocolHandler(NeoSystem system) { this.system = system; this.pendingKnownHashes = new PendingKnownHashesCollection(); - this.knownHashes = new FIFOSet(Blockchain.Singleton.MemPool.Capacity * 2); - this.sentHashes = new FIFOSet(Blockchain.Singleton.MemPool.Capacity * 2); + this.knownHashes = new HashSetCache(Blockchain.Singleton.MemPool.Capacity * 2 / 5); + this.sentHashes = new HashSetCache(Blockchain.Singleton.MemPool.Capacity * 2 / 5); } protected override void OnReceive(object message) diff --git a/src/neo/Network/P2P/TaskManager.cs b/src/neo/Network/P2P/TaskManager.cs index 818973ea4b..73fa49441a 100644 --- a/src/neo/Network/P2P/TaskManager.cs +++ b/src/neo/Network/P2P/TaskManager.cs @@ -32,7 +32,7 @@ private class Timer { } /// /// A set of known hashes, of inventories or payloads, already received. /// - private readonly FIFOSet knownHashes; + private readonly HashSetCache knownHashes; private readonly Dictionary globalTasks = new Dictionary(); private readonly Dictionary sessions = new Dictionary(); private readonly ICancelable timer = Context.System.Scheduler.ScheduleTellRepeatedlyCancelable(TimerInterval, TimerInterval, Context.Self, new Timer(), ActorRefs.NoSender); @@ -43,7 +43,7 @@ private class Timer { } public TaskManager(NeoSystem system) { this.system = system; - this.knownHashes = new FIFOSet(Blockchain.Singleton.MemPool.Capacity * 2); + this.knownHashes = new HashSetCache(Blockchain.Singleton.MemPool.Capacity * 2 / 5); } private void OnHeaderTaskCompleted() diff --git a/tests/neo.UnitTests/IO/Caching/UT_FIFOSet.cs b/tests/neo.UnitTests/IO/Caching/UT_HashSetCache.cs similarity index 64% rename from tests/neo.UnitTests/IO/Caching/UT_FIFOSet.cs rename to tests/neo.UnitTests/IO/Caching/UT_HashSetCache.cs index d785127778..5f227c7e35 100644 --- a/tests/neo.UnitTests/IO/Caching/UT_FIFOSet.cs +++ b/tests/neo.UnitTests/IO/Caching/UT_HashSetCache.cs @@ -8,63 +8,50 @@ namespace Neo.UnitTests.IO.Caching { [TestClass] - public class UT_FIFOSet + public class UT_HashSetCache { [TestMethod] - public void FIFOSetTest() + public void TestHashSetCache() { - var a = UInt256.Zero; - var b = new UInt256(); - var c = new UInt256(new byte[32] { - 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, - 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, - 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, - 0x01, 0x01 - }); - - var set = new FIFOSet(3); - - Assert.IsTrue(set.Add(a)); - Assert.IsFalse(set.Add(a)); - Assert.IsFalse(set.Add(b)); - Assert.IsTrue(set.Add(c)); - - CollectionAssert.AreEqual(set.ToArray(), new UInt256[] { a, c }); - - var d = new UInt256(new byte[32] { - 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, - 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, - 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, - 0x01, 0x02 - }); + var bucket = new HashSetCache(10); + for (int i = 1; i <= 100; i++) + { + bucket.Add(i); + } + bucket.Count.Should().Be(100); - // Testing Fifo max size - Assert.IsTrue(set.Add(d)); - CollectionAssert.AreEqual(set.ToArray(), new UInt256[] { a, c, d }); + int sum = 0; + foreach (var ele in bucket) + { + sum += ele; + } + sum.Should().Be(5050); - var e = new UInt256(new byte[32] { - 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, - 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, - 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, - 0x01, 0x03 - }); + bucket.Add(101); + bucket.Count.Should().Be(91); - Assert.IsTrue(set.Add(e)); - Assert.IsFalse(set.Add(e)); - CollectionAssert.AreEqual(set.ToArray(), new UInt256[] { c, d, e }); + var items = new int[10]; + var value = 11; + for (int i = 0; i < 10; i++) + { + items[i] = value; + value += 2; + } + bucket.ExceptWith(items); + bucket.Count.Should().Be(81); + + bucket.Contains(13).Should().BeFalse(); + bucket.Contains(50).Should().BeTrue(); } [TestMethod] public void TestConstructor() { - Action action1 = () => new FIFOSet(-1); + Action action1 = () => new HashSetCache(-1); action1.Should().Throw(); - Action action2 = () => new FIFOSet(1, -1); + Action action2 = () => new HashSetCache(1, -1); action2.Should().Throw(); - - Action action3 = () => new FIFOSet(1, 2); - action3.Should().Throw(); } [TestMethod] @@ -82,7 +69,7 @@ public void TestAdd() 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x02 }); - var set = new FIFOSet(1, 1) + var set = new HashSetCache(1, 1) { a, b @@ -105,7 +92,7 @@ public void TestGetEnumerator() 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x02 }); - var set = new FIFOSet(1, 1) + var set = new HashSetCache(1, 1) { a, b @@ -136,7 +123,7 @@ public void TestExceptWith() 0x01, 0x03 }); - var set = new FIFOSet(10) + var set = new HashSetCache(10) { a, b, @@ -145,7 +132,7 @@ public void TestExceptWith() set.ExceptWith(new UInt256[] { b, c }); CollectionAssert.AreEqual(set.ToArray(), new UInt256[] { a }); - set = new FIFOSet(10) + set = new HashSetCache(10) { a, b, @@ -154,7 +141,7 @@ public void TestExceptWith() set.ExceptWith(new UInt256[] { a }); CollectionAssert.AreEqual(set.ToArray(), new UInt256[] { b, c }); - set = new FIFOSet(10) + set = new HashSetCache(10) { a, b, From 49b8518a4805aa8dea40a432211a3037e8896be3 Mon Sep 17 00:00:00 2001 From: Charis Zhao Date: Mon, 13 Jan 2020 23:54:53 +0800 Subject: [PATCH 2/6] Add ContainsTransaction Recheck when ParallelVerifiedTransaction Received (#1408) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * add containsTx recheck * add UT * Update to just View check * update * revert * Update UT_Blockchain.cs Co-authored-by: Vitor Nazário Coelho --- src/neo/Ledger/Blockchain.cs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/neo/Ledger/Blockchain.cs b/src/neo/Ledger/Blockchain.cs index 74d06806e4..48c7da914a 100644 --- a/src/neo/Ledger/Blockchain.cs +++ b/src/neo/Ledger/Blockchain.cs @@ -429,7 +429,9 @@ private void OnParallelVerified(ParallelVerified parallelVerified) RelayResultReason reason = parallelVerified.VerifyResult; if (reason == RelayResultReason.Succeed) { - if (!MemPool.CanTransactionFitInPool(parallelVerified.Transaction)) + if (View.ContainsTransaction(parallelVerified.Transaction.Hash)) + reason = RelayResultReason.AlreadyExists; + else if (!MemPool.CanTransactionFitInPool(parallelVerified.Transaction)) reason = RelayResultReason.OutOfMemory; else if (!MemPool.TryAdd(parallelVerified.Transaction.Hash, parallelVerified.Transaction)) reason = RelayResultReason.OutOfMemory; From ab4830cae3242e80fd60e7df6c0d9143803b7be3 Mon Sep 17 00:00:00 2001 From: cn1010 <1062108372@qq.com> Date: Tue, 14 Jan 2020 16:07:16 +0800 Subject: [PATCH 3/6] Enhance security (1) (#1403) * inputs validation * more validation * recover constructor in ECPoint * format * minor changes * format * changes based on review * recover validation in AesEncrypt --- src/neo/Cryptography/Base58.cs | 1 + src/neo/Cryptography/BloomFilter.cs | 2 + src/neo/Cryptography/ECC/ECDsa.cs | 107 ------------------ src/neo/Cryptography/ECC/ECFieldElement.cs | 5 +- src/neo/Cryptography/ECC/ECPoint.cs | 3 +- src/neo/Cryptography/MerkleTree.cs | 2 +- .../Cryptography/ECC/UT_ECDsa.cs | 55 --------- .../Cryptography/ECC/UT_ECFieldElement.cs | 24 ++++ .../Cryptography/ECC/UT_ECPoint.cs | 18 ++- 9 files changed, 41 insertions(+), 176 deletions(-) delete mode 100644 src/neo/Cryptography/ECC/ECDsa.cs delete mode 100644 tests/neo.UnitTests/Cryptography/ECC/UT_ECDsa.cs diff --git a/src/neo/Cryptography/Base58.cs b/src/neo/Cryptography/Base58.cs index 316f1de72b..f571f9540d 100644 --- a/src/neo/Cryptography/Base58.cs +++ b/src/neo/Cryptography/Base58.cs @@ -12,6 +12,7 @@ public static class Base58 public static byte[] Base58CheckDecode(this string input) { + if (input is null) throw new ArgumentNullException(nameof(input)); byte[] buffer = Decode(input); if (buffer.Length < 4) throw new FormatException(); byte[] checksum = buffer.Sha256(0, buffer.Length - 4).Sha256(); diff --git a/src/neo/Cryptography/BloomFilter.cs b/src/neo/Cryptography/BloomFilter.cs index c4b79d01f2..32cfa50dd4 100644 --- a/src/neo/Cryptography/BloomFilter.cs +++ b/src/neo/Cryptography/BloomFilter.cs @@ -1,3 +1,4 @@ +using System; using System.Collections; using System.Linq; @@ -16,6 +17,7 @@ public class BloomFilter public BloomFilter(int m, int k, uint nTweak, byte[] elements = null) { + if (k < 0 || m < 0) throw new ArgumentOutOfRangeException(); this.seeds = Enumerable.Range(0, k).Select(p => (uint)p * 0xFBA4C795 + nTweak).ToArray(); this.bits = elements == null ? new BitArray(m) : new BitArray(elements); this.bits.Length = m; diff --git a/src/neo/Cryptography/ECC/ECDsa.cs b/src/neo/Cryptography/ECC/ECDsa.cs deleted file mode 100644 index 08f48b1dc3..0000000000 --- a/src/neo/Cryptography/ECC/ECDsa.cs +++ /dev/null @@ -1,107 +0,0 @@ -using System; -using System.Numerics; -using System.Security.Cryptography; - -namespace Neo.Cryptography.ECC -{ - public class ECDsa - { - private readonly byte[] privateKey; - private readonly ECPoint publicKey; - private readonly ECCurve curve; - - public ECDsa(byte[] privateKey, ECCurve curve) - : this(curve.G * privateKey) - { - this.privateKey = privateKey; - } - - public ECDsa(ECPoint publicKey) - { - this.publicKey = publicKey; - this.curve = publicKey.Curve; - } - - private BigInteger CalculateE(BigInteger n, ReadOnlySpan message) - { - int messageBitLength = message.Length * 8; - BigInteger trunc = new BigInteger(message, isUnsigned: true, isBigEndian: true); - if (n.GetBitLength() < messageBitLength) - { - trunc >>= messageBitLength - n.GetBitLength(); - } - return trunc; - } - - public BigInteger[] GenerateSignature(ReadOnlySpan message) - { - if (privateKey == null) throw new InvalidOperationException(); - BigInteger e = CalculateE(curve.N, message); - BigInteger d = new BigInteger(privateKey, isUnsigned: true, isBigEndian: true); - BigInteger r, s; - using (RandomNumberGenerator rng = RandomNumberGenerator.Create()) - { - do - { - BigInteger k; - do - { - do - { - k = rng.NextBigInteger(curve.N.GetBitLength()); - } - while (k.Sign == 0 || k.CompareTo(curve.N) >= 0); - ECPoint p = ECPoint.Multiply(curve.G, k); - BigInteger x = p.X.Value; - r = x.Mod(curve.N); - } - while (r.Sign == 0); - s = (k.ModInverse(curve.N) * (e + d * r)).Mod(curve.N); - if (s > curve.N / 2) - { - s = curve.N - s; - } - } - while (s.Sign == 0); - } - return new BigInteger[] { r, s }; - } - - private static ECPoint SumOfTwoMultiplies(ECPoint P, BigInteger k, ECPoint Q, BigInteger l) - { - int m = Math.Max(k.GetBitLength(), l.GetBitLength()); - ECPoint Z = P + Q; - ECPoint R = P.Curve.Infinity; - for (int i = m - 1; i >= 0; --i) - { - R = R.Twice(); - if (k.TestBit(i)) - { - if (l.TestBit(i)) - R = R + Z; - else - R = R + P; - } - else - { - if (l.TestBit(i)) - R = R + Q; - } - } - return R; - } - - public bool VerifySignature(ReadOnlySpan message, BigInteger r, BigInteger s) - { - if (r.Sign < 1 || s.Sign < 1 || r.CompareTo(curve.N) >= 0 || s.CompareTo(curve.N) >= 0) - return false; - BigInteger e = CalculateE(curve.N, message); - BigInteger c = s.ModInverse(curve.N); - BigInteger u1 = (e * c).Mod(curve.N); - BigInteger u2 = (r * c).Mod(curve.N); - ECPoint point = SumOfTwoMultiplies(curve.G, u1, publicKey, u2); - BigInteger v = point.X.Value.Mod(curve.N); - return v.Equals(r); - } - } -} diff --git a/src/neo/Cryptography/ECC/ECFieldElement.cs b/src/neo/Cryptography/ECC/ECFieldElement.cs index 948afc263d..9ebd48082d 100644 --- a/src/neo/Cryptography/ECC/ECFieldElement.cs +++ b/src/neo/Cryptography/ECC/ECFieldElement.cs @@ -10,6 +10,8 @@ internal class ECFieldElement : IComparable, IEquatable= curve.Q) throw new ArgumentException("x value too large in field element"); this.Value = value; @@ -19,6 +21,7 @@ public ECFieldElement(BigInteger value, ECCurve curve) public int CompareTo(ECFieldElement other) { if (ReferenceEquals(this, other)) return 0; + if (!curve.Equals(other.curve)) throw new InvalidOperationException("Invalid comparision for points with different curves"); return Value.CompareTo(other.Value); } @@ -35,7 +38,7 @@ public override bool Equals(object obj) public bool Equals(ECFieldElement other) { - return Value.Equals(other.Value); + return Value.Equals(other.Value) && curve.Equals(other.curve); } private static BigInteger[] FastLucasSequence(BigInteger p, BigInteger P, BigInteger Q, BigInteger k) diff --git a/src/neo/Cryptography/ECC/ECPoint.cs b/src/neo/Cryptography/ECC/ECPoint.cs index 4189dcbb9b..f0b0b19a36 100644 --- a/src/neo/Cryptography/ECC/ECPoint.cs +++ b/src/neo/Cryptography/ECC/ECPoint.cs @@ -22,7 +22,7 @@ public ECPoint() : this(null, null, ECCurve.Secp256r1) { } internal ECPoint(ECFieldElement x, ECFieldElement y, ECCurve curve) { - if ((x != null && y == null) || (x == null && y != null)) + if ((x is null ^ y is null) || (curve is null)) throw new ArgumentException("Exactly one of the field elements is null"); this.X = x; this.Y = y; @@ -31,6 +31,7 @@ internal ECPoint(ECFieldElement x, ECFieldElement y, ECCurve curve) public int CompareTo(ECPoint other) { + if (!Curve.Equals(other.Curve)) throw new InvalidOperationException("Invalid comparision for points with different curves"); if (ReferenceEquals(this, other)) return 0; int result = X.CompareTo(other.X); if (result != 0) return result; diff --git a/src/neo/Cryptography/MerkleTree.cs b/src/neo/Cryptography/MerkleTree.cs index 89d0a275b5..870eaabbb9 100644 --- a/src/neo/Cryptography/MerkleTree.cs +++ b/src/neo/Cryptography/MerkleTree.cs @@ -15,7 +15,7 @@ public class MerkleTree internal MerkleTree(UInt256[] hashes) { - if (hashes.Length == 0) throw new ArgumentException(); + if (hashes is null || hashes.Length == 0) throw new ArgumentException(); this.root = Build(hashes.Select(p => new MerkleTreeNode { Hash = p }).ToArray()); int depth = 1; for (MerkleTreeNode i = root; i.LeftChild != null; i = i.LeftChild) diff --git a/tests/neo.UnitTests/Cryptography/ECC/UT_ECDsa.cs b/tests/neo.UnitTests/Cryptography/ECC/UT_ECDsa.cs deleted file mode 100644 index 62206741cb..0000000000 --- a/tests/neo.UnitTests/Cryptography/ECC/UT_ECDsa.cs +++ /dev/null @@ -1,55 +0,0 @@ -using FluentAssertions; -using Microsoft.VisualStudio.TestTools.UnitTesting; -using Neo.Wallets; -using System; -using System.Numerics; -using ECDsa = Neo.Cryptography.ECC.ECDsa; - -namespace Neo.UnitTests.Cryptography -{ - [TestClass] - public class UT_ECDsa - { - private KeyPair key = null; - - [TestInitialize] - public void TestSetup() - { - key = UT_Crypto.generateCertainKey(32); - } - - [TestMethod] - public void TestECDsaConstructor() - { - Action action = () => new ECDsa(key.PublicKey); - action.Should().NotThrow(); - action = () => new ECDsa(key.PrivateKey, key.PublicKey.Curve); - action.Should().NotThrow(); - } - - [TestMethod] - public void TestGenerateSignature() - { - ECDsa sa = new ECDsa(key.PrivateKey, key.PublicKey.Curve); - byte[] message = System.Text.Encoding.Default.GetBytes("HelloWorld"); - for (int i = 0; i < 10; i++) - { - BigInteger[] result = sa.GenerateSignature(message); - result.Length.Should().Be(2); - } - sa = new ECDsa(key.PublicKey); - Action action = () => sa.GenerateSignature(message); - action.Should().Throw(); - } - - [TestMethod] - public void TestVerifySignature() - { - ECDsa sa = new ECDsa(key.PrivateKey, key.PublicKey.Curve); - byte[] message = System.Text.Encoding.Default.GetBytes("HelloWorld"); - BigInteger[] result = sa.GenerateSignature(message); - sa.VerifySignature(message, result[0], result[1]).Should().BeTrue(); - sa.VerifySignature(message, new BigInteger(-100), result[1]).Should().BeFalse(); - } - } -} diff --git a/tests/neo.UnitTests/Cryptography/ECC/UT_ECFieldElement.cs b/tests/neo.UnitTests/Cryptography/ECC/UT_ECFieldElement.cs index 6b2965b41f..725fc46962 100644 --- a/tests/neo.UnitTests/Cryptography/ECC/UT_ECFieldElement.cs +++ b/tests/neo.UnitTests/Cryptography/ECC/UT_ECFieldElement.cs @@ -23,6 +23,29 @@ public void TestECFieldElementConstructor() action.Should().Throw(); } + [TestMethod] + public void TestCompareTo() + { + ECFieldElement X1 = new ECFieldElement(new BigInteger(100), ECCurve.Secp256k1); + ECFieldElement Y1 = new ECFieldElement(new BigInteger(200), ECCurve.Secp256k1); + ECFieldElement X2 = new ECFieldElement(new BigInteger(300), ECCurve.Secp256k1); + ECFieldElement Y2 = new ECFieldElement(new BigInteger(400), ECCurve.Secp256k1); + ECFieldElement X3 = new ECFieldElement(new BigInteger(100), ECCurve.Secp256r1); + ECFieldElement Y3 = new ECFieldElement(new BigInteger(400), ECCurve.Secp256r1); + ECPoint point1 = new ECPoint(X1, Y1, ECCurve.Secp256k1); + ECPoint point2 = new ECPoint(X2, Y1, ECCurve.Secp256k1); + ECPoint point3 = new ECPoint(X1, Y2, ECCurve.Secp256k1); + ECPoint point4 = new ECPoint(X3, Y3, ECCurve.Secp256r1); + + point1.CompareTo(point1).Should().Be(0); + point1.CompareTo(point2).Should().Be(-1); + point2.CompareTo(point1).Should().Be(1); + point1.CompareTo(point3).Should().Be(-1); + point3.CompareTo(point1).Should().Be(1); + Action action = () => point3.CompareTo(point4); + action.Should().Throw(); + } + [TestMethod] public void TestEquals() { @@ -30,6 +53,7 @@ public void TestEquals() object element = new ECFieldElement(input, ECCurve.Secp256k1); element.Equals(element).Should().BeTrue(); element.Equals(1).Should().BeFalse(); + element.Equals(new ECFieldElement(input, ECCurve.Secp256r1)).Should().BeFalse(); input = new BigInteger(200); element.Equals(new ECFieldElement(input, ECCurve.Secp256k1)).Should().BeFalse(); diff --git a/tests/neo.UnitTests/Cryptography/ECC/UT_ECPoint.cs b/tests/neo.UnitTests/Cryptography/ECC/UT_ECPoint.cs index b38df0fa44..9a8232d9fb 100644 --- a/tests/neo.UnitTests/Cryptography/ECC/UT_ECPoint.cs +++ b/tests/neo.UnitTests/Cryptography/ECC/UT_ECPoint.cs @@ -28,18 +28,12 @@ public static byte[] generatePrivateKey(int privateKeyLength) public void TestCompareTo() { ECFieldElement X1 = new ECFieldElement(new BigInteger(100), ECCurve.Secp256k1); - ECFieldElement Y1 = new ECFieldElement(new BigInteger(200), ECCurve.Secp256k1); - ECFieldElement X2 = new ECFieldElement(new BigInteger(300), ECCurve.Secp256k1); - ECFieldElement Y2 = new ECFieldElement(new BigInteger(400), ECCurve.Secp256k1); - ECPoint point1 = new ECPoint(X1, Y1, ECCurve.Secp256k1); - ECPoint point2 = new ECPoint(X2, Y1, ECCurve.Secp256k1); - ECPoint point3 = new ECPoint(X1, Y2, ECCurve.Secp256k1); + ECFieldElement X2 = new ECFieldElement(new BigInteger(200), ECCurve.Secp256k1); + ECFieldElement X3 = new ECFieldElement(new BigInteger(100), ECCurve.Secp256r1); - point1.CompareTo(point1).Should().Be(0); - point1.CompareTo(point2).Should().Be(-1); - point2.CompareTo(point1).Should().Be(1); - point1.CompareTo(point3).Should().Be(-1); - point3.CompareTo(point1).Should().Be(1); + X1.CompareTo(X2).Should().Be(-1); + Action action = () => X1.CompareTo(X3); + action.Should().Throw(); } [TestMethod] @@ -60,6 +54,8 @@ public void TestECPointConstructor() action.Should().Throw(); action = () => new ECPoint(null, Y, ECCurve.Secp256k1); action.Should().Throw(); + action = () => new ECPoint(null, Y, null); + action.Should().Throw(); } [TestMethod] From fc1a64b77c8a901c048e7b973f204d3b7c9870e3 Mon Sep 17 00:00:00 2001 From: John Date: Wed, 15 Jan 2020 14:44:59 +0000 Subject: [PATCH 4/6] Fix undefined layout behaviour, and remove pinning: Uint256 and Uint160 (#1387) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Fix undefined layout behaviour, and remove pinning * Update UInt256.cs * Update UInt256.cs * add StructLayout * set pack=1 * Explicit * Remove UIntBase * Revert constructor * add unsafe * Comment * Comment Uint256 * Removing comments from value 1 * Removing comments from value1 Co-authored-by: Luchuan Co-authored-by: Vitor Nazário Coelho Co-authored-by: Erik Zhang Co-authored-by: Shargon --- src/neo/UInt160.cs | 25 ++++++--- src/neo/UInt256.cs | 27 ++++++--- src/neo/UIntBase.cs | 90 ------------------------------ tests/neo.UnitTests/UT_UIntBase.cs | 49 ---------------- 4 files changed, 35 insertions(+), 156 deletions(-) delete mode 100644 src/neo/UIntBase.cs delete mode 100644 tests/neo.UnitTests/UT_UIntBase.cs diff --git a/src/neo/UInt160.cs b/src/neo/UInt160.cs index 69c2f85ed1..a7cb19fc2a 100644 --- a/src/neo/UInt160.cs +++ b/src/neo/UInt160.cs @@ -1,22 +1,26 @@ +using Neo.IO; using System; using System.Globalization; using System.IO; +using System.Runtime.InteropServices; namespace Neo { /// /// This class stores a 160 bit unsigned int, represented as a 20-byte little-endian byte array + /// It is composed by ulong(64) + ulong(64) + uint(32) = UInt160(160) /// - public class UInt160 : UIntBase, IComparable, IEquatable + [StructLayout(LayoutKind.Explicit, Size = 20)] + public class UInt160 : IComparable, IEquatable, ISerializable { public const int Length = 20; public static readonly UInt160 Zero = new UInt160(); - private ulong value1; - private ulong value2; - private uint value3; + [FieldOffset(0)] private ulong value1; + [FieldOffset(8)] private ulong value2; + [FieldOffset(16)] private uint value3; - public override int Size => Length; + public int Size => Length; public UInt160() { @@ -44,7 +48,7 @@ public int CompareTo(UInt160 other) return value1.CompareTo(other.value1); } - public override void Deserialize(BinaryReader reader) + public void Deserialize(BinaryReader reader) { value1 = reader.ReadUInt64(); value2 = reader.ReadUInt64(); @@ -80,7 +84,7 @@ public override int GetHashCode() /// Method Parse receives a big-endian hex string and stores as a UInt160 little-endian 20-bytes array /// Example: Parse("0xa400ff00ff00ff00ff00ff00ff00ff00ff00ff01") should create UInt160 01ff00ff00ff00ff00ff00ff00ff00ff00ff00a4 /// - public static new UInt160 Parse(string value) + public static UInt160 Parse(string value) { if (value == null) throw new ArgumentNullException(); @@ -93,13 +97,18 @@ public override int GetHashCode() return new UInt160(data); } - public override void Serialize(BinaryWriter writer) + public void Serialize(BinaryWriter writer) { writer.Write(value1); writer.Write(value2); writer.Write(value3); } + public override string ToString() + { + return "0x" + this.ToArray().ToHexString(reverse: true); + } + /// /// Method TryParse tries to parse a big-endian hex string and store it as a UInt160 little-endian 20-bytes array /// Example: TryParse("0xa400ff00ff00ff00ff00ff00ff00ff00ff00ff01", result) should create result UInt160 01ff00ff00ff00ff00ff00ff00ff00ff00ff00a4 diff --git a/src/neo/UInt256.cs b/src/neo/UInt256.cs index bee5743c3c..d0c01f4f51 100644 --- a/src/neo/UInt256.cs +++ b/src/neo/UInt256.cs @@ -1,23 +1,27 @@ +using Neo.IO; using System; using System.Globalization; using System.IO; +using System.Runtime.InteropServices; namespace Neo { /// /// This class stores a 256 bit unsigned int, represented as a 32-byte little-endian byte array + /// Composed by ulong(64) + ulong(64) + ulong(64) + ulong(64) = UInt256(256) /// - public class UInt256 : UIntBase, IComparable, IEquatable + [StructLayout(LayoutKind.Explicit, Size = 32)] + public class UInt256 : IComparable, IEquatable, ISerializable { public const int Length = 32; public static readonly UInt256 Zero = new UInt256(); - private ulong value1; - private ulong value2; - private ulong value3; - private ulong value4; + [FieldOffset(0)] private ulong value1; + [FieldOffset(8)] private ulong value2; + [FieldOffset(16)] private ulong value3; + [FieldOffset(24)] private ulong value4; - public override int Size => Length; + public int Size => Length; public UInt256() { @@ -47,7 +51,7 @@ public int CompareTo(UInt256 other) return value1.CompareTo(other.value1); } - public override void Deserialize(BinaryReader reader) + public void Deserialize(BinaryReader reader) { value1 = reader.ReadUInt64(); value2 = reader.ReadUInt64(); @@ -85,7 +89,7 @@ public override int GetHashCode() /// Method Parse receives a big-endian hex string and stores as a UInt256 little-endian 32-bytes array /// Example: Parse("0xa400ff00ff00ff00ff00ff00ff00ff00ff00ff00ff00ff00ff00ff00ff00ff01") should create UInt256 01ff00ff00ff00ff00ff00ff00ff00ff00ff00ff00ff00ff00ff00ff00ff00a4 /// - public static new UInt256 Parse(string s) + public static UInt256 Parse(string s) { if (s == null) throw new ArgumentNullException(); @@ -98,7 +102,7 @@ public override int GetHashCode() return new UInt256(data); } - public override void Serialize(BinaryWriter writer) + public void Serialize(BinaryWriter writer) { writer.Write(value1); writer.Write(value2); @@ -106,6 +110,11 @@ public override void Serialize(BinaryWriter writer) writer.Write(value4); } + public override string ToString() + { + return "0x" + this.ToArray().ToHexString(reverse: true); + } + /// /// Method TryParse tries to parse a big-endian hex string and store it as a UInt256 little-endian 32-bytes array /// Example: TryParse("0xa400ff00ff00ff00ff00ff00ff00ff00ff00ff00ff00ff00ff00ff00ff00ff01", result) should create result UInt256 01ff00ff00ff00ff00ff00ff00ff00ff00ff00ff00ff00ff00ff00ff00ff00a4 diff --git a/src/neo/UIntBase.cs b/src/neo/UIntBase.cs deleted file mode 100644 index b78ccd3023..0000000000 --- a/src/neo/UIntBase.cs +++ /dev/null @@ -1,90 +0,0 @@ -using Neo.IO; -using System; -using System.IO; - -namespace Neo -{ - /// - /// Base class for little-endian unsigned integers. Two classes inherit from this: UInt160 and UInt256. - /// Only basic comparison/serialization are proposed for these classes. For arithmetic purposes, use BigInteger class. - /// - public abstract class UIntBase : ISerializable - { - /// - /// Number of bytes of the unsigned int. - /// Currently, inherited classes use 20-bytes (UInt160) or 32-bytes (UInt256) - /// - public abstract int Size { get; } - - public abstract void Deserialize(BinaryReader reader); - - public abstract override bool Equals(object obj); - - /// - /// Method GetHashCode returns a 32-bit int representing a hash code, composed of the first 4 bytes. - /// - public abstract override int GetHashCode(); - - /// - /// Method Parse receives a big-endian hex string and stores as a UInt160 or UInt256 little-endian byte array - /// Example: Parse("0xa400ff00ff00ff00ff00ff00ff00ff00ff00ff01") should create UInt160 01ff00ff00ff00ff00ff00ff00ff00ff00ff00a4 - /// - public static UIntBase Parse(string s) - { - if (s.Length == 40 || s.Length == 42) - return UInt160.Parse(s); - else if (s.Length == 64 || s.Length == 66) - return UInt256.Parse(s); - else - throw new FormatException(); - } - - public abstract void Serialize(BinaryWriter writer); - - /// - /// Method ToString returns a big-endian string starting by "0x" representing the little-endian unsigned int - /// Example: if this is storing 20-bytes 01ff00ff00ff00ff00ff00ff00ff00ff00ff00a4, ToString() should return "0xa400ff00ff00ff00ff00ff00ff00ff00ff00ff01" - /// - public override string ToString() - { - return "0x" + this.ToArray().ToHexString(reverse: true); - } - - /// - /// Method TryParse tries to parse a big-endian hex string and stores it as a UInt160 or UInt256 little-endian bytes array - /// Example: TryParse("0xa400ff00ff00ff00ff00ff00ff00ff00ff00ff01", result) should create result UInt160 01ff00ff00ff00ff00ff00ff00ff00ff00ff00a4 - /// - public static bool TryParse(string s, out T result) where T : UIntBase - { - int size; - if (typeof(T) == typeof(UInt160)) - size = 20; - else if (typeof(T) == typeof(UInt256)) - size = 32; - else if (s.Length == 40 || s.Length == 42) - size = 20; - else if (s.Length == 64 || s.Length == 66) - size = 32; - else - size = 0; - if (size == 20) - { - if (UInt160.TryParse(s, out UInt160 r)) - { - result = (T)(UIntBase)r; - return true; - } - } - else if (size == 32) - { - if (UInt256.TryParse(s, out UInt256 r)) - { - result = (T)(UIntBase)r; - return true; - } - } - result = null; - return false; - } - } -} diff --git a/tests/neo.UnitTests/UT_UIntBase.cs b/tests/neo.UnitTests/UT_UIntBase.cs deleted file mode 100644 index 5f02b92c25..0000000000 --- a/tests/neo.UnitTests/UT_UIntBase.cs +++ /dev/null @@ -1,49 +0,0 @@ -using FluentAssertions; -using Microsoft.VisualStudio.TestTools.UnitTesting; -using System; - -namespace Neo.UnitTests.IO -{ - [TestClass] - public class UT_UIntBase - { - [TestMethod] - public void TestParse() - { - UInt160 uInt1601 = (UInt160)UIntBase.Parse("0x0000000000000000000000000000000000000000"); - UInt256 uInt2561 = (UInt256)UIntBase.Parse("0x0000000000000000000000000000000000000000000000000000000000000000"); - UInt160 uInt1602 = (UInt160)UIntBase.Parse("0000000000000000000000000000000000000000"); - UInt256 uInt2562 = (UInt256)UIntBase.Parse("0000000000000000000000000000000000000000000000000000000000000000"); - Assert.AreEqual(UInt160.Zero, uInt1601); - Assert.AreEqual(UInt256.Zero, uInt2561); - Assert.AreEqual(UInt160.Zero, uInt1602); - Assert.AreEqual(UInt256.Zero, uInt2562); - Action action = () => UIntBase.Parse("0000000"); - action.Should().Throw(); - } - - [TestMethod] - public void TestTryParse() - { - UInt160 uInt160 = new UInt160(); - Assert.AreEqual(true, UIntBase.TryParse("0x0000000000000000000000000000000000000000", out uInt160)); - Assert.AreEqual(UInt160.Zero, uInt160); - Assert.AreEqual(false, UIntBase.TryParse("0x00000000000000000000000000000000000000", out uInt160)); - UInt256 uInt256 = new UInt256(); - Assert.AreEqual(true, UIntBase.TryParse("0x0000000000000000000000000000000000000000000000000000000000000000", out uInt256)); - Assert.AreEqual(UInt256.Zero, uInt256); - Assert.AreEqual(false, UIntBase.TryParse("0x00000000000000000000000000000000000000000000000000000000000000", out uInt256)); - UIntBase uIntBase = new UInt160(); - Assert.AreEqual(true, UIntBase.TryParse("0x0000000000000000000000000000000000000000", out uIntBase)); - Assert.AreEqual(UInt160.Zero, uIntBase); - Assert.AreEqual(true, UIntBase.TryParse("0000000000000000000000000000000000000000", out uIntBase)); - Assert.AreEqual(UInt160.Zero, uIntBase); - uIntBase = new UInt256(); - Assert.AreEqual(true, UIntBase.TryParse("0x0000000000000000000000000000000000000000000000000000000000000000", out uIntBase)); - Assert.AreEqual(UInt256.Zero, uIntBase); - Assert.AreEqual(true, UIntBase.TryParse("0000000000000000000000000000000000000000000000000000000000000000", out uIntBase)); - Assert.AreEqual(UInt256.Zero, uIntBase); - Assert.AreEqual(false, UIntBase.TryParse("00000000000000000000000000000000000000000000000000000000000000", out uIntBase)); - } - } -} From d5ec7151e24f288bad2b634485729ab0ed83869c Mon Sep 17 00:00:00 2001 From: Shargon Date: Fri, 17 Jan 2020 17:36:06 +0100 Subject: [PATCH 5/6] Allow to verify import blocks (#1415) * Verify Import * Default true --- src/neo/Ledger/Blockchain.cs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/neo/Ledger/Blockchain.cs b/src/neo/Ledger/Blockchain.cs index 48c7da914a..9090a5b339 100644 --- a/src/neo/Ledger/Blockchain.cs +++ b/src/neo/Ledger/Blockchain.cs @@ -23,7 +23,7 @@ public sealed partial class Blockchain : UntypedActor { public partial class ApplicationExecuted { } public class PersistCompleted { public Block Block; } - public class Import { public IEnumerable Blocks; } + public class Import { public IEnumerable Blocks; public bool Verify = true; } public class ImportCompleted { } public class FillMemoryPool { public IEnumerable Transactions; } public class FillCompleted { } @@ -239,13 +239,15 @@ public Transaction GetTransaction(UInt256 hash) return View.GetTransaction(hash); } - private void OnImport(IEnumerable blocks) + private void OnImport(IEnumerable blocks, bool verify) { foreach (Block block in blocks) { if (block.Index <= Height) continue; if (block.Index != Height + 1) throw new InvalidOperationException(); + if (verify && !block.Verify(currentSnapshot)) + throw new InvalidOperationException(); Persist(block); SaveHeaderHashList(); } @@ -453,7 +455,7 @@ protected override void OnReceive(object message) switch (message) { case Import import: - OnImport(import.Blocks); + OnImport(import.Blocks, import.Verify); break; case FillMemoryPool fill: OnFillMemoryPool(fill.Transactions); From 35c73cecf6538f70005fab1567367a6384b24092 Mon Sep 17 00:00:00 2001 From: Shargon Date: Sat, 18 Jan 2020 08:54:59 +0100 Subject: [PATCH 6/6] Fix notifications (#1422) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Vitor Nazário Coelho --- src/neo/SmartContract/Native/Tokens/Nep5Token.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/neo/SmartContract/Native/Tokens/Nep5Token.cs b/src/neo/SmartContract/Native/Tokens/Nep5Token.cs index bb92fbbd17..3e331de487 100644 --- a/src/neo/SmartContract/Native/Tokens/Nep5Token.cs +++ b/src/neo/SmartContract/Native/Tokens/Nep5Token.cs @@ -86,7 +86,7 @@ internal protected virtual void Mint(ApplicationEngine engine, UInt160 account, BigInteger totalSupply = new BigInteger(storage.Value); totalSupply += amount; storage.Value = totalSupply.ToByteArrayStandard(); - engine.SendNotification(Hash, new Array(engine.ReferenceCounter, new StackItem[] { "Transfer", StackItem.Null, account.ToArray(), amount })); + engine.SendNotification(Hash, new Array(new StackItem[] { "Transfer", StackItem.Null, account.ToArray(), amount })); } internal protected virtual void Burn(ApplicationEngine engine, UInt160 account, BigInteger amount) @@ -112,7 +112,7 @@ internal protected virtual void Burn(ApplicationEngine engine, UInt160 account, BigInteger totalSupply = new BigInteger(storage.Value); totalSupply -= amount; storage.Value = totalSupply.ToByteArrayStandard(); - engine.SendNotification(Hash, new Array(engine.ReferenceCounter, new StackItem[] { "Transfer", account.ToArray(), StackItem.Null, amount })); + engine.SendNotification(Hash, new Array(new StackItem[] { "Transfer", account.ToArray(), StackItem.Null, amount })); } [ContractMethod(0, ContractParameterType.String, Name = "name", SafeMethod = true)] @@ -222,7 +222,7 @@ protected virtual bool Transfer(ApplicationEngine engine, UInt160 from, UInt160 storage_to.Value = state_to.ToByteArray(); } } - engine.SendNotification(Hash, new Array(engine.ReferenceCounter, new StackItem[] { "Transfer", from.ToArray(), to.ToArray(), amount })); + engine.SendNotification(Hash, new Array(new StackItem[] { "Transfer", from.ToArray(), to.ToArray(), amount })); return true; }