add tokens tracker

neo-modules.sln

@@ -10,8 +10,6 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ApplicationLogs", "src\Appl
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "StatesDumper", "src\StatesDumper\StatesDumper.csproj", "{86531DB1-A231-46C4-823F-BE60972F7523}"
 EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "RpcNep17Tracker", "src\RpcNep17Tracker\RpcNep17Tracker.csproj", "{BBE8AC15-12DF-4AF0-ABC1-F1557EB5DC8E}"
-EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "LevelDBStore", "src\LevelDBStore\LevelDBStore.csproj", "{C66214CD-0B97-4EA5-B7A2-164F54346F19}"
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "RocksDBStore", "src\RocksDBStore\RocksDBStore.csproj", "{0E2AAF05-C55A-4B36-8750-F55743FBE4B3}"
@@ -34,6 +32,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DBFTPlugin", "src\DBFTPlugi
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Neo.Plugins.RpcServer.Tests", "tests\Neo.Plugins.RpcServer.Tests\Neo.Plugins.RpcServer.Tests.csproj", "{0DDAF738-0FD3-40A7-A433-C514BCDDF542}"
 EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "TokensTracker", "src\TokensTracker\TokensTracker.csproj", "{48CB0583-26CE-48FC-9F53-F7DC6F1727D8}"
+EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "MPTTrie", "src\MPTTrie\MPTTrie.csproj", "{D167FA6B-D2A3-4D8A-A65D-686DD06650F6}"
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Neo.Cryptography.MPTTrie.Tests", "tests\Neo.Cryptography.MPTTrie.Tests\Neo.Cryptography.MPTTrie.Tests.csproj", "{8D2EE375-2E2D-45FE-A4E9-0254D12C7554}"
@@ -52,10 +52,6 @@ Global
 		{86531DB1-A231-46C4-823F-BE60972F7523}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{86531DB1-A231-46C4-823F-BE60972F7523}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{86531DB1-A231-46C4-823F-BE60972F7523}.Release|Any CPU.Build.0 = Release|Any CPU
-		{BBE8AC15-12DF-4AF0-ABC1-F1557EB5DC8E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{BBE8AC15-12DF-4AF0-ABC1-F1557EB5DC8E}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{BBE8AC15-12DF-4AF0-ABC1-F1557EB5DC8E}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{BBE8AC15-12DF-4AF0-ABC1-F1557EB5DC8E}.Release|Any CPU.Build.0 = Release|Any CPU
 		{C66214CD-0B97-4EA5-B7A2-164F54346F19}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{C66214CD-0B97-4EA5-B7A2-164F54346F19}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{C66214CD-0B97-4EA5-B7A2-164F54346F19}.Release|Any CPU.ActiveCfg = Release|Any CPU
@@ -100,6 +96,10 @@ Global
 		{0DDAF738-0FD3-40A7-A433-C514BCDDF542}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{0DDAF738-0FD3-40A7-A433-C514BCDDF542}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{0DDAF738-0FD3-40A7-A433-C514BCDDF542}.Release|Any CPU.Build.0 = Release|Any CPU
+		{48CB0583-26CE-48FC-9F53-F7DC6F1727D8}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{48CB0583-26CE-48FC-9F53-F7DC6F1727D8}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{48CB0583-26CE-48FC-9F53-F7DC6F1727D8}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{48CB0583-26CE-48FC-9F53-F7DC6F1727D8}.Release|Any CPU.Build.0 = Release|Any CPU
 		{D167FA6B-D2A3-4D8A-A65D-686DD06650F6}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{D167FA6B-D2A3-4D8A-A65D-686DD06650F6}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{D167FA6B-D2A3-4D8A-A65D-686DD06650F6}.Release|Any CPU.ActiveCfg = Release|Any CPU
@@ -115,7 +115,6 @@ Global
 	GlobalSection(NestedProjects) = preSolution
 		{84DA8EA6-EF60-4FCD-B1C6-65C1A8323B3F} = {97E81C78-1637-481F-9485-DA1225E94C23}
 		{86531DB1-A231-46C4-823F-BE60972F7523} = {97E81C78-1637-481F-9485-DA1225E94C23}
-		{BBE8AC15-12DF-4AF0-ABC1-F1557EB5DC8E} = {97E81C78-1637-481F-9485-DA1225E94C23}
 		{C66214CD-0B97-4EA5-B7A2-164F54346F19} = {97E81C78-1637-481F-9485-DA1225E94C23}
 		{0E2AAF05-C55A-4B36-8750-F55743FBE4B3} = {97E81C78-1637-481F-9485-DA1225E94C23}
 		{8DC57A45-A192-4953-81B1-6907FB7C28D2} = {97E81C78-1637-481F-9485-DA1225E94C23}
@@ -127,6 +126,7 @@ Global
 		{A0F4A66F-6F87-4B99-B8BE-A779BC002F47} = {97E81C78-1637-481F-9485-DA1225E94C23}
 		{90185D3E-4813-4BC1-98FE-26FD34311403} = {97E81C78-1637-481F-9485-DA1225E94C23}
 		{0DDAF738-0FD3-40A7-A433-C514BCDDF542} = {59D802AB-C552-422A-B9C3-64D329FBCDCC}
+		{48CB0583-26CE-48FC-9F53-F7DC6F1727D8} = {97E81C78-1637-481F-9485-DA1225E94C23}
 		{D167FA6B-D2A3-4D8A-A65D-686DD06650F6} = {97E81C78-1637-481F-9485-DA1225E94C23}
 		{8D2EE375-2E2D-45FE-A4E9-0254D12C7554} = {59D802AB-C552-422A-B9C3-64D329FBCDCC}
 	EndGlobalSection

src/TokensTracker/Extensions.cs

@@ -0,0 +1,56 @@
+using System;
+using System.Collections.Generic;
+using System.Numerics;
+using Neo.IO;
+using Neo.Persistence;
+using Neo.VM.Types;
+
+namespace Neo.Plugins
+{
+    public static class Extensions
+    {
+        public static bool NotNull(this StackItem item)
+        {
+            return !item.IsNull;
+        }
+
+        public static string ToBase64(this ReadOnlySpan<byte> item)
+        {
+            return item == null ? String.Empty : Convert.ToBase64String(item);
+        }
+
+        public static int GetVarSize(this ByteString item)
+        {
+            var length = item.GetSpan().Length;
+            return IO.Helper.GetVarSize(length) + length;
+        }
+
+        public static int GetVarSize(this BigInteger item)
+        {
+            var length = item.GetByteCount();
+            return IO.Helper.GetVarSize(length) + length;
+        }
+
+        public static IEnumerable<(TKey, TValue)> FindPrefix<TKey, TValue>(this IStore db, byte[] prefix)
+            where TKey : ISerializable, new()
+            where TValue : class, ISerializable, new()
+        {
+            foreach (var (key, value) in db.Seek(prefix, SeekDirection.Forward))
+            {
+                if (!key.AsSpan().StartsWith(prefix)) break;
+                yield return (key.AsSerializable<TKey>(1), value.AsSerializable<TValue>());
+            }
+        }
+
+        public static IEnumerable<(TKey, TValue)> FindRange<TKey, TValue>(this IStore db, byte[] startKey, byte[] endKey)
+            where TKey : ISerializable, new()
+            where TValue : class, ISerializable, new()
+        {
+            foreach (var (key, value) in db.Seek(startKey, SeekDirection.Forward))
+            {
+                if (key.AsSpan().SequenceCompareTo(endKey) > 0) break;
+                yield return (key.AsSerializable<TKey>(1), value.AsSerializable<TValue>());
+            }
+        }
+    }
+}

src/TokensTracker/TokensTracker.cs

@@ -0,0 +1,84 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using Microsoft.Extensions.Configuration;
+using Neo.IO;
+using Neo.Ledger;
+using Neo.Network.P2P.Payloads;
+using Neo.Persistence;
+using Neo.Plugins.Trackers;
+using static System.IO.Path;
+
+namespace Neo.Plugins
+{
+    public class TokensTracker : Plugin, IPersistencePlugin
+    {
+        private string _dbPath;
+        private bool _shouldTrackHistory;
+        private uint _maxResults;
+        private uint _network;
+        private string[] _enabledTrackers;
+        private IStore _db;
+        private NeoSystem neoSystem;
+        private readonly List<TrackerBase> trackers = new();
+
+        public override string Description => "Enquiries balances and transaction history of accounts through RPC";
+
+        protected override void Configure()
+        {
+            IConfigurationSection config = GetConfiguration();
+            _dbPath = config.GetValue("DBPath", "TokensBalanceData");
+            _shouldTrackHistory = config.GetValue("TrackHistory", true);
+            _maxResults = config.GetValue("MaxResults", 1000u);
+            _network = config.GetValue("Network", 860833102u);
+            _enabledTrackers = config.GetSection("EnabledTrackers").GetChildren().Select(p => p.Value).ToArray();
+        }
+
+        protected override void OnSystemLoaded(NeoSystem system)
+        {
+            if (system.Settings.Network != _network) return;
+            neoSystem = system;
+            string path = string.Format(_dbPath, neoSystem.Settings.Network.ToString("X8"));
+            _db = neoSystem.LoadStore(GetFullPath(path));
+            if (_enabledTrackers.Contains("NEP-11"))
+                trackers.Add(new Trackers.NEP_11.Nep11Tracker(_db, _maxResults, _shouldTrackHistory, neoSystem));
+            if (_enabledTrackers.Contains("NEP-17"))
+                trackers.Add(new Trackers.NEP_17.Nep17Tracker(_db, _maxResults, _shouldTrackHistory, neoSystem));
+            foreach (TrackerBase tracker in trackers)
+                RpcServerPlugin.RegisterMethods(tracker, _network);
+        }
+
+        private void ResetBatch()
+        {
+            foreach (var tracker in trackers)
+            {
+                tracker.ResetBatch();
+            }
+        }
+
+        void IPersistencePlugin.OnPersist(NeoSystem system, Block block, DataCache snapshot, IReadOnlyList<Blockchain.ApplicationExecuted> applicationExecutedList)
+        {
+            if (system.Settings.Network != _network) return;
+            // Start freshly with a new DBCache for each block.
+            ResetBatch();
+            foreach (var tracker in trackers)
+            {
+                tracker.OnPersist(system, block, snapshot, applicationExecutedList);
+            }
+        }
+
+        void IPersistencePlugin.OnCommit(NeoSystem system, Block block, DataCache snapshot)
+        {
+            if (system.Settings.Network != _network) return;
+            foreach (var tracker in trackers)
+            {
+                tracker.Commit();
+            }
+        }
+
+        bool IPersistencePlugin.ShouldThrowExceptionFromCommit(Exception ex)
+        {
+            return true;
+        }
+    }
+}

src/TokensTracker/TokensTracker.csproj

@@ -0,0 +1,19 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <PackageId>Neo.Plugins.TokensTracker</PackageId>
+    <RootNamespace>Neo.Plugins</RootNamespace>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\RpcServer\RpcServer.csproj" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <None Update="TokensTracker\config.json">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+      <CopyToPublishDirectory>PreserveNewest</CopyToPublishDirectory>
+    </None>
+  </ItemGroup>
+
+</Project>

src/TokensTracker/TokensTracker/config.json

@@ -0,0 +1,12 @@
+{
+  "PluginConfiguration": {
+    "DBPath": "TokenBalanceData",
+    "TrackHistory": true,
+    "MaxResults": 1000,
+    "Network": 860833102,
+    "EnabledTrackers": [ "NEP-11", "NEP-17" ]
+  },
+  "Dependency": [
+    "RpcServer"
+  ]
+}

src/TokensTracker/Trackers/NEP-11/Nep11BalanceKey.cs

@@ -0,0 +1,70 @@
+using System;
+using System.IO;
+using Neo.IO;
+using Neo.VM.Types;
+
+namespace Neo.Plugins.Trackers.NEP_11
+{
+    public class Nep11BalanceKey : IComparable<Nep11BalanceKey>, IEquatable<Nep11BalanceKey>, ISerializable
+    {
+        public readonly UInt160 UserScriptHash;
+        public readonly UInt160 AssetScriptHash;
+        public ByteString Token;
+        public int Size => UInt160.Length + UInt160.Length + Token.GetVarSize();
+
+        public Nep11BalanceKey() : this(new UInt160(), new UInt160(), ByteString.Empty)
+        {
+        }
+
+        public Nep11BalanceKey(UInt160 userScriptHash, UInt160 assetScriptHash, ByteString tokenId)
+        {
+            if (userScriptHash == null || assetScriptHash == null || tokenId == null)
+                throw new ArgumentNullException();
+            UserScriptHash = userScriptHash;
+            AssetScriptHash = assetScriptHash;
+            Token = tokenId;
+        }
+
+        public int CompareTo(Nep11BalanceKey other)
+        {
+            if (other is null) return 1;
+            if (ReferenceEquals(this, other)) return 0;
+            int result = UserScriptHash.CompareTo(other.UserScriptHash);
+            if (result != 0) return result;
+            result = AssetScriptHash.CompareTo(other.AssetScriptHash);
+            if (result != 0) return result;
+            return (Token.GetInteger() - other.Token.GetInteger()).Sign;
+        }
+
+        public bool Equals(Nep11BalanceKey other)
+        {
+            if (other is null) return false;
+            if (ReferenceEquals(this, other)) return true;
+            return UserScriptHash.Equals(other.UserScriptHash) && AssetScriptHash.Equals(AssetScriptHash) && Token.Equals(other.Token);
+        }
+
+        public override bool Equals(Object other)
+        {
+            return other is Nep11BalanceKey otherKey && Equals(otherKey);
+        }
+
+        public override int GetHashCode()
+        {
+            return HashCode.Combine(UserScriptHash.GetHashCode(), AssetScriptHash.GetHashCode(), Token.GetHashCode());
+        }
+
+        public void Serialize(BinaryWriter writer)
+        {
+            writer.Write(UserScriptHash);
+            writer.Write(AssetScriptHash);
+            writer.WriteVarBytes(Token.GetSpan());
+        }
+
+        public void Deserialize(BinaryReader reader)
+        {
+            ((ISerializable)UserScriptHash).Deserialize(reader);
+            ((ISerializable)AssetScriptHash).Deserialize(reader);
+            Token = new ByteString(reader.ReadVarBytes());
+        }
+    }
+}