Refac statedump

src/StatesDumper/Settings.cs

@@ -26,10 +26,6 @@ internal class Settings
         /// </summary>
         public uint HeightToBegin { get; }
         /// <summary>
-        /// Height to begin real-time syncing and dumping on, consequently, dumping every block into a single files
-        /// </summary>
-        public int HeightToStartRealTimeSyncing { get; }
-        /// <summary>
         /// Persisting actions
         /// </summary>
         public PersistActions PersistAction { get; }
@@ -42,7 +38,6 @@ private Settings(IConfigurationSection section)
             /// Geting settings for storage changes state dumper
             this.BlockCacheSize = section.GetValue("BlockCacheSize", 1000u);
             this.HeightToBegin = section.GetValue("HeightToBegin", 0u);
-            this.HeightToStartRealTimeSyncing = section.GetValue("HeightToStartRealTimeSyncing", -1);
             this.PersistAction = section.GetValue("PersistAction", PersistActions.StorageChanges);
             this.Exclude = section.GetSection("Exclude").Exists()
                 ? section.GetSection("Exclude").GetChildren().Select(p => int.Parse(p.Value)).ToArray()

src/StatesDumper/StatesDumper.cs

@@ -24,9 +24,13 @@ namespace Neo.Plugins
 {
     public class StatesDumper : Plugin
     {
-        private readonly Dictionary<uint, JArray> bs_cache = new Dictionary<uint, JArray>();
         private readonly Dictionary<uint, NeoSystem> systems = new Dictionary<uint, NeoSystem>();
 
+        private StreamWriter _writer;
+        private JObject _currentBlock;
+        private string _lastCreateDirectory;
+
+
         public override string Description => "Exports Neo-CLI status data";
 
         public StatesDumper()
@@ -82,7 +86,10 @@ private void OnDumpStorage(uint network, UInt160 contractHash = null)
         private void OnCommitting(NeoSystem system, Block block, DataCache snapshot, IReadOnlyList<Blockchain.ApplicationExecuted> applicationExecutedList)
         {
             if (Settings.Default.PersistAction.HasFlag(PersistActions.StorageChanges))
+            {
+                InitFileWriter(system.Settings.Network, snapshot);
                 OnPersistStorage(system.Settings.Network, snapshot);
+            }
         }
 
         private void OnPersistStorage(uint network, DataCache snapshot)
@@ -122,45 +129,63 @@ private void OnPersistStorage(uint network, DataCache snapshot)
                 bs_item["block"] = blockIndex;
                 bs_item["size"] = array.Count;
                 bs_item["storage"] = array;
-                if (!bs_cache.TryGetValue(network, out JArray cache))
-                {
-                    cache = new JArray();
-                }
-                cache.Add(bs_item);
-                bs_cache[network] = cache;
+                _currentBlock = bs_item;
             }
         }
 
+
         private void OnCommitted(NeoSystem system, Block block)
         {
             if (Settings.Default.PersistAction.HasFlag(PersistActions.StorageChanges))
+            {
                 OnCommitStorage(system.Settings.Network, system.StoreView);
+            }
         }
 
         void OnCommitStorage(uint network, DataCache snapshot)
         {
-            if (!bs_cache.TryGetValue(network, out JArray cache)) return;
-            if (cache.Count == 0) return;
+            if (_currentBlock != null)
+            {
+                _writer.WriteLine(_currentBlock.ToString());
+                _writer.Flush();
+            }
+        }
+
+        private void InitFileWriter(uint network, DataCache snapshot)
+        {
             uint blockIndex = NativeContract.Ledger.CurrentIndex(snapshot);
-            if ((blockIndex % Settings.Default.BlockCacheSize == 0) || (Settings.Default.HeightToStartRealTimeSyncing != -1 && blockIndex >= Settings.Default.HeightToStartRealTimeSyncing))
+            if (_writer == null
+                || blockIndex % Settings.Default.BlockCacheSize == 0)
+            {
+                string path = GetOrCreateDirectory(network, blockIndex);
+                var filepart = (blockIndex / Settings.Default.BlockCacheSize) * Settings.Default.BlockCacheSize;
+                path = $"{path}/dump-block-{filepart}.dump";
+                if (_writer != null)
+                {
+                    _writer.Dispose();
+                }
+                _writer = new StreamWriter(new FileStream(path, FileMode.Append));
+            }
+        }
+
+        private string GetOrCreateDirectory(uint network, uint blockIndex)
+        {
+            string dirPathWithBlock = GetDirectoryPath(network, blockIndex);
+            if (_lastCreateDirectory != dirPathWithBlock)
             {
-                string path = HandlePaths(network, blockIndex);
-                path = $"{path}/dump-block-{blockIndex}.json";
-                File.WriteAllText(path, cache.ToString());
-                cache.Clear();
+                Directory.CreateDirectory(dirPathWithBlock);
+                _lastCreateDirectory = dirPathWithBlock;
             }
+            return dirPathWithBlock;
         }
 
-        private static string HandlePaths(uint network, uint blockIndex)
+        private string GetDirectoryPath(uint network, uint blockIndex)
         {
             //Default Parameter
             uint storagePerFolder = 100000;
-            uint folder = (((blockIndex - 1) / storagePerFolder) + 1) * storagePerFolder;
-            if (blockIndex == 0)
-                folder = 0;
-            string dirPathWithBlock = $"./Storage_{network:x8}/BlockStorage_{folder}";
-            Directory.CreateDirectory(dirPathWithBlock);
-            return dirPathWithBlock;
+            uint folder = (blockIndex / storagePerFolder) * storagePerFolder;
+            return $"./Storage_{network}/BlockStorage_{folder}";
         }
+
     }
 }

src/StatesDumper/config.json

@@ -3,7 +3,6 @@
     "PersistAction": "StorageChanges",
     "BlockCacheSize": 1000,
     "HeightToBegin": 0,
-    "HeightToStartRealTimeSyncing": -1,
     "Exclude": [ -4 ]
   }
 }