diff --git a/neo.sln b/neo.sln index 46ff7c64e8..b0de1c27b1 100644 --- a/neo.sln +++ b/neo.sln @@ -70,8 +70,6 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "RpcServer", "src\Plugins\Rp EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "SQLiteWallet", "src\Plugins\SQLiteWallet\SQLiteWallet.csproj", "{F53D5FF0-5D3D-4E8B-A44F-C4C5D9B563B1}" EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "StatesDumper", "src\Plugins\StatesDumper\StatesDumper.csproj", "{90CCA7D4-C277-4112-A036-BBB90C3FE3BE}" -EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "StateService", "src\Plugins\StateService\StateService.csproj", "{88975A8D-4797-45A4-BC3E-15962A425A54}" EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "StorageDumper", "src\Plugins\StorageDumper\StorageDumper.csproj", "{FF76D8A4-356B-461A-8471-BC1B83E57BBC}" @@ -202,10 +200,6 @@ Global {F53D5FF0-5D3D-4E8B-A44F-C4C5D9B563B1}.Debug|Any CPU.Build.0 = Debug|Any CPU {F53D5FF0-5D3D-4E8B-A44F-C4C5D9B563B1}.Release|Any CPU.ActiveCfg = Release|Any CPU {F53D5FF0-5D3D-4E8B-A44F-C4C5D9B563B1}.Release|Any CPU.Build.0 = Release|Any CPU - {90CCA7D4-C277-4112-A036-BBB90C3FE3BE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {90CCA7D4-C277-4112-A036-BBB90C3FE3BE}.Debug|Any CPU.Build.0 = Debug|Any CPU - {90CCA7D4-C277-4112-A036-BBB90C3FE3BE}.Release|Any CPU.ActiveCfg = Release|Any CPU - {90CCA7D4-C277-4112-A036-BBB90C3FE3BE}.Release|Any CPU.Build.0 = Release|Any CPU {88975A8D-4797-45A4-BC3E-15962A425A54}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {88975A8D-4797-45A4-BC3E-15962A425A54}.Debug|Any CPU.Build.0 = Debug|Any CPU {88975A8D-4797-45A4-BC3E-15962A425A54}.Release|Any CPU.ActiveCfg = Release|Any CPU @@ -257,7 +251,6 @@ Global {3DE59148-59D6-4CD3-8086-0BC74E3D4E0B} = {C2DC830A-327A-42A7-807D-295216D30DBB} {A3941551-E72C-42D7-8C4D-5122CB60D73D} = {C2DC830A-327A-42A7-807D-295216D30DBB} {F53D5FF0-5D3D-4E8B-A44F-C4C5D9B563B1} = {C2DC830A-327A-42A7-807D-295216D30DBB} - {90CCA7D4-C277-4112-A036-BBB90C3FE3BE} = {C2DC830A-327A-42A7-807D-295216D30DBB} {88975A8D-4797-45A4-BC3E-15962A425A54} = {C2DC830A-327A-42A7-807D-295216D30DBB} {FF76D8A4-356B-461A-8471-BC1B83E57BBC} = {C2DC830A-327A-42A7-807D-295216D30DBB} {5E4947F3-05D3-4806-B0F3-30DAC71B5986} = {C2DC830A-327A-42A7-807D-295216D30DBB} diff --git a/src/Plugins/StatesDumper/PersistActions.cs b/src/Plugins/StatesDumper/PersistActions.cs deleted file mode 100644 index 9c1aa2210c..0000000000 --- a/src/Plugins/StatesDumper/PersistActions.cs +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright (C) 2015-2024 The Neo Project. -// -// PersistActions.cs file belongs to the neo project and is free -// software distributed under the MIT software license, see the -// accompanying file LICENSE in the main directory of the -// repository or http://www.opensource.org/licenses/mit-license.php -// for more details. -// -// Redistribution and use in source and binary forms with or without -// modifications are permitted. - -using System; - -namespace Neo.Plugins -{ - [Flags] - internal enum PersistActions : byte - { - StorageChanges = 0b00000001 - } -} diff --git a/src/Plugins/StatesDumper/Settings.cs b/src/Plugins/StatesDumper/Settings.cs deleted file mode 100644 index 8fd3b9e163..0000000000 --- a/src/Plugins/StatesDumper/Settings.cs +++ /dev/null @@ -1,58 +0,0 @@ -// Copyright (C) 2015-2024 The Neo Project. -// -// Settings.cs file belongs to the neo project and is free -// software distributed under the MIT software license, see the -// accompanying file LICENSE in the main directory of the -// repository or http://www.opensource.org/licenses/mit-license.php -// for more details. -// -// Redistribution and use in source and binary forms with or without -// modifications are permitted. - -using Microsoft.Extensions.Configuration; -using Neo.SmartContract.Native; -using System.Collections.Generic; -using System.Linq; - -namespace Neo.Plugins -{ - internal class Settings - { - /// - /// Amount of storages states (heights) to be dump in a given json file - /// - public uint BlockCacheSize { get; } - /// - /// Height to begin storage dump - /// - public uint HeightToBegin { get; } - /// - /// Height to begin real-time syncing and dumping on, consequently, dumping every block into a single files - /// - public int HeightToStartRealTimeSyncing { get; } - /// - /// Persisting actions - /// - public PersistActions PersistAction { get; } - public IReadOnlyList Exclude { get; } - - public static Settings Default { get; private set; } - - private Settings(IConfigurationSection section) - { - /// Geting settings for storage changes state dumper - BlockCacheSize = section.GetValue("BlockCacheSize", 1000u); - HeightToBegin = section.GetValue("HeightToBegin", 0u); - HeightToStartRealTimeSyncing = section.GetValue("HeightToStartRealTimeSyncing", -1); - PersistAction = section.GetValue("PersistAction", PersistActions.StorageChanges); - Exclude = section.GetSection("Exclude").Exists() - ? section.GetSection("Exclude").GetChildren().Select(p => int.Parse(p.Value)).ToArray() - : new[] { NativeContract.Ledger.Id }; - } - - public static void Load(IConfigurationSection section) - { - Default = new Settings(section); - } - } -} diff --git a/src/Plugins/StatesDumper/StatesDumper.cs b/src/Plugins/StatesDumper/StatesDumper.cs deleted file mode 100644 index da1bbd3f69..0000000000 --- a/src/Plugins/StatesDumper/StatesDumper.cs +++ /dev/null @@ -1,169 +0,0 @@ -// Copyright (C) 2015-2024 The Neo Project. -// -// StatesDumper.cs file belongs to the neo project and is free -// software distributed under the MIT software license, see the -// accompanying file LICENSE in the main directory of the -// repository or http://www.opensource.org/licenses/mit-license.php -// for more details. -// -// Redistribution and use in source and binary forms with or without -// modifications are permitted. - -using Neo.ConsoleService; -using Neo.IO; -using Neo.Json; -using Neo.Ledger; -using Neo.Network.P2P.Payloads; -using Neo.Persistence; -using Neo.SmartContract.Native; -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; - -namespace Neo.Plugins -{ - public class StatesDumper : Plugin - { - private readonly Dictionary bs_cache = new Dictionary(); - private readonly Dictionary systems = new Dictionary(); - - public override string Description => "Exports Neo-CLI status data"; - - public override string ConfigFile => System.IO.Path.Combine(RootPath, "StatesDumper.json"); - - public StatesDumper() - { - Blockchain.Committing += OnCommitting; - Blockchain.Committed += OnCommitted; - } - - public override void Dispose() - { - Blockchain.Committing -= OnCommitting; - Blockchain.Committed -= OnCommitted; - } - - protected override void Configure() - { - Settings.Load(GetConfiguration()); - } - - protected override void OnSystemLoaded(NeoSystem system) - { - systems.Add(system.Settings.Network, system); - } - - /// - /// Process "dump storage" command - /// - [ConsoleCommand("dump storage", Category = "Storage", Description = "You can specify the contract script hash or use null to get the corresponding information from the storage")] - private void OnDumpStorage(uint network, UInt160 contractHash = null) - { - if (!systems.ContainsKey(network)) throw new InvalidOperationException("invalid network"); - string path = $"dump_{network:x8}.json"; - byte[] prefix = null; - if (contractHash is not null) - { - var contract = NativeContract.ContractManagement.GetContract(systems[network].StoreView, contractHash); - if (contract is null) throw new InvalidOperationException("contract not found"); - prefix = BitConverter.GetBytes(contract.Id); - } - var states = systems[network].StoreView.Find(prefix); - JArray array = new JArray(states.Where(p => !Settings.Default.Exclude.Contains(p.Key.Id)).Select(p => new JObject - { - ["key"] = Convert.ToBase64String(p.Key.ToArray()), - ["value"] = Convert.ToBase64String(p.Value.ToArray()) - })); - File.WriteAllText(path, array.ToString()); - ConsoleHelper.Info("States", - $"({array.Count})", - " have been dumped into file ", - $"{path}"); - } - - private void OnCommitting(NeoSystem system, Block block, DataCache snapshot, IReadOnlyList applicationExecutedList) - { - if (Settings.Default.PersistAction.HasFlag(PersistActions.StorageChanges)) - OnPersistStorage(system.Settings.Network, snapshot); - } - - private void OnPersistStorage(uint network, DataCache snapshot) - { - uint blockIndex = NativeContract.Ledger.CurrentIndex(snapshot); - if (blockIndex >= Settings.Default.HeightToBegin) - { - JArray array = new JArray(); - - foreach (var trackable in snapshot.GetChangeSet()) - { - if (Settings.Default.Exclude.Contains(trackable.Key.Id)) - continue; - JObject state = new JObject(); - switch (trackable.State) - { - case TrackState.Added: - state["state"] = "Added"; - state["key"] = Convert.ToBase64String(trackable.Key.ToArray()); - state["value"] = Convert.ToBase64String(trackable.Item.ToArray()); - // Here we have a new trackable.Key and trackable.Item - break; - case TrackState.Changed: - state["state"] = "Changed"; - state["key"] = Convert.ToBase64String(trackable.Key.ToArray()); - state["value"] = Convert.ToBase64String(trackable.Item.ToArray()); - break; - case TrackState.Deleted: - state["state"] = "Deleted"; - state["key"] = Convert.ToBase64String(trackable.Key.ToArray()); - break; - } - array.Add(state); - } - - JObject bs_item = new JObject(); - bs_item["block"] = blockIndex; - bs_item["size"] = array.Count; - bs_item["storage"] = array; - if (!bs_cache.TryGetValue(network, out JArray cache)) - { - cache = new JArray(); - } - cache.Add(bs_item); - bs_cache[network] = cache; - } - } - - private void OnCommitted(NeoSystem system, Block block) - { - if (Settings.Default.PersistAction.HasFlag(PersistActions.StorageChanges)) - OnCommitStorage(system.Settings.Network, system.StoreView); - } - - void OnCommitStorage(uint network, DataCache snapshot) - { - if (!bs_cache.TryGetValue(network, out JArray cache)) return; - if (cache.Count == 0) return; - uint blockIndex = NativeContract.Ledger.CurrentIndex(snapshot); - if ((blockIndex % Settings.Default.BlockCacheSize == 0) || (Settings.Default.HeightToStartRealTimeSyncing != -1 && blockIndex >= Settings.Default.HeightToStartRealTimeSyncing)) - { - string path = HandlePaths(network, blockIndex); - path = $"{path}/dump-block-{blockIndex}.json"; - File.WriteAllText(path, cache.ToString()); - cache.Clear(); - } - } - - private static string HandlePaths(uint network, uint blockIndex) - { - //Default Parameter - uint storagePerFolder = 100000; - uint folder = (((blockIndex - 1) / storagePerFolder) + 1) * storagePerFolder; - if (blockIndex == 0) - folder = 0; - string dirPathWithBlock = $"./Storage_{network:x8}/BlockStorage_{folder}"; - Directory.CreateDirectory(dirPathWithBlock); - return dirPathWithBlock; - } - } -} diff --git a/src/Plugins/StatesDumper/StatesDumper.csproj b/src/Plugins/StatesDumper/StatesDumper.csproj deleted file mode 100644 index 4ebf25f2c1..0000000000 --- a/src/Plugins/StatesDumper/StatesDumper.csproj +++ /dev/null @@ -1,18 +0,0 @@ - - - - net8.0 - Neo.Plugins.StatesDumper - - - - - - - - - PreserveNewest - - - - diff --git a/src/Plugins/StatesDumper/StatesDumper.json b/src/Plugins/StatesDumper/StatesDumper.json deleted file mode 100644 index c3b73767dd..0000000000 --- a/src/Plugins/StatesDumper/StatesDumper.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "PluginConfiguration": { - "PersistAction": "StorageChanges", - "BlockCacheSize": 1000, - "HeightToBegin": 0, - "HeightToStartRealTimeSyncing": -1, - "Exclude": [ -4 ] - } -} diff --git a/src/Plugins/StorageDumper/Settings.cs b/src/Plugins/StorageDumper/Settings.cs index 84fe2b2231..aac37ea6d8 100644 --- a/src/Plugins/StorageDumper/Settings.cs +++ b/src/Plugins/StorageDumper/Settings.cs @@ -24,7 +24,10 @@ internal class Settings /// Height to begin storage dump /// public uint HeightToBegin { get; } - + /// + /// Default number of items per folder + /// + public uint StoragePerFolder { get; } public IReadOnlyList Exclude { get; } public static Settings? Default { get; private set; } @@ -34,6 +37,7 @@ private Settings(IConfigurationSection section) /// Geting settings for storage changes state dumper BlockCacheSize = section.GetValue("BlockCacheSize", 1000u); HeightToBegin = section.GetValue("HeightToBegin", 0u); + StoragePerFolder = section.GetValue("StoragePerFolder", 100000u); Exclude = section.GetSection("Exclude").Exists() ? section.GetSection("Exclude").GetChildren().Select(p => int.Parse(p.Value)).ToArray() : new[] { NativeContract.Ledger.Id }; diff --git a/src/Plugins/StorageDumper/StorageDumper.cs b/src/Plugins/StorageDumper/StorageDumper.cs index 696f5014ac..eb9a56c72b 100644 --- a/src/Plugins/StorageDumper/StorageDumper.cs +++ b/src/Plugins/StorageDumper/StorageDumper.cs @@ -24,6 +24,9 @@ public class StorageDumper : Plugin private readonly Dictionary systems = new Dictionary(); private StreamWriter _writer; + /// + /// _currentBlock stores the last cached item + /// private JObject _currentBlock; private string _lastCreateDirectory; @@ -93,7 +96,7 @@ private void OnPersistStorage(uint network, DataCache snapshot) uint blockIndex = NativeContract.Ledger.CurrentIndex(snapshot); if (blockIndex >= Settings.Default.HeightToBegin) { - JArray array = new JArray(); + JArray stateChangeArray = new JArray(); foreach (var trackable in snapshot.GetChangeSet()) { @@ -106,7 +109,6 @@ private void OnPersistStorage(uint network, DataCache snapshot) state["state"] = "Added"; state["key"] = Convert.ToBase64String(trackable.Key.ToArray()); state["value"] = Convert.ToBase64String(trackable.Item.ToArray()); - // Here we have a new trackable.Key and trackable.Item break; case TrackState.Changed: state["state"] = "Changed"; @@ -118,13 +120,13 @@ private void OnPersistStorage(uint network, DataCache snapshot) state["key"] = Convert.ToBase64String(trackable.Key.ToArray()); break; } - array.Add(state); + stateChangeArray.Add(state); } JObject bs_item = new JObject(); bs_item["block"] = blockIndex; - bs_item["size"] = array.Count; - bs_item["storage"] = array; + bs_item["size"] = stateChangeArray.Count; + bs_item["storage"] = stateChangeArray; _currentBlock = bs_item; } } @@ -174,9 +176,7 @@ private string GetOrCreateDirectory(uint network, uint blockIndex) private string GetDirectoryPath(uint network, uint blockIndex) { - //Default Parameter - uint storagePerFolder = 100000; - uint folder = (blockIndex / storagePerFolder) * storagePerFolder; + uint folder = (blockIndex / Settings.Default.StoragePerFolder) * Settings.Default.StoragePerFolder; return $"./StorageDumper_{network}/BlockStorage_{folder}"; } diff --git a/src/Plugins/StorageDumper/StorageDumper.json b/src/Plugins/StorageDumper/StorageDumper.json index 3f5c0537f0..b327c37e0c 100644 --- a/src/Plugins/StorageDumper/StorageDumper.json +++ b/src/Plugins/StorageDumper/StorageDumper.json @@ -2,6 +2,7 @@ "PluginConfiguration": { "BlockCacheSize": 1000, "HeightToBegin": 0, + "StoragePerFolder": 100000, "Exclude": [ -4 ] } }