Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Remove StatesDumper - Minor Refactor on StorageDumper #3281

Merged
merged 5 commits into from
May 28, 2024
Merged

Remove StatesDumper - Minor Refactor on StorageDumper #3281

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
f7f19ec
Remove StatesDumper - Minor Refactor on StorageDumper
vncoelho May 27, 2024
6072400
remove extra line
vncoelho May 27, 2024
f591b77
Merge branch 'master' into organize-plugins-statesStorageDumper
vncoelho May 27, 2024
dfe1e15
Merge branch 'master' into organize-plugins-statesStorageDumper
Jim8y May 27, 2024
6223937
Merge branch 'master' into organize-plugins-statesStorageDumper
Jim8y May 28, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 0 additions & 7 deletions neo.sln
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,8 +70,6 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "RpcServer", "src\Plugins\Rp
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "SQLiteWallet", "src\Plugins\SQLiteWallet\SQLiteWallet.csproj", "{F53D5FF0-5D3D-4E8B-A44F-C4C5D9B563B1}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "StatesDumper", "src\Plugins\StatesDumper\StatesDumper.csproj", "{90CCA7D4-C277-4112-A036-BBB90C3FE3BE}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "StateService", "src\Plugins\StateService\StateService.csproj", "{88975A8D-4797-45A4-BC3E-15962A425A54}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "StorageDumper", "src\Plugins\StorageDumper\StorageDumper.csproj", "{FF76D8A4-356B-461A-8471-BC1B83E57BBC}"
Expand Down Expand Up @@ -202,10 +200,6 @@ Global
{F53D5FF0-5D3D-4E8B-A44F-C4C5D9B563B1}.Debug|Any CPU.Build.0 = Debug|Any CPU
{F53D5FF0-5D3D-4E8B-A44F-C4C5D9B563B1}.Release|Any CPU.ActiveCfg = Release|Any CPU
{F53D5FF0-5D3D-4E8B-A44F-C4C5D9B563B1}.Release|Any CPU.Build.0 = Release|Any CPU
{90CCA7D4-C277-4112-A036-BBB90C3FE3BE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{90CCA7D4-C277-4112-A036-BBB90C3FE3BE}.Debug|Any CPU.Build.0 = Debug|Any CPU
{90CCA7D4-C277-4112-A036-BBB90C3FE3BE}.Release|Any CPU.ActiveCfg = Release|Any CPU
{90CCA7D4-C277-4112-A036-BBB90C3FE3BE}.Release|Any CPU.Build.0 = Release|Any CPU
{88975A8D-4797-45A4-BC3E-15962A425A54}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{88975A8D-4797-45A4-BC3E-15962A425A54}.Debug|Any CPU.Build.0 = Debug|Any CPU
{88975A8D-4797-45A4-BC3E-15962A425A54}.Release|Any CPU.ActiveCfg = Release|Any CPU
Expand Down Expand Up @@ -257,7 +251,6 @@ Global
{3DE59148-59D6-4CD3-8086-0BC74E3D4E0B} = {C2DC830A-327A-42A7-807D-295216D30DBB}
{A3941551-E72C-42D7-8C4D-5122CB60D73D} = {C2DC830A-327A-42A7-807D-295216D30DBB}
{F53D5FF0-5D3D-4E8B-A44F-C4C5D9B563B1} = {C2DC830A-327A-42A7-807D-295216D30DBB}
{90CCA7D4-C277-4112-A036-BBB90C3FE3BE} = {C2DC830A-327A-42A7-807D-295216D30DBB}
{88975A8D-4797-45A4-BC3E-15962A425A54} = {C2DC830A-327A-42A7-807D-295216D30DBB}
{FF76D8A4-356B-461A-8471-BC1B83E57BBC} = {C2DC830A-327A-42A7-807D-295216D30DBB}
{5E4947F3-05D3-4806-B0F3-30DAC71B5986} = {C2DC830A-327A-42A7-807D-295216D30DBB}
Expand Down
21 changes: 0 additions & 21 deletions src/Plugins/StatesDumper/PersistActions.cs
View file
Open in desktop

This file was deleted.

58 changes: 0 additions & 58 deletions src/Plugins/StatesDumper/Settings.cs
View file
Open in desktop

This file was deleted.

169 changes: 0 additions & 169 deletions src/Plugins/StatesDumper/StatesDumper.cs
View file
Open in desktop

This file was deleted.

18 changes: 0 additions & 18 deletions src/Plugins/StatesDumper/StatesDumper.csproj
View file
Open in desktop

This file was deleted.

9 changes: 0 additions & 9 deletions src/Plugins/StatesDumper/StatesDumper.json
View file
Open in desktop

This file was deleted.

6 changes: 5 additions & 1 deletion src/Plugins/StorageDumper/Settings.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,10 @@
/// Height to begin storage dump
/// </summary>
public uint HeightToBegin { get; }

/// <summary>
/// Default number of items per folder
/// </summary>
public uint StoragePerFolder { get; }
public IReadOnlyList<int> Exclude { get; }

public static Settings? Default { get; private set; }
Expand All @@ -34,8 +37,9 @@
/// Geting settings for storage changes state dumper
BlockCacheSize = section.GetValue("BlockCacheSize", 1000u);
HeightToBegin = section.GetValue("HeightToBegin", 0u);
StoragePerFolder = section.GetValue("StoragePerFolder", 100000u);
Exclude = section.GetSection("Exclude").Exists()
? section.GetSection("Exclude").GetChildren().Select(p => int.Parse(p.Value)).ToArray()

Check warning on line 42 in src/Plugins/StorageDumper/Settings.cs

View workflow job for this annotation

GitHub Actions / Test (windows-latest) 

Possible null reference argument for parameter 's' in 'int int.Parse(string s)'.

Check warning on line 42 in src/Plugins/StorageDumper/Settings.cs

View workflow job for this annotation

GitHub Actions / Test (macos-latest) 

Possible null reference argument for parameter 's' in 'int int.Parse(string s)'.
: new[] { NativeContract.Ledger.Id };
}

Expand Down
16 changes: 8 additions & 8 deletions src/Plugins/StorageDumper/StorageDumper.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,9 @@
private readonly Dictionary<uint, NeoSystem> systems = new Dictionary<uint, NeoSystem>();

private StreamWriter _writer;
/// <summary>
/// _currentBlock stores the last cached item
/// </summary>
private JObject _currentBlock;
private string _lastCreateDirectory;

Expand All @@ -32,7 +35,7 @@

public override string ConfigFile => System.IO.Path.Combine(RootPath, "StorageDumper.json");

public StorageDumper()

Check warning on line 38 in src/Plugins/StorageDumper/StorageDumper.cs

View workflow job for this annotation

GitHub Actions / Test (windows-latest) 

Non-nullable field '_writer' must contain a non-null value when exiting constructor. Consider declaring the field as nullable.

Check warning on line 38 in src/Plugins/StorageDumper/StorageDumper.cs

View workflow job for this annotation

GitHub Actions / Test (windows-latest) 

Non-nullable field '_currentBlock' must contain a non-null value when exiting constructor. Consider declaring the field as nullable.

Check warning on line 38 in src/Plugins/StorageDumper/StorageDumper.cs

View workflow job for this annotation

GitHub Actions / Test (windows-latest) 

Non-nullable field '_lastCreateDirectory' must contain a non-null value when exiting constructor. Consider declaring the field as nullable.

Check warning on line 38 in src/Plugins/StorageDumper/StorageDumper.cs

View workflow job for this annotation

GitHub Actions / Test (macos-latest) 

Non-nullable field '_writer' must contain a non-null value when exiting constructor. Consider declaring the field as nullable.

Check warning on line 38 in src/Plugins/StorageDumper/StorageDumper.cs

View workflow job for this annotation

GitHub Actions / Test (macos-latest) 

Non-nullable field '_currentBlock' must contain a non-null value when exiting constructor. Consider declaring the field as nullable.

Check warning on line 38 in src/Plugins/StorageDumper/StorageDumper.cs

View workflow job for this annotation

GitHub Actions / Test (macos-latest) 

Non-nullable field '_lastCreateDirectory' must contain a non-null value when exiting constructor. Consider declaring the field as nullable.
{
Blockchain.Committing += OnCommitting;
Blockchain.Committed += OnCommitted;
Expand Down Expand Up @@ -70,7 +73,7 @@
prefix = BitConverter.GetBytes(contract.Id);
}
var states = systems[network].StoreView.Find(prefix);
JArray array = new JArray(states.Where(p => !Settings.Default.Exclude.Contains(p.Key.Id)).Select(p => new JObject

Check warning on line 76 in src/Plugins/StorageDumper/StorageDumper.cs

View workflow job for this annotation

GitHub Actions / Test (windows-latest) 

Dereference of a possibly null reference.

Check warning on line 76 in src/Plugins/StorageDumper/StorageDumper.cs

View workflow job for this annotation

GitHub Actions / Test (macos-latest) 

Dereference of a possibly null reference.
{
["key"] = Convert.ToBase64String(p.Key.ToArray()),
["value"] = Convert.ToBase64String(p.Value.ToArray())
Expand All @@ -91,9 +94,9 @@
private void OnPersistStorage(uint network, DataCache snapshot)
{
uint blockIndex = NativeContract.Ledger.CurrentIndex(snapshot);
if (blockIndex >= Settings.Default.HeightToBegin)

Check warning on line 97 in src/Plugins/StorageDumper/StorageDumper.cs

View workflow job for this annotation

GitHub Actions / Test (windows-latest) 

Dereference of a possibly null reference.

Check warning on line 97 in src/Plugins/StorageDumper/StorageDumper.cs

View workflow job for this annotation

GitHub Actions / Test (macos-latest) 

Dereference of a possibly null reference.
{
JArray array = new JArray();
JArray stateChangeArray = new JArray();

foreach (var trackable in snapshot.GetChangeSet())
{
Expand All @@ -106,7 +109,6 @@
state["state"] = "Added";
state["key"] = Convert.ToBase64String(trackable.Key.ToArray());
state["value"] = Convert.ToBase64String(trackable.Item.ToArray());
// Here we have a new trackable.Key and trackable.Item
break;
case TrackState.Changed:
state["state"] = "Changed";
Expand All @@ -118,13 +120,13 @@
state["key"] = Convert.ToBase64String(trackable.Key.ToArray());
break;
}
array.Add(state);
stateChangeArray.Add(state);
}

JObject bs_item = new JObject();
bs_item["block"] = blockIndex;
bs_item["size"] = array.Count;
bs_item["storage"] = array;
bs_item["size"] = stateChangeArray.Count;
bs_item["storage"] = stateChangeArray;
_currentBlock = bs_item;
}
}
Expand All @@ -148,7 +150,7 @@
{
uint blockIndex = NativeContract.Ledger.CurrentIndex(snapshot);
if (_writer == null
|| blockIndex % Settings.Default.BlockCacheSize == 0)

Check warning on line 153 in src/Plugins/StorageDumper/StorageDumper.cs

View workflow job for this annotation

GitHub Actions / Test (windows-latest) 

Dereference of a possibly null reference.

Check warning on line 153 in src/Plugins/StorageDumper/StorageDumper.cs

View workflow job for this annotation

GitHub Actions / Test (macos-latest) 

Dereference of a possibly null reference.
{
string path = GetOrCreateDirectory(network, blockIndex);
var filepart = (blockIndex / Settings.Default.BlockCacheSize) * Settings.Default.BlockCacheSize;
Expand All @@ -174,9 +176,7 @@

private string GetDirectoryPath(uint network, uint blockIndex)
{
//Default Parameter
uint storagePerFolder = 100000;
uint folder = (blockIndex / storagePerFolder) * storagePerFolder;
uint folder = (blockIndex / Settings.Default.StoragePerFolder) * Settings.Default.StoragePerFolder;
return $"./StorageDumper_{network}/BlockStorage_{folder}";
}

Expand Down
1 change: 1 addition & 0 deletions src/Plugins/StorageDumper/StorageDumper.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
"PluginConfiguration": {
"BlockCacheSize": 1000,
"HeightToBegin": 0,
"StoragePerFolder": 100000,
"Exclude": [ -4 ]
}
}
Loading