From 499a7b0bf043a0e7251c2df10796b613a32357f0 Mon Sep 17 00:00:00 2001 From: Atralupus Date: Mon, 18 Nov 2024 11:37:35 +0900 Subject: [PATCH 1/3] init --- Mimir.Initializer/Mimir.Initializer.csproj | 10 ++++++++++ Mimir.Initializer/Program.cs | 2 ++ Mimir.sln | 6 ++++++ 3 files changed, 18 insertions(+) create mode 100644 Mimir.Initializer/Mimir.Initializer.csproj create mode 100644 Mimir.Initializer/Program.cs diff --git a/Mimir.Initializer/Mimir.Initializer.csproj b/Mimir.Initializer/Mimir.Initializer.csproj new file mode 100644 index 00000000..206b89a9 --- /dev/null +++ b/Mimir.Initializer/Mimir.Initializer.csproj @@ -0,0 +1,10 @@ + + + + Exe + net8.0 + enable + enable + + + diff --git a/Mimir.Initializer/Program.cs b/Mimir.Initializer/Program.cs new file mode 100644 index 00000000..83fa4f4d --- /dev/null +++ b/Mimir.Initializer/Program.cs @@ -0,0 +1,2 @@ +// See https://aka.ms/new-console-template for more information +Console.WriteLine("Hello, World!"); diff --git a/Mimir.sln b/Mimir.sln index 17cec53d..4f0b353f 100644 --- a/Mimir.sln +++ b/Mimir.sln @@ -19,6 +19,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Mimir.MongoDB", "Mimir.Mong EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Mimir.MongoDB.Tests", "Mimir.MongoDB.Tests\Mimir.MongoDB.Tests.csproj", "{F9A5000C-7C42-4A15-B17C-D165D4322E84}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Mimir.Initializer", "Mimir.Initializer\Mimir.Initializer.csproj", "{53A4A7FD-CD63-425E-9FFB-4C023AFDFC59}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -64,5 +66,9 @@ Global {A678F294-FD8E-44D9-BBA6-D9FE1949C053}.Debug|Any CPU.Build.0 = Debug|Any CPU {A678F294-FD8E-44D9-BBA6-D9FE1949C053}.Release|Any CPU.ActiveCfg = Release|Any CPU {A678F294-FD8E-44D9-BBA6-D9FE1949C053}.Release|Any CPU.Build.0 = Release|Any CPU + {53A4A7FD-CD63-425E-9FFB-4C023AFDFC59}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {53A4A7FD-CD63-425E-9FFB-4C023AFDFC59}.Debug|Any CPU.Build.0 = Debug|Any CPU + {53A4A7FD-CD63-425E-9FFB-4C023AFDFC59}.Release|Any CPU.ActiveCfg = Release|Any CPU + {53A4A7FD-CD63-425E-9FFB-4C023AFDFC59}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection EndGlobal From 5371ff528d4089a7d341ed019dad07795005bf00 Mon Sep 17 00:00:00 2001 From: Atralupus Date: Wed, 20 Nov 2024 01:30:55 +0900 Subject: [PATCH 2/3] Implement account address initializer --- Mimir.Initializer/Configuration.cs | 16 ++ .../Initializer/ConverterMappings.cs | 21 +++ .../Initializer/SnapshotInitializer.cs | 145 ++++++++++++++++++ Mimir.Initializer/Mimir.Initializer.csproj | 4 + Mimir.Initializer/Program.cs | 64 +++++++- .../Util/ChainUtil.cs | 2 +- .../Util/MockAction.cs | 2 +- Mimir.Initializer/appsettings.json | 39 +++++ 8 files changed, 289 insertions(+), 4 deletions(-) create mode 100644 Mimir.Initializer/Configuration.cs create mode 100644 Mimir.Initializer/Initializer/ConverterMappings.cs create mode 100644 Mimir.Initializer/Initializer/SnapshotInitializer.cs rename {Mimir.Worker => Mimir.Initializer}/Util/ChainUtil.cs (99%) rename {Mimir.Worker => Mimir.Initializer}/Util/MockAction.cs (95%) create mode 100644 Mimir.Initializer/appsettings.json diff --git a/Mimir.Initializer/Configuration.cs b/Mimir.Initializer/Configuration.cs new file mode 100644 index 00000000..0149ed03 --- /dev/null +++ b/Mimir.Initializer/Configuration.cs @@ -0,0 +1,16 @@ +using Libplanet.Crypto; +using Mimir.Worker.Constants; + +namespace Mimir.Initializer; + +public class Configuration +{ + public string MongoDbConnectionString { get; init; } + public PlanetType PlanetType { get; init; } + public string? MongoDbCAFile { get; init; } + public string ChainStorePath { get; init; } + public string[] TargetAccounts { get; init; } + + public Address[] GetTargetAddresses() => + TargetAccounts.Select(adr => new Address(adr)).ToArray(); +} diff --git a/Mimir.Initializer/Initializer/ConverterMappings.cs b/Mimir.Initializer/Initializer/ConverterMappings.cs new file mode 100644 index 00000000..916fdcdc --- /dev/null +++ b/Mimir.Initializer/Initializer/ConverterMappings.cs @@ -0,0 +1,21 @@ +using Libplanet.Crypto; +using Mimir.Worker.StateDocumentConverter; +using Nekoyume; + +namespace Mimir.Initializer.Initializer; + +public static class ConverterMappings +{ + private static Dictionary pairs = new(); + + static ConverterMappings() + { + pairs.Add(Addresses.Agent, new AgentStateDocumentConverter()); + pairs.Add(Addresses.Avatar, new AgentStateDocumentConverter()); + } + + public static IStateDocumentConverter GetConverter(Address accountAddress) + { + return pairs[accountAddress]; + } +} diff --git a/Mimir.Initializer/Initializer/SnapshotInitializer.cs b/Mimir.Initializer/Initializer/SnapshotInitializer.cs new file mode 100644 index 00000000..0ff3b349 --- /dev/null +++ b/Mimir.Initializer/Initializer/SnapshotInitializer.cs @@ -0,0 +1,145 @@ +using Bencodex.Types; +using Libplanet.Action.State; +using Libplanet.Blockchain; +using Libplanet.Common; +using Libplanet.Crypto; +using Libplanet.Store; +using Libplanet.Store.Trie; +using Mimir.Initializer.Util; +using Mimir.MongoDB; +using Mimir.MongoDB.Bson; +using Mimir.Worker.Services; +using Mimir.Worker.StateDocumentConverter; +using Serilog; +using ILogger = Serilog.ILogger; + +namespace Mimir.Initializer.Initializer; + +public class SnapshotInitializer +{ + private readonly MongoDbService _dbService; + private readonly ILogger _logger; + private readonly string _chainStorePath; + private Address[] _targetAccounts; + + public SnapshotInitializer( + MongoDbService dbService, + string chainStorePath, + Address[] targetAccounts + ) + { + _dbService = dbService; + _chainStorePath = chainStorePath; + _targetAccounts = targetAccounts; + _logger = Log.ForContext(); + } + + public async Task RunAsync(CancellationToken stoppingToken) + { + var started = DateTime.UtcNow; + + (BlockChain blockChain, IStore store, IStateStore stateStore) = ChainUtil.LoadBlockChain( + _chainStorePath + ); + + foreach (var address in _targetAccounts) + { + await ProcessByAccountAddress(blockChain, stateStore, address, stoppingToken); + + if (stoppingToken.IsCancellationRequested) + { + break; + } + } + + store.Dispose(); + stateStore.Dispose(); + + _logger.Information( + "Finished SnapshotInitializer. Elapsed {TotalElapsedMinutes} minutes", + DateTime.UtcNow.Subtract(started).Minutes + ); + } + + private async Task ProcessByAccountAddress( + BlockChain blockChain, + IStateStore stateStore, + Address accountAddress, + CancellationToken stoppingToken + ) + { + int predicateLength = Address.Size * 2; + + ITrie worldTrie = ChainUtil.GetWorldTrie(blockChain); + IWorldState world = new WorldBaseState(worldTrie, stateStore); + IAccountState account = world.GetAccountState(accountAddress); + ITrie accountTrie = account.Trie; + _logger.Information( + "Iterating over trie with state root hash {StateRootHash}", + accountTrie.Hash + ); + + long addressCount = 0; + string? currentAddress = null; + + foreach ((KeyBytes keyBytes, IValue value) in accountTrie.IterateValues()) + { + if (keyBytes.Length == predicateLength) + { + addressCount++; + Address address = ChainUtil.ToAddress(keyBytes); + currentAddress = ByteUtil.Hex(address.ByteArray); + } + + if (currentAddress is string hex) + { + await HandleByAccount( + accountAddress, + new Address(currentAddress), + value, + blockChain.Tip.Index + ); + } + + if (stoppingToken.IsCancellationRequested) + { + break; + } + } + + _logger.Information("Total address count: {AddressCount}", addressCount); + } + + private async Task HandleByAccount( + Address accountAddress, + Address address, + IValue state, + long blockIndex + ) + { + var collectionName = CollectionNames.GetCollectionName(accountAddress); + var documents = new List(); + var document = ConverterMappings + .GetConverter(accountAddress) + .ConvertToDocument( + new AddressStatePair + { + BlockIndex = blockIndex, + Address = address, + RawState = state + } + ); + + documents.Add(document); + + if (documents.Count > 0) + await _dbService.UpsertStateDataManyAsync(collectionName, documents, null, default); + + _logger.Information( + "{DocumentCount} Handled, {CollectionName} - {Address}", + documents.Count, + collectionName, + address + ); + } +} diff --git a/Mimir.Initializer/Mimir.Initializer.csproj b/Mimir.Initializer/Mimir.Initializer.csproj index 206b89a9..153a11cd 100644 --- a/Mimir.Initializer/Mimir.Initializer.csproj +++ b/Mimir.Initializer/Mimir.Initializer.csproj @@ -7,4 +7,8 @@ enable + + + + diff --git a/Mimir.Initializer/Program.cs b/Mimir.Initializer/Program.cs index 83fa4f4d..24f1f166 100644 --- a/Mimir.Initializer/Program.cs +++ b/Mimir.Initializer/Program.cs @@ -1,2 +1,62 @@ -// See https://aka.ms/new-console-template for more information -Console.WriteLine("Hello, World!"); +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Options; +using Mimir.Initializer; +using Mimir.Initializer.Initializer; +using Mimir.Worker.Services; +using Serilog; + +var builder = Host.CreateDefaultBuilder(args) + .ConfigureAppConfiguration( + (hostingContext, config) => + { + string configPath = + Environment.GetEnvironmentVariable("INITIALIZER_CONFIG_FILE") ?? "appsettings.json"; + config + .AddJsonFile(configPath, optional: true, reloadOnChange: true) + .AddEnvironmentVariables("INITIALIZER_"); + } + ) + .ConfigureServices( + (hostContext, services) => + { + services.Configure( + hostContext.Configuration.GetSection("Configuration") + ); + + services.AddSingleton(); + + services.AddSingleton(serviceProvider => + { + var config = serviceProvider.GetRequiredService>().Value; + return new MongoDbService( + config.MongoDbConnectionString, + config.PlanetType, + config.MongoDbCAFile + ); + }); + + services.AddTransient(serviceProvider => + { + var config = serviceProvider.GetRequiredService>().Value; + var dbService = serviceProvider.GetRequiredService(); + var targetAccounts = config.GetTargetAddresses(); + return new SnapshotInitializer(dbService, config.ChainStorePath, targetAccounts); + }); + } + ) + .UseSerilog( + (context, configuration) => + { + configuration.ReadFrom.Configuration(context.Configuration); + } + ) + .Build(); + +using var scope = builder.Services.CreateScope(); +var initializer = scope.ServiceProvider.GetRequiredService(); + +var stoppingToken = new CancellationTokenSource().Token; + +await initializer.RunAsync(stoppingToken); diff --git a/Mimir.Worker/Util/ChainUtil.cs b/Mimir.Initializer/Util/ChainUtil.cs similarity index 99% rename from Mimir.Worker/Util/ChainUtil.cs rename to Mimir.Initializer/Util/ChainUtil.cs index e8a4d61a..a7d4d868 100644 --- a/Mimir.Worker/Util/ChainUtil.cs +++ b/Mimir.Initializer/Util/ChainUtil.cs @@ -11,7 +11,7 @@ using Libplanet.Types.Blocks; using Serilog; -namespace Mimir.Worker.Util; +namespace Mimir.Initializer.Util; // Copy From Census public static class ChainUtil diff --git a/Mimir.Worker/Util/MockAction.cs b/Mimir.Initializer/Util/MockAction.cs similarity index 95% rename from Mimir.Worker/Util/MockAction.cs rename to Mimir.Initializer/Util/MockAction.cs index 53967dfa..8126c117 100644 --- a/Mimir.Worker/Util/MockAction.cs +++ b/Mimir.Initializer/Util/MockAction.cs @@ -2,7 +2,7 @@ using Libplanet.Action; using Libplanet.Action.State; -namespace Mimir.Worker.Util; +namespace Mimir.Initializer.Util; /// /// A mock . diff --git a/Mimir.Initializer/appsettings.json b/Mimir.Initializer/appsettings.json new file mode 100644 index 00000000..bee0016b --- /dev/null +++ b/Mimir.Initializer/appsettings.json @@ -0,0 +1,39 @@ +{ + "Serilog": { + "Using": [], + "MinimumLevel": { + "Default": "Information", + "Override": { + "Microsoft": "Warning", + "System": "Warning" + } + }, + "WriteTo": [ + { + "Name": "Console", + "Args": { + "formatter": "Serilog.Formatting.Compact.CompactJsonFormatter, Serilog.Formatting.Compact", + "outputTemplate": "[{Timestamp:HH:mm:ss} {Level:u3}] [{SourceContext}] [{AccountAddress}] {Message:lj}{NewLine}{Exception}" + } + } + ], + "Enrich": [ + "FromLogContext", + "WithMachineName", + "WithThreadId" + ], + "Properties": { + "Application": "Mimir.Worker" + } + }, + "Configuration": { + "MongoDbConnectionString": "mongodb://rootuser:rootpass@localhost:27017", + "PlanetType": "heimdall", + "ChainStorePath": "your path", + "TargetAccounts": [ + "000000000000000000000000000000000000001a", + "000000000000000000000000000000000000001b" + ], + "EnableInitializing": true + } +} \ No newline at end of file From d0b7395ced22992e30bc1614d90af3faab9e6449 Mon Sep 17 00:00:00 2001 From: Atralupus Date: Wed, 20 Nov 2024 01:39:28 +0900 Subject: [PATCH 3/3] vscode debugger --- .vscode/launch.json | 13 +++++++ .vscode/tasks.json | 36 +++++++++++++++++++ .../Initializer/ConverterMappings.cs | 2 +- Mimir.Initializer/appsettings.json | 2 +- 4 files changed, 51 insertions(+), 2 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index bfffa20d..a83ce4ba 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -26,6 +26,19 @@ "env": { "WORKER_CONFIG_FILE": "appsettings.local.json" } + }, + { + "name": "Debug Mimir.Initializer", + "type": "coreclr", + "request": "launch", + "program": "${workspaceFolder}/Mimir.Initializer/bin/Debug/net8.0/Mimir.Initializer.dll", + "cwd": "${workspaceFolder}/Mimir.Initializer", + "stopAtEntry": false, + "console": "internalConsole", + "preLaunchTask": "build-initializer", + "env": { + "INITIALIZER_CONFIG_FILE": "appsettings.local.json" + } } ] } \ No newline at end of file diff --git a/.vscode/tasks.json b/.vscode/tasks.json index 1cf952d8..13f8fefc 100644 --- a/.vscode/tasks.json +++ b/.vscode/tasks.json @@ -72,6 +72,42 @@ "${workspaceFolder}/Mimir.Worker/Mimir.Worker.csproj" ], "problemMatcher": "$msCompile" + }, + { + "label": "build-initializer", + "command": "dotnet", + "type": "process", + "args": [ + "build", + "${workspaceFolder}/Mimir.Initializer/Mimir.Initializer.csproj", + "/property:GenerateFullPaths=true", + "/consoleloggerparameters:NoSummary" + ], + "problemMatcher": "$msCompile" + }, + { + "label": "publish-initializer", + "command": "dotnet", + "type": "process", + "args": [ + "publish", + "${workspaceFolder}/Mimir.Initializer/Mimir.Initializer.csproj", + "/property:GenerateFullPaths=true", + "/consoleloggerparameters:NoSummary" + ], + "problemMatcher": "$msCompile" + }, + { + "label": "watch-initializer", + "command": "dotnet", + "type": "process", + "args": [ + "watch", + "run", + "--project", + "${workspaceFolder}/Mimir.Initializer/Mimir.Initializer.csproj" + ], + "problemMatcher": "$msCompile" } ] } \ No newline at end of file diff --git a/Mimir.Initializer/Initializer/ConverterMappings.cs b/Mimir.Initializer/Initializer/ConverterMappings.cs index 916fdcdc..9e7772e9 100644 --- a/Mimir.Initializer/Initializer/ConverterMappings.cs +++ b/Mimir.Initializer/Initializer/ConverterMappings.cs @@ -11,7 +11,7 @@ public static class ConverterMappings static ConverterMappings() { pairs.Add(Addresses.Agent, new AgentStateDocumentConverter()); - pairs.Add(Addresses.Avatar, new AgentStateDocumentConverter()); + pairs.Add(Addresses.Avatar, new AvatarStateDocumentConverter()); } public static IStateDocumentConverter GetConverter(Address accountAddress) diff --git a/Mimir.Initializer/appsettings.json b/Mimir.Initializer/appsettings.json index bee0016b..f7586cd9 100644 --- a/Mimir.Initializer/appsettings.json +++ b/Mimir.Initializer/appsettings.json @@ -23,7 +23,7 @@ "WithThreadId" ], "Properties": { - "Application": "Mimir.Worker" + "Application": "Mimir.Initializer" } }, "Configuration": {