Merge pull request #491 from Atralupus/feat/initializer

Introduce Initializer

.vscode/launch.json

@@ -26,6 +26,19 @@
       "env": {
         "WORKER_CONFIG_FILE": "appsettings.local.json"
       }
+    },
+    {
+      "name": "Debug Mimir.Initializer",
+      "type": "coreclr",
+      "request": "launch",
+      "program": "${workspaceFolder}/Mimir.Initializer/bin/Debug/net8.0/Mimir.Initializer.dll",
+      "cwd": "${workspaceFolder}/Mimir.Initializer",
+      "stopAtEntry": false,
+      "console": "internalConsole",
+      "preLaunchTask": "build-initializer",
+      "env": {
+        "INITIALIZER_CONFIG_FILE": "appsettings.local.json"
+      }
     }
   ]
 }

.vscode/tasks.json

@@ -72,6 +72,42 @@
                 "${workspaceFolder}/Mimir.Worker/Mimir.Worker.csproj"
             ],
             "problemMatcher": "$msCompile"
+        },
+        {
+            "label": "build-initializer",
+            "command": "dotnet",
+            "type": "process",
+            "args": [
+                "build",
+                "${workspaceFolder}/Mimir.Initializer/Mimir.Initializer.csproj",
+                "/property:GenerateFullPaths=true",
+                "/consoleloggerparameters:NoSummary"
+            ],
+            "problemMatcher": "$msCompile"
+        },
+        {
+            "label": "publish-initializer",
+            "command": "dotnet",
+            "type": "process",
+            "args": [
+                "publish",
+                "${workspaceFolder}/Mimir.Initializer/Mimir.Initializer.csproj",
+                "/property:GenerateFullPaths=true",
+                "/consoleloggerparameters:NoSummary"
+            ],
+            "problemMatcher": "$msCompile"
+        },
+        {
+            "label": "watch-initializer",
+            "command": "dotnet",
+            "type": "process",
+            "args": [
+                "watch",
+                "run",
+                "--project",
+                "${workspaceFolder}/Mimir.Initializer/Mimir.Initializer.csproj"
+            ],
+            "problemMatcher": "$msCompile"
         }
     ]
 }

Mimir.Initializer/Configuration.cs

@@ -0,0 +1,16 @@
+using Libplanet.Crypto;
+using Mimir.Worker.Constants;
+
+namespace Mimir.Initializer;
+
+public class Configuration
+{
+    public string MongoDbConnectionString { get; init; }
+    public PlanetType PlanetType { get; init; }
+    public string? MongoDbCAFile { get; init; }
+    public string ChainStorePath { get; init; }
+    public string[] TargetAccounts { get; init; }
+
+    public Address[] GetTargetAddresses() =>
+        TargetAccounts.Select(adr => new Address(adr)).ToArray();
+}

Mimir.Initializer/Initializer/ConverterMappings.cs

@@ -0,0 +1,21 @@
+using Libplanet.Crypto;
+using Mimir.Worker.StateDocumentConverter;
+using Nekoyume;
+
+namespace Mimir.Initializer.Initializer;
+
+public static class ConverterMappings
+{
+    private static Dictionary<Address, IStateDocumentConverter> pairs = new();
+
+    static ConverterMappings()
+    {
+        pairs.Add(Addresses.Agent, new AgentStateDocumentConverter());
+        pairs.Add(Addresses.Avatar, new AvatarStateDocumentConverter());
+    }
+
+    public static IStateDocumentConverter GetConverter(Address accountAddress)
+    {
+        return pairs[accountAddress];
+    }
+}

Mimir.Initializer/Initializer/SnapshotInitializer.cs

@@ -0,0 +1,145 @@
+using Bencodex.Types;
+using Libplanet.Action.State;
+using Libplanet.Blockchain;
+using Libplanet.Common;
+using Libplanet.Crypto;
+using Libplanet.Store;
+using Libplanet.Store.Trie;
+using Mimir.Initializer.Util;
+using Mimir.MongoDB;
+using Mimir.MongoDB.Bson;
+using Mimir.Worker.Services;
+using Mimir.Worker.StateDocumentConverter;
+using Serilog;
+using ILogger = Serilog.ILogger;
+
+namespace Mimir.Initializer.Initializer;
+
+public class SnapshotInitializer
+{
+    private readonly MongoDbService _dbService;
+    private readonly ILogger _logger;
+    private readonly string _chainStorePath;
+    private Address[] _targetAccounts;
+
+    public SnapshotInitializer(
+        MongoDbService dbService,
+        string chainStorePath,
+        Address[] targetAccounts
+    )
+    {
+        _dbService = dbService;
+        _chainStorePath = chainStorePath;
+        _targetAccounts = targetAccounts;
+        _logger = Log.ForContext<SnapshotInitializer>();
+    }
+
+    public async Task RunAsync(CancellationToken stoppingToken)
+    {
+        var started = DateTime.UtcNow;
+
+        (BlockChain blockChain, IStore store, IStateStore stateStore) = ChainUtil.LoadBlockChain(
+            _chainStorePath
+        );
+
+        foreach (var address in _targetAccounts)
+        {
+            await ProcessByAccountAddress(blockChain, stateStore, address, stoppingToken);
+
+            if (stoppingToken.IsCancellationRequested)
+            {
+                break;
+            }
+        }
+
+        store.Dispose();
+        stateStore.Dispose();
+
+        _logger.Information(
+            "Finished SnapshotInitializer. Elapsed {TotalElapsedMinutes} minutes",
+            DateTime.UtcNow.Subtract(started).Minutes
+        );
+    }
+
+    private async Task ProcessByAccountAddress(
+        BlockChain blockChain,
+        IStateStore stateStore,
+        Address accountAddress,
+        CancellationToken stoppingToken
+    )
+    {
+        int predicateLength = Address.Size * 2;
+
+        ITrie worldTrie = ChainUtil.GetWorldTrie(blockChain);
+        IWorldState world = new WorldBaseState(worldTrie, stateStore);
+        IAccountState account = world.GetAccountState(accountAddress);
+        ITrie accountTrie = account.Trie;
+        _logger.Information(
+            "Iterating over trie with state root hash {StateRootHash}",
+            accountTrie.Hash
+        );
+
+        long addressCount = 0;
+        string? currentAddress = null;
+
+        foreach ((KeyBytes keyBytes, IValue value) in accountTrie.IterateValues())
+        {
+            if (keyBytes.Length == predicateLength)
+            {
+                addressCount++;
+                Address address = ChainUtil.ToAddress(keyBytes);
+                currentAddress = ByteUtil.Hex(address.ByteArray);
+            }
+
+            if (currentAddress is string hex)
+            {
+                await HandleByAccount(
+                    accountAddress,
+                    new Address(currentAddress),
+                    value,
+                    blockChain.Tip.Index
+                );
+            }
+
+            if (stoppingToken.IsCancellationRequested)
+            {
+                break;
+            }
+        }
+
+        _logger.Information("Total address count: {AddressCount}", addressCount);
+    }
+
+    private async Task HandleByAccount(
+        Address accountAddress,
+        Address address,
+        IValue state,
+        long blockIndex
+    )
+    {
+        var collectionName = CollectionNames.GetCollectionName(accountAddress);
+        var documents = new List<MimirBsonDocument>();
+        var document = ConverterMappings
+            .GetConverter(accountAddress)
+            .ConvertToDocument(
+                new AddressStatePair
+                {
+                    BlockIndex = blockIndex,
+                    Address = address,
+                    RawState = state
+                }
+            );
+
+        documents.Add(document);
+
+        if (documents.Count > 0)
+            await _dbService.UpsertStateDataManyAsync(collectionName, documents, null, default);
+
+        _logger.Information(
+            "{DocumentCount} Handled, {CollectionName} - {Address}",
+            documents.Count,
+            collectionName,
+            address
+        );
+    }
+}

Mimir.Initializer/Mimir.Initializer.csproj

@@ -0,0 +1,14 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net8.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\Mimir.Worker\Mimir.Worker.csproj" />
+  </ItemGroup>
+
+</Project>

Mimir.Initializer/Program.cs

@@ -0,0 +1,62 @@
+using Microsoft.Extensions.Configuration;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Hosting;
+using Microsoft.Extensions.Options;
+using Mimir.Initializer;
+using Mimir.Initializer.Initializer;
+using Mimir.Worker.Services;
+using Serilog;
+
+var builder = Host.CreateDefaultBuilder(args)
+    .ConfigureAppConfiguration(
+        (hostingContext, config) =>
+        {
+            string configPath =
+                Environment.GetEnvironmentVariable("INITIALIZER_CONFIG_FILE") ?? "appsettings.json";
+            config
+                .AddJsonFile(configPath, optional: true, reloadOnChange: true)
+                .AddEnvironmentVariables("INITIALIZER_");
+        }
+    )
+    .ConfigureServices(
+        (hostContext, services) =>
+        {
+            services.Configure<Configuration>(
+                hostContext.Configuration.GetSection("Configuration")
+            );
+
+            services.AddSingleton<MongoDbService>();
+
+            services.AddSingleton(serviceProvider =>
+            {
+                var config = serviceProvider.GetRequiredService<IOptions<Configuration>>().Value;
+                return new MongoDbService(
+                    config.MongoDbConnectionString,
+                    config.PlanetType,
+                    config.MongoDbCAFile
+                );
+            });
+
+            services.AddTransient<SnapshotInitializer>(serviceProvider =>
+            {
+                var config = serviceProvider.GetRequiredService<IOptions<Configuration>>().Value;
+                var dbService = serviceProvider.GetRequiredService<MongoDbService>();
+                var targetAccounts = config.GetTargetAddresses();
+                return new SnapshotInitializer(dbService, config.ChainStorePath, targetAccounts);
+            });
+        }
+    )
+    .UseSerilog(
+        (context, configuration) =>
+        {
+            configuration.ReadFrom.Configuration(context.Configuration);
+        }
+    )
+    .Build();
+
+using var scope = builder.Services.CreateScope();
+var initializer = scope.ServiceProvider.GetRequiredService<SnapshotInitializer>();
+
+var stoppingToken = new CancellationTokenSource().Token;
+
+await initializer.RunAsync(stoppingToken);

Mimir.Worker/Util/ChainUtil.cs → Mimir.Initializer/Util/ChainUtil.cs

@@ -11,7 +11,7 @@
 using Libplanet.Types.Blocks;
 using Serilog;
 
-namespace Mimir.Worker.Util;
+namespace Mimir.Initializer.Util;
 
 // Copy From Census
 public static class ChainUtil

Mimir.Worker/Util/MockAction.cs → Mimir.Initializer/Util/MockAction.cs

@@ -2,7 +2,7 @@
 using Libplanet.Action;
 using Libplanet.Action.State;
 
-namespace Mimir.Worker.Util;
+namespace Mimir.Initializer.Util;
 
 /// <summary>
 /// A mock <see cref="IAction"/>.

Mimir.Initializer/appsettings.json

@@ -0,0 +1,39 @@
+{
+    "Serilog": {
+        "Using": [],
+        "MinimumLevel": {
+            "Default": "Information",
+            "Override": {
+                "Microsoft": "Warning",
+                "System": "Warning"
+            }
+        },
+        "WriteTo": [
+            {
+                "Name": "Console",
+                "Args": {
+                    "formatter": "Serilog.Formatting.Compact.CompactJsonFormatter, Serilog.Formatting.Compact",
+                    "outputTemplate": "[{Timestamp:HH:mm:ss} {Level:u3}] [{SourceContext}] [{AccountAddress}] {Message:lj}{NewLine}{Exception}"
+                }
+            }
+        ],
+        "Enrich": [
+            "FromLogContext",
+            "WithMachineName",
+            "WithThreadId"
+        ],
+        "Properties": {
+            "Application": "Mimir.Initializer"
+        }
+    },
+    "Configuration": {
+        "MongoDbConnectionString": "mongodb://rootuser:rootpass@localhost:27017",
+        "PlanetType": "heimdall",
+        "ChainStorePath": "your path",
+        "TargetAccounts": [
+            "000000000000000000000000000000000000001a",
+            "000000000000000000000000000000000000001b"
+        ],
+        "EnableInitializing": true
+    }
+}