Refactor and improve DB size generator (#7450)

NethermindEth · Sep 20, 2024 · 7795c96 · 7795c96
1 parent 1bd8303
commit 7795c96
Show file tree

Hide file tree

Showing 9 changed files with 153 additions and 74 deletions.
diff --git a/.github/workflows/update-docs.yml b/.github/workflows/update-docs.yml
@@ -15,18 +15,20 @@ jobs:
     name: Update docs
     runs-on: ubuntu-latest
     steps:
-      - name: Check out Nethermind repository
-        uses: actions/checkout@v4
-        with:
-          ref: ${{ github.event.inputs.ref || github.ref }}
-          path: n
-      - name: Authenticate App
+      - name: Create GitHub app token
         id: gh-app
         uses: actions/create-github-app-token@v1
         with:
           app-id: ${{ vars.APP_ID }}
           private-key: ${{ secrets.APP_PRIVATE_KEY }}
-          repositories: "nethermind,docs"
+          repositories: |
+            nethermind
+            docs
+      - name: Check out Nethermind repository
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.inputs.ref || github.ref }}
+          path: n
       - name: Check out Nethermind docs repository
         uses: actions/checkout@v4
         with:
@@ -41,20 +43,7 @@ jobs:
         working-directory: n
         run: dotnet build tools/docgen/DocGen.csproj -c release -o docgen
       - name: Generate docs
-        run: |
-          mv d/docs/fundamentals/configuration.md n/docgen/configuration.md
-          mv d/docs/interacting/json-rpc-ns/eth_subscribe.md n/docgen/eth_subscribe.md
-          mv d/docs/interacting/json-rpc-ns/eth_unsubscribe.md n/docgen/eth_unsubscribe.md
-          mv d/docs/monitoring/metrics/metrics.md n/docgen/metrics.md
-          cd n/docgen
-          ./DocGen --config --jsonrpc --metrics
-          cd ../..
-          mv n/docgen/configuration.md d/docs/fundamentals/configuration.md
-          rm -f d/docs/interacting/json-rpc-ns/*.md
-          mv n/docgen/eth_subscribe.md d/docs/interacting/json-rpc-ns/eth_subscribe.md
-          mv n/docgen/eth_unsubscribe.md d/docs/interacting/json-rpc-ns/eth_unsubscribe.md
-          mv n/docgen/json-rpc-ns/*.md d/docs/interacting/json-rpc-ns
-          mv n/docgen/metrics.md d/docs/monitoring/metrics/metrics.md
+        run: n/docgen/DocGen $GITHUB_WORKSPACE/d --config --jsonrpc --metrics
       - name: Tag a new version
         if: github.event_name == 'release' && !github.event.release.prerelease
         working-directory: d

diff --git a/tools/docgen/ConfigGenerator.cs b/tools/docgen/ConfigGenerator.cs
@@ -4,29 +4,33 @@
 using System.ComponentModel;
 using System.Reflection;
 using Nethermind.Config;
+using Spectre.Console;
 
 namespace Nethermind.DocGen;
 
 internal static class ConfigGenerator
 {
-    internal static void Generate()
+    internal static void Generate(string path)
     {
+        path = Path.Join(path, "docs", "fundamentals");
+
         var startMark = "<!--[start autogen]-->";
         var endMark = "<!--[end autogen]-->";
-        var fileName = "configuration.md";
         var excluded = Enumerable.Empty<string>();
-
         var types = Directory
             .GetFiles(AppDomain.CurrentDomain.BaseDirectory, "Nethermind.*.dll")
             .SelectMany(a => Assembly.LoadFrom(a).GetExportedTypes())
             .Where(t => t.IsInterface && typeof(IConfig).IsAssignableFrom(t) &&
                 !excluded.Any(x => t.FullName?.Contains(x, StringComparison.Ordinal) ?? false))
             .OrderBy(t => t.Name);
+        var fileName = Path.Join(path, "configuration.md");
+        var tempFileName = Path.Join(path, "~configuration.md");
 
-        File.Delete($"~{fileName}");
+        // Delete the temp file if it exists
+        File.Delete(tempFileName);
 
         using var readStream = new StreamReader(File.OpenRead(fileName));
-        using var writeStream = new StreamWriter(File.OpenWrite($"~{fileName}"));
+        using var writeStream = new StreamWriter(File.OpenWrite(tempFileName));
 
         writeStream.NewLine = "\n";
 
@@ -63,9 +67,10 @@ internal static void Generate()
         readStream.Close();
         writeStream.Close();
 
-        File.Move($"~{fileName}", fileName, true);
+        File.Move(tempFileName, fileName, true);
+        File.Delete(tempFileName);
 
-        Console.WriteLine($"Updated {fileName}");
+        AnsiConsole.MarkupLine($"[green]Updated[/] {fileName}");
     }
 
     private static void WriteMarkdown(StreamWriter file, Type configType)

diff --git a/tools/docgen/DBSizeGenerator.cs b/tools/docgen/DBSizeGenerator.cs
@@ -1,13 +1,16 @@
 // SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited
 // SPDX-License-Identifier: LGPL-3.0-only
 
+using Spectre.Console;
 using System.Text.Json;
 
 namespace Nethermind.DocGen;
 
 internal static class DBSizeGenerator
 {
     private const string _chainSizesDir = "chainSizes";
+    private const string _startMark = "<!--[start autogen]-->";
+    private const string _endMark = "<!--[end autogen]-->";
 
     private static readonly List<string> _dbList =
     [
@@ -20,7 +23,7 @@ internal static class DBSizeGenerator
         "blobTransactions"
     ];
 
-    internal static void Generate()
+    internal static void Generate(string path)
     {
         IList<string> chainOrder =
         [
@@ -32,11 +35,8 @@ internal static void Generate()
             "energyweb",
             "volta"
         ];
-        var startMark = "<!--[start autogen]-->";
-        var endMark = "<!--[end autogen]-->";
-        var fileName = "database.md";
 
-        var chainSizesPath = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, _chainSizesDir);
+        var chainSizesPath = Path.Join(AppDomain.CurrentDomain.BaseDirectory, _chainSizesDir);
         var chains = Directory
             .GetFiles(chainSizesPath)
             .Select(Path.GetFileNameWithoutExtension)
@@ -48,10 +48,20 @@ internal static void Generate()
             })
             .ToList();
 
-        File.Delete($"~{fileName}");
+        GenerateFile(Path.Join(path, "docs", "fundamentals"), chains!);
+        GenerateFile(Path.Join(path, "versioned_docs", $"version-{GetLatestVersion(path)}", "fundamentals"), chains!);
+    }
+
+    private static void GenerateFile(string path, IList<string> chains)
+    {
+        var fileName = Path.Join(path, "database.md");
+        var tempFileName = Path.Join(path, "~database.md");
+
+        // Delete the temp file if it exists
+        File.Delete(tempFileName);
 
         using var readStream = new StreamReader(File.OpenRead(fileName));
-        using var writeStream = new StreamWriter(File.OpenWrite($"~{fileName}"));
+        using var writeStream = new StreamWriter(File.OpenWrite(tempFileName));
 
         writeStream.NewLine = "\n";
 
@@ -63,7 +73,7 @@ internal static void Generate()
 
             writeStream.WriteLine(line);
         }
-        while (!line?.Equals(startMark, StringComparison.Ordinal) ?? false);
+        while (!line?.Equals(_startMark, StringComparison.Ordinal) ?? false);
 
         writeStream.WriteLine();
 
@@ -75,7 +85,7 @@ internal static void Generate()
         {
             if (skip)
             {
-                if (line?.Equals(endMark, StringComparison.Ordinal) ?? false)
+                if (line?.Equals(_endMark, StringComparison.Ordinal) ?? false)
                     skip = false;
                 else
                     continue;
@@ -87,12 +97,13 @@ internal static void Generate()
         readStream.Close();
         writeStream.Close();
 
-        File.Move($"~{fileName}", fileName, true);
+        File.Move(tempFileName, fileName, true);
+        File.Delete(tempFileName);
 
-        Console.WriteLine($"Updated {fileName}");
+        AnsiConsole.MarkupLine($"[green]Updated[/] {fileName}");
     }
 
-    private static void WriteMarkdown(StreamWriter file, List<string> chains)
+    private static void WriteMarkdown(StreamWriter file, IList<string> chains)
     {
         file.WriteLine("<Tabs>");
 
@@ -146,4 +157,12 @@ private static string FormatSize(string value) => value
         .Replace("G", " GB")
         .Replace("M", " MB")
         .Replace("K", " KB");
+
+    private static string GetLatestVersion(string path)
+    {
+        using var versionsJson = File.OpenRead(Path.Join(path, "versions.json"));
+        var versions = JsonSerializer.Deserialize<string[]>(versionsJson)!;
+
+        return versions[0];
+    }
 }
diff --git a/tools/docgen/DocGen.csproj b/tools/docgen/DocGen.csproj
@@ -7,6 +7,10 @@
     <Nullable>enable</Nullable>
   </PropertyGroup>
 
+  <ItemGroup>
+    <PackageReference Include="Spectre.Console.Cli" Version="0.49.1" />
+  </ItemGroup>
+
   <ItemGroup>
     <ProjectReference Include="..\..\src\Nethermind\Nethermind.Runner\Nethermind.Runner.csproj" />
   </ItemGroup>

diff --git a/tools/docgen/JsonRpcGenerator.cs b/tools/docgen/JsonRpcGenerator.cs
@@ -7,16 +7,18 @@
 using Nethermind.JsonRpc.Modules.Rpc;
 using Nethermind.JsonRpc.Modules.Subscribe;
 using Newtonsoft.Json;
+using Spectre.Console;
 
 namespace Nethermind.DocGen;
 
 internal static class JsonRpcGenerator
 {
-    private const string _directory = "json-rpc-ns";
     private const string _objectTypeName = "*object*";
 
-    internal static void Generate()
+    internal static void Generate(string path)
     {
+        path = Path.Join(path, "docs", "interacting", "json-rpc-ns");
+
         var excluded = new[] {
             typeof(IContextAwareRpcModule).FullName,
             typeof(IEvmRpcModule).FullName,
@@ -30,22 +32,29 @@ internal static void Generate()
                 !excluded.Any(x => x is not null && (t.FullName?.Contains(x, StringComparison.Ordinal) ?? false)))
             .OrderBy(t => t.Name);
 
-        if (Directory.Exists(_directory))
-            Directory.Delete(_directory, true);
-
-        Directory.CreateDirectory(_directory);
+        foreach (var file in Directory.EnumerateFiles(path))
+        {
+            if (file.EndsWith(".md", StringComparison.Ordinal) &&
+                // Skip eth_subscribe.md and eth_unsubscribe.md
+                !file.EndsWith("subscribe.md", StringComparison.Ordinal))
+            {
+                File.Delete(file);
+            }
+        }
 
         var i = 0;
 
         foreach (var type in types)
-            WriteMarkdown(type, i++);
+            WriteMarkdown(path, type, i++);
     }
 
-    private static void WriteMarkdown(Type rpcType, int sidebarIndex)
+    private static void WriteMarkdown(string path, Type rpcType, int sidebarIndex)
     {
         var rpcName = rpcType.Name[1..].Replace("RpcModule", null).ToLowerInvariant();
+        var fileName = Path.Join(path, $"{rpcName}.md");
 
-        using var file = new StreamWriter(File.OpenWrite($"{_directory}/{rpcName}.md"));
+        using var stream = File.Open(fileName, FileMode.Create);
+        using var file = new StreamWriter(stream);
         file.NewLine = "\n";
 
         file.WriteLine($"""
@@ -79,7 +88,7 @@ private static void WriteMarkdown(Type rpcType, int sidebarIndex)
             if (method.Name.Equals("eth_subscribe", StringComparison.Ordinal) ||
                 method.Name.Equals("eth_unsubscribe", StringComparison.Ordinal))
             {
-                WriteFromFile(file, $"{method.Name}.md");
+                WriteFromFile(file, Path.Join(path, $"{method.Name}.md"));
 
                 continue;
             }
@@ -111,7 +120,7 @@ private static void WriteMarkdown(Type rpcType, int sidebarIndex)
 
         file.Close();
 
-        Console.WriteLine($"Generated {_directory}/{rpcName}.md");
+        AnsiConsole.MarkupLine($"[green]Generated[/] {fileName}");
     }
 
     private static void WriteParameters(StreamWriter file, MethodInfo method)
@@ -254,9 +263,11 @@ private static void WriteFromFile(StreamWriter file, string fileName)
     {
         file.Flush();
 
+        using var sourceFile = File.OpenRead(fileName);
+
         try
         {
-            File.OpenRead(fileName).CopyTo(file.BaseStream);
+            sourceFile.CopyTo(file.BaseStream);
         }
         catch (Exception)
         {

diff --git a/tools/docgen/MetricsGenerator.cs b/tools/docgen/MetricsGenerator.cs
@@ -4,31 +4,35 @@
 using System.ComponentModel;
 using System.Reflection;
 using System.Text.RegularExpressions;
+using Spectre.Console;
 
 namespace Nethermind.DocGen;
 
 internal static partial class MetricsGenerator
 {
     private static readonly Regex _regex = TransformRegex();
 
-    internal static void Generate()
+    internal static void Generate(string path)
     {
+        path = Path.Join(path, "docs", "monitoring", "metrics");
+
         var startMark = "<!--[start autogen]-->";
         var endMark = "<!--[end autogen]-->";
-        var fileName = "metrics.md";
         var excluded = Array.Empty<string>();
-
         var types = Directory
             .GetFiles(AppDomain.CurrentDomain.BaseDirectory, "Nethermind.*.dll")
             .SelectMany(a => Assembly.LoadFile(a).GetExportedTypes())
             .Where(t => t.Name.Equals("Metrics", StringComparison.Ordinal) &&
                 !excluded.Any(x => t.FullName?.Contains(x, StringComparison.Ordinal) ?? false))
             .OrderBy(t => GetNamespace(t.FullName));
+        var fileName = Path.Join(path, "metrics.md");
+        var tempFileName = Path.Join(path, "~metrics.md");
 
-        File.Delete($"~{fileName}");
+        // Delete the temp file if it exists
+        File.Delete(tempFileName);
 
         using var readStream = new StreamReader(File.OpenRead(fileName));
-        using var writeStream = new StreamWriter(File.OpenWrite($"~{fileName}"));
+        using var writeStream = new StreamWriter(File.OpenWrite(tempFileName));
 
         writeStream.NewLine = "\n";
 
@@ -65,9 +69,10 @@ internal static void Generate()
         readStream.Close();
         writeStream.Close();
 
-        File.Move($"~{fileName}", fileName, true);
+        File.Move(tempFileName, fileName, true);
+        File.Delete(tempFileName);
 
-        Console.WriteLine($"Updated {fileName}");
+        AnsiConsole.MarkupLine($"[green]Updated[/] {fileName}");
     }
 
     private static void WriteMarkdown(StreamWriter file, Type metricsType)