feat: Adding debug mode to local node. (#465)

Signed-off-by: Stefan Stefanov <stefan.stefanooov@gmail.com>

.env

@@ -16,7 +16,7 @@ PLATFORM_JAVA_OPTS="-XX:+UnlockExperimentalVMOptions -XX:+UseZGC -Xlog:gc*:gc.lo
 NETWORK_NODE_LOGS_ROOT_PATH=./network-logs/node
 APPLICATION_ROOT_PATH=./compose-network/network-node
 APPLICATION_CONFIG_PATH=./compose-network/network-node/data/config
-RECORD_PARSER_ROOT_PATH=./src/record-parser
+RECORD_PARSER_ROOT_PATH=./src/services/record-parser
 
 #### Network Node Memory Limits ####
 NETWORK_NODE_MEM_LIMIT=8gb
@@ -78,6 +78,7 @@ RELAY_RATE_LIMIT_DISABLED=true
 
 #### Record Stream Uploader ####
 STREAM_EXTENSION=rcd.gz
+STREAM_SIG_EXTENSION=rcd_sig
 
 #### ENVOY ####
 ENVOY_IMAGE_PREFIX=envoyproxy/

README.md

@@ -109,13 +109,17 @@ Available commands:
             --balance to set starting hbar balance of the created accounts.
             --async to enable or disable asynchronous creation of accounts.
             --b or --blocklist to enable or disable account blocklisting. Depending on how many private keys are blocklisted, this will affect the generated on startup accounts.
+            --enable-debug  Enable or disable debugging of the local node [boolean] [default: false]
     stop - Stops the local hedera network and delete all the existing data.
     restart - Restart the local hedera network.
     generate-accounts <n> - Generates N accounts, default 10.
         options:
             --h or --host to override the default host.
             --balance to set starting hbar balance of the created accounts.
             --async to enable or disable asynchronous creation of accounts.
+    debug [timestamp] - Parses and prints the contents of the record file that has been created
+                        during the selected timestamp.
+                        Important: Local node must be started with the -g, --enable-debug flag to enable this feature
 ```
 
 Note: Generated accounts are 3 types (ECDSA, Alias ECDSA and ED25519). All of them are usable via HederaSDK. Only Alias ECDSA accounts can be imported into wallet like Metamask or used in ethers.

src/Errors/LocalNodeErrors.ts

@@ -18,7 +18,7 @@
  *
  */
 
-export class LocalNodeErrors extends Error{
+export class LocalNodeErrors extends Error {
     public message: string;
     public name: string;
 
@@ -39,5 +39,8 @@ export class LocalNodeErrors extends Error{
 
 export const Errors = {
     CONNECTION_ERROR: (port?: number) => new LocalNodeErrors("Connection Error", `Something went wrong, while trying to connect ${port ? `to port ${port}` : `to local node`}`),
-    CLEINT_ERROR: (msg?: string) => new LocalNodeErrors("Client Error", `Something went wrong, while trying to create SDK Client${msg ? `: ${msg}` : ``}`)
+    CLEINT_ERROR: (msg?: string) => new LocalNodeErrors("Client Error", `Something went wrong, while trying to create SDK Client${msg ? `: ${msg}` : ``}`),
+    NO_RECORD_FILE_FOUND_ERROR: () => new LocalNodeErrors('No record file found Error', "This record file doesn't not exist, check if timestamp is correct and local-node was started in debug mode using --enable-debug option"),
+    INVALID_TIMESTAMP_ERROR: () => new LocalNodeErrors('Invalid Timestamp Error', 'Invalid timestamp string. Accepted formats are: 0000000000.000000000 and 0000000000-000000000'),
+    DEBUG_MODE_CHECK_ERROR: () => new LocalNodeErrors('Debug Mode check Error', 'Debug mode is not enabled to use this command. Please use the --enable-debug flag to enable it.'),
 }

src/configuration/originalNodeConfiguration.json

@@ -33,5 +33,8 @@
             "accountId": "0.0.6",
             "host": "network-node-3"
         }
-    ]
+    ],
+    "local": {
+        "deleteAfterProcessing": false
+    }
 }

src/constants.ts

@@ -41,6 +41,9 @@ export const MIRROR_NODE_LABEL = "mirror-node-rest";
 export const RELAY_LABEL = "json-rpc-relay";
 export const IS_WINDOWS = process.platform === "win32";
 export const UNKNOWN_VERSION = "Unknown";
-export const EVM_ADDRESSES_BLOCKLIST_FILE_RELATIVE_PATH = '../../compose-network/network-node';
 export const NECESSARY_PORTS = [5551, 8545, 5600, 5433, 50211, 8082];
 export const OPTIONAL_PORTS = [7546, 8080, 6379, 3000];
+export const EVM_ADDRESSES_BLOCKLIST_FILE_RELATIVE_PATH = '../../compose-network/network-node'
+export const RELATIVE_TMP_DIR_PATH = '../../src/services/record-parser/temp';
+export const RELATIVE_RECORDS_DIR_PATH = '../../network-logs/node/recordStreams/record0.0.3';
+export const APPLICATION_YML_RELATIVE_PATH = '../../compose-network/mirror-node/application.yml';

src/data/StateData.ts

@@ -26,6 +26,7 @@ import { NetworkPrepState } from '../state/NetworkPrepState';
 import { StartState } from '../state/StartState';
 import { StopState } from '../state/StopState';
 import { StateConfiguration } from '../types/StateConfiguration';
+import { DebugState } from '../state/DebugState';
 
 export class StateData {
 
@@ -39,6 +40,8 @@ export class StateData {
                 return this.getStopConfiguration();
             case 'accountCreation':
                 return this.getAccountCreationConfiguration();
+            case 'debug':
+                return this.getDebugConfiguration();
             default:
                 return undefined;
         }
@@ -92,4 +95,13 @@ export class StateData {
             ]
         }
     }
+
+    private getDebugConfiguration(): StateConfiguration {
+        return {
+            'stateMachineName' : 'debug',
+            'states' : [
+                new DebugState()
+            ]
+        }
+    }
 }

src/services/CLIService.ts

@@ -57,6 +57,7 @@ export class CLIService implements IService{
         CLIService.userComposeOption(yargs);
         CLIService.userComposeDirOption(yargs);
         CLIService.blocklistingOption(yargs);
+        CLIService.enableDebugOption(yargs);
     }
 
     public static loadDebugOptions(yargs: Argv<{}>): void {
@@ -97,6 +98,8 @@ export class CLIService implements IService{
         const blocklisting = argv.blocklist as boolean;
         const startup = argv.startup as boolean;
         const verbose = CLIService.resolveVerboseLevel(argv.verbose as string);
+        const timestamp = argv.timestamp as string;
+        const enableDebug = argv.enableDebug as boolean;
 
         const currentArgv: CLIOptions = {
             accounts,
@@ -113,7 +116,9 @@ export class CLIService implements IService{
             userComposeDir,
             blocklisting,
             startup,
-            verbose
+            verbose,
+            timestamp,
+            enableDebug
         };
 
         return currentArgv;
@@ -282,6 +287,15 @@ export class CLIService implements IService{
             demandOption: false,
             choices: ['info', 'trace'],
             default: 'info',
+        })
+    }
+
+    private static enableDebugOption(yargs: Argv<{}>): void {
+        yargs.option('enable-debug', {
+            type: 'boolean',
+            describe: 'Enable or disable debugging of the local node',
+            demandOption: false,
+            default: false
           });
     }
 

src/services/record-parser/src/Parser.java

@@ -1,4 +1,4 @@
-import static com.hedera.services.utils.forensics.RecordParsers.parseV6RecordStreamEntriesIn;
+import static com.hedera.node.app.service.mono.utils.forensics.RecordParsers.parseV6RecordStreamEntriesIn;
 public class Parser {
     public static void main(String[] args) {
         try {

src/state/CleanUpState.ts

@@ -60,6 +60,7 @@ export class CleanUpState implements IState{
         const application = yaml.load(readFileSync(propertiesFilePath).toString()) as any;
         delete application.hedera.mirror.importer.dataPath;
         delete application.hedera.mirror.importer.downloader.sources;
+        delete application.hedera.mirror.importer.downloader.local
 
         application.hedera.mirror.monitor.nodes = originalNodeConfiguration.fullNodeProperties;
         writeFileSync(propertiesFilePath, yaml.dump(application, { lineWidth: 256 }));

src/state/DebugState.ts

@@ -18,17 +18,26 @@
  *
  */
 
+import { resolve } from 'path';
+import { readdirSync, copyFileSync, unlinkSync } from 'fs';
+import shell from 'shelljs';
 import { IOBserver } from '../controller/IObserver';
 import { LoggerService } from '../services/LoggerService';
 import { ServiceLocator } from '../services/ServiceLocator';
 import { IState } from './IState';
+import { CLIService } from '../services/CLIService';
+import { Errors } from '../Errors/LocalNodeErrors';
+import { RELATIVE_RECORDS_DIR_PATH, RELATIVE_TMP_DIR_PATH } from '../constants';
 
 export class DebugState implements IState{
     private logger: LoggerService;
 
     private observer: IOBserver | undefined;
 
     private stateName: string;
+
+    private static readonly recordExt = `.${process.env.STREAM_EXTENSION}`;
+    private static readonly sigExt = `.${process.env.STREAM_SIG_EXTENSION}`;
 
     constructor() {
         this.stateName = DebugState.name;
@@ -41,6 +50,104 @@ export class DebugState implements IState{
     }
 
     public async onStart(): Promise<void> {
-        throw new Error('Method not implemented.');
+        try {
+            const { timestamp } = ServiceLocator.Current.get<CLIService>(CLIService.name).getCurrentArgv();
+            // DebugState.checkForDebugMode();
+            this.logger.trace('Debug State Starting...', this.stateName);
+            const jsTimestampNum = DebugState.getAndValidateTimestamp(timestamp)
+
+            const tempDir = resolve(__dirname, RELATIVE_TMP_DIR_PATH);
+            const recordFilesDirPath = resolve(__dirname, RELATIVE_RECORDS_DIR_PATH);
+            this.findAndCopyRecordFileToTmpDir(jsTimestampNum, recordFilesDirPath, tempDir)
+            // Perform the parsing
+            await shell.exec(
+                'docker exec network-node bash /opt/hgcapp/recordParser/parse.sh'
+            );
+
+            DebugState.cleanTempDir(tempDir);
+        } catch (error: any) {
+            this.logger.error(error.message);
+            return
+        }
+    }
+
+    private static cleanTempDir(dirPath: string): void {
+        for (const tempFile of readdirSync(dirPath)) {
+            if (tempFile !== '.gitignore') {
+              unlinkSync(resolve(dirPath, tempFile));
+            }
+        }
+    }
+
+    private static getAndValidateTimestamp(timestamp: string): number {
+        const timestampRegEx = /^\d{10}[.-]\d{9}$/;
+        if (!timestampRegEx.test(timestamp)) {          
+          throw Errors.INVALID_TIMESTAMP_ERROR();
+        }
+
+        // Parse the timestamp to a record file filename
+        let jsTimestamp = timestamp
+          .replace('.', '')
+          .replace('-', '')
+          .substring(0, 13);
+        return parseInt(jsTimestamp);
+    }
+
+    private findAndCopyRecordFileToTmpDir(jsTimestampNum: number, recordFilesDirPath: string, tmpDirPath: string): void {
+        // Copy the record file to a temp directory
+          const files = readdirSync(recordFilesDirPath);
+
+          for (let i = 1; i < files.length; i++) {
+            const file = files[i];
+            const recordFileName = file.replace(DebugState.recordExt, '');
+            const fileTimestamp = new Date(recordFileName.replace(/_/g, ':')).getTime();
+            if (fileTimestamp >= jsTimestampNum) {
+              const fileToCopy = [
+                files[i - 2],
+                files[i]
+              ];
+
+              this.copyFilesToTmpDir(fileToCopy, tmpDirPath, recordFilesDirPath)
+              return
+            }
+          }
+
+          throw Errors.NO_RECORD_FILE_FOUND_ERROR();
+    }
+
+    private copyFilesToTmpDir(
+      filesToCopy: string | Array<string>,
+      tmpDirPath: string,
+      recordFilesDirPath: string
+    ): void {
+        if (Array.isArray(filesToCopy)) {
+          for (const file of filesToCopy) {
+            this.copyFileToDir(file, recordFilesDirPath, tmpDirPath)
+          }
+          return
+        }
+
+        this.copyFileToDir(filesToCopy, recordFilesDirPath, tmpDirPath)
+    }
+
+    private copyFileToDir(
+      fileToCopy: string,
+      srcPath: string,
+      destinationPath: string,
+    ): void {
+        if (fileToCopy.endsWith(DebugState.recordExt)) {
+          this.logger.trace(`Parsing record file [${fileToCopy}]\n`);
+        }
+
+        const fileToCopyName = fileToCopy.replace(DebugState.recordExt, '');
+        const sigFile = fileToCopyName + DebugState.sigExt;
+        copyFileSync(
+          resolve(srcPath, fileToCopy),
+          resolve(destinationPath, fileToCopy)
+        );
+        copyFileSync(
+          resolve(srcPath, sigFile),
+          resolve(destinationPath, sigFile)
+        );
     }
 }

src/state/InitState.ts

@@ -130,6 +130,7 @@ export class InitState implements IState{
     private configureMirrorNodeProperties() {
         this.logger.trace('Configuring required mirror node properties, depending on selected configuration...', this.stateName);
         const turboMode = !this.cliOptions.fullMode;
+        const debugMode = this.cliOptions.enableDebug;
 
         // const multiNode = this.cliOptions.multiNode;
 
@@ -141,6 +142,10 @@ export class InitState implements IState{
             application.hedera.mirror.importer.downloader.sources = originalNodeConfiguration.turboNodeProperties.sources;
         }
 
+        if (debugMode) {
+            application.hedera.mirror.importer.downloader.local = originalNodeConfiguration.local
+        }
+
         // if (multiNode) {
         //     application['hedera']['mirror']['monitor']['nodes'] = originalNodeConfiguration.multiNodeProperties
         // }

src/state/StartState.ts

@@ -105,7 +105,7 @@ export class StartState implements IState{
         }
 
         return shell.exec(
-                  `docker compose -f ${composeFiles.join(' -f ')} up -d 2>${this.dockerService.getNullOutput()}`
+            `docker compose -f ${composeFiles.join(' -f ')} up -d 2>${this.dockerService.getNullOutput()}`
         );
     }
 

src/types/CLIOptions.ts

@@ -36,4 +36,6 @@ export interface CLIOptions {
     blocklisting: boolean,
     startup: boolean,
     verbose: number
+    timestamp: string,
+    enableDebug: boolean
 }

src/utils/config.ts

@@ -0,0 +1,33 @@
+/*-
+ *
+ * Hedera Local Node
+ *
+ * Copyright (C) 2023 Hedera Hashgraph, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+import { join } from 'path';
+import yaml from 'js-yaml';
+import { readFileSync } from 'fs';
+import { APPLICATION_YML_RELATIVE_PATH } from '../constants';
+
+export default function readApplicationYML() {
+    const propertiesFilePath = join(__dirname, APPLICATION_YML_RELATIVE_PATH);
+    const application = yaml.load(readFileSync(propertiesFilePath).toString()) as any;
+
+    return {
+        propertiesFilePath,
+        application
+    }
+}