feat: ability to deploy different software package to a subset of nodes

.github/workflows/zxc-e2e-test.yaml

@@ -101,6 +101,29 @@ jobs:
       - name: Pull Kind Docker Image
         run: docker image pull kindest/node:v1.27.3@sha256:3966ac761ae0136263ffdb6cfd4db23ef8a83cba8a463690e98317add2c9ba72
 
+      - name: Setup Java
+        if: ${{ runner.os == 'linux' && inputs.enable-e2e-tests && !cancelled() && !failure() }}
+        uses: actions/setup-java@387ac29b308b003ca37ba93a6cab5eb57c8f5f93 # v4.0.0
+        with:
+          distribution: temurin
+          java-version: 21.0.1
+
+      - name: Setup Gradle
+        if: ${{ runner.os == 'linux' && inputs.enable-e2e-tests && !cancelled() && !failure() }}
+        uses: gradle/gradle-build-action@29c0906b64b8fc82467890bfb7a0a7ef34bda89e # v3.1.0
+        with:
+          cache-read-only: false
+
+      - name: Build Hedera code locally
+        if: ${{ runner.os == 'linux' && inputs.enable-e2e-tests && !cancelled() && !failure() }}
+        run: |
+          cd ..
+          git clone https://github.com/hashgraph/hedera-services.git
+          cd hedera-services
+          ls -ltr
+          ./gradlew assemble
+          cd ../solo
+
       - name: Setup E2E Tests
         run: | 
           npm link

README.md.template

@@ -351,7 +351,24 @@ $ solo node start
 
 # output is similar to example-1 
 ```
+## For Developer Working on Hedera Repo
 
+To set customized `settings.txt` file, edit the file
+`./resources/templates/settings.txt` before `node setup` command.
+
+To deploy node with local build hedera code, first run `./gradlew assemble` at the root directory of hedera repo, then
+  run the following command:
+```
+$ solo node setup --local-build-path node0=<path-to-hedera-repo>,node1=<path-to-hedera-repo>,node2=<path-to-hedera-repo>
+```
+To have nodes running with different versions of hedera application, please clone the repo in multiple directories,
+checkout to the respective version and build the code. Then run the above command
+and provide the path to the respective node.
+
+To deploy node with local build PTT code, run the following command:
+```
+$ solo node setup --local-build-path node0=<path-to-hedera-repo>,node1=<path-to-hedera-repo>,node2=<path-to-hedera-repo> --ptt-test-config PlatformTestingTool.jar,<path-to-test-json>
+```
 ## Support
 
 If you have a question on how to use the product, please see our [support guide](https://github.com/hashgraph/.github/blob/main/SUPPORT.md).

package.json

@@ -15,7 +15,7 @@
     "test-e2e-all": "NODE_OPTIONS=--experimental-vm-modules JEST_SUITE_NAME='Jest E2E All Tests' JEST_JUNIT_OUTPUT_NAME='junit-e2e.xml' jest --runInBand --detectOpenHandles --forceExit --coverage --coverageDirectory='coverage/e2e' --testPathIgnorePatterns=\".*/unit/.*\"",
     "test-e2e": "NODE_OPTIONS=--experimental-vm-modules JEST_SUITE_NAME='Jest E2E Tests' JEST_JUNIT_OUTPUT_NAME='junit-e2e.xml' jest --runInBand --detectOpenHandles --forceExit --coverage --coverageDirectory='coverage/e2e' --testPathIgnorePatterns=\".*/unit/.*\" --testPathIgnorePatterns=\".*/e2e/commands/mirror_node.*\" --testPathIgnorePatterns=\".*/e2e/commands/node.*\" --testPathIgnorePatterns=\".*/e2e/commands/relay.*\"",
     "test-e2e-mirror-node": "NODE_OPTIONS=--experimental-vm-modules JEST_SUITE_NAME='Jest E2E Mirror Node Tests' JEST_JUNIT_OUTPUT_NAME='junit-e2e-mirror-node.xml' jest --runInBand --detectOpenHandles --forceExit --coverage --coverageDirectory='coverage/e2e-mirror-node' --testRegex=\".*\\/e2e\\/commands\\/mirror_node\\.test\\.mjs\"",
-    "test-e2e-node": "NODE_OPTIONS=--experimental-vm-modules JEST_SUITE_NAME='Jest E2E Node Tests' JEST_JUNIT_OUTPUT_NAME='junit-e2e-node.xml' jest --runInBand --detectOpenHandles --forceExit --coverage --coverageDirectory='coverage/e2e-node' --testRegex=\".*\\/e2e\\/commands\\/node\\.test\\.mjs\"",
+    "test-e2e-node": "NODE_OPTIONS=--experimental-vm-modules JEST_SUITE_NAME='Jest E2E Node Tests' JEST_JUNIT_OUTPUT_NAME='junit-e2e-node.xml' jest --runInBand --detectOpenHandles --forceExit --coverage --coverageDirectory='coverage/e2e-node' --testRegex=\".*\\/e2e\\/commands\\/node*\\.test\\.mjs\"",
     "test-e2e-relay": "NODE_OPTIONS=--experimental-vm-modules JEST_SUITE_NAME='Jest E2E Relay Tests' JEST_JUNIT_OUTPUT_NAME='junit-e2e-relay.xml' jest --runInBand --detectOpenHandles --forceExit --coverage --coverageDirectory='coverage/e2e-relay' --testRegex=\".*\\/e2e\\/commands\\/relay\\.test\\.mjs\"",
     "merge-clean": "rm -rf .nyc_output && mkdir  .nyc_output && rm -rf coverage/lcov-report && rm -rf coverage/solo && rm coverage/*.*",
     "merge-e2e": "nyc merge ./coverage/e2e/ .nyc_output/coverage.json",

src/commands/flags.mjs

@@ -399,6 +399,24 @@ export const settingTxt = {
   }
 }
 
+export const pttTestConfig = {
+  name: 'ptt-test-config',
+  definition: {
+    describe: 'PTT test jar and config file',
+    defaultValue: '',
+    type: 'string'
+  }
+}
+
+export const localBuildPath = {
+  name: 'local-build-path',
+  definition: {
+    describe: 'path of hedera local repo',
+    defaultValue: '',
+    type: 'string'
+  }
+}
+
 export const log4j2Xml = {
   name: 'log4j2-xml',
   definition: {
@@ -507,7 +525,9 @@ export const allFlags = [
   amount,
   applicationEnv,
   profileFile,
-  profileName
+  profileName,
+  localBuildPath,
+  pttTestConfig
 ]
 
 export const allFlagsMap = new Map(allFlags.map(f => [f.name, f]))

src/commands/node.mjs

@@ -88,13 +88,14 @@
     }
   }
 
-  async checkNetworkNodeState (nodeId, maxAttempt = 100, status = 'ACTIVE') {
+  async checkNetworkNodeState (nodeId, maxAttempt = 100, status = 'ACTIVE', logfile = 'output/hgcaa.log') {
     nodeId = nodeId.trim()
     const podName = Templates.renderNetworkPodName(nodeId)
-    const logfilePath = `${constants.HEDERA_HAPI_PATH}/output/hgcaa.log`
+    const logfilePath = `${constants.HEDERA_HAPI_PATH}/${logfile}`
     let attempt = 0
     let isActive = false
 
+    this.logger.debug(`Checking if node ${nodeId} is ${status}...`)
     // check log file is accessible
     let logFileAccessible = false
     while (attempt++ < maxAttempt) {
@@ -116,9 +117,10 @@
     attempt = 0
     while (attempt < maxAttempt) {
       try {
-        const output = await this.k8.execContainer(podName, constants.ROOT_CONTAINER, ['tail', '-10', logfilePath])
+        const output = await this.k8.execContainer(podName, constants.ROOT_CONTAINER, ['tail', '-100', logfilePath])
         if (output && output.indexOf('Terminating Netty') < 0 && // make sure we are not at the beginning of a restart
             (output.indexOf(`Now current platform status = ${status}`) > 0 ||
+            output.indexOf(`Platform Status Change ${status}`) > 0 ||
             output.indexOf(`is ${status}`) > 0)) { // 'is ACTIVE' is for newer versions, first seen in v0.49.0
           this.logger.debug(`Node ${nodeId} is ${status} [ attempt: ${attempt}/${maxAttempt}]`)
           isActive = true
@@ -356,6 +358,50 @@
     }
   }
 
+  uploadPlatformSoftware (ctx, task, localBuildPath) {
+    const self = this
+    const subTasks = []
+
+    const config = ctx.config
+    self.logger.debug('no need to fetch, use local build jar files')
+    // localBuildPath example input 'node0=../hedera-services/,node1=../hedera-services/,node2=../hedera2/hedera-services/'
+    // split the input string by ','
+    const parameterPairs = localBuildPath.split(',')
+    const pttTestConfig = self.configManager.getFlag(flags.pttTestConfig)
+    for (const parameterPair of parameterPairs) {
+      // split the localBuildPath by '='
+      const [nodeId, localBuildPath] = parameterPair.split('=')
+      let localDataLibBuildPath = path.join(localBuildPath, 'hedera-node', 'data')
+      if (pttTestConfig !== '' && fs.existsSync(pttTestConfig)) {
+        localDataLibBuildPath = path.join(localBuildPath, 'platform-sdk', 'sdk', 'data')
+      }
+
+      // if the path does not exist, throw an error
+      if (!fs.existsSync(localDataLibBuildPath)) {
+        throw new FullstackTestingError(`local build path does not exist: ${localDataLibBuildPath}`)
+      }
+      const podName = config.podNames[nodeId]
+      const [pttJar, testJson] = pttTestConfig.split(',')
+      self.logger.debug(`PTT test jar: ${pttJar}, test json: ${testJson}`)
+      subTasks.push({
+        title: `Copy local build to Node: ${chalk.yellow(nodeId)} from ${localDataLibBuildPath}`,
+        task: async () => {
+          this.logger.debug(`Copying build files to pod: ${podName} from ${localDataLibBuildPath}`)
+          await self.k8.copyTo(podName, constants.ROOT_CONTAINER, localDataLibBuildPath, `${constants.HEDERA_HAPI_PATH}`)
+          const pttTestConfig = self.configManager.getFlag(flags.pttTestConfig)
+          if (pttTestConfig !== '' && fs.existsSync(pttTestConfig)) {
+            await self.k8.copyTo(podName, constants.ROOT_CONTAINER, testJson, `${constants.HEDERA_HAPI_PATH}`)
+          }
+        }
+      })
+    }
+    // set up the sub-tasks
+    return task.newListr(subTasks, {
+      concurrent: true,
+      rendererOptions: constants.LISTR_DEFAULT_RENDERER_OPTION
+    })
+  }
+
   fetchPlatformSoftware (ctx, task, platformInstaller) {
     const config = ctx.config
 
@@ -501,7 +547,17 @@
                 const config = ctx.config
                 const configTxtPath = `${config.stagingDir}/config.txt`
                 const template = `${constants.RESOURCES_DIR}/templates/config.template`
-                await self.platformInstaller.prepareConfigTxt(config.nodeIds, configTxtPath, config.releaseTag, config.chainId, template)
+                const pttTestConfig = self.configManager.getFlag(flags.pttTestConfig)
+                if (pttTestConfig !== '' && fs.existsSync(pttTestConfig)) {
+                  const [pttJar, testJson] = pttTestConfig.split(',')
+                  self.logger.debug(`PTT test jar: ${pttJar}, test json: ${testJson}`)
+                  if (!fs.existsSync(testJson)) {
+                    throw new FullstackTestingError(`test json file does not exist: ${testJson}`)
+                  }
+                  await self.platformInstaller.prepareConfigTxt(config.nodeIds, configTxtPath, config.releaseTag, config.chainId, template, pttJar)
+                } else {
+                  await self.platformInstaller.prepareConfigTxt(config.nodeIds, configTxtPath, config.releaseTag, config.chainId, template)
+                }
               }
             }
           ]
@@ -516,7 +572,12 @@
         title: 'Fetch platform software into network nodes',
         task:
           async (ctx, task) => {
-            return self.fetchPlatformSoftware(ctx, task, self.platformInstaller)
+            const localBuildPath = self.configManager.getFlag(flags.localBuildPath)
+            if (localBuildPath !== '') {
+              return self.uploadPlatformSoftware(ctx, task, localBuildPath)
+            } else {
+              return self.fetchPlatformSoftware(ctx, task, self.platformInstaller)
+            }
           }
       },
       {
@@ -616,10 +677,18 @@
         task: (ctx, task) => {
           const subTasks = []
           for (const nodeId of ctx.config.nodeIds) {
-            subTasks.push({
-              title: `Check node: ${chalk.yellow(nodeId)}`,
-              task: () => self.checkNetworkNodeState(nodeId)
-            })
+            const pttTestConfig = self.configManager.getFlag(flags.pttTestConfig)
+            if (pttTestConfig !== '' && fs.existsSync(pttTestConfig)) {
+              subTasks.push({
+                title: `Check node: ${chalk.yellow(nodeId)}`,
+                task: () => self.checkNetworkNodeState(nodeId, 100, 'ACTIVE', 'logs/swirlds.log')
+              })
+            } else {
+              subTasks.push({
+                title: `Check node: ${chalk.yellow(nodeId)}`,
+                task: () => self.checkNetworkNodeState(nodeId)
+              })
+            }
           }
 
           // set up the sub-tasks
@@ -650,7 +719,8 @@
               collapseSubtasks: false
             }
           })
-        }
+        },
+        skip: (ctx, _) => self.configManager.getFlag(flags.pttTestConfig) !== ''
       }], {
       concurrent: false,
       rendererOptions: constants.LISTR_DEFAULT_RENDERER_OPTION
@@ -1031,10 +1101,18 @@
         task: (ctx, task) => {
           const subTasks = []
           for (const nodeId of ctx.config.nodeIds) {
-            subTasks.push({
-              title: `Check node: ${chalk.yellow(nodeId)}`,
-              task: () => self.checkNetworkNodeState(nodeId)
-            })
+            const pttTestConfig = self.configManager.getFlag(flags.pttTestConfig)
+            if (pttTestConfig !== '' && fs.existsSync(pttTestConfig)) {
+              subTasks.push({
+                title: `Check node: ${chalk.yellow(nodeId)}`,
+                task: () => self.checkNetworkNodeState(nodeId, 100, 'ACTIVE', 'logs/swirlds.log')
+              })
+            } else {
+              subTasks.push({
+                title: `Check node: ${chalk.yellow(nodeId)}`,
+                task: () => self.checkNetworkNodeState(nodeId)
+              })
+            }
           }
 
           // set up the sub-tasks
@@ -1067,7 +1145,8 @@
               collapseSubtasks: false
             }
           })
-        }
+        },
+        skip: (ctx, _) => self.configManager.getFlag(flags.pttTestConfig) !== ''
       }], {
       concurrent: false,
       rendererOptions: constants.LISTR_DEFAULT_RENDERER_OPTION
@@ -1558,6 +1637,8 @@
               flags.apiPermissionProperties,
               flags.bootstrapProperties,
               flags.settingTxt,
+              flags.localBuildPath,
+              flags.pttTestConfig,
               flags.log4j2Xml
             ),
             handler: argv => {

src/core/platform_installer.mjs

@@ -269,7 +269,7 @@ export class PlatformInstaller {
    * @param chainId chain ID (298 for local network)
    * @returns {Promise<unknown>}
    */
-  async prepareConfigTxt (nodeIDs, destPath, releaseTag, chainId = constants.HEDERA_CHAIN_ID, template = `${constants.RESOURCES_DIR}/templates/config.template`) {
+  async prepareConfigTxt (nodeIDs, destPath, releaseTag, chainId = constants.HEDERA_CHAIN_ID, template = `${constants.RESOURCES_DIR}/templates/config.template`, appName = constants.HEDERA_APP_NAME) {
     if (!nodeIDs || nodeIDs.length === 0) throw new MissingArgumentError('list of node IDs is required')
     if (!destPath) throw new MissingArgumentError('destPath is required')
     if (!template) throw new MissingArgumentError('config templatePath is required')
@@ -283,7 +283,6 @@ export class PlatformInstaller {
     const accountIdPrefix = `${startAccountId.realm}.${startAccountId.shard}`
     const internalPort = constants.HEDERA_NODE_INTERNAL_GOSSIP_PORT
     const externalPort = constants.HEDERA_NODE_EXTERNAL_GOSSIP_PORT
-    const appName = constants.HEDERA_APP_NAME
     const nodeStakeAmount = constants.HEDERA_NODE_DEFAULT_STAKE_AMOUNT
 
     const releaseVersion = semver.parse(releaseTag, { includePrerelease: true })

test/e2e/commands/node-local-hedera.test.mjs

@@ -0,0 +1,51 @@
+/**
+ * Copyright (C) 2024 Hedera Hashgraph, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the ""License"");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an ""AS IS"" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+import {
+  afterAll,
+  describe
+} from '@jest/globals'
+import { flags } from '../../../src/commands/index.mjs'
+import {
+  constants
+} from '../../../src/core/index.mjs'
+import {
+  bootstrapNetwork,
+  getDefaultArgv,
+  TEST_CLUSTER
+} from '../../test_util.js'
+
+describe('Node local build', () => {
+  const LOCAL_HEDERA = 'local-hedera-app'
+  const argv = getDefaultArgv()
+  argv[flags.keyFormat.name] = constants.KEY_FORMAT_PFX
+  argv[flags.nodeIDs.name] = 'node0,node1,node2'
+  argv[flags.generateGossipKeys.name] = true
+  argv[flags.generateTlsKeys.name] = true
+  argv[flags.clusterName.name] = TEST_CLUSTER
+  let hederaK8
+  afterAll(async () => {
+    await hederaK8.deleteNamespace(LOCAL_HEDERA)
+  }, 120000)
+
+  describe('Node for hedera app should start successfully', () => {
+    console.log('Starting local build for Hedera app')
+    argv[flags.localBuildPath.name] = 'node0=../hedera-services/,node1=../hedera-services/,node2=../hedera-services/'
+    argv[flags.namespace.name] = LOCAL_HEDERA
+    const bootstrapResp = bootstrapNetwork(LOCAL_HEDERA, argv)
+    hederaK8 = bootstrapResp.opts.k8
+  })
+})