[forge] custom profiles & add processor latency metrics

.github/workflows/adhoc-forge.yaml

@@ -8,33 +8,33 @@ on:
       GIT_SHA:
         required: true
         type: string
-        description: The git SHA1 to checkout and test
+        description: The git SHA1 to checkout and test. This can also be a branch or tag
       IMAGE_TAG:
         required: false
         type: string
-        description: The docker image tag to test. If not specified, falls back on GIT_SHA
+        description: The docker image tag to test. If not specified, it is derived from the current check-out
       FORGE_IMAGE_TAG:
         required: false
         type: string
-        description: The docker image tag to use for forge runner. If not specified, falls back on GIT_SHA
+        description: The docker image tag to use for forge runner. If not specified, it is derived from the current check-out
       FORGE_RUNNER_DURATION_SECS:
         required: false
         type: string
         default: "480"
-        description: Duration of the forge test run
+        description: Duration of the forge test run. For multi-phasic tests this is the length of a single phase
       FORGE_TEST_SUITE:
         required: false
         type: string
         default: land_blocking
-        description: Test suite to run
+        description: Test suite to run. See https://github.com/aptos-labs/aptos-core/blob/main/testsuite/forge-cli/src/main.rs
       FORGE_CLUSTER_NAME:
         required: false
         type: string
         description: The Forge k8s cluster to be used for test
-      FORGE_ENABLE_INDEXER:
+      FORGE_INDEXER_DEPLOYER_PROFILE:
         required: false
-        type: boolean
-        description: Whether to use indexer
+        type: string
+        description: If set enables an indexer stack with your test, spinning up with the provided profile from https://github.com/aptos-labs/internal-ops/blob/main/infra/cli/commands/forge/profiles.ts
       FORGE_NUM_VALIDATORS:
         required: false
         type: string
@@ -66,21 +66,21 @@ jobs:
           echo "FORGE_RUNNER_DURATION_SECS: ${{ inputs.FORGE_RUNNER_DURATION_SECS }}"
           echo "FORGE_TEST_SUITE: ${{ inputs.FORGE_TEST_SUITE }}"
           echo "FORGE_CLUSTER_NAME: ${{ inputs.FORGE_CLUSTER_NAME }}"
-          echo "FORGE_ENABLE_INDEXER: ${{ inputs.FORGE_ENABLE_INDEXER }}"
           echo "FORGE_NUM_VALIDATORS: ${{ inputs.FORGE_NUM_VALIDATORS }}"
           echo "FORGE_NUM_VALIDATOR_FULLNODES: ${{ inputs.FORGE_NUM_VALIDATOR_FULLNODES }}"
           echo "FORGE_RETAIN_DEBUG_LOGS: ${{ inputs.FORGE_RETAIN_DEBUG_LOGS }}"
+          echo "FORGE_INDEXER_DEPLOYER_PROFILE: ${{ inputs.FORGE_INDEXER_DEPLOYER_PROFILE }}"
     outputs:
       gitSha: ${{ inputs.GIT_SHA }}
       imageTag: ${{ inputs.IMAGE_TAG }}
       forgeImageTag: ${{ inputs.FORGE_IMAGE_TAG }}
       forgeRunnerDurationSecs: ${{ inputs.FORGE_RUNNER_DURATION_SECS || 600 }}
       forgeTestSuite: ${{ inputs.FORGE_TEST_SUITE }}
       forgeClusterName: ${{ inputs.FORGE_CLUSTER_NAME }}
-      forgeEnableIndexer: ${{ inputs.FORGE_ENABLE_INDEXER }}
       forgeNumValidators: ${{ inputs.FORGE_NUM_VALIDATORS }}
       forgeNumValidatorFullnodes: ${{ inputs.FORGE_NUM_VALIDATOR_FULLNODES }}
       forgeRetainDebugLogs: ${{ inputs.FORGE_RETAIN_DEBUG_LOGS }}
+      forgeIndexerDeployerProfile: ${{ inputs.FORGE_INDEXER_DEPLOYER_PROFILE }}
   adhoc-forge-test:
     uses: aptos-labs/aptos-core/.github/workflows/workflow-run-forge.yaml@main
     needs: [determine-forge-run-metadata]
@@ -92,7 +92,9 @@ jobs:
       FORGE_TEST_SUITE: ${{ needs.determine-forge-run-metadata.outputs.forgeTestSuite }}
       FORGE_RUNNER_DURATION_SECS: ${{ fromJSON(needs.determine-forge-run-metadata.outputs.forgeRunnerDurationSecs) }} # fromJSON converts to integer
       FORGE_CLUSTER_NAME: ${{ needs.determine-forge-run-metadata.outputs.forgeClusterName }}
-      FORGE_ENABLE_INDEXER: ${{ needs.determine-forge-run-metadata.outputs.forgeEnableIndexer == 'true' }}
       FORGE_NUM_VALIDATORS: ${{ needs.determine-forge-run-metadata.outputs.forgeNumValidators }}
       FORGE_NUM_VALIDATOR_FULLNODES: ${{ needs.determine-forge-run-metadata.outputs.forgeNumValidatorFullnodes }}
       FORGE_RETAIN_DEBUG_LOGS: ${{ needs.determine-forge-run-metadata.outputs.forgeRetainDebugLogs == 'true' }}
+      # Indexer outputs are derived from a single one, to avoid using too many inputs for this workflow_dispatch trigger
+      FORGE_ENABLE_INDEXER: ${{ needs.determine-forge-run-metadata.outputs.forgeIndexerDeployerProfile != '' }}
+      FORGE_DEPLOYER_PROFILE: ${{ needs.determine-forge-run-metadata.outputs.forgeIndexerDeployerProfile }}

.github/workflows/forge-unstable.yaml

@@ -128,6 +128,7 @@ jobs:
     secrets: inherit
     with:
       IMAGE_TAG: ${{ needs.determine-test-metadata.outputs.IMAGE_TAG }}
+      FORGE_NAMESPACE: forge-indexer-test-${{ needs.determine-test-metadata.outputs.BRANCH_HASH }}
       COMMENT_HEADER: forge-indexer
       FORGE_TEST_SUITE: indexer_test
       FORGE_ENABLE_INDEXER: true

.github/workflows/workflow-run-forge.yaml

@@ -52,10 +52,6 @@ on:
         required: false
         type: string
         description: Whether to use HAPRoxy
-      FORGE_ENABLE_INDEXER:
-        required: false
-        type: boolean
-        description: Whether to use indexer
       FORGE_ENABLE_PERFORMANCE:
         required: false
         type: string
@@ -83,6 +79,14 @@ on:
         required: false
         type: string
         description: Number of validator fullnodes to use for the forge test
+      FORGE_ENABLE_INDEXER:
+        required: false
+        type: boolean
+        description: Whether to use indexer
+      FORGE_DEPLOYER_PROFILE:
+        required: false
+        type: string
+        description: The deployer profile used to spin up and configure forge infrastructure
 
 env:
   AWS_ACCOUNT_NUM: ${{ secrets.ENV_ECR_AWS_ACCOUNT_NUM }}
@@ -104,6 +108,7 @@ env:
   FORGE_NAMESPACE: ${{ inputs.FORGE_NAMESPACE }}
   FORGE_ENABLE_HAPROXY: ${{ inputs.FORGE_ENABLE_HAPROXY }}
   FORGE_ENABLE_INDEXER: ${{ inputs.FORGE_ENABLE_INDEXER }}
+  FORGE_DEPLOYER_PROFILE: ${{ inputs.FORGE_DEPLOYER_PROFILE }}
   FORGE_TEST_SUITE: ${{ inputs.FORGE_TEST_SUITE }}
   POST_TO_SLACK: ${{ inputs.POST_TO_SLACK }}
   FORGE_ENABLE_FAILPOINTS: ${{ inputs.FORGE_ENABLE_FAILPOINTS }}

testsuite/forge-cli/src/main.rs

@@ -62,7 +62,11 @@ use aptos_testcases::{
 };
 use async_trait::async_trait;
 use clap::{Parser, Subcommand};
-use futures::stream::{FuturesUnordered, StreamExt};
+use futures::{
+    future,
+    stream::{FuturesUnordered, StreamExt},
+    FutureExt,
+};
 use once_cell::sync::Lazy;
 use rand::{rngs::ThreadRng, seq::SliceRandom, Rng};
 use serde_json::{json, Value};
@@ -194,11 +198,14 @@ struct K8sSwarm {
         help = "Retain debug logs and above for all nodes instead of just the first 5 nodes"
     )]
     retain_debug_logs: bool,
+    #[clap(long, help = "If set, spins up an indexer stack alongside the testnet")]
+    enable_indexer: bool,
     #[clap(
         long,
-        help = "If set, spins up an indexer stack alongside the testnet. Same as --enable-indexer"
+        help = "The deployer profile used to spin up and configure forge infrastructure",
+        default_value = &DEFAULT_FORGE_DEPLOYER_PROFILE,
     )]
-    enable_indexer: bool,
+    deployer_profile: String,
 }
 
 #[derive(Parser, Debug)]
@@ -262,6 +269,13 @@ struct Create {
         requires = "enable_indexer"
     )]
     indexer_image_tag: Option<String>,
+    #[clap(
+        long,
+        help = "The deployer profile used to spin up and configure forge infrastructure",
+        default_value = &DEFAULT_FORGE_DEPLOYER_PROFILE,
+
+    )]
+    deployer_profile: String,
 }
 
 // common metrics thresholds:
@@ -408,6 +422,7 @@ fn main() -> Result<()> {
                             k8s.keep,
                             k8s.enable_haproxy,
                             k8s.enable_indexer,
+                            k8s.deployer_profile.clone(),
                         )
                         .unwrap(),
                         &args.options,
@@ -444,7 +459,7 @@ fn main() -> Result<()> {
                     .or(Some(create.validator_image_tag.clone()))
                     .expect("Expected indexer or validator image tag to use");
                 let config: Value = serde_json::from_value(json!({
-                    "profile": DEFAULT_FORGE_DEPLOYER_PROFILE.to_string(),
+                    "profile": create.deployer_profile,
                     "era": era.clone(),
                     "namespace": create.namespace.clone(),
                     "indexer-grpc-values": {
@@ -455,32 +470,44 @@ fn main() -> Result<()> {
                     },
                 }))?;
 
-                runtime.block_on(install_testnet_resources(
-                    era.clone(),
-                    create.namespace.clone(),
-                    create.num_validators,
-                    create.num_fullnodes,
-                    create.validator_image_tag,
-                    create.testnet_image_tag,
-                    create.move_modules_dir,
-                    false, // since we skip_collecting_running_nodes, we don't connect directly to the nodes to validatet their health
-                    create.enable_haproxy,
-                    create.enable_indexer,
-                    None,
-                    None,
-                    true,
-                ))?;
-
-                if create.enable_indexer {
-                    let indexer_deployer = ForgeDeployerManager::new(
-                        kube_client.clone(),
+                let deploy_testnet_fut = async {
+                    install_testnet_resources(
+                        era.clone(),
                         create.namespace.clone(),
-                        FORGE_INDEXER_DEPLOYER_DOCKER_IMAGE_REPO.to_string(),
+                        create.num_validators,
+                        create.num_fullnodes,
+                        create.validator_image_tag,
+                        create.testnet_image_tag,
+                        create.move_modules_dir,
+                        false, // since we skip_collecting_running_nodes, we don't connect directly to the nodes to validatet their health
+                        create.enable_haproxy,
+                        create.enable_indexer,
+                        create.deployer_profile,
                         None,
-                    );
-                    runtime.block_on(indexer_deployer.start(config))?;
-                    runtime.block_on(indexer_deployer.wait_completed())?;
+                        None,
+                        true,
+                    )
+                    .await
                 }
+                .boxed();
+
+                let deploy_indexer_fut = async {
+                    if create.enable_indexer {
+                        let indexer_deployer = ForgeDeployerManager::new(
+                            kube_client.clone(),
+                            create.namespace.clone(),
+                            FORGE_INDEXER_DEPLOYER_DOCKER_IMAGE_REPO.to_string(),
+                            None,
+                        );
+                        indexer_deployer.start(config).await?;
+                        indexer_deployer.wait_completed().await
+                    } else {
+                        Ok(())
+                    }
+                }
+                .boxed();
+
+                runtime.block_on(future::try_join(deploy_testnet_fut, deploy_indexer_fut))?;
                 Ok(())
             },
         },
@@ -2389,48 +2416,52 @@ fn indexer_test() -> ForgeConfig {
     // Define all the workloads and their corresponding success criteria upfront
     // The TransactionTypeArg is the workload per phase
     // The structure of the success criteria is generally (min_tps, latencies...). See below for the exact definition.
+    // NOTES on tuning these criteria:
+    // * The blockchain's TPS criteria are generally lower than that of other tests. This is because we want to only capture indexer performance regressions. Other tests with higher TPS requirements
+    //   for the blockchain would catch an earlier regression
+    // * The indexer latencies are inter-component within the indexer stack, but not that of the e2e wall-clock time vs the blockchain
     let workloads_and_criteria = vec![
         (
             TransactionWorkload::new(TransactionTypeArg::CoinTransfer, 20000),
-            (7000, 0.5, 0.5, 0.5),
+            (7000, 0.5, 1.0, 0.1),
         ),
         (
             TransactionWorkload::new(TransactionTypeArg::NoOp, 20000).with_num_modules(100),
-            (8500, 0.5, 0.5, 0.5),
+            (8500, 0.5, 1.0, 0.1),
         ),
         (
             TransactionWorkload::new(TransactionTypeArg::ModifyGlobalResource, 6000)
                 .with_transactions_per_account(1),
-            (2000, 0.5, 0.5, 0.5),
+            (2000, 0.5, 0.5, 0.1),
         ),
         (
             TransactionWorkload::new(TransactionTypeArg::TokenV2AmbassadorMint, 20000)
                 .with_unique_senders(),
-            (3200, 0.5, 0.5, 0.5),
+            (2000, 1.0, 1.0, 0.5),
         ),
         (
             TransactionWorkload::new(TransactionTypeArg::PublishPackage, 200)
                 .with_transactions_per_account(1),
-            (28, 0.5, 0.5, 0.5),
+            (28, 0.5, 1.0, 0.1),
         ),
         (
             TransactionWorkload::new(TransactionTypeArg::VectorPicture30k, 100),
-            (100, 1.0, 1.0, 1.0),
+            (100, 0.5, 1.0, 0.1),
         ),
         (
             TransactionWorkload::new(TransactionTypeArg::SmartTablePicture30KWith200Change, 100),
-            (100, 1.0, 1.0, 1.0),
+            (100, 0.5, 1.0, 0.1),
         ),
         (
             TransactionWorkload::new(
                 TransactionTypeArg::TokenV1NFTMintAndTransferSequential,
                 1000,
             ),
-            (500, 0.5, 0.5, 0.5),
+            (500, 0.5, 1.0, 0.1),
         ),
         (
             TransactionWorkload::new(TransactionTypeArg::TokenV1FTMintAndTransfer, 1000),
-            (500, 0.5, 0.5, 0.5),
+            (500, 0.5, 0.5, 0.1),
         ),
     ];
     let num_sweep = workloads_and_criteria.len();

testsuite/forge.env

@@ -0,0 +1,7 @@
+# This file is source-d when running Forge
+# LEAVE ALL LINES COMMENTED OUT UNLESS FOR TESTING. NEVER LAND THIS WITH VALUES UNCOMMENTED
+
+# FORGE_ENABLE_INDEXER=true
+# FORGE_ENABLE_HAPROXY=true
+# FORGE_RETAIN_DEBUG_LOGS=true
+

testsuite/forge.py

@@ -1150,6 +1150,7 @@ def create_forge_command(
     forge_namespace_keep: Optional[str],
     forge_enable_haproxy: Optional[str],
     forge_enable_indexer: Optional[str],
+    forge_deployer_profile: Optional[str],
     cargo_args: Optional[Sequence[str]],
     forge_cli_args: Optional[Sequence[str]],
     test_args: Optional[Sequence[str]],
@@ -1221,6 +1222,8 @@ def create_forge_command(
         forge_args.append("--enable-haproxy")
     if forge_enable_indexer == "true":
         forge_args.append("--enable-indexer")
+    if forge_deployer_profile:
+        forge_args.extend(["--deployer-profile", forge_deployer_profile])
 
     if test_args:
         forge_args.extend(test_args)
@@ -1334,6 +1337,7 @@ def seeded_random_choice(namespace: str, cluster_names: Sequence[str]) -> str:
 @envoption("FORGE_NAMESPACE_REUSE")
 @envoption("FORGE_ENABLE_HAPROXY")
 @envoption("FORGE_ENABLE_INDEXER")
+@envoption("FORGE_DEPLOYER_PROFILE")
 @envoption("FORGE_ENABLE_FAILPOINTS")
 @envoption("FORGE_ENABLE_PERFORMANCE")
 @envoption("FORGE_TEST_SUITE")
@@ -1380,6 +1384,7 @@ def test(
     forge_enable_performance: Optional[str],
     forge_enable_haproxy: Optional[str],
     forge_enable_indexer: Optional[str],
+    forge_deployer_profile: Optional[str],
     forge_test_suite: str,
     forge_runner_duration_secs: str,
     forge_image_tag: Optional[str],
@@ -1606,6 +1611,7 @@ def test(
         forge_namespace_keep=forge_namespace_keep,
         forge_enable_haproxy=forge_enable_haproxy,
         forge_enable_indexer=forge_enable_indexer,
+        forge_deployer_profile=forge_deployer_profile,
         cargo_args=cargo_args,
         forge_cli_args=forge_cli_args,
         test_args=test_args,

testsuite/forge/src/backend/k8s/cluster_helper.rs

@@ -6,8 +6,8 @@ use super::GENESIS_HELM_RELEASE_NAME;
 use crate::{
     get_validator_fullnodes, get_validators, k8s_wait_nodes_strategy, nodes_healthcheck,
     wait_stateful_set, ForgeDeployerManager, ForgeRunnerMode, GenesisConfigFn, K8sApi, K8sNode,
-    NodeConfigFn, ReadWrite, Result, APTOS_NODE_HELM_RELEASE_NAME, DEFAULT_FORGE_DEPLOYER_PROFILE,
-    DEFAULT_ROOT_KEY, DEFAULT_TEST_SUITE_NAME, DEFAULT_USERNAME, FORGE_KEY_SEED,
+    NodeConfigFn, ReadWrite, Result, APTOS_NODE_HELM_RELEASE_NAME, DEFAULT_ROOT_KEY,
+    DEFAULT_TEST_SUITE_NAME, DEFAULT_USERNAME, FORGE_KEY_SEED,
     FORGE_TESTNET_DEPLOYER_DOCKER_IMAGE_REPO, FULLNODE_HAPROXY_SERVICE_SUFFIX,
     FULLNODE_SERVICE_SUFFIX, HELM_BIN, KUBECTL_BIN, MANAGEMENT_CONFIGMAP_PREFIX,
     NAMESPACE_CLEANUP_THRESHOLD_SECS, POD_CLEANUP_THRESHOLD_SECS, VALIDATOR_HAPROXY_SERVICE_SUFFIX,
@@ -569,6 +569,7 @@ pub async fn install_testnet_resources(
     use_port_forward: bool,
     enable_haproxy: bool,
     enable_indexer: bool,
+    deployer_profile: String,
     genesis_helm_config_fn: Option<GenesisConfigFn>,
     node_helm_config_fn: Option<NodeConfigFn>,
     // If true, skip collecting running nodes after installing the testnet. This is useful when we only care about creating resources
@@ -624,7 +625,7 @@ pub async fn install_testnet_resources(
     }
 
     let config: serde_json::Value = serde_json::from_value(serde_json::json!({
-        "profile": DEFAULT_FORGE_DEPLOYER_PROFILE.to_string(),
+        "profile": deployer_profile,
         "era": new_era,
         "namespace": kube_namespace.clone(),
         "testnet-values": aptos_node_helm_values,