feat: improve built-in assertoor tests (ethpandaops#488)

Improves the built-in assertoor tests to make them more resilient against non-critical failures.
bharath-123 · Feb 8, 2024 · d596699 · d596699
1 parent b3418cf
commit d596699
Show file tree

Hide file tree

Showing 4 changed files with 308 additions and 73 deletions.
diff --git a/static_files/assertoor-config/tests/all-opcodes-transaction-test.yaml b/static_files/assertoor-config/tests/all-opcodes-transaction-test.yaml
diff --git a/static_files/assertoor-config/tests/blob-transactions-test.yaml b/static_files/assertoor-config/tests/blob-transactions-test.yaml
@@ -12,39 +12,40 @@ tasks:
     minClientCount: 1
 
 # check if all client pairs propose blocks with blob transactions
-- name: run_tasks_concurrent
+- name: run_task_background
   title: "Check if all clients propose blocks with blob transactions"
   config:
-    succeedTaskCount: 1
-    tasks:
-    - name: generate_blob_transactions
+    onBackgroundComplete: fail
+    backgroundTask:
+      name: generate_blob_transactions
       title: "Generate 2 blob transactions per block"
       config:
         childWallets: 5
-        limitPending: 2
+        walletSeed: "blob-test"
+        limitPending: 4
         limitPerBlock: 2
         randomTarget: true
         amount: 1000000
         randomAmount: true
         blobSidecars: 1
       configVars:
         privateKey: "walletPrivkey"
-
-    - name: run_task_matrix
+    foregroundTask:
+      name: run_task_matrix
       title: "Check transaction inclusion with all client pairs"
-      timeout: 1h
+      timeout: 30m
       configVars:
         matrixValues: "validatorPairNames"
       config:
         runConcurrent: true
         matrixVar: "validatorPairName"
         task:
           name: check_consensus_block_proposals
-          title: "Wait for block proposal with >= 2 blobs from ${validatorPairName}"
+          title: "Wait for block proposal with >= 1 blobs from ${validatorPairName}"
           configVars:
             validatorNamePattern: "validatorPairName"
           config:
-            minBlobCount: 2
+            minBlobCount: 1
 - name: "sleep"
   title: "Wait 30sec to clear up pending transactions"
   config:
@@ -63,16 +64,17 @@ tasks:
       title: "Check if blob transactions can be sent via ${clientPairName}"
       config:
         tasks:
-        - name: run_tasks_concurrent
+        - name: run_task_background
           title: "Check if blob transactions can be sent via ${clientPairName}"
           config:
-            succeedTaskCount: 1
-            tasks:
-            - name: generate_blob_transactions
+            onBackgroundComplete: fail
+            backgroundTask:
+              name: generate_blob_transactions
               title: "Generate 2 blob transactions per block and send via ${clientPairName}"
               config:
                 childWallets: 5
-                limitPending: 2
+                walletSeed: "blob-test"
+                limitPending: 4
                 limitPerBlock: 2
                 randomTarget: true
                 amount: 1000000
@@ -81,10 +83,11 @@ tasks:
               configVars:
                 privateKey: "walletPrivkey"
                 clientPattern: "clientPairName"
-            - name: check_consensus_block_proposals
-              title: "Wait for block proposal with >= 2 blobs"
+            foregroundTask:
+              name: check_consensus_block_proposals
+              title: "Wait for block proposal with >= 1 blobs"
               config:
-                minBlobCount: 2
+                minBlobCount: 1
         - name: "sleep"
           title: "Wait 30sec to clear up pending transactions"
           config:

diff --git a/static_files/assertoor-config/tests/eoa-transactions-test.yaml b/static_files/assertoor-config/tests/eoa-transactions-test.yaml
@@ -12,12 +12,12 @@ tasks:
     minClientCount: 1
 
 # check if all client pairs propose blocks with legacy EOA transactions
-- name: run_tasks_concurrent
+- name: run_task_background
   title: "Check if all clients propose blocks with legacy EOA transactions"
   config:
-    succeedTaskCount: 1
-    tasks:
-    - name: generate_eoa_transactions
+    onBackgroundComplete: fail
+    backgroundTask:
+      name: generate_eoa_transactions
       title: "Generate 10 EOA transaction per block"
       config:
         childWallets: 10
@@ -30,9 +30,10 @@ tasks:
       configVars:
         privateKey: "walletPrivkey"
 
-    - name: run_task_matrix
+    foregroundTask:
+      name: run_task_matrix
       title: "Check transaction inclusion with all client pairs"
-      timeout: 1h
+      timeout: 30m
       configVars:
         matrixValues: "validatorPairNames"
       config:
@@ -63,12 +64,12 @@ tasks:
       title: "Check if legacy EOA transactions can be sent via ${clientPairName}"
       config:
         tasks:
-        - name: run_tasks_concurrent
+        - name: run_task_background
           title: "Check if legacy EOA transactions can be sent via ${clientPairName}"
           config:
-            succeedTaskCount: 1
-            tasks:
-            - name: generate_eoa_transactions
+            onBackgroundComplete: fail
+            backgroundTask:
+              name: generate_eoa_transactions
               title: "Generate 10 EOA transaction per block and send via ${clientPairName}"
               config:
                 childWallets: 10
@@ -81,7 +82,9 @@ tasks:
               configVars:
                 privateKey: "walletPrivkey"
                 clientPattern: "clientPairName"
-            - name: check_consensus_block_proposals
+
+            foregroundTask:
+              name: check_consensus_block_proposals
               title: "Wait for block proposal with >= 5 transactions"
               config:
                 minTransactionCount: 5
@@ -91,12 +94,12 @@ tasks:
             duration: 30s
 
 # check if all client pairs propose blocks with dynfee EOA transactions
-- name: run_tasks_concurrent
+- name: run_task_background
   title: "Check if all clients propose blocks with dynfee EOA transactions"
   config:
-    succeedTaskCount: 1
-    tasks:
-    - name: generate_eoa_transactions
+    onBackgroundComplete: fail
+    backgroundTask:
+      name: generate_eoa_transactions
       title: "Generate 10 EOA transaction per block"
       config:
         childWallets: 10
@@ -108,9 +111,10 @@ tasks:
       configVars:
         privateKey: "walletPrivkey"
 
-    - name: run_task_matrix
+    foregroundTask:
+      name: run_task_matrix
       title: "Check transaction inclusion with all client pairs"
-      timeout: 1h
+      timeout: 30m
       configVars:
         matrixValues: "validatorPairNames"
       config:
@@ -141,12 +145,12 @@ tasks:
       title: "Check if dynfee EOA transactions can be sent via ${clientPairName}"
       config:
         tasks:
-        - name: run_tasks_concurrent
+        - name: run_task_background
           title: "Check if dynfee EOA transactions can be sent via ${clientPairName}"
           config:
-            succeedTaskCount: 1
-            tasks:
-            - name: generate_eoa_transactions
+            onBackgroundComplete: fail
+            backgroundTask:
+              name: generate_eoa_transactions
               title: "Generate 10 EOA transaction per block and send via ${clientPairName}"
               config:
                 childWallets: 10
@@ -158,7 +162,8 @@ tasks:
               configVars:
                 privateKey: "walletPrivkey"
                 clientPattern: "clientPairName"
-            - name: check_consensus_block_proposals
+            foregroundTask:
+              name: check_consensus_block_proposals
               title: "Wait for block proposal with >= 5 transactions"
               config:
                 minTransactionCount: 5