e2e: avoid starting nodes from the future (#6835) (#6838)

(cherry picked from commit a4cc831)

Co-authored-by: Sam Kleinman <garen@tychoish.com>

test/e2e/runner/start.go

@@ -25,6 +25,7 @@ func Start(testnet *e2e.Testnet) error {
  }
  return false
  })
+
  sort.SliceStable(nodeQueue, func(i, j int) bool {
  return nodeQueue[i].StartAt < nodeQueue[j].StartAt
  })
@@ -49,9 +50,15 @@ func Start(testnet *e2e.Testnet) error {
  logger.Info(fmt.Sprintf("Node %v up on http://127.0.0.1:%v", node.Name, node.ProxyPort))
  }
 
+ networkHeight := testnet.InitialHeight
+
  // Wait for initial height
- logger.Info(fmt.Sprintf("Waiting for initial height %v...", testnet.InitialHeight))
- block, blockID, err := waitForHeight(testnet, testnet.InitialHeight)
+ logger.Info("Waiting for initial height",
+ "height", networkHeight,
+ "nodes", len(testnet.Nodes)-len(nodeQueue),
+ "pending", len(nodeQueue))
+
+ block, blockID, err := waitForHeight(testnet, networkHeight)
  if err != nil {
  return err
  }
@@ -66,8 +73,26 @@ func Start(testnet *e2e.Testnet) error {
  }
  }
 
- // Start up remaining nodes
  for _, node := range nodeQueue {
+ if node.StartAt > networkHeight {
+ // if we're starting a node that's ahead of
+ // the last known height of the network, then
+ // we should make sure that the rest of the
+ // network has reached at least the height
+ // that this node will start at before we
+ // start the node.
+
+ networkHeight = node.StartAt
+
+ logger.Info("Waiting for network to advance before starting catch up node",
+ "node", node.Name,
+ "height", networkHeight)
+
+ if _, _, err := waitForHeight(testnet, networkHeight); err != nil {
+ return err
+ }
+ }
+
  logger.Info(fmt.Sprintf("Starting node %v at height %v...", node.Name, node.StartAt))
  if _, _, err := waitForHeight(testnet, node.StartAt); err != nil {
  return err