Use random port for github action to resolve unstable CI

.github/workflows/e2e.yml

@@ -64,20 +64,41 @@ jobs:
           if [ ${{ matrix.workflow }} = "router-pay" ]; then
             export START_BOOTNODE=y
           fi
+          export ON_GITHUB_ACTION=y
           ./tests/nodes/start.sh &
 
           # Wait for the nodes to start, the initialization takes some time
-          # check port 127.0.0.1:(41714 ~ 41716) are open
+          # check all the ports are open
+
+          # when .ports file is not generated, we will retry 20 times to check if all ports are open
+          port_file=./tests/nodes/.ports
+          retry_count=0
+          while [ $retry_count -lt 100 ]; do
+            if [ -f $port_file ]; then
+              break
+            else
+                echo "File $port_file not found. Retrying in 2 seconds..."
+                sleep 2
+                retry_count=$((retry_count + 1))
+            fi
+          done
+
+          ports=()
+          while IFS= read -r line; do
+            ports+=("$line")
+          done < ./tests/nodes/.ports
+
           for i in {1..20}; do
             all_open=true
-            for port in 41714 41715 41716 8114; do
+            for port in $ports; do
+              echo "Checking port $port"
               if ! nc -z 127.0.0.1 $port; then
                 all_open=false
                 break
               fi
             done
             if $all_open; then
-              echo "All ports are open"
+              echo "All ports are open now ..."
               break
             else
               echo "Not all ports are open, waiting 3 seconds before retrying"

.gitignore

@@ -7,3 +7,4 @@
 /tests/nodes/*/fiber/store
 /tests/nodes/*/config.yml
 /tests/deploy/udt-init/target
+tests/nodes/.ports

tests/bruno/environments/test.bru

@@ -1,8 +1,8 @@
 vars {
   CKB_RPC_URL: http://127.0.0.1:8114
-  NODE1_RPC_URL: http://127.0.0.1:41714
-  NODE2_RPC_URL: http://127.0.0.1:41715
-  NODE3_RPC_URL: http://127.0.0.1:41716
+  NODE1_RPC_URL: http://127.0.0.1:21714
+  NODE2_RPC_URL: http://127.0.0.1:21715
+  NODE3_RPC_URL: http://127.0.0.1:21716
   NODE1_ADDR: /ip4/127.0.0.1/tcp/8344/p2p/QmbvRjJHAQDmj3cgnUBGQ5zVnGxUKwb2qJygwNs2wk41h8
   NODE2_ADDR: /ip4/127.0.0.1/tcp/8345/p2p/QmSRcPqUn4aQrKHXyCDjGn2qBVf43tWBDS2Wj9QDUZXtZp
   NODE3_ADDR: /ip4/127.0.0.1/tcp/8346/p2p/QmaFDJb9CkMrXy7nhTWBY5y9mvuykre3EzzRsCJUAVXprZ

tests/bruno/environments/xudt-test.bru

@@ -1,8 +1,8 @@
 vars {
   CKB_RPC_URL: http://127.0.0.1:8114
-  NODE1_RPC_URL: http://127.0.0.1:41714
-  NODE2_RPC_URL: http://127.0.0.1:41715
-  NODE3_RPC_URL: http://127.0.0.1:41716
+  NODE1_RPC_URL: http://127.0.0.1:21714
+  NODE2_RPC_URL: http://127.0.0.1:21715
+  NODE3_RPC_URL: http://127.0.0.1:21716
   NODE1_ADDR: /ip4/127.0.0.1/tcp/8344/p2p/QmbvRjJHAQDmj3cgnUBGQ5zVnGxUKwb2qJygwNs2wk41h8
   NODE2_ADDR: /ip4/127.0.0.1/tcp/8345/p2p/QmSRcPqUn4aQrKHXyCDjGn2qBVf43tWBDS2Wj9QDUZXtZp
   NODE3_ADDR: /ip4/127.0.0.1/tcp/8346/p2p/QmaFDJb9CkMrXy7nhTWBY5y9mvuykre3EzzRsCJUAVXprZ

tests/deploy/deploy.sh

@@ -47,7 +47,7 @@ function deploy() {
 }
 
 generate_blocks() {
-    ./generate-blocks.sh 4
+    ./generate-blocks.sh 8
     sleep 1
 }
 

tests/deploy/init-dev-chain.sh

@@ -45,7 +45,15 @@ if ! [[ -d "$data_dir" ]]; then
     # Don't continue until the default account has some money.
     # Transfer some money from the default account (node 3) to node 1 for later use.
     echo "begin to setup wallet states for nodes"
-    sleep 3
+    for i in {1..20}; do
+        if ! nc -z 127.0.0.1 8114; then
+            echo "waiting CKB ready $i ..."
+            sleep 2
+        else
+            echo "CKB is ready now ..."
+            break
+        fi
+    done
 
     # Transfer some money to the node 1.
     # The address of node 1 can be seen with the following command:

tests/deploy/udt-init/Cargo.toml

@@ -15,3 +15,4 @@ serde_json = "1.0"
 thiserror = "1.0.30"
 hex = "0.4.3"
 serde_yaml = "0.9.34"
+rand = "0.8.5"

tests/deploy/udt-init/src/main.rs

@@ -16,7 +16,10 @@ use ckb_types::{
     H256,
 };
 use ckb_types::{packed::CellDep, prelude::Builder};
+use rand::Rng;
 use serde::{Deserialize, Serialize};
+use std::collections::HashSet;
+use std::net::TcpListener;
 
 use std::{error::Error as StdErr, str::FromStr};
 
@@ -196,6 +199,31 @@ struct UdtInfos {
     infos: Vec<UdtInfo>,
 }
 
+fn is_port_available(port: u16) -> bool {
+    match TcpListener::bind(("127.0.0.1", port)) {
+        Ok(listener) => {
+            drop(listener); // Close the listener
+            true
+        }
+        Err(_) => false,
+    }
+}
+
+fn generate_ports(num_ports: usize) -> Vec<u16> {
+    let mut ports = HashSet::new();
+    let mut rng = rand::thread_rng();
+
+    while ports.len() < num_ports {
+        // avoid https://en.wikipedia.org/wiki/Ephemeral_port
+        let port: u16 = rng.gen_range(1024..32768);
+        if is_port_available(port) {
+            ports.insert(port);
+        }
+    }
+
+    ports.into_iter().collect()
+}
+
 fn genrate_nodes_config() {
     let nodes_dir = std::env::var("NODES_DIR").expect("env var");
     let yaml_file_path = format!("{}/deployer/config.yml", nodes_dir);
@@ -225,18 +253,32 @@ fn genrate_nodes_config() {
         "# you can edit nodes/deployer/config.yml and run `REMOVE_OLD_STATE=y ./tests/nodes/start.sh` to regenerate"
     );
     let config_dirs = vec!["bootnode", "1", "2", "3"];
+    let mut ports_map = vec![];
+    let on_github_action = std::env::var("ON_GITHUB_ACTION").is_ok();
+    let gen_ports = generate_ports(6);
+    let mut ports_iter = gen_ports.iter();
     for (i, config_dir) in config_dirs.iter().enumerate() {
+        let use_gen_port = on_github_action && i != 0;
+        let default_fiber_port = (8343 + i) as u16;
+        let default_rpc_port = (21713 + i) as u16;
+        let (fiber_port, rpc_port) = if use_gen_port {
+            (*ports_iter.next().unwrap(), *ports_iter.next().unwrap())
+        } else {
+            (default_fiber_port, default_rpc_port)
+        };
+        ports_map.push((default_fiber_port, fiber_port));
+        ports_map.push((default_rpc_port, rpc_port));
         let mut data = data.clone();
         data["fiber"]["listening_addr"] =
-            serde_yaml::Value::String(format!("/ip4/0.0.0.0/tcp/{}", 8343 + i));
+            serde_yaml::Value::String(format!("/ip4/0.0.0.0/tcp/{}", fiber_port));
         data["fiber"]["announced_addrs"] =
             serde_yaml::Value::Sequence(vec![serde_yaml::Value::String(format!(
                 "/ip4/127.0.0.1/tcp/{}",
-                8343 + i
+                fiber_port
             ))]);
         data["fiber"]["announced_node_name"] = serde_yaml::Value::String(format!("fiber-{}", i));
         data["rpc"]["listening_addr"] =
-            serde_yaml::Value::String(format!("127.0.0.1:{}", 41713 + i));
+            serde_yaml::Value::String(format!("127.0.0.1:{}", rpc_port));
         data["ckb"]["udt_whitelist"] = serde_yaml::to_value(&udt_infos).unwrap();
 
         // Node 3 acts as a CCH node.
@@ -249,20 +291,49 @@ fn genrate_nodes_config() {
 
         let new_yaml = header.to_string() + &serde_yaml::to_string(&data).unwrap();
         let config_path = format!("{}/{}/config.yml", nodes_dir, config_dir);
+
         std::fs::write(config_path, new_yaml).expect("write failed");
     }
+
+    if on_github_action {
+        let bruno_dir = format!("{}/../bruno/environments/", nodes_dir);
+        for config in std::fs::read_dir(bruno_dir).expect("read dir") {
+            let config = config.expect("read config");
+            for (default_port, port) in ports_map.iter() {
+                eprintln!(
+                    "update bruno config: {:?} {} -> {}",
+                    config, default_port, port
+                );
+                let content = std::fs::read_to_string(config.path()).expect("read config");
+                let new_content = content.replace(&default_port.to_string(), &port.to_string());
+                std::fs::write(config.path(), new_content).expect("write config");
+            }
+        }
+    }
+
+    // write the real ports into a file so that later script can use it to double check the ports
+    let content = ports_map
+        .iter()
+        .skip(2) // bootnode node was not always started
+        .map(|(_, port)| port.to_string())
+        .collect::<Vec<_>>()
+        .join("\n");
+
+    let port_file_path = format!("{}/.ports", nodes_dir);
+    std::fs::write(port_file_path, content).expect("write ports list");
 }
 
 fn init_udt_accounts() -> Result<(), Box<dyn StdErr>> {
     let udt_owner = get_nodes_info("deployer");
     for udt in UDT_KINDS {
+        eprintln!("begin init udt: {} ...", udt);
         init_or_send_udt(udt, &udt_owner.0, &udt_owner, None, 1000000000000, true)
             .expect("init udt");
         generate_blocks(8).expect("ok");
         std::thread::sleep(std::time::Duration::from_millis(1000));
-
         for i in 0..3 {
             let wallet = get_nodes_info(&(i + 1).to_string());
+            eprintln!("begin send udt: {} to node {} ...", udt, i);
             init_or_send_udt(
                 udt,
                 &udt_owner.0,

tests/nodes/start.sh

@@ -1,14 +1,15 @@
 #!/usr/bin/env bash
-
 set -euo pipefail
 export SHELLOPTS
 export RUST_BACKTRACE=full RUST_LOG=info,fnn=debug
 
 should_remove_old_state="${REMOVE_OLD_STATE:-}"
 should_start_bootnode="${START_BOOTNODE:-}"
+should_generate_port="${ON_GITHUB_ACTION:-}"
 script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd)"
 nodes_dir="$(dirname "$script_dir")/nodes"
 deploy_dir="$(dirname "$script_dir")/deploy"
+bruno_dir="$(dirname "$script_dir")/bruno/environments"
 
 # The following environment variables are used in the contract tests.
 # We may load all contracts within the following folder to the test environment.
@@ -38,6 +39,9 @@ if [[ -f "$deploy_dir/.env.local" ]]; then
     export $(xargs <"$deploy_dir/.env.local")
 fi
 
+echo "Initializing finished, begin to start services ...."
+sleep 1
+
 ckb run -C "$deploy_dir/node-data" --indexer &
 
 # Start the dev node in the background.