feat: fix ci and tests

.github/workflows/ci.yml

@@ -64,11 +64,6 @@ jobs:
                 "cache-to": ["type=gha,mode=max,scope=bridge-0"],
                 "output": ["type=docker"]
               },
-              "bridge-1": {
-                "cache-from": ["type=gha,scope=bridge-1"],
-                "cache-to": ["type=gha,mode=max,scope=bridge-1"],
-                "output": ["type=docker"]
-              },
               "light-0": {
                 "cache-from": ["type=gha,scope=light-0"],
                 "cache-to": ["type=gha,mode=max,scope=light-0"],
@@ -82,10 +77,23 @@ jobs:
       - name: Run the docker-compose stack
         run: docker compose -f ci/docker-compose.yml up --no-build -d
 
-      - name: Wait for bridge node to start
+      - name: Wait for bridge node 0 to start
+        run: |
+          while ! docker compose -f ci/docker-compose.yml logs bridge-0 | grep -q 'Configuration finished. Running a bridge node'; do
+            sleep 1
+          done
+
+      - name: Wait for bridge node 1 to start
+        run: |
+          while ! docker compose -f ci/docker-compose.yml logs bridge-1 | grep -q 'Configuration finished. Running a bridge node'; do
+            sleep 1
+          done
+
+      - name: Wait for light node 0 to start
         run: |
-          docker compose -f ci/docker-compose.yml logs -f |
-            awk '/Configuration finished. Running a bridge node/ {print;exit}'
+          while ! docker compose -f ci/docker-compose.yml logs light-0 | grep -q 'Configuration finished. Running a light node'; do
+            sleep 1
+          done
 
       - name: Run integration tests
         run: cargo test -p prism-tests --lib --release --features mock_prover

README.md

@@ -39,7 +39,7 @@ We are currently experimenting with various proof systems and have handwritten g
 
 ### Install Dependencies
 
-We use `just` as a task runner. Once installed, you can install the rest of the dependencies with:
+We use [`just`](https://github.com/casey/just?tab=readme-ov-file#packages) as a task runner. Once installed, you can install the rest of the dependencies with:
 
 ```bash
 just install-deps

ci/Dockerfile.bridge

@@ -3,7 +3,7 @@
 # A dockerfile for the celestia bridge node in DA layer
 # Based on:
 # https://github.com/celestiaorg/celestia-node/blob/main/Dockerfile
-FROM docker.io/alpine:3.19.1
+FROM docker.io/alpine:3.21.0
 
 ENV CELESTIA_HOME=/root
 

ci/Dockerfile.lightnode

@@ -3,7 +3,7 @@
 # A dockerfile for the celestia bridge node in DA layer
 # Based on:
 # https://github.com/celestiaorg/celestia-node/blob/main/Dockerfile
-FROM docker.io/alpine:3.19.1
+FROM docker.io/alpine:3.21.0
 
 ENV CELESTIA_HOME=/root
 

ci/Dockerfile.validator

@@ -3,7 +3,7 @@
 # A dockerfile for the celestia validator in consensus layer
 # Based on:
 # https://github.com/celestiaorg/celestia-app/blob/main/Dockerfile
-FROM docker.io/alpine:3.19.1
+FROM docker.io/alpine:3.21.0
 
 ENV CELESTIA_HOME=/root
 

ci/docker-compose.yml

@@ -11,7 +11,7 @@ services:
       - LIGHT_COUNT=1
     volumes:
       - credentials:/credentials
-      - genesis:/genesis
+      - shared:/shared
 
   bridge-0:
     image: bridge
@@ -23,12 +23,16 @@ services:
       # provide an id for the bridge node (default: 0)
       # each node should have a next natural number starting from 0
       - NODE_ID=0
+      # setting SKIP_AUTH to true disables the use of JWT for authentication
       - SKIP_AUTH=true
+      # this must match the service name in the docker-compose.yml file
+      # used for the trusted peers string for the light node
+      - CONTAINER_NAME=bridge-0
     ports:
       - 26658:26658
     volumes:
       - credentials:/credentials
-      - genesis:/genesis
+      - shared:/shared
 
   bridge-1:
     image: bridge
@@ -42,11 +46,14 @@ services:
       - NODE_ID=1
       # setting SKIP_AUTH to true disables the use of JWT for authentication
       - SKIP_AUTH=true
+      # this must match the service name in the docker-compose.yml file
+      # used for the trusted peers string for the light node
+      - CONTAINER_NAME=bridge-1
     ports:
       - 36658:26658
     volumes:
       - credentials:/credentials
-      - genesis:/genesis
+      - shared:/shared
 
   light-0:
     image: light
@@ -55,33 +62,19 @@ services:
       context: .
       dockerfile: Dockerfile.lightnode
     environment:
-      # provide an id for the bridge node (default: 0)
+      # provide an id for the light node (default: 0)
       # each node should have a next natural number starting from 0
       - NODE_ID=0
+      # setting SKIP_AUTH to true disables the use of JWT for authentication
       - SKIP_AUTH=true
+      # depending on the number of bridge nodes, provide the count
+      # is used for the trusted peers string for the light node
+      - BRIDGE_COUNT=2
     ports:
       - 46658:26658
     volumes:
       - credentials:/credentials
-      - genesis:/genesis
-
-  # Uncomment for another bridge node
-  # remember to adjust services.validator.command
-  # bridge-1:
-  #   image: bridge
-  #   platform: "linux/amd64"
-  #   build:
-  #     context: .
-  #     dockerfile: Dockerfile.bridge
-  #   environment:
-  #     # provide an id for the bridge node (default: 0)
-  #     # each node should have a next natural number starting from 0
-  #     - NODE_ID=1
-  #   ports:
-  #     - 36658:26658
-  #   volumes:
-  #     - credentials:/credentials
-  #     - genesis:/genesis
+      - shared:/shared
 
 volumes:
   # local volume where node's credentials can persist
@@ -91,8 +84,8 @@ volumes:
       type: "none"
       o: "bind"
       device: "./credentials"
-  # a temporary fs where the genesis hash is announced
-  genesis:
+  # a temporary fs where the nodes share data
+  shared:
     driver_opts:
       type: tmpfs
       device: tmpfs

ci/run-bridge.sh

@@ -6,6 +6,7 @@ set -euo pipefail
 # Name for this node or `bridge-0` if not provided
 NODE_ID="${NODE_ID:-0}"
 SKIP_AUTH="${SKIP_AUTH:-false}"
+CONTAINER_NAME="${CONTAINER_NAME:-bridge-$NODE_ID}"
 NODE_NAME="bridge-$NODE_ID"
 # a private local network
 P2P_NETWORK="private"
@@ -16,9 +17,10 @@ CREDENTIALS_DIR="/credentials"
 # node credentials
 NODE_KEY_FILE="$CREDENTIALS_DIR/$NODE_NAME.key"
 NODE_JWT_FILE="$CREDENTIALS_DIR/$NODE_NAME.jwt"
-# directory where validator will write the genesis hash
-GENESIS_DIR="/genesis"
-GENESIS_HASH_FILE="$GENESIS_DIR/genesis_hash"
+# directory where validator will write the genesis hash and the bridge node their peers addresses
+SHARED_DIR="/shared"
+GENESIS_HASH_FILE="$SHARED_DIR/genesis_hash"
+TRUSTED_PEERS_FILE="$SHARED_DIR/trusted_peers"
 
 # Wait for the validator to set up and provision us via shared dir
 wait_for_provision() {
@@ -53,6 +55,52 @@ write_jwt_token() {
   celestia bridge auth admin --p2p.network "$P2P_NETWORK" > "$NODE_JWT_FILE"
 }
 
+append_trusted_peers() {
+  peer_id=""
+  start_time=$(date +%s)
+  timeout=30
+
+  while [[ -z "$peer_id" ]]; do
+    peer_id=$(celestia p2p info | jq -r '.result.id' || echo "")
+    if [[ -z "$peer_id" ]]; then
+      echo "Node is not running yet. Retrying..."
+      sleep 1
+    fi
+
+    current_time=$(date +%s)
+    elapsed=$((current_time - start_time))
+    if [[ $elapsed -ge $timeout ]]; then
+      echo "Failed to retrieve Peer ID after $timeout seconds. Exiting."
+      exit 1
+    fi
+  done
+
+  #multiaddr: /dns/$CONTAINER_NAME/tcp/$RPC_PORT/p2p/$peer_id
+  multiaddr="/dns/$CONTAINER_NAME/tcp/2121/p2p/$peer_id"
+  echo "Appending trusted peer: $multiaddr"
+
+  # Lock the file to prevent race conditions
+  exec 9>"$TRUSTED_PEERS_FILE.lock"
+  flock -x 9
+
+  # Read existing peers into a variable
+  existing_peers=""
+  if [[ -s "$TRUSTED_PEERS_FILE" ]]; then
+    existing_peers=$(cat "$TRUSTED_PEERS_FILE")
+  fi
+
+  # Append the new multiaddr to the existing peers
+  if [[ -n "$existing_peers" ]]; then
+    echo "$existing_peers,$multiaddr" > "$TRUSTED_PEERS_FILE"
+  else
+    echo "$multiaddr" > "$TRUSTED_PEERS_FILE"
+  fi
+
+  # Unlock the file
+  flock -u 9
+  exec 9>&-
+}
+
 main() {
   # Initialize the bridge node
   celestia bridge init --p2p.network "$P2P_NETWORK"
@@ -64,6 +112,8 @@ main() {
   add_trusted_genesis
   # Update the JWT token
   write_jwt_token
+  # Append the peer multiaddr to the trusted peers (run in background, as the node needs to be running)
+  append_trusted_peers &
   # give validator some time to set up
   sleep 4
   # Start the bridge node

ci/run-lightnode.sh

@@ -6,6 +6,7 @@ set -euo pipefail
 # Name for this node or `light-0` if not provided
 NODE_ID="${NODE_ID:-0}"
 SKIP_AUTH="${SKIP_AUTH:-false}"
+BRIDGE_COUNT="${BRIDGE_COUNT}"
 NODE_NAME="light-$NODE_ID"
 # a private local network
 P2P_NETWORK="private"
@@ -16,18 +17,51 @@ CREDENTIALS_DIR="/credentials"
 # node credentials
 NODE_KEY_FILE="$CREDENTIALS_DIR/$NODE_NAME.key"
 NODE_JWT_FILE="$CREDENTIALS_DIR/$NODE_NAME.jwt"
-# directory where validator will write the genesis hash
-GENESIS_DIR="/genesis"
-GENESIS_HASH_FILE="$GENESIS_DIR/genesis_hash"
+# directory where validator will write the genesis hash and the bridge node their peers addresses
+SHARED_DIR="/shared"
+GENESIS_HASH_FILE="$SHARED_DIR/genesis_hash"
+TRUSTED_PEERS_FILE="$SHARED_DIR/trusted_peers"
 
 # Wait for the validator to set up and provision us via shared dir
 wait_for_provision() {
   echo "Waiting for the validator node to start"
   while [[ ! ( -e "$GENESIS_HASH_FILE" && -e "$NODE_KEY_FILE" ) ]]; do
     sleep 0.1
   done
-
   echo "Validator is ready"
+
+  echo "Waiting for $BRIDGE_COUNT bridge nodes to start"
+  start_time=$(date +%s)
+  timeout=30
+
+  while true; do
+    if [[ -e "$TRUSTED_PEERS_FILE" ]]; then
+
+      echo "Trusted peers file exists"
+
+      trusted_peers="$(cat "$TRUSTED_PEERS_FILE")"
+      echo "Trusted peers: $trusted_peers"
+      peer_count=$(echo "$trusted_peers" | tr ',' '\n' | wc -l)
+      echo "Peer count: $peer_count"
+      if [[ $peer_count -eq $BRIDGE_COUNT ]]; then
+        echo "$BRIDGE_COUNT bridge nodes are ready"
+        break
+      else
+        echo "Trusted peers file does not contain the expected number of commas. Retrying..."
+      fi
+    else
+      echo "Trusted peers file does not exist yet. Retrying..."
+    fi
+
+    current_time=$(date +%s)
+    elapsed=$((current_time - start_time))
+    if [[ $elapsed -ge $timeout ]]; then
+      echo "Timeout reached. Exiting."
+      exit 1
+    fi
+
+    sleep 1
+  done
 }
 
 # Import the test account key shared by the validator
@@ -43,11 +77,19 @@ add_trusted_genesis() {
 
   # Read the hash of the genesis block
   genesis_hash="$(cat "$GENESIS_HASH_FILE")"
+  trusted_peers="$(cat "$TRUSTED_PEERS_FILE")"
   # and make it trusted in the node's config
   echo "Trusting a genesis: $genesis_hash"
   sed -i'.bak' "s/TrustedHash = .*/TrustedHash = $genesis_hash/" "$CONFIG_DIR/config.toml"
 }
 
+add_trusted_peers() {
+  local trusted_peers="$(cat "$TRUSTED_PEERS_FILE")"
+  local formatted_peers=$(echo "$trusted_peers" | sed 's/\([^,]*\)/"\1"/g')
+  echo "Trusting peers: $formatted_peers"
+  sed -i'.bak' "s|TrustedPeers = .*|TrustedPeers = [$formatted_peers]|" "$CONFIG_DIR/config.toml"
+}
+
 write_jwt_token() {
   echo "Saving jwt token to $NODE_JWT_FILE"
   celestia light auth admin --p2p.network "$P2P_NETWORK" > "$NODE_JWT_FILE"
@@ -62,6 +104,8 @@ main() {
   import_shared_key
   # Trust the private blockchain
   add_trusted_genesis
+  # Trust the bridge nodes
+  add_trusted_peers
   # Update the JWT token
   write_jwt_token
   # give validator some time to set up

ci/run-validator.sh

@@ -18,9 +18,9 @@ BRIDGE_COINS="200000000000000utia"
 VALIDATOR_COINS="1000000000000000utia"
 # a directory and the files shared with the bridge nodes
 CREDENTIALS_DIR="/credentials"
-# directory where validator will write the genesis hash
-GENESIS_DIR="/genesis"
-GENESIS_HASH_FILE="$GENESIS_DIR/genesis_hash"
+# directory where validator will write the genesis hash and the bridge node their peers addresses
+SHARED_DIR="/shared"
+GENESIS_HASH_FILE="$SHARED_DIR/genesis_hash"
 
 # Get the address of the node of given name
 node_address() {

crates/tests/src/lib.rs

@@ -34,11 +34,11 @@ async fn test_light_client_prover_talking() -> Result<()> {
     pretty_env_logger::init();
 
     let bridge_cfg = CelestiaConfig {
-        connection_string: "ws://0.0.0.0:36658".to_string(),
+        connection_string: "ws://localhost:26658".to_string(),
         ..CelestiaConfig::default()
     };
     let lc_cfg = CelestiaConfig {
-        connection_string: "ws://0.0.0.0:26658".to_string(),
+        connection_string: "ws://localhost:46658".to_string(),
         ..CelestiaConfig::default()
     };