Forest & Lotus devnet

.dockerignore

@@ -5,3 +5,8 @@ Dockerfile
 *.md
 target/
 scripts/
+
+# Ignore CAR files fetched to the project directory which tends to happen during development.
+# Without it, the Docker context may bloat to hundreds of gigabytes of data.
+/*.car
+/*.car.zst

CHANGELOG.md

@@ -29,6 +29,8 @@
 
 ### Added
 
+- [#2816](https://github.com/ChainSafe/forest/issues/2816): Support `2k` devnet.
+
 - [#3026](https://github.com/ChainSafe/forest/pull/3026): Expose
   `forest-cli state diff ...`
 

documentation/src/developer_documentation/devnet_notes.md

@@ -9,6 +9,7 @@
     export LOTUS_SKIP_GENESIS_CHECK=_yes_
     export CGO_CFLAGS_ALLOW="-D__BLST_PORTABLE__"
     export CGO_CFLAGS="-D__BLST_PORTABLE__"
+    # For MacOS: LIBRARY_PATH=/opt/homebrew/Cellar/hwloc/2.9.1/lib
     make 2k
     ./lotus fetch-params 2048
     ./lotus-seed pre-seal --sector-size 2KiB --num-sectors 2

scripts/devnet/.env

@@ -0,0 +1,7 @@
+FOREST_DATA_DIR=/forest_data
+LOTUS_DATA_DIR=/lotus_data
+FILECOIN_PROOFS_DIR=/var/tmp/filecoin-proof-parameters
+LOTUS_RPC_PORT=1234
+LOTUS_P2P_PORT=1235
+MINER_RPC_PORT=2345
+FOREST_RPC_PORT=3456

scripts/devnet/README.md

@@ -0,0 +1,63 @@
+# Local devnet setup
+
+The devnet consists of a:
+
+- Lotus miner,
+- Lotus node,
+- Forest node.
+
+It's packed in a docker-compose setup for convenience and ease of usage. By
+default, running it will expose relevant RPC and P2P ports to the host:
+
+- 1234 - Lotus RPC,
+- 9090 - Lotus P2P port,
+- 2345 - Miner RPC,
+- 3456 - Forest RPC.
+
+## Running the devnet
+
+Run it with:
+
+```shell
+docker-compose up --build
+```
+
+This will build the local Forest (using the Dockerfile in the project's root)
+image, tagged Lotus and setup the devnet. Initial setup may be slow due to
+fetching params and setting everyting up. Consecutive starts will be quick.
+
+Stop the devnet with:
+
+```shell
+docker-compose down
+```
+
+Remove the devnet:
+
+```shell
+docker-compose rm
+```
+
+## Interacting with the devnet via CLI
+
+Exec into the `forest` container:
+
+```shell
+docker exec -it forest /bin/bash
+```
+
+and setup credentials. Then run any command:
+
+```shell
+export TOKEN=$(cat /forest_data/token.jwt)
+export FULLNODE_API_INFO=$TOKEN:/dns/forest/tcp/1234/http
+
+forest-cli net peers
+```
+
+## Local devnet development
+
+If you prefer to have Forest running directly on the host, you can comment it
+out and draw inspiration from the `docker-compose.yml` on how to connect it to
+Lotus. In short, you will need to obtain the peer id, network name and the
+genesis file.

scripts/devnet/docker-compose.yml

@@ -0,0 +1,192 @@
+# Docker compose file to run a local devnet.
+# The devnet consists of a:
+# - Lotus node (2k build),
+# - Lotus miner (2k build),
+# - Forest node.
+
+version: "3.8"
+
+services:
+  # Basic devnet initialisation. This will populate Lotus data volume with necessary artifacts
+  # to run a devnet. The initialisation is a lengthy process and occurs only at the first
+  # `docker-compose up`.
+  lotus_init:
+    build:
+      context: .
+      dockerfile: lotus.dockerfile
+    volumes:
+      - filecoin-proofs:${FILECOIN_PROOFS_DIR}
+      - lotus-data:${LOTUS_DATA_DIR}
+    env_file:
+      - lotus.env
+    entrypoint: ["/bin/bash", "-c" ]
+    command:
+      - |
+        if [ ! -f ${LOTUS_DATA_DIR}/NODE_INITIALISED ]; then
+          lotus fetch-params 2048
+          lotus-seed --sector-dir ${LOTUS_DATA_DIR}/genesis-sectors pre-seal --sector-size 2KiB --num-sectors 2
+          lotus-seed --sector-dir ${LOTUS_DATA_DIR}/genesis-sectors genesis new ${LOTUS_DATA_DIR}/localnet.json
+          lotus-seed --sector-dir ${LOTUS_DATA_DIR}/genesis-sectors genesis add-miner ${LOTUS_DATA_DIR}/localnet.json ${LOTUS_DATA_DIR}/genesis-sectors/pre-seal-t01000.json
+          touch ${LOTUS_DATA_DIR}/NODE_INITIALISED
+        fi
+
+  # Lotus node that communicates with the miner and the Forest node.
+  # The RPC and P2P listening ports are exposed so Forest does not necessarily need to be running inside Docker.
+  # For development purposes it might be practical to comment out Forest part altogether and use a local build.
+  lotus_node:
+    depends_on:
+      lotus_init:
+        condition: service_completed_successfully
+    build:
+      context: .
+      dockerfile: lotus.dockerfile
+    healthcheck:
+      test: >-
+        curl -s -x post -h "content-type: application/json"
+        --data '{ "jsonrpc": "2.0", "method": "filecoin.chainhead", "params": [], "id": 1 }'
+        http://lotus_node:${LOTUS_RPC_PORT}/rpc/v0 || exit 1
+      interval: 10s
+      retries: 10
+      timeout: 5s
+    container_name: lotus
+    networks:
+      - devnet
+    volumes:
+      - filecoin-proofs:${FILECOIN_PROOFS_DIR}
+      - lotus-data:${LOTUS_DATA_DIR}
+    ports:
+      - ${LOTUS_RPC_PORT}:${LOTUS_RPC_PORT}
+      - ${LOTUS_P2P_PORT}:${LOTUS_P2P_PORT}
+    env_file:
+      - lotus.env
+    entrypoint: ["/bin/bash", "-c" ]
+    command:
+      - |
+        lotus daemon --lotus-make-genesis=${LOTUS_DATA_DIR}/devgen.car --genesis-template=${LOTUS_DATA_DIR}/localnet.json --bootstrap=false
+
+  # Lotus miner container. It communicates only with Lotus and not with Forest.
+  lotus_miner:
+    depends_on:
+      lotus_node:
+        condition: service_healthy
+    build:
+      context: .
+      dockerfile: lotus.dockerfile
+    container_name: lotus-miner
+    healthcheck:
+      test: >-
+        curl -s http://lotus-miner:${MINER_RPC_PORT} || exit 1
+      interval: 20s
+      retries: 5
+      start_period: 6000s
+      timeout: 10s
+    networks:
+      - devnet
+    volumes:
+      - filecoin-proofs:${FILECOIN_PROOFS_DIR}
+      - lotus-data:${LOTUS_DATA_DIR}
+    ports:
+      - ${MINER_RPC_PORT}:${MINER_RPC_PORT}
+    env_file:
+      - lotus-miner.env
+    restart: on-failure # lotus node might not be ready
+    entrypoint: ["/bin/bash", "-c" ]
+    command:
+      - |
+        if [ ! -f ${LOTUS_DATA_DIR}/MINER_INITIALISED ]; then
+          lotus wallet import --as-default ${LOTUS_DATA_DIR}/genesis-sectors/pre-seal-t01000.key
+          lotus-miner init --genesis-miner --actor=t01000 --sector-size=2KiB --pre-sealed-sectors=${LOTUS_DATA_DIR}/genesis-sectors --pre-sealed-metadata=${LOTUS_DATA_DIR}/genesis-sectors/pre-seal-t01000.json --nosync
+          touch ${LOTUS_DATA_DIR}/MINER_INITIALISED
+        fi
+        lotus-miner run --nosync
+
+  # This container dumps relevant (for Forest) Lotus configuration to the shared volume.
+  lotus_config:
+    depends_on:
+      lotus_node:
+        condition: service_healthy
+    build:
+      context: .
+      dockerfile: lotus.dockerfile
+    volumes:
+      - lotus-data:${LOTUS_DATA_DIR}
+    entrypoint: ["/bin/bash", "-c" ]
+    env_file:
+      - lotus.env
+    networks:
+      - devnet
+    command:
+      - lotus net id > ${LOTUS_DATA_DIR}/PEER_ID
+
+  forest:
+    user: root
+    depends_on:
+      lotus_config:
+        condition: service_completed_successfully
+    build:
+      context: ../../.
+      dockerfile: Dockerfile
+    container_name: forest
+    healthcheck:
+      test: |
+        export TOKEN=$$(cat ${FOREST_DATA_DIR}/token.jwt)
+        export FULLNODE_API_INFO=$$TOKEN:/ip4/127.0.0.1/tcp/${FOREST_RPC_PORT}/http
+        forest-cli sync status || exit 1
+      interval: 10s
+      retries: 10
+      timeout: 5s
+      start_period: 15s
+    volumes:
+      - lotus-data:${LOTUS_DATA_DIR}
+      - filecoin-proofs:${FILECOIN_PROOFS_DIR}
+      - forest-data:${FOREST_DATA_DIR}
+      - ./forest_config.toml.tpl:/forest/forest_config.toml.tpl
+    environment:
+      - FIL_PROOFS_PARAMETER_CACHE=${FILECOIN_PROOFS_DIR}
+    networks:
+      - devnet
+    ports:
+      - ${FOREST_RPC_PORT}:${FOREST_RPC_PORT}
+    entrypoint: ["/bin/bash", "-c" ]
+    command:
+      - |
+        set -eu
+        if [ ! -f ${FOREST_DATA_DIR}/forest_config.toml ]; then
+          export NETWORK_NAME=$$(grep -o \"localnet.*\" ${LOTUS_DATA_DIR}/localnet.json)
+          cp /forest/forest_config.toml.tpl ${FOREST_DATA_DIR}/forest_config.toml
+          echo name = $$NETWORK_NAME >> ${FOREST_DATA_DIR}/forest_config.toml
+        fi
+        forest --genesis ${LOTUS_DATA_DIR}/devgen.car --config ${FOREST_DATA_DIR}/forest_config.toml --save-token ${FOREST_DATA_DIR}/token.jwt --rpc-address 0.0.0.0:${FOREST_RPC_PORT}
+
+  # At the moment of writing, Forest was not able to connect to a devnet node using its config.
+  # This is a workaround to force the connection.
+  forest_connecter:
+    user: root
+    depends_on:
+      forest:
+        condition: service_healthy
+    build:
+      context: ../../.
+      dockerfile: Dockerfile
+    volumes:
+      - lotus-data:${LOTUS_DATA_DIR}
+      - forest-data:${FOREST_DATA_DIR}
+    networks:
+      - devnet
+    entrypoint: ["/bin/bash", "-c" ]
+    command:
+      - |
+        set -eu
+        export TOKEN=$$(cat ${FOREST_DATA_DIR}/token.jwt)
+        export FULLNODE_API_INFO=$$TOKEN:/dns/forest/tcp/${FOREST_RPC_PORT}/http
+        forest-cli net connect /dns/lotus/tcp/${LOTUS_P2P_PORT}/p2p/$$(cat ${LOTUS_DATA_DIR}/PEER_ID)
+
+volumes:
+  # Shared proof parameter files. It is re-used by both Lotus and Forest.
+  filecoin-proofs:
+  # Lotus node, miner and network data.
+  lotus-data:
+  forest-data:
+
+networks:
+  devnet:

scripts/devnet/forest_config.toml.tpl

@@ -0,0 +1,12 @@
+[client]
+encrypt_keystore = false
+data_dir = "/forest_data"
+
+[network]
+kademlia = false
+target_peer_count = 1
+
+# Note that this has to come last. The actual TOML file will have
+# the chain name appended.
+[chain.network]
+type = "devnet"

scripts/devnet/lotus-miner.env

@@ -0,0 +1,5 @@
+LOTUS_PATH=/lotus_data/lotus-local-net
+LOTUS_MINER_PATH=/lotus_data/lotus-miner-local-net
+LOTUS_SKIP_GENESIS_CHECK=_yes_
+LOTUS_API_LISTENADDRESS=/dns/lotus/tcp/2345/http
+LOTUS_API_LISTENADDRESS=/dns/lotus-miner/tcp/2345/http

scripts/devnet/lotus.dockerfile

@@ -0,0 +1,36 @@
+# Lotus binaries image, to be used in the local devnet with Forest.
+FROM golang:1.19.7-buster AS lotus-builder
+
+ARG LOTUS_TAG=v1.23.0
+
+RUN apt-get update && apt-get install -y ca-certificates build-essential clang ocl-icd-opencl-dev ocl-icd-libopencl1 jq libhwloc-dev
+
+WORKDIR /lotus
+RUN git clone --depth 1 --branch ${LOTUS_TAG} https://github.com/filecoin-project/lotus.git .
+RUN CGO_CFLAGS_ALLOW="-D__BLST_PORTABLE__" \
+    CGO_CFLAGS="-D__BLST_PORTABLE__" \
+    make 2k
+
+FROM ubuntu:22.04
+
+# Needed for the healthcheck
+RUN apt-get update && apt-get install -y curl
+
+# Need to copy the relevant shared libraries from the builder image.
+# See https://github.com/filecoin-project/lotus/blob/master/Dockerfile
+COPY --from=lotus-builder /etc/ssl/certs            /etc/ssl/certs
+COPY --from=lotus-builder /lib/*/libdl.so.2         /lib/
+COPY --from=lotus-builder /lib/*/librt.so.1         /lib/
+COPY --from=lotus-builder /lib/*/libgcc_s.so.1      /lib/
+COPY --from=lotus-builder /lib/*/libutil.so.1       /lib/
+COPY --from=lotus-builder /usr/lib/*/libltdl.so.7   /lib/
+COPY --from=lotus-builder /usr/lib/*/libnuma.so.1   /lib/
+COPY --from=lotus-builder /usr/lib/*/libhwloc.so.5  /lib/
+COPY --from=lotus-builder /usr/lib/*/libOpenCL.so.1 /lib/
+
+# Copy only the binaries relevant for the devnet
+COPY --from=lotus-builder /lotus/lotus /lotus/lotus-miner /lotus/lotus-seed /usr/local/bin/
+
+WORKDIR /lotus
+
+CMD ["/bin/bash"]

scripts/devnet/lotus.env

@@ -0,0 +1,5 @@
+LOTUS_PATH=/lotus_data/lotus-local-net
+LOTUS_MINER_PATH=/lotus_data/lotus-miner-local-net
+LOTUS_SKIP_GENESIS_CHECK=_yes_
+LOTUS_API_LISTENADDRESS=/dns/lotus/tcp/1234/http
+LOTUS_LIBP2P_LISTENADDRESSES=/ip4/0.0.0.0/tcp/1235

src/chain_sync/chain_muxer.rs

@@ -17,7 +17,7 @@ use crate::libp2p::{
 };
 use crate::message::SignedMessage;
 use crate::message_pool::{MessagePool, Provider};
-use crate::shim::{clock::EPOCHS_IN_DAY, message::Message};
+use crate::shim::{clock::SECONDS_IN_DAY, message::Message};
 use crate::state_manager::StateManager;
 use cid::Cid;
 use futures::{
@@ -472,7 +472,9 @@ where
             }
         };
 
-        if tipset.epoch() + EPOCHS_IN_DAY < chain_store.heaviest_tipset().epoch() {
+        if tipset.epoch() + (SECONDS_IN_DAY / block_delay as i64)
+            < chain_store.heaviest_tipset().epoch()
+        {
             debug!(
                 "Skip processing tipset at epoch {} from {source} that is too old",
                 tipset.epoch()

src/cli/main.rs

@@ -31,6 +31,8 @@ pub async fn main() -> anyhow::Result<()> {
                 if let Ok(name) = chain_get_name((), &config.client.rpc_token).await {
                     if name == "calibnet" {
                         config.chain = Arc::new(ChainConfig::calibnet());
+                    } else if name == "devnet" {
+                        config.chain = Arc::new(ChainConfig::devnet());
                     }
                 }
             }

src/cli_shared/cli/mod.rs

@@ -159,6 +159,10 @@ impl CliOpts {
         if let Some(chain) = &self.chain {
             // override the chain configuration
             cfg.chain = Arc::new(ChainConfig::from_chain(chain));
+        } else {
+            // override any custom changes to the chain configuration based on the used
+            // network.
+            cfg.chain = Arc::new(ChainConfig::from_chain(&cfg.chain.network));
         }
 
         if let Some(genesis_file) = &self.genesis {