diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 31c1e70cb599..d6a2b693227c 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -71,32 +71,6 @@ jobs: echo "Not a PR event." fi - - name: Set up Python 3.10 - uses: actions/setup-python@v5 - with: - python-version: "3.10" - - - name: Install uv - uses: astral-sh/setup-uv@v5 - - - name: Install dependencies - run: | - bash -x .github/scripts/setup.sh - # We're going to check BOLT quotes, so get the latest version - git clone https://github.com/lightning/bolts.git ../${BOLTDIR} - - name: Configure - run: ./configure --enable-debugbuild --enable-rust - - name: Check source - env: - VALGRIND: 0 - PYTEST_OPTS: --timeout=1200 --durations=10 - run: | - uv run make check-source BASE_REF="origin/${{ github.base_ref }}" - - name: Check Generated Files have been updated - run: uv run make check-gen-updated - - name: Check docs - run: uv run make check-doc - compile: name: Compile CLN ${{ matrix.cfg }} runs-on: ubuntu-22.04 @@ -151,16 +125,74 @@ jobs: set -e ./configure --enable-debugbuild CC="$COMPILER" ${{ matrix.COPTFLAGS_VAR }} - uv run make -j $(nproc) testpack.tar.bz2 + uv run make -j $(nproc) testpack.tar.gz # Rename now so we don't clash - mv testpack.tar.bz2 cln-${CFG}.tar.bz2 + mv testpack.tar.gz cln-${CFG}.tar.gz - name: Check rust packages run: cargo test --all - uses: actions/upload-artifact@v4 with: - name: cln-${{ matrix.CFG }}.tar.bz2 - path: cln-${{ matrix.CFG }}.tar.bz2 + name: cln-${{ matrix.CFG }}.tar.gz + path: cln-${{ matrix.CFG }}.tar.gz + + postbuild: + name: Postbuild checks + runs-on: ubuntu-22.04 + timeout-minutes: 30 + env: + BOLTDIR: bolts + needs: + - compile + strategy: + fail-fast: true + matrix: + include: + - CFG: compile-gcc + VALGRIND: 1 + - CFG: compile-clang-sanitizers + VALGRIND: 0 + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Python 3.10 + uses: actions/setup-python@v5 + with: + python-version: "3.10" + + - name: Install uv + uses: astral-sh/setup-uv@v5 + + - name: Install dependencies + run: | + bash -x .github/scripts/setup.sh + sudo apt-get update -qq + # We're going to check BOLT quotes, so get the latest version + git clone https://github.com/lightning/bolts.git ../${BOLTDIR} + + - name: Download + uses: actions/download-artifact@v4 + with: + name: cln-${{ matrix.CFG }}.tar.gz + + - name: Unpack prebuilt binaries + run: | + git submodule sync && git submodule update --init --recursive + # Make sure source appears older than what we're about to unpack + find . -type f -print0 | xargs -0 touch -d yesterday + tar xaf cln-${{ matrix.CFG }}.tar.gz + + - name: Check source + env: + VALGRIND: 0 + PYTEST_OPTS: --timeout=1200 --durations=10 + run: | + uv run make check-source BASE_REF="origin/${{ github.base_ref }}" CARGO=false CC=devtools/cc-nobuild SUPPRESS_GENERATION=1 + - name: Check Generated Files have been updated + run: uv run make check-gen-updated CARGO=false CC=devtools/cc-nobuild SUPPRESS_GENERATION=1 + - name: Check docs + run: uv run make check-doc CARGO=false CC=devtools/cc-nobuild SUPPRESS_GENERATION=1 check-units: # The unit test checks are not in the critical path (not dependent @@ -202,12 +234,18 @@ jobs: - name: Download build uses: actions/download-artifact@v4 with: - name: cln-${{ matrix.CFG }}.tar.bz2 + name: cln-${{ matrix.CFG }}.tar.gz + + - name: Unpack prebuilt binaries + run: | + git submodule sync && git submodule update --init --recursive + # Make sure source appears older than what we're about to unpack + find . -type f -print0 | xargs -0 touch -d yesterday + tar xaf cln-${{ matrix.CFG }}.tar.gz - name: Check run: | - tar -xaf cln-${{ matrix.CFG }}.tar.bz2 - uv run eatmydata make -j $(nproc) check-units installcheck VALGRIND=${{ matrix.VALGRIND }} + uv run eatmydata make -j $(nproc) check-units installcheck VALGRIND=${{ matrix.VALGRIND }} CARGO=false CC=devtools/cc-nobuild SUPPRESS_GENERATION=1 check-fuzz: name: Run fuzz regression tests @@ -310,13 +348,14 @@ jobs: - name: Download build uses: actions/download-artifact@v4 with: - name: cln-${{ matrix.CFG }}.tar.bz2 + name: cln-${{ matrix.CFG }}.tar.gz - - name: Unpack pre-built CLN - env: - CFG: ${{ matrix.CFG }} + - name: Unpack prebuilt binaries run: | - tar -xaf cln-${CFG}.tar.bz2 + git submodule sync && git submodule update --init --recursive + # Make sure source appears older than what we're about to unpack + find . -type f -print0 | xargs -0 touch -d yesterday + tar xaf cln-${{ matrix.CFG }}.tar.gz - name: Switch network if: ${{ matrix.TEST_NETWORK == 'liquid-regtest' }} @@ -324,6 +363,7 @@ jobs: # Loading the network from config.vars rather than the envvar is a terrible idea... sed -i 's/TEST_NETWORK=regtest/TEST_NETWORK=liquid-regtest/g' config.vars cat config.vars + touch -d yesterday config.vars - name: Test env: @@ -401,10 +441,14 @@ jobs: - name: Download build uses: actions/download-artifact@v4 with: - name: cln-compile-gcc.tar.bz2 + name: cln-compile-gcc.tar.gz - - name: Unpack build - run: tar -xvjf cln-compile-gcc.tar.bz2 + - name: Unpack prebuilt binaries + run: | + git submodule sync && git submodule update --init --recursive + # Make sure source appears older than what we're about to unpack + find . -type f -print0 | xargs -0 touch -d yesterday + tar xaf cln-compile-gcc.tar.gz - name: Test env: @@ -470,10 +514,14 @@ jobs: - name: Download build uses: actions/download-artifact@v4 with: - name: cln-compile-clang-sanitizers.tar.bz2 + name: cln-compile-clang-sanitizers.tar.gz - - name: Unpack build - run: tar -xvjf cln-compile-clang-sanitizers.tar.bz2 + - name: Unpack prebuilt binaries + run: | + git submodule sync && git submodule update --init --recursive + # Make sure source appears older than what we're about to unpack + find . -type f -print0 | xargs -0 touch -d yesterday + tar xaf cln-compile-clang-sanitizers.tar.gz - name: Test run: | @@ -514,10 +562,14 @@ jobs: - name: Download build uses: actions/download-artifact@v4 with: - name: cln-compile-gcc.tar.bz2 - - name: Unpack pre-built CLN + name: cln-compile-gcc.tar.gz + - name: Unpack prebuilt binaries run: | - tar -xaf cln-compile-gcc.tar.bz2 + git submodule sync && git submodule update --init --recursive + # Make sure source appears older than what we're about to unpack + find . -type f -print0 | xargs -0 touch -d yesterday + tar xaf cln-compile-gcc.tar.gz + - name: Test run: | uv run eatmydata make -j $(nproc) check-doc-examples @@ -572,13 +624,14 @@ jobs: - name: Download build uses: actions/download-artifact@v4 with: - name: cln-${{ matrix.CFG }}.tar.bz2 + name: cln-${{ matrix.CFG }}.tar.gz - - name: Unpack pre-built CLN - env: - CFG: ${{ matrix.CFG }} + - name: Unpack prebuilt binaries run: | - tar -xaf cln-${CFG}.tar.bz2 + git submodule sync && git submodule update --init --recursive + # Make sure source appears older than what we're about to unpack + find . -type f -print0 | xargs -0 touch -d yesterday + tar xaf cln-${{ matrix.CFG }}.tar.gz - name: Test env: @@ -608,17 +661,19 @@ jobs: - integration-valgrind - integration-sanitizers - min-btc-support + - postbuild if: ${{ always() }} steps: - name: Complete env: - JOB_NAMES: "INTEGRATION CHECK_UNITS VALGRIND SANITIZERS BTC" + JOB_NAMES: "INTEGRATION CHECK_UNITS VALGRIND SANITIZERS BTC POSTBUILD" INTEGRATION: ${{ needs.integration.result }} CHECK_UNITS: ${{ needs['check-units'].result }} VALGRIND: ${{ needs['integration-valgrind'].result }} SANITIZERS: ${{ needs['integration-sanitizers'].result }} DOCS: ${{ needs['update-docs-examples'].result }} BTC: ${{ needs['min-btc-support'].result }} + POSTBUILD: ${{ needs['postbuild'].result }} run: | failed="" for name in $JOB_NAMES; do diff --git a/Makefile b/Makefile index e995867361e3..36e42b42e304 100644 --- a/Makefile +++ b/Makefile @@ -15,6 +15,7 @@ ECHO := echo SUPPRESS_OUTPUT := endif +CARGO := cargo DISTRO=$(shell lsb_release -is 2>/dev/null || echo unknown)-$(shell lsb_release -rs 2>/dev/null || echo unknown) OS=$(shell uname -s) ARCH=$(shell uname -m) @@ -98,6 +99,7 @@ CCAN_OBJS := \ ccan-bitmap.o \ ccan-bitops.o \ ccan-breakpoint.o \ + ccan-cdump.o \ ccan-closefrom.o \ ccan-crc32c.o \ ccan-crypto-hmac.o \ @@ -223,8 +225,6 @@ CCAN_HEADERS := \ $(CCANDIR)/ccan/typesafe_cb/typesafe_cb.h \ $(CCANDIR)/ccan/utf8/utf8.h -CDUMP_OBJS := ccan-cdump.o ccan-strmap.o - BOLT_GEN := tools/generate-wire.py WIRE_GEN := $(BOLT_GEN) @@ -472,10 +472,6 @@ mkdocs.yml: $(MANPAGES:=.md) ) - -# Don't delete these intermediaries. -.PRECIOUS: $(ALL_GEN_HEADERS) $(ALL_GEN_SOURCES) $(PYTHON_GENERATED) - # Every single object file. ALL_OBJS := $(ALL_C_SOURCES:.c=.o) @@ -680,7 +676,7 @@ TAGS: tags: $(RM) tags; find * -name test -type d -prune -o \( -name '*.[ch]' -o -name '*.py' \) -print0 | xargs -0 ctags --append -ccan/ccan/cdump/tools/cdump-enumstr: ccan/ccan/cdump/tools/cdump-enumstr.o $(CDUMP_OBJS) $(CCAN_OBJS) +ccan/ccan/cdump/tools/cdump-enumstr: ccan/ccan/cdump/tools/cdump-enumstr.o libccan.a ALL_PROGRAMS += ccan/ccan/cdump/tools/cdump-enumstr # Can't add to ALL_OBJS, as that makes a circular dep. @@ -702,6 +698,9 @@ endif header_versions_gen.h: tools/headerversions $(FORCE) @tools/headerversions $@ +# Once you have libccan.a, you don't need these. +.INTERMEDIATE: $(CCAN_OBJS) + # We make a static library, this way linker can discard unused parts. libccan.a: $(CCAN_OBJS) @$(call VERBOSE, "ar $@", $(AR) r $@ $(CCAN_OBJS)) @@ -923,26 +922,20 @@ install-data: installdirs $(MAN1PAGES) $(MAN5PAGES) $(MAN7PAGES) $(MAN8PAGES) $( install: install-program install-data -# Non-artifacts that are needed for testing. These are added to the -# testpack.tar, used to transfer things between builder and tester -# phase. If you get a missing file/executable while testing on CI it -# is likely missing from this variable. -TESTBINS = \ - $(CLN_PLUGIN_EXAMPLES) \ - tests/plugins/test_libplugin \ - tests/plugins/channeld_fakenet \ - tests/plugins/test_selfdisable_after_getmanifest \ - tools/hsmtool +# We exclude most of target/ and external, but we need: +# 1. config files (we only tar up files *newer* than these) +# 2. $(DEFAULT_TARGETS) for rust stuff. +# 3. $(EXTERNAL_LIBS) for prebuild external libraries. +TESTPACK_EXTRAS := \ + config.vars ccan/config.h \ + $(DEFAULT_TARGETS) \ + $(EXTERNAL_LIBS) # The testpack is used in CI to transfer built artefacts between the -# build and the test phase. This is necessary because the fixtures in -# `tests/` explicitly use the binaries built in the current directory -# rather than using `$PATH`, as that may pick up some other installed -# version of `lightningd` leading to bogus results. We bundle up all -# built artefacts here, and will unpack them on the tester (overlaying -# on top of the checked out repo as if we had just built it in place). -testpack.tar.bz2: $(BIN_PROGRAMS) $(PKGLIBEXEC_PROGRAMS) $(PLUGINS) $(PY_PLUGINS) $(MAN1PAGES) $(MAN5PAGES) $(MAN7PAGES) $(MAN8PAGES) $(DOC_DATA) config.vars $(TESTBINS) $(DEVTOOLS) - tar -caf $@ $^ +# build and the test phase. Only useful on a freshly build tree! +# We use Posix format for timestamps with subsecond accuracy. +testpack.tar.gz: all-programs all-fuzz-programs all-test-programs default-targets + (find * -path external -prune -o -path target -prune -o -newer config.vars -print; ls $(TESTPACK_EXTRAS)) | tar --verbatim-files-from -T- -c --format=posix -f - | gzip -5 > $@ uninstall: @$(NORMAL_UNINSTALL) diff --git a/cln-rpc/Makefile b/cln-rpc/Makefile index c43467ab0c94..808dd71cca13 100644 --- a/cln-rpc/Makefile +++ b/cln-rpc/Makefile @@ -9,13 +9,13 @@ DEFAULT_TARGETS += $(CLN_RPC_EXAMPLES) $(CLN_RPC_GENALL) MSGGEN_GENALL += $(CLN_RPC_GENALL) target/${RUST_PROFILE}/examples/cln-rpc-getinfo: ${CLN_RPC_SOURCES} cln-rpc/examples/getinfo.rs - cargo build ${CARGO_OPTS} --example cln-rpc-getinfo + $(CARGO) build ${CARGO_OPTS} --example cln-rpc-getinfo target/${RUST_PROFILE}/examples/cln-plugin-startup: ${CLN_RPC_SOURCES} plugins/examples/cln-plugin-startup.rs - cargo build ${CARGO_OPTS} --example cln-plugin-startup + $(CARGO) build ${CARGO_OPTS} --example cln-plugin-startup target/${RUST_PROFILE}/examples/cln-plugin-reentrant: ${CLN_RPC_SOURCES} plugins/examples/cln-plugin-reentrant.rs - cargo build ${CARGO_OPTS} --example cln-plugin-reentrant + $(CARGO) build ${CARGO_OPTS} --example cln-plugin-reentrant cln-rpc-all: ${CLN_RPC_GENALL} ${CLN_RPC_EXAMPLES} diff --git a/devtools/cc-nobuild b/devtools/cc-nobuild new file mode 100755 index 000000000000..118385581f41 --- /dev/null +++ b/devtools/cc-nobuild @@ -0,0 +1,9 @@ +#! /bin/sh +# Version of CC which only supports -dumpmachine (for external/Makefile), and fails otherwise +set -e + +if [ x"$*" = x"-dumpmachine" ]; then + CC="$(grep ^CC= config.vars | cut -d= -f2-)" + exec ${CC:=cc} "$@" +fi +exit 1 diff --git a/external/Makefile b/external/Makefile index 839cbac2c6d2..4ff220539eb5 100644 --- a/external/Makefile +++ b/external/Makefile @@ -42,6 +42,8 @@ endif EXTERNAL_LDLIBS := -L${TARGET_DIR} $(patsubst lib%.a,-l%,$(notdir $(EXTERNAL_LIBS))) +.PHONY: submodcheck + submodcheck: $(FORCE) ifneq ($(VERSION),) @tools/refresh-submodules.sh $(SUBMODULES) @@ -60,6 +62,8 @@ ifeq ($(DEBUGBUILD),1) WALLY_OPTS=--enable-debug endif +.INTERMEDIATE: $(TARGET_DIR)/libwally-core-build/src/secp256k1/libsecp256k1.la $(TARGET_DIR)/libwally-core-build/src/libwallycore.la + $(TARGET_DIR)/libwally-core-build/src/libwallycore.% $(TARGET_DIR)/libwally-core-build/src/secp256k1/libsecp256k1.%: $(LIBWALLY_HEADERS) $(LIBSECP_HEADERS) cd external/libwally-core && ./tools/autogen.sh mkdir -p ${TARGET_DIR}/libwally-core-build @@ -85,6 +89,8 @@ $(TARGET_DIR)/jsmn-build/jsmn.o: external/jsmn/jsmn.c Makefile $(TARGET_DIR)/libjsmn.a: $(TARGET_DIR)/jsmn-build/jsmn.o $(AR) rc $@ $< +.INTERMEDIATE: $(TARGET_DIR)/jsmn-build/jsmn.o + # Need separate build dir: changes inside submodule make git think it's dirty. $(TARGET_DIR)/libbacktrace.a: external/libbacktrace/backtrace.h mkdir -p $(TARGET_DIR)/libbacktrace-build diff --git a/plugins/Makefile b/plugins/Makefile index 335e3847a3fe..463fdd4bee9b 100644 --- a/plugins/Makefile +++ b/plugins/Makefile @@ -241,7 +241,7 @@ plugins/list_of_builtin_plugins_gen.h: plugins/Makefile Makefile config.vars @$(call VERBOSE,GEN $@,echo "static const char *list_of_builtin_plugins[] = { $(PLUGIN_BASES:%=\"%\",) NULL };" > $@) $(RUST_TARGET_DIR)/examples/cln-subscribe-wildcard: ${CLN_PLUGIN_SRC} plugins/examples/cln-subscribe-wildcard.rs - cargo build ${CARGO_OPTS} --example cln-subscribe-wildcard + $(CARGO) build ${CARGO_OPTS} --example cln-subscribe-wildcard CLN_PLUGIN_EXAMPLES := \ $(RUST_TARGET_DIR)/examples/cln-plugin-startup \ @@ -257,17 +257,17 @@ CLN_WSS_PROXY_PLUGIN_SRC = $(shell find plugins/wss-proxy-plugin/src -name "*.rs CLN_BIP353_PLUGIN_SRC = $(shell find plugins/bip353-plugin/src -name "*.rs") $(RUST_TARGET_DIR)/cln-grpc: ${CLN_PLUGIN_SRC} ${CLN_GRPC_PLUGIN_SRC} $(MSGGEN_GENALL) $(MSGGEN_GEN_ALL) - cargo build ${CARGO_OPTS} --bin cln-grpc + $(CARGO) build ${CARGO_OPTS} --bin cln-grpc $(RUST_TARGET_DIR)/clnrest: ${CLN_REST_PLUGIN_SRC} - cargo build ${CARGO_OPTS} --bin clnrest + $(CARGO) build ${CARGO_OPTS} --bin clnrest $(RUST_TARGET_DIR)/cln-lsps-client: ${CLN_LSPS_PLUGIN_SRC} - cargo build ${CARGO_OPTS} --bin cln-lsps-client + $(CARGO) build ${CARGO_OPTS} --bin cln-lsps-client $(RUST_TARGET_DIR)/cln-lsps-service: ${CLN_LSPS_PLUGIN_SRC} - cargo build ${CARGO_OPTS} --bin cln-lsps-service + $(CARGO) build ${CARGO_OPTS} --bin cln-lsps-service $(RUST_TARGET_DIR)/wss-proxy: ${CLN_WSS_PROXY_PLUGIN_SRC} - cargo build ${CARGO_OPTS} --bin wss-proxy + $(CARGO) build ${CARGO_OPTS} --bin wss-proxy $(RUST_TARGET_DIR)/cln-bip353: ${CLN_BIP353_PLUGIN_SRC} - cargo build ${CARGO_OPTS} --bin cln-bip353 + $(CARGO) build ${CARGO_OPTS} --bin cln-bip353 ifneq ($(RUST),0) include plugins/rest-plugin/Makefile