From 09517be9bb2428a3c2d7089d355aa1fc6aba3ccc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emilio=20L=C3=B3pez?= Date: Tue, 29 Aug 2023 23:11:29 -0300 Subject: [PATCH 1/7] flake.nix: static build support Adds a "redistributable" flavor of Echidna that is fully static on Linux, and mostly static on macOS. --- flake.nix | 74 ++++++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 71 insertions(+), 3 deletions(-) diff --git a/flake.nix b/flake.nix index 0a740d00b..88f0674cd 100644 --- a/flake.nix +++ b/flake.nix @@ -15,7 +15,10 @@ outputs = { self, nixpkgs, flake-utils, nix-bundle-exe, ... }: flake-utils.lib.eachDefaultSystem (system: let - pkgs = nixpkgs.legacyPackages.${system}; + systemPkgs = nixpkgs.legacyPackages.${system}; + # prefer musl on Linux, static glibc + threading does not work properly + # TODO: maybe only override it for echidna-redistributable? + pkgs = if systemPkgs.stdenv.hostPlatform.isLinux then systemPkgs.pkgsMusl else systemPkgs; # this is not perfect for development as it hardcodes solc to 0.5.7, test suite runs fine though # would be great to integrate solc-select to be more flexible, improve this in future solc = pkgs.stdenv.mkDerivation { @@ -38,6 +41,12 @@ ''; }; + secp256k1-static = stripDylib (pkgs.secp256k1.overrideAttrs (attrs: { + configureFlags = attrs.configureFlags ++ [ "--enable-static" ]; + })); + + ncurses-static = pkgs.ncurses.override { enableStatic = true; }; + hevm = pkgs.haskell.lib.dontCheck ( pkgs.haskellPackages.callCabal2nix "hevm" (pkgs.fetchFromGitHub { owner = "elopez"; @@ -55,12 +64,71 @@ (haskell.lib.compose.addTestToolDepends [ haskellPackages.hpack slither-analyzer solc ]) (haskell.lib.compose.disableCabalFlag "static") ]); + + echidna-static = with pkgs; lib.pipe + echidna + [ + (haskell.lib.compose.appendConfigureFlags + ([ + "--extra-lib-dirs=${stripDylib (gmp.override { withStatic = true; })}/lib" + "--extra-lib-dirs=${stripDylib secp256k1-static}/lib" + "--extra-lib-dirs=${stripDylib (libff.override { enableStatic = true; })}/lib" + "--extra-lib-dirs=${zlib.static}/lib" + "--extra-lib-dirs=${stripDylib (libffi.overrideAttrs (_: { dontDisableStatic = true; }))}/lib" + "--extra-lib-dirs=${stripDylib (ncurses-static)}/lib" + ] ++ (if stdenv.hostPlatform.isDarwin then [ + "--extra-lib-dirs=${stripDylib (libiconv.override { enableStatic = true; })}/lib" + ] else []))) + (haskell.lib.compose.enableCabalFlag "static") + ]; + + # "static" binary for distribution + # on linux this is actually a real fully static binary + # on macos this has everything except libcxx and libsystem + # statically linked. we can be confident that these two will always + # be provided in a well known location by macos itself. + echidnaRedistributable = let + grep = "${pkgs.gnugrep}/bin/grep"; + perl = "${pkgs.perl}/bin/perl"; + otool = "${pkgs.darwin.binutils.bintools}/bin/otool"; + install_name_tool = "${pkgs.darwin.binutils.bintools}/bin/install_name_tool"; + codesign_allocate = "${pkgs.darwin.binutils.bintools}/bin/codesign_allocate"; + codesign = "${pkgs.darwin.sigtool}/bin/codesign"; + in if pkgs.stdenv.isLinux + then pkgs.haskell.lib.dontCheck echidna-static + else pkgs.runCommand "echidna-stripNixRefs" {} '' + mkdir -p $out/bin + cp ${pkgs.haskell.lib.dontCheck echidna-static}/bin/echidna $out/bin/ + # get the list of dynamic libs from otool and tidy the output + libs=$(${otool} -L $out/bin/echidna | tail -n +2 | sed 's/^[[:space:]]*//' | cut -d' ' -f1) + # get the path for libcxx + cxx=$(echo "$libs" | ${grep} '^/nix/store/.*-libcxx') + # rewrite /nix/... library paths to point to /usr/lib + chmod 777 $out/bin/echidna + ${install_name_tool} -change "$cxx" /usr/lib/libc++.1.dylib $out/bin/echidna + # fix TERMINFO path in ncurses + ${perl} -i -pe 's#(${ncurses-static}/share/terminfo)#"/usr/share/terminfo" . "\x0" x (length($1) - 19)#e' $out/bin/echidna + # re-sign binary + CODESIGN_ALLOCATE=${codesign_allocate} ${codesign} -f -s - $out/bin/echidna + chmod 555 $out/bin/echidna + ''; + + # if we pass a library folder to ghc via --extra-lib-dirs that contains + # only .a files, then ghc will link that library statically instead of + # dynamically (even if --enable-executable-static is not passed to cabal). + # we use this trick to force static linking of some libraries on macos. + stripDylib = drv : pkgs.runCommand "${drv.name}-strip-dylibs" {} '' + mkdir -p $out + mkdir -p $out/lib + cp -r ${drv}/* $out/ + rm -rf $out/**/*.dylib + ''; + in rec { packages.echidna = echidna; packages.default = echidna; - packages.echidna-bundle = - pkgs.callPackage nix-bundle-exe {} (pkgs.haskell.lib.dontCheck echidna); + packages.echidna-redistributable = echidnaRedistributable; devShell = with pkgs; haskellPackages.shellFor { From aa48485ec5fe02b1a1dc86a670a4bcc925b0e09f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emilio=20L=C3=B3pez?= Date: Thu, 26 Oct 2023 12:33:27 -0300 Subject: [PATCH 2/7] ci: add Nix & release workflow Replaces previous Nix workflow --- .github/workflows/nix.yml | 24 -------- .github/workflows/release.yml | 100 ++++++++++++++++++++++++++++++++++ 2 files changed, 100 insertions(+), 24 deletions(-) delete mode 100644 .github/workflows/nix.yml create mode 100644 .github/workflows/release.yml diff --git a/.github/workflows/nix.yml b/.github/workflows/nix.yml deleted file mode 100644 index 4f15517e2..000000000 --- a/.github/workflows/nix.yml +++ /dev/null @@ -1,24 +0,0 @@ -name: Nix - -on: - push: - branches: - - master - pull_request: - branches: - - master - -jobs: - test: - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: - - ubuntu-latest - - macos-latest - steps: - - uses: actions/checkout@v4 - - uses: cachix/install-nix-action@v23 - with: - nix_path: nixpkgs=channel:nixos-unstable - - run: nix-build diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 000000000..7af42899f --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,100 @@ +name: "Nix and release" +on: + push: + branches: + - master + tags: + - "v*" + pull_request: + branches: + - master + +jobs: + nixBuild: + name: Build ${{ matrix.name }} binary + runs-on: ${{ matrix.os }} + permissions: + contents: read + outputs: + version: ${{ steps.version.outputs.version }} + strategy: + matrix: + include: + - os: ubuntu-latest + name: Linux (x86_64) + tuple: x86_64-linux + - os: macos-latest + name: macOS (x86_64) + tuple: x86_64-macos + - os: macos-latest-xlarge + name: macOS (aarch64) + tuple: aarch64-macos + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Install Nix + uses: DeterminateSystems/nix-installer-action@v6 + + - name: Configure Nix cache + if: runner.arch == 'X64' + # Unfortunately the action does not work on ARM runners + uses: DeterminateSystems/magic-nix-cache-action@v2 + + - name: Obtain version number + id: version + run: | + if [[ "$GIT_REF" =~ ^refs/tags/v.* ]]; then + echo "version=$(echo "$GIT_REF" | sed 's#^refs/tags/v##')" >> "$GITHUB_OUTPUT" + else + echo "version=HEAD-$(echo "$GIT_SHA" | cut -c1-7)" >> "$GITHUB_OUTPUT" + fi + env: + GIT_REF: ${{ github.ref }} + GIT_SHA: ${{ github.sha }} + + - name: Build dynamic echidna + run: | + nix build .#echidna + + - name: Build redistributable echidna + run: | + nix build .#echidna-redistributable --out-link redistributable + tar -czf "echidna-${{ steps.version.outputs.version }}-${{ matrix.tuple }}.tar.gz" -C ./redistributable/bin/ echidna + + - name: Upload artifact + uses: actions/upload-artifact@v3 + with: + name: echidna-redistributable + path: echidna-${{ steps.version.outputs.version }}-${{ matrix.tuple }}.tar.gz + + release: + name: Create release + needs: [nixBuild] + if: startsWith(github.ref, 'refs/tags/') + runs-on: ubuntu-latest + permissions: + contents: write + id-token: write + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Download binaries + uses: actions/download-artifact@v3 + with: + name: echidna-redistributable + + - name: Sign binaries + uses: sigstore/gh-action-sigstore-python@v2.1.0 + with: + inputs: ./echidna-*.tar.gz + + - name: Create GitHub release and upload binaries + uses: softprops/action-gh-release@v0.1.15 + with: + draft: true + name: "Echidna ${{ needs.nixBuild.outputs.version }}" + files: | + ./echidna-*.tar.gz + ./echidna-*.tar.gz.sigstore From b734792a6c072e33deb2243de9cfe4bb4a963c58 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emilio=20L=C3=B3pez?= Date: Thu, 26 Oct 2023 20:19:15 -0300 Subject: [PATCH 3/7] ci: release: add job timeouts --- .github/workflows/release.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 7af42899f..c17492cca 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -12,6 +12,7 @@ on: jobs: nixBuild: name: Build ${{ matrix.name }} binary + timeout-minutes: ${{ matrix.timeout || 30 }} runs-on: ${{ matrix.os }} permissions: contents: read @@ -23,6 +24,7 @@ jobs: - os: ubuntu-latest name: Linux (x86_64) tuple: x86_64-linux + timeout: 180 - os: macos-latest name: macOS (x86_64) tuple: x86_64-macos @@ -70,6 +72,7 @@ jobs: release: name: Create release + timeout-minutes: 10 needs: [nixBuild] if: startsWith(github.ref, 'refs/tags/') runs-on: ubuntu-latest From f6886d9b62716806fba785202a1dc4bd4708b3bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emilio=20L=C3=B3pez?= Date: Tue, 31 Oct 2023 09:55:57 -0300 Subject: [PATCH 4/7] ci: release: configure Cachix --- .github/workflows/release.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c17492cca..fad150b10 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -38,10 +38,18 @@ jobs: - name: Install Nix uses: DeterminateSystems/nix-installer-action@v6 + - name: Configure Cachix + uses: cachix/cachix-action@v12 + with: + name: trailofbits + authToken: ${{ secrets.CACHIX_AUTH_TOKEN }} + - name: Configure Nix cache if: runner.arch == 'X64' # Unfortunately the action does not work on ARM runners uses: DeterminateSystems/magic-nix-cache-action@v2 + with: + upstream-cache: https://trailofbits.cachix.org - name: Obtain version number id: version From 693f3c77656fe94f383255808df3bf013275608c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emilio=20L=C3=B3pez?= Date: Tue, 31 Oct 2023 13:24:19 -0300 Subject: [PATCH 5/7] README: update echidna-bundle references to echidna-redistributable --- README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 48eed773c..7a747eb13 100644 --- a/README.md +++ b/README.md @@ -272,10 +272,11 @@ $ nix run github:crytic/echidna/v2.1.1 # specific ref (tag/branch/commit) ``` To build a standalone release for non-Nix macOS systems, the following will -bundle Echidna and all linked dylibs: +build Echidna in a mostly static binary. This can also be used on Linux systems +to produce a fully static binary. ```sh -$ nix build .#echidna-bundle +$ nix build .#echidna-redistributable ``` Nix will automatically install all the dependencies required for development From a94e55a580aaa38353e22c3a055d2026c1466c45 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emilio=20L=C3=B3pez?= Date: Tue, 31 Oct 2023 13:25:17 -0300 Subject: [PATCH 6/7] Fix TERMINFO path for Nix release builds on Linux ncurses in Nix is built with a TERMINFO path that references `/nix`. This causes the binaries fail when ran on non-nix systems, unless TERMINFO=/usr/share/terminfo is exported. This patches the binaries to use a more sensible default TERMINFO path. See also commit f76a7f4a --- flake.nix | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/flake.nix b/flake.nix index 88f0674cd..d017e8f0c 100644 --- a/flake.nix +++ b/flake.nix @@ -95,8 +95,13 @@ codesign_allocate = "${pkgs.darwin.binutils.bintools}/bin/codesign_allocate"; codesign = "${pkgs.darwin.sigtool}/bin/codesign"; in if pkgs.stdenv.isLinux - then pkgs.haskell.lib.dontCheck echidna-static - else pkgs.runCommand "echidna-stripNixRefs" {} '' + then pkgs.runCommand "echidna-stripNixRefs" {} '' + mkdir -p $out/bin + cp ${pkgs.haskell.lib.dontCheck echidna-static}/bin/echidna $out/bin/ + # fix TERMINFO path in ncurses + ${perl} -i -pe 's#(${ncurses-static}/share/terminfo)#"/usr/share/terminfo" . "\x0" x (length($1) - 19)#e' $out/bin/echidna + chmod 555 $out/bin/echidna + '' else pkgs.runCommand "echidna-stripNixRefs" {} '' mkdir -p $out/bin cp ${pkgs.haskell.lib.dontCheck echidna-static}/bin/echidna $out/bin/ # get the list of dynamic libs from otool and tidy the output From 77d75eb4ea0a8724bb86dd36141e139aa9e701a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emilio=20L=C3=B3pez?= Date: Tue, 31 Oct 2023 13:41:00 -0300 Subject: [PATCH 7/7] flake.nix: remove redundant stripping --- flake.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/flake.nix b/flake.nix index d017e8f0c..4206fa77a 100644 --- a/flake.nix +++ b/flake.nix @@ -41,9 +41,9 @@ ''; }; - secp256k1-static = stripDylib (pkgs.secp256k1.overrideAttrs (attrs: { + secp256k1-static = pkgs.secp256k1.overrideAttrs (attrs: { configureFlags = attrs.configureFlags ++ [ "--enable-static" ]; - })); + }); ncurses-static = pkgs.ncurses.override { enableStatic = true; };