From daece1c50597e603f4f6027bf93240a0771eb5d3 Mon Sep 17 00:00:00 2001 From: strophy Date: Tue, 8 Mar 2022 15:58:07 +1100 Subject: [PATCH] chore: ff changes from other branches --- .github/workflows/release_alpha.yml | 116 ++++++--- .../deploy/Dockerfile.GitHubActions.Gitian | 46 ++++ contrib/gitian-build.py | 12 +- .../gitian-linux-64-only.yml | 226 ++++++++++++++++++ 4 files changed, 362 insertions(+), 38 deletions(-) create mode 100644 contrib/containers/deploy/Dockerfile.GitHubActions.Gitian create mode 100755 contrib/gitian-descriptors/gitian-linux-64-only.yml diff --git a/.github/workflows/release_alpha.yml b/.github/workflows/release_alpha.yml index 7dc2c92eddf0a..4bf3708db059d 100644 --- a/.github/workflows/release_alpha.yml +++ b/.github/workflows/release_alpha.yml @@ -1,4 +1,4 @@ -name: Build alpha release for Docker Hub +name: Build and push image on: workflow_dispatch: @@ -12,20 +12,42 @@ jobs: name: Run Gitian build runs-on: self-hosted steps: - - name: Set up QEMU to run multi-arch builds - uses: docker/setup-qemu-action@v1 + - name: Checkout + uses: actions/checkout@v2 + with: + path: dash + + - name: Checkout Gitian builder + uses: actions/checkout@v2 + with: + repository: devrandom/gitian-builder + path: gitian-builder - - name: Set up Docker BuildX - id: buildx - uses: docker/setup-buildx-action@v1 + - name: Checkout detached sigs + uses: actions/checkout@v2 with: - install: true + repository: dashpay/dash-detached-sigs + path: dash-detached-sigs - - name: Login to DockerHub - uses: docker/login-action@v1 + - name: Checkout gitian sigs + uses: actions/checkout@v2 with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} + repository: dashpay/gitian.sigs + path: gitian.sigs + + - name: Import GPG key + id: import_gpg + uses: crazy-max/ghaction-import-gpg@v4 + with: + gpg_private_key: ${{ secrets.GPG_PRIVATE_KEY }} + passphrase: ${{ secrets.GPG_PASSPHRASE }} + + - name: GPG user IDs + run: | + echo "fingerprint: ${{ steps.import_gpg.outputs.fingerprint }}" + echo "keyid: ${{ steps.import_gpg.outputs.keyid }}" + echo "name: ${{ steps.import_gpg.outputs.name }}" + echo "email: ${{ steps.import_gpg.outputs.email }}" - name: Checkout uses: actions/checkout@v2 @@ -64,30 +86,62 @@ jobs: with: images: strophy/dashd tags: | - type=semver,pattern={{version}},value=${{github.event.inputs.tag}} + type=semver,pattern={{version}},value=${{ github.event.inputs.tag }} type=raw,value=latest-dev flavor: | latest=false + - name: Install apt-cacher-ng + run: sudo DEBIAN_FRONTEND=noninteractive apt-get install -yq apt-cacher-ng + + - name: Get branch name + id: branch-name + uses: tj-actions/branch-names@v5 + - name: Prepare Gitian - run: dash/contrib/gitian-build.py --setup "Leon White" "${{steps.docker_meta.outputs.tags}}" + run: dash/contrib/gitian-build.py --setup "strophy" "${{ steps.branch-name.outputs.current_branch }}" + + - uses: pat-s/always-upload-cache@v2 + id: ccache + with: + path: /opt/actions-runner/_work/dash/dash/gitian-builder/cache/dash-linux-0.17 + key: ${{ runner.os }}-ccache-${{ hashFiles('**/ccache.tar') }} + restore-keys: ${{ runner.os }}-ccache- - name: Run Gitian build - run: dash/contrib/gitian-build.py -b -n -j $(nproc) -m `awk '/^Mem/ {print $7}' <(free -m)` "Leon White" "${{steps.docker_meta.outputs.tags}}" - - - name: Setup tmate session - uses: mxschmitt/action-tmate@v3 - - - # - name: Build and push Docker image - # id: docker_build - # uses: docker/build-push-action@v2 - # with: - # file: ./docker/Dockerfile.GitHubActions.Dispatch - # push: true - # tags: ${{ steps.docker_meta.outputs.tags }} - # labels: ${{ steps.docker_meta.outputs.labels }} - # build-args: TAG=${{ steps.docker_meta.outputs.version }} - # cache-from: type=gha - # cache-to: type=gha,mode=max - # platforms: linux/amd64,linux/arm64 + run: | + export GPG_TTY=$(tty) + dash/contrib/gitian-build.py -c -b -n -o l \ + -u https://github.com/strophy/dash \ + -j $(nproc) -m `awk '/^Mem/ {print $7}' <(free -m)` \ + strophy ${{ steps.branch-name.outputs.current_branch }} + + - name: Set up QEMU to run multi-arch builds + uses: docker/setup-qemu-action@v1 + + - name: Set up Docker BuildX + id: buildx + uses: docker/setup-buildx-action@v1 + with: + version: v0.7.0 + install: true + + - name: Login to DockerHub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Build and push Docker image + id: docker_build + uses: docker/build-push-action@v2 + with: + context: ${{ github.workspace }} + file: ./dash/contrib/containers/deploy/Dockerfile.GitHubActions.Gitian + push: true + tags: ${{ steps.docker_meta.outputs.tags }} + labels: ${{ steps.docker_meta.outputs.labels }} + platforms: linux/amd64,linux/arm64 + build-args: | + TAG=${{ steps.docker_meta.outputs.version }} + BRANCH=${{ steps.branch-name.outputs.current_branch }} diff --git a/contrib/containers/deploy/Dockerfile.GitHubActions.Gitian b/contrib/containers/deploy/Dockerfile.GitHubActions.Gitian new file mode 100644 index 0000000000000..57fa0c7213021 --- /dev/null +++ b/contrib/containers/deploy/Dockerfile.GitHubActions.Gitian @@ -0,0 +1,46 @@ +# syntax=docker/dockerfile:1.3 +FROM ubuntu:focal +LABEL maintainer="Dash Developers " +LABEL description="Dockerised DashCore" + +ARG USER_ID +ARG GROUP_ID +ARG TAG +ARG BRANCH + +ENV HOME /dash + +# add user with specified (or default) user/group ids +ENV USER_ID ${USER_ID:-1000} +ENV GROUP_ID ${GROUP_ID:-1000} +RUN groupadd -g ${GROUP_ID} dash && \ + useradd -u ${USER_ID} -g dash -s /bin/bash -m -d /dash dash && \ + mkdir /dash/.dashcore && \ + chown dash:dash -R /dash + +RUN apt-get update && \ + apt-get -y install --no-install-recommends \ + wget \ + ca-certificates \ + && rm -rf /var/lib/apt/lists/* + +COPY dashcore-binaries/${BRANCH}/dashcore* /dash + +RUN mach=$(uname -m) \ + && case $mach in aarch64) arch="aarch64-linux-gnu"; ;; x86_64) arch="x86_64-linux-gnu"; ;; *) echo "ERROR: Machine type $mach not supported."; ;; esac \ + && cd /dash \ + && tar xvzf dashcore*$arch.tar.gz \ + && echo $(ls -1 /dash) \ + && cp dashcore-0.17.0/bin/* /usr/local/bin \ + && rm -rf dash* + +USER dash + +VOLUME ["/dash"] + +COPY dash/contrib/containers/deploy/docker-entrypoint.sh /docker-entrypoint.sh +ENTRYPOINT ["/docker-entrypoint.sh"] + +EXPOSE 9998 9999 19998 19999 + +WORKDIR /dash diff --git a/contrib/gitian-build.py b/contrib/gitian-build.py index 4fbd7416bcd1d..a16c8057c7d63 100755 --- a/contrib/gitian-build.py +++ b/contrib/gitian-build.py @@ -36,7 +36,8 @@ def setup(): if args.docker: make_image_prog += ['--docker'] elif args.lxc: - make_image_prog += ['--lxc', '--disksize', '13000'] + make_image_prog += ['--lxc'] + print('\nAbout to run ', make_image_prog ) subprocess.check_call(make_image_prog) os.chdir(workdir) if args.is_focal and not args.kvm and not args.docker: @@ -58,8 +59,8 @@ def build(): if args.linux: print('\nCompiling ' + args.version + ' Linux') - subprocess.check_call(['bin/gbuild', '-j', args.jobs, '-m', args.memory, '--commit', 'dash='+args.commit, '--url', 'dash='+args.url, '../dash/contrib/gitian-descriptors/gitian-linux.yml']) - subprocess.check_call(['bin/gsign', '-p', args.sign_prog, '--signer', args.signer, '--release', args.version+'-linux', '--destination', '../gitian.sigs/', '../dash/contrib/gitian-descriptors/gitian-linux.yml']) + subprocess.check_call(['bin/gbuild', '-j', args.jobs, '-m', args.memory, '--commit', 'dash='+args.commit, '--url', 'dash='+args.url, '../dash/contrib/gitian-descriptors/gitian-linux-64-only.yml']) + subprocess.check_call(['bin/gsign', '-p', args.sign_prog, '--signer', args.signer, '--release', args.version+'-linux', '--destination', '../gitian.sigs/', '../dash/contrib/gitian-descriptors/gitian-linux-64-only.yml']) subprocess.check_call('mv build/out/dashcore-*.tar.gz build/out/src/dashcore-*.tar.gz ../dashcore-binaries/'+args.version, shell=True) if args.windows: @@ -123,10 +124,7 @@ def verify(): os.chdir('gitian-builder') print('\nVerifying v'+args.version+' Linux\n') - if subprocess.call(['bin/gverify', '-v', '-d', '../gitian.sigs/', '-r', args.version+'-linux', '../dash/contrib/gitian-descriptors/gitian-linux.yml']): - print('Verifying v'+args.version+' Linux FAILED\n') - rc = 1 - + subprocess.call(['bin/gverify', '-v', '-d', '../gitian.sigs/', '-r', args.version+'-linux', '../dash/contrib/gitian-descriptors/gitian-linux-64-only.yml']) print('\nVerifying v'+args.version+' Windows\n') if subprocess.call(['bin/gverify', '-v', '-d', '../gitian.sigs/', '-r', args.version+'-win-unsigned', '../dash/contrib/gitian-descriptors/gitian-win.yml']): print('Verifying v'+args.version+' Windows FAILED\n') diff --git a/contrib/gitian-descriptors/gitian-linux-64-only.yml b/contrib/gitian-descriptors/gitian-linux-64-only.yml new file mode 100755 index 0000000000000..10767b77a3e33 --- /dev/null +++ b/contrib/gitian-descriptors/gitian-linux-64-only.yml @@ -0,0 +1,226 @@ +--- +name: "dash-linux-0.17" +enable_cache: true +suites: +- "bionic" +architectures: +- "amd64" +packages: +- "curl" +- "g++-aarch64-linux-gnu" +- "g++-7-aarch64-linux-gnu" +- "gcc-7-aarch64-linux-gnu" +- "binutils-aarch64-linux-gnu" +- "g++-7-multilib" +- "gcc-7-multilib" +- "binutils-gold" +- "git" +- "pkg-config" +- "autoconf" +- "libtool" +- "automake" +- "faketime" +- "bsdmainutils" +- "ca-certificates" +- "python" +- "python3" +- "libxkbcommon0" +- "ccache" +remotes: +- "url": "https://github.com/dashpay/dash.git" + "dir": "dash" +files: [] +script: | + + WRAP_DIR=$HOME/wrapped + HOSTS="x86_64-linux-gnu aarch64-linux-gnu" + CONFIGFLAGS="--enable-glibc-back-compat --enable-reduce-exports --disable-bench --disable-gui-tests --enable-crash-hooks" + FAKETIME_HOST_PROGS="" + FAKETIME_PROGS="date ar ranlib nm" + HOST_CFLAGS="-O2 -g" + HOST_CXXFLAGS="-O2 -g" + HOST_LDFLAGS=-static-libstdc++ + + export QT_RCC_TEST=1 + export QT_RCC_SOURCE_DATE_OVERRIDE=1 + export GZIP="-9n" + export TAR_OPTIONS="--mtime="$REFERENCE_DATE\\\ $REFERENCE_TIME"" + export TZ="UTC" + export BUILD_DIR=`pwd` + mkdir -p ${WRAP_DIR} + if test -n "$GBUILD_CACHE_ENABLED"; then + export SOURCES_PATH=${GBUILD_COMMON_CACHE} + export BASE_CACHE=${GBUILD_PACKAGE_CACHE}/depends + mkdir -p ${BASE_CACHE} ${SOURCES_PATH} + + # Setup ccache to use correct cache directories and fix the compiler check of ccache + CONFIGFLAGS="${CONFIGFLAGS} --enable-ccache" + export CCACHE_DIR=${GBUILD_PACKAGE_CACHE}/ccache + # As we later wrap the gcc binaries, this is fast + export CCACHE_COMPILERCHECK="content" + if [ -f ${GBUILD_PACKAGE_CACHE}/ccache.tar ]; then + pushd ${GBUILD_PACKAGE_CACHE} + tar xf ccache.tar + rm ccache.tar + popd + fi + # instead of compressing ccache.tar, we let ccache handle it by itself + # Otherwise we end up uncompressing/compressing a lot of cache files which we actually never use + export CCACHE_COMPRESS=1 + else + CONFIGFLAGS="${CONFIGFLAGS} --disable-ccache" + fi + + # We include the GCC version in all wrappers so that ccache can detect compiler upgrades when hashing the wrappers + GCCVERSION=`gcc --version | head -1` + + function create_global_faketime_wrappers { + for prog in ${FAKETIME_PROGS}; do + echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${prog} + echo "# GCCVERSION=${GCCVERSION}" >> ${WRAP_DIR}/${prog} + echo "REAL=\`which -a ${prog} | grep -v ${WRAP_DIR}/${prog} | head -1\`" >> ${WRAP_DIR}/${prog} + echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${prog} + echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${prog} + echo "\$REAL \$@" >> $WRAP_DIR/${prog} + chmod +x ${WRAP_DIR}/${prog} + touch -d "${REFERENCE_DATETIME}" ${WRAP_DIR}/${prog} + done + } + + function create_per-host_faketime_wrappers { + for i in $HOSTS; do + for prog in ${FAKETIME_HOST_PROGS}; do + echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${i}-${prog} + echo "# GCCVERSION=${GCCVERSION}" >> ${WRAP_DIR}/${i}-${prog} + echo "REAL=\`which -a ${i}-${prog} | grep -v ${WRAP_DIR}/${i}-${prog} | head -1\`" >> ${WRAP_DIR}/${i}-${prog} + echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${i}-${prog} + echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${i}-${prog} + echo "\$REAL \$@" >> $WRAP_DIR/${i}-${prog} + chmod +x ${WRAP_DIR}/${i}-${prog} + touch -d "${REFERENCE_DATETIME}" ${WRAP_DIR}/${i}-${prog} + done + done + } + + # Faketime for depends so intermediate results are comparable + export PATH_orig=${PATH} + create_global_faketime_wrappers "2000-01-01 12:00:00" + create_per-host_faketime_wrappers "2000-01-01 12:00:00" + export PATH=${WRAP_DIR}:${PATH} + + EXTRA_INCLUDES_BASE=$WRAP_DIR/extra_includes + mkdir -p $EXTRA_INCLUDES_BASE + + # x86 needs /usr/include/i386-linux-gnu/asm pointed to /usr/include/x86_64-linux-gnu/asm, + # but we can't write there. Instead, create a link here and force it to be included in the + # search paths by wrapping gcc/g++. + + mkdir -p $EXTRA_INCLUDES_BASE/i686-pc-linux-gnu + rm -f $WRAP_DIR/extra_includes/i686-pc-linux-gnu/asm + ln -s /usr/include/x86_64-linux-gnu/asm $EXTRA_INCLUDES_BASE/i686-pc-linux-gnu/asm + + for prog in gcc g++; do + rm -f ${WRAP_DIR}/${prog} + cat << EOF > ${WRAP_DIR}/${prog} + #!/usr/bin/env bash + # GCCVERSION=${GCCVERSION} + REAL="`which -a ${prog} | grep -v ${WRAP_DIR}/${prog} | head -1`" + for var in "\$@" + do + if [ "\$var" = "-m32" ]; then + export C_INCLUDE_PATH="$EXTRA_INCLUDES_BASE/i686-pc-linux-gnu" + export CPLUS_INCLUDE_PATH="$EXTRA_INCLUDES_BASE/i686-pc-linux-gnu" + break + fi + done + \$REAL \$@ + EOF + chmod +x ${WRAP_DIR}/${prog} + touch -d "${REFERENCE_DATETIME}" ${WRAP_DIR}/${prog} + done + + cd dash + BASEPREFIX=`pwd`/depends + # Build dependencies for each host + for i in $HOSTS; do + EXTRA_INCLUDES="$EXTRA_INCLUDES_BASE/$i" + if [ -d "$EXTRA_INCLUDES" ]; then + export HOST_ID_SALT="$EXTRA_INCLUDES" + fi + make ${MAKEOPTS} -C ${BASEPREFIX} HOST="${i}" + unset HOST_ID_SALT + done + + # Faketime for binaries + export PATH=${PATH_orig} + create_global_faketime_wrappers "${REFERENCE_DATETIME}" + create_per-host_faketime_wrappers "${REFERENCE_DATETIME}" + export PATH=${WRAP_DIR}:${PATH} + + # Create the release tarball using (arbitrarily) the first host + ./autogen.sh + CONFIG_SITE=${BASEPREFIX}/`echo "${HOSTS}" | awk '{print $1;}'`/share/config.site ./configure --prefix=/ + make dist + SOURCEDIST=`echo dashcore-*.tar.gz` + DISTNAME=`echo ${SOURCEDIST} | sed 's/.tar.*//'` + # Correct tar file order + mkdir -p temp + pushd temp + tar xf ../$SOURCEDIST + find dashcore-* | sort | tar --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ../$SOURCEDIST + popd + + # Workaround for tarball not building with the bare tag version (prep) + make -C src obj/build.h + + ORIGPATH="$PATH" + # Extract the release tarball into a dir for each host and build + for i in ${HOSTS}; do + export PATH=${BASEPREFIX}/${i}/native/bin:${ORIGPATH} + mkdir -p distsrc-${i} + cd distsrc-${i} + INSTALLPATH=`pwd`/installed/${DISTNAME} + mkdir -p ${INSTALLPATH} + tar --strip-components=1 -xf ../$SOURCEDIST + + # Workaround for tarball not building with the bare tag version + echo '#!/bin/true' >share/genbuild.sh + mkdir src/obj + cp ../src/obj/build.h src/obj/ + + CONFIG_SITE=${BASEPREFIX}/${i}/share/config.site ./configure --prefix=/ --disable-maintainer-mode --disable-dependency-tracking ${CONFIGFLAGS} CFLAGS="${HOST_CFLAGS}" CXXFLAGS="${HOST_CXXFLAGS}" LDFLAGS="${HOST_LDFLAGS}" + make ${MAKEOPTS} + make ${MAKEOPTS} -C src check-security + + #TODO: This is a quick hack that disables symbol checking for arm. + # Instead, we should investigate why these are popping up. + # For aarch64, we'll need to bump up the min GLIBC version, as the abi + # support wasn't introduced until 2.17. + case $i in + aarch64-*) : ;; + arm-*) : ;; + *) make ${MAKEOPTS} -C src check-symbols ;; + esac + + make install DESTDIR=${INSTALLPATH} + cd installed + find . -name "lib*.la" -delete + find . -name "lib*.a" -delete + rm -rf ${DISTNAME}/lib/pkgconfig + find ${DISTNAME}/bin -type f -executable -exec ../contrib/devtools/split-debug.sh {} {} {}.dbg \; + find ${DISTNAME}/lib -type f -exec ../contrib/devtools/split-debug.sh {} {} {}.dbg \; + find ${DISTNAME} -not -name "*.dbg" | sort | tar --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}-${i}.tar.gz + find ${DISTNAME} -name "*.dbg" | sort | tar --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}-${i}-debug.tar.gz + cd ../../ + rm -rf distsrc-${i} + done + mkdir -p $OUTDIR/src + mv $SOURCEDIST $OUTDIR/src + + # Compress ccache (otherwise the assert file will get too huge) + if [ "$CCACHE_DIR" != "" ]; then + pushd ${GBUILD_PACKAGE_CACHE} + tar cf ccache.tar ccache + rm -rf ccache + popd + fi