From 340090e1c65574ca5ceb2687c3ab399791e4bdbd Mon Sep 17 00:00:00 2001 From: Aaron Dewes Date: Tue, 30 Jun 2020 14:45:02 +0200 Subject: [PATCH] Improve build speed --- .travis.yml | 3 ++ docker/arm64/Dockerfile | 31 ----------- script/build-arm64-git.sh | 52 ------------------ script/build-arm64.sh | 91 +++++++++++++++++++++++--------- script/build-ubuntu.sh | 2 +- script/generate-travis-config.ts | 3 ++ script/verify-arm64-git.sh | 28 ---------- 7 files changed, 74 insertions(+), 136 deletions(-) delete mode 100644 docker/arm64/Dockerfile delete mode 100755 script/build-arm64-git.sh delete mode 100644 script/verify-arm64-git.sh diff --git a/.travis.yml b/.travis.yml index aaccc3b6..9f5f9006 100644 --- a/.travis.yml +++ b/.travis.yml @@ -31,6 +31,7 @@ matrix: env: - TARGET_PLATFORM=ubuntu - GIT_LFS_CHECKSUM=89f5aa2c29800bbb71f5d4550edd69c5f83e3ee9e30f770446436dd7f4ef1d4c + arch: amd64 - os: osx language: c env: @@ -56,6 +57,7 @@ matrix: env: - TARGET_PLATFORM=arm64 - GIT_LFS_CHECKSUM=ea628d95158d5c76d9c7fe9432f28e49cc1a1b7ae3928b7089b1f4f97748d7a0 + arch: arm64 compiler: - gcc script: @@ -64,6 +66,7 @@ branches: only: - master - /^v[0-9]*.[0-9]*.[0.9]*.*$/ +cache: ccache deploy: provider: releases api_key: $GITHUB_TOKEN diff --git a/docker/arm64/Dockerfile b/docker/arm64/Dockerfile deleted file mode 100644 index 2e494a23..00000000 --- a/docker/arm64/Dockerfile +++ /dev/null @@ -1,31 +0,0 @@ -# currently tagged on Docker Hub as shiftkey/dugite-native:arm64-jessie-git-with-curl -FROM multiarch/debian-debootstrap:arm64-jessie - -RUN apt-get update -RUN apt-get install -y \ - build-essential \ - autoconf \ - libexpat-dev \ - curl \ - zlib1g-dev \ - libssl-dev \ - gettext - -ENV CURL_INSTALL_DIR "/tmp/build/curl" -ENV CURL_VERSION "curl-7.61.1" - -# extract curl to temp directory -RUN mkdir -p $CURL_INSTALL_DIR -WORKDIR /tmp -RUN curl -LO "https://curl.haxx.se/download/$CURL_VERSION.tar.gz" -RUN tar -xf "$CURL_VERSION.tar.gz" - -# configure and install curl -WORKDIR $CURL_VERSION -RUN ./configure --prefix=$CURL_INSTALL_DIR -RUN make install - -#cleanup -WORKDIR /tmp -RUN rm -rf "$CURL_VERSION.tar.gz" -RUN rm -rf "$CURL_VERSION" \ No newline at end of file diff --git a/script/build-arm64-git.sh b/script/build-arm64-git.sh deleted file mode 100755 index 5a3eebbf..00000000 --- a/script/build-arm64-git.sh +++ /dev/null @@ -1,52 +0,0 @@ -#!/bin/bash -e -# -# Compiling Git for ARM64 Linux (should be run inside a container) - -if [[ -z "${SOURCE}" ]]; then - echo "Required environment variable SOURCE was not set" - exit 1 -fi - -if [[ -z "${DESTINATION}" ]]; then - echo "Required environment variable DESTINATION was not set" - exit 1 -fi - -if [[ -z "${CURL_INSTALL_DIR}" ]]; then - echo "Required environment variable CURL_INSTALL_DIR was not set" - exit 1 -fi - -echo " -- Building git at $SOURCE to $DESTINATION" - -cd "$SOURCE" || exit 1 -make clean -make configure -CC='gcc' \ - CFLAGS='-Wall -g -O2 -fstack-protector --param=ssp-buffer-size=4 -Wformat -Werror=format-security -U_FORTIFY_SOURCE' \ - LDFLAGS='-Wl,-Bsymbolic-functions -Wl,-z,relro' \ - CURL_CONFIG="$CURL_INSTALL_DIR/bin/curl-config" \ - ./configure \ - --with-curl="$CURL_INSTALL_DIR" \ - --prefix=/ - -DESTDIR="$DESTINATION" \ - NO_PERL=1 \ - NO_TCLTK=1 \ - NO_GETTEXT=1 \ - NO_INSTALL_HARDLINKS=1 \ - NO_R_TO_GCC_LINKER=1 \ - make strip install - -echo "-- Removing server-side programs" -rm "$DESTINATION/bin/git-cvsserver" -rm "$DESTINATION/bin/git-receive-pack" -rm "$DESTINATION/bin/git-upload-archive" -rm "$DESTINATION/bin/git-upload-pack" -rm "$DESTINATION/bin/git-shell" - -echo "-- Removing unsupported features" -rm "$DESTINATION/libexec/git-core/git-svn" -rm "$DESTINATION/libexec/git-core/git-remote-testsvn" -rm "$DESTINATION/libexec/git-core/git-p4" -chmod 777 "$DESTINATION/libexec/git-core" diff --git a/script/build-arm64.sh b/script/build-arm64.sh index 4648fb73..02491f0d 100755 --- a/script/build-arm64.sh +++ b/script/build-arm64.sh @@ -1,8 +1,9 @@ #!/bin/bash -e # -# Building Git for ARM64 Linux and bundling Git LFS from upstream. +# Compiling Git for Linux and bundling Git LFS from upstream. # + if [[ -z "${SOURCE}" ]]; then echo "Required environment variable SOURCE was not set" exit 1 @@ -18,29 +19,48 @@ if [[ -z "${CURL_INSTALL_DIR}" ]]; then exit 1 fi -if [[ -z "${BASEDIR}" ]]; then - echo "Required environment variable BASEDIR was not set" - exit 1 -fi - - CURRENT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" # shellcheck source=script/compute-checksum.sh source "$CURRENT_DIR/compute-checksum.sh" # shellcheck source=script/check-static-linking.sh source "$CURRENT_DIR/check-static-linking.sh" -mkdir -p "$DESTINATION" +# Missing on arm64 +sudo apt-get install gettext + +echo " -- Building vanilla curl at $CURL_INSTALL_DIR instead of distro-specific version" -docker run --rm --privileged multiarch/qemu-user-static:register --reset -docker run -it \ ---mount type=bind,source="$BASEDIR",target="$BASEDIR" \ ---mount type=bind,source="$DESTINATION",target="$DESTINATION" \ --e "SOURCE=$SOURCE" \ --e "DESTINATION=$DESTINATION" \ --e "CURL_INSTALL_DIR=$CURL_INSTALL_DIR" \ --w="$BASEDIR" \ ---rm shiftkey/dugite-native:arm64-jessie-git-with-curl bash "$BASEDIR/script/build-arm64-git.sh" +CURL_FILE_NAME="curl-7.61.1" +CURL_FILE="$CURL_FILE_NAME.tar.gz" + +cd /tmp || exit 1 +curl -LO "https://curl.haxx.se/download/$CURL_FILE" +tar -xf $CURL_FILE + +( +cd $CURL_FILE_NAME || exit 1 +./configure --prefix="$CURL_INSTALL_DIR" +make install -j "$(nproc)" +) +echo " -- Building git at $SOURCE to $DESTINATION" + +( +cd "$SOURCE" || exit 1 +make clean +make configure +CC='gcc' \ + CFLAGS='-Wall -g -O2 -fstack-protector --param=ssp-buffer-size=4 -Wformat -Werror=format-security -U_FORTIFY_SOURCE' \ + LDFLAGS='-Wl,-Bsymbolic-functions -Wl,-z,relro' \ + ./configure \ + --with-curl="$CURL_INSTALL_DIR" \ + --prefix=/ +DESTDIR="$DESTINATION" \ + NO_TCLTK=1 \ + NO_GETTEXT=1 \ + NO_INSTALL_HARDLINKS=1 \ + NO_R_TO_GCC_LINKER=1 \ + make strip install -j "$(nproc)" +) if [[ "$GIT_LFS_VERSION" ]]; then echo "-- Bundling Git LFS" @@ -82,13 +102,36 @@ if [[ ! -f "$DESTINATION/ssl/cacert.pem" ]]; then echo "-- Skipped bundling of CA certificates (failed to download them)" fi -echo "-- Verifying environment" -docker run -it \ - --mount type=bind,source="$BASEDIR",target="$BASEDIR" \ - --mount type=bind,source="$DESTINATION",target="$DESTINATION" \ - -e "DESTINATION=$DESTINATION" \ - -w="$BASEDIR" \ - --rm shiftkey/dugite-native:arm64-jessie-git-with-curl sh "$BASEDIR/script/verify-arm64-git.sh" + +echo "-- Removing server-side programs" +rm "$DESTINATION/bin/git-cvsserver" +rm "$DESTINATION/bin/git-receive-pack" +rm "$DESTINATION/bin/git-upload-archive" +rm "$DESTINATION/bin/git-upload-pack" +rm "$DESTINATION/bin/git-shell" + +echo "-- Removing unsupported features" +rm "$DESTINATION/libexec/git-core/git-svn" +rm "$DESTINATION/libexec/git-core/git-remote-testsvn" +rm "$DESTINATION/libexec/git-core/git-p4" echo "-- Static linking research" check_static_linking "$DESTINATION" + +echo "-- Testing clone operation with generated binary" + +rm -rf "$CURL_OUTPUT_DIR" + +TEMP_CLONE_DIR=/tmp/clones +mkdir -p $TEMP_CLONE_DIR + +( +cd "$DESTINATION/bin" || exit 1 +./git --version +GIT_CURL_VERBOSE=1 \ + GIT_TEMPLATE_DIR="$DESTINATION/share/git-core/templates" \ + GIT_SSL_CAINFO="$DESTINATION/ssl/cacert.pem" \ + GIT_EXEC_PATH="$DESTINATION/libexec/git-core" \ + PREFIX="$DESTINATION" \ + ./git clone https://github.com/git/git.github.io "$TEMP_CLONE_DIR/git.github.io" +) diff --git a/script/build-ubuntu.sh b/script/build-ubuntu.sh index 6ded0d23..f9f1f17c 100755 --- a/script/build-ubuntu.sh +++ b/script/build-ubuntu.sh @@ -131,4 +131,4 @@ GIT_CURL_VERBOSE=1 \ GIT_EXEC_PATH="$DESTINATION/libexec/git-core" \ PREFIX="$DESTINATION" \ ./git clone https://github.com/git/git.github.io "$TEMP_CLONE_DIR/git.github.io" -) \ No newline at end of file +) diff --git a/script/generate-travis-config.ts b/script/generate-travis-config.ts index 448cf780..7eec8e17 100644 --- a/script/generate-travis-config.ts +++ b/script/generate-travis-config.ts @@ -81,12 +81,14 @@ function getConfig(platform: string, arch: string) { os: 'linux', language: 'c', env: ['TARGET_PLATFORM=ubuntu', `GIT_LFS_CHECKSUM=${lfsFile.checksum}`], + arch: 'amd64', } } else if (arch === 'arm64') { return { os: 'linux', language: 'c', env: ['TARGET_PLATFORM=arm64', `GIT_LFS_CHECKSUM=${lfsFile.checksum}`], + arch: 'arm64', } } } @@ -133,6 +135,7 @@ const baseConfig = { branches: { only: ['master', '/^v[0-9]*.[0-9]*.[0.9]*.*$/'], }, + cache: 'ccache', deploy: { provider: 'releases', api_key: '$GITHUB_TOKEN', diff --git a/script/verify-arm64-git.sh b/script/verify-arm64-git.sh deleted file mode 100644 index 15f0f029..00000000 --- a/script/verify-arm64-git.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash -e -# -# Verify Git for ARM64 Linux (should be run inside a container) - -if [[ -z "${DESTINATION}" ]]; then - echo "Required environment variable DESTINATION was not set" - exit 1 -fi - -echo "-- Test external Git LFS" - -"$DESTINATION/libexec/git-core/git-lfs" --version - -echo "-- Test clone operation with generated binary" - -TEMP_CLONE_DIR=/tmp/clones -mkdir -p $TEMP_CLONE_DIR - -( -cd "$DESTINATION/bin" || exit 1 -./git --version -GIT_CURL_VERBOSE=1 \ - GIT_TEMPLATE_DIR="$DESTINATION/share/git-core/templates" \ - GIT_SSL_CAINFO="$DESTINATION/ssl/cacert.pem" \ - GIT_EXEC_PATH="$DESTINATION/libexec/git-core" \ - PREFIX="$DESTINATION" \ - ./git clone https://github.com/git/git.github.io "$TEMP_CLONE_DIR/git.github.io" -)