diff --git a/.travis.yml b/.travis.yml index 69dc6deb..482c80f9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -28,6 +28,7 @@ matrix: - npm run check && npm run prettier - os: linux language: c + cache: ccache env: - TARGET_PLATFORM=ubuntu - GIT_LFS_CHECKSUM=89f5aa2c29800bbb71f5d4550edd69c5f83e3ee9e30f770446436dd7f4ef1d4c @@ -51,6 +52,13 @@ matrix: - GIT_FOR_WINDOWS_URL=https://github.com/git-for-windows/git/releases/download/v2.23.0.windows.5/MinGit-2.23.0.windows.5-32-bit.zip - GIT_FOR_WINDOWS_CHECKSUM=17e8dcffcc3c42e37e6beedcf51ef0457bef45f24290ee6e854f622a9133dd76 - GIT_LFS_CHECKSUM=f4f49e9261584711c337f566a62bd9645cc0e10cef4dc54de1e1e0d31a7b2f71 + - os: linux + language: c + arch: arm64 + cache: ccache + env: + - TARGET_PLATFORM=arm64 + - GIT_LFS_CHECKSUM=ea628d95158d5c76d9c7fe9432f28e49cc1a1b7ae3928b7089b1f4f97748d7a0 compiler: - gcc script: diff --git a/dependencies.json b/dependencies.json index 004a99af..3ed4ef8f 100644 --- a/dependencies.json +++ b/dependencies.json @@ -27,6 +27,12 @@ "name": "git-lfs-linux-amd64-v2.7.2.tar.gz", "checksum": "89f5aa2c29800bbb71f5d4550edd69c5f83e3ee9e30f770446436dd7f4ef1d4c" }, + { + "platform": "linux", + "arch": "arm64", + "name": "git-lfs-linux-arm64-v2.7.2.tar.gz", + "checksum": "ea628d95158d5c76d9c7fe9432f28e49cc1a1b7ae3928b7089b1f4f97748d7a0" + }, { "platform": "windows", "arch": "x86", diff --git a/script/build-arm64.sh b/script/build-arm64.sh new file mode 100755 index 00000000..965895d7 --- /dev/null +++ b/script/build-arm64.sh @@ -0,0 +1,138 @@ +#!/bin/bash -e +# +# Compiling Git for ARM64 Linux and bundling Git LFS from upstream. +# + + +if [[ -z "${SOURCE}" ]]; then + echo "Required environment variable SOURCE was not set" + exit 1 +fi + +if [[ -z "${DESTINATION}" ]]; then + echo "Required environment variable DESTINATION was not set" + exit 1 +fi + +if [[ -z "${CURL_INSTALL_DIR}" ]]; then + echo "Required environment variable CURL_INSTALL_DIR was not set" + exit 1 +fi + +CURRENT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +# shellcheck source=script/compute-checksum.sh +source "$CURRENT_DIR/compute-checksum.sh" +# shellcheck source=script/check-static-linking.sh +source "$CURRENT_DIR/check-static-linking.sh" + +# Missing on arm64 +sudo apt-get install gettext + +echo " -- Building vanilla curl at $CURL_INSTALL_DIR instead of distro-specific version" + +CURL_FILE_NAME="curl-7.61.1" +CURL_FILE="$CURL_FILE_NAME.tar.gz" + +cd /tmp || exit 1 +curl -LO "https://curl.haxx.se/download/$CURL_FILE" +tar -xf $CURL_FILE + +( +cd $CURL_FILE_NAME || exit 1 +./configure --prefix="$CURL_INSTALL_DIR" +make install -j "$(nproc)" +) +echo " -- Building git at $SOURCE to $DESTINATION" + +( +cd "$SOURCE" || exit 1 +make clean +make configure +CC='gcc' \ + CFLAGS='-Wall -g -O2 -fstack-protector --param=ssp-buffer-size=4 -Wformat -Werror=format-security -U_FORTIFY_SOURCE' \ + LDFLAGS='-Wl,-Bsymbolic-functions -Wl,-z,relro' \ + ./configure \ + --with-curl="$CURL_INSTALL_DIR" \ + --prefix=/ +DESTDIR="$DESTINATION" \ + NO_TCLTK=1 \ + NO_GETTEXT=1 \ + NO_INSTALL_HARDLINKS=1 \ + NO_R_TO_GCC_LINKER=1 \ + make strip install -j "$(nproc)" +) + +if [[ "$GIT_LFS_VERSION" ]]; then + echo "-- Bundling Git LFS" + GIT_LFS_FILE=git-lfs.tar.gz + GIT_LFS_URL="https://github.com/git-lfs/git-lfs/releases/download/v${GIT_LFS_VERSION}/git-lfs-linux-arm64-v${GIT_LFS_VERSION}.tar.gz" + echo "-- Downloading from $GIT_LFS_URL" + curl -sL -o $GIT_LFS_FILE "$GIT_LFS_URL" + COMPUTED_SHA256=$(compute_checksum $GIT_LFS_FILE) + if [ "$COMPUTED_SHA256" = "$GIT_LFS_CHECKSUM" ]; then + echo "Git LFS: checksums match" + SUBFOLDER="$DESTINATION/libexec/git-core" + tar -xvf $GIT_LFS_FILE -C "$SUBFOLDER" --exclude='*.sh' --exclude="*.md" + + if [[ ! -f "$SUBFOLDER/git-lfs" ]]; then + echo "After extracting Git LFS the file was not found under libexec/git-core/" + echo "aborting..." + exit 1 + fi + else + echo "Git LFS: expected checksum $GIT_LFS_CHECKSUM but got $COMPUTED_SHA256" + echo "aborting..." + exit 1 + fi +else + echo "-- Skipped bundling Git LFS (set GIT_LFS_VERSION to include it in the bundle)" +fi + + +( +# download CA bundle and write straight to temp folder +# for more information: https://curl.haxx.se/docs/caextract.html +echo "-- Adding CA bundle" +cd "$DESTINATION" || exit 1 +mkdir -p ssl +curl -sL -o ssl/cacert.pem https://curl.haxx.se/ca/cacert.pem +) + +if [[ ! -f "$DESTINATION/ssl/cacert.pem" ]]; then + echo "-- Skipped bundling of CA certificates (failed to download them)" +fi + + +echo "-- Removing server-side programs" +rm "$DESTINATION/bin/git-cvsserver" +rm "$DESTINATION/bin/git-receive-pack" +rm "$DESTINATION/bin/git-upload-archive" +rm "$DESTINATION/bin/git-upload-pack" +rm "$DESTINATION/bin/git-shell" + +echo "-- Removing unsupported features" +rm "$DESTINATION/libexec/git-core/git-svn" +rm "$DESTINATION/libexec/git-core/git-remote-testsvn" +rm "$DESTINATION/libexec/git-core/git-p4" + +echo "-- Static linking research" +check_static_linking "$DESTINATION" + +echo "-- Testing clone operation with generated binary" + +rm -rf "$CURL_OUTPUT_DIR" + +TEMP_CLONE_DIR=/tmp/clones +mkdir -p $TEMP_CLONE_DIR + +( +cd "$DESTINATION/bin" || exit 1 +./git --version +GIT_CURL_VERBOSE=1 \ + GIT_TEMPLATE_DIR="$DESTINATION/share/git-core/templates" \ + GIT_SSL_CAINFO="$DESTINATION/ssl/cacert.pem" \ + GIT_EXEC_PATH="$DESTINATION/libexec/git-core" \ + PREFIX="$DESTINATION" \ + ./git clone https://github.com/git/git.github.io "$TEMP_CLONE_DIR/git.github.io" +) + diff --git a/script/build.sh b/script/build.sh index 8233e568..4289f655 100755 --- a/script/build.sh +++ b/script/build.sh @@ -12,6 +12,8 @@ elif [ "$TARGET_PLATFORM" == "macOS" ]; then SCRIPT="$CURRENT_DIR/build-macos.sh" elif [ "$TARGET_PLATFORM" == "win32" ]; then SCRIPT="$CURRENT_DIR/build-win32.sh" +elif [ "$TARGET_PLATFORM" == "arm64" ]; then + SCRIPT="$CURRENT_DIR/build-arm64.sh" else echo "Unable to build Git for platform $TARGET_PLATFORM" exit 1 diff --git a/script/generate-travis-config.ts b/script/generate-travis-config.ts index fd043ab5..453531d2 100644 --- a/script/generate-travis-config.ts +++ b/script/generate-travis-config.ts @@ -75,11 +75,22 @@ function getConfig(platform: string, arch: string) { } } - if (platform === 'linux' && arch === 'amd64') { - return { - os: 'linux', - language: 'c', - env: ['TARGET_PLATFORM=ubuntu', `GIT_LFS_CHECKSUM=${lfsFile.checksum}`], + if (platform === 'linux') { + if (arch === 'amd64') { + return { + os: 'linux', + language: 'c', + cache: 'ccache', + env: ['TARGET_PLATFORM=ubuntu', `GIT_LFS_CHECKSUM=${lfsFile.checksum}`], + } + } else if (arch === 'arm64') { + return { + os: 'linux', + language: 'c', + arch: 'arm64', + cache: 'ccache', + env: ['TARGET_PLATFORM=arm64', `GIT_LFS_CHECKSUM=${lfsFile.checksum}`], + } } } @@ -115,6 +126,7 @@ const baseConfig = { getConfig('darwin', 'amd64'), getConfig('windows', 'amd64'), getConfig('windows', 'x86'), + getConfig('linux', 'arm64'), ], }, diff --git a/script/package.sh b/script/package.sh index ea24be71..cc13f3bd 100755 --- a/script/package.sh +++ b/script/package.sh @@ -41,6 +41,9 @@ elif [ "$TARGET_PLATFORM" == "win32" ]; then if [ "$WIN_ARCH" -eq "64" ]; then ARCH="x64"; else ARCH="x86"; fi GZIP_FILE="dugite-native-$VERSION-$BUILD_HASH-windows-$ARCH.tar.gz" LZMA_FILE="dugite-native-$VERSION-$BUILD_HASH-windows-$ARCH.lzma" +elif [ "$TARGET_PLATFORM" == "arm64" ]; then + GZIP_FILE="dugite-native-$VERSION-$BUILD_HASH-arm64.tar.gz" + LZMA_FILE="dugite-native-$VERSION-$BUILD_HASH-arm64.lzma" else echo "Unable to package Git for platform $TARGET_PLATFORM" exit 1 diff --git a/script/update-git-lfs.ts b/script/update-git-lfs.ts index 9a1a8017..fd627a03 100644 --- a/script/update-git-lfs.ts +++ b/script/update-git-lfs.ts @@ -27,6 +27,9 @@ function getArch(fileName: string) { if (fileName.match(/-386-/)) { return 'x86' } + if (fileName.match(/-arm64-/)) { + return 'arm64' + } throw new Error(`Unable to find arch for file: ${fileName}`) } @@ -93,6 +96,7 @@ async function run() { const files = [ `git-lfs-linux-amd64-${version}.tar.gz`, + `git-lfs-linux-arm64-${version}.tar.gz`, `git-lfs-windows-386-${version}.zip`, `git-lfs-windows-amd64-${version}.zip`, ] diff --git a/script/update-test-harness.ts b/script/update-test-harness.ts index c6258a76..0683bd7d 100644 --- a/script/update-test-harness.ts +++ b/script/update-test-harness.ts @@ -55,6 +55,9 @@ elif [ "$TARGET_PLATFORM" == "win32" ]; then if [ "$WIN_ARCH" -eq "64" ]; then ARCH="x64"; else ARCH="x86"; fi GZIP_FILE="dugite-native-$VERSION-$BUILD_HASH-windows-$ARCH.tar.gz" LZMA_FILE="dugite-native-$VERSION-$BUILD_HASH-windows-$ARCH.lzma" +elif [ "$TARGET_PLATFORM" == "arm64" ]; then + GZIP_FILE="dugite-native-$VERSION-$BUILD_HASH-arm64.tar.gz" + LZMA_FILE="dugite-native-$VERSION-$BUILD_HASH-arm64.lzma" else echo "Unable to package Git for platform $TARGET_PLATFORM" exit 1 diff --git a/script/verify-arm64-git.sh b/script/verify-arm64-git.sh new file mode 100644 index 00000000..15f0f029 --- /dev/null +++ b/script/verify-arm64-git.sh @@ -0,0 +1,28 @@ +#!/bin/bash -e +# +# Verify Git for ARM64 Linux (should be run inside a container) + +if [[ -z "${DESTINATION}" ]]; then + echo "Required environment variable DESTINATION was not set" + exit 1 +fi + +echo "-- Test external Git LFS" + +"$DESTINATION/libexec/git-core/git-lfs" --version + +echo "-- Test clone operation with generated binary" + +TEMP_CLONE_DIR=/tmp/clones +mkdir -p $TEMP_CLONE_DIR + +( +cd "$DESTINATION/bin" || exit 1 +./git --version +GIT_CURL_VERBOSE=1 \ + GIT_TEMPLATE_DIR="$DESTINATION/share/git-core/templates" \ + GIT_SSL_CAINFO="$DESTINATION/ssl/cacert.pem" \ + GIT_EXEC_PATH="$DESTINATION/libexec/git-core" \ + PREFIX="$DESTINATION" \ + ./git clone https://github.com/git/git.github.io "$TEMP_CLONE_DIR/git.github.io" +) diff --git a/test/macos.sh b/test/macos.sh index 92bbd9af..959745a5 100755 --- a/test/macos.sh +++ b/test/macos.sh @@ -47,6 +47,9 @@ elif [ "$TARGET_PLATFORM" == "win32" ]; then if [ "$WIN_ARCH" -eq "64" ]; then ARCH="x64"; else ARCH="x86"; fi GZIP_FILE="dugite-native-$VERSION-$BUILD_HASH-windows-$ARCH.tar.gz" LZMA_FILE="dugite-native-$VERSION-$BUILD_HASH-windows-$ARCH.lzma" +elif [ "$TARGET_PLATFORM" == "arm64" ]; then + GZIP_FILE="dugite-native-$VERSION-$BUILD_HASH-arm64.tar.gz" + LZMA_FILE="dugite-native-$VERSION-$BUILD_HASH-arm64.lzma" else echo "Unable to package Git for platform $TARGET_PLATFORM" exit 1 diff --git a/test/ubuntu.sh b/test/ubuntu.sh index 4e3339c5..c743d48d 100755 --- a/test/ubuntu.sh +++ b/test/ubuntu.sh @@ -48,6 +48,9 @@ elif [ "$TARGET_PLATFORM" == "win32" ]; then if [ "$WIN_ARCH" -eq "64" ]; then ARCH="x64"; else ARCH="x86"; fi GZIP_FILE="dugite-native-$VERSION-$BUILD_HASH-windows-$ARCH.tar.gz" LZMA_FILE="dugite-native-$VERSION-$BUILD_HASH-windows-$ARCH.lzma" +elif [ "$TARGET_PLATFORM" == "arm64" ]; then + GZIP_FILE="dugite-native-$VERSION-$BUILD_HASH-arm64.tar.gz" + LZMA_FILE="dugite-native-$VERSION-$BUILD_HASH-arm64.lzma" else echo "Unable to package Git for platform $TARGET_PLATFORM" exit 1 diff --git a/test/win32.sh b/test/win32.sh index d95caca3..fb676114 100755 --- a/test/win32.sh +++ b/test/win32.sh @@ -51,6 +51,9 @@ elif [ "$TARGET_PLATFORM" == "win32" ]; then if [ "$WIN_ARCH" -eq "64" ]; then ARCH="x64"; else ARCH="x86"; fi GZIP_FILE="dugite-native-$VERSION-$BUILD_HASH-windows-$ARCH.tar.gz" LZMA_FILE="dugite-native-$VERSION-$BUILD_HASH-windows-$ARCH.lzma" +elif [ "$TARGET_PLATFORM" == "arm64" ]; then + GZIP_FILE="dugite-native-$VERSION-$BUILD_HASH-arm64.tar.gz" + LZMA_FILE="dugite-native-$VERSION-$BUILD_HASH-arm64.lzma" else echo "Unable to package Git for platform $TARGET_PLATFORM" exit 1