setup azure pipelines build config and extract ARM64-specific scripts

azure-pipelines.yml

@@ -0,0 +1,204 @@
+variables:
+ GIT_LFS_VERSION: "2.5.2"
+
+resources:
+ repositories:
+ - repository: self
+ checkoutOptions:
+ submodules: true
+
+jobs:
+ - job: Ubuntu
+ displayName: "Ubuntu Linux (AMD64)"
+ pool:
+ vmImage: ubuntu-16.04
+ variables:
+ TARGET_PLATFORM: ubuntu
+ GIT_LFS_CHECKSUM: 624396e8994578ac38c3e5987889be56dba453c378c0675d56cffbc5b8972aa5
+ steps:
+ - script: |
+ sudo apt-get update
+ sudo apt-get install -y --no-install-recommends gettext
+ - script: script/build.sh
+ name: Build
+ - script: script/package.sh
+ name: Package
+ # TODO: figure out a way to unify these into a single task
+ - task: CopyFiles@2
+ inputs:
+ contents: "output/*.tar.gz"
+ targetFolder: $(Build.ArtifactStagingDirectory)
+ flattenFolders: true
+ - task: CopyFiles@2
+ inputs:
+ contents: "output/*.lzma"
+ targetFolder: $(Build.ArtifactStagingDirectory)
+ flattenFolders: true
+ - task: CopyFiles@2
+ inputs:
+ contents: "output/*.sha256"
+ targetFolder: $(Build.ArtifactStagingDirectory)
+ flattenFolders: true
+ - task: PublishBuildArtifacts@1
+
+ - job: Linux_ARM64
+ displayName: "Linux (arm64)"
+ pool:
+ vmImage: "ubuntu-16.04"
+ variables:
+ TARGET_PLATFORM: arm64
+ steps:
+ - script: docker run --rm --privileged multiarch/qemu-user-static:register --reset
+ displayName: "Register Docker QEMU"
+ - task: docker@0
+ displayName: "Build Git in container"
+ inputs:
+ action: "Run an image"
+ imageName: shiftkey/dugite-native:arm64-jessie-git
+ volumes: |
+ $(Build.SourcesDirectory):/src
+ $(Build.SourcesDirectory)/build/git:/output
+ envVars: |
+ SOURCE=/src/git
+ DESTINATION=/output
+ workDir: /src
+ containerCommand: "sh /src/script/arm64/build-git.sh"
+ detached: false
+ - bash: script/arm64/build-git-lfs.sh
+ displayName: "Verify Git"
+ env:
+ DESTINATION: $(Build.SourcesDirectory)/build/git
+ - bash: script/arm64/verify-git.sh
+ displayName: "Build Git LFS"
+ env:
+ DESTINATION: $(Build.SourcesDirectory)/build/git
+ - task: docker@0
+ displayName: Verify Git LFS in container
+ inputs:
+ action: "Run an image"
+ imageName: shiftkey/dugite-native:arm64-jessie-git
+ volumes: |
+ $(Build.SourcesDirectory):/src
+ $(Build.SourcesDirectory)/build/git:/output
+ envVars: |
+ DESTINATION=/output
+ workDir: /src
+ containerCommand: "sh /src/script/arm64/verify-git-lfs.sh"
+ detached: false
+ - bash: script/arm64/package.sh
+ name: Package
+ env:
+ SOURCE: $(Build.SourcesDirectory)/git
+ DESTINATION: $(Build.SourcesDirectory)/build/git
+ # TODO: figure out a way to unify these into a single task
+ - task: CopyFiles@2
+ inputs:
+ contents: "output/*.tar.gz"
+ targetFolder: $(Build.ArtifactStagingDirectory)
+ flattenFolders: true
+ - task: CopyFiles@2
+ inputs:
+ contents: "output/*.lzma"
+ targetFolder: $(Build.ArtifactStagingDirectory)
+ flattenFolders: true
+ - task: CopyFiles@2
+ inputs:
+ contents: "output/*.sha256"
+ targetFolder: $(Build.ArtifactStagingDirectory)
+ flattenFolders: true
+ - task: PublishBuildArtifacts@1
+
+ - job: macOS
+ pool:
+ vmImage: xcode9-macos10.13
+ variables:
+ TARGET_PLATFORM: macOS
+ GIT_LFS_CHECKSUM: eedb80c79f1d3106aa5f1496ddc505e1c1c86c290293d81fb20c5358c615fd74
+ steps:
+ - bash: script/build.sh
+ name: Build
+ - bash: script/package.sh
+ name: Package
+ # TODO: figure out a way to unify these into a single task
+ - task: CopyFiles@2
+ inputs:
+ contents: "output/*.tar.gz"
+ targetFolder: $(Build.ArtifactStagingDirectory)
+ flattenFolders: true
+ - task: CopyFiles@2
+ inputs:
+ contents: "output/*.lzma"
+ targetFolder: $(Build.ArtifactStagingDirectory)
+ flattenFolders: true
+ - task: CopyFiles@2
+ inputs:
+ contents: "output/*.sha256"
+ targetFolder: $(Build.ArtifactStagingDirectory)
+ flattenFolders: true
+ - task: PublishBuildArtifacts@1
+
+ - job: Windows_64bit
+ displayName: "Windows (64-bit)"
+ pool:
+ vmImage: vs2017-win2016
+ variables:
+ TARGET_PLATFORM: win32
+ WIN_ARCH: 64
+ GIT_LFS_CHECKSUM: d5276eb61dea32b3978c2f68c5cf3ad4a45bf70f1a245ddc86b555db7299c7c9
+ GIT_FOR_WINDOWS_URL: https://github.com/git-for-windows/git/releases/download/v2.19.1.windows.1/MinGit-2.19.1-64-bit.zip
+ GIT_FOR_WINDOWS_CHECKSUM: f89e103a41bda8e12efeaab198a8c20bb4a84804683862da518ee2cb66a5a5b3
+ steps:
+ - bash: script/build.sh
+ name: Build
+ - bash: script/package.sh
+ name: Package
+ # TODO: figure out a way to unify these into a single task
+ - task: CopyFiles@2
+ inputs:
+ contents: "output/*.tar.gz"
+ targetFolder: $(Build.ArtifactStagingDirectory)
+ flattenFolders: true
+ - task: CopyFiles@2
+ inputs:
+ contents: "output/*.lzma"
+ targetFolder: $(Build.ArtifactStagingDirectory)
+ flattenFolders: true
+ - task: CopyFiles@2
+ inputs:
+ contents: "output/*.sha256"
+ targetFolder: $(Build.ArtifactStagingDirectory)
+ flattenFolders: true
+ - task: PublishBuildArtifacts@1
+
+ - job: Windows_32bit
+ displayName: "Windows (32-bit)"
+ pool:
+ vmImage: vs2017-win2016
+ variables:
+ TARGET_PLATFORM: win32
+ WIN_ARCH: 32
+ GIT_LFS_CHECKSUM: 6cf7d4c169a17dd5b326f903708829e7471368b7e1235ab150ce77555f47b213
+ GIT_FOR_WINDOWS_URL: https://github.com/git-for-windows/git/releases/download/v2.19.1.windows.1/MinGit-2.19.1-32-bit.zip
+ GIT_FOR_WINDOWS_CHECKSUM: 9bde728fe03f66a022b3e41408902ccfceb56a34067db1f35d6509375b9be922
+ steps:
+ - bash: script/build.sh
+ name: Build
+ - bash: script/package.sh
+ name: Package
+ # TODO: figure out a way to unify these into a single task
+ - task: CopyFiles@2
+ inputs:
+ contents: "output/*.tar.gz"
+ targetFolder: $(Build.ArtifactStagingDirectory)
+ flattenFolders: true
+ - task: CopyFiles@2
+ inputs:
+ contents: "output/*.lzma"
+ targetFolder: $(Build.ArtifactStagingDirectory)
+ flattenFolders: true
+ - task: CopyFiles@2
+ inputs:
+ contents: "output/*.sha256"
+ targetFolder: $(Build.ArtifactStagingDirectory)
+ flattenFolders: true
+ - task: PublishBuildArtifacts@1

script/arm64/build-git-lfs.sh

@@ -0,0 +1,19 @@
+
+if [[ "$GIT_LFS_VERSION" ]]; then
+ echo "-- Building Git LFS"
+ go get github.com/git-lfs/git-lfs
+ GOPATH=`go env GOPATH`
+ cd $GOPATH/src/github.com/git-lfs/git-lfs
+ git checkout "v${GIT_LFS_VERSION}"
+ # Make the 'mangen' target first, without setting GOOS/GOARCH.
+ make mangen
+ make GOARCH=arm64 GOOS=linux
+ GIT_LFS_OUTPUT_DIR=$GOPATH/src/github.com/git-lfs/git-lfs/bin/
+
+ echo "-- Bundling Git LFS"
+ GIT_LFS_FILE=$GIT_LFS_OUTPUT_DIR/git-lfs
+ SUBFOLDER="$DESTINATION/libexec/git-core"
+ cp $GIT_LFS_FILE $SUBFOLDER
+else
+ echo "-- Skipped bundling Git LFS (set GIT_LFS_VERSION to include it in the bundle)"
+fi

script/arm64/build-git.sh

@@ -0,0 +1,65 @@
+echo " -- Building git at $SOURCE to $DESTINATION"
+
+cd $SOURCE
+make clean
+DESTDIR="$DESTINATION" make strip install prefix=/ \
+ NO_PERL=1 \
+ NO_TCLTK=1 \
+ NO_GETTEXT=1 \
+ NO_INSTALL_HARDLINKS=1 \
+ CC='gcc' \
+ CFLAGS='-Wall -g -O2 -fstack-protector --param=ssp-buffer-size=4 -Wformat -Werror=format-security -U_FORTIFY_SOURCE' \
+ LDFLAGS='-Wl,-Bsymbolic-functions -Wl,-z,relro'
+
+echo "-- Removing server-side programs"
+rm -f "$DESTINATION/bin/git-cvsserver"
+rm -f "$DESTINATION/bin/git-receive-pack"
+rm -f "$DESTINATION/bin/git-upload-archive"
+rm -f "$DESTINATION/bin/git-upload-pack"
+rm -f "$DESTINATION/bin/git-shell"
+
+echo "-- Removing unsupported features"
+rm -f "$DESTINATION/libexec/git-core/git-svn"
+rm -f "$DESTINATION/libexec/git-core/git-remote-testsvn"
+rm -f "$DESTINATION/libexec/git-core/git-p4"
+
+# because we are building Git in a container, we need to ensure the regular
+# user is able to modify the output outside of this script
+chmod 777 "$DESTINATION/libexec/git-core"
+
+sh $SOURCE/../script/clean-artefacts.sh $DESTINATION
+
+# download CA bundle and write straight to temp folder
+# for more information: https://curl.haxx.se/docs/caextract.html
+echo "-- Adding CA bundle"
+cd $DESTINATION
+mkdir -p ssl
+curl -sL -o ssl/cacert.pem https://curl.haxx.se/ca/cacert.pem
+cd - > /dev/null
+
+checkStaticLinking() {
+ if [ -z "$1" ] ; then
+ # no parameter provided, fail hard
+ exit 1
+ fi
+
+ # ermagherd there's two whitespace characters between 'LSB' and 'executable'
+ # when running this on Travis - why is everything so terrible?
+ if file $1 | grep -q 'ELF 64-bit LSB'; then
+ if readelf -d $1 | grep -q 'Shared library'; then
+ echo "File: $file"
+ # this is done twice rather than storing in a bash variable because
+ # it's easier than trying to preserve the line endings
+ readelf -d $1 | grep 'Shared library'
+ fi
+ fi
+}
+
+echo "-- Static linking research"
+cd "$DESTINATION"
+# check all files for ELF exectuables
+find . -type f -print0 | while read -d $'\0' file
+do
+ checkStaticLinking $file
+done
+cd - > /dev/null

script/arm64/package.sh

@@ -0,0 +1,55 @@
+#!/bin/bash
+#
+# Script for packaging artefacts into gzipped archive.
+# Build scripts should handle platform-specific differences, so this
+# script works off the assumption that everything at $DESTINATION is
+# intended to be part of the archive.
+#
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+cd $SOURCE
+VERSION=$(git describe --exact-match HEAD)
+EXIT_CODE=$?
+
+if [ "$EXIT_CODE" == "128" ]; then
+ echo "Git commit does not have tag, cannot use version to build from"
+ exit 1
+fi
+cd - > /dev/null
+
+BUILD_HASH=$(git rev-parse --short HEAD)
+
+if ! [ -d "$DESTINATION" ]; then
+ echo "No output found, exiting..."
+ exit 1
+fi
+
+GZIP_FILE="dugite-native-$VERSION-$BUILD_HASH-arm64.tar.gz"
+LZMA_FILE="dugite-native-$VERSION-$BUILD_HASH-arm64.lzma"
+
+echo ""
+echo "Creating archives..."
+mkdir output
+cd output
+if [ "$(uname -s)" == "Darwin" ]; then
+ tar -czf $GZIP_FILE -C $DESTINATION .
+ tar --lzma -cf $LZMA_FILE -C $DESTINATION .
+else
+ tar -caf $GZIP_FILE -C $DESTINATION .
+ tar -caf $LZMA_FILE -C $DESTINATION .
+fi
+
+GZIP_CHECKSUM=$(shasum -a 256 $GZIP_FILE | awk '{print $1;}')
+LZMA_CHECKSUM=$(shasum -a 256 $LZMA_FILE | awk '{print $1;}')
+
+echo "$GZIP_CHECKSUM" | tr -d '\n' > "${GZIP_FILE}.sha256"
+echo "$LZMA_CHECKSUM" | tr -d '\n' > "${LZMA_FILE}.sha256"
+
+GZIP_SIZE=$(du -h $GZIP_FILE | cut -f1)
+LZMA_SIZE=$(du -h $LZMA_FILE | cut -f1)
+
+echo "Packages created:"
+echo "${GZIP_FILE} - ${GZIP_SIZE} - checksum: ${GZIP_CHECKSUM}"
+echo "${LZMA_FILE} - ${LZMA_SIZE} - checksum: ${LZMA_CHECKSUM}"
+cd - > /dev/null

script/arm64/verify-git-lfs.sh

@@ -0,0 +1,5 @@
+if [[ "$GIT_LFS_VERSION" ]]; then
+ $DESTINATION/libexec/git-core/git-lfs --version
+else
+ echo "-- Skipped verifying Git LFS (set GIT_LFS_VERSION to include it in the bundle)"
+fi

script/arm64/verify-git.sh

@@ -0,0 +1,26 @@
+checkStaticLinking() {
+ if [ -z "$1" ] ; then
+ # no parameter provided, fail hard
+ exit 1
+ fi
+
+ # ermagherd there's two whitespace characters between 'LSB' and 'executable'
+ # when running this on Travis - why is everything so terrible?
+ if file $1 | grep -q 'ELF 64-bit LSB'; then
+ if readelf -d $1 | grep -q 'Shared library'; then
+ echo "File: $file"
+ # this is done twice rather than storing in a bash variable because
+ # it's easier than trying to preserve the line endings
+ readelf -d $1 | grep 'Shared library'
+ fi
+ fi
+}
+
+echo "-- Static linking research"
+cd "$DESTINATION"
+# check all files for ELF exectuables
+find . -type f -print0 | while read -d $'\0' file
+do
+ checkStaticLinking $file
+done
+cd - > /dev/null