libgit2 build cherry pick

.dockerignore

@@ -0,0 +1 @@
+hack/libgit2/

.github/actions/run-tests/Dockerfile

@@ -1,19 +1,35 @@
-FROM golang:1.16-buster as builder
-
-# Up-to-date libgit2 dependencies are only available in sid (unstable).
-RUN echo "deb http://deb.debian.org/debian unstable main" >> /etc/apt/sources.list \
-    && echo "deb-src http://deb.debian.org/debian unstable main" >> /etc/apt/sources.list
-RUN set -eux; \
-    apt-get update \
-    && apt-get install -y libgit2-dev/unstable zlib1g-dev/unstable libssh2-1-dev/unstable libpcre3-dev/unstable \
-    && apt-get clean \
-    && apt-get autoremove --purge -y \
-    && rm -rf /var/lib/apt/lists/*
+ARG BASE_VARIANT=bullseye
+ARG GO_VERSION=1.16.8
+ARG XX_VERSION=1.0.0-rc.2
 
+ARG LIBGIT2_IMG=ghcr.io/fluxcd/golang-with-libgit2
+ARG LIBGIT2_TAG=libgit2-1.1.1-1
+
+FROM tonistiigi/xx:${XX_VERSION} AS xx
+FROM ${LIBGIT2_IMG}:${LIBGIT2_TAG} as libgit2
+
+FROM golang:${GO_VERSION}-${BASE_VARIANT} as gostable
+
+# Copy the build utiltiies
+COPY --from=xx / /
+COPY --from=libgit2 /Makefile /libgit2/
+
+# Install the libgit2 build dependencies
+RUN make -C /libgit2 cmake
+
+RUN make -C /libgit2 dependencies
+
+# Compile and install libgit2
+RUN FLAGS=$(xx-clang --print-cmake-defines) make -C /libgit2 libgit2
+
+# Use the GitHub Actions uid:gid combination for proper fs permissions
 RUN groupadd -g 116 test && \
     useradd -u 1001 --gid test --shell /bin/sh --create-home test
 
 # Run as test user
 USER test
 
+# Set path to envtest binaries.
+ENV PATH="/github/workspace/envtest:${PATH}"
+
 ENTRYPOINT ["/bin/sh", "-c"]

.github/workflows/build.yaml

@@ -22,8 +22,11 @@ jobs:
           ${{ runner.os }}-go-
     - name: Set up kubebuilder
       uses: fluxcd/pkg/actions/kubebuilder@main
+    - name: Setup envtest
+      uses: fluxcd/pkg/actions/envtest@main
+      with:
+        version: "1.19.2"
     - name: Run tests
       uses: ./.github/actions/run-tests
       env:
         GOPATH: /github/home/go
-        KUBEBUILDER_ASSETS: ${{ github.workspace }}/kubebuilder/bin

.gitignore

@@ -27,3 +27,6 @@ bin
 *.swp
 *.swo
 *~
+
+# Exclude all libgit2 related files
+hack/libgit2/

CONTRIBUTING.md

@@ -27,6 +27,40 @@ to join the conversation (this will also add an invitation to your
 Google calendar for our [Flux
 meeting](https://docs.google.com/document/d/1l_M0om0qUEN_NNiGgpqJ2tvsF2iioHkaARDeh6b70B0/edit#)).
 
+### Installing required dependencies
+
+The dependency [libgit2](https://libgit2.org/) needs to be installed to be able
+to run source-controller or its test-suite locally (not in a container).
+
+In case this dependency is not present on your system (at the expected
+version), the first invocation of a `make` target that requires the
+dependency will attempt to compile it locally to `hack/libgit2`. For this build
+to succeed; CMake, OpenSSL 1.1 and LibSSH2 must be present on the system.
+
+Triggering a manual build of the dependency is possible as well by running
+`make libgit2`. To enforce the build, for example if your system dependencies
+match but are not linked in a compatible way, append `LIBGIT2_FORCE=1` to the
+`make` command.
+
+#### macOS
+
+```console
+$ # Ensure libgit2 dependencies are available
+$ brew install cmake openssl@1.1 libssh2 pkg-config
+$ LIBGIT2_FORCE=1 make libgit2
+```
+
+#### Linux
+
+```console
+$ # Ensure libgit2 dependencies are available
+$ pacman -S cmake openssl libssh2
+$ LIBGIT2_FORCE=1 make libgit2
+```
+
+**Note:** Example shown is for Arch Linux, but likewise procedure can be
+followed using any other package manager, e.g. `apt`.
+
 ### How to run the test suite
 
 Prerequisites:

Dockerfile

@@ -1,26 +1,49 @@
-FROM golang:1.16-buster as builder
-
-# Up-to-date libgit2 dependencies are only available in sid (unstable).
-# The libgit2 dependencies must be listed here to be able to build on ARM64.
-RUN echo "deb http://deb.debian.org/debian unstable main" >> /etc/apt/sources.list \
-    && echo "deb-src http://deb.debian.org/debian unstable main" >> /etc/apt/sources.list
-RUN set -eux; \
-    apt-get update \
-    && apt-get install -y libgit2-dev/unstable zlib1g-dev/unstable libssh2-1-dev/unstable libpcre3-dev/unstable \
-    && apt-get clean \
-    && apt-get autoremove --purge -y \
-    && rm -rf /var/lib/apt/lists/*
+ARG BASE_VARIANT=bullseye
+ARG GO_VERSION=1.16.8
+ARG XX_VERSION=1.0.0-rc.2
 
-WORKDIR /workspace
+ARG LIBGIT2_IMG=ghcr.io/fluxcd/golang-with-libgit2
+ARG LIBGIT2_TAG=libgit2-1.1.1-1
 
-# Copy the Go Modules manifests
-COPY go.mod go.mod
-COPY go.sum go.sum
+FROM --platform=$BUILDPLATFORM tonistiigi/xx:${XX_VERSION} AS xx
+FROM ${LIBGIT2_IMG}:${LIBGIT2_TAG} as libgit2
+
+FROM --platform=$BUILDPLATFORM golang:${GO_VERSION}-${BASE_VARIANT} as gostable
+FROM --platform=$BUILDPLATFORM golang:1.17rc1-${BASE_VARIANT} AS golatest
+
+FROM gostable AS go-linux
+
+FROM go-${TARGETOS} AS build-base-bullseye
+
+# Copy the build utiltiies
+COPY --from=xx / /
+COPY --from=libgit2 /Makefile /libgit2/
+
+# Install the libgit2 build dependencies
+RUN make -C /libgit2 cmake
+
+ARG TARGETPLATFORM
+RUN make -C /libgit2 dependencies
+
+FROM build-base-${BASE_VARIANT} as libgit2-bullseye
+
+# Compile and install libgit2
+ARG TARGETPLATFORM
+RUN FLAGS=$(xx-clang --print-cmake-defines) make -C /libgit2 libgit2
+
+FROM libgit2-${BASE_VARIANT} as build-bullseye
+
+# Configure workspace
+WORKDIR /workspace
 
 # This has its own go.mod, which needs to be present so go mod
 # download works.
 COPY api/ api/
 
+# Copy modules manifests
+COPY go.mod go.mod
+COPY go.sum go.sum
+
 # cache deps before building and copying source so that we don't need to re-download as much
 # and so that source changes don't invalidate our downloaded layer
 RUN go mod download
@@ -30,30 +53,43 @@ COPY main.go main.go
 COPY pkg/ pkg/
 COPY controllers/ controllers/
 
-# Build
-RUN CGO_ENABLED=1 go build -o image-automation-controller main.go
+# Build the binary
+ENV CGO_ENABLED=1
+ARG TARGETPLATFORM
+RUN xx-go build -o image-automation-controller -trimpath \
+    main.go
 
-FROM debian:buster-slim as controller
+FROM build-${BASE_VARIANT} as prepare-bullseye
 
-LABEL org.opencontainers.image.source="https://github.com/fluxcd/image-automation-controller"
+# Move libgit2 lib to generic and predictable location
+ARG TARGETPLATFORM
+RUN mkdir -p /libgit2/lib/ \
+    && cp -d /usr/lib/$(xx-info triple)/libgit2.so* /libgit2/lib/
 
-# Up-to-date libgit2 dependencies are only available in
-# unstable, as libssh2 in testing/bullseye has been linked
-# against gcrypt which causes issues with PKCS* formats.
-RUN echo "deb http://deb.debian.org/debian unstable main" >> /etc/apt/sources.list \
-    && echo "deb-src http://deb.debian.org/debian unstable main" >> /etc/apt/sources.list
-RUN set -eux; \
-    apt-get update \
-    && apt-get install -y ca-certificates libgit2-1.1 \
-    && apt-get clean \
-    && apt-get autoremove --purge -y \
-    && rm -rf /var/lib/apt/lists/*
+FROM prepare-${BASE_VARIANT} as build
 
-COPY --from=builder /workspace/image-automation-controller /usr/local/bin/
+FROM debian:${BASE_VARIANT}-slim as controller
 
+# Configure user
 RUN groupadd controller && \
     useradd --gid controller --shell /bin/sh --create-home controller
 
-USER controller
+# Copy libgit2
+COPY --from=build /libgit2/lib/ /usr/local/lib/
+RUN ldconfig
+
+# Upgrade packages and install runtime dependencies
+RUN echo "deb http://deb.debian.org/debian sid main" >> /etc/apt/sources.list \
+    && echo "deb-src http://deb.debian.org/debian sid main" >> /etc/apt/sources.list \
+    && apt update \
+    && apt install --no-install-recommends -y zlib1g/sid libssl1.1/sid libssh2-1/sid \
+    && apt install --no-install-recommends -y ca-certificates \
+    && apt clean \
+    && apt autoremove --purge -y \
+    && rm -rf /var/lib/apt/lists/*
+
+# Copy over binary from build
+COPY --from=build /workspace/image-automation-controller /usr/local/bin/
 
+USER controller
 ENTRYPOINT [ "image-automation-controller" ]