Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add multistage docker builds #825

Merged
merged 21 commits into from
Jan 31, 2020
Merged

Add multistage docker builds #825

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
21 commits
Select commit Hold shift + click to select a range
09af09e
Convert spaces in Dockerfiles to tabs
Dean-Coakley Dec 2, 2019
14aa31b
Add multi stage docker builds
Dean-Coakley Dec 4, 2019
8d6342a
Update docker for travisci
Dean-Coakley Jan 16, 2020
6085b7f
Try update docker in travisci
Dean-Coakley Jan 20, 2020
baea83e
Improve targets
Dean-Coakley Jan 20, 2020
12dd336
Fix travis file
Dean-Coakley Jan 20, 2020
05a74fd
Test docker versions
Dean-Coakley Jan 21, 2020
1ac75e4
Try docker 18.09
Dean-Coakley Jan 21, 2020
5043119
Upgrade Docker version to 19.03.5
martialonline Jan 21, 2020
4007b14
Remove Docker upgrade and BUILDKIT from travis
martialonline Jan 21, 2020
1fbf9bb
Fixed typo in local-container
martialonline Jan 21, 2020
f597981
Re-add test target to Makefile
Dean-Coakley Jan 21, 2020
be8cd0b
Update Makefile docs
Dean-Coakley Jan 21, 2020
d1d1fdb
Make default behaviour consistent with master. Docs
Dean-Coakley Jan 22, 2020
3953b3e
More consistency improvements
Dean-Coakley Jan 23, 2020
21540d7
Remove local-container target
Dean-Coakley Jan 23, 2020
c466ee8
Fix docs
Dean-Coakley Jan 23, 2020
932080f
Remove unnecessary Makefile args
Dean-Coakley Jan 23, 2020
0008a51
Fix setting build args. PR feedback
Dean-Coakley Jan 24, 2020
71e02af
Add golang:latest default for jenkins
Dean-Coakley Jan 24, 2020
ecc5245
Fix typo in opentracing build
Dean-Coakley Jan 25, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 0 additions & 3 deletions .gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,8 +39,5 @@ cmd/nginx-ingress/nginx-ingress
# Default certificate and key
default.pem

# Dockerfiles for building
/Dockerfile

# IntelliJ IDEA
.idea
4 changes: 2 additions & 2 deletions .travis.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ go:
- "1.13"
script:
- echo "Building ingress controller commit:${TRAVIS_COMMIT}"
- make BUILD_IN_CONTAINER=0 container;
- make container BUILD_IN_CONTAINER=0 DOCKER_BUILDKIT=0;
lucacome marked this conversation as resolved.
Show resolved Hide resolved
- echo "Helm smoke test"
- cd ${TRAVIS_BUILD_DIR} && wget -O helm.tar.gz ${helm_download} &&
tar xzfv helm.tar.gz -C ./ --strip-components=1 linux-amd64/helm
Expand All @@ -20,4 +20,4 @@ before_install:
&& ./fossa init
&& GO111MODULE=on FOSSA_API_KEY=${fossapush} ./fossa analyze -t kubernetes-ingress -b ${TRAVIS_BRANCH}; fi
- echo "Checking ingress controller for linting errors" && wget -O - -q ${golangci_lint} | sh -s v1.17.1
- export PATH="$PATH:./bin" && make lint
- export PATH="$PATH:./bin" && make lint
32 changes: 15 additions & 17 deletions Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,6 @@ VERSION = edge
TAG = $(VERSION)
PREFIX = nginx/nginx-ingress

DOCKER_TEST_RUN = docker run --rm -v $(shell pwd):/go/src/github.com/nginxinc/kubernetes-ingress -w /go/src/github.com/nginxinc/kubernetes-ingress
DOCKER_BUILD_RUN = docker run --rm -v $(shell pwd):/go/src/github.com/nginxinc/kubernetes-ingress -w /go/src/github.com/nginxinc/kubernetes-ingress/cmd/nginx-ingress/
GOLANG_CONTAINER = golang:1.13
Dean-Coakley marked this conversation as resolved.
Show resolved Hide resolved
DOCKERFILEPATH = build
DOCKERFILE = Dockerfile # note, this can be overwritten e.g. can be DOCKERFILE=DockerFileForPlus
Expand All @@ -15,27 +13,20 @@ PUSH_TO_GCR =
GENERATE_DEFAULT_CERT_AND_KEY =
DOCKER_BUILD_OPTIONS =

GIT_COMMIT=$(shell git rev-parse --short HEAD)
GIT_COMMIT = $(shell git rev-parse --short HEAD)

nginx-ingress:
ifeq ($(BUILD_IN_CONTAINER),1)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

similar thing should be done to the test stage - run it inside the builder container, not here

$(DOCKER_BUILD_RUN) -e CGO_ENABLED=0 -e GO111MODULE=on -e GOFLAGS='-mod=vendor' $(GOLANG_CONTAINER) go build -installsuffix cgo -ldflags "-w -X main.version=${VERSION} -X main.gitCommit=${GIT_COMMIT}" -o /go/src/github.com/nginxinc/kubernetes-ingress/nginx-ingress
else
CGO_ENABLED=0 GO111MODULE=on GOFLAGS='-mod=vendor' GOOS=linux go build -installsuffix cgo -ldflags "-w -X main.version=${VERSION} -X main.gitCommit=${GIT_COMMIT}" -o nginx-ingress github.com/nginxinc/kubernetes-ingress/cmd/nginx-ingress
endif
export DOCKER_BUILDKIT = 1

lint:
golangci-lint run

test:
ifeq ($(BUILD_IN_CONTAINER),1)
$(DOCKER_TEST_RUN) -e GO111MODULE=on -e GOFLAGS='-mod=vendor' $(GOLANG_CONTAINER) go test ./...
else
ifneq ($(BUILD_IN_CONTAINER),1)
GO111MODULE=on GOFLAGS='-mod=vendor' go test ./...
endif

verify-codegen:
ifneq ($(BUILD_IN_CONTAINER), 1)
ifneq ($(BUILD_IN_CONTAINER),1)
./hack/verify-codegen.sh
endif

Expand All @@ -47,9 +38,17 @@ ifeq ($(GENERATE_DEFAULT_CERT_AND_KEY),1)
./build/generate_default_cert_and_key.sh
endif

container: test verify-codegen nginx-ingress certificate-and-key
cp $(DOCKERFILEPATH)/$(DOCKERFILE) ./Dockerfile
docker build $(DOCKER_BUILD_OPTIONS) --build-arg IC_VERSION=$(VERSION)-$(GIT_COMMIT) -f Dockerfile -t $(PREFIX):$(TAG) .
binary:
ifneq ($(BUILD_IN_CONTAINER),1)
CGO_ENABLED=0 GO111MODULE=on GOFLAGS='-mod=vendor' GOOS=linux go build -installsuffix cgo -ldflags "-w -X main.version=${VERSION} -X main.gitCommit=${GIT_COMMIT}" -o nginx-ingress github.com/nginxinc/kubernetes-ingress/cmd/nginx-ingress
endif

container: test verify-codegen binary certificate-and-key
ifeq ($(BUILD_IN_CONTAINER),1)
Dean-Coakley marked this conversation as resolved.
Show resolved Hide resolved
docker build $(DOCKER_BUILD_OPTIONS) --build-arg IC_VERSION=$(VERSION)-$(GIT_COMMIT) --build-arg GIT_COMMIT=$(GIT_COMMIT) --build-arg VERSION=$(VERSION) --build-arg GOLANG_CONTAINER=$(GOLANG_CONTAINER) --target container -f $(DOCKERFILEPATH)/$(DOCKERFILE) -t $(PREFIX):$(TAG) .
else
docker build $(DOCKER_BUILD_OPTIONS) --build-arg IC_VERSION=$(VERSION)-$(GIT_COMMIT) --target local -f $(DOCKERFILEPATH)/$(DOCKERFILE) -t $(PREFIX):$(TAG) .
endif

push: container
ifeq ($(PUSH_TO_GCR),1)
Expand All @@ -60,4 +59,3 @@ endif

clean:
rm -f nginx-ingress
rm -f Dockerfile
Dean-Coakley marked this conversation as resolved.
Show resolved Hide resolved
Dean-Coakley marked this conversation as resolved.
Show resolved Hide resolved
25 changes: 23 additions & 2 deletions build/Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,6 @@
FROM nginx:1.17.7
ARG GOLANG_CONTAINER=golang:latest
Dean-Coakley marked this conversation as resolved.
Show resolved Hide resolved

FROM nginx:1.17.7 AS base

# forward nginx access and error logs to stdout and stderr of the ingress
# controller process
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I've never seen /proc/1/fd/1 being used instead of /dev/stdout in a Dockerfile and looks weird to me 😄. Any chance you could change it since you're modifying the files anyway? Also, the nginx base image already has the forward for error and access log, so we really need just the one for stream-access.log.

Expand All @@ -19,7 +21,9 @@ RUN mkdir -p /var/lib/nginx \
&& rm /etc/nginx/conf.d/* \
&& rm -rf /var/lib/apt/lists/*

COPY nginx-ingress internal/configs/version1/nginx.ingress.tmpl internal/configs/version1/nginx.tmpl internal/configs/version2/nginx.virtualserver.tmpl /
COPY internal/configs/version1/nginx.ingress.tmpl \
internal/configs/version1/nginx.tmpl \
internal/configs/version2/nginx.virtualserver.tmpl /

# Uncomment the line below if you would like to add the default.pem to the image
# and use it as a certificate and key for the default server
Expand All @@ -28,3 +32,20 @@ COPY nginx-ingress internal/configs/version1/nginx.ingress.tmpl internal/configs
USER nginx

ENTRYPOINT ["/nginx-ingress"]


FROM base AS local
COPY nginx-ingress /


FROM $GOLANG_CONTAINER AS builder
ARG VERSION
ARG GIT_COMMIT
WORKDIR /go/src/github.com/nginxinc/kubernetes-ingress/nginx-ingress/cmd/nginx-ingress
COPY . /go/src/github.com/nginxinc/kubernetes-ingress/nginx-ingress/
RUN CGO_ENABLED=0 GOFLAGS='-mod=vendor' \
go build -installsuffix cgo -ldflags "-w -X main.version=${VERSION} -X main.gitCommit=${GIT_COMMIT}" -o /nginx-ingress


FROM base AS container
COPY --from=builder /nginx-ingress /
25 changes: 23 additions & 2 deletions build/DockerfileForAlpine
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,6 @@
FROM nginx:1.17.7-alpine
ARG GOLANG_CONTAINER=golang:latest

FROM nginx:1.17.7-alpine AS base
Dean-Coakley marked this conversation as resolved.
Show resolved Hide resolved

# forward nginx access and error logs to stdout and stderr of the ingress
# controller process
Expand All @@ -18,7 +20,9 @@ RUN mkdir -p /etc/nginx/secrets \
&& rm /etc/nginx/conf.d/* \
&& rm -rf /var/cache/apk/*

COPY nginx-ingress internal/configs/version1/nginx.ingress.tmpl internal/configs/version1/nginx.tmpl internal/configs/version2/nginx.virtualserver.tmpl /
COPY internal/configs/version1/nginx.ingress.tmpl \
internal/configs/version1/nginx.tmpl \
internal/configs/version2/nginx.virtualserver.tmpl /

# Uncomment the line below if you would like to add the default.pem to the image
# and use it as a certificate and key for the default server
Expand All @@ -27,3 +31,20 @@ COPY nginx-ingress internal/configs/version1/nginx.ingress.tmpl internal/configs
USER nginx

ENTRYPOINT ["/nginx-ingress"]


FROM base AS local
COPY nginx-ingress /


FROM $GOLANG_CONTAINER AS builder
ARG VERSION
ARG GIT_COMMIT
WORKDIR /go/src/github.com/nginxinc/kubernetes-ingress/nginx-ingress/cmd/nginx-ingress
COPY . /go/src/github.com/nginxinc/kubernetes-ingress/nginx-ingress/
RUN CGO_ENABLED=0 GOFLAGS='-mod=vendor' \
go build -installsuffix cgo -ldflags "-w -X main.version=${VERSION} -X main.gitCommit=${GIT_COMMIT}" -o /nginx-ingress


FROM base AS container
COPY --from=builder /nginx-ingress /
104 changes: 62 additions & 42 deletions build/DockerfileForPlus
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,6 @@
FROM debian:stretch-slim
ARG GOLANG_CONTAINER=golang:latest

FROM debian:stretch-slim AS base

LABEL maintainer="NGINX Docker Maintainers <docker-maint@nginx.com>"

Expand All @@ -7,66 +9,84 @@ ARG IC_VERSION

# Download certificate and key from the customer portal (https://cs.nginx.com)
# and copy to the build context
COPY nginx-repo.crt /etc/ssl/nginx/
COPY nginx-repo.key /etc/ssl/nginx/
COPY nginx-repo.crt nginx-repo.key /etc/ssl/nginx/

# Make sure the certificate and key have correct permissions
RUN chmod 644 /etc/ssl/nginx/*

# Install NGINX Plus
RUN set -x \
&& apt-get update \
&& apt-get install --no-install-recommends --no-install-suggests -y apt-transport-https ca-certificates gnupg1 libcap2-bin \
&& \
NGINX_GPGKEY=573BFD6B3D8FBC641079A6ABABF5BD827BD9BF62; \
found=''; \
for server in \
ha.pool.sks-keyservers.net \
hkp://keyserver.ubuntu.com:80 \
hkp://p80.pool.sks-keyservers.net:80 \
pgp.mit.edu \
; do \
echo "Fetching GPG key $NGINX_GPGKEY from $server"; \
apt-key adv --keyserver "$server" --keyserver-options timeout=10 --recv-keys "$NGINX_GPGKEY" && found=yes && break; \
done; \
test -z "$found" && echo >&2 "error: failed to fetch GPG key $NGINX_GPGKEY" && exit 1; \
echo "Acquire::https::plus-pkgs.nginx.com::Verify-Peer \"true\";" >> /etc/apt/apt.conf.d/90nginx \
&& echo "Acquire::https::plus-pkgs.nginx.com::Verify-Host \"true\";" >> /etc/apt/apt.conf.d/90nginx \
&& echo "Acquire::https::plus-pkgs.nginx.com::SslCert \"/etc/ssl/nginx/nginx-repo.crt\";" >> /etc/apt/apt.conf.d/90nginx \
&& echo "Acquire::https::plus-pkgs.nginx.com::SslKey \"/etc/ssl/nginx/nginx-repo.key\";" >> /etc/apt/apt.conf.d/90nginx \
&& echo "Acquire::https::plus-pkgs.nginx.com::User-Agent \"k8s-ic-$IC_VERSION-apt\";" >> /etc/apt/apt.conf.d/90nginx \
&& printf "deb https://plus-pkgs.nginx.com/debian stretch nginx-plus\n" > /etc/apt/sources.list.d/nginx-plus.list \
&& apt-get update && apt-get install -y nginx-plus=${NGINX_PLUS_VERSION} \
&& setcap 'cap_net_bind_service=+ep' /usr/sbin/nginx \
&& setcap 'cap_net_bind_service=+ep' /usr/sbin/nginx-debug \
&& apt-get remove --purge --auto-remove -y gnupg1 \
&& rm -rf /var/lib/apt/lists/* \
&& rm -rf /etc/ssl/nginx \
&& rm /etc/apt/apt.conf.d/90nginx /etc/apt/sources.list.d/nginx-plus.list
&& apt-get update \
&& apt-get install --no-install-recommends --no-install-suggests -y apt-transport-https ca-certificates gnupg1 libcap2-bin \
&& \
NGINX_GPGKEY=573BFD6B3D8FBC641079A6ABABF5BD827BD9BF62; \
found=''; \
for server in \
ha.pool.sks-keyservers.net \
hkp://keyserver.ubuntu.com:80 \
hkp://p80.pool.sks-keyservers.net:80 \
pgp.mit.edu \
; do \
echo "Fetching GPG key $NGINX_GPGKEY from $server"; \
apt-key adv --keyserver "$server" --keyserver-options timeout=10 --recv-keys "$NGINX_GPGKEY" && found=yes && break; \
done; \
test -z "$found" && echo >&2 "error: failed to fetch GPG key $NGINX_GPGKEY" && exit 1; \
echo "Acquire::https::plus-pkgs.nginx.com::Verify-Peer \"true\";" >> /etc/apt/apt.conf.d/90nginx \
&& echo "Acquire::https::plus-pkgs.nginx.com::Verify-Host \"true\";" >> /etc/apt/apt.conf.d/90nginx \
&& echo "Acquire::https::plus-pkgs.nginx.com::SslCert \"/etc/ssl/nginx/nginx-repo.crt\";" >> /etc/apt/apt.conf.d/90nginx \
&& echo "Acquire::https::plus-pkgs.nginx.com::SslKey \"/etc/ssl/nginx/nginx-repo.key\";" >> /etc/apt/apt.conf.d/90nginx \
&& echo "Acquire::https::plus-pkgs.nginx.com::User-Agent \"k8s-ic-$IC_VERSION-apt\";" >> /etc/apt/apt.conf.d/90nginx \
&& printf "deb https://plus-pkgs.nginx.com/debian stretch nginx-plus\n" > /etc/apt/sources.list.d/nginx-plus.list \
&& apt-get update && apt-get install -y nginx-plus=${NGINX_PLUS_VERSION} \
&& setcap 'cap_net_bind_service=+ep' /usr/sbin/nginx \
&& setcap 'cap_net_bind_service=+ep' /usr/sbin/nginx-debug \
&& apt-get remove --purge --auto-remove -y gnupg1 \
&& rm -rf /var/lib/apt/lists/* \
&& rm -rf /etc/ssl/nginx \
&& rm /etc/apt/apt.conf.d/90nginx /etc/apt/sources.list.d/nginx-plus.list


# forward nginx access and error logs to stdout and stderr of the ingress
# controller process
RUN ln -sf /proc/1/fd/1 /var/log/nginx/access.log \
&& ln -sf /proc/1/fd/1 /var/log/nginx/stream-access.log \
&& ln -sf /proc/1/fd/2 /var/log/nginx/error.log
&& ln -sf /proc/1/fd/1 /var/log/nginx/stream-access.log \
&& ln -sf /proc/1/fd/2 /var/log/nginx/error.log

RUN mkdir -p /var/lib/nginx \
&& mkdir -p /etc/nginx/secrets \
&& chown -R nginx:0 /etc/nginx \
&& chown -R nginx:0 /var/cache/nginx \
&& chown -R nginx:0 /var/lib/nginx/ \
&& apt-get remove --purge -y libcap2-bin \
&& rm /etc/nginx/conf.d/*
RUN mkdir -p /var/lib/nginx \
&& mkdir -p /etc/nginx/secrets \
&& chown -R nginx:0 /etc/nginx \
&& chown -R nginx:0 /var/cache/nginx \
&& chown -R nginx:0 /var/lib/nginx/ \
&& apt-get remove --purge -y libcap2-bin \
&& rm /etc/nginx/conf.d/*

EXPOSE 80 443

COPY nginx-ingress internal/configs/version1/nginx-plus.ingress.tmpl internal/configs/version1/nginx-plus.tmpl internal/configs/version2/nginx-plus.virtualserver.tmpl /
COPY internal/configs/version1/nginx-plus.ingress.tmpl \
internal/configs/version1/nginx-plus.tmpl \
internal/configs/version2/nginx-plus.virtualserver.tmpl /

# Uncomment the line below if you would like to add the default.pem to the image
# and use it as a certificate and key for the default server
# ADD default.pem /etc/nginx/secrets/default

USER nginx

ENTRYPOINT ["/nginx-ingress"]
ENTRYPOINT ["/nginx-ingress"]


FROM base AS local
COPY nginx-ingress /


FROM $GOLANG_CONTAINER AS builder
ARG VERSION
ARG GIT_COMMIT
WORKDIR /go/src/github.com/nginxinc/kubernetes-ingress/nginx-ingress/cmd/nginx-ingress
COPY . /go/src/github.com/nginxinc/kubernetes-ingress/nginx-ingress/
RUN CGO_ENABLED=0 GOFLAGS='-mod=vendor' \
go build -installsuffix cgo -ldflags "-w -X main.version=${VERSION} -X main.gitCommit=${GIT_COMMIT}" -o /nginx-ingress


FROM base AS container
COPY --from=builder /nginx-ingress /
Loading