Merge pull request kubernetes#88 from Shopify/sync-upstream

Sync upstream

.travis.yml

@@ -15,9 +15,10 @@ notifications:
 # New secure variables can be added using travis encrypt -r kubernetes/ingress-nginx --add K=V
 env:
  global:
- - CHANGE_MINIKUBE_NONE_USER=true
- - KUBERNETES_VERSION=v1.10.0
  - DOCKER=docker
+ - SKIP_SNAPSHOT=true
+ - NODE_IP=10.192.0.3
+ - E2E_NODES=6
  - GH_REF=github.com/kubernetes/ingress-nginx
 
 jobs:
@@ -34,7 +35,6 @@ jobs:
  - stage: e2e
  before_script:
  - test/e2e/up.sh
- - SKIP_MINIKUBE_START=true make dev-env
  script:
  - make e2e-test
  # split builds to avoid job timeouts

Makefile

@@ -26,7 +26,9 @@ GOHOSTOS ?= $(shell go env GOHOSTOS)
 # Allow limiting the scope of the e2e tests. By default run everything
 FOCUS ?= .*
 # number of parallel test
-E2E_NODES ?= 3
+E2E_NODES ?= 4
+# slow test only if takes > 40s
+SLOW_E2E_THRESHOLD ?= 40
 
 NODE_IP ?= $(shell minikube ip)
 
@@ -179,11 +181,12 @@ lua-test:
 
 .PHONY: e2e-test
 e2e-test:
- @$(DEF_VARS) \
- FOCUS=$(FOCUS) \
- E2E_NODES=$(E2E_NODES) \
- DOCKER_OPTS="-i --net=host" \
- NODE_IP=$(NODE_IP) \
+ @$(DEF_VARS) \
+ FOCUS=$(FOCUS) \
+ E2E_NODES=$(E2E_NODES) \
+ DOCKER_OPTS="-i --net=host" \
+ NODE_IP=$(NODE_IP) \
+ SLOW_E2E_THRESHOLD=$(SLOW_E2E_THRESHOLD) \
  build/go-in-docker.sh build/e2e-tests.sh
 
 .PHONY: cover

build/build.sh

@@ -18,29 +18,25 @@ set -o errexit
 set -o nounset
 set -o pipefail
 
-if [ -z "${PKG}" ]; then
- echo "PKG must be set"
- exit 1
-fi
-if [ -z "${ARCH}" ]; then
- echo "ARCH must be set"
- exit 1
-fi
-if [ -z "${GIT_COMMIT}" ]; then
- echo "GIT_COMMIT must be set"
- exit 1
-fi
-if [ -z "${REPO_INFO}" ]; then
- echo "REPO_INFO must be set"
- exit 1
-fi
-if [ -z "${TAG}" ]; then
- echo "TAG must be set"
- exit 1
-fi
-if [ -z "${TAG}" ]; then
- echo "TAG must be set"
- exit 1
+declare -a mandatory
+mandatory=(
+ PKG
+ ARCH
+ GIT_COMMIT
+ REPO_INFO
+ TAG
+)
+
+missing=false
+for var in ${mandatory[@]}; do
+ if [[ -z "${!var+x}" ]]; then
+ echo "Environment variable $var must be set"
+ missing=true
+ fi
+done
+
+if [ "$missing" = true ];then
+ exit 1
 fi
 
 export CGO_ENABLED=0

build/e2e-tests.sh

@@ -18,21 +18,25 @@ set -o errexit
 set -o nounset
 set -o pipefail
 
-if [ -z "${PKG}" ]; then
- echo "PKG must be set"
- exit 1
-fi
-if [ -z "${FOCUS}" ]; then
- echo "FOCUS must be set"
- exit 1
-fi
-if [ -z "${E2E_NODES}" ]; then
- echo "E2E_NODES must be set"
- exit 1
-fi
-if [ -z "${NODE_IP}" ]; then
- echo "NODE_IP must be set"
- exit 1
+declare -a mandatory
+mandatory=(
+ NODE_IP
+ SLOW_E2E_THRESHOLD
+ PKG
+ FOCUS
+ E2E_NODES
+)
+
+missing=false
+for var in ${mandatory[@]}; do
+ if [[ -z "${!var+x}" ]]; then
+ echo "Environment variable $var must be set"
+ missing=true
+ fi
+done
+
+if [ "$missing" = true ];then
+ exit 1
 fi
 
 SCRIPT_ROOT=$(dirname ${BASH_SOURCE})/..
@@ -52,13 +56,14 @@ fi
 
 ginkgo build ./test/e2e
 
-exec -- \
-ginkgo \
- -randomizeSuites \
- -randomizeAllSpecs \
- -flakeAttempts=2 \
- --focus=${FOCUS} \
- -p \
- -trace \
- -nodes=${E2E_NODES} \
+exec -- \
+ginkgo \
+ -randomizeSuites \
+ -randomizeAllSpecs \
+ -flakeAttempts=2 \
+ --focus=${FOCUS} \
+ -p \
+ -trace \
+ -nodes=${E2E_NODES} \
+ -slowSpecThreshold=${SLOW_E2E_THRESHOLD} \
  test/e2e/e2e.test

build/go-in-docker.sh

@@ -18,42 +18,39 @@ set -o errexit
 set -o nounset
 set -o pipefail
 
-if [ -z "${PKG}" ]; then
- echo "PKG must be set"
- exit 1
-fi
-if [ -z "${ARCH}" ]; then
- echo "ARCH must be set"
- exit 1
-fi
-if [ -z "${GIT_COMMIT}" ]; then
- echo "GIT_COMMIT must be set"
- exit 1
-fi
-if [ -z "${REPO_INFO}" ]; then
- echo "REPO_INFO must be set"
- exit 1
-fi
-if [ -z "${TAG}" ]; then
- echo "TAG must be set"
- exit 1
-fi
-if [ -z "${HOME}" ]; then
- echo "HOME must be set"
- exit 1
+declare -a mandatory
+mandatory=(
+ PKG
+ ARCH
+ GIT_COMMIT
+ REPO_INFO
+ TAG
+ HOME
+)
+
+missing=false
+for var in ${mandatory[@]}; do
+ if [[ -z "${!var+x}" ]]; then
+ echo "Environment variable $var must be set"
+ missing=true
+ fi
+done
+
+if [ "$missing" = true ];then
+ exit 1
 fi
 
-DOCKER_OPTS=${DOCKER_OPTS:-""}
+E2E_IMAGE=quay.io/kubernetes-ingress-controller/e2e:v07282018-45ba1672c
 
-docker build -t ingress-nginx:build build
+DOCKER_OPTS=${DOCKER_OPTS:-""}
 
 FLAGS=$@
 
 tee .env << EOF
 PKG=${PKG:-""}
 ARCH=${ARCH:-""}
 GIT_COMMIT=${GIT_COMMIT:-""}
-E2E_NODES=${E2E_NODES:-3}
+E2E_NODES=${E2E_NODES:-4}
 FOCUS=${FOCUS:-.*}
 TAG=${TAG:-"0.0"}
 HOME=${HOME:-/root}
@@ -63,6 +60,7 @@ PWD=${PWD}
 BUSTED_ARGS=${BUSTED_ARGS:-""}
 REPO_INFO=${REPO_INFO:-local}
 NODE_IP=${NODE_IP:-127.0.0.1}
+SLOW_E2E_THRESHOLD=${SLOW_E2E_THRESHOLD:-40}
 EOF
 
 docker run \
@@ -76,6 +74,6 @@ docker run \
  -v ${PWD}/bin/${ARCH}:/go/bin/linux_${ARCH} \
  -w /go/src/${PKG} \
  --env-file .env \
- ingress-nginx:build ${FLAGS}
+ ${E2E_IMAGE} ${FLAGS}
 
 rm .env

docs/deploy/index.md

@@ -31,10 +31,8 @@ There are cloud provider specific yaml files.
 
 #### Docker for Mac
 
-Kubernetes is available for Docker for Mac's Edge channel. Switch to the [Edge
-channel][edge] and [enable Kubernetes][enable].
+Kubernetes is available in Docker for Mac (from [version 18.06.0-ce](https://docs.docker.com/docker-for-mac/release-notes/#stable-releases-of-2018))
 
-[edge]: https://docs.docker.com/docker-for-mac/install/
 [enable]: https://docs.docker.com/docker-for-mac/#kubernetes
 
 Create a service

docs/how-it-works.md

@@ -1,16 +1,16 @@
 # How it works
 
-The objective of this document explains how the NGINX Ingress controller works, in particular how the NGINX model is built and why we need a one.
+The objective of this document is to explain how the NGINX Ingress controller works, in particular how the NGINX model is built and why we need a one.
 
 ## NGINX configuration
 
-The goal of this Ingress controller is the assembly of a configuration file (nginx.conf). The main implication of this requirement is the need to reload NGINX after any change in the configuration file.
+The goal of this Ingress controller is the assembly of a configuration file (nginx.conf). The main implication of this requirement is the need to reload NGINX after any change in the configuration file. _Though it is important to note that we don't reload Nginx on changes that impact only an `upstream` configuration (i.e Endpoints change when you deploy your app)_. We use https://github.com/openresty/lua-nginx-module to achieve this. Check [below](#avoiding-reloads-on-endpoints-changes) to learn more about how it's done.
 
 ## NGINX model
 
 Usually, a Kubernetes Controller utilizes the [synchronization loop pattern](1) to check if the desired state in the controller is updated or a change is required. To this purpose, we need to build a model using different objects from the cluster, in particular (in no special order) Ingresses, Services, Endpoints, Secrets, and Configmaps to generate a point in time configuration file that reflects the state of the cluster.
 
-To get this object from the cluster, we use [Kubernetes Informers](2), in particular, `FilteredSharedInformer`. This informers allows reacting to changes in using [callbacks](3) to individual changes when a new object is added, modified or removed. Unfortunately, there is no way to know if a particular change is going to affect the final configuration file. Therefore on every change, we have to rebuild a new model from scratch based on the state of cluster and compare it to the current model. If the new model equals to the current one, then we avoid generating a new NGINX configuration and [trigger a reload](7). Otherwise, we create a new NGINX configuration based on the new model, replace the current model and [trigger a reload](7).
+To get this object from the cluster, we use [Kubernetes Informers](2), in particular, `FilteredSharedInformer`. This informers allows reacting to changes in using [callbacks](3) to individual changes when a new object is added, modified or removed. Unfortunately, there is no way to know if a particular change is going to affect the final configuration file. Therefore on every change, we have to rebuild a new model from scratch based on the state of cluster and compare it to the current model. If the new model equals to the current one, then we avoid generating a new NGINX configuration and triggering a reload. Otherwise, we check if the difference is only about Endpoints. If so we then send the new list of Endpoints to a Lua handler running inside Nginx using HTTP POST request and again avoid generating a new NGINX configuration and triggering a reload. If the difference between running and new model is about more than just Endpoints we create a new NGINX configuration based on the new model, replace the current model and trigger a reload.
 
 One of the uses of the model is to avoid unnecessary reloads when there's no change in the state and to detect conflicts in definitions.
 
@@ -39,21 +39,26 @@ The next list describes the scenarios when a reload is required:
 
 - New Ingress Resource Created.
 - TLS section is added to existing Ingress.
-- Change in Ingress annotations.
+- Change in Ingress annotations that impacts more than just upstream configuration. For instance `load-balance` annotation does not require a reload.
 - A path is added/removed from an Ingress.
 - An Ingress, Service, Secret is removed.
-- Some missing referenced object from the Ingress is available, like a Service, Secret or Endpoint.
+- Some missing referenced object from the Ingress is available, like a Service or Secret.
 - A Secret is updated.
 
 ## Avoiding reloads
 
 In some cases, it is possible to avoid reloads, in particular when there is a change in the endpoints, i.e., a pod is started or replaced. It is out of the scope of this Ingress controller to remove reloads completely. This would require an incredible amount of work and at some point makes no sense. This can change only if NGINX changes the way new configurations are read, basically, new changes do not replace worker processes.
 
+### Avoiding reloads on Endpoints changes
+On every endpoint change the controller fetches endpoints from all the services it sees and generates corresponding Backend objects. It then sends these objects to a Lua handler running inside Nginx. The Lua code in turn stores those backends in a shared memory zone. Then for every request Lua code running in [`balancer_by_lua`](https://github.com/openresty/lua-resty-core/blob/master/lib/ngx/balancer.md) context detects what endpoints it should choose upstream peer from and applies the configured load balancing algorithm to choose the peer. Then Nginx takes care of the rest. This way we avoid reloading Nginx on endpoint changes. _Note_ that this includes annotation changes that affects only `upstream` configuration in Nginx as well.
+
+In a relatively big clusters with frequently deploying apps this feature saves significant number of Nginx reloads which can otherwise affect response latency, load balancing quality (after every reload Nginx resets the state of load balancing) and so on.
+
 [0]: https://github.com/openresty/lua-nginx-module/pull/1259
 [1]: https://coreos.com/kubernetes/docs/latest/replication-controller.html#the-reconciliation-loop-in-detail
 [2]: https://godoc.org/k8s.io/client-go/informers#NewFilteredSharedInformerFactory
 [3]: https://godoc.org/k8s.io/client-go/tools/cache#ResourceEventHandlerFuncs
 [4]: https://github.com/kubernetes/ingress-nginx/blob/master/internal/task/queue.go#L38
 [5]: https://golang.org/pkg/sync/#Mutex
 [6]: https://github.com/kubernetes/ingress-nginx/blob/master/rootfs/etc/nginx/template/nginx.tmpl
-[7]: http://nginx.org/en/docs/beginners_guide.html#control
+[7]: http://nginx.org/en/docs/beginners_guide.html#control

docs/user-guide/nginx-configuration/annotations.md

@@ -43,6 +43,7 @@ You can add these Kubernetes annotations to specific Ingress objects to customiz
 |[nginx.ingress.kubernetes.io/limit-connections](#rate-limiting)|number|
 |[nginx.ingress.kubernetes.io/limit-rps](#rate-limiting)|number|
 |[nginx.ingress.kubernetes.io/permanent-redirect](#permanent-redirect)|string|
+|[nginx.ingress.kubernetes.io/permanent-redirect-code](#permanent-redirect-code)|number|
 |[nginx.ingress.kubernetes.io/proxy-body-size](#custom-max-body-size)|string|
 |[nginx.ingress.kubernetes.io/proxy-cookie-domain](#proxy-cookie-domain)|string|
 |[nginx.ingress.kubernetes.io/proxy-connect-timeout](#custom-timeouts)|number|
@@ -366,6 +367,10 @@ To configure this setting globally for all Ingress rules, the `limit-rate-after`
 
 This annotation allows to return a permanent redirect instead of sending data to the upstream. For example `nginx.ingress.kubernetes.io/permanent-redirect: https://www.google.com` would redirect everything to Google.
 
+### Permanent Redirect Code
+
+This annotation allows you to modify the status code used for permanent redirects. For example `nginx.ingress.kubernetes.io/permanent-redirect-code: '308'` would return your permanet-redirect with a 308.
+
 ### SSL Passthrough
 
 The annotation `nginx.ingress.kubernetes.io/ssl-passthrough` allows to configure TLS termination in the pod and not in NGINX.