diff --git a/build/dev-env.sh b/build/dev-env.sh index e0a828ca33..59b883cdcf 100755 --- a/build/dev-env.sh +++ b/build/dev-env.sh @@ -47,16 +47,21 @@ make build container docker save "${DEV_IMAGE}" | (eval $(minikube docker-env --shell bash) && docker load) || true -echo "[dev-env] installing kubectl" -kubectl version || brew install kubectl +for tool in kubectl kustomize; do + echo "[dev-env] installing $tool" + $tool version || brew install $tool +done + +if ! kubectl get namespace $NAMESPACE; then + kubectl create namespace $NAMESPACE +fi + +ROOT=./deploy/minikube + +pushd $ROOT +kustomize edit set namespace $NAMESPACE +kustomize edit set image quay.io/kubernetes-ingress-controller/nginx-ingress-controller=${DEV_IMAGE} +popd echo "[dev-env] deploying NGINX Ingress controller in namespace $NAMESPACE" -cat ./deploy/mandatory.yaml | kubectl apply --namespace=$NAMESPACE -f - -cat ./deploy/provider/baremetal/service-nodeport.yaml | kubectl apply --namespace=$NAMESPACE -f - - -echo "updating image..." -kubectl set image \ - deployments \ - --namespace ingress-nginx \ - --selector app.kubernetes.io/name=ingress-nginx \ - nginx-ingress-controller=${DEV_IMAGE} +kustomize build $ROOT | kubectl apply -f - diff --git a/deploy/aws/l4/kustomization.yaml b/deploy/aws/l4/kustomization.yaml new file mode 100644 index 0000000000..a17bd9156e --- /dev/null +++ b/deploy/aws/l4/kustomization.yaml @@ -0,0 +1,11 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +bases: +- ../../cloud-generic +patchesStrategicMerge: +- service-l4.yaml +configMapGenerator: +- name: nginx-configuration + behavior: merge + literals: + - use-proxy-protocol=true diff --git a/deploy/provider/aws/service-l4.yaml b/deploy/aws/l4/service-l4.yaml similarity index 56% rename from deploy/provider/aws/service-l4.yaml rename to deploy/aws/l4/service-l4.yaml index 893b5a03d6..3d96424917 100644 --- a/deploy/provider/aws/service-l4.yaml +++ b/deploy/aws/l4/service-l4.yaml @@ -2,10 +2,6 @@ kind: Service apiVersion: v1 metadata: name: ingress-nginx - namespace: ingress-nginx - labels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx annotations: # Enable PROXY protocol service.beta.kubernetes.io/aws-load-balancer-proxy-protocol: "*" @@ -14,17 +10,4 @@ metadata: # increased to '3600' to avoid any potential issues. service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout: "60" spec: - type: LoadBalancer - selector: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - ports: - - name: http - port: 80 - targetPort: http - - name: https - port: 443 - targetPort: https - ---- - + externalTrafficPolicy: Cluster diff --git a/deploy/aws/l7/kustomization.yaml b/deploy/aws/l7/kustomization.yaml new file mode 100644 index 0000000000..35dbc67e47 --- /dev/null +++ b/deploy/aws/l7/kustomization.yaml @@ -0,0 +1,13 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +bases: +- ../../cloud-generic +patchesStrategicMerge: +- service-l7.yaml +configMapGenerator: +- name: nginx-configuration + behavior: merge + literals: + - use-proxy-protocol=false + - use-forwarded-headers=true + - proxy-real-ip-cidr=0.0.0.0/0 # restrict this to the IP addresses of ELB diff --git a/deploy/provider/aws/service-l7.yaml b/deploy/aws/l7/service-l7.yaml similarity index 68% rename from deploy/provider/aws/service-l7.yaml rename to deploy/aws/l7/service-l7.yaml index 6616108a23..b3b0b64d81 100644 --- a/deploy/provider/aws/service-l7.yaml +++ b/deploy/aws/l7/service-l7.yaml @@ -2,10 +2,6 @@ kind: Service apiVersion: v1 metadata: name: ingress-nginx - namespace: ingress-nginx - labels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx annotations: # replace with the correct value of the generated certificate in the AWS console service.beta.kubernetes.io/aws-load-balancer-ssl-cert: "arn:aws:acm:us-west-2:XXXXXXXX:certificate/XXXXXX-XXXXXXX-XXXXXXX-XXXXXXXX" @@ -18,17 +14,4 @@ metadata: # increased to '3600' to avoid any potential issues. service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout: "60" spec: - type: LoadBalancer - selector: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - ports: - - name: http - port: 80 - targetPort: http - - name: https - port: 443 - targetPort: http - ---- - + externalTrafficPolicy: Cluster diff --git a/deploy/aws/nlb/kustomization.yaml b/deploy/aws/nlb/kustomization.yaml new file mode 100644 index 0000000000..cfffbefc42 --- /dev/null +++ b/deploy/aws/nlb/kustomization.yaml @@ -0,0 +1,6 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +bases: +- ../../cloud-generic +patchesStrategicMerge: +- service-nlb.yaml diff --git a/deploy/aws/nlb/service-nlb.yaml b/deploy/aws/nlb/service-nlb.yaml new file mode 100644 index 0000000000..a0438c207e --- /dev/null +++ b/deploy/aws/nlb/service-nlb.yaml @@ -0,0 +1,7 @@ +kind: Service +apiVersion: v1 +metadata: + name: ingress-nginx + annotations: + # by default the type is elb (classic load balancer). + service.beta.kubernetes.io/aws-load-balancer-type: nlb diff --git a/deploy/baremetal/kustomization.yaml b/deploy/baremetal/kustomization.yaml new file mode 100644 index 0000000000..3512703b83 --- /dev/null +++ b/deploy/baremetal/kustomization.yaml @@ -0,0 +1,6 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +bases: +- ../cloud-generic +patchesStrategicMerge: +- service-nodeport.yaml diff --git a/test/e2e-image/manifests/service.yaml b/deploy/baremetal/service-nodeport.yaml similarity index 69% rename from test/e2e-image/manifests/service.yaml rename to deploy/baremetal/service-nodeport.yaml index 4226bb0e03..0aadea1576 100644 --- a/test/e2e-image/manifests/service.yaml +++ b/deploy/baremetal/service-nodeport.yaml @@ -3,6 +3,7 @@ kind: Service metadata: name: ingress-nginx spec: + type: NodePort ports: - name: http port: 80 @@ -12,6 +13,4 @@ spec: port: 443 targetPort: 443 protocol: TCP - selector: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx + externalTrafficPolicy: Cluster diff --git a/deploy/with-rbac.yaml b/deploy/cloud-generic/deployment.yaml similarity index 73% rename from deploy/with-rbac.yaml rename to deploy/cloud-generic/deployment.yaml index 3e3661cdba..71de9202b0 100644 --- a/deploy/with-rbac.yaml +++ b/deploy/cloud-generic/deployment.yaml @@ -2,21 +2,10 @@ apiVersion: apps/v1 kind: Deployment metadata: name: nginx-ingress-controller - namespace: ingress-nginx - labels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx spec: replicas: 1 - selector: - matchLabels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx template: metadata: - labels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx annotations: prometheus.io/port: "10254" prometheus.io/scrape: "true" @@ -27,10 +16,10 @@ spec: image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.24.1 args: - /nginx-ingress-controller - - --configmap=$(POD_NAMESPACE)/nginx-configuration - - --tcp-services-configmap=$(POD_NAMESPACE)/tcp-services - - --udp-services-configmap=$(POD_NAMESPACE)/udp-services - - --publish-service=$(POD_NAMESPACE)/ingress-nginx + - --configmap=$(POD_NAMESPACE)/$(NGINX_CONFIGMAP_NAME) + - --tcp-services-configmap=$(POD_NAMESPACE)/$(TCP_CONFIGMAP_NAME) + - --udp-services-configmap=$(POD_NAMESPACE)/$(UDP_CONFIGMAP_NAME) + - --publish-service=$(POD_NAMESPACE)/$(SERVICE_NAME) - --annotations-prefix=nginx.ingress.kubernetes.io securityContext: allowPrivilegeEscalation: true @@ -74,6 +63,3 @@ spec: periodSeconds: 10 successThreshold: 1 timeoutSeconds: 10 - ---- - diff --git a/deploy/cloud-generic/kustomization.yaml b/deploy/cloud-generic/kustomization.yaml new file mode 100644 index 0000000000..c2b03ddbf6 --- /dev/null +++ b/deploy/cloud-generic/kustomization.yaml @@ -0,0 +1,50 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: ingress-nginx +commonLabels: + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/part-of: ingress-nginx +resources: +- deployment.yaml +- role-binding.yaml +- role.yaml +- service-account.yaml +- service.yaml +images: +- name: quay.io/kubernetes-ingress-controller/nginx-ingress-controller + newTag: 0.24.1 +vars: +- fieldref: + fieldPath: metadata.name + name: NGINX_CONFIGMAP_NAME + objref: + apiVersion: v1 + kind: ConfigMap + name: nginx-configuration +- fieldref: + fieldPath: metadata.name + name: TCP_CONFIGMAP_NAME + objref: + apiVersion: v1 + kind: ConfigMap + name: tcp-services +- fieldref: + fieldPath: metadata.name + name: UDP_CONFIGMAP_NAME + objref: + apiVersion: v1 + kind: ConfigMap + name: udp-services +- fieldref: + fieldPath: metadata.name + name: SERVICE_NAME + objref: + apiVersion: v1 + kind: Service + name: ingress-nginx +configMapGenerator: +- name: nginx-configuration +- name: tcp-services +- name: udp-services +generatorOptions: + disableNameSuffixHash: true diff --git a/deploy/cloud-generic/role-binding.yaml b/deploy/cloud-generic/role-binding.yaml new file mode 100644 index 0000000000..228588e6dc --- /dev/null +++ b/deploy/cloud-generic/role-binding.yaml @@ -0,0 +1,11 @@ +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: RoleBinding +metadata: + name: nginx-ingress-role-nisa-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: nginx-ingress-role +subjects: + - kind: ServiceAccount + name: nginx-ingress-serviceaccount diff --git a/deploy/cloud-generic/role.yaml b/deploy/cloud-generic/role.yaml new file mode 100644 index 0000000000..936b63d729 --- /dev/null +++ b/deploy/cloud-generic/role.yaml @@ -0,0 +1,39 @@ +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: Role +metadata: + name: nginx-ingress-role +rules: + - apiGroups: + - "" + resources: + - configmaps + - pods + - secrets + - namespaces + verbs: + - get + - apiGroups: + - "" + resources: + - configmaps + resourceNames: + # Defaults to "-" + # Here: "-" + # This has to be adapted if you change either parameter + # when launching the nginx-ingress-controller. + - "ingress-controller-leader-nginx" + verbs: + - get + - update + - apiGroups: + - "" + resources: + - configmaps + verbs: + - create + - apiGroups: + - "" + resources: + - endpoints + verbs: + - get diff --git a/deploy/cloud-generic/service-account.yaml b/deploy/cloud-generic/service-account.yaml new file mode 100644 index 0000000000..a52fb8ac85 --- /dev/null +++ b/deploy/cloud-generic/service-account.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: nginx-ingress-serviceaccount diff --git a/deploy/provider/cloud-generic.yaml b/deploy/cloud-generic/service.yaml similarity index 51% rename from deploy/provider/cloud-generic.yaml rename to deploy/cloud-generic/service.yaml index 8bbac569bf..3a3a3e2a82 100644 --- a/deploy/provider/cloud-generic.yaml +++ b/deploy/cloud-generic/service.yaml @@ -2,16 +2,9 @@ kind: Service apiVersion: v1 metadata: name: ingress-nginx - namespace: ingress-nginx - labels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx spec: externalTrafficPolicy: Local type: LoadBalancer - selector: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx ports: - name: http port: 80 @@ -19,6 +12,3 @@ spec: - name: https port: 443 targetPort: https - ---- - diff --git a/deploy/cluster-wide/cluster-role-binding.yaml b/deploy/cluster-wide/cluster-role-binding.yaml new file mode 100644 index 0000000000..7293fb37d7 --- /dev/null +++ b/deploy/cluster-wide/cluster-role-binding.yaml @@ -0,0 +1,11 @@ +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: nginx-ingress-clusterrole-nisa-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: nginx-ingress-clusterrole +subjects: + - kind: ServiceAccount + name: nginx-ingress-serviceaccount diff --git a/test/e2e-image/manifests/rbac.yaml b/deploy/cluster-wide/cluster-role.yaml similarity index 88% rename from test/e2e-image/manifests/rbac.yaml rename to deploy/cluster-wide/cluster-role.yaml index 7f4f79fe8d..9e5d39ca30 100644 --- a/test/e2e-image/manifests/rbac.yaml +++ b/deploy/cluster-wide/cluster-role.yaml @@ -2,9 +2,6 @@ apiVersion: rbac.authorization.k8s.io/v1beta1 kind: ClusterRole metadata: name: nginx-ingress-clusterrole - labels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx rules: - apiGroups: - "" diff --git a/deploy/cluster-wide/kustomization.yaml b/deploy/cluster-wide/kustomization.yaml new file mode 100644 index 0000000000..aeef6ed6b3 --- /dev/null +++ b/deploy/cluster-wide/kustomization.yaml @@ -0,0 +1,8 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +commonLabels: + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/part-of: ingress-nginx +resources: +- cluster-role.yaml +- cluster-role-binding.yaml diff --git a/deploy/configmap.yaml b/deploy/configmap.yaml deleted file mode 100644 index 436b660a94..0000000000 --- a/deploy/configmap.yaml +++ /dev/null @@ -1,30 +0,0 @@ -kind: ConfigMap -apiVersion: v1 -metadata: - name: nginx-configuration - namespace: ingress-nginx - labels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - ---- -kind: ConfigMap -apiVersion: v1 -metadata: - name: tcp-services - namespace: ingress-nginx - labels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - ---- -kind: ConfigMap -apiVersion: v1 -metadata: - name: udp-services - namespace: ingress-nginx - labels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - ---- diff --git a/deploy/monitoring/grafana.yaml b/deploy/grafana/deployment.yaml similarity index 50% rename from deploy/monitoring/grafana.yaml rename to deploy/grafana/deployment.yaml index 7c7b000007..765b2f64fb 100644 --- a/deploy/monitoring/grafana.yaml +++ b/deploy/grafana/deployment.yaml @@ -1,27 +1,15 @@ apiVersion: apps/v1 kind: Deployment metadata: - labels: - app.kubernetes.io/name: grafana - app.kubernetes.io/part-of: ingress-nginx - name: grafana namespace: ingress-nginx spec: - selector: - matchLabels: - app.kubernetes.io/name: grafana - app.kubernetes.io/part-of: ingress-nginx strategy: rollingUpdate: maxSurge: 1 maxUnavailable: 1 type: RollingUpdate template: - metadata: - labels: - app.kubernetes.io/name: grafana - app.kubernetes.io/part-of: ingress-nginx spec: containers: - image: grafana/grafana @@ -43,26 +31,3 @@ spec: volumes: - emptyDir: {} name: data - ---- -apiVersion: v1 -kind: Service -metadata: - name: grafana - namespace: ingress-nginx - labels: - app.kubernetes.io/name: grafana - app.kubernetes.io/part-of: ingress-nginx - -spec: - ports: - - port: 3000 - protocol: TCP - targetPort: 3000 - selector: - app.kubernetes.io/name: grafana - app.kubernetes.io/part-of: ingress-nginx - type: NodePort - ---- - diff --git a/deploy/grafana/kustomization.yaml b/deploy/grafana/kustomization.yaml new file mode 100644 index 0000000000..2069c1a7a9 --- /dev/null +++ b/deploy/grafana/kustomization.yaml @@ -0,0 +1,12 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: ingress-nginx +commonLabels: + app.kubernetes.io/name: grafana + app.kubernetes.io/part-of: ingress-nginx +resources: +- deployment.yaml +- service.yaml +images: +- name: grafana/grafana + newTag: 6.1.6 diff --git a/deploy/grafana/service.yaml b/deploy/grafana/service.yaml new file mode 100644 index 0000000000..16d69ac7a8 --- /dev/null +++ b/deploy/grafana/service.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: Service +metadata: + name: grafana +spec: + ports: + - port: 3000 + protocol: TCP + targetPort: 3000 + type: NodePort diff --git a/deploy/mandatory.yaml b/deploy/mandatory.yaml deleted file mode 100644 index 921bc482d7..0000000000 --- a/deploy/mandatory.yaml +++ /dev/null @@ -1,265 +0,0 @@ -apiVersion: v1 -kind: Namespace -metadata: - name: ingress-nginx - labels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - ---- - -kind: ConfigMap -apiVersion: v1 -metadata: - name: nginx-configuration - namespace: ingress-nginx - labels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - ---- -kind: ConfigMap -apiVersion: v1 -metadata: - name: tcp-services - namespace: ingress-nginx - labels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - ---- -kind: ConfigMap -apiVersion: v1 -metadata: - name: udp-services - namespace: ingress-nginx - labels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: nginx-ingress-serviceaccount - namespace: ingress-nginx - labels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - ---- -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: ClusterRole -metadata: - name: nginx-ingress-clusterrole - labels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx -rules: - - apiGroups: - - "" - resources: - - configmaps - - endpoints - - nodes - - pods - - secrets - verbs: - - list - - watch - - apiGroups: - - "" - resources: - - nodes - verbs: - - get - - apiGroups: - - "" - resources: - - services - verbs: - - get - - list - - watch - - apiGroups: - - "extensions" - resources: - - ingresses - verbs: - - get - - list - - watch - - apiGroups: - - "" - resources: - - events - verbs: - - create - - patch - - apiGroups: - - "extensions" - resources: - - ingresses/status - verbs: - - update - ---- -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: Role -metadata: - name: nginx-ingress-role - namespace: ingress-nginx - labels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx -rules: - - apiGroups: - - "" - resources: - - configmaps - - pods - - secrets - - namespaces - verbs: - - get - - apiGroups: - - "" - resources: - - configmaps - resourceNames: - # Defaults to "-" - # Here: "-" - # This has to be adapted if you change either parameter - # when launching the nginx-ingress-controller. - - "ingress-controller-leader-nginx" - verbs: - - get - - update - - apiGroups: - - "" - resources: - - configmaps - verbs: - - create - - apiGroups: - - "" - resources: - - endpoints - verbs: - - get - ---- -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: RoleBinding -metadata: - name: nginx-ingress-role-nisa-binding - namespace: ingress-nginx - labels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: nginx-ingress-role -subjects: - - kind: ServiceAccount - name: nginx-ingress-serviceaccount - namespace: ingress-nginx - ---- -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: ClusterRoleBinding -metadata: - name: nginx-ingress-clusterrole-nisa-binding - labels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: nginx-ingress-clusterrole -subjects: - - kind: ServiceAccount - name: nginx-ingress-serviceaccount - namespace: ingress-nginx - ---- - -apiVersion: apps/v1 -kind: Deployment -metadata: - name: nginx-ingress-controller - namespace: ingress-nginx - labels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx -spec: - replicas: 1 - selector: - matchLabels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - template: - metadata: - labels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - annotations: - prometheus.io/port: "10254" - prometheus.io/scrape: "true" - spec: - serviceAccountName: nginx-ingress-serviceaccount - containers: - - name: nginx-ingress-controller - image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.24.1 - args: - - /nginx-ingress-controller - - --configmap=$(POD_NAMESPACE)/nginx-configuration - - --tcp-services-configmap=$(POD_NAMESPACE)/tcp-services - - --udp-services-configmap=$(POD_NAMESPACE)/udp-services - - --publish-service=$(POD_NAMESPACE)/ingress-nginx - - --annotations-prefix=nginx.ingress.kubernetes.io - securityContext: - allowPrivilegeEscalation: true - capabilities: - drop: - - ALL - add: - - NET_BIND_SERVICE - # www-data -> 33 - runAsUser: 33 - env: - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - ports: - - name: http - containerPort: 80 - - name: https - containerPort: 443 - livenessProbe: - failureThreshold: 3 - httpGet: - path: /healthz - port: 10254 - scheme: HTTP - initialDelaySeconds: 10 - periodSeconds: 10 - successThreshold: 1 - timeoutSeconds: 10 - readinessProbe: - failureThreshold: 3 - httpGet: - path: /healthz - port: 10254 - scheme: HTTP - periodSeconds: 10 - successThreshold: 1 - timeoutSeconds: 10 - ---- diff --git a/deploy/minikube/kustomization.yaml b/deploy/minikube/kustomization.yaml new file mode 100644 index 0000000000..34ea278ad0 --- /dev/null +++ b/deploy/minikube/kustomization.yaml @@ -0,0 +1,10 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: ingress-nginx +bases: +- ../baremetal +- ../cluster-wide +images: +- name: quay.io/kubernetes-ingress-controller/nginx-ingress-controller + newName: ingress-controller/nginx-ingress-controller + newTag: dev diff --git a/deploy/monitoring/configuration.yaml b/deploy/monitoring/configuration.yaml deleted file mode 100644 index 36b9d17739..0000000000 --- a/deploy/monitoring/configuration.yaml +++ /dev/null @@ -1,45 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: prometheus-configuration - labels: - app.kubernetes.io/name: prometheus - app.kubernetes.io/part-of: ingress-nginx - name: prometheus-configuration - namespace: ingress-nginx -data: - prometheus.yml: |- - global: - scrape_interval: 10s - scrape_configs: - - job_name: 'ingress-nginx-endpoints' - kubernetes_sd_configs: - - role: pod - namespaces: - names: - - ingress-nginx - - relabel_configs: - - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape] - action: keep - regex: true - - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scheme] - action: replace - target_label: __scheme__ - regex: (https?) - - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path] - action: replace - target_label: __metrics_path__ - regex: (.+) - - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port] - action: replace - target_label: __address__ - regex: ([^:]+)(?::\d+)?;(\d+) - replacement: $1:$2 - - - source_labels: [__meta_kubernetes_service_name] - regex: prometheus-server - action: drop - ---- - diff --git a/deploy/monitoring/prometheus.yaml b/deploy/monitoring/prometheus.yaml deleted file mode 100644 index aee13d286d..0000000000 --- a/deploy/monitoring/prometheus.yaml +++ /dev/null @@ -1,109 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: prometheus-server - namespace: ingress-nginx - labels: - app.kubernetes.io/name: prometheus - app.kubernetes.io/part-of: ingress-nginx -rules: - - apiGroups: [""] - resources: - - services - - endpoints - - pods - verbs: ["get", "list", "watch"] - ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: prometheus-server - namespace: ingress-nginx - labels: - app.kubernetes.io/name: prometheus - app.kubernetes.io/part-of: ingress-nginx - ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: prometheus-server - namespace: ingress-nginx - labels: - app.kubernetes.io/name: prometheus - app.kubernetes.io/part-of: ingress-nginx - -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: prometheus-server -subjects: - - kind: ServiceAccount - name: prometheus-server - namespace: ingress-nginx - ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: prometheus-server - namespace: ingress-nginx - labels: - app.kubernetes.io/name: prometheus - app.kubernetes.io/part-of: ingress-nginx - -spec: - replicas: 1 - selector: - matchLabels: - app.kubernetes.io/name: prometheus - app.kubernetes.io/part-of: ingress-nginx - template: - metadata: - labels: - app.kubernetes.io/name: prometheus - app.kubernetes.io/part-of: ingress-nginx - spec: - serviceAccountName: prometheus-server - containers: - - name: prometheus - image: prom/prometheus:v2.3.2 - args: - - "--config.file=/etc/prometheus/prometheus.yml" - - "--storage.tsdb.path=/prometheus/" - ports: - - containerPort: 9090 - volumeMounts: - - name: prometheus-config-volume - mountPath: /etc/prometheus/ - - name: prometheus-storage-volume - mountPath: /prometheus/ - volumes: - - name: prometheus-config-volume - configMap: - name: prometheus-configuration - - name: prometheus-storage-volume - emptyDir: {} - ---- -apiVersion: v1 -kind: Service -metadata: - name: prometheus-server - namespace: ingress-nginx - labels: - app.kubernetes.io/name: prometheus - app.kubernetes.io/part-of: ingress-nginx - -spec: - selector: - app.kubernetes.io/name: prometheus - app.kubernetes.io/part-of: ingress-nginx - type: NodePort - ports: - - port: 9090 - targetPort: 9090 - ---- - diff --git a/deploy/namespace.yaml b/deploy/namespace.yaml deleted file mode 100644 index 9196d6d16d..0000000000 --- a/deploy/namespace.yaml +++ /dev/null @@ -1,10 +0,0 @@ -apiVersion: v1 -kind: Namespace -metadata: - name: ingress-nginx - labels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - ---- - diff --git a/deploy/prometheus/deployment.yaml b/deploy/prometheus/deployment.yaml new file mode 100644 index 0000000000..2b15c686b2 --- /dev/null +++ b/deploy/prometheus/deployment.yaml @@ -0,0 +1,28 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: prometheus-server +spec: + replicas: 1 + template: + spec: + serviceAccountName: prometheus-server + containers: + - name: prometheus + image: prom/prometheus + args: + - "--config.file=/etc/prometheus/prometheus.yaml" + - "--storage.tsdb.path=/prometheus/" + ports: + - containerPort: 9090 + volumeMounts: + - name: prometheus-config-volume + mountPath: /etc/prometheus/ + - name: prometheus-storage-volume + mountPath: /prometheus/ + volumes: + - name: prometheus-config-volume + configMap: + name: prometheus-configuration + - name: prometheus-storage-volume + emptyDir: {} diff --git a/deploy/prometheus/kustomization.yaml b/deploy/prometheus/kustomization.yaml new file mode 100644 index 0000000000..2d6e0a7e26 --- /dev/null +++ b/deploy/prometheus/kustomization.yaml @@ -0,0 +1,19 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: ingress-nginx +commonLabels: + app.kubernetes.io/name: prometheus + app.kubernetes.io/part-of: ingress-nginx +resources: +- role.yaml +- service-account.yaml +- role-binding.yaml +- deployment.yaml +- service.yaml +images: +- name: prom/prometheus + newTag: v2.3.2 +configMapGenerator: +- name: prometheus-configuration + files: + - prometheus.yaml diff --git a/deploy/prometheus/prometheus.yaml b/deploy/prometheus/prometheus.yaml new file mode 100644 index 0000000000..fe8cefe961 --- /dev/null +++ b/deploy/prometheus/prometheus.yaml @@ -0,0 +1,29 @@ +global: + scrape_interval: 10s +scrape_configs: +- job_name: 'ingress-nginx-endpoints' + kubernetes_sd_configs: + - role: pod + namespaces: + names: + - ingress-nginx + relabel_configs: + - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape] + action: keep + regex: true + - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scheme] + action: replace + target_label: __scheme__ + regex: (https?) + - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path] + action: replace + target_label: __metrics_path__ + regex: (.+) + - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port] + action: replace + target_label: __address__ + regex: ([^:]+)(?::\d+)?;(\d+) + replacement: $1:$2 + - source_labels: [__meta_kubernetes_service_name] + regex: prometheus-server + action: drop diff --git a/deploy/prometheus/role-binding.yaml b/deploy/prometheus/role-binding.yaml new file mode 100644 index 0000000000..b73b94f3a9 --- /dev/null +++ b/deploy/prometheus/role-binding.yaml @@ -0,0 +1,11 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: prometheus-server +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: prometheus-server +subjects: + - kind: ServiceAccount + name: prometheus-server diff --git a/deploy/prometheus/role.yaml b/deploy/prometheus/role.yaml new file mode 100644 index 0000000000..0328f80baf --- /dev/null +++ b/deploy/prometheus/role.yaml @@ -0,0 +1,11 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: prometheus-server +rules: + - apiGroups: [""] + resources: + - services + - endpoints + - pods + verbs: ["get", "list", "watch"] diff --git a/deploy/prometheus/service-account.yaml b/deploy/prometheus/service-account.yaml new file mode 100644 index 0000000000..3e2c9a7700 --- /dev/null +++ b/deploy/prometheus/service-account.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: prometheus-server diff --git a/deploy/prometheus/service.yaml b/deploy/prometheus/service.yaml new file mode 100644 index 0000000000..9a2de8ce44 --- /dev/null +++ b/deploy/prometheus/service.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: Service +metadata: + name: prometheus-server +spec: + type: NodePort + ports: + - port: 9090 + targetPort: 9090 diff --git a/deploy/provider/aws/patch-configmap-l4.yaml b/deploy/provider/aws/patch-configmap-l4.yaml deleted file mode 100644 index 1d612289fb..0000000000 --- a/deploy/provider/aws/patch-configmap-l4.yaml +++ /dev/null @@ -1,10 +0,0 @@ -kind: ConfigMap -apiVersion: v1 -metadata: - name: nginx-configuration - namespace: ingress-nginx - labels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx -data: - use-proxy-protocol: "true" diff --git a/deploy/provider/aws/patch-configmap-l7.yaml b/deploy/provider/aws/patch-configmap-l7.yaml deleted file mode 100644 index b1bcd2a971..0000000000 --- a/deploy/provider/aws/patch-configmap-l7.yaml +++ /dev/null @@ -1,14 +0,0 @@ -kind: ConfigMap -apiVersion: v1 -metadata: - name: nginx-configuration - namespace: ingress-nginx - labels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx -data: - use-proxy-protocol: "false" - use-forwarded-headers: "true" - proxy-real-ip-cidr: "0.0.0.0/0" # restrict this to the IP addresses of ELB ---- - diff --git a/deploy/provider/aws/service-nlb.yaml b/deploy/provider/aws/service-nlb.yaml deleted file mode 100644 index 244460b6d1..0000000000 --- a/deploy/provider/aws/service-nlb.yaml +++ /dev/null @@ -1,28 +0,0 @@ -kind: Service -apiVersion: v1 -metadata: - name: ingress-nginx - namespace: ingress-nginx - labels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - annotations: - # by default the type is elb (classic load balancer). - service.beta.kubernetes.io/aws-load-balancer-type: nlb -spec: - # this setting is to make sure the source IP address is preserved. - externalTrafficPolicy: Local - type: LoadBalancer - selector: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - ports: - - name: http - port: 80 - targetPort: http - - name: https - port: 443 - targetPort: https - ---- - diff --git a/deploy/provider/baremetal/service-nodeport.yaml b/deploy/provider/baremetal/service-nodeport.yaml deleted file mode 100644 index 24e302818b..0000000000 --- a/deploy/provider/baremetal/service-nodeport.yaml +++ /dev/null @@ -1,25 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: ingress-nginx - namespace: ingress-nginx - labels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx -spec: - type: NodePort - ports: - - name: http - port: 80 - targetPort: 80 - protocol: TCP - - name: https - port: 443 - targetPort: 443 - protocol: TCP - selector: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - ---- - diff --git a/deploy/rbac.yaml b/deploy/rbac.yaml deleted file mode 100644 index 103bd98cc0..0000000000 --- a/deploy/rbac.yaml +++ /dev/null @@ -1,147 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - name: nginx-ingress-serviceaccount - namespace: ingress-nginx - labels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - ---- -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: ClusterRole -metadata: - name: nginx-ingress-clusterrole - labels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx -rules: - - apiGroups: - - "" - resources: - - configmaps - - endpoints - - nodes - - pods - - secrets - verbs: - - list - - watch - - apiGroups: - - "" - resources: - - nodes - verbs: - - get - - apiGroups: - - "" - resources: - - services - verbs: - - get - - list - - watch - - apiGroups: - - "extensions" - resources: - - ingresses - verbs: - - get - - list - - watch - - apiGroups: - - "" - resources: - - events - verbs: - - create - - patch - - apiGroups: - - "extensions" - resources: - - ingresses/status - verbs: - - update - ---- -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: Role -metadata: - name: nginx-ingress-role - namespace: ingress-nginx - labels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx -rules: - - apiGroups: - - "" - resources: - - configmaps - - pods - - secrets - - namespaces - verbs: - - get - - apiGroups: - - "" - resources: - - configmaps - resourceNames: - # Defaults to "-" - # Here: "-" - # This has to be adapted if you change either parameter - # when launching the nginx-ingress-controller. - - "ingress-controller-leader-nginx" - verbs: - - get - - update - - apiGroups: - - "" - resources: - - configmaps - verbs: - - create - - apiGroups: - - "" - resources: - - endpoints - verbs: - - get - ---- -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: RoleBinding -metadata: - name: nginx-ingress-role-nisa-binding - namespace: ingress-nginx - labels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: nginx-ingress-role -subjects: - - kind: ServiceAccount - name: nginx-ingress-serviceaccount - namespace: ingress-nginx - ---- -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: ClusterRoleBinding -metadata: - name: nginx-ingress-clusterrole-nisa-binding - labels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: nginx-ingress-clusterrole -subjects: - - kind: ServiceAccount - name: nginx-ingress-serviceaccount - namespace: ingress-nginx - ---- - diff --git a/docs/deploy/index.md b/docs/deploy/index.md index 9f56f6e882..786d1b4bfc 100644 --- a/docs/deploy/index.md +++ b/docs/deploy/index.md @@ -18,6 +18,9 @@ The following **Mandatory Command** is required for all deployments. +!!! attention + These commands depend on having kubectl version 1.14 or newer. + !!! attention The default configuration watches Ingress object from all the namespaces. To change this behavior use the flag `--watch-namespace` to limit the scope to a particular namespace. @@ -25,17 +28,25 @@ The following **Mandatory Command** is required for all deployments. !!! warning If multiple Ingresses define different paths for the same host, the ingress controller will merge the definitions. -!!! attention - If you're using GKE you need to initialize your user as a cluster-admin with the following command: - ```kubectl create clusterrolebinding cluster-admin-binding --clusterrole cluster-admin --user $(gcloud config get-value account)``` ```console -kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/master/deploy/mandatory.yaml +kubectl create namespace ingress-nginx +``` + +```console +cat << EOF > kustomization.yaml +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: ingress-nginx +bases: +- github.com/kubernetes/ingress-nginx/deploy/cluster-wide +- # provider-specific, see below +EOF ``` ### Provider Specific Steps -There are cloud provider specific yaml files. +There are cloud provider specific kustomize bases. #### Docker for Mac @@ -43,11 +54,7 @@ Kubernetes is available in Docker for Mac (from [version 18.06.0-ce](https://doc [enable]: https://docs.docker.com/docker-for-mac/#kubernetes -Create a service - -```console -kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/master/deploy/provider/cloud-generic.yaml -``` +Add `github.com/kubernetes/ingress-nginx/deploy/cloud-generic` to the `bases` list in `kustomization.yaml` and run `kubectl apply --kustomize .`. #### minikube @@ -88,30 +95,57 @@ This setup requires to choose in which layer (L4 or L7) we want to configure the - [Layer 4](https://en.wikipedia.org/wiki/OSI_model#Layer_4:_Transport_Layer): use TCP as the listener protocol for ports 80 and 443. - [Layer 7](https://en.wikipedia.org/wiki/OSI_model#Layer_7:_Application_Layer): use HTTP as the listener protocol for port 80 and terminate TLS in the ELB -For L4: -Check that no change is necessary with regards to the ELB idle timeout. In some scenarios, users may want to modify the ELB idle timeout, so please check the [ELB Idle Timeouts section](#elb-idle-timeouts) for additional information. If a change is required, users will need to update the value of `service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout` in `provider/aws/service-l4.yaml` +Check that no change is necessary with regards to the ELB idle timeout. In some scenarios, users may want to modify the ELB idle timeout, so please check the [ELB Idle Timeouts section](#elb-idle-timeouts) for additional information. If a change is required, users will need to override the value of the annotation `service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout` on the service object. -Then execute: +To do this, create a patch file which will replace the annotation. -```console -kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/master/deploy/provider/aws/service-l4.yaml -kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/master/deploy/provider/aws/patch-configmap-l4.yaml +``` +cat << EOF > elb-timeout.yaml +kind: Service +apiVersion: v1 +metadata: + name: ingress-nginx + annotations: + service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout: "3600" # Recommended value for WebSockets +EOF ``` -For L7: +After creating the patch file, reference it in your `kustomization.yaml`: +```yaml +patchesStrategicMerge: +- elb-timeout.yaml +``` + +For L4: -Change line of the file `provider/aws/service-l7.yaml` replacing the dummy id with a valid one `"arn:aws:acm:us-west-2:XXXXXXXX:certificate/XXXXXX-XXXXXXX-XXXXXXX-XXXXXXXX"` +To deploy the default example, add the base ` github.com/kubernetes/ingress-nginx/deploy/aws/l4` and then run `kubectl apply --kustomize .` -Check that no change is necessary with regards to the ELB idle timeout. In some scenarios, users may want to modify the ELB idle timeout, so please check the [ELB Idle Timeouts section](#elb-idle-timeouts) for additional information. If a change is required, users will need to update the value of `service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout` in `provider/aws/service-l7.yaml` +For L7: -Then execute: +Create a a patch that will annotate the ingress-controller's service with your ssl certificate id. ```console -kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/master/deploy/provider/aws/service-l7.yaml -kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/master/deploy/provider/aws/patch-configmap-l7.yaml +cat << EOF > elb-ssl.yaml +kind: Service +apiVersion: v1 +metadata: + name: ingress-nginx + annotations: + # replace with the correct value of the generated certificate in the AWS console + service.beta.kubernetes.io/aws-load-balancer-ssl-cert: "arn:aws:acm:us-west-2:XXXXXXXX:certificate/XXXXXX-XXXXXXX-XXXXXXX-XXXXXXXX" +EOF ``` +Reference this patch in your `kustomization.yaml`: + +```yaml +patchesStrategicMerge: +- elb-ssl.yaml +``` + +Then add the l7 base, `github.com/kubernetes/ingress-nginx/deploy/aws/l7` and execute `kubectl apply --kustomize .` + This example creates an ELB with just two listeners, one in port 80 and another in port 443 ![Listeners](../images/elb-l7-listener.png) @@ -127,35 +161,31 @@ More information with regards to idle timeouts for your Load Balancer can be fou ##### Network Load Balancer (NLB) -This type of load balancer is supported since v1.10.0 as an ALPHA feature. +This type of load balancer is supported since v1.10.0 as an ALPHA feature. Use the base `github.com/kubernetes/ingress-nginx/deploy/aws/nlb` and execute `kubectl apply --kustomize .` -```console -kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/master/deploy/provider/aws/service-nlb.yaml -``` #### GCE-GKE -```console -kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/master/deploy/provider/cloud-generic.yaml -``` +!!! attention + If you're using GKE you need to initialize your user as a cluster-admin with the following command: + ```kubectl create clusterrolebinding cluster-admin-binding --clusterrole cluster-admin --user $(gcloud config get-value account)``` + +Use the base `github.com/kubernetes/ingress-nginx/deploy/cloud-generic` and execute `kubectl apply --kustomize .` **Important Note:** proxy protocol is not supported in GCE/GKE + #### Azure - -```console -kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/master/deploy/provider/cloud-generic.yaml -``` +Use the base `github.com/kubernetes/ingress-nginx/deploy/cloud-generic` and execute `kubectl apply --kustomize .` #### Bare-metal Using [NodePort](https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport): -```console -kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/master/deploy/provider/baremetal/service-nodeport.yaml -``` + +Use the base `github.com/kubernetes/ingress-nginx/deploy/baremetal` and execute `kubectl apply --kustomize .` !!! tip For extended notes regarding deployments on bare-metal, see [Bare-metal considerations](./baremetal.md). diff --git a/docs/user-guide/monitoring.md b/docs/user-guide/monitoring.md index f295302cc9..ed5de73e78 100644 --- a/docs/user-guide/monitoring.md +++ b/docs/user-guide/monitoring.md @@ -9,7 +9,7 @@ This tutorial will show you how to install [Prometheus](https://prometheus.io/) The NGINX Ingress controller should already be deployed according to the deployment instructions [here](../deploy/index.md). -Note that the yaml files used in this tutorial are stored in the [deploy/monitoring](https://github.com/kubernetes/ingress-nginx/tree/master/deploy/monitoring) folder of the GitHub repository [kubernetes/ingress-nginx](https://github.com/kubernetes/ingress-nginx). +Note that the kustomize bases used in this tutorial are stored in the [deploy](https://github.com/kubernetes/ingress-nginx/tree/master/deploy) folder of the GitHub repository [kubernetes/ingress-nginx](https://github.com/kubernetes/ingress-nginx). ## Deploy and configure Prometheus Server @@ -17,22 +17,16 @@ The Prometheus server must be configured so that it can discover endpoints of se If there is no existing Prometheus server running, the rest of this tutorial will guide you through the steps needed to deploy a properly configured Prometheus server. -Running the following command deploys the prometheus configuration in Kubernetes: - -```console -kubectl create -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/master/deploy/monitoring/configuration.yaml -configmap "prometheus-configuration" created -``` - Running the following command deploys prometheus in Kubernetes: ```console -kubectl create -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/master/deploy/monitoring/prometheus.yaml -clusterrole "prometheus-server" created -serviceaccount "prometheus-server" created -clusterrolebinding "prometheus-server" created -deployment "prometheus-server" created -service "prometheus-server" created +kubectl apply --kustomize github.com/kubernetes/ingress-nginx/deploy/prometheus/ +serviceaccount/prometheus-server created +role.rbac.authorization.k8s.io/prometheus-server created +rolebinding.rbac.authorization.k8s.io/prometheus-server created +configmap/prometheus-configuration-bc6bcg7b65 created +service/prometheus-server created +deployment.apps/prometheus-server created ``` ### Prometheus Dashboard @@ -69,7 +63,7 @@ According to the above example, this URL will be http://10.192.0.3:32630 ### Grafana ```console -kubectl create -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/master/deploy/monitoring/grafana.yaml +kubectl apply --kustomize github.com/kubernetes/ingress-nginx/deploy/grafana/ ``` ```console diff --git a/hack/build-single-manifest-sh b/hack/build-single-manifest-sh deleted file mode 100755 index 0e236e7820..0000000000 --- a/hack/build-single-manifest-sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash - -# Copyright 2018 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -o errexit -set -o nounset -set -o pipefail - -SCRIPT_ROOT=$(dirname ${BASH_SOURCE})/.. - -INPUT="namespace.yaml configmap.yaml rbac.yaml with-rbac.yaml" -MANIFEST=$(cd ${SCRIPT_ROOT}/deploy; cat ${INPUT}) - -echo "${MANIFEST}" > ${SCRIPT_ROOT}/deploy/mandatory.yaml diff --git a/test/e2e-image/.gitignore b/test/e2e-image/.gitignore index 811b620e0d..260b7c282d 100644 --- a/test/e2e-image/.gitignore +++ b/test/e2e-image/.gitignore @@ -1,3 +1,5 @@ e2e.test ginkgo kubectl +/cloud-generic/ +/cluster-wide/ diff --git a/test/e2e-image/Dockerfile b/test/e2e-image/Dockerfile index efc9813114..fbdff334fe 100644 --- a/test/e2e-image/Dockerfile +++ b/test/e2e-image/Dockerfile @@ -15,7 +15,10 @@ RUN curl -Lo /usr/local/bin/kubectl \ COPY --from=BASE /go/bin/ginkgo /usr/local/bin/ COPY e2e.sh /e2e.sh -COPY manifests /manifests +COPY cloud-generic /cloud-generic +COPY cluster-wide /cluster-wide +COPY overlay /overlay +RUN sed -E -i 's|^- .*deploy/cloud-generic$|- ../cloud-generic|' /overlay/kustomization.yaml COPY wait-for-nginx.sh / COPY e2e.test / diff --git a/test/e2e-image/Makefile b/test/e2e-image/Makefile index f64d2c26a1..f5ea8d88fa 100644 --- a/test/e2e-image/Makefile +++ b/test/e2e-image/Makefile @@ -9,6 +9,8 @@ container: cp ../e2e/e2e.test . cp ../e2e/wait-for-nginx.sh . + cp -r ../../deploy/cloud-generic . + cp -r ../../deploy/cluster-wide . docker build -t $(IMAGE) . diff --git a/test/e2e-image/e2e.sh b/test/e2e-image/e2e.sh index f185c60ec7..3178c8a3b6 100755 --- a/test/e2e-image/e2e.sh +++ b/test/e2e-image/e2e.sh @@ -27,8 +27,6 @@ if [ ! -f ${HOME}/.kube/config ]; then kubectl config use-context default fi -kubectl apply -f manifests/rbac.yaml - ginkgo_args=( "-randomizeSuites" "-randomizeAllSpecs" diff --git a/test/e2e-image/manifests/mandatory.yaml b/test/e2e-image/manifests/mandatory.yaml deleted file mode 100644 index 7380a67a19..0000000000 --- a/test/e2e-image/manifests/mandatory.yaml +++ /dev/null @@ -1,205 +0,0 @@ -kind: ConfigMap -apiVersion: v1 -metadata: - name: nginx-configuration - labels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx -data: - worker-processes: "1" - ---- -kind: ConfigMap -apiVersion: v1 -metadata: - name: tcp-services - labels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - ---- -kind: ConfigMap -apiVersion: v1 -metadata: - name: udp-services - labels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: nginx-ingress-serviceaccount - labels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - ---- -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: Role -metadata: - name: nginx-ingress-role - labels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx -rules: - - apiGroups: - - "" - resources: - - configmaps - - pods - - secrets - - namespaces - verbs: - - get - - apiGroups: - - "" - resources: - - configmaps - resourceNames: - # Defaults to "-" - # Here: "-" - # This has to be adapted if you change either parameter - # when launching the nginx-ingress-controller. - - "ingress-controller-leader-nginx" - verbs: - - get - - update - - apiGroups: - - "" - resources: - - configmaps - verbs: - - create - - apiGroups: - - "" - resources: - - endpoints - verbs: - - get - ---- -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: RoleBinding -metadata: - name: nginx-ingress-role-${NAMESPACE} - labels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: nginx-ingress-role -subjects: - - kind: ServiceAccount - name: nginx-ingress-serviceaccount - namespace: ${NAMESPACE} - ---- -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: ClusterRoleBinding -metadata: - name: nginx-ingress-clusterrole-${NAMESPACE} - labels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: nginx-ingress-clusterrole -subjects: - - kind: ServiceAccount - name: nginx-ingress-serviceaccount - namespace: ${NAMESPACE} - ---- -apiVersion: extensions/v1beta1 -kind: Deployment -metadata: - name: nginx-ingress-controller - labels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx -spec: - replicas: 1 - selector: - matchLabels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - template: - metadata: - labels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - annotations: - prometheus.io/port: "10254" - prometheus.io/scrape: "true" - spec: - terminationGracePeriodSeconds: 0 - serviceAccountName: nginx-ingress-serviceaccount - initContainers: - - name: enable-coredump - image: busybox - command: - - /bin/sh - - -c - - | - ulimit -c unlimited - echo "/tmp/core.%e.%p" > /proc/sys/kernel/core_pattern - sysctl -w fs.suid_dumpable=2 - securityContext: - privileged: true - containers: - - name: nginx-ingress-controller - image: ingress-controller/nginx-ingress-controller:dev - args: - - /nginx-ingress-controller - - --configmap=$(POD_NAMESPACE)/nginx-configuration - - --tcp-services-configmap=$(POD_NAMESPACE)/tcp-services - - --udp-services-configmap=$(POD_NAMESPACE)/udp-services - - --publish-service=$(POD_NAMESPACE)/ingress-nginx - - --annotations-prefix=nginx.ingress.kubernetes.io - - --watch-namespace=${NAMESPACE} - securityContext: - allowPrivilegeEscalation: true - capabilities: - drop: - - ALL - add: - - NET_BIND_SERVICE - # www-data -> 33 - runAsUser: 33 - env: - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - ports: - - name: http - containerPort: 80 - - name: https - containerPort: 443 - livenessProbe: - failureThreshold: 3 - httpGet: - path: /healthz - port: 10254 - scheme: HTTP - initialDelaySeconds: 10 - periodSeconds: 10 - successThreshold: 1 - timeoutSeconds: 1 - readinessProbe: - failureThreshold: 3 - httpGet: - path: /healthz - port: 10254 - scheme: HTTP - periodSeconds: 10 - successThreshold: 1 - timeoutSeconds: 1 diff --git a/test/e2e-image/overlay/deployment-e2e.yaml b/test/e2e-image/overlay/deployment-e2e.yaml new file mode 100644 index 0000000000..f8df465a65 --- /dev/null +++ b/test/e2e-image/overlay/deployment-e2e.yaml @@ -0,0 +1,26 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nginx-ingress-controller +spec: + template: + spec: + terminationGracePeriodSeconds: 0 + initContainers: + - name: enable-coredump + image: busybox + command: + - /bin/sh + - -c + - | + ulimit -c unlimited + echo "/tmp/core.%e.%p" > /proc/sys/kernel/core_pattern + sysctl -w fs.suid_dumpable=2 + securityContext: + privileged: true + containers: + - name: nginx-ingress-controller + livenessProbe: + timeoutSeconds: 1 + readinessProbe: + timeoutSeconds: 1 diff --git a/test/e2e-image/overlay/deployment-extension-group-patch.yaml b/test/e2e-image/overlay/deployment-extension-group-patch.yaml new file mode 100644 index 0000000000..837a5f7e12 --- /dev/null +++ b/test/e2e-image/overlay/deployment-extension-group-patch.yaml @@ -0,0 +1,3 @@ +- op: replace + path: /apiVersion + value: extensions/v1beta1 diff --git a/test/e2e-image/overlay/deployment-namespace-patch.yaml b/test/e2e-image/overlay/deployment-namespace-patch.yaml new file mode 100644 index 0000000000..f0f1fddd8a --- /dev/null +++ b/test/e2e-image/overlay/deployment-namespace-patch.yaml @@ -0,0 +1,3 @@ +- op: add + path: /spec/template/spec/containers/0/args/-1 + value: "--watch-namespace=$(POD_NAMESPACE)" diff --git a/test/e2e-image/overlay/kustomization.yaml b/test/e2e-image/overlay/kustomization.yaml new file mode 100644 index 0000000000..a79dfb73cb --- /dev/null +++ b/test/e2e-image/overlay/kustomization.yaml @@ -0,0 +1,34 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +bases: +- ../../../deploy/cloud-generic +configMapGenerator: +- name: nginx-configuration + behavior: merge + literals: + - worker-processes=1 +patchesStrategicMerge: +- deployment-e2e.yaml +- service-protocol-tcp.yaml +patchesJson6902: +- path: deployment-namespace-patch.yaml + target: + group: apps + kind: Deployment + name: nginx-ingress-controller + version: v1 +- path: service-cluster-patch.yaml + target: + kind: Service + name: ingress-nginx + version: v1 +- path: deployment-extension-group-patch.yaml + target: + group: apps + kind: Deployment + name: nginx-ingress-controller + version: v1 +images: +- name: quay.io/kubernetes-ingress-controller/nginx-ingress-controller + newName: ingress-controller/nginx-ingress-controller + newTag: dev diff --git a/test/e2e-image/overlay/service-cluster-patch.yaml b/test/e2e-image/overlay/service-cluster-patch.yaml new file mode 100644 index 0000000000..0465d38046 --- /dev/null +++ b/test/e2e-image/overlay/service-cluster-patch.yaml @@ -0,0 +1,4 @@ +- op: remove + path: /spec/externalTrafficPolicy +- op: remove + path: /spec/type diff --git a/test/e2e-image/overlay/service-protocol-tcp.yaml b/test/e2e-image/overlay/service-protocol-tcp.yaml new file mode 100644 index 0000000000..c49626fcd1 --- /dev/null +++ b/test/e2e-image/overlay/service-protocol-tcp.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: Service +metadata: + name: ingress-nginx +spec: + ports: + - name: http + port: 80 + targetPort: 80 + protocol: TCP + - name: https + port: 443 + targetPort: 443 + protocol: TCP diff --git a/test/e2e/defaultbackend/custom_default_backend.go b/test/e2e/defaultbackend/custom_default_backend.go index 463d302ae6..4bdb9ed17f 100644 --- a/test/e2e/defaultbackend/custom_default_backend.go +++ b/test/e2e/defaultbackend/custom_default_backend.go @@ -17,7 +17,6 @@ limitations under the License. package defaultbackend import ( - "fmt" "net/http" "strings" @@ -40,7 +39,7 @@ var _ = framework.IngressNginxDescribe("Custom Default Backend", func() { framework.UpdateDeployment(f.KubeClientSet, f.Namespace, "nginx-ingress-controller", 1, func(deployment *appsv1beta1.Deployment) error { args := deployment.Spec.Template.Spec.Containers[0].Args - args = append(args, fmt.Sprintf("--default-backend-service=%s/%s", f.Namespace, "http-svc")) + args = append(args, "--default-backend-service=$(POD_NAMESPACE)/http-svc") deployment.Spec.Template.Spec.Containers[0].Args = args _, err := f.KubeClientSet.AppsV1beta1().Deployments(f.Namespace).Update(deployment) diff --git a/test/e2e/settings/pod_security_policy.go b/test/e2e/settings/pod_security_policy.go index 58cd227c09..fbf833a340 100644 --- a/test/e2e/settings/pod_security_policy.go +++ b/test/e2e/settings/pod_security_policy.go @@ -17,6 +17,7 @@ limitations under the License. package settings import ( + "fmt" "net/http" "strings" @@ -48,7 +49,7 @@ var _ = framework.IngressNginxDescribe("[Serial] Pod Security Policies", func() Expect(err).NotTo(HaveOccurred(), "creating Pod Security Policy") } - role, err := f.KubeClientSet.RbacV1().ClusterRoles().Get("nginx-ingress-clusterrole", metav1.GetOptions{}) + role, err := f.KubeClientSet.RbacV1().ClusterRoles().Get(fmt.Sprintf("nginx-ingress-clusterrole-%v", f.Namespace), metav1.GetOptions{}) Expect(err).NotTo(HaveOccurred(), "getting ingress controller cluster role") Expect(role).NotTo(BeNil()) @@ -78,7 +79,7 @@ var _ = framework.IngressNginxDescribe("[Serial] Pod Security Policies", func() }) AfterEach(func() { - role, err := f.KubeClientSet.RbacV1().ClusterRoles().Get("nginx-ingress-clusterrole", metav1.GetOptions{}) + role, err := f.KubeClientSet.RbacV1().ClusterRoles().Get(fmt.Sprintf("nginx-ingress-clusterrole-%v", f.Namespace), metav1.GetOptions{}) Expect(err).NotTo(HaveOccurred(), "getting ingress controller cluster role") Expect(role).NotTo(BeNil()) diff --git a/test/e2e/wait-for-nginx.sh b/test/e2e/wait-for-nginx.sh index f4e01d9680..a9c6658bdd 100755 --- a/test/e2e/wait-for-nginx.sh +++ b/test/e2e/wait-for-nginx.sh @@ -35,10 +35,32 @@ function on_exit { } trap on_exit EXIT -kubectl apply --namespace=$NAMESPACE -f $DIR/manifests/service.yaml +CLUSTER_WIDE="$DIR/cluster-wide-$NAMESPACE" -sed "s@\${NAMESPACE}@${NAMESPACE}@" $DIR/manifests/mandatory.yaml | kubectl apply --namespace=$NAMESPACE -f - -cat $DIR/manifests/service.yaml | kubectl apply --namespace=$NAMESPACE -f - +mkdir "$CLUSTER_WIDE" + +cat << EOF > "$CLUSTER_WIDE/kustomization.yaml" +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +bases: +- ../cluster-wide +nameSuffix: "-$NAMESPACE" +EOF + +OVERLAY="$DIR/overlay-$NAMESPACE" + +mkdir "$OVERLAY" + +cat << EOF > "$OVERLAY/kustomization.yaml" +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: $NAMESPACE +bases: +- ../overlay +- ../cluster-wide-$NAMESPACE +EOF + +kubectl apply --kustomize "$OVERLAY" # wait for the deployment and fail if there is an error before starting the execution of any test kubectl rollout status \