From 4d675e2eef3d7857078eb182b04328d083e7ba8c Mon Sep 17 00:00:00 2001 From: Manuel Alejandro de Brito Fontes Date: Tue, 6 Jul 2021 09:22:40 -0400 Subject: [PATCH 1/9] Remove session affinity --- chart/templates/_default.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/chart/templates/_default.yaml b/chart/templates/_default.yaml index 61262d632f06c9..ee1112a3a3a9c8 100644 --- a/chart/templates/_default.yaml +++ b/chart/templates/_default.yaml @@ -51,7 +51,6 @@ spec: {{- if $comp.clusterIP }} clusterIP: {{ $comp.clusterIP }} {{- end }} - sessionAffinity: {{ $comp.serviceSessionAffinity | default "None" }} {{ if $comp.serviceExternalTrafficPolicy -}} externalTrafficPolicy: {{ $comp.serviceExternalTrafficPolicy }} {{- end }} From 7b246f7173531edc49e892846787e7f95d9a59e3 Mon Sep 17 00:00:00 2001 From: Manuel Alejandro de Brito Fontes Date: Tue, 6 Jul 2021 09:23:26 -0400 Subject: [PATCH 2/9] Add update ca-certificates initContainer --- chart/templates/_helpers.tpl | 77 ++++++++++++++++++++---------------- 1 file changed, 42 insertions(+), 35 deletions(-) diff --git a/chart/templates/_helpers.tpl b/chart/templates/_helpers.tpl index c7f4aa5f786007..0f647fd7f77dbd 100644 --- a/chart/templates/_helpers.tpl +++ b/chart/templates/_helpers.tpl @@ -89,42 +89,13 @@ affinity: {{- define "gitpod.workspaceAffinity" -}} {{- $ := .root -}} {{- $gp := .gp -}} -{{- $expr := dict -}} -{{- if $gp.components.workspace.affinity -}} -{{- if $gp.components.workspace.affinity.default -}}{{- $_ := set $expr $gp.components.workspace.affinity.default "" -}}{{- end -}} -{{- if $gp.components.workspace.affinity.prebuild -}}{{- $_ := set $expr $gp.components.workspace.affinity.prebuild "" -}}{{- end -}} -{{- if $gp.components.workspace.affinity.probe -}}{{- $_ := set $expr $gp.components.workspace.affinity.probe "" -}}{{- end -}} -{{- if $gp.components.workspace.affinity.regular -}}{{- $_ := set $expr $gp.components.workspace.affinity.regular "" -}}{{- end -}} -{{- end -}} -{{- /* - In a previous iteration of the templates the node affinity was part of the workspace pod template. - In that case we need to extract the affinity from the template and add it to the workspace affinity set. -*/ -}} -{{- if $gp.components.workspace.template -}} -{{- if $gp.components.workspace.template.spec -}} -{{- if $gp.components.workspace.template.spec.affinity -}} -{{- if $gp.components.workspace.template.spec.affinity.nodeAffinity -}} -{{- if $gp.components.workspace.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution -}} -{{- range $_, $t := $gp.components.workspace.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms -}} -{{- range $_, $m := $t.matchExpressions -}} - {{- $_ := set $expr $m.key "" -}} -{{- end -}} -{{- end -}} -{{- end -}} -{{- end -}} -{{- end -}} -{{- end -}} -{{- end -}} -{{- if not (eq (len $expr) 0) -}} +{{- $comp := .comp -}} +{{- if $comp.affinity -}} affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - {{- range $key, $val := $expr }} - - matchExpressions: - - key: {{ $key }} - operator: Exists - {{- end }} +{{ $comp.affinity | toYaml | indent 2 }} +{{- else if $gp.affinity -}} +affinity: +{{ $gp.affinity | toYaml | indent 2 }} {{- end -}} {{- end -}} @@ -375,3 +346,39 @@ storage: runAsUser: 65532 terminationMessagePolicy: FallbackToLogsOnError {{- end -}} + +{{/* Container definition to update ca-certificates and add gitpod self-signed CA certificate */}} +{{- define "gitpod.ca-certificates.container" -}} +- name: update-ca-certificates + image: alpine:3.14 + command: + - sh + - -c + - | + set -e + apk add --update ca-certificates + cp /etc/ssl/gitpod-ca.crt /usr/local/share/ca-certificates + update-ca-certificates + cp /etc/ssl/certs/* /ssl-certs + volumeMounts: + - name: cacerts + mountPath: "/ssl-certs" + - name: registry-certs + subPath: ca.crt + mountPath: /etc/ssl/gitpod-ca.crt +{{- end -}} + +{{/* Volume mount for updated ca-certificates */}} +{{- define "gitpod.ca-certificates.volumeMount" }} +- name: cacerts + mountPath: /etc/ssl/certs +{{- end -}} + +{{/* emptyDir volume ca-certificates */}} +{{- define "gitpod.ca-certificates.volume" }} +- name: cacerts + emptyDir: {} +- name: registry-certs + secret: + secretName: builtin-registry-certs +{{- end -}} From 2a0f82a4aa8667dabe5be19d8693141c35f60df5 Mon Sep 17 00:00:00 2001 From: Manuel Alejandro de Brito Fontes Date: Tue, 6 Jul 2021 09:26:32 -0400 Subject: [PATCH 3/9] Refactor generation of self-signed certificates to use cert-manager --- .../templates/builtin-proxy-certs-secret.yaml | 34 ++++++++++++ .../builtin-registry-certs-secret.yaml | 29 +---------- chart/templates/certmanager-ca.yaml | 39 ++++++++------ chart/templates/ws-daemon-tlssecret.yaml | 25 ++------- chart/templates/ws-manager-tlssecret.yaml | 52 ++----------------- 5 files changed, 68 insertions(+), 111 deletions(-) create mode 100644 chart/templates/builtin-proxy-certs-secret.yaml diff --git a/chart/templates/builtin-proxy-certs-secret.yaml b/chart/templates/builtin-proxy-certs-secret.yaml new file mode 100644 index 00000000000000..76fa286c6f79a6 --- /dev/null +++ b/chart/templates/builtin-proxy-certs-secret.yaml @@ -0,0 +1,34 @@ +# Copyright (c) 2021 Gitpod GmbH. All rights reserved. +# Licensed under the MIT License. See License-MIT.txt in the project root for license information. + +{{ $comp := .Values.components.proxy -}} +{{ $certManager := (index .Values "cert-manager") }} +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: proxy-config-certificates +spec: + secretName: proxy-config-certificates + commonName: {{ .Values.hostname | quote }} + dnsNames: + - {{ .Values.hostname | quote }} + - "*.{{ .Values.hostname }}" + - "*.ws{{- if .Values.installation.shortname -}}-{{ .Values.installation.shortname }}{{- end -}}.{{ .Values.hostname }}" + {{- range $index, $dnsName := $certManager.certificate.additionalNames }} + - {{ $dnsName | quote }} + {{ end -}} + {{- if $comp.certManager }} {{/* custom proxy certificate issuer */}} + {{ if $comp.certManager.issuerRef }} {{/* use custom issuer */}} + issuerRef: {{ $comp.certManager.issuerRef }} + {{ else }} + issuerRef: + name: {{ if $comp.certManager.issuerName }}{{$comp.certManager.issuerName}}{{ else }}gitpod{{ end }} + kind: {{ if $comp.certManager.issuerKind }}{{$comp.certManager.issuerKind}}{{ else }}Issuer{{ end }} + group: cert-manager.io + {{ end }} + {{ else }} + issuerRef: + name: gitpod + kind: Issuer + group: cert-manager.io + {{ end }} diff --git a/chart/templates/builtin-registry-certs-secret.yaml b/chart/templates/builtin-registry-certs-secret.yaml index 315a5d664d46c7..7c0e9e2742806d 100644 --- a/chart/templates/builtin-registry-certs-secret.yaml +++ b/chart/templates/builtin-registry-certs-secret.yaml @@ -3,9 +3,7 @@ {{ if (index .Values "docker-registry" "enabled") }} {{- $regName := include "gitpod.builtinRegistry.name" . -}} -{{ $cm := (index .Values "cert-manager") }} -{{- if $cm.enabled }} - +{{ $certManager := (index .Values "cert-manager") }} apiVersion: cert-manager.io/v1 kind: Certificate metadata: @@ -14,30 +12,7 @@ spec: secretName: builtin-registry-certs dnsNames: {{ ( list $regName ) | toJson }} issuerRef: - name: {{ $cm.ca.issuerName }} + name: {{ $certManager.ca.issuerName }} kind: Issuer group: cert-manager.io - -{{- else }} -{{- $ca := genCA "wsdaemon-ca" 365 -}} -{{- $cert := genSignedCert (include "gitpod.fullname" . ) nil ( list $regName ) 365 $ca -}} -apiVersion: v1 -kind: Secret -metadata: - name: builtin-registry-certs - labels: - app: {{ template "gitpod.fullname" . }} - chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" - release: "{{ .Release.Name }}" - heritage: "{{ .Release.Service }}" - annotations: - checksum/cert: {{ $cert.Cert | indent 2 | sha256sum }} -data: - ca.crt: {{ $ca.Cert | b64enc }} - tls.key: {{ $cert.Key | b64enc }} - # Docker registry needs this file to end with .crt - tls.crt: {{ $cert.Cert | b64enc }} - # Docker daemon needs this file to end with .cert - tls.cert: {{ $cert.Cert | b64enc }} -{{- end }} {{- end }} diff --git a/chart/templates/certmanager-ca.yaml b/chart/templates/certmanager-ca.yaml index 5cac4a5e02e96c..a70d70309ae518 100644 --- a/chart/templates/certmanager-ca.yaml +++ b/chart/templates/certmanager-ca.yaml @@ -1,27 +1,36 @@ # Copyright (c) 2020 Gitpod GmbH. All rights reserved. # Licensed under the MIT License. See License-MIT.txt in the project root for license information. -{{ $cm := (index .Values "cert-manager") }} -{{- if $cm.enabled }} -{{- if $cm.ca.certificate.selfSigned }} -{{ $tls := genCA "gitpod-ca" 365 }} -kind: Secret -apiVersion: v1 +{{ $certManager := (index .Values "cert-manager") }} +apiVersion: cert-manager.io/v1 +kind: Issuer metadata: - name: {{ $cm.ca.certificate.secretName }} + name: gitpod-selfsigned-issuer +spec: + selfSigned: {} +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: {{ $certManager.ca.secretName }} namespace: {{ .Release.Namespace }} -data: - tls.crt: {{ $tls.Cert | b64enc }} - tls.key: {{ $tls.Key | b64enc }} -type: kubernetes.io/tls +spec: + isCA: true + commonName: {{ $certManager.ca.issuerName }} + secretName: {{ $certManager.ca.secretName }} + privateKey: + algorithm: ECDSA + size: 256 + issuerRef: + name: gitpod-selfsigned-issuer + kind: Issuer + group: cert-manager.io --- -{{- end }} apiVersion: cert-manager.io/v1 kind: Issuer metadata: - name: {{ $cm.ca.issuerName }} + name: {{ $certManager.ca.issuerName }} namespace: {{ .Release.Namespace }} spec: ca: - secretName: {{ $cm.ca.certificate.secretName }} -{{- end }} \ No newline at end of file + secretName: {{ $certManager.ca.secretName }} diff --git a/chart/templates/ws-daemon-tlssecret.yaml b/chart/templates/ws-daemon-tlssecret.yaml index 5f7b67c8d15a57..f0975ab2a022f1 100644 --- a/chart/templates/ws-daemon-tlssecret.yaml +++ b/chart/templates/ws-daemon-tlssecret.yaml @@ -5,8 +5,8 @@ {{- $this := dict "root" . "gp" $.Values "comp" $comp -}} {{- if not $comp.disabled -}} {{- $altNames := list ( printf "%s.%s" (include "gitpod.fullname" .) .Release.Namespace ) ( printf "%s.%s.svc" "ws-daemon" .Release.Namespace ) ( printf "wsdaemon" ) -}} -{{ $cm := (index .Values "cert-manager") }} -{{- if $cm.enabled }} + +{{ $certManager := (index .Values "cert-manager") }} apiVersion: cert-manager.io/v1 kind: Certificate metadata: @@ -16,26 +16,7 @@ spec: secretName: ws-daemon-tls dnsNames: {{ $altNames | toJson }} issuerRef: - name: {{ $cm.ca.issuerName }} + name: {{ $certManager.ca.issuerName }} kind: Issuer group: cert-manager.io -{{- else }} -{{- $ca := genCA "wsdaemon-ca" 365 -}} -{{- $cert := genSignedCert (include "gitpod.fullname" . ) nil $altNames 365 $ca -}} -apiVersion: v1 -kind: Secret -metadata: - name: ws-daemon-tls - labels: - app: {{ template "gitpod.fullname" $ }} - chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" - release: "{{ .Release.Name }}" - heritage: "{{ .Release.Service }}" - annotations: - checksum/checksd-config: {{ $cert.Cert | indent 2 | sha256sum }} -data: - ca.crt: {{ $ca.Cert | b64enc }} - tls.crt: {{ $cert.Cert | b64enc }} - tls.key: {{ $cert.Key | b64enc }} {{- end }} -{{- end }} \ No newline at end of file diff --git a/chart/templates/ws-manager-tlssecret.yaml b/chart/templates/ws-manager-tlssecret.yaml index 5ed82b4d336564..d98d19d1812570 100644 --- a/chart/templates/ws-manager-tlssecret.yaml +++ b/chart/templates/ws-manager-tlssecret.yaml @@ -9,9 +9,7 @@ {{- $server := $comp.tls.server }} {{- $client := $comp.tls.client }} - -{{ $cm := (index $.Values "cert-manager") }} -{{- if $cm.enabled }} +{{ $certManager := (index .Values "cert-manager") }} apiVersion: cert-manager.io/v1 kind: Certificate metadata: @@ -26,14 +24,14 @@ spec: secretName: "{{ $server.secretName }}" dnsNames: {{ $serverAltNames | toJson }} issuerRef: - name: {{ $cm.ca.issuerName }} + name: {{ $certManager.ca.issuerName }} kind: Issuer group: cert-manager.io --- apiVersion: cert-manager.io/v1 kind: Certificate metadata: - name: "{{ $client.secretName }}" + name: {{ $client.secretName }} namespace: {{ .Release.Namespace }} labels: app: {{ template "gitpod.fullname" $ }} @@ -44,47 +42,7 @@ spec: secretName: "{{ $client.secretName }}" dnsNames: {{ $clientAltNames | toJson }} issuerRef: - name: {{ $cm.ca.issuerName }} + name: {{ $certManager.ca.issuerName }} kind: Issuer group: cert-manager.io -{{- else }} -{{- $ca := genCA "wsmanager-ca" 365 -}} -{{- if not $server.crtFile }} -{{- $cert := genSignedCert (include "gitpod.fullname" . ) nil $serverAltNames 365 $ca -}} -apiVersion: v1 -kind: Secret -metadata: - name: "{{ $server.secretName }}" - labels: - app: {{ template "gitpod.fullname" $ }} - chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" - release: "{{ .Release.Name }}" - heritage: "{{ .Release.Service }}" - annotations: - checksum/checksd-config: {{ $cert.Cert | indent 2 | sha256sum }} -data: - ca.crt: {{ $ca.Cert | b64enc }} - tls.crt: {{ $cert.Cert | b64enc }} - tls.key: {{ $cert.Key | b64enc }} -{{- end }} ---- -{{- if not $client.crtFile }} -{{- $cert := genSignedCert (include "gitpod.fullname" . ) nil $clientAltNames 365 $ca -}} -apiVersion: v1 -kind: Secret -metadata: - name: "{{ $client.secretName }}" - labels: - app: {{ template "gitpod.fullname" $ }} - chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" - release: "{{ .Release.Name }}" - heritage: "{{ .Release.Service }}" - annotations: - checksum/checksd-config: {{ $cert.Cert | indent 2 | sha256sum }} -data: - ca.crt: {{ $ca.Cert | b64enc }} - tls.crt: {{ $cert.Cert | b64enc }} - tls.key: {{ $cert.Key | b64enc }} -{{- end }} -{{- end }} -{{- end }} \ No newline at end of file +{{ end }} From f7b2694a8487a1ae3b65ac766a3d9e33814908d7 Mon Sep 17 00:00:00 2001 From: Manuel Alejandro de Brito Fontes Date: Tue, 6 Jul 2021 09:27:12 -0400 Subject: [PATCH 4/9] Create Job to ensure database gitpod-session exists --- chart/templates/db-initscripts-configmap.yaml | 4 +- chart/templates/db-serviceaccount.yaml | 2 +- chart/templates/db-sessions-job.yaml | 57 +++++++++++++++++++ 3 files changed, 59 insertions(+), 4 deletions(-) create mode 100644 chart/templates/db-sessions-job.yaml diff --git a/chart/templates/db-initscripts-configmap.yaml b/chart/templates/db-initscripts-configmap.yaml index 076a9d2444fd1f..2875c87600d9a4 100644 --- a/chart/templates/db-initscripts-configmap.yaml +++ b/chart/templates/db-initscripts-configmap.yaml @@ -1,7 +1,6 @@ # Copyright (c) 2020 Gitpod GmbH. All rights reserved. # Licensed under the MIT License. See License-MIT.txt in the project root for license information. -{{ if not .Values.components.db.gcloudSqlProxy.enabled -}} apiVersion: v1 kind: ConfigMap metadata: @@ -17,7 +16,6 @@ metadata: data: init.sql: |- {{- $root := . }} -{{- range $path, $bytes := .Files.Glob "config/db/init/**.sql" }} +{{- range $path, $bytes := .Files.Glob "config/db/init/02-create-and-init-sessions-db.sql" }} {{ $root.Files.Get $path | indent 4 }} {{- end }} -{{- end -}} diff --git a/chart/templates/db-serviceaccount.yaml b/chart/templates/db-serviceaccount.yaml index f27ff8d5bc1d4d..03d57a5f80dd28 100644 --- a/chart/templates/db-serviceaccount.yaml +++ b/chart/templates/db-serviceaccount.yaml @@ -10,4 +10,4 @@ metadata: component: db kind: service-account stage: {{ .Values.installation.stage }} -automountServiceAccountToken: false \ No newline at end of file +automountServiceAccountToken: false diff --git a/chart/templates/db-sessions-job.yaml b/chart/templates/db-sessions-job.yaml new file mode 100644 index 00000000000000..45eb61322c2c88 --- /dev/null +++ b/chart/templates/db-sessions-job.yaml @@ -0,0 +1,57 @@ +# Copyright (c) 2020 Gitpod GmbH. All rights reserved. +# Licensed under the MIT License. See License-MIT.txt in the project root for license information. + +{{ $comp := .Values.components.db }} +{{- $this := dict "root" . "gp" $.Values "comp" $comp -}} + +apiVersion: batch/v1 +kind: Job +metadata: + name: database-sessions + labels: + app: {{ template "gitpod.fullname" . }} + component: database-sessions + kind: job + stage: {{ .Values.installation.stage }} + annotations: + # This is what defines this resource as a hook. Without this line, the + # job is considered part of the release. + # see https://github.com/helm/helm/blob/master/docs/charts_hooks.md + "helm.sh/hook": post-install,post-upgrade + "helm.sh/hook-weight": "5" + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded,hook-failed + "helm.sh/hook-delete-timeout": "0" +spec: + template: + metadata: + name: database-sessions + labels: + app: {{ template "gitpod.fullname" . }} + component: database-sessions + kind: job + stage: {{ .Values.installation.stage }} + spec: +{{ include "gitpod.pod.affinity" $this | indent 6 }} + restartPolicy: Never + serviceAccountName: db + imagePullSecrets: + enableServiceLinks: false + containers: + - name: database-sessions + image: mysql:5.7.34 + env: + - name: "MYSQL_PWD" + value: "{{ $.Values.db.password }}" + - name: "DB_PORT" + value: "{{ $.Values.db.port }}" + - name: "DB_HOST" + value: "{{ $.Values.db.host }}" + command: ["sh","-c","mysql -h $DB_HOST --port $DB_PORT -u gitpod < /db-init-scripts/init.sql"] + volumeMounts: + - name: db-init-scripts + mountPath: "/db-init-scripts" + readOnly: true + volumes: + - name: db-init-scripts + configMap: + name: db-init-scripts From c64946b0d6ac6b9b4ec589212aa199ec414f5691 Mon Sep 17 00:00:00 2001 From: Manuel Alejandro de Brito Fontes Date: Tue, 6 Jul 2021 09:34:32 -0400 Subject: [PATCH 5/9] Remove priorityClassName workaround --- chart/templates/ws-daemon-daemonset.yaml | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) diff --git a/chart/templates/ws-daemon-daemonset.yaml b/chart/templates/ws-daemon-daemonset.yaml index 0fb239a8bfa400..dd72891f36bc26 100644 --- a/chart/templates/ws-daemon-daemonset.yaml +++ b/chart/templates/ws-daemon-daemonset.yaml @@ -38,21 +38,7 @@ spec: {{- end }} spec: {{ include "gitpod.workspaceAffinity" $this | indent 6 }} - # see https://kubernetes.io/docs/tasks/administer-cluster/guaranteed-scheduling-critical-addon-pods/ for more - # details on this priority class. - # Pior to Kubernetes 1.17 critical pods can only be scheduled in kube-system: https://github.com/kubernetes/kubernetes/pull/76310 - # Leaving this in here so that others might enable it and we don't forget about this when 1.17 comes around in GKE. - # priorityClassName: system-node-critical - tolerations: - - key: node.kubernetes.io/disk-pressure - operator: "Exists" - effect: "NoExecute" - - key: node.kubernetes.io/memory-pressure - operator: "Exists" - effect: "NoExecute" - - key: node.kubernetes.io/out-of-disk - operator: "Exists" - effect: "NoExecute" + priorityClassName: system-node-critical serviceAccountName: ws-daemon hostPID: true volumes: @@ -195,4 +181,4 @@ spec: procMount: Default {{ include "gitpod.kube-rbac-proxy" $this | indent 6 }} {{ toYaml .Values.defaults | indent 6 }} -{{ end }} \ No newline at end of file +{{ end }} From 82a739b3eb6462e01a82f1d21ead08d85b8b8d59 Mon Sep 17 00:00:00 2001 From: Manuel Alejandro de Brito Fontes Date: Tue, 6 Jul 2021 09:35:50 -0400 Subject: [PATCH 6/9] Mount self-signed SSL certificate for in cluster docker registry access --- chart/templates/image-builder-deployment.yaml | 12 +++++++++++- chart/templates/registry-facade-daemonset.yaml | 4 ++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/chart/templates/image-builder-deployment.yaml b/chart/templates/image-builder-deployment.yaml index 151e47e6a0fe7e..8595c975c737bd 100644 --- a/chart/templates/image-builder-deployment.yaml +++ b/chart/templates/image-builder-deployment.yaml @@ -77,8 +77,18 @@ spec: - mountPath: /var/lib/docker name: dind-storage {{- range $idx, $sec := $comp.registryCerts }} - - mountPath: /etc/docker/certs.d/{{- if eq $sec.name "builtin" -}}{{ template "gitpod.builtinRegistry.name" $this.root }}{{ else }}{{ $sec.name }}{{ end }} + - mountPath: /etc/docker/certs.d/{{- if eq $sec.name "builtin" -}}{{ template "gitpod.builtinRegistry.name" $this.root }}{{ else }}{{ $sec.name }}{{ end }}/tls.cert name: docker-tls-certs-{{ $idx }} + subPath: tls.crt + - mountPath: /etc/docker/certs.d/{{- if eq $sec.name "builtin" -}}{{ template "gitpod.builtinRegistry.name" $this.root }}{{ else }}{{ $sec.name }}{{ end }}/tls.crt + name: docker-tls-certs-{{ $idx }} + subPath: tls.crt + - mountPath: /etc/docker/certs.d/{{- if eq $sec.name "builtin" -}}{{ template "gitpod.builtinRegistry.name" $this.root }}{{ else }}{{ $sec.name }}{{ end }}/tls.key + name: docker-tls-certs-{{ $idx }} + subPath: tls.key + - mountPath: /etc/docker/certs.d/{{- if eq $sec.name "builtin" -}}{{ template "gitpod.builtinRegistry.name" $this.root }}{{ else }}{{ $sec.name }}{{ end }}/ca.crt + name: docker-tls-certs-{{ $idx }} + subPath: ca.crt {{- end }} {{- if $comp.dindResources }} resources: diff --git a/chart/templates/registry-facade-daemonset.yaml b/chart/templates/registry-facade-daemonset.yaml index 8998a2d2f93148..4537895cbe8484 100644 --- a/chart/templates/registry-facade-daemonset.yaml +++ b/chart/templates/registry-facade-daemonset.yaml @@ -35,6 +35,8 @@ spec: {{ include "gitpod.workspaceAffinity" $this | indent 6 }} serviceAccountName: registry-facade enableServiceLinks: false + initContainers: +{{ include "gitpod.ca-certificates.container" $this | indent 8 }} containers: - name: registry-facade image: {{ template "gitpod.comp.imageFull" $this }} @@ -53,6 +55,7 @@ spec: - name: GRPC_GO_RETRY value: "on" volumeMounts: +{{ include "gitpod.ca-certificates.volumeMount" $this | indent 8 }} - name: cache mountPath: "/mnt/cache" - name: config @@ -90,5 +93,6 @@ spec: secret: secretName: {{ .Values.certificatesSecret.secretName }} {{- end }} +{{- include "gitpod.ca-certificates.volume" $this | indent 6 }} {{ toYaml .Values.defaults | indent 6 }} {{ end }} From 08614948e3906225aeebb8c195bc47a15409ba59 Mon Sep 17 00:00:00 2001 From: Manuel Alejandro de Brito Fontes Date: Tue, 6 Jul 2021 09:39:02 -0400 Subject: [PATCH 7/9] Update default values --- chart/values.yaml | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/chart/values.yaml b/chart/values.yaml index c0e7d950196cb4..9540b3dadde1a1 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -22,7 +22,7 @@ installNetworkPolicies: true installPodSecurityPolicies: true imagePullPolicy: IfNotPresent -# configure default log level +# configure default log level of gitpod.io components log: level: trace @@ -225,7 +225,6 @@ components: cpu: 100m memory: 128Mi alpineImage: alpine:3.14 - selfBuildBaseImage: "" ports: rpc: expose: true @@ -304,7 +303,6 @@ components: debug: expose: false containerPort: 9229 - serviceSessionAffinity: None serverContainer: volumeMounts: null env: null @@ -362,7 +360,6 @@ components: containerPort: 443 loadBalancerIP: null serviceType: "LoadBalancer" - serviceSessionAffinity: "None" serviceExternalTrafficPolicy: null wsManager: @@ -506,7 +503,6 @@ mysql: serviceAccount: create: false name: db - initdbScriptsConfigMap: db-init-scripts volumePermissions: enabled: true @@ -648,10 +644,17 @@ rabbitmq: {{- end }} cert-manager: - enabled: false + # assume cert-manager is already installed in the cluster + install: false + # gitpod Certificate + certificate: + # by default use the gitpod CA issuer + issuerName: null + # allow additional names to *.ws. and *. + additionalDNSNames: [] installCRDs: true + # custom CA for certificates used by gitpod + # this certificates are internal to the namespace where gitpod is running ca: issuerName: ca-issuer - certificate: - selfSigned: true - secretName: gitpod-ca-certificate + secretName: gitpod-ca-certificate From 5e8805c0bc5b959c6dcc3e925174e279d039a209 Mon Sep 17 00:00:00 2001 From: Manuel Alejandro de Brito Fontes Date: Mon, 12 Jul 2021 06:43:51 -0400 Subject: [PATCH 8/9] Sync changes --- chart/templates/_helpers.tpl | 1 + chart/templates/server-deployment.yaml | 4 +++- chart/templates/ws-daemon-daemonset.yaml | 4 ++++ chart/values.yaml | 4 ++++ 4 files changed, 12 insertions(+), 1 deletion(-) diff --git a/chart/templates/_helpers.tpl b/chart/templates/_helpers.tpl index 0f647fd7f77dbd..60dfcaef7fc0c5 100644 --- a/chart/templates/_helpers.tpl +++ b/chart/templates/_helpers.tpl @@ -360,6 +360,7 @@ storage: cp /etc/ssl/gitpod-ca.crt /usr/local/share/ca-certificates update-ca-certificates cp /etc/ssl/certs/* /ssl-certs + echo "OK" volumeMounts: - name: cacerts mountPath: "/ssl-certs" diff --git a/chart/templates/server-deployment.yaml b/chart/templates/server-deployment.yaml index 1b708baa906bd9..245b5eef1a0022 100644 --- a/chart/templates/server-deployment.yaml +++ b/chart/templates/server-deployment.yaml @@ -147,6 +147,8 @@ spec: value: "{{ .Values.components.workspace.ports.http.containerPort | toString }}" - name: THEIA_IMAGE_REPO value: "{{ template "gitpod.comp.imageRepo" (dict "root" . "gp" $.Values "comp" .Values.components.workspace.theiaImage) }}" + - name: THEIA_VERSION + value: "{{ template "gitpod.comp.version" (dict "root" . "gp" $.Values "comp" .Values.components.workspace.theiaImage) }}" - name: CODE_IMAGE_REPO value: "{{ template "gitpod.comp.imageRepo" (dict "root" . "gp" $.Values "comp" .Values.components.workspace.codeImage) }}" - name: EXPERIMENTAL_FEATURE_FLAGS @@ -246,4 +248,4 @@ spec: {{ toYaml $comp.volumes | indent 6 }} {{- end }} {{ toYaml .Values.defaults | indent 6 }} -{{ end }} \ No newline at end of file +{{ end }} diff --git a/chart/templates/ws-daemon-daemonset.yaml b/chart/templates/ws-daemon-daemonset.yaml index dd72891f36bc26..9f978e882e0fcf 100644 --- a/chart/templates/ws-daemon-daemonset.yaml +++ b/chart/templates/ws-daemon-daemonset.yaml @@ -40,6 +40,10 @@ spec: {{ include "gitpod.workspaceAffinity" $this | indent 6 }} priorityClassName: system-node-critical serviceAccountName: ws-daemon + hostAliases: + - ip: "127.0.0.1" + hostnames: + - {{ (printf "reg.%s" (.Values.components.registryFacade.hostname | default .Values.hostname)) | quote }} hostPID: true volumes: - hostPath: diff --git a/chart/values.yaml b/chart/values.yaml index 9540b3dadde1a1..1babe38058bdfe 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -419,6 +419,10 @@ components: seccompProfileInstaller: enabled: true imageName: "seccomp-profile-installer" + ports: + rpc: + expose: true + containerPort: 8080 wsScheduler: name: "ws-scheduler" From f607e50c7e1f72aa5a9f416d363e9e23625a96bb Mon Sep 17 00:00:00 2001 From: Manuel Alejandro de Brito Fontes Date: Mon, 12 Jul 2021 06:44:23 -0400 Subject: [PATCH 9/9] [registry-facade] Install gitpod self-signed CA certificate --- .../templates/registry-facade-daemonset.yaml | 33 +++++++++++++++++++ .../registry-facade-podsecuritypolicy.yaml | 4 +-- 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/chart/templates/registry-facade-daemonset.yaml b/chart/templates/registry-facade-daemonset.yaml index 4537895cbe8484..f575f036a06dfb 100644 --- a/chart/templates/registry-facade-daemonset.yaml +++ b/chart/templates/registry-facade-daemonset.yaml @@ -32,11 +32,31 @@ spec: stage: {{ .Values.installation.stage }} gitpod.io/nodeService: registry-facade spec: + hostPID: true {{ include "gitpod.workspaceAffinity" $this | indent 6 }} serviceAccountName: registry-facade enableServiceLinks: false initContainers: {{ include "gitpod.ca-certificates.container" $this | indent 8 }} + # Add gitpod self-signed CA certificate to containerd + # in order to pull images from the host without x509 errors + - name: update-containerd-certificates + command: ["nsenter"] + args: ["--mount=/proc/1/ns/mnt", "--", "sh", "-c", "$(SETUP_SCRIPT)"] + image: alpine:3.14 + env: + - name: GITPOD_CA_CERT + valueFrom: + secretKeyRef: + name: builtin-registry-certs + key: ca.crt + - name: SETUP_SCRIPT + valueFrom: + configMapKeyRef: + name: update-containerd-certificates + key: setup.sh + securityContext: + privileged: true containers: - name: registry-facade image: {{ template "gitpod.comp.imageFull" $this }} @@ -95,4 +115,17 @@ spec: {{- end }} {{- include "gitpod.ca-certificates.volume" $this | indent 6 }} {{ toYaml .Values.defaults | indent 6 }} + +--- + +# Install gitpod ca.crt in containerd to allow pulls from the host +# https://github.com/containerd/containerd/blob/main/docs/hosts.md +apiVersion: v1 +kind: ConfigMap +metadata: + name: update-containerd-certificates +data: + setup.sh: | + mkdir -p /etc/containerd/certs.d/{{- (printf "reg.%s:%v" (.Values.components.registryFacade.hostname | default .Values.hostname) .Values.components.registryFacade.ports.registry.servicePort) }} && echo "$GITPOD_CA_CERT" > /etc/containerd/certs.d/{{- (printf "reg.%s:%v" (.Values.components.registryFacade.hostname | default .Values.hostname) .Values.components.registryFacade.ports.registry.servicePort) -}}/ca.crt && echo "OK" + {{ end }} diff --git a/chart/templates/registry-facade-podsecuritypolicy.yaml b/chart/templates/registry-facade-podsecuritypolicy.yaml index 76c0d7c132a734..3c5fb9175f33f4 100644 --- a/chart/templates/registry-facade-podsecuritypolicy.yaml +++ b/chart/templates/registry-facade-podsecuritypolicy.yaml @@ -40,7 +40,7 @@ spec: - 'hostPath' hostNetwork: true hostIPC: false - hostPID: false + hostPID: true hostPorts: - min: 30000 max: 33000 @@ -62,4 +62,4 @@ spec: - min: 1 max: 65535 readOnlyRootFilesystem: false - {{- end -}} \ No newline at end of file + {{- end -}}