diff --git a/.github/workflows/k8s-testing.yml b/.github/workflows/k8s-testing.yml index 824a63e1c6c..6d75f8202e8 100644 --- a/.github/workflows/k8s-testing.yml +++ b/.github/workflows/k8s-testing.yml @@ -108,9 +108,22 @@ jobs: strategy: matrix: - databases: [pgsql, mysql] - brokers: [redis, rabbit] - k8s: ['v1.16.0','v1.20.0'] + include: + # databases, broker and k8s are independent, so we don't need to test each combination + # lastest k8s version (https://kubernetes.io/releases/) and oldest supported version from aws + # are tested (https://docs.aws.amazon.com/eks/latest/userguide/kubernetes-versions.html#available-versions) + - databases: pgsql + brokers: redis + k8s: 'v1.18.16' + - databases: mysql + brokers: rabbit + k8s: 'v1.18.16' + - databases: pgsql + brokers: rabbit + k8s: 'v1.22.0' + - databases: mysql + brokers: redis + k8s: 'v1.22.0' steps: # - name: Login to DockerHub @@ -124,7 +137,7 @@ jobs: - name: Setup Minikube uses: manusa/actions-setup-minikube@v2.4.2 with: - minikube version: 'v1.22.0' + minikube version: 'v1.24.0' kubernetes version: ${{ matrix.k8s }} driver: docker start args: '--addons=ingress' @@ -145,7 +158,6 @@ jobs: - name: Configure HELM repos run: |- - helm repo add stable https://charts.helm.sh/stable helm repo add bitnami https://charts.bitnami.com/bitnami helm dependency list ./helm/defectdojo helm dependency update ./helm/defectdojo @@ -172,7 +184,7 @@ jobs: ${{ steps.set.outputs[matrix.databases] }} \ ${{ steps.set.outputs[matrix.brokers] }} \ --set createSecret=true \ - # --set imagePullSecrets=defectdojoregistrykey + # --set imagePullSecrets=defectdojoregistrykey - name: Check deployment status run: |- diff --git a/.github/workflows/new-release-chart.yml b/.github/workflows/new-release-chart.yml index 10ef828715b..b5f05688f48 100644 --- a/.github/workflows/new-release-chart.yml +++ b/.github/workflows/new-release-chart.yml @@ -35,7 +35,6 @@ jobs: version: v3.4.0 - name: Configure HELM repos run: |- - helm repo add stable https://charts.helm.sh/stable helm repo add bitnami https://charts.bitnami.com/bitnami helm dependency list ./helm/defectdojo helm dependency update ./helm/defectdojo diff --git a/.github/workflows/new-release-tag-docker.yml b/.github/workflows/new-release-tag-docker.yml index fada2a564dd..0ff96d5ae2c 100644 --- a/.github/workflows/new-release-tag-docker.yml +++ b/.github/workflows/new-release-tag-docker.yml @@ -36,7 +36,6 @@ jobs: version: v3.4.0 - name: Configure Helm repos run: | - helm repo add stable https://charts.helm.sh/stable helm repo add bitnami https://charts.bitnami.com/bitnami helm dependency list ./helm/defectdojo helm dependency update ./helm/defectdojo diff --git a/.github/workflows/test-helm-chart.yml b/.github/workflows/test-helm-chart.yml index d83d336ec47..8c28ae43f18 100644 --- a/.github/workflows/test-helm-chart.yml +++ b/.github/workflows/test-helm-chart.yml @@ -30,7 +30,6 @@ jobs: - name: Configure Helm repos run: |- - helm repo add stable https://charts.helm.sh/stable helm repo add bitnami https://charts.bitnami.com/bitnami helm dependency list ./helm/defectdojo helm dependency update ./helm/defectdojo diff --git a/docs/content/en/getting_started/upgrading.md b/docs/content/en/getting_started/upgrading.md index de4a88d5e2f..12aa8e34fed 100644 --- a/docs/content/en/getting_started/upgrading.md +++ b/docs/content/en/getting_started/upgrading.md @@ -84,7 +84,9 @@ notifications but also for sending the reset password emails. It is highly recom value if you are satisfied. If you installed DefectDojo earlier, you can expect `"from@example.com"` there. A fresh installation will use `"no-reply@example.com"` +This release [updates](https://github.com/DefectDojo/django-DefectDojo/pull/5450) our helm dependencies. There is a breaking change if you are using the mysql database from the helm chart because we replaced the deprecated chart from the stable repo with a chart from bitnami. If you have persistance enabled, ensure to backup your data before upgrading. All data get lost when replacing the mysql chart during the upgrade. For data migration take a look at the mysql backup and restore process. +Furthermore we updated our kubernetes version. Current tests run on 1.18.16 and 1.22.0. ## Upgrading to DefectDojo Version 2.4.x. (Security Release) diff --git a/helm/defectdojo/requirements.lock b/helm/defectdojo/requirements.lock index 979d0076fd1..1afabfae3e9 100644 --- a/helm/defectdojo/requirements.lock +++ b/helm/defectdojo/requirements.lock @@ -1,15 +1,15 @@ dependencies: - name: mysql - repository: https://charts.helm.sh/stable - version: 1.6.9 + repository: https://charts.bitnami.com/bitnami + version: 8.8.12 - name: postgresql repository: https://charts.bitnami.com/bitnami - version: 10.3.5 + version: 10.13.4 - name: rabbitmq repository: https://charts.bitnami.com/bitnami - version: 8.7.0 + version: 8.24.2 - name: redis repository: https://charts.bitnami.com/bitnami - version: 12.0.0 -digest: sha256:971488f759f2dc91f38b92a281bca1ecec1cbfa51870655cfd20667c6078e201 -generated: "2021-02-22T20:25:51.156231357+01:00" + version: 15.5.4 +digest: sha256:703b13af68b77d817409b3bb4f3c6427cd96b6000b46b21c15125d21fa47a8ac +generated: "2021-11-15T12:49:03.596417+01:00" diff --git a/helm/defectdojo/requirements.yaml b/helm/defectdojo/requirements.yaml index e2cfecd2730..aa26e429f28 100644 --- a/helm/defectdojo/requirements.yaml +++ b/helm/defectdojo/requirements.yaml @@ -1,17 +1,17 @@ dependencies: - name: mysql - version: 1.6.9 - repository: "@stable" + version: 8.8.12 + repository: "@bitnami" condition: mysql.enabled - name: postgresql - version: 10.3.5 + version: 10.13.4 repository: "@bitnami" condition: postgresql.enabled - name: rabbitmq - version: 8.7.0 + version: 8.24.2 repository: "@bitnami" condition: rabbitmq.enabled - name: redis - version: 12.0.0 + version: 15.5.4 repository: "@bitnami" condition: redis.enabled diff --git a/helm/defectdojo/templates/celery-beat-deployment.yaml b/helm/defectdojo/templates/celery-beat-deployment.yaml index b5db6e1c478..df4b6c61132 100644 --- a/helm/defectdojo/templates/celery-beat-deployment.yaml +++ b/helm/defectdojo/templates/celery-beat-deployment.yaml @@ -65,7 +65,7 @@ spec: {{- if eq .Values.database "postgresql" }} - "-instances={{ .Values.cloudsql.instance }}=tcp:{{ .Values.postgresql.service.port }}" {{- else if eq .Values.database "mysql" }} - - "-instances={{ .Values.cloudsql.instance }}=tcp:{{ .Values.mysql.service.port }}" + - "-instances={{ .Values.cloudsql.instance }}=tcp:{{ .Values.mysql.master.service.port }}" {{- end }} {{- if .Values.cloudsql.enable_iam_login }} - "-enable_iam_login" @@ -111,8 +111,8 @@ spec: name: {{ .Values.postgresql.existingSecret }} key: {{ .Values.postgresql.secretKey }} {{- else if eq .Values.database "mysql" }} - name: {{ .Values.mysql.existingSecret }} - key: {{ .Values.mysql.secretKey }} + name: {{ .Values.mysql.auth.existingSecret }} + key: {{ .Values.mysql.auth.secretKey }} {{- end }} - name: DD_SECRET_KEY valueFrom: diff --git a/helm/defectdojo/templates/celery-worker-deployment.yaml b/helm/defectdojo/templates/celery-worker-deployment.yaml index bab6aaba244..b1794cf8496 100644 --- a/helm/defectdojo/templates/celery-worker-deployment.yaml +++ b/helm/defectdojo/templates/celery-worker-deployment.yaml @@ -63,7 +63,7 @@ spec: {{- if eq .Values.database "postgresql" }} - "-instances={{ .Values.cloudsql.instance }}=tcp:{{ .Values.postgresql.service.port }}" {{- else if eq .Values.database "mysql" }} - - "-instances={{ .Values.cloudsql.instance }}=tcp:{{ .Values.mysql.service.port }}" + - "-instances={{ .Values.cloudsql.instance }}=tcp:{{ .Values.mysql.master.service.port }}" {{- end }} {{- if .Values.cloudsql.enable_iam_login }} - "-enable_iam_login" @@ -106,8 +106,8 @@ spec: name: {{ .Values.postgresql.existingSecret }} key: {{ .Values.postgresql.secretKey }} {{- else if eq .Values.database "mysql" }} - name: {{ .Values.mysql.existingSecret }} - key: {{ .Values.mysql.secretKey }} + name: {{ .Values.mysql.auth.existingSecret }} + key: {{ .Values.mysql.auth.secretKey }} {{- end }} - name: DD_SECRET_KEY valueFrom: diff --git a/helm/defectdojo/templates/configmap.yaml b/helm/defectdojo/templates/configmap.yaml index 75e5c24d73f..aed0869b50e 100644 --- a/helm/defectdojo/templates/configmap.yaml +++ b/helm/defectdojo/templates/configmap.yaml @@ -27,9 +27,9 @@ data: DD_CELERY_WORKER_PREFETCH_MULTIPLIER: '{{ if eq .Values.celery.worker.app_settings.pool_type "prefork" }}128{{ end }}' DD_DATABASE_ENGINE: django.db.backends.{{ if eq .Values.database "postgresql" }}postgresql{{ end }}{{ if eq .Values.database "mysql" }}mysql{{ end }} DD_DATABASE_HOST: {{ if eq .Values.database "postgresql" }}{{ template "postgresql.hostname" . }}{{ end }}{{ if eq .Values.database "mysql" }}{{ template "mysql.hostname" . }}{{ end }} - DD_DATABASE_PORT: '{{ if eq .Values.database "postgresql" }}{{ .Values.postgresql.service.port }}{{ end }}{{ if eq .Values.database "mysql" }}{{ .Values.mysql.service.port }}{{ end }}' - DD_DATABASE_USER: {{ if eq .Values.database "postgresql" }}{{ .Values.postgresql.postgresqlUsername }}{{ end }}{{ if eq .Values.database "mysql" }}{{ .Values.mysql.mysqlUser }}{{ end }} - DD_DATABASE_NAME: {{ if eq .Values.database "postgresql" }}{{ .Values.postgresql.postgresqlDatabase }}{{ end }}{{ if eq .Values.database "mysql" }}{{ .Values.mysql.mysqlDatabase }}{{ end }} + DD_DATABASE_PORT: '{{ if eq .Values.database "postgresql" }}{{ .Values.postgresql.service.port }}{{ end }}{{ if eq .Values.database "mysql" }}{{ .Values.mysql.master.service.port }}{{ end }}' + DD_DATABASE_USER: {{ if eq .Values.database "postgresql" }}{{ .Values.postgresql.postgresqlUsername }}{{ end }}{{ if eq .Values.database "mysql" }}{{ .Values.mysql.auth.username }}{{ end }} + DD_DATABASE_NAME: {{ if eq .Values.database "postgresql" }}{{ .Values.postgresql.postgresqlDatabase }}{{ end }}{{ if eq .Values.database "mysql" }}{{ .Values.mysql.auth.database }}{{ end }} DD_INITIALIZE: '{{ .Values.initializer.run }}' DD_UWSGI_ENDPOINT: /run/defectdojo/uwsgi.sock DD_UWSGI_HOST: localhost diff --git a/helm/defectdojo/templates/django-deployment.yaml b/helm/defectdojo/templates/django-deployment.yaml index d7fcd28288f..89b636c758d 100644 --- a/helm/defectdojo/templates/django-deployment.yaml +++ b/helm/defectdojo/templates/django-deployment.yaml @@ -89,7 +89,7 @@ spec: {{- if eq .Values.database "postgresql" }} - "-instances={{ .Values.cloudsql.instance }}=tcp:{{ .Values.postgresql.service.port }}" {{- else if eq .Values.database "mysql" }} - - "-instances={{ .Values.cloudsql.instance }}=tcp:{{ .Values.mysql.service.port }}" + - "-instances={{ .Values.cloudsql.instance }}=tcp:{{ .Values.mysql.master.service.port }}" {{- end }} {{- if .Values.cloudsql.enable_iam_login }} - "-enable_iam_login" @@ -169,8 +169,8 @@ spec: name: {{ .Values.postgresql.existingSecret }} key: {{ .Values.postgresql.secretKey }} {{- else if eq .Values.database "mysql" }} - name: {{ .Values.mysql.existingSecret }} - key: {{ .Values.mysql.secretKey }} + name: {{ .Values.mysql.auth.existingSecret }} + key: {{ .Values.mysql.auth.secretKey }} {{- end }} - name: DD_SECRET_KEY valueFrom: diff --git a/helm/defectdojo/templates/initializer-job.yaml b/helm/defectdojo/templates/initializer-job.yaml index 8dd30d1d7ef..457491efedf 100644 --- a/helm/defectdojo/templates/initializer-job.yaml +++ b/helm/defectdojo/templates/initializer-job.yaml @@ -36,7 +36,7 @@ spec: {{- if eq .Values.database "postgresql" }} - "-instances={{ .Values.cloudsql.instance }}=tcp:{{ .Values.postgresql.service.port }}" {{- else if eq .Values.database "mysql" }} - - "-instances={{ .Values.cloudsql.instance }}=tcp:{{ .Values.mysql.service.port }}" + - "-instances={{ .Values.cloudsql.instance }}=tcp:{{ .Values.mysql.master.service.port }}" {{- end }} {{- if .Values.cloudsql.enable_iam_login }} - "-enable_iam_login" @@ -67,8 +67,8 @@ spec: name: {{ .Values.postgresql.existingSecret }} key: {{ .Values.postgresql.secretKey }} {{- else if eq .Values.database "mysql" }} - name: {{ .Values.mysql.existingSecret }} - key: {{ .Values.mysql.secretKey }} + name: {{ .Values.mysql.auth.existingSecret }} + key: {{ .Values.mysql.auth.secretKey }} {{- end }} resources: {{- toYaml .Values.initializer.resources | nindent 10 }} diff --git a/helm/defectdojo/templates/secret-mysql.yaml b/helm/defectdojo/templates/secret-mysql.yaml index 53e14c65364..24aee10c9b8 100644 --- a/helm/defectdojo/templates/secret-mysql.yaml +++ b/helm/defectdojo/templates/secret-mysql.yaml @@ -2,7 +2,7 @@ apiVersion: v1 kind: Secret metadata: - name: {{ .Values.mysql.existingSecret }} + name: {{ .Values.mysql.auth.existingSecret }} labels: app.kubernetes.io/name: {{ include "defectdojo.name" . }} app.kubernetes.io/instance: {{ .Release.Name }} @@ -15,17 +15,18 @@ metadata: type: Opaque data: {{- if .Values.mysql.enabled }} -{{- if .Values.mysql.mysqlRootPassword }} - mysql-root-password: {{ .Values.mysql.mysqlRootPassword | b64enc | quote }} +{{- if .Values.mysql.auth.rootPassword }} + mysql-root-password: {{ .Values.mysql.auth.rootPassword | b64enc | quote }} {{- else }} mysql-root-password: {{ randAlphaNum 10 | b64enc | quote }} {{- end }} -{{- if .Values.mysql.mysqlPassword }} - {{ .Values.mysql.secretKey }}: {{ .Values.mysql.mysqlPassword | b64enc | quote }} +{{- if .Values.mysql.auth.password }} + {{ .Values.mysql.auth.secretKey }}: {{ .Values.mysql.auth.password | b64enc | quote }} {{- else }} - {{ .Values.mysql.secretKey }}: {{ randAlphaNum 10 | b64enc | quote }} + {{ .Values.mysql.auth.secretKey }}: {{ randAlphaNum 10 | b64enc | quote }} {{- end}} + mysql-replication-password: {{ randAlphaNum 10 | b64enc | quote }} {{- else }} - {{ .Values.mysql.secretKey }}: {{ .Values.mysql.mysqlPassword | b64enc | quote }} + {{ .Values.mysql.auth.secretKey }}: {{ .Values.mysql.auth.password | b64enc | quote }} {{- end }} {{- end }} diff --git a/helm/defectdojo/templates/tests/unit-tests.yaml b/helm/defectdojo/templates/tests/unit-tests.yaml index 8e6d8559bc6..ebac25aed05 100644 --- a/helm/defectdojo/templates/tests/unit-tests.yaml +++ b/helm/defectdojo/templates/tests/unit-tests.yaml @@ -46,8 +46,8 @@ spec: name: {{ .Values.postgresql.existingSecret }} key: {{ .Values.postgresql.secretKey }} {{- else if eq .Values.database "mysql" }} - name: {{ .Values.mysql.existingSecret }} - key: {{ .Values.mysql.secretKey }} + name: {{ .Values.mysql.auth.existingSecret }} + key: {{ .Values.mysql.auth.secretKey }} {{- end }} key: {{ if eq .Values.database "postgresql" }}{{ .Values.database }}-password{{ end }}{{ if eq .Values.database "mysql" }}{{ .Values.database }}-root-password{{ end }} - name: DD_DEBUG diff --git a/helm/defectdojo/values.yaml b/helm/defectdojo/values.yaml index e9eb1556559..a3f09d04b2c 100644 --- a/helm/defectdojo/values.yaml +++ b/helm/defectdojo/values.yaml @@ -260,16 +260,19 @@ initializer: mysql: enabled: false - mysqlUser: defectdojo - mysqlPassword: "" - mysqlRootPassword: "" - existingSecret: defectdojo-mysql-specific - secretKey: mysql-password - mysqlDatabase: defectdojo - service: + auth: + username: defectdojo + password: "" + rootPassword: "" + database: defectdojo + existingSecret: defectdojo-mysql-specific + secretKey: mysql-password + master: + service: + port: 3306 # To use an external mySQL instance, set enabled to false and uncomment - # the line below: - mysqlServer: "127.0.0.1" + # the line below / add external address: + # mysqlServer: "127.0.0.1" postgresql: enabled: true diff --git a/readme-docs/KUBERNETES.md b/readme-docs/KUBERNETES.md index c58ba7bfc08..39c894464b7 100644 --- a/readme-docs/KUBERNETES.md +++ b/readme-docs/KUBERNETES.md @@ -9,6 +9,9 @@ For development purposes, and [Helm](https://helm.sh/) can be installed locally by following this [guide](https://helm.sh/docs/using_helm/#installing-helm). +## Supported Kubernetes Versions +The tests cover the deployment on the lastest [kubernetes version](https://kubernetes.io/releases/) and the oldest supported [version from AWS](https://docs.aws.amazon.com/eks/latest/userguide/kubernetes-versions.html#available-versions). The assumption is that version in between do not have significant differences. Current tested versions can looks up in the [github k8s workflow](https://github.com/DefectDojo/django-DefectDojo/blob/master/.github/workflows/k8s-testing.yml). + ## Helm chart Starting with version 1.14.0, a helm chart will be pushed onto the `helm-charts` branch during the release process. Don't look for a chart museum, we're leveraging the "raw" capabilities of GitHub at this time. @@ -52,7 +55,6 @@ helm repo update Helm >= v3 ```zsh -helm repo add stable https://charts.helm.sh/stable helm repo add bitnami https://charts.bitnami.com/bitnami helm repo update ``` @@ -399,5 +401,6 @@ helm uninstall defectdojo To remove persistent objects not removed by uninstall (this will remove any database): ``` kubectl delete secrets defectdojo defectdojo-redis-specific defectdojo-rabbitmq-specific defectdojo-postgresql-specific defectdojo-mysql-specific -kubectl delete pvc data-defectdojo-rabbitmq-0 data-defectdojo-postgresql-0 +kubectl delete serviceAccount defectdojo +kubectl delete pvc data-defectdojo-rabbitmq-0 data-defectdojo-postgresql-0 data-defectdojo-mysql-0 ```