From e02a1a715f87b8e6cf6c8551d4d1baabce70b3e4 Mon Sep 17 00:00:00 2001 From: se-wo Date: Sat, 12 Mar 2022 13:56:04 +0100 Subject: [PATCH 1/4] Fix frontend helm labels taken. Previously those were erronously taken from backend. --- helm-chart/templates/cvat_frontend/deployment.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/helm-chart/templates/cvat_frontend/deployment.yml b/helm-chart/templates/cvat_frontend/deployment.yml index 9274469e54f2..a7b5648a3b08 100644 --- a/helm-chart/templates/cvat_frontend/deployment.yml +++ b/helm-chart/templates/cvat_frontend/deployment.yml @@ -22,10 +22,10 @@ spec: {{- include "cvat.labels" . | nindent 8 }} app: cvat-app tier: frontend - {{- with .Values.cvat.backend.labels }} + {{- with .Values.cvat.frontend.labels }} {{- toYaml . | nindent 8 }} {{- end }} - {{- with .Values.cvat.backend.annotations }} + {{- with .Values.cvat.frontend.annotations }} annotations: {{- toYaml . | nindent 8 }} {{- end }} @@ -62,4 +62,4 @@ spec: {{- with .Values.imagePullSecrets }} imagePullSecrets: {{- toYaml . | nindent 8 }} - {{- end }} + {{- end }} \ No newline at end of file From 687268990e0321899c93fa3b3052a4499e89b97b Mon Sep 17 00:00:00 2001 From: se-wo Date: Sat, 12 Mar 2022 13:56:25 +0100 Subject: [PATCH 2/4] Add support for CVAT 2.X in helm charts Added templates for OPA and extended readme on how to deploy. Moreover modernized the ingress chart to support k8s version 1.22+. --- helm-chart/Chart.yaml | 2 +- helm-chart/README.md | 56 +++++++++---- helm-chart/templates/cvat_opa/config.yml | 17 ++++ helm-chart/templates/cvat_opa/deployment.yml | 82 ++++++++++++++++++++ helm-chart/templates/cvat_opa/service.yml | 21 +++++ helm-chart/templates/ingress.yaml | 16 +++- helm-chart/values.yaml | 45 +++++++++++ 7 files changed, 224 insertions(+), 15 deletions(-) create mode 100644 helm-chart/templates/cvat_opa/config.yml create mode 100644 helm-chart/templates/cvat_opa/deployment.yml create mode 100644 helm-chart/templates/cvat_opa/service.yml diff --git a/helm-chart/Chart.yaml b/helm-chart/Chart.yaml index 567bb5e4d0b8..381331e669f9 100644 --- a/helm-chart/Chart.yaml +++ b/helm-chart/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.2.0 +version: 0.3.0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/helm-chart/README.md b/helm-chart/README.md index b135b0584483..d310dac331e7 100644 --- a/helm-chart/README.md +++ b/helm-chart/README.md @@ -1,58 +1,88 @@ # FAQ + ## What should be configured before installation? + 1. You should have configured connection to existed k8s cluster 2. Helm must be installed 3. You should download chart external dependencies, using following commands: -``` - helm repo add bitnami https://charts.bitnami.com/bitnami - helm repo update - helm dependency update -``` + + ```shell + helm repo add bitnami + helm repo update + helm dependency update + ``` + 4. (Optional) Install ingress of your choice (for example: ) 5. (Optional) Create certificates for https (for example: ) 6. (Optional) Create values.override.yaml and override there parameters you want 7. Change postgresql password as described below -8. Add ingress to values.override.yaml(example also below) -7. Deploy cvat using command below +8. Create a rules.tar.gz archive containing at least all rego files in /cvat/apps/iam/rules not containing test in their name +9. Add ingress to values.override.yaml(example also below) +10. Deploy cvat using command below + +> **Warning:** The k8s service name of Open Policy Agent is fixed to opa by default. This is done to be compatible with CVAT 2.0 but limits this helm chart to a single release. The OPA url currently can´t be set as an environment variable. As soon as this is possible you can set cvat.opa.composeCompatibleServiceName to false in your value.override.yaml and configure those as additional envs. + ## How to deploy new version of chart to cluster? + Execute following command: ```helm upgrade --install ./helm-chart -f ./helm-chart/values.yaml -f values.override.yaml(if exists) --namespace ``` + ## How to create superuser? -``` + +```shell HELM_RELEASE_NAMESPACE="" &&\ HELM_RELEASE_NAME="" &&\ BACKEND_POD_NAME=$(kubectl get pod --namespace $HELM_RELEASE_NAMESPACE -l tier=backend,app.kubernetes.io/instance=$HELM_RELEASE_NAME -o jsonpath='{.items[0].metadata.name}') &&\ kubectl exec -it --namespace $HELM_RELEASE_NAMESPACE $BACKEND_POD_NAME -c cvat-backend-app-container -- python manage.py createsuperuser ``` + ## How to change embedded postgresql password? + There are several passwords used here, for security reasons - better change them all. -``` + +```yaml postgresql: secret: password: cvat_postgresql postgres_password: cvat_postgresql_postgres replication_password: cvat_postgresql_replica ``` + Or, if you know how to work with k8s - you could create your own secret and use it here: -``` + +```yaml postgresql: global: postgresql: existingSecret: cvat-postgres-secret ``` -## How to describe ingress: + +## How to describe ingress + Just set `ingress.enabled:` to `true`, then copy example, uncomment it and change values there + ## How to understand what diff will be inflicted by 'helm upgrade'? + You can use for that -## I want to use my own postgresql/redis with your chart. + +## I want to use my own postgresql/redis with your chart + Just set `postgresql.enabled` or `redis.enabled` to `false`, as described below. Then - put your instance params to "external" field -## I want to override some settings in values.yaml. + +## I want to override some settings in values.yaml + Just create file `values.override.yaml` and place your changes here, using same structure as in `values.yaml`. Then reference it in helm update/install command using `-f` flag + ## Why you used external charts to provide redis and postgres? + Because they definitely know what they do better then we are, so we are getting more quality and less support + ## What is kubernetes and how it is working? + See + ## What is helm and how it is working? + See diff --git a/helm-chart/templates/cvat_opa/config.yml b/helm-chart/templates/cvat_opa/config.yml new file mode 100644 index 000000000000..e29038509ecb --- /dev/null +++ b/helm-chart/templates/cvat_opa/config.yml @@ -0,0 +1,17 @@ +{{- if .Values.cvat.opa.defaultStorage.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Release.Name }}-opa-rules + namespace: {{ .Release.Namespace }} + labels: + {{- include "cvat.labels" . | nindent 4 }} + app: cvat-app + tier: opa +{{- if semverCompare ">=1.21-0" .Capabilities.KubeVersion.GitVersion }} +immutable: true +{{- end }} +binaryData: + rules.tar.gz: |- + {{ .Files.Get "rules.tar.gz" | b64enc }} +{{- end}} \ No newline at end of file diff --git a/helm-chart/templates/cvat_opa/deployment.yml b/helm-chart/templates/cvat_opa/deployment.yml new file mode 100644 index 000000000000..e1db2b0a3d60 --- /dev/null +++ b/helm-chart/templates/cvat_opa/deployment.yml @@ -0,0 +1,82 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Release.Name }}-opa + namespace: {{ .Release.Namespace }} + labels: + {{- include "cvat.labels" . | nindent 4 }} + app: cvat-app + tier: opa +spec: + replicas: {{ .Values.cvat.opa.replicas }} + strategy: + type: Recreate + selector: + matchLabels: + {{- include "cvat.labels" . | nindent 6 }} + app: cvat-app + tier: opa + template: + metadata: + labels: + {{- include "cvat.labels" . | nindent 8 }} + app: cvat-app + tier: opa + {{- with .Values.cvat.opa.labels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.cvat.opa.annotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + containers: + - name: cvat-opa-app-container + image: {{ .Values.cvat.opa.image }}:{{ .Values.cvat.opa.tag }} + args: + - run + - --server + - --addr + - :8181 + - --set=decision_logs.console=true + - /rules/rules.tar.gz + {{- with .Values.cvat.opa.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + ports: + - containerPort: 8181 + {{- with .Values.cvat.opa.additionalEnv }} + env: + {{- toYaml . | nindent 10 }} + {{- end }} + volumeMounts: + - mountPath: /rules + name: cvat-opa-rules + {{- with .Values.cvat.opa.additionalVolumeMounts }} + {{- toYaml . | nindent 10 }} + {{- end }} + volumes: + {{- if .Values.cvat.opa.defaultStorage.enabled }} + - name: cvat-opa-rules + configMap: + name: "{{ .Release.Name }}-opa-rules" + items: + - key: "rules.tar.gz" + path: "rules.tar.gz" + {{- end }} + {{- with .Values.cvat.opa.additionalVolumes }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.cvat.opa.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.cvat.opa.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/helm-chart/templates/cvat_opa/service.yml b/helm-chart/templates/cvat_opa/service.yml new file mode 100644 index 000000000000..add6430aa667 --- /dev/null +++ b/helm-chart/templates/cvat_opa/service.yml @@ -0,0 +1,21 @@ +apiVersion: v1 +kind: Service +metadata: + {{- if .Values.cvat.opa.composeCompatibleServiceName }} + name: opa + {{- else }} + name: {{ .Release.Name }}-opa-service + {{- end }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "cvat.labels" . | nindent 4 }} + app: cvat-app + tier: opa +spec: + selector: + app: cvat-app + tier: opa + {{- include "cvat.labels" . | nindent 4 }} + {{- with .Values.cvat.opa.service }} + {{- toYaml . | nindent 2 }} + {{- end }} diff --git a/helm-chart/templates/ingress.yaml b/helm-chart/templates/ingress.yaml index cab76c1353d7..86fc4651679e 100644 --- a/helm-chart/templates/ingress.yaml +++ b/helm-chart/templates/ingress.yaml @@ -1,6 +1,9 @@ +{{- $version := .Capabilities.KubeVersion.GitVersion -}} {{- if .Values.ingress.enabled -}} {{- $fullName := include "cvat.fullname" . -}} -{{- if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} +{{- if semverCompare ">=1.19-0" $version -}} +apiVersion: networking.k8s.io/v1 +{{- else if semverCompare ">=1.14-0" $version -}} apiVersion: networking.k8s.io/v1beta1 {{- else -}} apiVersion: extensions/v1beta1 @@ -16,6 +19,9 @@ metadata: {{- toYaml . | nindent 4 }} {{- end }} spec: +{{- if semverCompare ">=1.19-0" $version }} + ingressClassName: nginx +{{- end }} {{- if .Values.ingress.tls }} tls: {{- range .Values.ingress.tls }} @@ -36,9 +42,17 @@ spec: {{- if .pathType }} pathType: {{ .pathType }} {{- end }} + {{- if semverCompare ">=1.19-0" $version }} + backend: + service: + name: {{ .service.name }} + port: + number: {{ .service.port }} + {{- else }} backend: serviceName: {{ .service.name }} servicePort: {{ .service.port }} + {{- end }} {{- end }} {{- end }} {{- end }} diff --git a/helm-chart/values.yaml b/helm-chart/values.yaml index f4d1b6137c98..cbd3a01f4675 100644 --- a/helm-chart/values.yaml +++ b/helm-chart/values.yaml @@ -100,6 +100,51 @@ cvat: targetPort: 80 protocol: TCP name: http + opa: + replicas: 1 + image: openpolicyagent/opa + tag: 0.34.2-rootless + labels: {} + # test: test + annotations: {} + # test.io/test: test + resources: {} + affinity: {} + tolerations: [] + # nodeAffinity: + # requiredDuringSchedulingIgnoredDuringExecution: + # nodeSelectorTerms: + # - matchExpressions: + # - key: kubernetes.io/e2e-az-name + # operator: In + # values: + # - e2e-az1 + # - e2e-az2 + additionalEnv: [] + # Example: + # - name: volume-from-secret + # - name: TEST + # value: "test" + additionalVolumes: [] + # Example(assumes that pvc was already created): + # - name: tmp + # persistentVolumeClaim: + # claimName: tmp + additionalVolumeMounts: [] + # Example: + # - mountPath: /tmp + # name: tmp + # subPath: test + composeCompatibleServiceName: true # Sets service name to opa in order to be docker-compose compatible. Necessary because changing IAM_OPA_DATA_URL via environment variables in current images. Hinders multiple deployment due to duplicate name + service: + type: ClusterIP + ports: + - port: 8181 + targetPort: 8181 + protocol: TCP + name: http + defaultStorage: + enabled: true postgresql: #See https://github.com/bitnami/charts/blob/master/bitnami/postgresql/ for more info From 186ecc8bf58f9c92d8d22ce28751a0a272c66121 Mon Sep 17 00:00:00 2001 From: se-wo Date: Wed, 16 Mar 2022 22:00:48 +0100 Subject: [PATCH 3/4] Update Changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d8dd16242ae7..8e02098593bd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed - Added missing geos dependency into Dockerfile () - Improved helm chart readme () +- Added helm chart support for CVAT 2.X and made ingress compatible with Kubernetes >=1.22 () ### Deprecated - TDB From 6961830d4f1f1034b86314103198079761db40b7 Mon Sep 17 00:00:00 2001 From: se-wo Date: Fri, 25 Mar 2022 22:52:22 +0100 Subject: [PATCH 4/4] Fix helm readme linter error Fix linter error due to markdown line longer than 120 characters --- helm-chart/README.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/helm-chart/README.md b/helm-chart/README.md index dd032302c376..f4052e39f447 100644 --- a/helm-chart/README.md +++ b/helm-chart/README.md @@ -103,7 +103,11 @@ ingress: ## Deployment Make sure you are using correct kubernetes context. You can check it with `kubectl config current-context`. -> **Warning:** The k8s service name of Open Policy Agent is fixed to opa by default. This is done to be compatible with CVAT 2.0 but limits this helm chart to a single release per namespace. The OPA url currently can´t be set as an environment variable. As soon as this is possible you can set cvat.opa.composeCompatibleServiceName to false in your value.override.yaml and configure the opa url as additional env. +> **Warning:** The k8s service name of Open Policy Agent is fixed to opa by default. +This is done to be compatible with CVAT 2.0 but limits this helm chart to a single release per namespace. +The OPA url currently can´t be set as an environment variable. +As soon as this is possible you can set cvat.opa.composeCompatibleServiceName +to false in your value.override.yaml and configure the opa url as additional env. Execute following command from repo root directory ### With overrides: