Asset syncer and Assetsvc (#1323)

vmware-tanzu · Jan 7, 2020 · 1ef9929 · 1ef9929
1 parent 37db2fa
commit 1ef9929
Show file tree

Hide file tree

Showing 64 changed files with 5,446 additions and 316 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -35,7 +35,14 @@ workflows:
           <<: *build_always
       - build_dashboard:
           <<: *build_always
-      - local_e2e_tests:
+      - local_e2e_tests_mongodb:
+          <<: *build_always
+          requires:
+            - test_go
+            - test_dashboard
+            - build_go_images
+            - build_dashboard
+      - local_e2e_tests_postgresql:
           <<: *build_always
           requires:
             - test_go
@@ -73,20 +80,26 @@ workflows:
       - sync_chart:
           <<: *build_on_master
           requires:
+            - local_e2e_tests_postgresql
+            - local_e2e_tests_mongodb
             - GKE_1_13_MASTER
             - GKE_1_13_LATEST_RELEASE
             - GKE_1_12_MASTER
             - GKE_1_12_LATEST_RELEASE
       - push_images:
-          <<: *build_always
+          <<: *build_on_master
           requires:
+            - local_e2e_tests_postgresql
+            - local_e2e_tests_mongodb
             - GKE_1_13_MASTER
             - GKE_1_13_LATEST_RELEASE
             - GKE_1_12_MASTER
             - GKE_1_12_LATEST_RELEASE
       - release:
           <<: *build_on_tag
           requires:
+            - local_e2e_tests_postgresql
+            - local_e2e_tests_mongodb
             - GKE_1_13_MASTER
             - GKE_1_13_LATEST_RELEASE
             - GKE_1_12_MASTER
@@ -207,6 +220,27 @@ gke_test: &gke_test
         name: Cleanup GKE Cluster
         command: gcloud container clusters delete --async --zone $GKE_ZONE $ESCAPED_GKE_CLUSTER
         when: always
+local_e2e_steps: &local_e2e_steps
+  steps:
+    - checkout
+    - <<: *exports
+    # Install kind
+    - run: |
+        curl -LO https://github.com/kubernetes-sigs/kind/releases/download/0.2.1/kind-linux-amd64
+        chmod +x kind-linux-amd64
+        sudo mv kind-linux-amd64 /usr/local/bin/kind
+    - run: kind create cluster
+    - run: echo "export KUBECONFIG=$(kind get kubeconfig-path --name=kind)" >> $BASH_ENV
+    - <<: *install_kubectl
+    - <<: *install_helm_cli
+    # Load images from other jobs
+    - attach_workspace:
+        at: /tmp/images
+    - run: for image in /tmp/images/*; do kind load image-archive "$image"; done
+    - <<: *run_e2e_tests
+    - store_artifacts:
+        path: integration/reports
+    - run: exit $TEST_RESULT
 ###
 
 jobs:
@@ -243,7 +277,7 @@ jobs:
     working_directory: /go/src/github.com/kubeapps/kubeapps
     environment:
       GOPATH: /home/circleci/.go_workspace
-      IMAGES: "apprepository-controller tiller-proxy"
+      IMAGES: "apprepository-controller tiller-proxy asset-syncer assetsvc"
     <<: *build_images
   build_dashboard:
     docker:
@@ -257,28 +291,16 @@ jobs:
     steps:
       - checkout
       - run: REPO_DOMAIN=kubeapps REPO_NAME=kubeapps ./script/create_release.sh ${CIRCLE_TAG}
-  local_e2e_tests:
+  local_e2e_tests_mongodb:
     machine: true
-    steps:
-      - checkout
-      - <<: *exports
-      # Install kind
-      - run: |
-          curl -LO https://github.com/kubernetes-sigs/kind/releases/download/0.2.1/kind-linux-amd64
-          chmod +x kind-linux-amd64
-          sudo mv kind-linux-amd64 /usr/local/bin/kind
-      - run: kind create cluster
-      - run: echo "export KUBECONFIG=$(kind get kubeconfig-path --name=kind)" >> $BASH_ENV
-      - <<: *install_kubectl
-      - <<: *install_helm_cli
-      # Load images from other jobs
-      - attach_workspace:
-          at: /tmp/images
-      - run: for image in /tmp/images/*; do kind load image-archive "$image"; done
-      - <<: *run_e2e_tests
-      - store_artifacts:
-          path: integration/reports
-      - run: exit $TEST_RESULT
+    environment:
+      KUBEAPPS_DB: "mongodb"
+    <<: *local_e2e_steps
+  local_e2e_tests_postgresql:
+    machine: true
+    environment:
+      KUBEAPPS_DB: "postgresql"
+    <<: *local_e2e_steps
   GKE_1_13_MASTER:
     <<: *gke_test
     environment:

diff --git a/chart/kubeapps/README.md b/chart/kubeapps/README.md
@@ -57,11 +57,11 @@ Specify each parameter using the `--set key=value[,key=value]` argument to `helm
 
 ```console
 $ helm install --name kubeapps --namespace kubeapps \
-  --set chartsvc.service.port=9090 \
+  --set assetsvc.service.port=9090 \
     bitnami/kubeapps
 ```
 
-The above command sets the port for the chartsvc Service to 9090.
+The above command sets the port for the assetsvc Service to 9090.
 
 Alternatively, a YAML file that specifies the values for parameters can be provided while installing the chart. For example,
 

diff --git a/chart/kubeapps/requirements.lock b/chart/kubeapps/requirements.lock
@@ -1,6 +1,9 @@
 dependencies:
-  - name: mongodb
-    repository: https://kubernetes-charts.storage.googleapis.com
-    version: 7.2.9
-digest: sha256:415440e73af7d4b02a10a15f28bb2fc095cbdffdc2e1676d76e0f0eaa1632d50
-generated: 2019-09-24T07:04:11.615576228Z
+- name: mongodb
+  repository: https://kubernetes-charts.storage.googleapis.com
+  version: 7.6.1
+- name: postgresql
+  repository: https://kubernetes-charts.storage.googleapis.com
+  version: 8.1.0
+digest: sha256:50d16adab428d7a83c2c394ddd532cff6edb208bd2523e4dd837f950162a7c02
+generated: "2019-12-18T13:23:55.241155459+01:00"
diff --git a/chart/kubeapps/requirements.yaml b/chart/kubeapps/requirements.yaml
@@ -2,3 +2,8 @@ dependencies:
   - name: mongodb
     version: ">= 0"
     repository: https://kubernetes-charts.storage.googleapis.com
+    condition: mongodb.enabled
+  - name: postgresql
+    version: ">= 0"
+    repository: https://kubernetes-charts.storage.googleapis.com
+    condition: postgresql.enabled
diff --git a/chart/kubeapps/templates/_helpers.tpl b/chart/kubeapps/templates/_helpers.tpl
@@ -80,6 +80,16 @@ We truncate at 63 chars because some Kubernetes name fields are limited to this
 {{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
 {{- end -}}
 
+
+{{/*
+Create a default fully qualified app name for PostgreSQL dependency.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+*/}}
+{{- define "kubeapps.postgresql.fullname" -}}
+{{- $name := default "postgresql" .Values.postgresql.nameOverride -}}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
 {{/*
 Create name for the apprepository-controller based on the fullname
 */}}
@@ -102,10 +112,10 @@ Create name for the apprepository cleanup job
 {{- end -}}
 
 {{/*
-Create name for the mongodb secret bootstrap job
+Create name for the db-secret secret bootstrap job
 */}}
-{{- define "kubeapps.mongodb-jobs-cleanup.fullname" -}}
-{{ template "kubeapps.fullname" . }}-internal-mongodb-jobs-cleanup
+{{- define "kubeapps.db-secret-jobs-cleanup.fullname" -}}
+{{ template "kubeapps.fullname" . }}-internal-db-secret-jobs-cleanup
 {{- end -}}
 
 {{/*
@@ -116,10 +126,10 @@ Create name for the kubeapps upgrade job
 {{- end -}}
 
 {{/*
-Create name for the chartsvc based on the fullname
+Create name for the assetsvc based on the fullname
 */}}
-{{- define "kubeapps.chartsvc.fullname" -}}
-{{ template "kubeapps.fullname" . }}-internal-chartsvc
+{{- define "kubeapps.assetsvc.fullname" -}}
+{{ template "kubeapps.fullname" . }}-internal-assetsvc
 {{- end -}}
 
 {{/*

diff --git a/chart/kubeapps/templates/apprepository-deployment.yaml b/chart/kubeapps/templates/apprepository-deployment.yaml
@@ -44,9 +44,24 @@ spec:
             - --logtostderr
             - --user-agent-comment=kubeapps/{{ .Chart.AppVersion }}
             - --repo-sync-image={{ template "kubeapps.image" (list .Values.apprepository.syncImage .Values.global) }}
+            - --repo-sync-cmd=/asset-syncer
             - --namespace={{ .Release.Namespace }}
-            - --mongo-url={{ template "kubeapps.mongodb.fullname" . }}
-            - --mongo-secret-name={{ .Values.mongodb.existingSecret }}
+            {{- if .Values.mongodb.enabled }}
+            - --database-secret-name={{ .Values.mongodb.existingSecret }}
+            - --database-secret-key=mongodb-root-password
+            - --database-type=mongodb
+            - --database-url={{ template "kubeapps.mongodb.fullname" . }}
+            - --database-user=root
+            - --database-name=charts
+            {{- end }}
+            {{- if .Values.postgresql.enabled }}
+            - --database-secret-name={{ .Values.postgresql.existingSecret }}
+            - --database-secret-key=postgresql-password
+            - --database-type=postgresql
+            - --database-url={{ template "kubeapps.postgresql.fullname" . }}:5432
+            - --database-user=postgres
+            - --database-name=assets
+            {{- end }}
             {{- if .Values.apprepository.crontab }}
             - --crontab={{ .Values.apprepository.crontab }}
             {{- end }}

diff --git a/chart/kubeapps/templates/assetsvc-deployment.yaml b/chart/kubeapps/templates/assetsvc-deployment.yaml
@@ -0,0 +1,79 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ template "kubeapps.assetsvc.fullname" . }}
+  labels:
+    app: {{ template "kubeapps.assetsvc.fullname" . }}
+    chart: {{ template "kubeapps.chart" . }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+spec:
+  replicas: {{ .Values.assetsvc.replicaCount }}
+  selector:
+    matchLabels:
+      app: {{ template "kubeapps.assetsvc.fullname" . }}
+      release: {{ .Release.Name }}
+  template:
+    metadata:
+      labels:
+        app: {{ template "kubeapps.assetsvc.fullname" . }}
+        release: {{ .Release.Name }}
+    spec:
+{{- include "kubeapps.imagePullSecrets" . | indent 6 }}
+      {{- if .Values.assetsvc.affinity }}
+      affinity: {{- include "kubeapps.tplValue" (dict "value" .Values.assetsvc.affinity "context" $) | nindent 8 }}
+      {{- end }}
+      {{- if .Values.assetsvc.nodeSelector }}
+      nodeSelector: {{- include "kubeapps.tplValue" (dict "value" .Values.assetsvc.nodeSelector "context" $) | nindent 8 }}
+      {{- end }}
+      {{- if .Values.assetsvc.tolerations }}
+      tolerations: {{- include "kubeapps.tplValue" (dict "value" .Values.assetsvc.tolerations "context" $) | nindent 8 }}
+      {{- end }}
+      {{- if .Values.securityContext.enabled }}
+      securityContext:
+        fsGroup: {{ .Values.securityContext.fsGroup }}
+        runAsUser: {{ .Values.securityContext.runAsUser }}
+      {{- end }}
+      containers:
+        - name: assetsvc
+          image: {{ template "kubeapps.image" (list .Values.assetsvc.image .Values.global) }}
+          command:
+            - /assetsvc
+          {{- if .Values.mongodb.enabled }}
+          args:
+            - --database-type=mongodb
+            - --database-user=root
+            - --database-name=charts
+            - --database-url={{ template "kubeapps.mongodb.fullname" . }}
+          env:
+            - name: DB_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: {{ .Values.mongodb.existingSecret }}
+                  key: mongodb-root-password
+          {{- end }}
+          {{- if .Values.postgresql.enabled }}
+          args:
+            - --database-type=postgresql
+            - --database-user=postgres
+            - --database-name=assets
+            - --database-url={{ template "kubeapps.postgresql.fullname" . }}:5432
+          env:
+            - name: DB_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: {{ .Values.postgresql.existingSecret }}
+                  key: postgresql-password
+          {{- end }}
+          ports:
+            - name: http
+              containerPort: {{ .Values.assetsvc.service.port }}
+          {{- if .Values.assetsvc.livenessProbe }}
+          livenessProbe: {{- toYaml .Values.assetsvc.livenessProbe | nindent 12 }}
+          {{- end }}
+          {{- if .Values.assetsvc.readinessProbe }}
+          readinessProbe: {{- toYaml .Values.assetsvc.readinessProbe | nindent 12 }}
+          {{- end }}
+          {{- if .Values.assetsvc.resource }}
+          resources: {{- toYaml .Values.assetsvc.resources | nindent 12 }}
+          {{- end }}
diff --git a/.../kubeapps/templates/chartsvc-service.yaml → .../kubeapps/templates/assetsvc-service.yaml b/.../kubeapps/templates/chartsvc-service.yaml → .../kubeapps/templates/assetsvc-service.yaml
@@ -1,7 +1,7 @@
 apiVersion: v1
 kind: Service
 metadata:
-  name: {{ template "kubeapps.chartsvc.fullname" . }}
+  name: {{ template "kubeapps.assetsvc.fullname" . }}
   labels:
     app: {{ template "kubeapps.name" . }}
     chart: {{ template "kubeapps.chart" . }}
@@ -10,10 +10,10 @@ metadata:
 spec:
   type: ClusterIP
   ports:
-    - port: {{ .Values.chartsvc.service.port }}
+    - port: {{ .Values.assetsvc.service.port }}
       targetPort: http
       protocol: TCP
       name: http
   selector:
-    app: {{ template "kubeapps.chartsvc.fullname" . }}
+    app: {{ template "kubeapps.assetsvc.fullname" . }}
     release: {{ .Release.Name }}
diff --git a/chart/kubeapps/templates/chartsvc-deployment.yaml b/chart/kubeapps/templates/chartsvc-deployment.yaml