From fe460ce83c0dc8183ff8236d78a7cc0e6d91ed9e Mon Sep 17 00:00:00 2001 From: Jonathan Stacks Date: Fri, 17 Sep 2021 00:21:16 -0500 Subject: [PATCH] =?UTF-8?q?=F0=9F=8E=89=20Create=20a=20Helm=20Chart=20For?= =?UTF-8?q?=20Airbyte=20(#5891)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See number #1868. This creates an initial helm chart for installing Airbyte in Kubernetes to make it easier for users who are more familiar with helm. It also includes GitHub actions to help continually test that the chart works in the most basic case. All of the templates are based off of the kustomize folder, but minio and postgres have been removed in favor of adding the bitnami helm charts as dependencies since they have an active community and allow easily tweaking their install. --- .github/workflows/helm.yaml | 74 ++++ charts/airbyte/.gitignore | 2 + charts/airbyte/.helmignore | 25 ++ charts/airbyte/Chart.lock | 12 + charts/airbyte/Chart.yaml | 39 ++ charts/airbyte/README.md | 149 +++++++ charts/airbyte/ci.sh | 90 ++++ charts/airbyte/files/sweep-pod.sh | 40 ++ charts/airbyte/templates/NOTES.txt | 22 + charts/airbyte/templates/_helpers.tpl | 176 ++++++++ charts/airbyte/templates/env-configmap.yaml | 44 ++ .../templates/gcs-log-creds-secret.yaml | 7 + .../templates/pod-sweeper/configmap.yaml | 8 + .../templates/pod-sweeper/deployment.yaml | 54 +++ .../templates/scheduler/deployment.yaml | 218 ++++++++++ .../airbyte/templates/server/deployment.yaml | 196 +++++++++ charts/airbyte/templates/server/pvc-data.yaml | 14 + charts/airbyte/templates/server/service.yaml | 14 + charts/airbyte/templates/serviceaccount.yaml | 33 ++ .../airbyte/templates/temporal/configmap.yaml | 48 +++ .../templates/temporal/deployment.yaml | 58 +++ .../airbyte/templates/temporal/service.yaml | 13 + .../airbyte/templates/tests/test-webapp.yaml | 15 + .../airbyte/templates/webapp/deployment.yaml | 79 ++++ charts/airbyte/templates/webapp/ingress.yaml | 61 +++ charts/airbyte/templates/webapp/service.yaml | 14 + charts/airbyte/values.yaml | 393 ++++++++++++++++++ 27 files changed, 1898 insertions(+) create mode 100644 .github/workflows/helm.yaml create mode 100644 charts/airbyte/.gitignore create mode 100644 charts/airbyte/.helmignore create mode 100644 charts/airbyte/Chart.lock create mode 100644 charts/airbyte/Chart.yaml create mode 100644 charts/airbyte/README.md create mode 100755 charts/airbyte/ci.sh create mode 100644 charts/airbyte/files/sweep-pod.sh create mode 100644 charts/airbyte/templates/NOTES.txt create mode 100644 charts/airbyte/templates/_helpers.tpl create mode 100644 charts/airbyte/templates/env-configmap.yaml create mode 100644 charts/airbyte/templates/gcs-log-creds-secret.yaml create mode 100644 charts/airbyte/templates/pod-sweeper/configmap.yaml create mode 100644 charts/airbyte/templates/pod-sweeper/deployment.yaml create mode 100644 charts/airbyte/templates/scheduler/deployment.yaml create mode 100644 charts/airbyte/templates/server/deployment.yaml create mode 100644 charts/airbyte/templates/server/pvc-data.yaml create mode 100644 charts/airbyte/templates/server/service.yaml create mode 100644 charts/airbyte/templates/serviceaccount.yaml create mode 100644 charts/airbyte/templates/temporal/configmap.yaml create mode 100644 charts/airbyte/templates/temporal/deployment.yaml create mode 100644 charts/airbyte/templates/temporal/service.yaml create mode 100644 charts/airbyte/templates/tests/test-webapp.yaml create mode 100644 charts/airbyte/templates/webapp/deployment.yaml create mode 100644 charts/airbyte/templates/webapp/ingress.yaml create mode 100644 charts/airbyte/templates/webapp/service.yaml create mode 100644 charts/airbyte/values.yaml diff --git a/.github/workflows/helm.yaml b/.github/workflows/helm.yaml new file mode 100644 index 000000000000..6c738d4e7c00 --- /dev/null +++ b/.github/workflows/helm.yaml @@ -0,0 +1,74 @@ +name: Helm +on: + push: + paths: + - '.github/workflows/helm.yaml' + - 'charts/**' + pull_request: + paths: + - '.github/workflows/helm.yaml' + - 'charts/**' +jobs: + lint: + name: Lint + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@v2 + - name: Setup Kubectl + uses: azure/setup-kubectl@v1 + - name: Setup Helm + uses: azure/setup-helm@v1 + with: + version: '3.6.3' + - name: Lint Chart + working-directory: ./charts/airbyte + run: ./ci.sh lint + + generate-docs: + name: Generate Docs Parameters + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@v2 + - name: Checkout bitnami-labs/readme-generator-for-helm + uses: actions/checkout@v2 + with: + repository: 'bitnami-labs/readme-generator-for-helm' + ref: '55cab5dd2191c4ffa7245cfefa428d4d9bb12730' + path: readme-generator-for-helm + - name: Install readme-generator-for-helm dependencies + working-directory: readme-generator-for-helm + run: npm install -g + - name: Test can update README with generated parameters + working-directory: charts/airbyte + run: ./ci.sh check-docs-updated + + install: + name: Install + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@v2 + - name: Setup Kubectl + uses: azure/setup-kubectl@v1 + - name: Setup Helm + uses: azure/setup-helm@v1 + with: + version: '3.6.3' + - name: Setup Kind Cluster + uses: helm/kind-action@v1.2.0 + with: + version: "v0.11.1" + image: "kindest/node:v1.21.1" + - name: Install airbyte chart + working-directory: ./charts/airbyte + run: ./ci.sh install + - if: always() + name: Print diagnostics + working-directory: ./charts/airbyte + run: ./ci.sh diagnostics + - if: success() + name: Test airbyte chart + working-directory: ./charts/airbyte + run: ./ci.sh test diff --git a/charts/airbyte/.gitignore b/charts/airbyte/.gitignore new file mode 100644 index 000000000000..88e91e8a8f34 --- /dev/null +++ b/charts/airbyte/.gitignore @@ -0,0 +1,2 @@ +# Charts are downloaded at install time with `helm dep build`. +charts diff --git a/charts/airbyte/.helmignore b/charts/airbyte/.helmignore new file mode 100644 index 000000000000..f885da3fd491 --- /dev/null +++ b/charts/airbyte/.helmignore @@ -0,0 +1,25 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ + +ci.sh diff --git a/charts/airbyte/Chart.lock b/charts/airbyte/Chart.lock new file mode 100644 index 000000000000..c5b73545be82 --- /dev/null +++ b/charts/airbyte/Chart.lock @@ -0,0 +1,12 @@ +dependencies: +- name: common + repository: https://charts.bitnami.com/bitnami + version: 1.8.0 +- name: postgresql + repository: https://charts.bitnami.com/bitnami + version: 10.9.4 +- name: minio + repository: https://charts.bitnami.com/bitnami + version: 7.2.0 +digest: sha256:6b5428c4bffa53a16e1015635d712f28b1c1991eea4d048928e4fbb88974cf4f +generated: "2021-08-31T22:58:47.835303672-05:00" diff --git a/charts/airbyte/Chart.yaml b/charts/airbyte/Chart.yaml new file mode 100644 index 000000000000..873f69a47211 --- /dev/null +++ b/charts/airbyte/Chart.yaml @@ -0,0 +1,39 @@ +apiVersion: v2 +name: airbyte +description: Helm chart to deploy airbyte + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.1.0 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +# It is recommended to use it with quotes. +appVersion: "0.29.13-alpha" + +dependencies: +- name: common + repository: https://charts.bitnami.com/bitnami + tags: + - bitnami-common + version: 1.x.x +- condition: postgresql.enabled + name: postgresql + version: 10.x.x + repository: https://charts.bitnami.com/bitnami +- condition: minio.enabled + name: minio + version: 7.x.x + repository: https://charts.bitnami.com/bitnami diff --git a/charts/airbyte/README.md b/charts/airbyte/README.md new file mode 100644 index 000000000000..e1d69efe702e --- /dev/null +++ b/charts/airbyte/README.md @@ -0,0 +1,149 @@ +# airbyte + +## Parameters + +### Global Parameters + +| Name | Description | Value | +| --------------------- | -------------------------------------------- | ----- | +| `global.storageClass` | Global StorageClass for Persistent Volume(s) | `""` | + + +### Common Parameters + +| Name | Description | Value | +| ---------------------------- | ------------------------------------------------------------------------------------------------------------------- | --------------- | +| `nameOverride` | String to partially override airbyte.fullname template with a string (will prepend the release name) | `""` | +| `fullnameOverride` | String to fully override airbyte.fullname template with a string | `""` | +| `serviceAccount.annotations` | Annotations for service account. Evaluated as a template. Only used if `create` is `true`. | `{}` | +| `serviceAccount.create` | Specifies whether a ServiceAccount should be created | `true` | +| `serviceAccount.name` | Name of the service account to use. If not set and create is true, a name is generated using the fullname template. | `airbyte-admin` | + + +### Webapp Parameters + +| Name | Description | Value | +| ---------------------------- | ---------------------------------------------------------------- | ---------------- | +| `webapp.replicaCount` | Number of webapp replicas | `1` | +| `webapp.image.repository` | The repository to use for the airbyte webapp image. | `airbyte/webapp` | +| `webapp.image.pullPolicy` | the pull policy to use for the airbyte webapp image | `IfNotPresent` | +| `webapp.image.tag` | The airbyte webapp image tag. Defaults to the chart's AppVersion | `0.29.13-alpha` | +| `webapp.podAnnotations` | Add extra annotations to the scheduler pod | `{}` | +| `webapp.service.type` | The service type to use for the webapp service | `ClusterIP` | +| `webapp.service.port` | The service port to expose the webapp on | `80` | +| `webapp.resources.limits` | The resources limits for the Web container | `{}` | +| `webapp.resources.requests` | The requested resources for the Web container | `{}` | +| `webapp.nodeSelector` | Node labels for pod assignment | `{}` | +| `webapp.tolerations` | Tolerations for webapp pod assignment. | `[]` | +| `webapp.ingress.enabled` | Set to true to enable ingress record generation | `false` | +| `webapp.ingress.className` | Specifies ingressClassName for clusters >= 1.18+ | `""` | +| `webapp.ingress.hosts` | Ingress Hosts configuration | `[]` | +| `webapp.ingress.annotations` | Ingress annotations done as key:value pairs | `{}` | +| `webapp.ingress.hosts` | The list of hostnames to be covered with this ingress record. | `[]` | +| `webapp.ingress.tls` | Custom ingress TLS configuration | `[]` | + + +### Scheduler Parameters + +| Name | Description | Value | +| ------------------------------ | ------------------------------------------------------------------- | ------------------- | +| `scheduler.replicaCount` | Number of scheduler replicas | `1` | +| `scheduler.image.repository` | The repository to use for the airbyte scheduler image. | `airbyte/scheduler` | +| `scheduler.image.pullPolicy` | the pull policy to use for the airbyte scheduler image | `IfNotPresent` | +| `scheduler.image.tag` | The airbyte scheduler image tag. Defaults to the chart's AppVersion | `0.29.13-alpha` | +| `scheduler.podAnnotations` | Add extra annotations to the scheduler pod | `{}` | +| `scheduler.resources.limits` | The resources limits for the scheduler container | `{}` | +| `scheduler.resources.requests` | The requested resources for the scheduler container | `{}` | +| `scheduler.nodeSelector` | Node labels for pod assignment | `{}` | +| `scheduler.tolerations` | Tolerations for scheduler pod assignment. | `[]` | + + +### Pod Sweeper parameters + +| Name | Description | Value | +| ------------------------------- | ---------------------------------------------------- | ----------------- | +| `podSweeper.image.repository` | The image repository to use for the pod sweeper | `bitnami/kubectl` | +| `podSweeper.image.pullPolicy` | The pull policy for the pod sweeper image | `IfNotPresent` | +| `podSweeper.image.tag` | The pod sweeper image tag to use | `latest` | +| `podSweeper.podAnnotations` | Add extra annotations to the podSweeper pod | `{}` | +| `podSweeper.resources.limits` | The resources limits for the podSweeper container | `{}` | +| `podSweeper.resources.requests` | The requested resources for the podSweeper container | `{}` | +| `podSweeper.nodeSelector` | Node labels for pod assignment | `{}` | +| `podSweeper.tolerations` | Tolerations for podSweeper pod assignment. | `[]` | + + +### Server parameters + +| Name | Description | Value | +| ------------------------------------------- | ---------------------------------------------------------------- | ---------------- | +| `server.replicaCount` | Number of server replicas | `1` | +| `server.image.repository` | The repository to use for the airbyte server image. | `airbyte/server` | +| `server.image.pullPolicy` | the pull policy to use for the airbyte server image | `IfNotPresent` | +| `server.image.tag` | The airbyte server image tag. Defaults to the chart's AppVersion | `0.29.13-alpha` | +| `server.podAnnotations` | Add extra annotations to the server pod | `{}` | +| `server.livenessProbe.enabled` | Enable livenessProbe on the server | `true` | +| `server.livenessProbe.initialDelaySeconds` | Initial delay seconds for livenessProbe | `30` | +| `server.livenessProbe.periodSeconds` | Period seconds for livenessProbe | `10` | +| `server.livenessProbe.timeoutSeconds` | Timeout seconds for livenessProbe | `1` | +| `server.livenessProbe.failureThreshold` | Failure threshold for livenessProbe | `3` | +| `server.livenessProbe.successThreshold` | Success threshold for livenessProbe | `1` | +| `server.readinessProbe.enabled` | Enable readinessProbe on the server | `true` | +| `server.readinessProbe.initialDelaySeconds` | Initial delay seconds for readinessProbe | `10` | +| `server.readinessProbe.periodSeconds` | Period seconds for readinessProbe | `10` | +| `server.readinessProbe.timeoutSeconds` | Timeout seconds for readinessProbe | `1` | +| `server.readinessProbe.failureThreshold` | Failure threshold for readinessProbe | `3` | +| `server.readinessProbe.successThreshold` | Success threshold for readinessProbe | `1` | +| `server.resources.limits` | The resources limits for the server container | `{}` | +| `server.resources.requests` | The requested resources for the server container | `{}` | +| `server.service.type` | The service type to use for the API server | `ClusterIP` | +| `server.service.port` | The service port to expose the API server on | `8001` | +| `server.persistence.accessMode` | The access mode for the airbyte server pvc | `ReadWriteOnce` | +| `server.persistence.size` | The size of the pvc to use for the airbyte server pvc | `1Gi` | +| `server.persistence.storageClass` | The storage class to use for the airbyte server pvc | `""` | +| `server.nodeSelector` | Node labels for pod assignment | `{}` | +| `server.tolerations` | Tolerations for server pod assignment. | `[]` | + + +### Temporal parameters + +| Name | Description | Value | +| --------------------------- | --------------------------------------------- | ----------------------- | +| `temporal.replicaCount` | The number of temporal replicas to deploy | `1` | +| `temporal.image.repository` | The temporal image repository to use | `temporalio/auto-setup` | +| `temporal.image.pullPolicy` | The pull policy for the temporal image | `IfNotPresent` | +| `temporal.image.tag` | The temporal image tag to use | `1.7.0` | +| `temporal.service.type` | The Kubernetes Service Type | `ClusterIP` | +| `temporal.service.port` | The temporal port and exposed kubernetes port | `7233` | +| `temporal.nodeSelector` | Node labels for pod assignment | `{}` | +| `temporal.tolerations` | Tolerations for pod assignment. | `[]` | + + +### Airbyte Database parameters + +| Name | Description | Value | +| -------------------------------------------- | ----------------------------------------------------------------------------------------- | ------------ | +| `postgresql.enabled` | Switch to enable or disable the PostgreSQL helm chart | `true` | +| `postgresql.postgresqlUsername` | Airbyte Postgresql username | `airbyte` | +| `postgresql.postgresqlPassword` | Airbyte Postgresql password | `airbyte` | +| `postgresql.postgresqlDatabase` | Airbyte Postgresql database | `db-airbyte` | +| `postgresql.existingSecret` | Name of an existing secret containing the PostgreSQL password ('postgresql-password' key) | `""` | +| `externalDatabase.host` | Database host | `localhost` | +| `externalDatabase.user` | non-root Username for Airbyte Database | `airbyte` | +| `externalDatabase.password` | Database password | `""` | +| `externalDatabase.existingSecret` | Name of an existing secret resource containing the DB password | `""` | +| `externalDatabase.existingSecretPasswordKey` | Name of an existing secret key containing the DB password | `""` | +| `externalDatabase.database` | Database name | `db-airbyte` | +| `externalDatabase.port` | Database port number | `5432` | + + +### Minio parameters + +| Name | Description | Value | +| -------------------------- | ------------------------------------------------ | ----------- | +| `minio.enabled` | Switch to enable or disable the Minio helm chart | `true` | +| `minio.accessKey.password` | Minio Access Key | `minio` | +| `minio.secretKey.password` | Minio Secret Key | `minio123` | +| `externalMinio.host` | Minio Host | `localhost` | +| `externalMinio.port` | Minio Port | `9000` | + + diff --git a/charts/airbyte/ci.sh b/charts/airbyte/ci.sh new file mode 100755 index 000000000000..9b0bc77b3a93 --- /dev/null +++ b/charts/airbyte/ci.sh @@ -0,0 +1,90 @@ +#!/bin/bash + +set -e + +export RELEASE_NAME="${RELEASE_NAME:-airbyte}" +export NAMESPACE="${NAMESPACE:-airbyte}" +export INSTALL_TIMEOUT="${INSTALL_TIMEOUT:-600s}" + +usage() { + echo "Airbyte Helm Chart CI Script" + echo "" + echo "Usage:" + echo " ./ci.sh [command]" + echo "" + echo "Available Commands" + echo "" + echo "lint Lints the helm chart" + echo "test Runs 'helm test' for the helm chart" + echo "install Installs the helm chart" + echo "diagnostics Prints diagnostics & logs of pods that aren't running" + echo "template Templates out the default kubernetes manifests generated by helm" + echo "update-docs Regenerates the README.md documentation after making changes to the values.yaml" + echo "check-docs-updated Fails if changes values.yaml and README.md documentation are out of sync" + echo "help Displays help about this command" +} + +if ! helm repo list | grep "bitnami" > /dev/null 2>&1; then + helm repo add bitnami https://charts.bitnami.com/bitnami +fi + +if [ ! -d ./charts ]; then + helm dep build +fi + + +case "$1" in + lint) + helm lint . + ;; + + test) + helm test "${RELEASE_NAME}" --logs --debug --namespace "${NAMESPACE}" + ;; + + install) + helm upgrade --install \ + --create-namespace \ + --namespace "${NAMESPACE}" \ + --debug \ + --wait \ + --timeout "${INSTALL_TIMEOUT}" \ + "${RELEASE_NAME}" . + ;; + + diagnostics) + kubectl -n "${NAMESPACE}" get pods -o wide + kubectl -n "${NAMESPACE}" get pods --no-headers | grep -v "Running" | awk '{print $1}' | xargs -L 1 -r kubectl -n "${NAMESPACE}" logs + ;; + + template) + helm template --release-name "${RELEASE_NAME}" . + ;; + + # Uses the readme-generator command to update the README.md + # with changes in the values.yaml file. See: https://github.com/bitnami-labs/readme-generator-for-helm + update-docs) + readme-generator -v ./values.yaml -r README.md + ;; + + # Checks if the docs were updated as a result of changes in the values.yaml file and if they were + # fails CI with a message about needing to run the update-docs command. + check-docs-updated) + readme-generator -v ./values.yaml -r README.md + if git status --porcelain | grep "charts/airbyte/README.md"; then + echo "It appears the README.md has not been updated with changes from the values.yaml." + echo "Please run './ci.sh update-docs' locally and commit your changes." + exit 1 + else + echo "No uncommitted changes to the README.md detected" + fi + ;; + + help) + usage + ;; + + *) + usage + ;; +esac diff --git a/charts/airbyte/files/sweep-pod.sh b/charts/airbyte/files/sweep-pod.sh new file mode 100644 index 000000000000..15422f204a2f --- /dev/null +++ b/charts/airbyte/files/sweep-pod.sh @@ -0,0 +1,40 @@ +#!/bin/bash + +get_worker_pods () { + kubectl -n ${KUBE_NAMESPACE} -L airbyte -l airbyte=worker-pod --field-selector status.phase!=Running get pods -o go-template --template '{{range .items}} {{.metadata.name}} {{.status.phase}} {{.metadata.creationTimestamp}}{{"\n"}}{{end}}' +} + +delete_worker_pod() { + printf "From status '%s' since '%s', " $2 $3 + echo "$1" | grep -v "STATUS" | awk '{print $1}' | xargs --no-run-if-empty kubectl -n ${KUBE_NAMESPACE} delete pod +} + +while : +do + # Shorter time window for completed pods + SUCCESS_DATE_STR=`date -d 'now - 2 hours' --utc -Ins` + SUCCESS_DATE=`date -d $SUCCESS_DATE_STR +%s` + # Longer time window for pods in error (to debug) + NON_SUCCESS_DATE_STR=`date -d 'now - 24 hours' --utc -Ins` + NON_SUCCESS_DATE=`date -d $NON_SUCCESS_DATE_STR +%s` + ( + IFS=$'\n' + for POD in `get_worker_pods`; do + IFS=' ' + POD_NAME=`echo $POD | cut -d " " -f 1` + POD_STATUS=`echo $POD | cut -d " " -f 2` + POD_DATE_STR=`echo $POD | cut -d " " -f 3` + POD_DATE=`date -d $POD_DATE_STR '+%s'` + if [ "$POD_STATUS" = "Succeeded" ]; then + if [ "$POD_DATE" -lt "$SUCCESS_DATE" ]; then + delete_worker_pod "$POD_NAME" "$POD_STATUS" "$POD_DATE_STR" + fi + else + if [ "$POD_DATE" -lt "$NON_SUCCESS_DATE" ]; then + delete_worker_pod "$POD_NAME" "$POD_STATUS" "$POD_DATE_STR" + fi + fi + done + ) + sleep 60 +done diff --git a/charts/airbyte/templates/NOTES.txt b/charts/airbyte/templates/NOTES.txt new file mode 100644 index 000000000000..f39c2932cfbd --- /dev/null +++ b/charts/airbyte/templates/NOTES.txt @@ -0,0 +1,22 @@ +1. Get the application URL by running these commands: +{{- if .Values.webapp.ingress.enabled }} +{{- range $host := .Values.webapp.ingress.hosts }} + {{- range .paths }} + http{{ if $.Values.webapp.ingress.tls }}s{{ end }}://{{ $host.host }}{{ .path }} + {{- end }} +{{- end }} +{{- else if contains "NodePort" .Values.webapp.service.type }} + export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "airbyte.fullname" . }}) + export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") + echo http://$NODE_IP:$NODE_PORT +{{- else if contains "LoadBalancer" .Values.webapp.service.type }} + NOTE: It may take a few minutes for the LoadBalancer IP to be available. + You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "airbyte.fullname" . }}' + export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "airbyte.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}") + echo http://$SERVICE_IP:{{ .Values.webapp.service.port }} +{{- else if contains "ClusterIP" .Values.webapp.service.type }} + export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "airbyte=webapp" -o jsonpath="{.items[0].metadata.name}") + export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}") + echo "Visit http://127.0.0.1:8080 to use your application" + kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT +{{- end }} diff --git a/charts/airbyte/templates/_helpers.tpl b/charts/airbyte/templates/_helpers.tpl new file mode 100644 index 000000000000..f6f4c8fa67e8 --- /dev/null +++ b/charts/airbyte/templates/_helpers.tpl @@ -0,0 +1,176 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "airbyte.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "airbyte.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "airbyte.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "airbyte.labels" -}} +helm.sh/chart: {{ include "airbyte.chart" . }} +{{ include "airbyte.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "airbyte.selectorLabels" -}} +app.kubernetes.io/name: {{ include "airbyte.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "airbyte.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "airbyte.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} + +{{/* +Create a default fully qualified postgresql name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +*/}} +{{- define "airbyte.postgresql.fullname" -}} +{{- $name := default "postgresql" .Values.postgresql.nameOverride -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} + + +{{/* +Get the Postgresql credentials secret. +*/}} +{{- define "airbyte.postgresql.secretName" -}} +{{- if and (.Values.postgresql.enabled) (not .Values.postgresql.existingSecret) -}} + {{- printf "%s" (include "airbyte.postgresql.fullname" .) -}} +{{- else if and (.Values.postgresql.enabled) (.Values.postgresql.existingSecret) -}} + {{- printf "%s" .Values.postgresql.existingSecret -}} +{{- else }} + {{- if .Values.externalDatabase.existingSecret -}} + {{- printf "%s" .Values.externalDatabase.existingSecret -}} + {{- else -}} + {{ printf "%s-%s" .Release.Name "externaldb" }} + {{- end -}} +{{- end -}} +{{- end -}} + + +{{/* +Add environment variables to configure database values +*/}} +{{- define "airbyte.database.host" -}} +{{- ternary (include "airbyte.postgresql.fullname" .) .Values.externalDatabase.host .Values.postgresql.enabled -}} +{{- end -}} + +{{/* +Add environment variables to configure database values +*/}} +{{- define "airbyte.database.user" -}} +{{- ternary .Values.postgresql.postgresqlUsername .Values.externalDatabase.user .Values.postgresql.enabled -}} +{{- end -}} + +{{/* +Add environment variables to configure database values +*/}} +{{- define "airbyte.database.name" -}} +{{- ternary .Values.postgresql.postgresqlDatabase .Values.externalDatabase.database .Values.postgresql.enabled -}} +{{- end -}} + +{{/* +Add environment variables to configure database values +*/}} +{{- define "airbyte.database.existingsecret.key" -}} +{{- if .Values.postgresql.enabled -}} + {{- printf "%s" "postgresql-password" -}} +{{- else -}} + {{- if .Values.externalDatabase.existingSecret -}} + {{- if .Values.externalDatabase.existingSecretPasswordKey -}} + {{- printf "%s" .Values.externalDatabase.existingSecretPasswordKey -}} + {{- else -}} + {{- printf "%s" "postgresql-password" -}} + {{- end -}} + {{- else -}} + {{- printf "%s" "postgresql-password" -}} + {{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Add environment variables to configure database values +*/}} +{{- define "airbyte.database.port" -}} +{{- ternary "5432" .Values.externalDatabase.port .Values.postgresql.enabled -}} +{{- end -}} + +{{/* +Add environment variables to configure database values +*/}} +{{- define "airbyte.database.url" -}} +{{- $host := (include "airbyte.database.host" .) -}} +{{- $dbName := (include "airbyte.database.name" .) -}} +{{- $port := (include "airbyte.database.port" . ) -}} +{{- printf "jdbc:postgresql://%s:%s/%s" $host $port $dbName -}} +{{- end -}} + +{{/* +Create a default fully qualified minio name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +*/}} +{{- define "airbyte.minio.fullname" -}} +{{- $name := default "minio" .Values.minio.nameOverride -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Add environment variables to configure minio +*/}} +{{- define "airbyte.minio.host" -}} +{{- ternary (include "airbyte.minio.fullname" .) .Values.externalMinio.host .Values.minio.enabled -}} +{{- end -}} + +{{/* +Add environment variables to configure minio +*/}} +{{- define "airbyte.minio.port" -}} +{{- ternary "9000" .Values.externalMinio.port .Values.minio.enabled -}} +{{- end -}} + +{{- define "airbyte.minio.endpoint" -}} +{{- $host := (include "airbyte.minio.host" .) -}} +{{- $port := (include "airbyte.minio.port" .) -}} +{{- printf "http://%s:%s" $host $port -}} +{{- end -}} diff --git a/charts/airbyte/templates/env-configmap.yaml b/charts/airbyte/templates/env-configmap.yaml new file mode 100644 index 000000000000..dfc38afdebd2 --- /dev/null +++ b/charts/airbyte/templates/env-configmap.yaml @@ -0,0 +1,44 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: airbyte-env +data: + AIRBYTE_VERSION: {{ .Chart.AppVersion }} + API_URL: /api/v1/ + AWS_ACCESS_KEY_ID: {{ .Values.minio.accessKey.password }} + AWS_SECRET_ACCESS_KEY: {{ .Values.minio.secretKey.password }} + CONFIG_ROOT: /configs + DATA_DOCKER_MOUNT: airbyte_data + DATABASE_DB: {{ include "airbyte.database.name" . }} + DATABASE_HOST: {{ include "airbyte.database.host" . }} + DATABASE_PASSWORD: {{ .Values.postgresql.postgresqlPassword }} + DATABASE_PORT: {{ include "airbyte.database.port" . | quote }} + DATABASE_URL: {{ include "airbyte.database.url" . | quote }} + DATABASE_USER: {{ include "airbyte.database.user" . }} + DB_DOCKER_MOUNT: airbyte_db + FULLSTORY: enabled + GCP_STORAGE_BUCKET: "" + GOOGLE_APPLICATION_CREDENTIALS: "" + INTERNAL_API_HOST: {{ include "common.names.fullname" . }}-server:{{ .Values.server.service.port }} + IS_DEMO: "false" + LOCAL_ROOT: /tmp/airbyte_local + LOG_LEVEL: INFO + OPENREPLAY: enabled + PAPERCUPS_STORYTIME: enabled + RESOURCE_CPU_LIMIT: "" + RESOURCE_CPU_REQUEST: "" + RESOURCE_MEMORY_LIMIT: "" + RESOURCE_MEMORY_REQUEST: "" + RUN_DATABASE_MIGRATION_ON_STARTUP: "true" + S3_LOG_BUCKET: airbyte-dev-logs + S3_LOG_BUCKET_REGION: "" + S3_MINIO_ENDPOINT: {{ include "airbyte.minio.endpoint" . }} + S3_PATH_STYLE_ACCESS: "true" + SUBMITTER_NUM_THREADS: "10" + TEMPORAL_HOST: {{ include "common.names.fullname" . }}-temporal:{{ .Values.temporal.service.port }} + TEMPORAL_WORKER_PORTS: 9001,9002,9003,9004,9005,9006,9007,9008,9009,9010,9011,9012,9013,9014,9015,9016,9017,9018,9019,9020,9021,9022,9023,9024,9025,9026,9027,9028,9029,9030,9031,9032,9033,9034,9035,9036,9037,9038,9039,9040 + TRACKING_STRATEGY: segment + WEBAPP_URL: http://{{ include "common.names.fullname" . }}-webapp:{{ .Values.webapp.service.port }} + WORKER_ENVIRONMENT: kubernetes + WORKSPACE_DOCKER_MOUNT: airbyte_workspace + WORKSPACE_ROOT: /workspace diff --git a/charts/airbyte/templates/gcs-log-creds-secret.yaml b/charts/airbyte/templates/gcs-log-creds-secret.yaml new file mode 100644 index 000000000000..27e1f7a61d0b --- /dev/null +++ b/charts/airbyte/templates/gcs-log-creds-secret.yaml @@ -0,0 +1,7 @@ +# TODO: Make this more configurable or support an existing secret +apiVersion: v1 +kind: Secret +metadata: + name: gcs-log-creds +data: + gcp.json: "" diff --git a/charts/airbyte/templates/pod-sweeper/configmap.yaml b/charts/airbyte/templates/pod-sweeper/configmap.yaml new file mode 100644 index 000000000000..cf19ce7e1340 --- /dev/null +++ b/charts/airbyte/templates/pod-sweeper/configmap.yaml @@ -0,0 +1,8 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "airbyte.fullname" . }}-sweep-pod-script +data: + sweep-pod.sh: | + {{- (.Files.Get "files/sweep-pod.sh") | nindent 4 }} diff --git a/charts/airbyte/templates/pod-sweeper/deployment.yaml b/charts/airbyte/templates/pod-sweeper/deployment.yaml new file mode 100644 index 000000000000..b7c0b1bab2fa --- /dev/null +++ b/charts/airbyte/templates/pod-sweeper/deployment.yaml @@ -0,0 +1,54 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "airbyte.fullname" . }}-pod-sweeper + labels: + {{- include "airbyte.labels" . | nindent 4 }} +spec: + replicas: 1 + selector: + matchLabels: + airbyte: pod-sweeper + template: + metadata: + labels: + airbyte: pod-sweeper + {{- if .Values.podSweeper.podAnnotations }} + annotations: + {{- include "common.tplvalues.render" (dict "value" .Values.podSweeper.podAnnotations "context" $) | nindent 8 }} + {{- end }} + spec: + serviceAccountName: {{ include "airbyte.serviceAccountName" . }} + {{- if .Values.podSweeper.nodeSelector }} + nodeSelector: {{- include "common.tplvalues.render" (dict "value" .Values.podSweeper.nodeSelector "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.podSweeper.tolerations }} + tolerations: {{- include "common.tplvalues.render" (dict "value" .Values.podSweeper.tolerations "context" $) | nindent 8 }} + {{- end }} + containers: + - name: airbyte-pod-sweeper + image: "{{ .Values.podSweeper.image.repository }}:{{ .Values.podSweeper.image.tag}}" + imagePullPolicy: "{{ .Values.podSweeper.image.pullPolicy }}" + env: + - name: KUBE_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + volumeMounts: + - mountPath: /script/sweep-pod.sh + subPath: sweep-pod.sh + name: sweep-pod-script + - mountPath: /.kube + name: kube-config + command: ["/bin/bash", "-c", /script/sweep-pod.sh] + {{- if .Values.podSweeper.resources }} + resources: {{- toYaml .Values.podSweeper.resources | nindent 10 }} + {{- end }} + volumes: + - name: kube-config + emptyDir: {} + - name: sweep-pod-script + configMap: + name: {{ include "airbyte.fullname" . }}-sweep-pod-script + defaultMode: 0755 diff --git a/charts/airbyte/templates/scheduler/deployment.yaml b/charts/airbyte/templates/scheduler/deployment.yaml new file mode 100644 index 000000000000..d4fde6a6742e --- /dev/null +++ b/charts/airbyte/templates/scheduler/deployment.yaml @@ -0,0 +1,218 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "airbyte.fullname" . }}-scheduler + labels: + {{- include "airbyte.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.scheduler.replicaCount }} + selector: + matchLabels: + airbyte: scheduler + template: + metadata: + labels: + airbyte: scheduler + {{- if .Values.scheduler.podAnnotations }} + annotations: + {{- include "common.tplvalues.render" (dict "value" .Values.scheduler.podAnnotations "context" $) | nindent 8 }} + {{- end }} + spec: + serviceAccountName: {{ include "airbyte.serviceAccountName" . }} + automountServiceAccountToken: true + {{- if .Values.scheduler.nodeSelector }} + nodeSelector: {{- include "common.tplvalues.render" (dict "value" .Values.scheduler.nodeSelector "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.scheduler.tolerations }} + tolerations: {{- include "common.tplvalues.render" (dict "value" .Values.scheduler.tolerations "context" $) | nindent 8 }} + {{- end }} + containers: + - name: airbyte-scheduler-container + image: "{{ .Values.scheduler.image.repository }}:{{ default .Chart.AppVersion .Values.scheduler.image.tag }}" + imagePullPolicy: "{{ .Values.scheduler.image.pullPolicy }}" + env: + - name: AIRBYTE_VERSION + value: {{ .Chart.AppVersion }} + - name: CONFIG_ROOT + valueFrom: + configMapKeyRef: + name: airbyte-env + key: CONFIG_ROOT + - name: DATABASE_HOST + valueFrom: + configMapKeyRef: + name: airbyte-env + key: DATABASE_HOST + - name: DATABASE_PORT + valueFrom: + configMapKeyRef: + name: airbyte-env + key: DATABASE_PORT + - name: DATABASE_PASSWORD + valueFrom: + configMapKeyRef: + name: airbyte-env + key: DATABASE_PASSWORD + - name: DATABASE_URL + valueFrom: + configMapKeyRef: + name: airbyte-env + key: DATABASE_URL + - name: DATABASE_USER + valueFrom: + configMapKeyRef: + name: airbyte-env + key: DATABASE_USER + - name: TRACKING_STRATEGY + valueFrom: + configMapKeyRef: + name: airbyte-env + key: TRACKING_STRATEGY + - name: WORKSPACE_DOCKER_MOUNT + value: workspace + - name: WORKSPACE_ROOT + valueFrom: + configMapKeyRef: + name: airbyte-env + key: WORKSPACE_ROOT + - name: WORKER_ENVIRONMENT + valueFrom: + configMapKeyRef: + name: airbyte-env + key: WORKER_ENVIRONMENT + - name: LOCAL_ROOT + valueFrom: + configMapKeyRef: + name: airbyte-env + key: LOCAL_ROOT + - name: WEBAPP_URL + valueFrom: + configMapKeyRef: + name: airbyte-env + key: WEBAPP_URL + - name: TEMPORAL_HOST + valueFrom: + configMapKeyRef: + name: airbyte-env + key: TEMPORAL_HOST + - name: TEMPORAL_WORKER_PORTS + valueFrom: + configMapKeyRef: + name: airbyte-env + key: TEMPORAL_WORKER_PORTS + - name: LOG_LEVEL + valueFrom: + configMapKeyRef: + name: airbyte-env + key: LOG_LEVEL + - name: KUBE_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: SUBMITTER_NUM_THREADS + valueFrom: + configMapKeyRef: + name: airbyte-env + key: SUBMITTER_NUM_THREADS + - name: RESOURCE_CPU_REQUEST + valueFrom: + configMapKeyRef: + name: airbyte-env + key: RESOURCE_CPU_REQUEST + - name: RESOURCE_CPU_LIMIT + valueFrom: + configMapKeyRef: + name: airbyte-env + key: RESOURCE_CPU_LIMIT + - name: RESOURCE_MEMORY_REQUEST + valueFrom: + configMapKeyRef: + name: airbyte-env + key: RESOURCE_MEMORY_REQUEST + - name: RESOURCE_MEMORY_LIMIT + valueFrom: + configMapKeyRef: + name: airbyte-env + key: RESOURCE_MEMORY_LIMIT + - name: S3_LOG_BUCKET + valueFrom: + configMapKeyRef: + name: airbyte-env + key: S3_LOG_BUCKET + - name: S3_LOG_BUCKET_REGION + valueFrom: + configMapKeyRef: + name: airbyte-env + key: S3_LOG_BUCKET_REGION + - name: AWS_ACCESS_KEY_ID + value: {{ .Values.minio.accessKey.password }} + - name: AWS_SECRET_ACCESS_KEY + value: {{ .Values.minio.secretKey.password }} + - name: S3_MINIO_ENDPOINT + valueFrom: + configMapKeyRef: + name: airbyte-env + key: S3_MINIO_ENDPOINT + - name: S3_PATH_STYLE_ACCESS + valueFrom: + configMapKeyRef: + name: airbyte-env + key: S3_PATH_STYLE_ACCESS + - name: GOOGLE_APPLICATION_CREDENTIALS + valueFrom: + configMapKeyRef: + name: airbyte-env + key: GOOGLE_APPLICATION_CREDENTIALS + - name: GCP_STORAGE_BUCKET + valueFrom: + configMapKeyRef: + name: airbyte-env + key: GCP_STORAGE_BUCKET + - name: INTERNAL_API_HOST + valueFrom: + configMapKeyRef: + name: airbyte-env + key: INTERNAL_API_HOST + ports: + - containerPort: 9000 # for heartbeat server + - containerPort: 9001 # start temporal worker port pool + - containerPort: 9002 + - containerPort: 9003 + - containerPort: 9004 + - containerPort: 9005 + - containerPort: 9006 + - containerPort: 9007 + - containerPort: 9008 + - containerPort: 9009 + - containerPort: 9010 + - containerPort: 9011 + - containerPort: 9012 + - containerPort: 9013 + - containerPort: 9014 + - containerPort: 9015 + - containerPort: 9016 + - containerPort: 9017 + - containerPort: 9018 + - containerPort: 9019 + - containerPort: 9020 + - containerPort: 9021 + - containerPort: 9022 + - containerPort: 9023 + - containerPort: 9024 + - containerPort: 9025 + - containerPort: 9026 + - containerPort: 9027 + - containerPort: 9028 + - containerPort: 9029 + - containerPort: 9030 # end temporal worker port pool + {{- if .Values.scheduler.resources }} + resources: {{- toYaml .Values.scheduler.resources | nindent 10 }} + {{- end }} + volumeMounts: + - name: gcs-log-creds-volume + mountPath: /secrets/gcs-log-creds + readOnly: true + volumes: + - name: gcs-log-creds-volume + secret: + secretName: gcs-log-creds diff --git a/charts/airbyte/templates/server/deployment.yaml b/charts/airbyte/templates/server/deployment.yaml new file mode 100644 index 000000000000..4e7cc99b0b49 --- /dev/null +++ b/charts/airbyte/templates/server/deployment.yaml @@ -0,0 +1,196 @@ + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "common.names.fullname" . }}-server + labels: + {{- include "airbyte.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.server.replicaCount }} + selector: + matchLabels: + airbyte: server + strategy: + type: Recreate # Needed due to volume claims + template: + metadata: + labels: + airbyte: server + {{- if .Values.server.podAnnotations }} + annotations: + {{- include "common.tplvalues.render" (dict "value" .Values.server.podAnnotations "context" $) | nindent 8 }} + {{- end }} + spec: + {{- if .Values.server.nodeSelector }} + nodeSelector: {{- include "common.tplvalues.render" (dict "value" .Values.server.nodeSelector "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.server.tolerations }} + tolerations: {{- include "common.tplvalues.render" (dict "value" .Values.server.tolerations "context" $) | nindent 8 }} + {{- end }} + containers: + - name: airbyte-server-container + image: "{{ .Values.server.image.repository }}:{{ default .Chart.AppVersion .Values.server.image.tag }}" + imagePullPolicy: "{{ .Values.server.image.pullPolicy }}" + env: + - name: AIRBYTE_VERSION + valueFrom: + configMapKeyRef: + name: airbyte-env + key: AIRBYTE_VERSION + - name: CONFIG_ROOT + valueFrom: + configMapKeyRef: + name: airbyte-env + key: CONFIG_ROOT + - name: DATABASE_PASSWORD + valueFrom: + configMapKeyRef: + name: airbyte-env + key: DATABASE_PASSWORD + - name: DATABASE_URL + valueFrom: + configMapKeyRef: + name: airbyte-env + key: DATABASE_URL + - name: DATABASE_USER + valueFrom: + configMapKeyRef: + name: airbyte-env + key: DATABASE_USER + - name: TRACKING_STRATEGY + valueFrom: + configMapKeyRef: + name: airbyte-env + key: TRACKING_STRATEGY + - name: WORKER_ENVIRONMENT + valueFrom: + configMapKeyRef: + name: airbyte-env + key: WORKER_ENVIRONMENT + - name: WORKSPACE_ROOT + valueFrom: + configMapKeyRef: + name: airbyte-env + key: WORKSPACE_ROOT + - name: WEBAPP_URL + valueFrom: + configMapKeyRef: + name: airbyte-env + key: WEBAPP_URL + - name: TEMPORAL_HOST + valueFrom: + configMapKeyRef: + name: airbyte-env + key: TEMPORAL_HOST + - name: LOG_LEVEL + valueFrom: + configMapKeyRef: + name: airbyte-env + key: LOG_LEVEL + - name: RESOURCE_CPU_REQUEST + valueFrom: + configMapKeyRef: + name: airbyte-env + key: RESOURCE_CPU_REQUEST + - name: RESOURCE_CPU_LIMIT + valueFrom: + configMapKeyRef: + name: airbyte-env + key: RESOURCE_CPU_LIMIT + - name: RESOURCE_MEMORY_REQUEST + valueFrom: + configMapKeyRef: + name: airbyte-env + key: RESOURCE_MEMORY_REQUEST + - name: RESOURCE_MEMORY_LIMIT + valueFrom: + configMapKeyRef: + name: airbyte-env + key: RESOURCE_MEMORY_LIMIT + - name: S3_LOG_BUCKET + valueFrom: + configMapKeyRef: + name: airbyte-env + key: S3_LOG_BUCKET + - name: S3_LOG_BUCKET_REGION + valueFrom: + configMapKeyRef: + name: airbyte-env + key: S3_LOG_BUCKET_REGION + - name: AWS_ACCESS_KEY_ID + valueFrom: + configMapKeyRef: + name: airbyte-env + key: AWS_ACCESS_KEY_ID + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + configMapKeyRef: + name: airbyte-env + key: AWS_SECRET_ACCESS_KEY + - name: S3_MINIO_ENDPOINT + valueFrom: + configMapKeyRef: + name: airbyte-env + key: S3_MINIO_ENDPOINT + - name: S3_PATH_STYLE_ACCESS + valueFrom: + configMapKeyRef: + name: airbyte-env + key: S3_PATH_STYLE_ACCESS + - name: GOOGLE_APPLICATION_CREDENTIALS + valueFrom: + configMapKeyRef: + name: airbyte-env + key: GOOGLE_APPLICATION_CREDENTIALS + - name: GCP_STORAGE_BUCKET + valueFrom: + configMapKeyRef: + name: airbyte-env + key: GCP_STORAGE_BUCKET + {{- if .Values.server.livenessProbe.enabled }} + livenessProbe: + httpGet: + path: /api/v1/health + port: http + initialDelaySeconds: {{ .Values.server.livenessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.server.livenessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.server.livenessProbe.timeoutSeconds }} + successThreshold: {{ .Values.server.livenessProbe.successThreshold }} + failureThreshold: {{ .Values.server.livenessProbe.failureThreshold }} + {{- end }} + {{- if .Values.server.readinessProbe.enabled }} + readinessProbe: + httpGet: + path: /api/v1/health + port: http + initialDelaySeconds: {{ .Values.server.readinessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.server.readinessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.server.readinessProbe.timeoutSeconds }} + successThreshold: {{ .Values.server.readinessProbe.successThreshold }} + failureThreshold: {{ .Values.server.readinessProbe.failureThreshold }} + {{- end }} + ports: + - name: http + containerPort: 8001 + protocol: TCP + {{- if .Values.server.resources }} + resources: {{- toYaml .Values.server.resources | nindent 10 }} + {{- end }} + volumeMounts: + - name: airbyte-data + mountPath: /configs + subPath: configs + - name: airbyte-data + mountPath: /workspace + subPath: workspace + - name: gcs-log-creds-volume + mountPath: /secrets/gcs-log-creds + readOnly: true + volumes: + - name: airbyte-data + persistentVolumeClaim: + claimName: {{ include "common.names.fullname" . }}-data + - name: gcs-log-creds-volume + secret: + secretName: gcs-log-creds diff --git a/charts/airbyte/templates/server/pvc-data.yaml b/charts/airbyte/templates/server/pvc-data.yaml new file mode 100644 index 000000000000..62ef1e3ccb38 --- /dev/null +++ b/charts/airbyte/templates/server/pvc-data.yaml @@ -0,0 +1,14 @@ +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ include "common.names.fullname" . }}-data + labels: + airbyte: volume-data +spec: + accessModes: + - {{ .Values.server.persistence.accessMode | quote }} + resources: + requests: + storage: {{ .Values.server.persistence.size | quote }} + {{- include "common.storage.class" (dict "persistence" .Values.server.persistence "global" .Values.global) | nindent 2 }} diff --git a/charts/airbyte/templates/server/service.yaml b/charts/airbyte/templates/server/service.yaml new file mode 100644 index 000000000000..a00b7021a17f --- /dev/null +++ b/charts/airbyte/templates/server/service.yaml @@ -0,0 +1,14 @@ +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ include "common.names.fullname" . }}-server +spec: + type: {{ .Values.server.service.type }} + ports: + - port: {{ .Values.server.service.port }} + targetPort: http + protocol: TCP + name: http + selector: + airbyte: server diff --git a/charts/airbyte/templates/serviceaccount.yaml b/charts/airbyte/templates/serviceaccount.yaml new file mode 100644 index 000000000000..2be26ab5e964 --- /dev/null +++ b/charts/airbyte/templates/serviceaccount.yaml @@ -0,0 +1,33 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "airbyte.serviceAccountName" . }} + labels: + {{- include "airbyte.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "airbyte.serviceAccountName" . }}-role +rules: + - apiGroups: ["*"] + resources: ["jobs", "pods", "pods/log", "pods/exec", "pods/attach"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] # over-permission for now +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "airbyte.serviceAccountName" . }}-binding +roleRef: + apiGroup: "" + kind: Role + name: {{ include "airbyte.serviceAccountName" . }}-role +subjects: + - kind: ServiceAccount + name: {{ include "airbyte.serviceAccountName" . }} +{{- end }} diff --git a/charts/airbyte/templates/temporal/configmap.yaml b/charts/airbyte/templates/temporal/configmap.yaml new file mode 100644 index 000000000000..bab965cf3fe6 --- /dev/null +++ b/charts/airbyte/templates/temporal/configmap.yaml @@ -0,0 +1,48 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: airbyte-temporal-dynamicconfig +data: + "development.yaml": | + # when modifying, remember to update the docker-compose version of this file in temporal/dynamicconfig/development.yaml + frontend.enableClientVersionCheck: + - value: true + constraints: {} + history.persistenceMaxQPS: + - value: 3000 + constraints: {} + frontend.persistenceMaxQPS: + - value: 3000 + constraints: {} + frontend.historyMgrNumConns: + - value: 30 + constraints: {} + frontend.throttledLogRPS: + - value: 20 + constraints: {} + history.historyMgrNumConns: + - value: 50 + constraints: {} + system.advancedVisibilityWritingMode: + - value: "off" + constraints: {} + history.defaultActivityRetryPolicy: + - value: + InitialIntervalInSeconds: 1 + MaximumIntervalCoefficient: 100.0 + BackoffCoefficient: 2.0 + MaximumAttempts: 0 + history.defaultWorkflowRetryPolicy: + - value: + InitialIntervalInSeconds: 1 + MaximumIntervalCoefficient: 100.0 + BackoffCoefficient: 2.0 + MaximumAttempts: 0 + # Limit for responses. This mostly impacts discovery jobs since they have the largest responses. + limit.blobSize.error: + - value: 15728640 # 15MB + constraints: {} + limit.blobSize.warn: + - value: 10485760 # 10MB + constraints: {} diff --git a/charts/airbyte/templates/temporal/deployment.yaml b/charts/airbyte/templates/temporal/deployment.yaml new file mode 100644 index 000000000000..48aba564aa86 --- /dev/null +++ b/charts/airbyte/templates/temporal/deployment.yaml @@ -0,0 +1,58 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "common.names.fullname" . }}-temporal + labels: + {{- include "airbyte.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.temporal.replicaCount }} + selector: + matchLabels: + airbyte: temporal + template: + metadata: + labels: + airbyte: temporal + spec: + {{- if .Values.temporal.nodeSelector }} + nodeSelector: {{- include "common.tplvalues.render" (dict "value" .Values.temporal.nodeSelector "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.temporal.tolerations }} + tolerations: {{- include "common.tplvalues.render" (dict "value" .Values.temporal.tolerations "context" $) | nindent 8 }} + {{- end }} + containers: + - name: airbyte-temporal + image: "{{ .Values.temporal.image.repository}}:{{ .Values.temporal.image.tag }}" + imagePullPolicy: {{ .Values.temporal.image.pullPolicy }} + env: + - name: AUTO_SETUP + value: "true" + - name: DB # The DB engine to use + value: "postgresql" + - name: DB_PORT + value: {{ include "airbyte.database.port" . | quote }} + - name: POSTGRES_USER + value: {{ include "airbyte.database.user" . }} + - name: POSTGRES_PWD + valueFrom: + secretKeyRef: + name: {{ include "airbyte.postgresql.secretName" . }} + key: {{ include "airbyte.database.existingsecret.key" . }} + - name: POSTGRES_SEEDS + value: {{ include "airbyte.database.host" . }} + - name: DYNAMIC_CONFIG_FILE_PATH + value: "config/dynamicconfig/development.yaml" + + ports: + - containerPort: 7233 + volumeMounts: + - name: airbyte-temporal-dynamicconfig + mountPath: "/etc/temporal/config/dynamicconfig/" + volumes: + - name: airbyte-temporal-dynamicconfig + configMap: + name: airbyte-temporal-dynamicconfig + items: + - key: development.yaml + path: development.yaml diff --git a/charts/airbyte/templates/temporal/service.yaml b/charts/airbyte/templates/temporal/service.yaml new file mode 100644 index 000000000000..7c949422aeac --- /dev/null +++ b/charts/airbyte/templates/temporal/service.yaml @@ -0,0 +1,13 @@ +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ include "common.names.fullname" . }}-temporal +spec: + type: {{ .Values.temporal.service.type }} + ports: + - port: {{ .Values.temporal.service.port }} + protocol: TCP + targetPort: 7233 + selector: + airbyte: temporal diff --git a/charts/airbyte/templates/tests/test-webapp.yaml b/charts/airbyte/templates/tests/test-webapp.yaml new file mode 100644 index 000000000000..9784832869aa --- /dev/null +++ b/charts/airbyte/templates/tests/test-webapp.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Pod +metadata: + name: "{{ include "airbyte.fullname" . }}-test-connection" + labels: + {{- include "airbyte.labels" . | nindent 4 }} + annotations: + "helm.sh/hook": test +spec: + containers: + - name: wget + image: busybox + command: ['wget'] + args: ['{{ include "common.names.fullname" . }}-webapp:{{ .Values.webapp.service.port }}'] + restartPolicy: Never diff --git a/charts/airbyte/templates/webapp/deployment.yaml b/charts/airbyte/templates/webapp/deployment.yaml new file mode 100644 index 000000000000..43a524cfe7ed --- /dev/null +++ b/charts/airbyte/templates/webapp/deployment.yaml @@ -0,0 +1,79 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "common.names.fullname" . }}-webapp + labels: + {{- include "airbyte.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.webapp.replicaCount }} + selector: + matchLabels: + airbyte: webapp + template: + metadata: + labels: + airbyte: webapp + {{- if .Values.webapp.podAnnotations }} + annotations: + {{- include "common.tplvalues.render" (dict "value" .Values.webapp.podAnnotations "context" $) | nindent 8 }} + {{- end }} + spec: + {{- if .Values.webapp.nodeSelector }} + nodeSelector: {{- include "common.tplvalues.render" (dict "value" .Values.webapp.nodeSelector "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.webapp.tolerations }} + tolerations: {{- include "common.tplvalues.render" (dict "value" .Values.webapp.tolerations "context" $) | nindent 8 }} + {{- end }} + containers: + - name: airbyte-webapp-container + image: "{{ .Values.webapp.image.repository }}:{{ default .Chart.AppVersion .Values.webapp.image.tag }}" + imagePullPolicy: "{{ .Values.webapp.image.pullPolicy }}" + env: + - name: AIRBYTE_VERSION + valueFrom: + configMapKeyRef: + name: airbyte-env + key: AIRBYTE_VERSION + - name: API_URL + valueFrom: + configMapKeyRef: + name: airbyte-env + key: API_URL + - name: TRACKING_STRATEGY + valueFrom: + configMapKeyRef: + name: airbyte-env + key: TRACKING_STRATEGY + - name: PAPERCUPS_STORYTIME + valueFrom: + configMapKeyRef: + name: airbyte-env + key: PAPERCUPS_STORYTIME + - name: FULLSTORY + valueFrom: + configMapKeyRef: + name: airbyte-env + key: FULLSTORY + - name: OPENREPLAY + valueFrom: + configMapKeyRef: + name: airbyte-env + key: OPENREPLAY + - name: IS_DEMO + valueFrom: + configMapKeyRef: + name: airbyte-env + key: IS_DEMO + - name: INTERNAL_API_HOST + valueFrom: + configMapKeyRef: + name: airbyte-env + key: INTERNAL_API_HOST + ports: + - name: http + containerPort: 80 + protocol: TCP + {{- if .Values.webapp.resources }} + resources: {{- toYaml .Values.webapp.resources | nindent 10 }} + {{- end }} diff --git a/charts/airbyte/templates/webapp/ingress.yaml b/charts/airbyte/templates/webapp/ingress.yaml new file mode 100644 index 000000000000..670d8d8783a4 --- /dev/null +++ b/charts/airbyte/templates/webapp/ingress.yaml @@ -0,0 +1,61 @@ +{{- if .Values.webapp.ingress.enabled -}} +{{- $fullName := include "airbyte.fullname" . -}} +{{- $svcPort := .Values.webapp.service.port -}} +{{- if and .Values.webapp.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }} + {{- if not (hasKey .Values.webapp.ingress.annotations "kubernetes.io/ingress.class") }} + {{- $_ := set .Values.webapp.ingress.annotations "kubernetes.io/ingress.class" .Values.webapp.ingress.className}} + {{- end }} +{{- end }} +{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}} +apiVersion: networking.k8s.io/v1 +{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} +apiVersion: networking.k8s.io/v1beta1 +{{- else -}} +apiVersion: extensions/v1beta1 +{{- end }} +kind: Ingress +metadata: + name: {{ $fullName }} + labels: + {{- include "airbyte.labels" . | nindent 4 }} + {{- with .Values.webapp.ingress.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- if and .Values.webapp.ingress.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }} + ingressClassName: {{ .Values.webapp.ingress.className }} + {{- end }} + {{- if .Values.webapp.ingress.tls }} + tls: + {{- range .Values.webapp.ingress.tls }} + - hosts: + {{- range .hosts }} + - {{ . | quote }} + {{- end }} + secretName: {{ .secretName }} + {{- end }} + {{- end }} + rules: + {{- range .Values.webapp.ingress.hosts }} + - host: {{ .host | quote }} + http: + paths: + {{- range .paths }} + - path: {{ .path }} + {{- if and .pathType (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }} + pathType: {{ .pathType }} + {{- end }} + backend: + {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} + service: + name: {{ $fullName }}-webapp + port: + number: {{ $svcPort }} + {{- else }} + serviceName: {{ $fullName }}-webapp + servicePort: {{ $svcPort }} + {{- end }} + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/airbyte/templates/webapp/service.yaml b/charts/airbyte/templates/webapp/service.yaml new file mode 100644 index 000000000000..925360ea77fb --- /dev/null +++ b/charts/airbyte/templates/webapp/service.yaml @@ -0,0 +1,14 @@ +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ include "common.names.fullname" . }}-webapp +spec: + type: {{ .Values.webapp.service.type }} + ports: + - port: {{ .Values.webapp.service.port }} + targetPort: http + protocol: TCP + name: http + selector: + airbyte: webapp diff --git a/charts/airbyte/values.yaml b/charts/airbyte/values.yaml new file mode 100644 index 000000000000..fe4e2d5cf7a0 --- /dev/null +++ b/charts/airbyte/values.yaml @@ -0,0 +1,393 @@ +## @section Global Parameters + +## @param global.storageClass Global StorageClass for Persistent Volume(s) +## +global: + storageClass: "" + +## @section Common Parameters + +## @param nameOverride String to partially override airbyte.fullname template with a string (will prepend the release name) +## +nameOverride: "" +## @param fullnameOverride String to fully override airbyte.fullname template with a string +## +fullnameOverride: "" + +## Pods Service Account +## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ +## @param serviceAccount.annotations [object] Annotations for service account. Evaluated as a template. Only used if `create` is `true`. +## @param serviceAccount.create Specifies whether a ServiceAccount should be created +## @param serviceAccount.name Name of the service account to use. If not set and create is true, a name is generated using the fullname template. +## +serviceAccount: + create: true + annotations: {} + name: airbyte-admin + +## @section Webapp Parameters + +webapp: + ## @param webapp.replicaCount Number of webapp replicas + replicaCount: 1 + + ## @param webapp.image.repository The repository to use for the airbyte webapp image. + ## @param webapp.image.pullPolicy the pull policy to use for the airbyte webapp image + ## @param webapp.image.tag The airbyte webapp image tag. Defaults to the chart's AppVersion + image: + repository: airbyte/webapp + pullPolicy: IfNotPresent + tag: 0.29.13-alpha + + ## @param webapp.podAnnotations [object] Add extra annotations to the scheduler pod + ## + podAnnotations: {} + + ## @param webapp.service.type The service type to use for the webapp service + ## @param webapp.service.port The service port to expose the webapp on + service: + type: ClusterIP + port: 80 + + ## Web app resource requests and limits + ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ + ## We usually recommend not to specify default resources and to leave this as a conscious + ## choice for the user. This also increases chances charts run on environments with little + ## resources, such as Minikube. If you do want to specify resources, uncomment the following + ## lines, adjust them as necessary, and remove the curly braces after 'resources:'. + ## @param webapp.resources.limits [object] The resources limits for the Web container + ## @param webapp.resources.requests [object] The requested resources for the Web container + resources: + ## Example: + ## limits: + ## cpu: 200m + ## memory: 1Gi + limits: {} + ## Examples: + ## requests: + ## memory: 256Mi + ## cpu: 250m + requests: {} + + ## @param webapp.nodeSelector [object] Node labels for pod assignment + ## Ref: https://kubernetes.io/docs/user-guide/node-selection/ + ## + nodeSelector: {} + + ## @param webapp.tolerations [array] Tolerations for webapp pod assignment. + ## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + ## + tolerations: [] + + ## Configure the ingress resource that allows you to access the Airbyte installation. + ## ref: http://kubernetes.io/docs/user-guide/ingress/ + ## @param webapp.ingress.enabled Set to true to enable ingress record generation + ## @param webapp.ingress.className Specifies ingressClassName for clusters >= 1.18+ + ## @param webapp.ingress.hosts Ingress Hosts configuration + ## @param webapp.ingress.annotations [object] Ingress annotations done as key:value pairs + ## @param webapp.ingress.hosts The list of hostnames to be covered with this ingress record. + ## @param webapp.ingress.tls [array] Custom ingress TLS configuration + ingress: + enabled: false + className: "" + annotations: {} + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" + hosts: + - host: chart-example.local + paths: + - path: / + pathType: ImplementationSpecific + tls: [] + # - secretName: chart-example-tls + # hosts: + # - chart-example.local + + +## @section Scheduler Parameters + +scheduler: + ## @param scheduler.replicaCount Number of scheduler replicas + replicaCount: 1 + + ## @param scheduler.image.repository The repository to use for the airbyte scheduler image. + ## @param scheduler.image.pullPolicy the pull policy to use for the airbyte scheduler image + ## @param scheduler.image.tag The airbyte scheduler image tag. Defaults to the chart's AppVersion + image: + repository: airbyte/scheduler + pullPolicy: IfNotPresent + tag: 0.29.13-alpha + + ## @param scheduler.podAnnotations [object] Add extra annotations to the scheduler pod + ## + podAnnotations: {} + + ## Scheduler resource requests and limits + ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ + ## We usually recommend not to specify default resources and to leave this as a conscious + ## choice for the user. This also increases chances charts run on environments with little + ## resources, such as Minikube. If you do want to specify resources, uncomment the following + ## lines, adjust them as necessary, and remove the curly braces after 'resources:'. + ## @param scheduler.resources.limits [object] The resources limits for the scheduler container + ## @param scheduler.resources.requests [object] The requested resources for the scheduler container + resources: + ## Example: + ## limits: + ## cpu: 200m + ## memory: 1Gi + limits: {} + ## Examples: + ## requests: + ## memory: 256Mi + ## cpu: 250m + requests: {} + + ## @param scheduler.nodeSelector [object] Node labels for pod assignment + ## Ref: https://kubernetes.io/docs/user-guide/node-selection/ + ## + nodeSelector: {} + + ## @param scheduler.tolerations [array] Tolerations for scheduler pod assignment. + ## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + ## + tolerations: [] + + +## @section Pod Sweeper parameters + +podSweeper: + ## @param podSweeper.image.repository The image repository to use for the pod sweeper + ## @param podSweeper.image.pullPolicy The pull policy for the pod sweeper image + ## @param podSweeper.image.tag The pod sweeper image tag to use + image: + repository: bitnami/kubectl + pullPolicy: IfNotPresent + tag: latest + + ## @param podSweeper.podAnnotations [object] Add extra annotations to the podSweeper pod + ## + podAnnotations: {} + + ## Pod Sweeper app resource requests and limits + ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ + ## We usually recommend not to specify default resources and to leave this as a conscious + ## choice for the user. This also increases chances charts run on environments with little + ## resources, such as Minikube. If you do want to specify resources, uncomment the following + ## lines, adjust them as necessary, and remove the curly braces after 'resources:'. + ## @param podSweeper.resources.limits [object] The resources limits for the podSweeper container + ## @param podSweeper.resources.requests [object] The requested resources for the podSweeper container + resources: + ## Example: + ## limits: + ## cpu: 200m + ## memory: 1Gi + limits: {} + ## Examples: + ## requests: + ## memory: 256Mi + ## cpu: 250m + requests: {} + + ## @param podSweeper.nodeSelector [object] Node labels for pod assignment + ## Ref: https://kubernetes.io/docs/user-guide/node-selection/ + ## + nodeSelector: {} + + ## @param podSweeper.tolerations [array] Tolerations for podSweeper pod assignment. + ## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + ## + tolerations: [] + + +## @section Server parameters + +server: + ## @param server.replicaCount Number of server replicas + replicaCount: 1 + + ## @param server.image.repository The repository to use for the airbyte server image. + ## @param server.image.pullPolicy the pull policy to use for the airbyte server image + ## @param server.image.tag The airbyte server image tag. Defaults to the chart's AppVersion + image: + repository: airbyte/server + pullPolicy: IfNotPresent + tag: 0.29.13-alpha + + ## @param server.podAnnotations [object] Add extra annotations to the server pod + ## + podAnnotations: {} + + ## Configure extra options for the server containers' liveness and readiness probes + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/#configure-probes + ## @param server.livenessProbe.enabled Enable livenessProbe on the server + ## @param server.livenessProbe.initialDelaySeconds Initial delay seconds for livenessProbe + ## @param server.livenessProbe.periodSeconds Period seconds for livenessProbe + ## @param server.livenessProbe.timeoutSeconds Timeout seconds for livenessProbe + ## @param server.livenessProbe.failureThreshold Failure threshold for livenessProbe + ## @param server.livenessProbe.successThreshold Success threshold for livenessProbe + ## + livenessProbe: + enabled: true + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 1 + failureThreshold: 3 + successThreshold: 1 + + ## @param server.readinessProbe.enabled Enable readinessProbe on the server + ## @param server.readinessProbe.initialDelaySeconds Initial delay seconds for readinessProbe + ## @param server.readinessProbe.periodSeconds Period seconds for readinessProbe + ## @param server.readinessProbe.timeoutSeconds Timeout seconds for readinessProbe + ## @param server.readinessProbe.failureThreshold Failure threshold for readinessProbe + ## @param server.readinessProbe.successThreshold Success threshold for readinessProbe + ## + readinessProbe: + enabled: true + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 1 + failureThreshold: 3 + successThreshold: 1 + + ## Server app resource requests and limits + ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ + ## We usually recommend not to specify default resources and to leave this as a conscious + ## choice for the user. This also increases chances charts run on environments with little + ## resources, such as Minikube. If you do want to specify resources, uncomment the following + ## lines, adjust them as necessary, and remove the curly braces after 'resources:'. + ## @param server.resources.limits [object] The resources limits for the server container + ## @param server.resources.requests [object] The requested resources for the server container + resources: + ## Example: + ## limits: + ## cpu: 200m + ## memory: 1Gi + limits: {} + ## Examples: + ## requests: + ## memory: 256Mi + ## cpu: 250m + requests: {} + + ## @param server.service.type The service type to use for the API server + ## @param server.service.port The service port to expose the API server on + service: + type: ClusterIP + port: 8001 + + ## @param server.persistence.accessMode The access mode for the airbyte server pvc + ## @param server.persistence.size The size of the pvc to use for the airbyte server pvc + persistence: + size: 1Gi + accessMode: ReadWriteOnce + ## @param server.persistence.storageClass The storage class to use for the airbyte server pvc + ## If defined, storageClassName: + ## If set to "-", storageClassName: "", which disables dynamic provisioning + ## If undefined (the default) or set to null, no storageClassName spec is + ## set, choosing the default provisioner. (gp2 on AWS, standard on + ## GKE, AWS & OpenStack) + ## + storageClass: "" + + ## @param server.nodeSelector [object] Node labels for pod assignment + ## Ref: https://kubernetes.io/docs/user-guide/node-selection/ + ## + nodeSelector: {} + + ## @param server.tolerations [array] Tolerations for server pod assignment. + ## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + ## + tolerations: [] + +## @section Temporal parameters +## TODO: Move to consuming temporal from a dedicated helm chart + +temporal: + ## @param temporal.replicaCount The number of temporal replicas to deploy + replicaCount: 1 + + ## @param temporal.image.repository The temporal image repository to use + ## @param temporal.image.pullPolicy The pull policy for the temporal image + ## @param temporal.image.tag The temporal image tag to use + image: + repository: temporalio/auto-setup + pullPolicy: IfNotPresent + tag: "1.7.0" + + ## @param temporal.service.type The Kubernetes Service Type + ## @param temporal.service.port The temporal port and exposed kubernetes port + service: + type: ClusterIP + port: 7233 + + ## @param temporal.nodeSelector [object] Node labels for pod assignment + ## Ref: https://kubernetes.io/docs/user-guide/node-selection/ + ## + nodeSelector: {} + + ## @param temporal.tolerations [array] Tolerations for pod assignment. + ## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + ## + tolerations: [] + + +## @section Airbyte Database parameters + +## PostgreSQL chart configuration +## ref: https://github.com/bitnami/charts/blob/master/bitnami/postgresql/values.yaml +## @param postgresql.enabled Switch to enable or disable the PostgreSQL helm chart +## @param postgresql.postgresqlUsername Airbyte Postgresql username +## @param postgresql.postgresqlPassword Airbyte Postgresql password +## @param postgresql.postgresqlDatabase Airbyte Postgresql database +## @param postgresql.existingSecret Name of an existing secret containing the PostgreSQL password ('postgresql-password' key) +## +postgresql: + enabled: true + postgresqlUsername: airbyte + postgresqlPassword: airbyte + postgresqlDatabase: db-airbyte + ## This secret is used in case of postgresql.enabled=true and we would like to specify password for newly created postgresql instance + ## + existingSecret: "" + + +## External PostgreSQL configuration +## All of these values are only used when postgresql.enabled is set to false +## @param externalDatabase.host Database host +## @param externalDatabase.user non-root Username for Airbyte Database +## @param externalDatabase.password Database password +## @param externalDatabase.existingSecret Name of an existing secret resource containing the DB password +## @param externalDatabase.existingSecretPasswordKey Name of an existing secret key containing the DB password +## @param externalDatabase.database Database name +## @param externalDatabase.port Database port number +## +externalDatabase: + host: localhost + user: airbyte + password: "" + existingSecret: "" + existingSecretPasswordKey: "" + database: db-airbyte + port: 5432 + +## @section Minio parameters + +## Minio chart configuration +## ref: https://github.com/bitnami/charts/blob/master/bitnami/minio/values.yaml +## @param minio.enabled Switch to enable or disable the Minio helm chart +## @param minio.accessKey.password Minio Access Key +## @param minio.secretKey.password Minio Secret Key +minio: + enabled: true + accessKey: + password: minio + secretKey: + password: minio123 + +## External Minio configuration +## All of these values are only used when minio.enabled is set to false +## @param externalMinio.host Minio Host +## @param externalMinio.port Minio Port +externalMinio: + host: localhost + port: 9000