diff --git a/charts/airbyte-metrics/.gitignore b/charts/airbyte-metrics/.gitignore new file mode 100644 index 000000000000..88e91e8a8f34 --- /dev/null +++ b/charts/airbyte-metrics/.gitignore @@ -0,0 +1,2 @@ +# Charts are downloaded at install time with `helm dep build`. +charts diff --git a/charts/airbyte-metrics/.helmignore b/charts/airbyte-metrics/.helmignore new file mode 100644 index 000000000000..f885da3fd491 --- /dev/null +++ b/charts/airbyte-metrics/.helmignore @@ -0,0 +1,25 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ + +ci.sh diff --git a/charts/airbyte-metrics/Chart.lock b/charts/airbyte-metrics/Chart.lock new file mode 100644 index 000000000000..017faffa076b --- /dev/null +++ b/charts/airbyte-metrics/Chart.lock @@ -0,0 +1,6 @@ +dependencies: +- name: common + repository: https://charts.bitnami.com/bitnami + version: 1.17.1 +digest: sha256:dacc73770a5640c011e067ff8840ddf89631fc19016c8d0a9e5ea160e7da8690 +generated: "2022-08-31T12:09:18.473209+01:00" diff --git a/charts/airbyte-metrics/Chart.yaml b/charts/airbyte-metrics/Chart.yaml new file mode 100644 index 000000000000..1acc08fea938 --- /dev/null +++ b/charts/airbyte-metrics/Chart.yaml @@ -0,0 +1,31 @@ +apiVersion: v2 +name: metrics +description: Helm chart to deploy airbyte-metrics + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: "0.39.36" + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +# It is recommended to use it with quotes. +appVersion: "0.40.3" + +dependencies: + - name: common + repository: https://charts.bitnami.com/bitnami + tags: + - bitnami-common + version: 1.x.x diff --git a/charts/airbyte-metrics/README.md b/charts/airbyte-metrics/README.md new file mode 100644 index 000000000000..952cdc3c508f --- /dev/null +++ b/charts/airbyte-metrics/README.md @@ -0,0 +1,43 @@ +# metrics + +![Version: 0.39.36](https://img.shields.io/badge/Version-0.39.36-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.40.3](https://img.shields.io/badge/AppVersion-0.40.3-informational?style=flat-square) + +Helm chart to deploy airbyte-metrics + +## Requirements + +| Repository | Name | Version | +|------------|------|---------| +| https://charts.bitnami.com/bitnami | common | 1.x.x | + +## Values + +| Key | Type | Default | Description | +|-----|------|---------|-------------| +| affinity | object | `{}` | | +| containerSecurityContext | object | `{}` | | +| enabled | bool | `true` | | +| env_vars | object | `{}` | | +| extraContainers | list | `[]` | | +| extraEnv | list | `[]` | | +| extraVolumeMounts | list | `[]` | | +| extraVolumes | list | `[]` | | +| global.database.host | string | `"example.com"` | | +| global.database.port | string | `"5432"` | | +| global.database.secretName | string | `""` | | +| global.database.secretValue | string | `""` | | +| global.deploymentMode | string | `"oss"` | | +| global.extraContainers | list | `[]` | | +| global.serviceAccountName | string | `"placeholderServiceAccount"` | | +| image.pullPolicy | string | `"IfNotPresent"` | | +| image.repository | string | `"airbyte/metrics-reporter"` | | +| nodeSelector | object | `{}` | | +| podAnnotations | object | `{}` | | +| replicaCount | int | `1` | | +| resources.limits | object | `{}` | | +| resources.requests | object | `{}` | | +| secrets | object | `{}` | | +| tolerations | list | `[]` | | + +---------------------------------------------- +Autogenerated from chart metadata using [helm-docs v1.11.0](https://github.com/norwoodj/helm-docs/releases/v1.11.0) diff --git a/charts/airbyte-metrics/templates/_helpers.tpl b/charts/airbyte-metrics/templates/_helpers.tpl new file mode 100644 index 000000000000..e6cc9d05cc14 --- /dev/null +++ b/charts/airbyte-metrics/templates/_helpers.tpl @@ -0,0 +1,39 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "airbyte.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "airbyte.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "airbyte.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Define db secret +*/}} + +{{- define "database.secret.name" -}} +{{- printf "%s-postgresql" .Release.Name }} +{{- end }} diff --git a/charts/airbyte-metrics/templates/deployment.yaml b/charts/airbyte-metrics/templates/deployment.yaml new file mode 100644 index 000000000000..1fe4e8132484 --- /dev/null +++ b/charts/airbyte-metrics/templates/deployment.yaml @@ -0,0 +1,130 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "common.names.fullname" . }} + labels: + app.kubernetes.io/name: {{ include "airbyte.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + helm.sh/chart: {{ include "airbyte.chart" . }} + app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.airbyte.io/fullname: {{ include "airbyte.fullname" . }} +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + airbyte: metrics + strategy: + type: Recreate # Needed due to volume claims + template: + metadata: + labels: + airbyte: metrics + {{- if .Values.podAnnotations }} + annotations: + {{- include "common.tplvalues.render" (dict "value" .Values.podAnnotations "context" $) | nindent 8 }} + {{- end }} + spec: + serviceAccountName: {{ .Values.global.serviceAccountName }} + {{- if .Values.global.imagePullSecrets }} + imagePullSecrets: + {{- range .Values.global.imagePullSecrets }} + {{- printf "- name: %s" .name | nindent 2 }} + {{- end }} + {{- end }} + {{- if .Values.nodeSelector }} + nodeSelector: {{- include "common.tplvalues.render" (dict "value" .Values.nodeSelector "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.tolerations }} + tolerations: {{- include "common.tplvalues.render" (dict "value" .Values.tolerations "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.affinity }} + affinity: {{- include "common.tplvalues.render" (dict "value" .Values.affinity "context" $) | nindent 8 }} + {{- end }} + containers: + - name: airbyte-metrics-container + image: {{ printf "%s:%s" .Values.image.repository (default .Chart.AppVersion .Values.image.tag) }} + imagePullPolicy: "{{ .Values.image.pullPolicy }}" + env: + {{- if eq .Values.global.deploymentMode "oss" }} + - name: AIRBYTE_VERSION + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: AIRBYTE_VERSION + - name: DATABASE_PASSWORD + valueFrom: + secretKeyRef: + name: {{ .Values.global.database.secretName | default (include "airbyte.chart" .) }} + key: {{ .Values.global.database.secretValue | default "postgresql-password" }} + - name: DATABASE_URL + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: DATABASE_URL + - name: DATABASE_USER + valueFrom: + secretKeyRef: + name: {{ .Release.Name }}-airbyte-secrets + key: DATABASE_USER + - name: CONFIGS_DATABASE_MINIMUM_FLYWAY_MIGRATION_VERSION + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: CONFIGS_DATABASE_MINIMUM_FLYWAY_MIGRATION_VERSION + - name: METRIC_CLIENT + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: METRIC_CLIENT + - name: OTEL_COLLECTOR_ENDPOINT + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: OTEL_COLLECTOR_ENDPOINT + {{- end }} + # Values from secret + {{- if .Values.secrets }} + {{- range $k, $v := .Values.secrets }} + - name: {{ $k }} + valueFrom: + secretKeyRef: + name: metrics-secrets + key: {{ $k }} + {{- end }} + {{- end }} + + # Values from env + {{- if .Values.env_vars }} + {{- range $k, $v := mergeOverwrite .Values.env_vars .Values.global.env_vars }} + - name: {{ $k }} + value: {{ $v | quote }} + {{- end }} + {{- end }} + + # Values from extraEnv for more compability(if you want to use external secret source or other stuff) + {{- if .Values.extraEnv }} + {{- toYaml .Values.extraEnv | nindent 8 }} + {{- end }} + + {{- if .Values.resources }} + resources: {{- toYaml .Values.resources | nindent 10 }} + {{- end }} + {{- if .Values.containerSecurityContext }} + securityContext: {{- toYaml .Values.containerSecurityContext | nindent 10 }} + {{- end }} + volumeMounts: + {{- if .Values.extraVolumeMounts }} + {{ toYaml .Values.extraVolumeMounts | nindent 8 }} + {{- end }} + {{- if .Values.extraContainers }} + {{ toYaml .Values.extraContainers | indent 8 }} + {{- end }} + {{- if .Values.global.extraContainers }} + {{ toYaml .Values.global.extraContainers | indent 8 }} + {{- end }} + volumes: + {{- if .Values.extraVolumes }} +{{ toYaml .Values.extraVolumes | nindent 6 }} + {{- end }} diff --git a/charts/airbyte-metrics/values.yaml b/charts/airbyte-metrics/values.yaml new file mode 100644 index 000000000000..7d79225b9de3 --- /dev/null +++ b/charts/airbyte-metrics/values.yaml @@ -0,0 +1,98 @@ + +global: + serviceAccountName: placeholderServiceAccount + deploymentMode: oss + extraContainers: [] + database: + secretName: "" + secretValue: "" + host: "example.com" + port: "5432" + +enabled: true +## @param metrics.replicaCount Number of metrics-reporter replicas +replicaCount: 1 + +## @param metrics.image.repository The repository to use for the airbyte metrics-reporter image. +## @param metrics.image.pullPolicy the pull policy to use for the airbyte metrics-reporter image +## @param metrics.image.tag The airbyte metrics-reporter image tag. Defaults to the chart's AppVersion +image: + repository: airbyte/metrics-reporter + pullPolicy: IfNotPresent + +## @param metrics.podAnnotations [object] Add extra annotations to the metrics-reporter pod +## +podAnnotations: {} + +## @param metrics.containerSecurityContext Security context for the container +## Examples: +## containerSecurityContext: +## runAsNonRoot: true +## runAsUser: 1000 +## readOnlyRootFilesystem: true +containerSecurityContext: {} + +## metrics-reporter app resource requests and limits +## ref: http://kubernetes.io/docs/user-guide/compute-resources/ +## We usually recommend not to specify default resources and to leave this as a conscious +## choice for the user. This also increases chances charts run on environments with little +## resources, such as Minikube. If you do want to specify resources, uncomment the following +## lines, adjust them as necessary, and remove the curly braces after 'resources:'. +## @param metrics.resources.limits [object] The resources limits for the metrics-reporter container +## @param metrics.resources.requests [object] The requested resources for the metrics-reporter container +resources: + ## Example: + ## limits: + ## cpu: 200m + ## memory: 1Gi + limits: {} + ## Examples: + ## requests: + ## memory: 256Mi + ## cpu: 250m + requests: {} + +## @param metrics.nodeSelector [object] Node labels for pod assignment +## Ref: https://kubernetes.io/docs/user-guide/node-selection/ +## +nodeSelector: {} + +## @param metrics.tolerations [array] Tolerations for metrics-reporter pod assignment. +## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ +## +tolerations: [] + +## @param metrics.affinity [object] Affinity and anti-affinity for metrics-reporter pod assignment. +## ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity +## +affinity: {} + +## @param metrics.extraEnv [array] Additional env vars for metrics-reporter pod(s). +## Example: +## +## extraEnv: +## - name: SAMPLE_ENV_VAR +## value: "key=sample-value" +extraEnv: [] + +## @param metrics.extraVolumeMounts [array] Additional volumeMounts for metrics-reporter container(s). +## Examples (when using `metrics.containerSecurityContext.readOnlyRootFilesystem=true`): +## extraVolumeMounts: +## - name: tmpdir +## mountPath: /tmp +## +extraVolumeMounts: [] + +## @param metrics.extraVolumes [array] Additional volumes for metrics-reporter pod(s). +## Examples (when using `metrics.containerSecurityContext.readOnlyRootFilesystem=true`): +## extraVolumes: +## - name: tmpdir +## emptyDir: {} +## +extraVolumes: [] + +extraContainers: [] + +secrets: {} + +env_vars: {} diff --git a/charts/airbyte/Chart.yaml b/charts/airbyte/Chart.yaml index 4fcb003ca1eb..80b6ed7eab51 100644 --- a/charts/airbyte/Chart.yaml +++ b/charts/airbyte/Chart.yaml @@ -61,3 +61,7 @@ dependencies: name: pod-sweeper repository: "https://airbytehq.github.io/helm-charts/" version: placeholder + - condition: metrics.enabled + name: metrics + repository: "https://airbytehq.github.io/helm-charts/" + version: placeholder diff --git a/charts/airbyte/templates/env-configmap.yaml b/charts/airbyte/templates/env-configmap.yaml index 1384297e2ad9..aaaa82bf2c6b 100644 --- a/charts/airbyte/templates/env-configmap.yaml +++ b/charts/airbyte/templates/env-configmap.yaml @@ -52,8 +52,8 @@ data: WORKER_ENVIRONMENT: kubernetes WORKSPACE_DOCKER_MOUNT: airbyte_workspace WORKSPACE_ROOT: /workspace - METRIC_CLIENT: "" - OTEL_COLLECTOR_ENDPOINT: "" + METRIC_CLIENT: {{ .Values.global.metrics.metricClient | default "" | quote }} + OTEL_COLLECTOR_ENDPOINT: {{ .Values.global.metrics.otelCollectorEndpoint | default "" | quote }} ACTIVITY_MAX_ATTEMPT: "" ACTIVITY_INITIAL_DELAY_BETWEEN_ATTEMPTS_SECONDS: "" ACTIVITY_MAX_DELAY_BETWEEN_ATTEMPTS_SECONDS: "" diff --git a/charts/airbyte/values.yaml b/charts/airbyte/values.yaml index 66243bb7d2b5..c423f292f889 100644 --- a/charts/airbyte/values.yaml +++ b/charts/airbyte/values.yaml @@ -59,6 +59,13 @@ global: credentials: "" # If credentialsJson is set then credentials auto resolves (to /secrets/gcs-log-creds/gcp.json) credentialsJson: "" + metrics: + ## These parameters are used in the airbyte-env ConfigMap, which is then mounted in deployments in the airbyte-worker and airbyte-metrics Charts. + ## @param global.metrics.metricClient The metric client to configure globally. Supports "otel" + metricClient: "" + + ## @param global.metrics.otelCollectorEndpoint The open-telemetry-collector endpoint that metrics will be sent to. + otelCollectorEndpoint: "" jobs: ## Jobs resource requests and limits ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ @@ -653,6 +660,97 @@ worker: hpa: enabled: false +## @section Metrics parameters +metrics: + enabled: false + + ## @param metrics.replicaCount Number of metrics-reporter replicas + replicaCount: 1 + + ## @param metrics.image.repository The repository to use for the airbyte metrics-reporter image. + ## @param metrics.image.pullPolicy the pull policy to use for the airbyte metrics-reporter image + ## @param metrics.image.tag The airbyte metrics-reporter image tag. Defaults to the chart's AppVersion + image: + repository: airbyte/metrics-reporter + pullPolicy: IfNotPresent + + ## @param metrics.podAnnotations [object] Add extra annotations to the metrics-reporter pod + ## + podAnnotations: {} + + ## @param metrics.containerSecurityContext Security context for the container + ## Examples: + ## containerSecurityContext: + ## runAsNonRoot: true + ## runAsUser: 1000 + ## readOnlyRootFilesystem: true + containerSecurityContext: {} + + ## metrics-reporter app resource requests and limits + ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ + ## We usually recommend not to specify default resources and to leave this as a conscious + ## choice for the user. This also increases chances charts run on environments with little + ## resources, such as Minikube. If you do want to specify resources, uncomment the following + ## lines, adjust them as necessary, and remove the curly braces after 'resources:'. + ## @param metrics.resources.limits [object] The resources limits for the metrics-reporter container + ## @param metrics.resources.requests [object] The requested resources for the metrics-reporter container + resources: + ## Example: + ## limits: + ## cpu: 200m + ## memory: 1Gi + limits: {} + ## Examples: + ## requests: + ## memory: 256Mi + ## cpu: 250m + requests: {} + + ## @param metrics.nodeSelector [object] Node labels for pod assignment + ## Ref: https://kubernetes.io/docs/user-guide/node-selection/ + ## + nodeSelector: {} + + ## @param metrics.tolerations [array] Tolerations for metrics-reporter pod assignment. + ## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + ## + tolerations: [] + + ## @param metrics.affinity [object] Affinity and anti-affinity for metrics-reporter pod assignment. + ## ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity + ## + affinity: {} + + ## @param metrics.extraEnv [array] Additional env vars for metrics-reporter pod(s). + ## Example: + ## + ## extraEnv: + ## - name: SAMPLE_ENV_VAR + ## value: "key=sample-value" + extraEnv: [] + + ## @param metrics.extraVolumeMounts [array] Additional volumeMounts for metrics-reporter container(s). + ## Examples (when using `metrics.containerSecurityContext.readOnlyRootFilesystem=true`): + ## extraVolumeMounts: + ## - name: tmpdir + ## mountPath: /tmp + ## + extraVolumeMounts: [] + + ## @param metrics.extraVolumes [array] Additional volumes for metrics-reporter pod(s). + ## Examples (when using `metrics.containerSecurityContext.readOnlyRootFilesystem=true`): + ## extraVolumes: + ## - name: tmpdir + ## emptyDir: {} + ## + extraVolumes: [] + + extraContainers: [] + + secrets: {} + + env_vars: {} + ## @section Bootloader Parameters airbyte-bootloader: