Add connector builder server to airbyte proxy, kube overlays, and helm charts

.env

@@ -61,6 +61,7 @@ CONNECTOR_BUILDER_API_HOST=airbyte-connector-builder-server:80
 WEBAPP_URL=http://localhost:8000/
 # Although not present as an env var, required for webapp configuration.
 API_URL=/api/v1/
+CONNECTOR_BUILDER_API_URL=/connector-builder-api
 
 ### JOBS ###
 # Relevant to scaling.

.github/workflows/publish-helm-charts.yml

@@ -70,7 +70,7 @@ jobs:
       - name: "Helm package"
         shell: bash
         run: |
-          declare -a StringArray=("airbyte-bootloader" "airbyte-server" "airbyte-temporal" "airbyte-webapp" "airbyte-pod-sweeper" "airbyte-worker" "airbyte-metrics" "airbyte-cron")
+          declare -a StringArray=("airbyte-bootloader" "airbyte-server" "airbyte-temporal" "airbyte-webapp" "airbyte-pod-sweeper" "airbyte-worker" "airbyte-metrics" "airbyte-cron" "airbyte-connector-builder-server")
           for val in ${StringArray[@]}; do
             cd ./airbyte/charts/${val} && helm dep update && cd $GITHUB_WORKSPACE
             sed -i -E  's/version: \"[0-9]+\.[0-9]+\.[0-9]+\"/version: \"${{ needs.generate-semantic-version.outputs.next-version }}\"/' ./airbyte/charts/${val}/Chart.yaml

airbyte-proxy/Dockerfile

@@ -9,9 +9,10 @@ ENV VERSION ${VERSION}
 
 RUN apt-get update -y && apt-get install -y apache2-utils && rm -rf /var/lib/apt/lists/*
 
-# This variable can be used to update the destintion containers that Nginx proxies to.
+# This variable can be used to update the destination containers that Nginx proxies to.
 ENV PROXY_PASS_WEB "http://airbyte-webapp:80"
 ENV PROXY_PASS_API "http://airbyte-server:8001"
+ENV CONNECTOR_BUILDER_SERVER_API "http://airbyte-connector-builder-server:80"
 
 # Nginx config file
 WORKDIR /

airbyte-proxy/nginx-auth.conf.template

@@ -42,4 +42,25 @@ http {
       }
     }
   }
+
+  server {
+    listen 8003;
+
+    location / {
+      proxy_set_header Host $host;
+      proxy_set_header X-Real-IP $remote_addr;
+      proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+
+      auth_basic "Welcome to Airbyte";
+      auth_basic_user_file /etc/nginx/.htpasswd;
+
+      proxy_pass "${CONNECTOR_BUILDER_SERVER_API}";
+
+      error_page 401 /etc/nginx/401.html;
+      location ~ (401.html)$ {
+        alias /etc/nginx/$1;
+        auth_basic off;
+      }
+    }
+  }
 }

airbyte-proxy/nginx-no-auth.conf.template

@@ -24,4 +24,16 @@ http {
       proxy_pass "${PROXY_PASS_API}";
     }
   }
+
+  server {
+    listen 8003;
+
+    location / {
+      proxy_set_header Host $host;
+      proxy_set_header X-Real-IP $remote_addr;
+      proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+
+      proxy_pass "${CONNECTOR_BUILDER_SERVER_API}";
+    }
+  }
 }

airbyte-proxy/run.sh

@@ -16,7 +16,7 @@ else
   TEMPLATE_PATH="/etc/nginx/templates/nginx-auth.conf.template"
 fi
 
-envsubst '${PROXY_PASS_WEB} ${PROXY_PASS_API} ${PROXY_PASS_RESOLVER}' < $TEMPLATE_PATH > /etc/nginx/nginx.conf
+envsubst '${PROXY_PASS_WEB} ${PROXY_PASS_API} ${CONNECTOR_BUILDER_SERVER_API} ${PROXY_PASS_RESOLVER}' < $TEMPLATE_PATH > /etc/nginx/nginx.conf
 
 echo "starting nginx..."
 nginx -v

airbyte-proxy/test.sh

@@ -11,7 +11,7 @@ VERSION="${VERSION:-dev}" # defaults to "dev", otherwise it is set by environmen
 echo "testing with proxy container airbyte/proxy:$VERSION"
 
 function start_container () {
-  CMD="docker run -d -p $PORT:8000 --env BASIC_AUTH_USERNAME=$1 --env BASIC_AUTH_PASSWORD=$2 --env PROXY_PASS_WEB=http://localhost --env PROXY_PASS_API=http://localhost --name $NAME airbyte/proxy:$VERSION"
+  CMD="docker run -d -p $PORT:8000 --env BASIC_AUTH_USERNAME=$1 --env BASIC_AUTH_PASSWORD=$2 --env PROXY_PASS_WEB=http://localhost --env PROXY_PASS_API=http://localhost --env CONNECTOR_BUILDER_SERVER_API=http://localhost --name $NAME airbyte/proxy:$VERSION"
   echo $CMD
   eval $CMD
   wait_for_docker;

airbyte-webapp/.env

@@ -3,4 +3,3 @@ REACT_APP_FULL_STORY_ORG=13AXQ4
 REACT_APP_SENTRY_DSN=
 REACT_APP_INTERCOM_APP_ID=nj1oam7s
 REACT_APP_OSANO=16A0CTTE7vE8m1Qif/67beec9b-e563-4736-bdb4-4fe4adc39d48
-REACT_APP_CONNECTOR_BUILDER_API=/connector-builder-api/

airbyte-webapp/nginx/default.conf.template

@@ -25,6 +25,7 @@ server {
                 window.TRACKING_STRATEGY = "$TRACKING_STRATEGY";
                 window.AIRBYTE_VERSION = "$AIRBYTE_VERSION";
                 window.API_URL = "$API_URL";
+                window.CONNECTOR_BUILDER_API_URL = "$CONNECTOR_BUILDER_API_URL";
                 </script>';
         sub_filter_once on;
     }

airbyte-webapp/src/config/configProviders.ts

@@ -11,6 +11,7 @@ const windowConfigProvider: ConfigProvider = async () => {
       token: window.SEGMENT_TOKEN,
     },
     apiUrl: window.API_URL,
+    connectorBuilderApiUrl: window.CONNECTOR_BUILDER_API_URL,
     version: window.AIRBYTE_VERSION,
     // cloud only start
     // TODO: remove when infra team supports proper webapp building
@@ -22,7 +23,6 @@ const windowConfigProvider: ConfigProvider = async () => {
 const envConfigProvider: ConfigProvider = async () => {
   return {
     apiUrl: process.env.REACT_APP_API_URL,
-    connectorBuilderUrl: process.env.REACT_APP_CONNECTOR_BUILDER_API,
     integrationUrl: process.env.REACT_APP_INTEGRATION_DOCS_URLS,
     segment: {
       token: process.env.REACT_APP_SEGMENT_TOKEN,

airbyte-webapp/src/config/types.ts

@@ -3,6 +3,7 @@ declare global {
     TRACKING_STRATEGY?: string;
     AIRBYTE_VERSION?: string;
     API_URL?: string;
+    CONNECTOR_BUILDER_API_URL?: string;
     CLOUD?: string;
     REACT_APP_DATADOG_APPLICATION_ID: string;
     REACT_APP_DATADOG_CLIENT_TOKEN: string;

charts/airbyte-connector-builder-server/.gitignore

@@ -0,0 +1,2 @@
+# Charts are downloaded at install time with `helm dep build`.
+charts

charts/airbyte-connector-builder-server/Chart.lock

@@ -0,0 +1,6 @@
+dependencies:
+- name: common
+  repository: https://charts.bitnami.com/bitnami
+  version: 1.17.1
+digest: sha256:dacc73770a5640c011e067ff8840ddf89631fc19016c8d0a9e5ea160e7da8690
+generated: "2022-10-17T18:35:15.123937677Z"

charts/airbyte-connector-builder-server/Chart.yaml

@@ -0,0 +1,31 @@
+apiVersion: v2
+name: connector-builder-server
+description: Helm chart to deploy airbyte-connector-builder-server
+
+# A chart can be either an 'application' or a 'library' chart.
+#
+# Application charts are a collection of templates that can be packaged into versioned archives
+# to be deployed.
+#
+# Library charts provide useful utilities or functions for the chart developer. They're included as
+# a dependency of application charts to inject those utilities and functions into the rendering
+# pipeline. Library charts do not define any templates and therefore cannot be deployed.
+type: application
+
+# This is the chart version. This version number should be incremented each time you make changes
+# to the chart and its templates, including the app version.
+# Versions are expected to follow Semantic Versioning (https://semver.org/)
+version: "0.40.46"
+
+# This is the version number of the application being deployed. This version number should be
+# incremented each time you make changes to the application. Versions are not expected to
+# follow Semantic Versioning. They should reflect the version the application is using.
+# It is recommended to use it with quotes.
+appVersion: "0.40.21"
+
+dependencies:
+  - name: common
+    repository: https://charts.bitnami.com/bitnami
+    tags:
+      - bitnami-common
+    version: 1.x.x

charts/airbyte-connector-builder-server/README.md

@@ -0,0 +1,79 @@
+# server
+
+![Version: 0.39.36](https://img.shields.io/badge/Version-0.39.36-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.40.4](https://img.shields.io/badge/AppVersion-0.40.4-informational?style=flat-square)
+
+Helm chart to deploy airbyte-server
+
+## Requirements
+
+| Repository | Name | Version |
+|------------|------|---------|
+| https://charts.bitnami.com/bitnami | common | 1.x.x |
+
+## Values
+
+| Key | Type | Default | Description |
+|-----|------|---------|-------------|
+| affinity | object | `{}` |  |
+| containerSecurityContext | object | `{}` |  |
+| enabled | bool | `true` |  |
+| env_vars | object | `{}` |  |
+| extraContainers | list | `[]` |  |
+| extraEnv | list | `[]` |  |
+| extraInitContainers | list | `[]` |  |
+| extraVolumeMounts | list | `[]` |  |
+| extraVolumes | list | `[]` |  |
+| global.configMapName | string | `""` |  |
+| global.credVolumeOverride | string | `""` |  |
+| global.database.host | string | `"example.com"` |  |
+| global.database.port | string | `"5432"` |  |
+| global.database.secretName | string | `""` |  |
+| global.database.secretValue | string | `""` |  |
+| global.deploymentMode | string | `"oss"` |  |
+| global.extraContainers | list | `[]` |  |
+| global.logs.accessKey.existingSecret | string | `""` |  |
+| global.logs.accessKey.existingSecretKey | string | `""` |  |
+| global.logs.accessKey.password | string | `"minio"` |  |
+| global.logs.externalMinio.enabled | bool | `false` |  |
+| global.logs.externalMinio.host | string | `"localhost"` |  |
+| global.logs.externalMinio.port | int | `9000` |  |
+| global.logs.gcs.bucket | string | `""` |  |
+| global.logs.gcs.credentials | string | `""` |  |
+| global.logs.gcs.credentialsJson | string | `""` |  |
+| global.logs.minio.enabled | bool | `true` |  |
+| global.logs.s3.bucket | string | `"airbyte-dev-logs"` |  |
+| global.logs.s3.bucketRegion | string | `""` |  |
+| global.logs.s3.enabled | bool | `false` |  |
+| global.logs.secretKey.existingSecret | string | `""` |  |
+| global.logs.secretKey.existingSecretKey | string | `""` |  |
+| global.logs.secretKey.password | string | `"minio123"` |  |
+| global.secretName | string | `""` |  |
+| global.serviceAccountName | string | `"placeholderServiceAccount"` |  |
+| image.pullPolicy | string | `"IfNotPresent"` |  |
+| image.repository | string | `"airbyte/server"` |  |
+| livenessProbe.enabled | bool | `true` |  |
+| livenessProbe.failureThreshold | int | `3` |  |
+| livenessProbe.initialDelaySeconds | int | `30` |  |
+| livenessProbe.periodSeconds | int | `10` |  |
+| livenessProbe.successThreshold | int | `1` |  |
+| livenessProbe.timeoutSeconds | int | `1` |  |
+| log.level | string | `"INFO"` |  |
+| nodeSelector | object | `{}` |  |
+| podAnnotations | object | `{}` |  |
+| readinessProbe.enabled | bool | `true` |  |
+| readinessProbe.failureThreshold | int | `3` |  |
+| readinessProbe.initialDelaySeconds | int | `10` |  |
+| readinessProbe.periodSeconds | int | `10` |  |
+| readinessProbe.successThreshold | int | `1` |  |
+| readinessProbe.timeoutSeconds | int | `1` |  |
+| replicaCount | int | `1` |  |
+| resources.limits | object | `{}` |  |
+| resources.requests | object | `{}` |  |
+| secrets | object | `{}` |  |
+| service.annotations | object | `{}` |  |
+| service.port | int | `8001` |  |
+| service.type | string | `"ClusterIP"` |  |
+| tolerations | list | `[]` |  |
+
+----------------------------------------------
+Autogenerated from chart metadata using [helm-docs v1.11.0](https://github.com/norwoodj/helm-docs/releases/v1.11.0)

charts/airbyte-connector-builder-server/templates/_helpers.tpl

@@ -0,0 +1,73 @@
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "airbyte.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "airbyte.fullname" -}}
+{{- if .Values.fullnameOverride }}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- $name := default .Chart.Name .Values.nameOverride }}
+{{- if contains $name .Release.Name }}
+{{- .Release.Name | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
+{{- end }}
+{{- end }}
+{{- end }}
+
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "airbyte.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Common labels
+*/}}
+{{- define "airbyte.labels" -}}
+helm.sh/chart: {{ include "airbyte.chart" . }}
+{{ include "airbyte.selectorLabels" . }}
+{{- if .Chart.AppVersion }}
+app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+{{- end }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- end }}
+
+{{/*
+Selector labels
+*/}}
+{{- define "airbyte.selectorLabels" -}}
+app.kubernetes.io/name: {{ include "airbyte.name" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end }}
+
+{{/*
+Define db secret
+*/}}
+
+{{- define "database.secret.name" -}}
+{{- printf "%s-postgresql" .Release.Name }}
+{{- end }}
+
+{{/* 
+Define imageTag
+*/}}
+
+{{- define "connectorBuilderServer.imageTag" -}}
+{{- if .Values.image.tag }}
+    {{- printf "%s" .Values.image.tag }}
+{{- else if ((.Values.global.image).tag) }}
+    {{- printf "%s" .Values.global.image.tag }}
+{{- else }}
+    {{- printf "%s" .Chart.AppVersion }}
+{{- end }}
+{{- end }}