Merge pull request feast-dev#10 from farfetch-test/ben_engin/helmsman…

…_automation

Ben engin/helmsman automation

.github/workflows/deploy-to-aks.yaml

@@ -0,0 +1,24 @@
+name: deploy-to-aks
+
+on:
+  push:
+    branches:
+    - master
+    paths:
+    - 'infra/charts/**'
+    - 'infra/desired-state/**'
+    - 'infra/scripts/deploy-to-aks.sh'
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    name: deploy
+    steps:
+      - uses: actions/checkout@v1
+
+      - uses: azure/login@v1
+        with:
+          creds: ${{ secrets.AZURE_CREDENTIALS }}
+
+      - name: deploy
+        run: ./infra/scripts/deploy-to-aks.sh

.gitignore

@@ -193,4 +193,5 @@ pom-ff-deploy.xml
 secrets/
 
 # Dumping ground
-tmp/
+tmp/
+.helmsman-tmp/

infra/desired-state/chart-values/feast-kafka.yaml

@@ -0,0 +1,14 @@
+heapOpts: -Xmx1024m -Xms1024m
+persistence:
+  enabled: false
+resources:
+  limits:
+    cpu: 1200m
+    memory: 1280Mi
+  requests:
+    cpu: 50m
+    memory: 1024Mi
+zookeeper:
+  enabled: false
+externalZookeeper:
+  servers: feast-zookeeper:2181

infra/desired-state/chart-values/feast-redis-jobstore.yaml

@@ -0,0 +1,32 @@
+cluster:
+  enabled: false
+sentinel:
+  enabled: false
+usePassword: false
+master:
+  resources:
+    limits:
+      cpu: 2
+      memory: 300Mi
+    requests:
+      cpu: 100m
+      memory: 220Mi
+  persistence:
+    enabled: false
+metrics:
+  enabled: true
+  resources:
+    limits:
+      cpu: 100m
+      memory: 64Mi
+    requests:
+      cpu: 25m
+      memory: 32Mi
+sysctlImage:
+  enabled: true
+  mountHostSys: true
+  command:
+    - /bin/sh
+    - -c
+    - |-
+      echo never > /host-sys/kernel/mm/transparent_hugepage/enabled

infra/desired-state/chart-values/feast-zookeeper.yaml

@@ -0,0 +1,10 @@
+heapSize: 256
+persistence:
+  enabled: false
+resources:
+  limits:
+    cpu: 100m
+    memory: 512Mi
+  requests:
+    cpu: 10m
+    memory: 256Mi

infra/desired-state/chart-values/feast.yaml

@@ -0,0 +1,81 @@
+kafka:
+  enabled: false
+redis:
+  enabled: false
+grafana:
+  enabled: true
+prometheus-statsd-exporter:
+  enabled: true
+prometheus:
+  enabled: true
+
+feast-core:
+  enabled:
+    true
+  prometheus:
+    enabled:
+      true
+  gcpServiceAccount:
+    enabled: true
+    existingSecret:
+      name: feast-gcp-service-account
+      key: dev-konnekt-data-deep-1_feast-server-azure-cluster.json
+  postgresql:
+    existingSecret: feast-postgresql
+  kafka:
+    enabled:
+      false
+  application-override.yaml:
+    feast:
+      stream:
+        type: kafka
+        options:
+          topic: feast-features
+          bootstrapServers: feast-kafka:9092
+
+feast-online-serving:
+  enabled: false
+
+feast-batch-serving:
+  enabled:
+    true
+  redis:
+    enabled: false
+  livenessProbe:
+    enabled: true
+  readinessProbe:
+    enabled: true
+  gcpServiceAccount:
+    enabled: true
+    existingSecret:
+      name: feast-gcp-service-account
+      key: dev-konnekt-data-deep-1_feast-server-azure-cluster.json
+  application-override.yaml:
+    feast:
+      active_store: historical
+      stores:
+        - name: historical
+          type: BIGQUERY
+          config:
+            project_id: dev-konnekt-data-deep-1
+            dataset_id: feast_warehouse
+            staging_location: gs://dev-konnekt-data-deep-1-dataflow-workspace/batch-staging/
+            initial_retry_delay_seconds: 3
+            total_timeout_seconds: 21600
+          subscriptions:
+            - name: "*"
+              project: "*"
+              version: "*"
+      stream:
+        type: kafka
+        options:
+          topic: feast-features
+          bootstrapServers: feast-kafka:9092
+      job-store:
+        redis_host: feast-redis-jobstore-master
+        redis_port: 6379
+
+
+
+postgresql:
+  existingSecret: feast-postgresql

infra/desired-state/desired-state.yaml

@@ -0,0 +1,50 @@
+settings:
+  kubeContext: feast-temp
+
+helmRepos:
+  stable: "https://kubernetes-charts.storage.googleapis.com"
+  incubator: "https://kubernetes-charts-incubator.storage.googleapis.com"
+  feast-charts: "https://feast-charts.storage.googleapis.com"
+  bitnami: "https://charts.bitnami.com/bitnami"
+  farfetch-public: "https://konnekt-public-charts.storage.googleapis.com"
+
+namespaces:
+  feast:
+
+apps:
+  feast:
+    name: feast
+    description: Feast
+    namespace: feast
+    enabled: true
+    chart: farfetch-public/feast
+    version: 0.5.0
+    valuesFiles:
+    - chart-values/feast.yaml
+  feast-redis-jobstore:
+    name: feast-redis-jobstore
+    description: Feast Redis Database for Job Store
+    namespace: feast
+    enabled: true
+    chart: stable/redis
+    version: 10.2.1
+    valuesFiles:
+    - chart-values/feast-redis-jobstore.yaml
+  feast-kafka:
+    name: feast-kafka
+    description: Feast Kafka instance
+    namespace: feast
+    enabled: true
+    chart: bitnami/kafka
+    version: 5.2.4
+    valuesFiles:
+    - chart-values/feast-kafka.yaml
+  feast-zookeeper:
+    name: feast-zookeeper
+    description: Feast Zookeeper instance
+    namespace: feast
+    enabled: true
+    chart: bitnami/zookeeper
+    version: 4.3.4
+    valuesFiles:
+    - chart-values/feast-zookeeper.yaml

infra/scripts/deploy-to-aks.sh

@@ -0,0 +1,45 @@
+#!/bin/bash
+
+# This script automates deploy Feast to Azure Kubernetes Service (a.k.a AKS)
+# Checks dependencies first, if there is a missing dependency, install it silently
+
+# Stop on error
+set -e
+
+# check kubectl
+
+if ! [ -x "$(command -v kubectl)" ]; then
+  echo 'Error: kubectl is not installed.' >&2
+
+  curl -LO https://storage.googleapis.com/kubernetes-release/release/`curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt`/bin/linux/amd64/kubectl
+  chmod +x ./kubectl
+  sudo mv ./kubectl /usr/local/bin/kubectl
+fi
+
+az aks get-credentials --name feast-temp --resource-group farfetch_rg
+
+# check helm
+
+if ! [ -x "$(command -v helm)" ]; then
+  echo 'Error: helm is not installed.' >&2
+
+  curl -Ls https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 | DESIRED_VERSION=v3.2.1 bash
+fi
+
+# install required plugins
+
+helm plugin list | grep -cE ^diff>/dev/null || helm plugin install "https://github.com/databus23/helm-diff" --version 3.1.1
+
+# check helmsman
+
+if ! [ -x "$(command -v helmsman)" ]; then
+  echo 'Error: helmsman is not installed.' >&2
+
+  curl -Ls https://github.com/Praqma/helmsman/releases/download/v3.2.0/helmsman_3.2.0_linux_amd64.tar.gz | tar zx
+
+  chmod +x helmsman
+fi
+
+# run helmsman
+
+./helmsman -apply -f ./infra/desired-state/desired-state.yaml -debug -no-ns