More deploy triggers and simplify docker (mindsdb#8609)

* Refactor bake and deploys. Add staging and prod deploys

* Log into ECR for prod push

* fix latest tag

* Add byom deps

* typo

.github/workflows/docker_build.yml → .github/workflows/deploy-dev.yml

@@ -1,4 +1,4 @@
-name: MindsDB Docker Build
+name: Deploy to dev
 
 on:
   pull_request:
@@ -64,6 +64,6 @@ jobs:
           owner: mindsdb
           repo: INTERNAL-mindsdb-build-deploy-to-kubernetes
           github_token: ${{ secrets.REPO_DISPATCH_PAT_TOKEN }}
-          workflow_file_name: dev-deploy.yml
+          workflow_file_name: deploy-dev.yml
           ref: master
           client_payload: '{"image-tag-prefix": "${{ env.CI_SHA }}", "deploy-env": "${{matrix.deploy-env}}"}'

.github/workflows/deploy-staging.yml

@@ -0,0 +1,41 @@
+name: Deploy to staging
+
+on:
+  push:
+    branches:
+      - stable
+
+jobs:
+  build:
+    # Build our docker images based on our bake file
+    runs-on: [self-hosted, dev]
+    steps:
+    - uses: actions/checkout@v2
+    # Get clean environment variables via https://github.com/marketplace/actions/github-environment-variables-action
+    - uses: FranzDiebold/github-env-vars-action@v2
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v3
+    - name: Login to Amazon ECR
+      uses: aws-actions/amazon-ecr-login@v1
+    - name: Build and push
+      shell: bash
+      run: |
+        docker buildx create --name=remote-buildkit-agent --driver=remote --use tcp://remote-buildkit-agent.infrastructure.svc.cluster.local:80 || true # Create the builder (might already exist)
+        VERSION=${{ env.CI_SHA }} docker buildx bake --push --progress plain -f docker/docker-bake.hcl
+
+  trigger_deploy:
+    # Trigger private repo to deploy to staging env
+    runs-on: [self-hosted, dev]
+    needs: [build]
+    environment:
+      name: staging
+    steps:
+      - uses: FranzDiebold/github-env-vars-action@v2
+      - uses: convictional/trigger-workflow-and-wait@v1.6.5
+        with:
+          owner: mindsdb
+          repo: INTERNAL-mindsdb-build-deploy-to-kubernetes
+          github_token: ${{ secrets.REPO_DISPATCH_PAT_TOKEN }}
+          workflow_file_name: deploy-dev.yml
+          ref: master
+          client_payload: '{"image-tag-prefix": "${{ env.CI_SHA }}", "deploy-env": "staging"}'

.github/workflows/release.yml

@@ -45,8 +45,9 @@ jobs:
         python setup.py sdist
         twine upload dist/*
 
-  deploy_to_dockerhub:
+  docker_build:
       # Build our docker images based on our bake file
+      # This will tag with the release version tag and push to both dockerhub and ECR
       runs-on: [self-hosted, dev]
       needs: check-version
       if: github.actor != 'mindsdbadmin'
@@ -61,8 +62,28 @@ jobs:
           with:
             username: ${{ secrets.DOCKER_USERNAME }}
             password: ${{ secrets.DOCKER_PASSWORD }}
+        - name: Login to Amazon ECR
+          uses: aws-actions/amazon-ecr-login@v1
         - name: Build and push
           shell: bash
           run: |
             docker buildx create --name=remote-buildkit-agent --driver=remote --use tcp://remote-buildkit-agent.infrastructure.svc.cluster.local:80 || true # Create the builder (might already exist)
-            VERSION=${{ env.CI_REF_NAME }} REGISTRY=mindsdb docker buildx bake --push --progress plain -f docker/docker-bake.hcl
+            VERSION=${{ env.CI_REF_NAME }} PUSH_TO_DOCKERHUB=true docker buildx bake --push --progress plain -f docker/docker-bake.hcl
+
+  trigger_deploy:
+    # Trigger private repo to deploy to prod env
+    runs-on: [self-hosted, dev]
+    needs: docker_build
+    if: github.actor != 'mindsdbadmin'
+    environment:
+      name: prod
+    steps:
+      - uses: FranzDiebold/github-env-vars-action@v2
+      - uses: convictional/trigger-workflow-and-wait@v1.6.5
+        with:
+          owner: mindsdb
+          repo: INTERNAL-mindsdb-build-deploy-to-kubernetes
+          github_token: ${{ secrets.REPO_DISPATCH_PAT_TOKEN }}
+          workflow_file_name: deploy-prod.yml
+          ref: master
+          client_payload: '{"image-tag-prefix": "${{ env.CI_REF_NAME }}", "deploy-env": "prod"}'

docker/docker-bake.hcl

@@ -1,18 +1,33 @@
+# The default targets to be built if none are specified
 group "default" {
   targets = ["bare", "devel", "cloud", "lightwood", "huggingface"]
 }
 
-
-variable "REGISTRY" {
-  default = "454861456664.dkr.ecr.us-east-2.amazonaws.com"
+variable "PUSH_TO_DOCKERHUB" {
+  default = false
 }
 variable "IMAGE" {
   default = "mindsdb"
 }
+# This is a semver for releases but otherwise is a github sha
 variable "VERSION" {
   default = "unknown"
 }
 
+# Generate the list of tags for a given image.
+# e.g. for the 'cloud' images this generates:
+# - "mindsdb:cloud"        - This functions as a 'latest' tag for the cloud image
+# - "mindsdb:v1.2.3-cloud" - For this specific version
+# The same tags are pushed to dockerhub as well if the PUSH_TO_DOCKERHUB variable is set.
+function "get_tags" {
+  params = [target]
+  result = [
+    "454861456664.dkr.ecr.us-east-2.amazonaws.com/${IMAGE}:${VERSION}${notequal(target, "") ? "-" : ""}${target}",
+    "454861456664.dkr.ecr.us-east-2.amazonaws.com/${IMAGE}:${notequal(target, "") ? target : "latest"}",
+    PUSH_TO_DOCKERHUB ? "mindsdb/${IMAGE}:${VERSION}${notequal(target, "") ? "-" : ""}${target}" : "",
+    PUSH_TO_DOCKERHUB ? "mindsdb/${IMAGE}:${notequal(target, "") ? target : "latest"}" : ""
+  ]
+} 
 
 # This is effectively the base image for all of our images.
 # We define it separately so we can use it as a base and only build it once.
@@ -21,50 +36,51 @@ target "builder" {
   target = "build"
   platforms = ["linux/amd64", "linux/arm64"]
 }
+
 # Common traits of every image that we use to reduce duplication below.
 target "_common" {
   dockerfile = "docker/mindsdb.Dockerfile" # If you change this, also change it in target:builder
   contexts = {
-    builder = "target:builder" # Use a target to only perform base build steps once
+    build = "target:builder" # Use a target to only perform base build steps once
   }
   platforms = ["linux/amd64", "linux/arm64"]
 }
 
 
 
-### IMAGES ###
+### OUTPUT IMAGES ###
 
 target "bare" {
   inherits = ["_common"]
-  tags = ["${REGISTRY}/${IMAGE}:${VERSION}", "${REGISTRY}/${IMAGE}:latest"]
+  tags = get_tags("")
 }
 
 target "devel" {
   inherits = ["_common"]
-  tags = ["${REGISTRY}/${IMAGE}:${VERSION}-dev", "${REGISTRY}/${IMAGE}:dev"]
+  tags = get_tags("dev")
   target = "dev"
 }
 
 target "cloud" {
   inherits = ["_common"]
+  tags = get_tags("cloud")
   args = {
-    EXTRAS = ".[lightwood,huggingface,statsforecast-extra,neuralforecast-extra,timegpt,surrealdb,youtube,ignite,gmail,pgvector]"
+    EXTRAS = ".[lightwood,huggingface,statsforecast-extra,neuralforecast-extra,timegpt,surrealdb,youtube,ignite,gmail,pgvector] darts datasetsforecast"
   }
-  tags = ["${REGISTRY}/${IMAGE}:${VERSION}-cloud", "${REGISTRY}/${IMAGE}:cloud"]
 }
 
 target "lightwood" {
   inherits = ["_common"]
+  tags = get_tags("lightwood")
   args = {
     EXTRAS = ".[lightwood]"
   }
-  tags = ["${REGISTRY}/${IMAGE}:${VERSION}-lightwood", "${REGISTRY}/${IMAGE}:lightwood"]
 }
 
 target "huggingface" {
   inherits = ["_common"]
+  tags = get_tags("huggingface")
   args = {
     EXTRAS = ".[huggingface]"
   }
-  tags = ["${REGISTRY}/${IMAGE}:${VERSION}-huggingface", "${REGISTRY}/${IMAGE}:huggingface"]
 }