From b62f485b12ce562c191b9b89a7cad0b810a2843e Mon Sep 17 00:00:00 2001 From: Hamish Fagg Date: Mon, 15 Jan 2024 10:07:10 +1300 Subject: [PATCH] More deploy triggers and simplify docker (#8609) * Refactor bake and deploys. Add staging and prod deploys * Log into ECR for prod push * fix latest tag * Add byom deps * typo --- .../{docker_build.yml => deploy-dev.yml} | 4 +- .github/workflows/deploy-staging.yml | 41 +++++++++++++++++++ .github/workflows/release.yml | 25 ++++++++++- docker/docker-bake.hcl | 38 ++++++++++++----- 4 files changed, 93 insertions(+), 15 deletions(-) rename .github/workflows/{docker_build.yml => deploy-dev.yml} (97%) create mode 100644 .github/workflows/deploy-staging.yml diff --git a/.github/workflows/docker_build.yml b/.github/workflows/deploy-dev.yml similarity index 97% rename from .github/workflows/docker_build.yml rename to .github/workflows/deploy-dev.yml index 0441b701e49..0f98341f25b 100644 --- a/.github/workflows/docker_build.yml +++ b/.github/workflows/deploy-dev.yml @@ -1,4 +1,4 @@ -name: MindsDB Docker Build +name: Deploy to dev on: pull_request: @@ -64,6 +64,6 @@ jobs: owner: mindsdb repo: INTERNAL-mindsdb-build-deploy-to-kubernetes github_token: ${{ secrets.REPO_DISPATCH_PAT_TOKEN }} - workflow_file_name: dev-deploy.yml + workflow_file_name: deploy-dev.yml ref: master client_payload: '{"image-tag-prefix": "${{ env.CI_SHA }}", "deploy-env": "${{matrix.deploy-env}}"}' diff --git a/.github/workflows/deploy-staging.yml b/.github/workflows/deploy-staging.yml new file mode 100644 index 00000000000..8f16cb19dae --- /dev/null +++ b/.github/workflows/deploy-staging.yml @@ -0,0 +1,41 @@ +name: Deploy to staging + +on: + push: + branches: + - stable + +jobs: + build: + # Build our docker images based on our bake file + runs-on: [self-hosted, dev] + steps: + - uses: actions/checkout@v2 + # Get clean environment variables via https://github.com/marketplace/actions/github-environment-variables-action + - uses: FranzDiebold/github-env-vars-action@v2 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Login to Amazon ECR + uses: aws-actions/amazon-ecr-login@v1 + - name: Build and push + shell: bash + run: | + docker buildx create --name=remote-buildkit-agent --driver=remote --use tcp://remote-buildkit-agent.infrastructure.svc.cluster.local:80 || true # Create the builder (might already exist) + VERSION=${{ env.CI_SHA }} docker buildx bake --push --progress plain -f docker/docker-bake.hcl + + trigger_deploy: + # Trigger private repo to deploy to staging env + runs-on: [self-hosted, dev] + needs: [build] + environment: + name: staging + steps: + - uses: FranzDiebold/github-env-vars-action@v2 + - uses: convictional/trigger-workflow-and-wait@v1.6.5 + with: + owner: mindsdb + repo: INTERNAL-mindsdb-build-deploy-to-kubernetes + github_token: ${{ secrets.REPO_DISPATCH_PAT_TOKEN }} + workflow_file_name: deploy-dev.yml + ref: master + client_payload: '{"image-tag-prefix": "${{ env.CI_SHA }}", "deploy-env": "staging"}' diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a5aca366125..de71f0c46b5 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -45,8 +45,9 @@ jobs: python setup.py sdist twine upload dist/* - deploy_to_dockerhub: + docker_build: # Build our docker images based on our bake file + # This will tag with the release version tag and push to both dockerhub and ECR runs-on: [self-hosted, dev] needs: check-version if: github.actor != 'mindsdbadmin' @@ -61,8 +62,28 @@ jobs: with: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} + - name: Login to Amazon ECR + uses: aws-actions/amazon-ecr-login@v1 - name: Build and push shell: bash run: | docker buildx create --name=remote-buildkit-agent --driver=remote --use tcp://remote-buildkit-agent.infrastructure.svc.cluster.local:80 || true # Create the builder (might already exist) - VERSION=${{ env.CI_REF_NAME }} REGISTRY=mindsdb docker buildx bake --push --progress plain -f docker/docker-bake.hcl + VERSION=${{ env.CI_REF_NAME }} PUSH_TO_DOCKERHUB=true docker buildx bake --push --progress plain -f docker/docker-bake.hcl + + trigger_deploy: + # Trigger private repo to deploy to prod env + runs-on: [self-hosted, dev] + needs: docker_build + if: github.actor != 'mindsdbadmin' + environment: + name: prod + steps: + - uses: FranzDiebold/github-env-vars-action@v2 + - uses: convictional/trigger-workflow-and-wait@v1.6.5 + with: + owner: mindsdb + repo: INTERNAL-mindsdb-build-deploy-to-kubernetes + github_token: ${{ secrets.REPO_DISPATCH_PAT_TOKEN }} + workflow_file_name: deploy-prod.yml + ref: master + client_payload: '{"image-tag-prefix": "${{ env.CI_REF_NAME }}", "deploy-env": "prod"}' diff --git a/docker/docker-bake.hcl b/docker/docker-bake.hcl index 04f9358fd5f..b299f30b532 100644 --- a/docker/docker-bake.hcl +++ b/docker/docker-bake.hcl @@ -1,18 +1,33 @@ +# The default targets to be built if none are specified group "default" { targets = ["bare", "devel", "cloud", "lightwood", "huggingface"] } - -variable "REGISTRY" { - default = "454861456664.dkr.ecr.us-east-2.amazonaws.com" +variable "PUSH_TO_DOCKERHUB" { + default = false } variable "IMAGE" { default = "mindsdb" } +# This is a semver for releases but otherwise is a github sha variable "VERSION" { default = "unknown" } +# Generate the list of tags for a given image. +# e.g. for the 'cloud' images this generates: +# - "mindsdb:cloud" - This functions as a 'latest' tag for the cloud image +# - "mindsdb:v1.2.3-cloud" - For this specific version +# The same tags are pushed to dockerhub as well if the PUSH_TO_DOCKERHUB variable is set. +function "get_tags" { + params = [target] + result = [ + "454861456664.dkr.ecr.us-east-2.amazonaws.com/${IMAGE}:${VERSION}${notequal(target, "") ? "-" : ""}${target}", + "454861456664.dkr.ecr.us-east-2.amazonaws.com/${IMAGE}:${notequal(target, "") ? target : "latest"}", + PUSH_TO_DOCKERHUB ? "mindsdb/${IMAGE}:${VERSION}${notequal(target, "") ? "-" : ""}${target}" : "", + PUSH_TO_DOCKERHUB ? "mindsdb/${IMAGE}:${notequal(target, "") ? target : "latest"}" : "" + ] +} # This is effectively the base image for all of our images. # We define it separately so we can use it as a base and only build it once. @@ -21,50 +36,51 @@ target "builder" { target = "build" platforms = ["linux/amd64", "linux/arm64"] } + # Common traits of every image that we use to reduce duplication below. target "_common" { dockerfile = "docker/mindsdb.Dockerfile" # If you change this, also change it in target:builder contexts = { - builder = "target:builder" # Use a target to only perform base build steps once + build = "target:builder" # Use a target to only perform base build steps once } platforms = ["linux/amd64", "linux/arm64"] } -### IMAGES ### +### OUTPUT IMAGES ### target "bare" { inherits = ["_common"] - tags = ["${REGISTRY}/${IMAGE}:${VERSION}", "${REGISTRY}/${IMAGE}:latest"] + tags = get_tags("") } target "devel" { inherits = ["_common"] - tags = ["${REGISTRY}/${IMAGE}:${VERSION}-dev", "${REGISTRY}/${IMAGE}:dev"] + tags = get_tags("dev") target = "dev" } target "cloud" { inherits = ["_common"] + tags = get_tags("cloud") args = { - EXTRAS = ".[lightwood,huggingface,statsforecast-extra,neuralforecast-extra,timegpt,surrealdb,youtube,ignite,gmail,pgvector]" + EXTRAS = ".[lightwood,huggingface,statsforecast-extra,neuralforecast-extra,timegpt,surrealdb,youtube,ignite,gmail,pgvector] darts datasetsforecast" } - tags = ["${REGISTRY}/${IMAGE}:${VERSION}-cloud", "${REGISTRY}/${IMAGE}:cloud"] } target "lightwood" { inherits = ["_common"] + tags = get_tags("lightwood") args = { EXTRAS = ".[lightwood]" } - tags = ["${REGISTRY}/${IMAGE}:${VERSION}-lightwood", "${REGISTRY}/${IMAGE}:lightwood"] } target "huggingface" { inherits = ["_common"] + tags = get_tags("huggingface") args = { EXTRAS = ".[huggingface]" } - tags = ["${REGISTRY}/${IMAGE}:${VERSION}-huggingface", "${REGISTRY}/${IMAGE}:huggingface"] } \ No newline at end of file