From 044ffd5295de7050411b6d16701cca2e58149455 Mon Sep 17 00:00:00 2001 From: Filipe Forattini Date: Wed, 6 Jul 2022 11:04:44 -0300 Subject: [PATCH] fix: added deploy for apps --- .github/workflows/app-deploy.yml | 370 +++++++++++++++++++++++++++++++ .github/workflows/app-push.yml | 4 - 2 files changed, 370 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/app-deploy.yml diff --git a/.github/workflows/app-deploy.yml b/.github/workflows/app-deploy.yml new file mode 100644 index 00000000..fd23d718 --- /dev/null +++ b/.github/workflows/app-deploy.yml @@ -0,0 +1,370 @@ +name: deploy + +concurrency: + group: ${{github.workflow}} + cancel-in-progress: true + +#--------------------------------------------------# +# Triggers # +#--------------------------------------------------# +on: + + workflow_call: + + inputs: + environment: + type: string + required: false + default: "dev" + description: "Environment target" + containerRegistry: + type: string + required: false + default: ghcr.io + description: "Container registry to upload container images" + environmentsAsNamespaces: + type: boolean + required: false + default: false + description: "Separate environments as namespaces" + + outputs: + PipelineConfig: + description: "Used pipeline config" + value: ${{ jobs.Setup.outputs.PipelineConfig }} + + +#--------------------------------------------------# +# Workflow Jobs # +#--------------------------------------------------# +jobs: + + #--------------------------------------------------# + # Setup Jobs # + #--------------------------------------------------# + Setup: + runs-on: ubuntu-latest + + outputs: + PipelineConfig: ${{ steps.script_setup.outputs.result }} + BuildNode: ${{ steps.define_builders.outputs.build_node }} + BuildPython: ${{ steps.define_builders.outputs.build_python }} + + steps: + + # pre-job + - name: Setup | Cloning repository + uses: actions/checkout@v3 + + - name: Setup | Cloning tools + uses: actions/checkout@v3 + with: + ref: main + path: .pipeline + repository: filipeforattini/ff-iac-github-actions + + # job + - name: Config | Pipeline config scrapper + uses: actions/github-script@v6 + id: script_setup + with: + result-encoding: string + script: | + return require('./.pipeline/src/steps/config-scrapper')({ + context, + inputs: { + containerRegistry: "${{ inputs.containerRegistry }}", + }, + }) + + - name: Docs | Configs summary + env: + PIPELINE_SETUP: ${{ steps.script_setup.outputs.result}} + run: | + echo -e "### $(echo $PIPELINE_SETUP | jq -r '.deploy.podName') pipeline\n\n" >> $GITHUB_STEP_SUMMARY + echo -e "
Config\n\n\`\`\`json \n$(echo $PIPELINE_SETUP | jq '.')\n \`\`\`\n
\n\n" >> $GITHUB_STEP_SUMMARY + echo -e "---\n\n" >> $GITHUB_STEP_SUMMARY + echo -e "Build started at: $(echo $PIPELINE_SETUP | jq -r '.run.startedAt')\n\n" >> $GITHUB_STEP_SUMMARY + + + #--------------------------------------------------# + # Deploy # + #--------------------------------------------------# + Deploy: + environment: ${{ inputs.environment }} + runs-on: ubuntu-latest + + needs: + - Setup + + steps: + + # pre-job + - name: Setup | Cloning repository + uses: actions/checkout@v3 + with: + fetch-depth: 0 + persist-credentials: false + + - name: Setup | Cloning tools + uses: actions/checkout@v3 + with: + ref: main + path: .pipeline + repository: filipeforattini/ff-iac-github-actions + + - name: Install | Kubectl + uses: azure/setup-kubectl@v2.1 + + - name: Install | Helm + uses: azure/setup-helm@v2.1 + + - name: Install | YTT + uses: vmware-tanzu/carvel-setup-action@v1 + with: + only: ytt + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Install | YQ + env: + VERSION: v4.25.2 + BINARY: yq_linux_amd64 + run: | + wget https://github.com/mikefarah/yq/releases/download/${VERSION}/${BINARY}.tar.gz -O - | tar xz && sudo mv ${BINARY} /usr/bin/yq + yq --version + + - name: Install | QEMU + uses: docker/setup-qemu-action@v2 + + - name: Install | Docker Buildx + uses: docker/setup-buildx-action@v1 + + - name: Config | Version + id: versioning + run: | + NEXT_VERSION=$(git for-each-ref --sort=authordate --format '%(refname)' refs/tags | tail -n 1 | sed -n -e 's/^.*refs\/tags\/v//p') + echo "next version = $NEXT_VERSION" + echo "::set-output name=version::$NEXT_VERSION" + + - name: Config | Load setup configs + env: + ENVIRONMENTS_AS_NAMESPACES: ${{ inputs.environmentsAsNamespaces }} + PIPELINE_SETUP: ${{ needs.Setup.outputs.PipelineConfig }} + id: deploy_setup + run: | + echo "::set-output name=deploy_as_k8s::$(echo $PIPELINE_SETUP | jq -r '.deploy.deployAsK8s')" + echo "::set-output name=deploy_as_chart::$(echo $PIPELINE_SETUP | jq -r '.deploy.deployAsChart')" + echo "::set-output name=has_dev_secrets::$(echo $PIPELINE_SETUP | jq -r '.deploy.secrets.dev')" + echo "::set-output name=has_dev_configs::$(echo $PIPELINE_SETUP | jq -r '.deploy.configs.dev')" + echo "::set-output name=has_dev_dependencies::$(echo $PIPELINE_SETUP | jq -r '.deploy.dependencies.dev')" + echo "::set-output name=deploy_ecosystem::$(echo $PIPELINE_SETUP | jq -r '.deploy.ecosystem')" + echo "::set-output name=deploy_organization::$(echo $PIPELINE_SETUP | jq -r '.deploy.organization')" + echo "::set-output name=deploy_container_registry::$(echo $PIPELINE_SETUP | jq -r '.deploy.containerRegistry')" + echo "::set-output name=deploy_repository::$(echo $PIPELINE_SETUP | jq -r '.deploy.repository')" + echo "::set-output name=deploy_tag::$(echo $PIPELINE_SETUP | jq -r '.deploy.commitTag')" + echo "::set-output name=run_started_at::$(echo $PIPELINE_SETUP | jq -r '.run.startedAt')" + case $ENVIRONMENTS_AS_NAMESPACES in + "true") echo "::set-output name=deploy_namespace::$(echo $PIPELINE_SETUP | jq -r '.deploy.namespaces.dev')";; + "false") echo "::set-output name=deploy_namespace::$(echo $PIPELINE_SETUP | jq -r '.deploy.namespace')";; + *) echo "::set-output name=deploy_namespace::$(echo $PIPELINE_SETUP | jq -r '.deploy.namespace')";; + esac + + - name: Config | Kubectl config file + env: + KUBE_CONFIG: ${{ secrets.KUBE_CONFIG }} + run: | + mkdir -p ~/.kube + echo "$KUBE_CONFIG" | base64 -d > ~/.kube/config + + # namespace + - name: K8s create namespace + run: kubectl create namespace ${{steps.deploy_setup.outputs.deploy_namespace}} --dry-run=client --validate=false --output=yaml | kubectl apply -f - + + # dependencies + - name: Dependencies | Resources render + if: steps.deploy_setup.outputs.has_dev_dependencies == 'true' + run: | + ytt \ + -f ./.pipeline/deploy/as-k8s/dependencies.schema.yml \ + -f ./.pipeline/deploy/as-k8s/dependencies.yml \ + -f ./manifests/dependencies/dev.yml \ + --data-value repository=${{steps.deploy_setup.outputs.deploy_repository}} \ + > ./manifests/k8s-dependencies.yml + + - name: Docs | Dependencies + if: steps.deploy_setup.outputs.deploy_as_k8s == 'true' + run: | + echo -e "### Dependencies\n\n" >> $GITHUB_STEP_SUMMARY + echo -e "
dependencies\n\n\`\`\`yml \n$(cat ./manifests/k8s-dependencies.yml)\n \`\`\`\n
\n\n" >> $GITHUB_STEP_SUMMARY + echo -e "| name | chart version | app version |" >> $GITHUB_STEP_SUMMARY + echo -e "| --- | :---: | :---: |" >> $GITHUB_STEP_SUMMARY + if [ $(cat ./manifests/k8s-dependencies.yml | yq -P '.postgres.enabled') = true ]; then + echo -e "| postgres | $(cat ./manifests/k8s-dependencies.yml | yq -P '.postgres.version') | $(cat ./manifests/k8s-dependencies.yml | yq -P '.postgres.helm.image.tag') |" >> $GITHUB_STEP_SUMMARY + fi + if [ $(cat ./manifests/k8s-dependencies.yml | yq -P '.mysql.enabled') = true ]; then + echo -e "| mysql | $(cat ./manifests/k8s-dependencies.yml | yq -P '.mysql.version') | $(cat ./manifests/k8s-dependencies.yml | yq -P '.mysql.helm.image.tag') |\n" >> $GITHUB_STEP_SUMMARY + fi + + - name: Dependencies | Helm update + if: steps.deploy_setup.outputs.has_dev_dependencies == 'true' + run: | + helm repo add bitnami https://charts.bitnami.com/bitnami + helm repo update + + - name: Dependencies | Mysql + if: steps.deploy_setup.outputs.has_dev_dependencies == 'true' + continue-on-error: true + run: | + K8S_NAMESPACE=${{steps.deploy_setup.outputs.deploy_namespace}} \ + DEPENDENCY_NAME=mysql \ + REPOSITORY_TAG_VERSION=${{steps.versioning.outputs.version}} \ + DEPENDENCY_FILE=./manifests/k8s-dependencies.yml \ + ./.pipeline/src/dependency-install.sh + + - name: Dependencies | Postgres + if: steps.deploy_setup.outputs.has_dev_dependencies == 'true' + continue-on-error: true + run: | + K8S_NAMESPACE=${{steps.deploy_setup.outputs.deploy_namespace}} \ + DEPENDENCY_NAME=postgres \ + REPOSITORY_TAG_VERSION=${{steps.versioning.outputs.version}} \ + DEPENDENCY_FILE=./manifests/k8s-dependencies.yml \ + ./.pipeline/src/dependency-install.sh + + - name: Dependencies | Rabbitmq + if: steps.deploy_setup.outputs.has_dev_dependencies == 'true' + continue-on-error: true + run: | + K8S_NAMESPACE=${{steps.deploy_setup.outputs.deploy_namespace}} \ + DEPENDENCY_NAME=rabbitmq \ + REPOSITORY_TAG_VERSION=${{steps.versioning.outputs.version}} \ + DEPENDENCY_FILE=./manifests/k8s-dependencies.yml \ + ./.pipeline/src/dependency-install.sh + + - name: Dependencies | Elasticsearch + if: steps.deploy_setup.outputs.has_dev_dependencies == 'true' + continue-on-error: true + run: | + K8S_NAMESPACE=${{steps.deploy_setup.outputs.deploy_namespace}} \ + DEPENDENCY_NAME=elasticsearch \ + REPOSITORY_TAG_VERSION=${{steps.versioning.outputs.version}} \ + DEPENDENCY_FILE=./manifests/k8s-dependencies.yml \ + ./.pipeline/src/dependency-install.sh + + - name: Dependencies | Redis + if: steps.deploy_setup.outputs.has_dev_dependencies == 'true' + continue-on-error: true + run: | + K8S_NAMESPACE=${{steps.deploy_setup.outputs.deploy_namespace}} \ + DEPENDENCY_NAME=redis \ + REPOSITORY_TAG_VERSION=${{steps.versioning.outputs.version}} \ + DEPENDENCY_FILE=./manifests/k8s-dependencies.yml \ + ./.pipeline/src/dependency-install.sh + + - name: Dependencies | Nats + if: steps.deploy_setup.outputs.has_dev_dependencies == 'true' + continue-on-error: true + run: | + K8S_NAMESPACE=${{steps.deploy_setup.outputs.deploy_namespace}} \ + DEPENDENCY_NAME=nats \ + REPOSITORY_TAG_VERSION=${{steps.versioning.outputs.version}} \ + DEPENDENCY_FILE=./manifests/k8s-dependencies.yml \ + ./.pipeline/src/dependency-install.sh + + - name: Dependencies | Etcd + if: steps.deploy_setup.outputs.has_dev_dependencies == 'true' + continue-on-error: true + run: | + K8S_NAMESPACE=${{steps.deploy_setup.outputs.deploy_namespace}} \ + DEPENDENCY_NAME=etcd \ + REPOSITORY_TAG_VERSION=${{steps.versioning.outputs.version}} \ + DEPENDENCY_FILE=./manifests/k8s-dependencies.yml \ + ./.pipeline/src/dependency-install.sh + + # configs + - name: K8s create config-map + if: steps.deploy_setup.outputs.has_dev_configs == 'true' + run: | + kubectl create configmap -n ${{steps.deploy_setup.outputs.deploy_namespace}} svc --from-env-file=./manifests/configs/dev.env --dry-run=client --validate=false --output=yaml | kubectl apply -f - + kubectl create configmap -n ${{steps.deploy_setup.outputs.deploy_namespace}} svc-${{steps.versioning.outputs.version}} --from-env-file=./manifests/configs/dev.env --dry-run=client --validate=false --output=yaml | kubectl apply -f - + kubectl get configmap -n ${{steps.deploy_setup.outputs.deploy_namespace}} svc -o jsonpath='{.data}' | jq -r 'keys[]' | tr '\n' '~' | sed 's/~/,/g;s/,$//' > ./manifests/k8s-configs-keys.txt + + # secrets + - name: Decrypt DEV secrets + if: steps.deploy_setup.outputs.has_dev_secrets == 'true' + run: | + gpg \ + --yes --batch --quiet --decrypt \ + --passphrase="${{ secrets.GPG_PASSPHRASE }}" \ + --output ./manifests/k8s-secrets.env \ + ./manifests/secrets/dev.gpg + + - name: K8s create secrets + if: steps.deploy_setup.outputs.has_dev_secrets == 'true' + run: | + kubectl create secret generic -n ${{steps.deploy_setup.outputs.deploy_namespace}} svc --from-env-file=./manifests/k8s-secrets.env --dry-run=client --validate=false --output=yaml | kubectl apply -f - + kubectl create secret generic -n ${{steps.deploy_setup.outputs.deploy_namespace}} svc-${{steps.versioning.outputs.version}} --from-env-file=./manifests/k8s-secrets.env --dry-run=client --validate=false --output=yaml | kubectl apply -f - + kubectl get secret -n ${{steps.deploy_setup.outputs.deploy_namespace}} svc -o jsonpath='{.data}' | jq -r 'keys[]' | tr '\n' '~' | sed 's/~/,/g;s/,$//' > ./manifests/k8s-secrets-keys.txt + + # secrets for registry auth + - name: Config | Login to Container Registry + uses: docker/login-action@v2 + with: + logout: false + registry: ${{ inputs.containerRegistry }} + username: ${{ secrets.REGISTRY_USERNAME }} + password: ${{ secrets.REGISTRY_PASSWORD }} + + - name: Config | Gives runner access to docker config file + if: steps.deploy_setup.outputs.deploy_as_k8s == 'true' + run: | + sudo chown $(whoami):docker /home/$(whoami)/.docker/config.json + cp /home/$(whoami)/.docker/config.json ./manifests/docker-config.json + + - name: K8s create registry-token secret + if: steps.deploy_setup.outputs.deploy_as_k8s == 'true' + run: kubectl create secret generic -n ${{steps.deploy_setup.outputs.deploy_namespace}} registry-token --type=kubernetes.io/dockerconfigjson --from-file=.dockerconfigjson=./manifests/docker-config.json --dry-run=client --validate=false --output=yaml | kubectl apply -f - + + # generate k8s manifests + - name: K8s generates final yml + if: steps.deploy_setup.outputs.deploy_as_k8s == 'true' + run: | + CONFIGS_LIST=$(if test -f ./manifests/k8s-configs-keys.txt; then cat ./manifests/k8s-configs-keys.txt; else echo ''; fi) + SECRETS_LIST=$(if test -f ./manifests/k8s-secrets-keys.txt; then cat ./manifests/k8s-secrets-keys.txt; else echo ''; fi) + DEPENDENCIES_LIST=$(if test -f ./manifests/k8s-dependencies.yml; then (cat ./manifests/k8s-dependencies.yml | yq -P '.dependencies'); else echo '' ; fi) + ytt \ + -f ./.pipeline/deploy/as-k8s/service.schema.yml \ + -f ./.pipeline/deploy/as-k8s/service.yml \ + -f ./manifests/k8s-values.yml \ + --data-value ecosystem=${{steps.deploy_setup.outputs.deploy_ecosystem}} \ + --data-value organization=${{steps.deploy_setup.outputs.deploy_organization}} \ + --data-value repository=${{steps.deploy_setup.outputs.deploy_repository}} \ + --data-value containerRegistry=${{steps.deploy_setup.outputs.deploy_container_registry}} \ + --data-value tag=${{steps.deploy_setup.outputs.deploy_tag}} \ + --data-value-yaml deployment.imagePullSecrets=true \ + --data-value-yaml envFromSecrets="[$SECRETS_LIST]" \ + --data-value-yaml envFromConfigMaps="[$CONFIGS_LIST]" \ + --data-value-yaml envFromDependencies="[$DEPENDENCIES_LIST]" \ + --data-value pipelineControl.datetime=${{steps.deploy_setup.outputs.run_started_at}} \ + --data-value-yaml pipelineControl.environmentsAsNamespaces=${{inputs.environmentsAsNamespaces}} \ + > ./manifests/k8s-to-apply.yml + + - name: Docs | K8s summary + if: steps.deploy_setup.outputs.deploy_as_k8s == 'true' + run: | + CONFIGS_LIST=$(if test -f ./manifests/k8s-configs-keys.txt; then cat ./manifests/k8s-configs-keys.txt; else echo ''; fi) + SECRETS_LIST=$(if test -f ./manifests/k8s-secrets-keys.txt; then cat ./manifests/k8s-secrets-keys.txt; else echo ''; fi) + DEPENDENCIES_LIST=$(if test -f ./manifests/k8s-dependencies.yml; then (cat ./manifests/k8s-dependencies.yml | yq -P '.dependencies'); else echo ''; fi) + echo -e "### k8s\n\n" >> $GITHUB_STEP_SUMMARY + echo -e "| param | value |" >> $GITHUB_STEP_SUMMARY + echo -e "| --- | :---: |" >> $GITHUB_STEP_SUMMARY + echo -e "| secrets | $SECRETS_LIST |" >> $GITHUB_STEP_SUMMARY + echo -e "| configs | $CONFIGS_LIST |" >> $GITHUB_STEP_SUMMARY + echo -e "| dependencies | $DEPENDENCIES_LIST |" >> $GITHUB_STEP_SUMMARY + echo -e "
kubefile\n\n\`\`\`yml \n$(cat ./manifests/k8s-to-apply.yml)\n \`\`\`\n
\n\n" >> $GITHUB_STEP_SUMMARY + + - name: K8s apply yml + if: steps.deploy_setup.outputs.deploy_as_k8s == 'true' + run: | + kubectl apply -f ./manifests/k8s-to-apply.yml + kubectl get pods -n ${{steps.deploy_setup.outputs.deploy_namespace}} diff --git a/.github/workflows/app-push.yml b/.github/workflows/app-push.yml index 0c5d756b..99095b06 100644 --- a/.github/workflows/app-push.yml +++ b/.github/workflows/app-push.yml @@ -45,10 +45,6 @@ on: type: string required: false default: '[14, 16, 17]' - pythonMatrix: - type: string - required: false - default: '["2.7", "3.6", "3.8", "3.10"]' platforms: type: string required: false