fix: added deploy for apps

.github/workflows/app-deploy.yml

@@ -0,0 +1,370 @@
+name: deploy
+
+concurrency:
+  group: ${{github.workflow}}
+  cancel-in-progress: true
+
+#--------------------------------------------------#
+#                     Triggers                     # 
+#--------------------------------------------------#
+on:
+
+  workflow_call:
+
+    inputs:
+      environment:
+        type: string
+        required: false
+        default: "dev"
+        description: "Environment target"
+      containerRegistry:
+        type: string
+        required: false
+        default: ghcr.io
+        description: "Container registry to upload container images"
+      environmentsAsNamespaces:
+        type: boolean
+        required: false
+        default: false
+        description: "Separate environments as namespaces"
+
+    outputs:
+      PipelineConfig: 
+        description: "Used pipeline config"
+        value: ${{ jobs.Setup.outputs.PipelineConfig }}
+
+
+#--------------------------------------------------#
+#                  Workflow Jobs                   # 
+#--------------------------------------------------#
+jobs:
+
+  #--------------------------------------------------#
+  #                    Setup Jobs                    # 
+  #--------------------------------------------------#
+  Setup:
+    runs-on: ubuntu-latest
+
+    outputs:
+      PipelineConfig: ${{ steps.script_setup.outputs.result }}
+      BuildNode: ${{ steps.define_builders.outputs.build_node }}
+      BuildPython: ${{ steps.define_builders.outputs.build_python }}
+
+    steps:
+
+      # pre-job
+      - name: Setup | Cloning repository
+        uses: actions/checkout@v3
+
+      - name: Setup | Cloning tools
+        uses: actions/checkout@v3
+        with:
+          ref: main
+          path: .pipeline
+          repository: filipeforattini/ff-iac-github-actions
+
+      # job
+      - name: Config | Pipeline config scrapper
+        uses: actions/github-script@v6
+        id: script_setup
+        with:
+          result-encoding: string
+          script: |
+            return require('./.pipeline/src/steps/config-scrapper')({ 
+              context, 
+              inputs: {
+                containerRegistry: "${{ inputs.containerRegistry }}",
+              },
+            })
+
+      - name: Docs | Configs summary
+        env:
+          PIPELINE_SETUP: ${{ steps.script_setup.outputs.result}}
+        run: | 
+          echo -e "### $(echo $PIPELINE_SETUP | jq -r '.deploy.podName') pipeline\n\n" >> $GITHUB_STEP_SUMMARY
+          echo -e "<details><summary>Config</summary>\n\n\`\`\`json \n$(echo $PIPELINE_SETUP | jq '.')\n \`\`\`\n </details>\n\n" >> $GITHUB_STEP_SUMMARY
+          echo -e "---\n\n" >> $GITHUB_STEP_SUMMARY
+          echo -e "Build started at: $(echo $PIPELINE_SETUP | jq -r '.run.startedAt')\n\n" >> $GITHUB_STEP_SUMMARY
+
+
+  #--------------------------------------------------#
+  #                      Deploy                      # 
+  #--------------------------------------------------#
+  Deploy:
+    environment: ${{ inputs.environment }}
+    runs-on: ubuntu-latest
+
+    needs:
+      - Setup
+
+    steps:
+
+      # pre-job
+      - name: Setup | Cloning repository
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+          persist-credentials: false
+
+      - name: Setup | Cloning tools
+        uses: actions/checkout@v3
+        with:
+          ref: main
+          path: .pipeline
+          repository: filipeforattini/ff-iac-github-actions
+
+      - name: Install | Kubectl
+        uses: azure/setup-kubectl@v2.1
+
+      - name: Install | Helm
+        uses: azure/setup-helm@v2.1
+
+      - name: Install | YTT
+        uses: vmware-tanzu/carvel-setup-action@v1
+        with:
+          only: ytt
+          token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Install | YQ
+        env:
+          VERSION: v4.25.2
+          BINARY: yq_linux_amd64
+        run: |
+          wget https://github.com/mikefarah/yq/releases/download/${VERSION}/${BINARY}.tar.gz -O - | tar xz && sudo mv ${BINARY} /usr/bin/yq
+          yq --version
+
+      - name: Install | QEMU
+        uses: docker/setup-qemu-action@v2
+
+      - name: Install | Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Config | Version
+        id: versioning
+        run: |
+          NEXT_VERSION=$(git for-each-ref --sort=authordate --format '%(refname)' refs/tags | tail -n 1 | sed -n -e 's/^.*refs\/tags\/v//p')
+          echo "next version = $NEXT_VERSION"
+          echo "::set-output name=version::$NEXT_VERSION"
+
+      - name: Config | Load setup configs
+        env:
+          ENVIRONMENTS_AS_NAMESPACES: ${{ inputs.environmentsAsNamespaces }}
+          PIPELINE_SETUP: ${{ needs.Setup.outputs.PipelineConfig }}
+        id: deploy_setup
+        run: |
+          echo "::set-output name=deploy_as_k8s::$(echo $PIPELINE_SETUP | jq -r '.deploy.deployAsK8s')"
+          echo "::set-output name=deploy_as_chart::$(echo $PIPELINE_SETUP | jq -r '.deploy.deployAsChart')"
+          echo "::set-output name=has_dev_secrets::$(echo $PIPELINE_SETUP | jq -r '.deploy.secrets.dev')"
+          echo "::set-output name=has_dev_configs::$(echo $PIPELINE_SETUP | jq -r '.deploy.configs.dev')"
+          echo "::set-output name=has_dev_dependencies::$(echo $PIPELINE_SETUP | jq -r '.deploy.dependencies.dev')"
+          echo "::set-output name=deploy_ecosystem::$(echo $PIPELINE_SETUP | jq -r '.deploy.ecosystem')"
+          echo "::set-output name=deploy_organization::$(echo $PIPELINE_SETUP | jq -r '.deploy.organization')"
+          echo "::set-output name=deploy_container_registry::$(echo $PIPELINE_SETUP | jq -r '.deploy.containerRegistry')"
+          echo "::set-output name=deploy_repository::$(echo $PIPELINE_SETUP | jq -r '.deploy.repository')"
+          echo "::set-output name=deploy_tag::$(echo $PIPELINE_SETUP | jq -r '.deploy.commitTag')"
+          echo "::set-output name=run_started_at::$(echo $PIPELINE_SETUP | jq -r '.run.startedAt')"
+          case $ENVIRONMENTS_AS_NAMESPACES in
+            "true")  echo "::set-output name=deploy_namespace::$(echo $PIPELINE_SETUP | jq -r '.deploy.namespaces.dev')";;
+            "false") echo "::set-output name=deploy_namespace::$(echo $PIPELINE_SETUP | jq -r '.deploy.namespace')";;
+            *)     echo "::set-output name=deploy_namespace::$(echo $PIPELINE_SETUP | jq -r '.deploy.namespace')";;
+          esac
+
+      - name: Config | Kubectl config file
+        env:
+          KUBE_CONFIG: ${{ secrets.KUBE_CONFIG }}
+        run: |
+          mkdir -p ~/.kube
+          echo "$KUBE_CONFIG" | base64 -d > ~/.kube/config
+
+      # namespace
+      - name: K8s create namespace
+        run: kubectl create namespace ${{steps.deploy_setup.outputs.deploy_namespace}} --dry-run=client --validate=false --output=yaml | kubectl apply -f -
+
+      # dependencies
+      - name: Dependencies | Resources render
+        if: steps.deploy_setup.outputs.has_dev_dependencies == 'true'
+        run: |
+          ytt \
+            -f ./.pipeline/deploy/as-k8s/dependencies.schema.yml \
+            -f ./.pipeline/deploy/as-k8s/dependencies.yml \
+            -f ./manifests/dependencies/dev.yml \
+            --data-value repository=${{steps.deploy_setup.outputs.deploy_repository}} \
+              > ./manifests/k8s-dependencies.yml
+
+      - name: Docs | Dependencies
+        if: steps.deploy_setup.outputs.deploy_as_k8s == 'true'
+        run: | 
+          echo -e "### Dependencies\n\n" >> $GITHUB_STEP_SUMMARY
+          echo -e "<details><summary>dependencies</summary>\n\n\`\`\`yml \n$(cat ./manifests/k8s-dependencies.yml)\n \`\`\`\n </details>\n\n" >> $GITHUB_STEP_SUMMARY
+          echo -e "| name | chart version | app version |" >> $GITHUB_STEP_SUMMARY
+          echo -e "| --- | :---: | :---: |" >> $GITHUB_STEP_SUMMARY
+          if [ $(cat ./manifests/k8s-dependencies.yml | yq -P '.postgres.enabled') = true ]; then
+            echo -e "| postgres | $(cat ./manifests/k8s-dependencies.yml | yq -P '.postgres.version') | $(cat ./manifests/k8s-dependencies.yml | yq -P '.postgres.helm.image.tag') |" >> $GITHUB_STEP_SUMMARY
+          fi
+          if [ $(cat ./manifests/k8s-dependencies.yml | yq -P '.mysql.enabled') = true ]; then
+            echo -e "| mysql | $(cat ./manifests/k8s-dependencies.yml | yq -P '.mysql.version') | $(cat ./manifests/k8s-dependencies.yml | yq -P '.mysql.helm.image.tag') |\n" >> $GITHUB_STEP_SUMMARY
+          fi
+
+      - name: Dependencies | Helm update
+        if: steps.deploy_setup.outputs.has_dev_dependencies == 'true'
+        run: |	
+          helm repo add bitnami https://charts.bitnami.com/bitnami
+          helm repo update
+
+      - name: Dependencies | Mysql
+        if: steps.deploy_setup.outputs.has_dev_dependencies == 'true'
+        continue-on-error: true
+        run: |	
+          K8S_NAMESPACE=${{steps.deploy_setup.outputs.deploy_namespace}} \
+          DEPENDENCY_NAME=mysql \
+          REPOSITORY_TAG_VERSION=${{steps.versioning.outputs.version}} \
+          DEPENDENCY_FILE=./manifests/k8s-dependencies.yml \
+            ./.pipeline/src/dependency-install.sh
+          
+      - name: Dependencies | Postgres
+        if: steps.deploy_setup.outputs.has_dev_dependencies == 'true'
+        continue-on-error: true
+        run: |	
+          K8S_NAMESPACE=${{steps.deploy_setup.outputs.deploy_namespace}} \
+          DEPENDENCY_NAME=postgres \
+          REPOSITORY_TAG_VERSION=${{steps.versioning.outputs.version}} \
+          DEPENDENCY_FILE=./manifests/k8s-dependencies.yml \
+            ./.pipeline/src/dependency-install.sh
+
+      - name: Dependencies | Rabbitmq
+        if: steps.deploy_setup.outputs.has_dev_dependencies == 'true'
+        continue-on-error: true
+        run: |	
+          K8S_NAMESPACE=${{steps.deploy_setup.outputs.deploy_namespace}} \
+          DEPENDENCY_NAME=rabbitmq \
+          REPOSITORY_TAG_VERSION=${{steps.versioning.outputs.version}} \
+          DEPENDENCY_FILE=./manifests/k8s-dependencies.yml \
+            ./.pipeline/src/dependency-install.sh
+
+      - name: Dependencies | Elasticsearch
+        if: steps.deploy_setup.outputs.has_dev_dependencies == 'true'
+        continue-on-error: true
+        run: |	
+          K8S_NAMESPACE=${{steps.deploy_setup.outputs.deploy_namespace}} \
+          DEPENDENCY_NAME=elasticsearch \
+          REPOSITORY_TAG_VERSION=${{steps.versioning.outputs.version}} \
+          DEPENDENCY_FILE=./manifests/k8s-dependencies.yml \
+            ./.pipeline/src/dependency-install.sh
+
+      - name: Dependencies | Redis
+        if: steps.deploy_setup.outputs.has_dev_dependencies == 'true'
+        continue-on-error: true
+        run: |	
+          K8S_NAMESPACE=${{steps.deploy_setup.outputs.deploy_namespace}} \
+          DEPENDENCY_NAME=redis \
+          REPOSITORY_TAG_VERSION=${{steps.versioning.outputs.version}} \
+          DEPENDENCY_FILE=./manifests/k8s-dependencies.yml \
+            ./.pipeline/src/dependency-install.sh
+
+      - name: Dependencies | Nats
+        if: steps.deploy_setup.outputs.has_dev_dependencies == 'true'
+        continue-on-error: true
+        run: |	
+          K8S_NAMESPACE=${{steps.deploy_setup.outputs.deploy_namespace}} \
+          DEPENDENCY_NAME=nats \
+          REPOSITORY_TAG_VERSION=${{steps.versioning.outputs.version}} \
+          DEPENDENCY_FILE=./manifests/k8s-dependencies.yml \
+            ./.pipeline/src/dependency-install.sh
+
+      - name: Dependencies | Etcd
+        if: steps.deploy_setup.outputs.has_dev_dependencies == 'true'
+        continue-on-error: true
+        run: |	
+          K8S_NAMESPACE=${{steps.deploy_setup.outputs.deploy_namespace}} \
+          DEPENDENCY_NAME=etcd \
+          REPOSITORY_TAG_VERSION=${{steps.versioning.outputs.version}} \
+          DEPENDENCY_FILE=./manifests/k8s-dependencies.yml \
+            ./.pipeline/src/dependency-install.sh
+
+      # configs
+      - name: K8s create config-map
+        if: steps.deploy_setup.outputs.has_dev_configs == 'true'
+        run: |
+          kubectl create configmap -n ${{steps.deploy_setup.outputs.deploy_namespace}} svc --from-env-file=./manifests/configs/dev.env --dry-run=client --validate=false --output=yaml | kubectl apply -f -
+          kubectl create configmap -n ${{steps.deploy_setup.outputs.deploy_namespace}} svc-${{steps.versioning.outputs.version}} --from-env-file=./manifests/configs/dev.env --dry-run=client --validate=false --output=yaml | kubectl apply -f -
+          kubectl get configmap -n ${{steps.deploy_setup.outputs.deploy_namespace}} svc -o jsonpath='{.data}' | jq -r 'keys[]' |  tr '\n' '~' | sed 's/~/,/g;s/,$//' > ./manifests/k8s-configs-keys.txt
+
+      # secrets
+      - name: Decrypt DEV secrets
+        if: steps.deploy_setup.outputs.has_dev_secrets == 'true'
+        run: |
+          gpg \
+            --yes --batch --quiet --decrypt \
+            --passphrase="${{ secrets.GPG_PASSPHRASE }}" \
+            --output ./manifests/k8s-secrets.env \
+            ./manifests/secrets/dev.gpg
+
+      - name: K8s create secrets
+        if: steps.deploy_setup.outputs.has_dev_secrets == 'true'
+        run: |
+          kubectl create secret generic -n ${{steps.deploy_setup.outputs.deploy_namespace}} svc --from-env-file=./manifests/k8s-secrets.env --dry-run=client --validate=false --output=yaml | kubectl apply -f -
+          kubectl create secret generic -n ${{steps.deploy_setup.outputs.deploy_namespace}} svc-${{steps.versioning.outputs.version}} --from-env-file=./manifests/k8s-secrets.env --dry-run=client --validate=false --output=yaml | kubectl apply -f -
+          kubectl get secret -n ${{steps.deploy_setup.outputs.deploy_namespace}} svc -o jsonpath='{.data}' | jq -r 'keys[]' |  tr '\n' '~' | sed 's/~/,/g;s/,$//' > ./manifests/k8s-secrets-keys.txt
+
+      # secrets for registry auth
+      - name: Config | Login to Container Registry
+        uses: docker/login-action@v2
+        with:
+          logout: false
+          registry: ${{ inputs.containerRegistry }}
+          username: ${{ secrets.REGISTRY_USERNAME }}
+          password: ${{ secrets.REGISTRY_PASSWORD }}
+
+      - name: Config | Gives runner access to docker config file 
+        if: steps.deploy_setup.outputs.deploy_as_k8s == 'true'
+        run: |
+          sudo chown $(whoami):docker /home/$(whoami)/.docker/config.json
+          cp /home/$(whoami)/.docker/config.json ./manifests/docker-config.json
+
+      - name: K8s create registry-token secret
+        if: steps.deploy_setup.outputs.deploy_as_k8s == 'true'
+        run: kubectl create secret generic -n ${{steps.deploy_setup.outputs.deploy_namespace}} registry-token --type=kubernetes.io/dockerconfigjson --from-file=.dockerconfigjson=./manifests/docker-config.json --dry-run=client --validate=false --output=yaml | kubectl apply -f -
+
+      # generate k8s manifests
+      - name: K8s generates final yml
+        if: steps.deploy_setup.outputs.deploy_as_k8s == 'true'
+        run: |
+          CONFIGS_LIST=$(if test -f ./manifests/k8s-configs-keys.txt; then cat ./manifests/k8s-configs-keys.txt; else echo ''; fi)
+          SECRETS_LIST=$(if test -f ./manifests/k8s-secrets-keys.txt; then cat ./manifests/k8s-secrets-keys.txt; else echo ''; fi)
+          DEPENDENCIES_LIST=$(if test -f ./manifests/k8s-dependencies.yml; then (cat ./manifests/k8s-dependencies.yml | yq -P '.dependencies'); else echo '' ; fi)
+          ytt \
+            -f ./.pipeline/deploy/as-k8s/service.schema.yml \
+            -f ./.pipeline/deploy/as-k8s/service.yml \
+            -f ./manifests/k8s-values.yml \
+            --data-value ecosystem=${{steps.deploy_setup.outputs.deploy_ecosystem}} \
+            --data-value organization=${{steps.deploy_setup.outputs.deploy_organization}} \
+            --data-value repository=${{steps.deploy_setup.outputs.deploy_repository}} \
+            --data-value containerRegistry=${{steps.deploy_setup.outputs.deploy_container_registry}} \
+            --data-value tag=${{steps.deploy_setup.outputs.deploy_tag}} \
+            --data-value-yaml deployment.imagePullSecrets=true \
+            --data-value-yaml envFromSecrets="[$SECRETS_LIST]" \
+            --data-value-yaml envFromConfigMaps="[$CONFIGS_LIST]" \
+            --data-value-yaml envFromDependencies="[$DEPENDENCIES_LIST]" \
+            --data-value pipelineControl.datetime=${{steps.deploy_setup.outputs.run_started_at}} \
+            --data-value-yaml pipelineControl.environmentsAsNamespaces=${{inputs.environmentsAsNamespaces}} \
+              > ./manifests/k8s-to-apply.yml
+
+      - name: Docs | K8s summary
+        if: steps.deploy_setup.outputs.deploy_as_k8s == 'true'
+        run: |
+          CONFIGS_LIST=$(if test -f ./manifests/k8s-configs-keys.txt; then cat ./manifests/k8s-configs-keys.txt; else echo ''; fi)
+          SECRETS_LIST=$(if test -f ./manifests/k8s-secrets-keys.txt; then cat ./manifests/k8s-secrets-keys.txt; else echo ''; fi)
+          DEPENDENCIES_LIST=$(if test -f ./manifests/k8s-dependencies.yml; then (cat ./manifests/k8s-dependencies.yml | yq -P '.dependencies'); else echo ''; fi)
+          echo -e "### k8s\n\n" >> $GITHUB_STEP_SUMMARY
+          echo -e "| param | value |" >> $GITHUB_STEP_SUMMARY
+          echo -e "| --- | :---: |" >> $GITHUB_STEP_SUMMARY
+          echo -e "| secrets | $SECRETS_LIST |" >> $GITHUB_STEP_SUMMARY
+          echo -e "| configs | $CONFIGS_LIST |" >> $GITHUB_STEP_SUMMARY
+          echo -e "| dependencies | $DEPENDENCIES_LIST |" >> $GITHUB_STEP_SUMMARY
+          echo -e "<details><summary>kubefile</summary>\n\n\`\`\`yml \n$(cat ./manifests/k8s-to-apply.yml)\n \`\`\`\n </details>\n\n" >> $GITHUB_STEP_SUMMARY
+
+      - name: K8s apply yml
+        if: steps.deploy_setup.outputs.deploy_as_k8s == 'true'
+        run: |
+          kubectl apply -f ./manifests/k8s-to-apply.yml
+          kubectl get pods -n ${{steps.deploy_setup.outputs.deploy_namespace}}

.github/workflows/app-push.yml

@@ -45,10 +45,6 @@ on:
         type: string
         required: false
         default: '[14, 16, 17]'
-      pythonMatrix:
-        type: string
-        required: false
-        default: '["2.7", "3.6", "3.8", "3.10"]'
       platforms:
         type: string
         required: false