Rollback the inbound_mail service in the Development environment #64
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Rollback | |
| run-name: Rollback the ${{ inputs.service }} service in the ${{ inputs.environment }} environment | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| service: | |
| type: choice | |
| description: The microservice which we want to rollback to the previous version. | |
| options: | |
| - inbound_mail | |
| - api | |
| - web | |
| - webhook | |
| - widget | |
| - worker | |
| - ws | |
| environment: | |
| type: choice | |
| description: The environment where the service is located. | |
| options: | |
| - Development | |
| - Production | |
| region: | |
| type: choice | |
| description: The region of the environment. It's needed to know TF workspace. When you choose DEV environment this option is not needed. | |
| options: | |
| - [EU] | |
| - [US] | |
| - [EU,US] | |
| mode: | |
| type: choice | |
| description: The Rollback mode. You can roll back to the previously deployed version or to the version that has the current commit hash of this branch in an image tag name or a deployment info. | |
| options: | |
| - Previous Version | |
| - Commit Hash | |
| jobs: | |
| ecs: | |
| if: contains(fromJson('["api", "inbound_mail", "webhook", "worker", "ws"]'), github.event.inputs.service) | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| region: ${{ fromJSON(github.event.inputs.region) }} | |
| timeout-minutes: 60 | |
| environment: ${{ github.event.inputs.environment }} | |
| permissions: | |
| contents: read | |
| packages: write | |
| deployments: write | |
| steps: | |
| - run: echo "Rolling back ${{ github.event.inputs.service }} in ${{ github.event.inputs.environment }}" | |
| - id: commit | |
| if: contains(fromJson('["Commit Hash"]'), github.event.inputs.mode) | |
| uses: prompt/actions-commit-hash@v3 | |
| - name: Prepare variables | |
| id: variables | |
| run: | | |
| if [[ "${{ matrix.region }}" == "EU" && "${{ github.event.inputs.environment }}" == "Production" ]]; then | |
| echo "Using Terraform Workspace: novu-prod-eu" | |
| echo "TF_WORKSPACE=novu-prod-eu" >> $GITHUB_ENV | |
| elif [[ "${{ matrix.region }}" == "US" && "${{ github.event.inputs.environment }}" == "Production" ]]; then | |
| echo "Using Terraform Workspace: novu-prod" | |
| echo "TF_WORKSPACE=novu-prod" >> $GITHUB_ENV | |
| elif [[ "${{ matrix.region }}" == "EU" && "${{ github.event.inputs.environment }}" == "Development" ]]; then | |
| echo "Using Terraform Workspace: novu-dev" | |
| echo "TF_WORKSPACE=novu-dev" >> $GITHUB_ENV | |
| elif [[ "${{ matrix.region }}" == "US" && "${{ github.event.inputs.environment }}" == "Development" ]]; then | |
| echo "Using Terraform Workspace: novu-dev" | |
| echo "TF_WORKSPACE=novu-dev" >> $GITHUB_ENV | |
| echo "Error: Development environment doesn't exist in the US region." >&2 | |
| exit 1 | |
| else | |
| echo "Using Terraform Workspace: novu-dev" | |
| echo "TF_WORKSPACE=novu-dev" >> $GITHUB_ENV | |
| fi | |
| - name: Checkout cloud infra | |
| uses: actions/checkout@master | |
| with: | |
| repository: novuhq/cloud-infra | |
| token: ${{ secrets.GH_PACKAGES }} | |
| path: cloud-infra | |
| - name: Terraform setup | |
| uses: hashicorp/setup-terraform@v1 | |
| with: | |
| cli_config_credentials_token: ${{ secrets.TF_API_TOKEN }} | |
| terraform_version: 1.5.5 | |
| terraform_wrapper: false | |
| - name: Terraform Init | |
| working-directory: cloud-infra/terraform/novu/aws | |
| run: terraform init | |
| - name: Terraform get output | |
| working-directory: cloud-infra/terraform/novu/aws | |
| id: terraform | |
| env: | |
| SERVICE_NAME: ${{ github.event.inputs.service }} | |
| run: | | |
| echo "ecs_container_name=$(terraform output -json ${{ env.SERVICE_NAME }}_ecs_container_name | jq -r .)" >> $GITHUB_ENV | |
| echo "ecs_service=$(terraform output -json ${{ env.SERVICE_NAME }}_ecs_service | jq -r .)" >> $GITHUB_ENV | |
| echo "ecs_cluster=$(terraform output -json ${{ env.SERVICE_NAME }}_ecs_cluster | jq -r .)" >> $GITHUB_ENV | |
| echo "task_name=$(terraform output -json ${{ env.SERVICE_NAME }}_task_name | jq -r .)" >> $GITHUB_ENV | |
| echo "aws_region=$(terraform output -json aws_region | jq -r .)" >> $GITHUB_ENV | |
| - name: Configure AWS credentials | |
| uses: aws-actions/configure-aws-credentials@v4 | |
| with: | |
| aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| aws-region: ${{ env.aws_region }} | |
| - name: ECS get output | |
| if: contains(fromJson('["Previous Version"]'), github.event.inputs.mode) | |
| id: ecs-output | |
| run: | | |
| echo "Retrieving current_task_definition_arn..." | |
| current_task_definition_arn=$(aws ecs describe-services --cluster ${{ env.ecs_cluster }} --services ${{ env.ecs_service }} --query 'services[0].taskDefinition' --output text) | |
| echo "current_task_definition_arn=$current_task_definition_arn" >> $GITHUB_ENV | |
| echo "Retrieving task_definition_family..." | |
| task_definition_family=$(aws ecs describe-task-definition --task-definition ${{ env.task_name }} --query 'taskDefinition.family' --output text) | |
| echo "task_definition_family=$task_definition_family" >> $GITHUB_ENV | |
| echo "Retrieving task_definition_list..." | |
| task_definition_list=$(aws ecs list-task-definitions --family-prefix "${task_definition_family}" --output text --sort DESC | grep 'TASKDEFINITIONARNS' | cut -f 2) | |
| task_definition_list_formatted=$(echo "$task_definition_list" | tr '\n' '|') # Replace newline with '|' | |
| echo "task_definition_list=$task_definition_list_formatted" >> $GITHUB_ENV | |
| if [ -n "$task_definition_list" ]; then | |
| echo "Retrieving previous_task_definition_arn..." | |
| index=$(echo "$task_definition_list" | grep -n "$current_task_definition_arn" | cut -d ':' -f 1) | |
| if [ -n "$index" ]; then | |
| if [ "$index" -ge 1 ]; then # Greater than or equal to 1 | |
| previous_index=$((index + 1)) | |
| previous_task_definition_arn=$(echo "$task_definition_list" | sed -n "${previous_index}p") | |
| echo "previous_task_definition_arn=$previous_task_definition_arn" >> $GITHUB_ENV | |
| else | |
| echo "Invalid index value: $index" | |
| fi | |
| else | |
| echo "Previous task definition not found. It seems to me someone deleted the current task from the list and that is why I can't find the previous task." | |
| exit 1 | |
| fi | |
| else | |
| echo "No task definitions found." | |
| exit 1 | |
| fi | |
| - name: ECS get output. By using commit hash | |
| if: contains(fromJson('["Commit Hash"]'), github.event.inputs.mode) | |
| id: ecs-output-commit-hash | |
| env: | |
| IMAGE_TAG: ${{ steps.commit.outputs.hash }} | |
| run: | | |
| task_definition_family=$(aws ecs describe-task-definition --task-definition ${{ env.task_name }} --query 'taskDefinition.family' --output text) | |
| task_definition_arns=$(aws ecs list-task-definitions --family-prefix "${task_definition_family}" --query 'taskDefinitionArns' --output text --sort DESC) | |
| found=false | |
| counter=0 | |
| for arn in $task_definition_arns; do | |
| task_definition=$(aws ecs describe-task-definition --task-definition $arn) | |
| if echo "$task_definition" | grep -q "$IMAGE_TAG"; then | |
| echo "Found task definition with image tag $IMAGE_TAG: $arn" | |
| needed_arn=$arn | |
| found=true | |
| break # Stop the loop | |
| fi | |
| ((counter++)) | |
| if [ $counter -ge 20 ]; then | |
| break # Stop the loop | |
| fi | |
| done | |
| if [ "$found" = false ]; then | |
| echo "Error: Task definition with image tag $IMAGE_TAG not found within the last 20 tasks." | |
| exit 1 | |
| fi | |
| current_task_definition_arn=$(aws ecs describe-services --cluster ${{ env.ecs_cluster }} --services ${{ env.ecs_service }} --query 'services[0].taskDefinition' --output text) | |
| echo "current_task_definition_arn=$current_task_definition_arn" >> $GITHUB_ENV | |
| echo "previous_task_definition_arn=$needed_arn" >> $GITHUB_ENV | |
| echo "Your task definition ARN is $needed_arn" | |
| - name: Rollback a service to the previous task definition | |
| id: rollback | |
| env: | |
| PREVIOUS_TASK: ${{ env.previous_task_definition_arn }} | |
| CURRENT_TASK: ${{ env.current_task_definition_arn }} | |
| run: | | |
| aws ecs update-service --cluster ${{ env.ecs_cluster }} --service ${{ env.ecs_service }} --task-definition ${{ env.PREVIOUS_TASK }} | |
| aws ecs wait services-stable --cluster ${{ env.ecs_cluster }} --service ${{ env.ecs_service }} | |
| echo "After Rollback:" | |
| echo "The previous task definition: $(echo $CURRENT_TASK | awk -F'task-definition/' '{print $2}')" | |
| echo "The current task definition: $(echo $PREVIOUS_TASK | awk -F'task-definition/' '{print $2}')" | |
| netlify: | |
| if: contains(fromJson('["web", "widget"]'), github.event.inputs.service) | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| region: ${{ fromJSON(github.event.inputs.region) }} | |
| timeout-minutes: 60 | |
| environment: ${{ github.event.inputs.environment }} | |
| permissions: | |
| contents: read | |
| packages: write | |
| deployments: write | |
| env: | |
| NETLIFY_ACCESS_TOKEN: ${{ secrets.NETLIFY_AUTH_TOKEN }} | |
| steps: | |
| - run: echo "Rolling back ${{ github.event.inputs.service }} in ${{ github.event.inputs.environment }}" | |
| - id: commit-netlify | |
| if: contains(fromJson('["Commit Hash"]'), github.event.inputs.mode) | |
| uses: prompt/actions-commit-hash@v3 | |
| - name: Prepare variables | |
| id: variables | |
| run: | | |
| if [[ "${{ github.event.inputs.service }}" == "widget" && "${{ github.event.inputs.environment }}" == "Development" && "${{ matrix.region }}" == "EU" ]]; then | |
| echo "Using netlify_site_id: b9147448-b835-4eb1-a2f0-11102f611f5f" | |
| echo "netlify_site_id=b9147448-b835-4eb1-a2f0-11102f611f5f" >> $GITHUB_ENV | |
| elif [[ "${{ github.event.inputs.service }}" == "web" && "${{ github.event.inputs.environment }}" == "Development" && "${{ matrix.region }}" == "EU" ]]; then | |
| echo "Using netlify_site_id: 45396446-dc86-4ad6-81e4-86d3eb78d06f" | |
| echo "netlify_site_id=45396446-dc86-4ad6-81e4-86d3eb78d06f" >> $GITHUB_ENV | |
| elif [[ "${{ github.event.inputs.environment }}" == "Development" && "${{ matrix.region }}" == "US" ]]; then | |
| echo "Error: Development environment doesn't exist in the US region." >&2 | |
| exit 1 | |
| elif [[ "${{ github.event.inputs.service }}" == "web" && "${{ github.event.inputs.environment }}" == "Production" && "${{ matrix.region }}" == "EU" ]]; then | |
| echo "Using netlify_site_id: d2e8b860-7016-4202-9256-ebca0f13259a" | |
| echo "netlify_site_id=d2e8b860-7016-4202-9256-ebca0f13259a" >> $GITHUB_ENV | |
| elif [[ "${{ github.event.inputs.service }}" == "web" && "${{ github.event.inputs.environment }}" == "Production" && "${{ matrix.region }}" == "US" ]]; then | |
| echo "Using netlify_site_id: 8639d8b9-81f9-44c3-b885-585a7fd2b5ff" | |
| echo "netlify_site_id=8639d8b9-81f9-44c3-b885-585a7fd2b5ff" >> $GITHUB_ENV | |
| elif [[ "${{ github.event.inputs.service }}" == "widget" && "${{ github.event.inputs.environment }}" == "Production" && "${{ matrix.region }}" == "EU" ]]; then | |
| echo "Using netlify_site_id: 20a64bdd-1934-4284-875f-862410c69a3b" | |
| echo "netlify_site_id=20a64bdd-1934-4284-875f-862410c69a3b" >> $GITHUB_ENV | |
| elif [[ "${{ github.event.inputs.service }}" == "widget" && "${{ github.event.inputs.environment }}" == "Production" && "${{ matrix.region }}" == "US" ]]; then | |
| echo "Using netlify_site_id: 6f927fd4-dcb0-4cf3-8c0b-8c5539d0d034" | |
| echo "netlify_site_id=6f927fd4-dcb0-4cf3-8c0b-8c5539d0d034" >> $GITHUB_ENV | |
| fi | |
| - name: Get Current Deploy ID | |
| if: contains(fromJson('["Previous Version"]'), github.event.inputs.mode) | |
| id: get_current_deploy | |
| env: | |
| NETLIFY_SITE_ID: ${{ env.netlify_site_id }} | |
| run: | | |
| response=$(curl -s -H "Authorization: Bearer $NETLIFY_ACCESS_TOKEN" "https://api.netlify.com/api/v1/sites/${NETLIFY_SITE_ID}") | |
| current_deploy_id=$(echo "$response" | jq -r '.published_deploy.id') | |
| echo "current_deploy_id=$current_deploy_id" >> $GITHUB_ENV | |
| - name: Find Previous Production Deployments and Determine Previous Deploy ID | |
| if: contains(fromJson('["Previous Version"]'), github.event.inputs.mode) | |
| id: previous_deploy_id | |
| env: | |
| NETLIFY_SITE_ID: ${{ env.netlify_site_id }} | |
| run: | | |
| response=$(curl -s -H "Authorization: Bearer $NETLIFY_ACCESS_TOKEN" "https://api.netlify.com/api/v1/sites/${NETLIFY_SITE_ID}/deploys?per_page=100") | |
| deploy_ids=$(echo "$response" | jq -r '.[] | select(.context == "production" and .state == "ready" and .published_at != null) | .id' | sort) | |
| current_index=$(echo "$deploy_ids" | grep -n "$current_deploy_id" | cut -d ":" -f 1) | |
| previous_index=$((current_index - 1)) | |
| previous_deploy_id=$(echo "$deploy_ids" | sed "${previous_index}q;d") | |
| echo "previous_deploy_id=$previous_deploy_id" >> $GITHUB_ENV | |
| - name: Determine Previous Deploy ID | |
| if: contains(fromJson('["Commit Hash"]'), github.event.inputs.mode) | |
| env: | |
| NETLIFY_SITE_ID: ${{ env.netlify_site_id }} | |
| COMMIT_REF: ${{ steps.commit-netlify.outputs.hash }} | |
| run: | | |
| response=$(curl -s -H "Authorization: Bearer $NETLIFY_ACCESS_TOKEN" "https://api.netlify.com/api/v1/sites/$NETLIFY_SITE_ID/deploys") | |
| deploy_id=$(echo "$response" | jq -r ".[] | select(.commit_ref == \"$COMMIT_REF\") | .id") | |
| if [ -n "$deploy_id" ]; then | |
| echo "Deploy ID for commit $COMMIT_REF: $deploy_id" | |
| echo "previous_deploy_id=$deploy_id" >> $GITHUB_ENV | |
| else | |
| echo "Deploy not found for commit $COMMIT_REF" | |
| exit 1 | |
| fi | |
| - name: Rollback to Previous Deploy | |
| if: env.previous_deploy_id != null | |
| env: | |
| NETLIFY_SITE_ID: ${{ env.netlify_site_id }} | |
| run: | | |
| echo "Restoring previous deploy..." | |
| curl -X POST -H "Authorization: Bearer $NETLIFY_ACCESS_TOKEN" "https://api.netlify.com/api/v1/sites/${{ env.netlify_site_id }}/deploys/${{ env.previous_deploy_id }}/restore" |