Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

ci: alert on slack when pipelines fail #1233

Merged
merged 2 commits into from
Oct 14, 2024
Merged

ci: alert on slack when pipelines fail #1233

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
4ff6aa6
ci: alert on slack when pipelines fail
arealmaas Oct 14, 2024
cfc8dfa
cleanup
arealmaas Oct 14, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
54 changes: 54 additions & 0 deletions .github/slack-templates/pipeline-failed.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
{
"attachments": [
{
"color": "#FF0000",
"blocks": [
{
"type": "header",
"text": {
"type": "plain_text",
"text": "Github pipeline failed",
"emoji": true
}
},
{
"type": "section",
"text": {
"type": "mrkdwn",
"text": "*Environment:* ${{ env.ENVIRONMENT }}"
}
},
{
"type": "section",
"text": {
"type": "mrkdwn",
"text": "*Job Status:*\n• Infrastructure: ${{ env.INFRA_STATUS }}\n• Apps: ${{ env.APPS_STATUS }}\n• Build and Test: ${{ env.BUILD_AND_TEST_STATUS }}\n• Docker Build and Push: ${{ env.DOCKER_BUILD_AND_PUSH_STATUS }}\n• Publish Node Logger to NPM: ${{ env.PUBLISH_NODE_LOGGER_TO_NPM_STATUS }}"
}
},
{
"type": "section",
"text": {
"type": "mrkdwn",
"text": "Please check the workflow for more details."
}
},
{
"type": "divider"
},
{
"type": "actions",
"elements": [
{
"type": "button",
"text": {
"type": "plain_text",
"text": "View Run"
},
"url": "${{ env.RUN_URL }}"
}
]
}
]
}
]
}
22 changes: 21 additions & 1 deletion .github/workflows/ci-cd-main.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,7 @@ jobs:
check-for-changes,
generate-git-short-sha,
docker-build-and-push,
deploy-infrastructure,
deploy-infrastructure
]
# we want deployment of apps to be dependent on deployment of infrastructure, but if infrastructure is skipped, we still want to deploy the apps
if: ${{ always() && !failure() && !cancelled() && (github.event_name == 'workflow_dispatch' || needs.check-for-changes.outputs.hasApplicationChanges == 'true') }}
Expand All @@ -116,3 +116,23 @@ jobs:
region: norwayeast
version: ${{ needs.get-current-version.outputs.version }}-${{ needs.generate-git-short-sha.outputs.gitShortSha }}
runMigration: ${{ github.event_name == 'workflow_dispatch' || needs.check-for-changes.outputs.hasMigrationChanges == 'true' }}

send-slack-message-on-failure:
name: Send Slack message on failure
needs: [
build-and-test,
docker-build-and-push,
deploy-infrastructure,
deploy-apps,
]
if: ${{ always() && failure() && !cancelled() }}
uses: ./.github/workflows/workflow-send-ci-cd-status-slack-message.yml
with:
environment: test
build_and_test_status: ${{ needs.build-and-test.result }}
docker_build_and_push_status: ${{ needs.docker-build-and-push.result }}
infra_status: ${{ needs.deploy-infrastructure.result }}
apps_status: ${{ needs.deploy-apps.result }}
secrets:
SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
SLACK_CHANNEL_ID: ${{ secrets.SLACK_CHANNEL_ID_FOR_CI_CD_STATUS }}
16 changes: 16 additions & 0 deletions .github/workflows/ci-cd-prod.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -114,3 +114,19 @@ jobs:
region: norwayeast
version: ${{ needs.get-current-version.outputs.version }}
runMigration: ${{ github.event_name == 'workflow_dispatch' || needs.check-for-changes.outputs.hasMigrationChanges == 'true' }}

send-slack-message-on-failure:
name: Send Slack message on failure
needs: [
deploy-infrastructure,
deploy-apps,
]
if: ${{ always() && failure() && !cancelled() }}
uses: ./.github/workflows/workflow-send-ci-cd-status-slack-message.yml
with:
environment: prod
infra_status: ${{ needs.deploy-infrastructure.result }}
apps_status: ${{ needs.deploy-apps.result }}
secrets:
SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
SLACK_CHANNEL_ID: ${{ secrets.SLACK_CHANNEL_ID_FOR_CI_CD_STATUS }}
24 changes: 23 additions & 1 deletion .github/workflows/ci-cd-staging.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -90,4 +90,26 @@ jobs:
version: ${{ needs.get-current-version.outputs.version }}
pathToPackage: packages/node-logger
secrets:
NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
NPM_TOKEN: ${{ secrets.NPM_TOKEN }}


send-slack-message-on-failure:
name: Send Slack message on failure
needs: [
docker-build-and-push,
deploy-infrastructure,
deploy-apps,
publish-node-logger-to-npm,
]
if: ${{ always() && failure() && !cancelled() }}
uses: ./.github/workflows/workflow-send-ci-cd-status-slack-message.yml
with:
environment: test
docker_build_and_push_status: ${{ needs.docker-build-and-push.result }}
infra_status: ${{ needs.deploy-infrastructure.result }}
apps_status: ${{ needs.deploy-apps.result }}
publish_node_logger_to_npm_status: ${{ needs.publish-node-logger-to-npm.result }}
secrets:
SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
SLACK_CHANNEL_ID: ${{ secrets.SLACK_CHANNEL_ID_FOR_CI_CD_STATUS }}

79 changes: 79 additions & 0 deletions .github/workflows/workflow-send-ci-cd-status-slack-message.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,79 @@
name: Send CI/CD Status Slack Message

on:
workflow_call:
inputs:
environment:
required: true
type: string
infra_status:
type: string
description: "Status of the infrastructure deployment job"
default: "skipped"
apps_status:
type: string
description: "Status of the apps deployment job"
default: "skipped"
docker_build_and_push_status:
type: string
description: "Status of the docker image publishing job"
default: "skipped"
build_and_test_status:
type: string
description: "Status of the build and test job"
default: "skipped"
publish_node_logger_to_npm_status:
type: string
description: "Status of the publish node logger to npm job"
default: "skipped"
secrets:
SLACK_BOT_TOKEN:
required: true
SLACK_CHANNEL_ID:
required: true

jobs:
send-slack-message:
name: Send Slack message
runs-on: ubuntu-latest
steps:
- name: Checkout repository
uses: actions/checkout@v4

- name: Determine status emojis
id: status-emojis
run: |
determine_emoji() {
local -A emoji_map=(
[success]=":white_check_mark:"
[failure]=":x:"
[cancelled]=":warning:"
[skipped]=":ballot_box_with_check:"
)
echo "${emoji_map[$1]:-Invalid status: $1}"
}

{
echo "INFRA_EMOJI=$(determine_emoji "${{ inputs.infra_status }}")"
echo "APPS_EMOJI=$(determine_emoji "${{ inputs.apps_status }}")"
echo "BUILD_AND_TEST_EMOJI=$(determine_emoji "${{ inputs.build_and_test_status }}")"
echo "DOCKER_BUILD_AND_PUSH_EMOJI=$(determine_emoji "${{ inputs.docker_build_and_push_status }}")"
echo "PUBLISH_NODE_LOGGER_TO_NPM_EMOJI=$(determine_emoji "${{ inputs.publish_node_logger_to_npm_status }}")"
} >> "$GITHUB_OUTPUT"

- name: Send GitHub slack message
id: slack
env:
SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
ENVIRONMENT: ${{ inputs.environment }}
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
# statuses
INFRA_STATUS: "${{ steps.status-emojis.outputs.INFRA_EMOJI }}"
APPS_STATUS: "${{ steps.status-emojis.outputs.APPS_EMOJI }}"
BUILD_AND_TEST_STATUS: "${{ steps.status-emojis.outputs.BUILD_AND_TEST_EMOJI }}"
DOCKER_BUILD_AND_PUSH_STATUS: "${{ steps.status-emojis.outputs.DOCKER_BUILD_AND_PUSH_EMOJI }}"
PUBLISH_NODE_LOGGER_TO_NPM_STATUS: "${{ steps.status-emojis.outputs.PUBLISH_NODE_LOGGER_TO_NPM_EMOJI }}"
uses: slackapi/slack-github-action@v1.27.0
with:
channel-id: ${{ secrets.SLACK_CHANNEL_ID }}
payload-file-path: "./.github/slack-templates/pipeline-failed.json"
Loading