From 702027e10a63a359257f63e187767b83af857c98 Mon Sep 17 00:00:00 2001 From: Volodymyr Pochtar Date: Thu, 12 Jan 2023 12:30:26 +0200 Subject: [PATCH] feat: mention user on failed master branch build (#21201) --- .../match-github-to-slack-user/action.yml | 24 +++ .github/workflows/gradle.yml | 44 +++--- tools/bin/match_github_user_to_slack | 139 ++++++++++++++++++ 3 files changed, 188 insertions(+), 19 deletions(-) create mode 100644 .github/actions/match-github-to-slack-user/action.yml create mode 100755 tools/bin/match_github_user_to_slack diff --git a/.github/actions/match-github-to-slack-user/action.yml b/.github/actions/match-github-to-slack-user/action.yml new file mode 100644 index 000000000000..ea2b3f49a5b6 --- /dev/null +++ b/.github/actions/match-github-to-slack-user/action.yml @@ -0,0 +1,24 @@ +# This action will try to match git commit author (GITHUB_ACTOR) with Slack user +# and add it to GITHUB_OUTPUT +# Following env variables should be provided. +# Provided by Github: +# GITHUB_ACTOR: commit author +# GITHUB_REPOSITORY: name of the repo we check the commit author, e.g. "airbytehq/airbyte-cloud" +# Required: +# AIRBYTE_TEAM_BOT_SLACK_TOKEN: ${{ secrets.AIRBYTE_TEAM_BOT_SLACK_TOKEN }} +# GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }} + +name: "Match Github user to Slack user" +description: "Match Github user to Slack by email or full name in Github profile." +outputs: + slack_user_ids: + description: "Comma separated slack user IDs that match to GITHUB_ACTOR (Github username)" + value: ${{ steps.match-github-to-slack-user.outputs.slack_user_ids }} +runs: + using: "composite" + steps: + - name: Match github user to slack user + id: match-github-to-slack-user + run: | + ./tools/bin/match_github_user_to_slack + shell: bash diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index 25abc23e4d60..8bda14232f84 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -11,10 +11,10 @@ on: debug_mode: description: "Enable or disable tmate session for debug during helm ac tests" type: choice - default: 'false' - options: - - 'true' - - 'false' + default: "false" + options: + - "true" + - "false" required: false schedule: - cron: "0 */1 * * *" @@ -756,7 +756,7 @@ jobs: github-token: ${{ env.PAT }} label: ${{ needs.start-platform-build-runner.outputs.label }} ec2-instance-id: ${{ needs.start-platform-build-runner.outputs.ec2-instance-id }} - + ## Kube Acceptance Tests # Docker acceptance tests run as part of the build job. # In case of self-hosted EC2 errors, remove this block. @@ -822,7 +822,7 @@ jobs: - uses: azure/setup-helm@v3 with: - version: 'latest' + version: "latest" token: ${{ secrets.GITHUB_TOKEN }} id: install @@ -1036,7 +1036,7 @@ jobs: - name: Fix EC-2 Runner run: | mkdir -p /actions-runner/_work/airbyte/airbyte && mkdir -p /actions-runner/_work/airbyte/airbyte/.kube - + - name: Checkout Airbyte uses: actions/checkout@v2 with: @@ -1059,7 +1059,7 @@ jobs: - uses: actions/setup-python@v4 with: - python-version: '3.9' + python-version: "3.9" - uses: actions/setup-java@v1 with: @@ -1071,13 +1071,13 @@ jobs: - uses: azure/setup-helm@v3 with: - version: 'latest' + version: "latest" token: ${{ secrets.GITHUB_TOKEN }} id: install - uses: azure/setup-kubectl@v3 with: - version: 'v1.21.2' # default is latest stable + version: "v1.21.2" # default is latest stable - name: Install tmate and it's dependencies if: inputs.debug_mode == 'true' @@ -1087,11 +1087,11 @@ jobs: sudo apt-get -y install tmate attempt_limit: 3 attempt_delay: 2000 # in ms - + - name: Start tmate session in background if: inputs.debug_mode == 'true' shell: bash - run: | + run: | tmate -S /tmp/tmate.sock new-session -d # Launch tmate in a headless mode tmate -S /tmp/tmate.sock wait tmate-ready # Blocks until the SSH connection is established tmate -S /tmp/tmate.sock display -p '#{tmate_ssh}' # Prints the SSH connection string @@ -1152,10 +1152,10 @@ jobs: - name: Generate Test Report uses: dorny/test-reporter@v1 - if: always() # run this step even if previous step failed + if: always() # run this step even if previous step failed with: name: Platform Helm E2E Test Report - path: './*/build/test-results/*/*.xml' + path: "./*/build/test-results/*/*.xml" reporter: java-junit - uses: actions/upload-artifact@v2 @@ -1163,7 +1163,7 @@ jobs: with: name: Kubernetes Logs path: /tmp/kubernetes_logs/* - + - name: Upload test results to BuildPulse for flaky test detection if: "!cancelled()" # Run this step even when the tests fail. Skip if the workflow is cancelled. uses: Workshop64/buildpulse-action@main @@ -1173,13 +1173,12 @@ jobs: path: "/actions-runner/_work/airbyte/airbyte/*" key: ${{ secrets.BUILDPULSE_ACCESS_KEY_ID }} secret: ${{ secrets.BUILDPULSE_SECRET_ACCESS_KEY }} - + - name: "Display logs of k3s" if: failure() shell: bash run: | journalctl -xeu k3s.service - # # In case of self-hosted EC2 errors, remove this block. stop-helm-acceptance-test-runner: @@ -1216,8 +1215,6 @@ jobs: label: ${{ needs.start-helm-acceptance-test-runner.outputs.label }} ec2-instance-id: ${{ needs.start-helm-acceptance-test-runner.outputs.ec2-instance-id }} - - notify-failure-slack-channel: name: "Notify Slack Channel on Build Failures" runs-on: ubuntu-latest @@ -1231,6 +1228,14 @@ jobs: # - helm-acceptance-test if: ${{ failure() && github.ref == 'refs/heads/master' }} steps: + - name: Checkout Airbyte + uses: actions/checkout@v3 + - name: Match GitHub User to Slack User + id: match-github-to-slack-user + uses: ./.github/actions/match-github-to-slack-user + env: + AIRBYTE_TEAM_BOT_SLACK_TOKEN: ${{ secrets.SLACK_AIRBYTE_TEAM_READ_USERS }} + GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Publish to OSS Build Failure Slack Channel uses: abinoda/slack-action@master env: @@ -1241,6 +1246,7 @@ jobs: {\"type\":\"divider\"}, {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\" Merge to OSS Master failed! :bangbang: \n\n\"}}, {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\"_merged by_: *${{ github.actor }}* \n\"}}, + {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\"<@${{ steps.match-github-to-slack-user.outputs.slack_user_ids }}> \n\"}}, {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\" :octavia-shocked: :octavia-shocked: \n\"}}, {\"type\":\"divider\"}]} diff --git a/tools/bin/match_github_user_to_slack b/tools/bin/match_github_user_to_slack new file mode 100755 index 000000000000..7de1add11ffa --- /dev/null +++ b/tools/bin/match_github_user_to_slack @@ -0,0 +1,139 @@ +#!/usr/bin/env bash + +set -e + +function usage() { + echo "Usage: $0 -r GITHUB_REPOSITORY [-l GITHUB_ACTOR]" + echo "Either of the parameters below have to be provided:" + echo " -l GITHUB_ACTOR - github user login" +} + +function error_exit() { + usage + echo $1 >>/dev/stderr; + exit 1 +} + +function lc() { + tr '[:upper:]' '[:lower:]' +} + +function uc() { + tr '[:lower:]' '[:upper:]' +} + +function jq_filter() { + local name=$(echo -n "$1" | lc) + filter='' + filter+="( .profile.email | tostring | ascii_downcase | startswith(\"$name \") )" + filter+=" or ( .profile.email | tostring | ascii_downcase == (\"$name\") )" + filter+=" or ( .profile.display_name | tostring | ascii_downcase | startswith(\"$name \") )" + filter+=" or ( .profile.display_name | tostring | ascii_downcase | endswith(\" $name\") )" + filter+=" or ( .profile.display_name | tostring | ascii_downcase == (\"$name\") )" + filter+=" or ( .name | tostring | ascii_downcase | startswith(\"$name \") )" + filter+=" or ( .name | tostring | ascii_downcase | endswith(\" $name\") )" + filter+=" or ( (.name | tostring | ascii_downcase) == \"$name\" )" + filter+=" or ( .real_name | tostring | ascii_downcase | startswith(\"$name \") )" + filter+=" or ( .real_name | tostring | ascii_downcase | endswith(\" $name\") )" + filter+=" or ( .real_name | tostring | ascii_downcase == (\"$name\") )" + + + echo "${filter}" +} + +function fetch_slack_users() { + local slack_api_key=$1 + echo "Fetching slack users ..." >>/dev/stderr + slack_users="$(curl -s -H "Authorization: Bearer ${slack_api_key}" -H "Content-type: application/json" https://slack.com/api/users.list?limit=2000)" + echo "$slack_users" > ./slack_users.json + slack_page_id="$(echo ${slack_users} | jq -r '.response_metadata.next_cursor')" + x=0 + echo ${slack_page_id} >>/dev/stderr + if [ -n "${slack_page_id}" ]; then + while [ -n "${slack_page_id}" ]; do + echo "Extracting page info ${slack_page_id} for part ${x}" >>/dev/stderr + slack_users_paginator="$(curl -s -H "Authorization: Bearer ${slack_api_key}" -G -H "Content-type: application/json" --data-urlencode "cursor=${slack_page_id}" https://slack.com/api/users.list)" + slack_page_id="$(echo ${slack_users_paginator} | jq -r '.response_metadata.next_cursor')" + echo "$slack_users_paginator" > ./slack_users_${x}.json + x=$(( $x + 1 )) + done + all_slack_users="$(jq -n '[ inputs.members ] | add ' slack_users*.json)" + else + all_slack_users="$(jq -n '[ inputs.members ] | add ' slack_users*.json)" + fi + echo ${all_slack_users} +} + +[ -z "${AIRBYTE_TEAM_BOT_SLACK_TOKEN}" ] && error_exit "ERROR: AIRBYTE_TEAM_BOT_SLACK_TOKEN is not set." + +while getopts ":r:c:u:l:" opt; do + case ${opt} in + r ) + GITHUB_REPOSITORY=$OPTARG + ;; + l ) + GITHUB_ACTOR=$OPTARG + ;; + \? ) usage + ;; + esac +done + + +[ -z "$GITHUB_ACTOR" ] && error_exit "ERROR: GITHUB_ACTOR (-l GITHUB_ACTOR) have to be provided." + +if [ -n "$GITHUB_ACTOR" ]; then + [ -z "${GITHUB_API_TOKEN}" ] && error_exit "ERROR: GITHUB_API_TOKEN is not set." +fi + + +if [ -n "$GITHUB_ACTOR" ]; then + echo "Fetching github user ..." >>/dev/stderr + github_user="$(curl -s -H "Authorization: token ${GITHUB_API_TOKEN}" https://api.github.com/users/${GITHUB_ACTOR})" + echo $github_user + email="$(echo "$github_user" | jq -r '.email | select(type == "string")')" + name="$(echo "$github_user" | jq -r '.name | select(type == "string")')" + if [ -z "$email" -a -z "$name" ]; then + echo "WARNING: Github user ($GITHUB_ACTOR) does not have either name nor email in the profile." >>/dev/stderr + echo "Fetching commits made by this user to detect user by commit author name and/or email." >>/dev/stderr + [ -z "$GITHUB_REPOSITORY" ] && error_exit "ERROR: Git reposuitory (-r GITHUB_REPOSITORY) is missing." + commit="$(curl -s -H "Authorization: token ${GITHUB_API_TOKEN}" https://api.github.com/repos/${GITHUB_REPOSITORY}/commits?author=$GITHUB_ACTOR | jq '.[0] | .commit | select(type == "object")' )" + github_user="$(echo "$commit" | jq '.author | select(type == "object")')" + fi + [ -z "$github_user" ] && error_exit "ERROR: Gitub user is not found for commit ${GITHUB_ACTOR}" +else + github_user="$GITHUB_USER" +fi + +airbyte_team_slack_users=$(fetch_slack_users ${AIRBYTE_TEAM_BOT_SLACK_TOKEN}) + +all_users_list=$(jq -s '.[0] + .[1]' < all_slack_users.json + +OLDIFS=$IFS; +IFS=$'\n' +for git_user in $(echo "$github_user" | jq -r '.email, .name | select(type == "string")'; echo "$github_user" | jq -r '.name | select(type == "string")' | tr ' ' "\n"); do + if [ -z "$git_user" ]; then + echo "Skipping empty string ..." >>/dev/stderr + continue + fi + echo "Checking $git_user string ..." >>/dev/stderr + slack_user_id=$(echo "$all_users_list" | lc | jq -r ".[] | select( $(jq_filter "${git_user}") ) | .id | select(type == \"string\")" | uc) + [ -n "$slack_user_id" ] && break +done +IFS=$OLDIFS + +if [ -z "$slack_user_id" ]; then + echo "WARN: Slack user is not found for ${GITHUB_ACTOR}" + slack_user_id="GitHub user do not match any user in Slack" + echo slack_user_ids=$(echo $slack_user_id) >> $GITHUB_OUTPUT +else + echo "Slack User ID is found $slack_user_id" >> /dev/stderr + slack_user_ids=$(echo $slack_user_id | tr -s '[:blank:]' ',') + echo $slack_user_ids + echo slack_user_ids=$slack_user_ids >> $GITHUB_OUTPUT +fi