diff --git a/.github/workflows/shell.yml b/.github/workflows/scripts-lint.yml similarity index 55% rename from .github/workflows/shell.yml rename to .github/workflows/scripts-lint.yml index 62310bbe006e..297657ba812a 100644 --- a/.github/workflows/shell.yml +++ b/.github/workflows/scripts-lint.yml @@ -1,4 +1,4 @@ -name: Shell +name: Scripts on: pull_request: @@ -16,3 +16,11 @@ jobs: uses: actions/checkout@v2 - name: Run shellcheck uses: ludeeus/action-shellcheck@1.1.0 + rubocop: + runs-on: ubuntu-latest + steps: + - name: RuboCop Linter Action + uses: andrewmcodes/rubocop-linter-action@v3.3.0 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + diff --git a/scripts/sync_check/.env b/scripts/sync_check/.env index 0a2b135b9177..33251753e1b8 100644 --- a/scripts/sync_check/.env +++ b/scripts/sync_check/.env @@ -6,4 +6,5 @@ FOREST_TARGET_DATA=/opt/forest FOREST_TARGET_LOGS=/opt/logs FOREST_TARGET_SNAPSHOTS=/opt/snapshots FOREST_TARGET_SCRIPTS=/opt/scripts -FOREST_CHECK_SLACK_HOOK= +FOREST_SLACK_API_TOKEN= +FOREST_SLACK_NOTIF_CHANNEL= diff --git a/scripts/sync_check/Dockerfile-tester b/scripts/sync_check/Dockerfile-tester new file mode 100644 index 000000000000..d2a4131bd175 --- /dev/null +++ b/scripts/sync_check/Dockerfile-tester @@ -0,0 +1,6 @@ +FROM fedora:36 + +RUN dnf install -y docker ruby ruby-devel make gcc + +# Install required Ruby packages +RUN gem install docker-api slack-ruby-client diff --git a/scripts/sync_check/README.md b/scripts/sync_check/README.md index e11edd084059..7a0d39474c00 100644 --- a/scripts/sync_check/README.md +++ b/scripts/sync_check/README.md @@ -7,7 +7,7 @@ Fedora Linux 36 (Cloud Edition) x86_64 16GB / 320GB Disk ``` * `s3fs-fuse` installed, -* Slack webhook: follow the instructions [here](https://api.slack.com/messaging/webhooks) to set up notifications. +* Slack OAuth token (you can find it in the Slack app settings) needs to be set as an environmental variable, along with the target slack channel. See [.env](.env) for naming. ## Installation * Download manually mainnet snapshot and put it in `$HOME/snapshots`. diff --git a/scripts/sync_check/docker-compose.yml b/scripts/sync_check/docker-compose.yml index 9f9c16558c75..48935f5dd851 100644 --- a/scripts/sync_check/docker-compose.yml +++ b/scripts/sync_check/docker-compose.yml @@ -66,7 +66,9 @@ services: com.centurylinklabs.watchtower.enable: true # Probe container to validate Forest syncing. Needs to be on the same network. forest_tester: - image: fedora:36 + build: + context: . + dockerfile: Dockerfile-tester networks: - mainnet - calibnet @@ -78,13 +80,17 @@ services: - type: bind source: ${FOREST_HOST_LOGS} target: ${FOREST_TARGET_LOGS} + - /var/run/docker.sock:/var/run/docker.sock environment: - - SLACK_HOOK=${FOREST_CHECK_SLACK_HOOK} + - LOG_DIR=${FOREST_TARGET_LOGS} + - SCRIPTS_DIR=${FOREST_TARGET_SCRIPTS} + - SLACK_API_TOKEN=${FOREST_SLACK_API_TOKEN} + - SLACK_NOTIF_CHANNEL=${FOREST_SLACK_NOTIF_CHANNEL} entrypoint: ["/bin/bash","-c"] command: - | - bash ${FOREST_TARGET_SCRIPTS}/sync_check.sh forest-calibnet & - bash ${FOREST_TARGET_SCRIPTS}/sync_check.sh forest-mainnet & + ruby ${FOREST_TARGET_SCRIPTS}/src/sync_check.rb forest-mainnet & + ruby ${FOREST_TARGET_SCRIPTS}/src/sync_check.rb forest-calibnet & sleep infinity depends_on: - forest_mainnet diff --git a/scripts/sync_check/src/docker_utils.rb b/scripts/sync_check/src/docker_utils.rb new file mode 100644 index 000000000000..6eee6a26dfa3 --- /dev/null +++ b/scripts/sync_check/src/docker_utils.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +require 'docker' + +# Tools to facilitate interacting with Docker +module DockerUtils + # returns the specified container logs as String + def self.get_container_logs(container_name) + container = Docker::Container.get container_name + container.streaming_logs(stdout: true, stderr: true) + end +end diff --git a/scripts/sync_check/src/slack_client.rb b/scripts/sync_check/src/slack_client.rb new file mode 100644 index 000000000000..54f582322ddb --- /dev/null +++ b/scripts/sync_check/src/slack_client.rb @@ -0,0 +1,49 @@ +# frozen_string_literal: true + +require 'slack-ruby-client' + +# Wrapper Slack client class to handle sending messages and uploading logs. +class SlackClient + @last_thread = nil + @channel = nil + @client = nil + + def initialize(channel, token) + raise "Invalid channel name: #{channel}, must start with \#" unless channel.start_with? '#' + raise 'Missing token' if token.nil? + + Slack.configure do |config| + config.token = token + end + + @channel = channel + @client = Slack::Web::Client.new + end + + # Posts a new message to configured channel. + def post_message(text) + msg = @client.chat_postMessage(channel: @channel, text: text) + @last_thread = msg[:ts] + end + + # Attaches files to the last posted thread. + def attach_files(*files) + files.each do |file| + attach_file file + end + end + + # Attaches a file to the latest posted thread. + def attach_file(file) + raise "No such file #{file}" unless File.exist? file + raise 'Need to create a thread before attaching a file.' if @last_thread.nil? + + @client.files_upload( + channels: @channel, + file: Faraday::UploadIO.new(file, 'text/plain'), + filename: File.basename(file), + initial_comment: 'Attached a file.', + thread_ts: @last_thread + ) + end +end diff --git a/scripts/sync_check/src/sync_check.rb b/scripts/sync_check/src/sync_check.rb new file mode 100644 index 000000000000..0c5e18b4a735 --- /dev/null +++ b/scripts/sync_check/src/sync_check.rb @@ -0,0 +1,53 @@ +# frozen_string_literal: true + +require_relative 'slack_client' +require_relative 'docker_utils' +require 'logger' +require 'fileutils' + +# Retrieves an environmental variable, failing if its not set or empty. +def get_and_assert_env_variable(name) + var = ENV[name] + raise "Please set #{name} environmental variable" if var.nil? || var.empty? + + var +end + +SLACK_TOKEN = get_and_assert_env_variable 'SLACK_API_TOKEN' +CHANNEL = get_and_assert_env_variable 'SLACK_NOTIF_CHANNEL' +SCRIPTS_DIR = get_and_assert_env_variable 'SCRIPTS_DIR' +LOG_DIR = get_and_assert_env_variable 'LOG_DIR' + +hostname = ARGV[0] +raise 'No arguments supplied. Please provide Forest hostname, e.g. forest-mainnet' if ARGV.empty? + +# Current datetime, to append to the log files +DATE = Time.new.strftime '%FT%H:%M:%S' +LOG_HEALTH = "#{LOG_DIR}/#{hostname}_#{DATE}_health" +LOG_FOREST = "#{LOG_DIR}/#{hostname}_#{DATE}_forest" +LOG_SYNC = "#{LOG_DIR}/#{hostname}_#{DATE}_sync" + +# Create log directory +FileUtils.mkdir_p LOG_DIR + +logger = Logger.new(LOG_SYNC) + +# Run the actual health check +logger.info 'Running the health check...' +health_check_passed = system("bash #{SCRIPTS_DIR}/health_check.sh #{hostname} > #{LOG_HEALTH} 2>&1") +logger.info 'Health check finished' + +# Save the log capture from the Forest container +container_logs = DockerUtils.get_container_logs hostname +File.write(LOG_FOREST, container_logs) + +client = SlackClient.new CHANNEL, SLACK_TOKEN + +if health_check_passed + client.post_message "✅ Sync check for #{hostname} passed. 🌲🌳🌲🌳🌲" +else + client.post_message "⛔ Sync check for #{hostname} fiascoed. 🔥🌲🔥 " +end +client.attach_files(LOG_HEALTH, LOG_SYNC, LOG_FOREST) + +logger.info 'Sync check finished' diff --git a/scripts/sync_check/sync_check.sh b/scripts/sync_check/sync_check.sh deleted file mode 100644 index c4b1eb149864..000000000000 --- a/scripts/sync_check/sync_check.sh +++ /dev/null @@ -1,48 +0,0 @@ -#!/usr/bin/env bash - -# Input: Forest hostname - -# Exit codes -RET_CHECK_FAILED=1 -RET_HOOK_NOT_SET=2 -RET_HOSTNAME_NOT_SET=3 - -if [ $# -eq 0 ]; then - echo "No arguments supplied. Need to provide Forest hostname, e.g. forest-mainnet." - exit "$RET_HOSTNAME_NOT_SET" -else - FOREST_HOSTNAME=$1 -fi - -# Hook is needed to send notifications to Slack channel. -# https://api.slack.com/messaging/webhooks -# It should not be kept in source code. -if [ -z "$SLACK_HOOK" ]; then - echo "Slack hook not set!" - exit "$RET_HOOK_NOT_SET" -fi - -# Directory where the nightly check logs are kept -export SCRIPTS_DIR=/opt/scripts -export LOG_DIR=/opt/logs - -DATE=$(date +"%FT%H:%M:%S") -mkdir -p "$LOG_DIR" -export LOG_FILE_CHECK="$LOG_DIR/forest_check_$DATE.log" - -function send_success_notification() { - curl -X POST -H 'Content-type: application/json' --data "{\"text\":\"✅ $FOREST_HOSTNAME check successfully passed! 💪🌲!\"}" "$SLACK_HOOK" -} - -function send_failure_notification() { - curl -X POST -H 'Content-type: application/json' --data "{\"text\":\"❌ $FOREST_HOSTNAME check miserably failed!\n $(tail -n20 "$LOG_FILE_CHECK")\"}" "$SLACK_HOOK" -} - -echo "Running the health check..." -if bash "$SCRIPTS_DIR"/health_check.sh "$FOREST_HOSTNAME" > "$LOG_FILE_CHECK" 2>&1 -then - send_success_notification -else - send_failure_notification - exit "$RET_CHECK_FAILED" -fi