rubify sync check, add logs as attachments (#1867)

.github/workflows/shell.yml → .github/workflows/scripts-lint.yml

@@ -1,4 +1,4 @@
-name: Shell
+name: Scripts
 
 on:
   pull_request:
@@ -16,3 +16,11 @@ jobs:
       uses: actions/checkout@v2
     - name: Run shellcheck
       uses: ludeeus/action-shellcheck@1.1.0
+  rubocop:
+    runs-on: ubuntu-latest
+    steps:
+    - name: RuboCop Linter Action
+      uses: andrewmcodes/rubocop-linter-action@v3.3.0
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+

scripts/sync_check/.env

@@ -6,4 +6,5 @@ FOREST_TARGET_DATA=/opt/forest
 FOREST_TARGET_LOGS=/opt/logs
 FOREST_TARGET_SNAPSHOTS=/opt/snapshots
 FOREST_TARGET_SCRIPTS=/opt/scripts
-FOREST_CHECK_SLACK_HOOK=
+FOREST_SLACK_API_TOKEN=
+FOREST_SLACK_NOTIF_CHANNEL=

scripts/sync_check/Dockerfile-tester

@@ -0,0 +1,6 @@
+FROM fedora:36
+
+RUN dnf install -y docker ruby ruby-devel make gcc
+
+# Install required Ruby packages
+RUN gem install docker-api slack-ruby-client

scripts/sync_check/README.md

@@ -7,7 +7,7 @@ Fedora Linux 36 (Cloud Edition) x86_64
 16GB / 320GB Disk
 ```
 * `s3fs-fuse` installed,
-* Slack webhook: follow the instructions [here](https://api.slack.com/messaging/webhooks) to set up notifications.
+* Slack OAuth token (you can find it in the Slack app settings) needs to be set as an environmental variable, along with the target slack channel. See [.env](.env) for naming.
 
 ## Installation
 * Download manually mainnet snapshot and put it in `$HOME/snapshots`. 

scripts/sync_check/docker-compose.yml

@@ -66,7 +66,9 @@ services:
       com.centurylinklabs.watchtower.enable: true
   # Probe container to validate Forest syncing. Needs to be on the same network.
   forest_tester:
-    image: fedora:36
+    build:
+      context: .
+      dockerfile: Dockerfile-tester
     networks:
       - mainnet
       - calibnet
@@ -78,13 +80,17 @@ services:
       - type: bind
         source: ${FOREST_HOST_LOGS}
         target: ${FOREST_TARGET_LOGS}
+      - /var/run/docker.sock:/var/run/docker.sock
     environment:
-      - SLACK_HOOK=${FOREST_CHECK_SLACK_HOOK}
+      - LOG_DIR=${FOREST_TARGET_LOGS}
+      - SCRIPTS_DIR=${FOREST_TARGET_SCRIPTS}
+      - SLACK_API_TOKEN=${FOREST_SLACK_API_TOKEN}
+      - SLACK_NOTIF_CHANNEL=${FOREST_SLACK_NOTIF_CHANNEL}
     entrypoint: ["/bin/bash","-c"]
     command:
       - |
-        bash ${FOREST_TARGET_SCRIPTS}/sync_check.sh forest-calibnet &
-        bash ${FOREST_TARGET_SCRIPTS}/sync_check.sh forest-mainnet &
+        ruby ${FOREST_TARGET_SCRIPTS}/src/sync_check.rb forest-mainnet &
+        ruby ${FOREST_TARGET_SCRIPTS}/src/sync_check.rb forest-calibnet &
         sleep infinity
     depends_on:
       - forest_mainnet

scripts/sync_check/src/docker_utils.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+require 'docker'
+
+# Tools to facilitate interacting with Docker
+module DockerUtils
+  # returns the specified container logs as String
+  def self.get_container_logs(container_name)
+    container = Docker::Container.get container_name
+    container.streaming_logs(stdout: true, stderr: true)
+  end
+end

scripts/sync_check/src/slack_client.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+require 'slack-ruby-client'
+
+# Wrapper Slack client class to handle sending messages and uploading logs.
+class SlackClient
+  @last_thread = nil
+  @channel = nil
+  @client = nil
+
+  def initialize(channel, token)
+    raise "Invalid channel name: #{channel}, must start with \#" unless channel.start_with? '#'
+    raise 'Missing token' if token.nil?
+
+    Slack.configure do |config|
+      config.token = token
+    end
+
+    @channel = channel
+    @client = Slack::Web::Client.new
+  end
+
+  # Posts a new message to configured channel.
+  def post_message(text)
+    msg = @client.chat_postMessage(channel: @channel, text: text)
+    @last_thread = msg[:ts]
+  end
+
+  # Attaches files to the last posted thread.
+  def attach_files(*files)
+    files.each do |file|
+      attach_file file
+    end
+  end
+
+  # Attaches a file to the latest posted thread.
+  def attach_file(file)
+    raise "No such file #{file}" unless File.exist? file
+    raise 'Need to create a thread before attaching a file.' if @last_thread.nil?
+
+    @client.files_upload(
+      channels: @channel,
+      file: Faraday::UploadIO.new(file, 'text/plain'),
+      filename: File.basename(file),
+      initial_comment: 'Attached a file.',
+      thread_ts: @last_thread
+    )
+  end
+end

scripts/sync_check/src/sync_check.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+require_relative 'slack_client'
+require_relative 'docker_utils'
+require 'logger'
+require 'fileutils'
+
+# Retrieves an environmental variable, failing if its not set or empty.
+def get_and_assert_env_variable(name)
+  var = ENV[name]
+  raise "Please set #{name} environmental variable" if var.nil? || var.empty?
+
+  var
+end
+
+SLACK_TOKEN = get_and_assert_env_variable 'SLACK_API_TOKEN'
+CHANNEL = get_and_assert_env_variable 'SLACK_NOTIF_CHANNEL'
+SCRIPTS_DIR = get_and_assert_env_variable 'SCRIPTS_DIR'
+LOG_DIR = get_and_assert_env_variable 'LOG_DIR'
+
+hostname = ARGV[0]
+raise 'No arguments supplied. Please provide Forest hostname, e.g. forest-mainnet' if ARGV.empty?
+
+# Current datetime, to append to the log files
+DATE = Time.new.strftime '%FT%H:%M:%S'
+LOG_HEALTH = "#{LOG_DIR}/#{hostname}_#{DATE}_health"
+LOG_FOREST = "#{LOG_DIR}/#{hostname}_#{DATE}_forest"
+LOG_SYNC = "#{LOG_DIR}/#{hostname}_#{DATE}_sync"
+
+# Create log directory
+FileUtils.mkdir_p LOG_DIR
+
+logger = Logger.new(LOG_SYNC)
+
+# Run the actual health check
+logger.info 'Running the health check...'
+health_check_passed = system("bash #{SCRIPTS_DIR}/health_check.sh #{hostname} > #{LOG_HEALTH} 2>&1")
+logger.info 'Health check finished'
+
+# Save the log capture from the Forest container
+container_logs = DockerUtils.get_container_logs hostname
+File.write(LOG_FOREST, container_logs)
+
+client = SlackClient.new CHANNEL, SLACK_TOKEN
+
+if health_check_passed
+  client.post_message "✅ Sync check for #{hostname} passed. 🌲🌳🌲🌳🌲"
+else
+  client.post_message "⛔ Sync check for #{hostname} fiascoed. 🔥🌲🔥 "
+end
+client.attach_files(LOG_HEALTH, LOG_SYNC, LOG_FOREST)
+
+logger.info 'Sync check finished'