Update setup for monitoring on the dedicated machine

README.md

@@ -2,19 +2,20 @@
 
 This is a substrate-only bot at the moment.
 
-## How to use
+# How to use
 
 The bot runs commands in response to pull request comments
 ([example](https://github.com/paritytech/polkadot/pull/2541)). The form is:
 
 `/bench [action] [...args]`
 
-For the response to work, [environment variables](#configuration) and
-[Github settings](#github-settings) have to properly configured upfront.
+[Environment variables](#configuration) and
+[Github settings](#required-github-settings) have to properly configured
+upfront for this interaction to work.
 
-## Configuration
+# Configuration
 
-Create an `.env` file in the root with the following:
+Create a `.env` file in the root with the following:
 
 ```
 APP_ID=<App id from Github App Settings>
@@ -26,71 +27,53 @@ WEBHOOK_SECRET=<Webhook secret from Github App Settings>
 WEBHOOK_PROXY_URL=<optional; webhook proxy for development>
 ```
 
-For development it's recommended to use [smee](https://smee.io) for
-`WEBHOOK_PROXY_URL`; that way you can test your changes locally without having
-to SSH into the dedicated machine - it avoids disrupting the production
-service.
+During development it's recommended to use [smee](https://smee.io) for
+`WEBHOOK_PROXY_URL` because it enables testing your bot's functionality
+locally, without having to SSH into the dedicated machine.
 
-## Running
+# Running
 
-### Locally
+## Locally
 
 `yarn && yarn start`
 
-### Dedicated machine
+## Dedicated machine
 
-Note: Before disrupting the production deployment, it's first recommended to
-check if some benchmark is running with `pgrep -au benchbot`. With SSH:
+_Note: Before disrupting the production deployment, it's first recommended to
+check if some benchmark is running through_ `pgrep -a cargo` _._
 
-`ssh user@remote 'sudo pgrep -au benchbot'`
-
-And check if the command above shows any `cargo` or `rust` command being ran
-currently (for the Rust benchmarks).
-
-#### Introduction
-
-The [run](./run) script is used to manage the application.
+The [run script](./run) is used to manage the application. Use `run help` for
+documentation about its options.
 
 `run bootstrap` will take care of creating and installing everything from
-scratch. After installation, a systemd service will be created for you to
-manage with `run {start,restart,stop,status}` which acts as a wrapper for
-`systemctl`.
-
-#### Updating branches
-
-The `update` subcommand will fetch and restart the bot with the selected branch. e.g.
-
-`ssh user@remote '/home/benchbot/bench-bot/run update master'`
-
-For pull requests, the format is `pull/${ID}/head:${BRANCH}` as per the
-[Github specification](https://docs.github.com/en/github/collaborating-with-pull-requests/reviewing-changes-in-pull-requests/checking-out-pull-requests-locally#modifying-an-inactive-pull-request-locally). e.g.
-
-`ssh user@remote '/home/benchbot/bench-bot/run update pull/1/head:branch'`
-
-#### Setting up
-
-By default the bot will be bootstrapped to `/home/benchbot/bench-bot` and
-executed by the `benchbot` user. From your machine, execute the `run` script
-remotely with SSH:
-
-`ssh user@remote '/home/benchbot/bench-bot/run [command]'`
+scratch. For it to fully work, you'll also need to set up [environment
+variables](#configuration) which optionally can be done through a `.env` file
+in the bot's directory.
 
-e.g.
+### Bot commands
 
-`ssh user@remote '/home/benchbot/bench-bot/run restart'`
+- `run {start,stop,restart}`: execute the relevant action for the bot.
+- `run update [ref]`: restart the bot with the branch or PR
+  - For branch: `ssh user@remote '/home/benchbot/bench-bot/run update master'`
+  - For PR: `ssh user@remote '/home/benchbot/bench-bot/run update pull/number/head:branch'` 
+    e.g. `pull/1/head:master`
 
+### Monitoring Service commands
 
-#### Additional information
+- `run monitor {install,uninstall}`: install or uninstall the monitoring
+  service
+- `run monitor {start,restart,stop,status,...}`: acts as a wrapper for
+  `systemctl`
 
-The full explanation for all commands is available with `run help`.
+### Logs
 
-After it's running, the logs will be to the systemd journal:
+The logs will be output to the systemd journal:
 
-`sudo journalctl -u benchbot.service`
+`sudo journalctl -u benchbot-monitor.service`
 
 As well as to `./log.txt`.
 
-# Github Settings
+# Required Github settings
 
 ## Permissions
 

bench.js

@@ -6,7 +6,6 @@ function errorResult(message, error) {
 }
 
 let cwd = process.cwd();
-console.log(`process cwd: ${cwd}`);
 
 const Mutex = require('async-mutex').Mutex;
 const mutex = new Mutex();
@@ -21,38 +20,43 @@ function BenchContext(app, config) {
 
     self.runTask = async function(cmd, { title, shouldLogOutput } = {}) {
       if (title) {
-        app.log(title)
+        app.log({ title, msg: `Running task on directory ${process.cwd()}` })
       }
 
       let stdout = "", stderr = "", error = false
 
       try {
-        if (shouldLogOutput) {
-          console.log(`<=== Start command output (cwd: ${process.cwd()})`)
-        }
-
         await new Promise(function (resolve) {
           const proc = cp.spawn("/bin/bash", ["-c", cmd], { stdio: "pipe" })
 
-          proc.stdout.on("data", function (data) {
-            data = data.toString()
-
-            if (data && shouldLogOutput) {
-              console.log(data.trim())
-            }
-
-            stdout += data
-          })
+          const getStreamCallback = function(channel) {
+            return function (data) {
+              data = data.toString()
 
-          proc.stderr.on("data", function (data) {
-            data = data.toString()
+              if (shouldLogOutput) {
+                const msg = data.trim()
+                if (msg) {
+                  app.log({ msg, channel })
+                }
+              }
 
-            if (data && shouldLogOutput) {
-              console.log(data.trim())
+              switch (channel) {
+                case "stderr": {
+                  stderr += data
+                  break
+                }
+                case "stdout": {
+                  stdout += data
+                  break
+                }
+                default: {
+                  throw new Error(`Got unexpected process channel ${channel}`)
+                }
+              }
             }
-
-            stderr += data
-          })
+          }
+          proc.stdout.on("data", getStreamCallback("stdout"))
+          proc.stderr.on("data", getStreamCallback("stderr"))
 
           proc.on("close", function (code) {
             error = !!code
@@ -65,12 +69,7 @@ function BenchContext(app, config) {
           stdout = err.stdout.toString()
           stderr = err.stderr.toString()
         } else {
-          app.log.error("Caught exception in command execution")
-          app.log.error(err)
-        }
-      } finally {
-        if (shouldLogOutput) {
-          console.log("===> Finished command output")
+          app.log.fatal({ msg: "Caught exception in command execution", err })
         }
       }
 

index.js

@@ -7,7 +7,31 @@ var { benchBranch, benchmarkRuntime } = require("./bench")
 const githubCommentLimitLength = 65536
 const githubCommentLimitTruncateMessage = "<truncated>..."
 
+let isTerminating = false
+let appFatalLogger = undefined
+
+for (const event of ["uncaughtException", "unhandledRejection"]) {
+  process.on(event, function (error, origin) {
+    if (isTerminating) {
+      return
+    }
+    isTerminating = true
+
+    try {
+      if (appFatalLogger) {
+        appFatalLogger({ event, error, origin })
+      }
+    } catch (error) {
+      console.error({ level: "error", event, error, origin, exception })
+    }
+
+    process.exit(1)
+  })
+}
+
 module.exports = (app) => {
+  appFatalLogger = app.log.fatal
+
   const baseBranch = process.env.BASE_BRANCH || "master"
   app.log.debug(`base branch: ${baseBranch}`)
 
@@ -54,10 +78,7 @@ module.exports = (app) => {
 
       const getPushDomain = async function () {
         const token = (
-          await authInstallation({
-            type: "installation",
-            installationId,
-          })
+          await authInstallation({ type: "installation", installationId })
         ).token
 
         const url = `https://x-access-token:${token}@github.com`
@@ -169,7 +190,13 @@ ${extraInfo}
         body,
       })
     } catch (error) {
-      app.log.error(error)
+      app.log.fatal({
+        error,
+        repo,
+        owner,
+        pull_number,
+        msg: "Caught exception in issue_comment's handler",
+      })
       await context.octokit.issues.createComment(
         context.issue({
           body: `Exception caught: \`${error.message}\`\n${error.stack}`,