[Argus] always parse Axios errors before logging

kdembler · Jan 11, 2024 · 92ccc49 · 92ccc49
1 parent 8e88b27
commit 92ccc49
Show file tree

Hide file tree

Showing 6 changed files with 26 additions and 9 deletions.
diff --git a/distributor-node/CHANGELOG.md b/distributor-node/CHANGELOG.md
@@ -1,3 +1,7 @@
+## 1.5.1
+
+- Added parsing of Axios errors on logger level so that we never log the whole Axios client instance (which is a circular object and causes the node to crash)
+
 ## 1.5.0
 
 - Changed Elasticsearch transport to use data streams instead of regular indices. Renamed `config.logs.elastic.index` to `config.logs.elastic.indexPrefix`. Node ID from config will be automatically appended to the index name.

diff --git a/distributor-node/package.json b/distributor-node/package.json
@@ -1,7 +1,7 @@
 {
   "name": "@joystream/distributor-cli",
   "description": "Joystream distributor node CLI",
-  "version": "1.5.0",
+  "version": "1.5.1",
   "author": "Joystream contributors",
   "bin": {
     "joystream-distributor": "./bin/run"

diff --git a/distributor-node/src/services/logging/LoggingService.ts b/distributor-node/src/services/logging/LoggingService.ts
@@ -1,3 +1,4 @@
+import axios from 'axios'
 import winston, { Logger, LoggerOptions } from 'winston'
 import escFormat from '@elastic/ecs-winston-format'
 import { ElasticsearchTransport } from 'winston-elasticsearch'
@@ -8,6 +9,7 @@ import stringify from 'fast-safe-stringify'
 import NodeCache from 'node-cache'
 import path from 'path'
 import 'winston-daily-rotate-file'
+import { parseAxiosError } from '../parsers/errors'
 
 const cliColors = {
   error: 'red',
@@ -51,6 +53,15 @@ const errorFormat: (opts: ErrorFormatOpts) => Format = winston.format((info, opt
   return info
 })
 
+const axiosErrorFormat = winston.format((info) => {
+  Object.entries(info).forEach(([key, value]) => {
+    if (axios.isAxiosError(value)) {
+      info[key] = parseAxiosError(value)
+    }
+  })
+  return info
+})
+
 const cliFormat = winston.format.combine(
   winston.format.timestamp({ format: 'YYYY-MM-DD HH:mm:ss:ms' }),
   errorFormat({ filedName: 'err' }),
@@ -91,7 +102,7 @@ export class LoggingService {
         index,
         dataStream: true,
         level: config.logs.elastic.level,
-        format: winston.format.combine(pauseFormat({ id: 'es' }), escFormat()),
+        format: winston.format.combine(axiosErrorFormat(), pauseFormat({ id: 'es' }), escFormat()),
         retryLimit: 10,
         flushInterval: 5000,
         // apply custom transform so that tracing data (if present) is placed in the top level of the log
@@ -144,15 +155,15 @@ export class LoggingService {
         maxSize: config.logs.file.maxSize,
         maxFiles: config.logs.file.maxFiles,
         level: config.logs.file.level,
-        format: winston.format.combine(pauseFormat({ id: 'file' }), escFormat()),
+        format: winston.format.combine(axiosErrorFormat(), pauseFormat({ id: 'file' }), escFormat()),
       })
       transports.push(fileTransport)
     }
 
     if (config.logs?.console) {
       const consoleTransport = new winston.transports.Console({
         level: config.logs.console.level,
-        format: winston.format.combine(pauseFormat({ id: 'cli' }), cliFormat),
+        format: winston.format.combine(axiosErrorFormat(), pauseFormat({ id: 'cli' }), cliFormat),
       })
       transports.push(consoleTransport)
     }

diff --git a/distributor-node/src/services/networking/NetworkingService.ts b/distributor-node/src/services/networking/NetworkingService.ts
@@ -332,9 +332,7 @@ export class NetworkingService {
       })
 
       objectDownloadQueue.on('error', (err) => {
-        this.logger.error('Download attempt from storage node failed after availability was confirmed:', {
-          err: axios.isAxiosError(err) ? parseAxiosError(err) : err,
-        })
+        this.logger.error('Download attempt from storage node failed after availability was confirmed:', { err })
       })
 
       objectDownloadQueue.on('end', () => {

diff --git a/distributor-node/src/services/networking/storage-node/api.ts b/distributor-node/src/services/networking/storage-node/api.ts
@@ -48,7 +48,7 @@ export class StorageNodeApi {
       return true
     } catch (err) {
       if (axios.isAxiosError(err)) {
-        this.logger.debug('Data object not available', { objectId, err: parseAxiosError(err) })
+        this.logger.debug('Data object not available', { objectId, err })
         return false
       }
       this.logger.error('Unexpected error while requesting data object', { objectId, err })

diff --git a/distributor-node/src/services/parsers/errors.ts b/distributor-node/src/services/parsers/errors.ts
@@ -4,13 +4,17 @@ type ParsedAxiosErrorResponse = Pick<AxiosResponse, 'data' | 'status' | 'statusT
 
 type ParsedAxiosError = Pick<AxiosError, 'message' | 'stack'> & {
   response?: ParsedAxiosErrorResponse
+  requestUrl?: string
 }
 
-export function parseAxiosError({ message, stack, response }: AxiosError): ParsedAxiosError {
+export function parseAxiosError({ message, stack, response, config }: AxiosError): ParsedAxiosError {
   const parsedError: ParsedAxiosError = {
     message,
     stack,
   }
+  if (config) {
+    parsedError.requestUrl = config.url
+  }
   if (response) {
     const { data, status, statusText, headers } = response
     parsedError.response = {