From f0ef6511c67daf74bfca144b3a5ddabcfc80420b Mon Sep 17 00:00:00 2001 From: Sokratis Vidros Date: Wed, 9 Oct 2024 13:49:05 +0300 Subject: [PATCH 1/4] fix(worker): Reduce distraction errors 1. Report queue metrics to Newrelic every 30 seconds, not every second. 2. Remove Better Stack heartbeat as it wasn't properly configured. We will add it again as part of future work. --- .../src/app/shared/utils/cron-health.ts | 20 ------------------- .../services/active-jobs-metric.service.ts | 3 +-- 2 files changed, 1 insertion(+), 22 deletions(-) delete mode 100644 apps/worker/src/app/shared/utils/cron-health.ts diff --git a/apps/worker/src/app/shared/utils/cron-health.ts b/apps/worker/src/app/shared/utils/cron-health.ts deleted file mode 100644 index 47d35278128..00000000000 --- a/apps/worker/src/app/shared/utils/cron-health.ts +++ /dev/null @@ -1,20 +0,0 @@ -import { Logger } from '@nestjs/common'; - -const LOG_CONTEXT = 'cronHealth'; -export async function checkingForCronJob(cronId?: string) { - const CRON_CHECKING_URL = process.env.CRON_CHECKING_URL - ? process.env.CRON_CHECKING_URL - : 'https://uptime.betterstack.com/api/v1/heartbeat/'; - - if (process.env.NOVU_MANAGED_SERVICE && process.env.NODE_ENV === 'production' && cronId && CRON_CHECKING_URL) { - Logger.verbose(`Calling health endpoint for ${cronId}`); - - const response = await fetch(CRON_CHECKING_URL + cronId); - - if (response.status !== 200) { - Logger.error(`Failed calling better Uptime: ${response.status}`, LOG_CONTEXT); - } else { - Logger.verbose(`Response from better Uptime: ${response.status}`, LOG_CONTEXT); - } - } -} diff --git a/apps/worker/src/app/workflow/services/active-jobs-metric.service.ts b/apps/worker/src/app/workflow/services/active-jobs-metric.service.ts index 622758f5369..47e90349a1c 100644 --- a/apps/worker/src/app/workflow/services/active-jobs-metric.service.ts +++ b/apps/worker/src/app/workflow/services/active-jobs-metric.service.ts @@ -26,7 +26,6 @@ export class ActiveJobsMetricService { this.activeJobsMetricWorkerService.createWorker(this.getWorkerProcessor(), this.getWorkerOptions()); this.activeJobsMetricWorkerService.worker.on('completed', async (job) => { - await checkingForCronJob(process.env.ACTIVE_CRON_ID); Logger.log({ jobId: job.id }, 'Metric Completed Job', LOG_CONTEXT); }); @@ -66,7 +65,7 @@ export class ActiveJobsMetricService { repeatJobKey: METRIC_JOB_ID, repeat: { immediately: true, - pattern: '* * * * * *', + pattern: '*/30 * * * * *', // Every 30 seconds }, removeOnFail: true, removeOnComplete: true, From 32ad6fe9533155439333c0373d0e0805e70f0204 Mon Sep 17 00:00:00 2001 From: Sokratis Vidros Date: Wed, 9 Oct 2024 14:01:30 +0300 Subject: [PATCH 2/4] Update active-jobs-metric.service.ts --- .../src/app/workflow/services/active-jobs-metric.service.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/apps/worker/src/app/workflow/services/active-jobs-metric.service.ts b/apps/worker/src/app/workflow/services/active-jobs-metric.service.ts index 47e90349a1c..a79569c3e31 100644 --- a/apps/worker/src/app/workflow/services/active-jobs-metric.service.ts +++ b/apps/worker/src/app/workflow/services/active-jobs-metric.service.ts @@ -5,6 +5,7 @@ import { QueueBaseService, WorkerOptions, } from '@novu/application-generic'; +import { CronExpressionEnum } from '@novu/shared'; import { Inject, Injectable, Logger } from '@nestjs/common'; import { checkingForCronJob } from '../../shared/utils'; @@ -65,7 +66,7 @@ export class ActiveJobsMetricService { repeatJobKey: METRIC_JOB_ID, repeat: { immediately: true, - pattern: '*/30 * * * * *', // Every 30 seconds + pattern: CronExpressionEnum.EVERY_30_SECONDS, }, removeOnFail: true, removeOnComplete: true, From 6144f69058ca245f3484f0f766ef89441e7046f4 Mon Sep 17 00:00:00 2001 From: Sokratis Vidros Date: Wed, 9 Oct 2024 15:00:26 +0300 Subject: [PATCH 3/4] fixup --- apps/worker/src/app/shared/utils/index.ts | 1 - .../src/app/workflow/services/active-jobs-metric.service.ts | 2 -- 2 files changed, 3 deletions(-) diff --git a/apps/worker/src/app/shared/utils/index.ts b/apps/worker/src/app/shared/utils/index.ts index daf676373fd..64fdd861816 100644 --- a/apps/worker/src/app/shared/utils/index.ts +++ b/apps/worker/src/app/shared/utils/index.ts @@ -1,3 +1,2 @@ export * from './constants'; export * from './exceptions'; -export * from './cron-health'; diff --git a/apps/worker/src/app/workflow/services/active-jobs-metric.service.ts b/apps/worker/src/app/workflow/services/active-jobs-metric.service.ts index a79569c3e31..05935f14e23 100644 --- a/apps/worker/src/app/workflow/services/active-jobs-metric.service.ts +++ b/apps/worker/src/app/workflow/services/active-jobs-metric.service.ts @@ -8,8 +8,6 @@ import { import { CronExpressionEnum } from '@novu/shared'; import { Inject, Injectable, Logger } from '@nestjs/common'; -import { checkingForCronJob } from '../../shared/utils'; - const nr = require('newrelic'); const LOG_CONTEXT = 'ActiveJobMetricService'; From bd45dfd551f80fe713dfe57eccbdb7b6a07e0591 Mon Sep 17 00:00:00 2001 From: Sokratis Vidros Date: Wed, 9 Oct 2024 15:02:44 +0300 Subject: [PATCH 4/4] Update distributed-lock.service.ts --- .../src/services/distributed-lock/distributed-lock.service.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/application-generic/src/services/distributed-lock/distributed-lock.service.ts b/libs/application-generic/src/services/distributed-lock/distributed-lock.service.ts index 2f01d274397..f62d450fe5a 100644 --- a/libs/application-generic/src/services/distributed-lock/distributed-lock.service.ts +++ b/libs/application-generic/src/services/distributed-lock/distributed-lock.service.ts @@ -63,7 +63,7 @@ export class DistributedLockService { * when an "error" event is emitted in the absence of listeners. */ this.distributedLock.on('error', (error) => { - Logger.error( + Logger.verbose( error, 'There has been an error in the Distributed Lock service', LOG_CONTEXT,