fix(dal): Reduce default Mongo connections (#6901)

novuhq · Nov 11, 2024 · 97d6739 · 97d6739
1 parent d445c37
commit 97d6739
Show file tree

Hide file tree

Showing 6 changed files with 41 additions and 17 deletions.
diff --git a/apps/api/package.json b/apps/api/package.json
@@ -20,6 +20,7 @@
     "lint": "eslint src",
     "lint:fix": "pnpm lint -- --fix",
     "lint:openapi": "spectral lint http://127.0.0.1:${PORT:-3000}/openapi.yaml",
+    "pretest": "pnpm build:metadata",
     "test": "cross-env TS_NODE_COMPILER_OPTIONS='{\"strictNullChecks\": false}' NODE_ENV=test E2E_RUNNER=true mocha --require ts-node/register --exit 'src/**/*.spec.ts'",
     "test:e2e": "cross-env TS_NODE_COMPILER_OPTIONS='{\"strictNullChecks\": false}' NODE_ENV=test E2E_RUNNER=true mocha --require ts-node/register --exit --file e2e/setup.ts src/**/*.e2e.ts ",
     "test:e2e:ee": "cross-env TS_NODE_COMPILER_OPTIONS='{\"strictNullChecks\": false}' NODE_ENV=test E2E_RUNNER=true CI_EE_TEST=true CLERK_ENABLED=true mocha --grep @skip-in-ee --invert --require ts-node/register --exit --file e2e/setup.ts 'src/**/*.e2e{,-ee}.ts'",

diff --git a/apps/api/src/config/env.validators.ts b/apps/api/src/config/env.validators.ts
@@ -25,9 +25,13 @@ export const envValidators = {
   REDIS_TLS: json({ default: undefined }),
   JWT_SECRET: str(),
   SENDGRID_API_KEY: str({ default: '' }),
-  MONGO_URL: str(),
+  /** @deprecated - use `MONGO_AUTO_CREATE_INDEXES` instead */
+  AUTO_CREATE_INDEXES: bool({ default: true }),
+  MONGO_AUTO_CREATE_INDEXES: bool({ default: true }),
+  MONGO_MAX_IDLE_TIME_IN_MS: num({ default: 1000 * 30 }),
+  MONGO_MAX_POOL_SIZE: num({ default: 50 }),
   MONGO_MIN_POOL_SIZE: num({ default: 10 }),
-  MONGO_MAX_POOL_SIZE: num({ default: 500 }),
+  MONGO_URL: str(),
   NOVU_API_KEY: str({ default: '' }),
   STORE_ENCRYPTION_KEY: str(),
   NEW_RELIC_APP_NAME: str({ default: '' }),

diff --git a/apps/webhook/src/config/env.validators.ts b/apps/webhook/src/config/env.validators.ts
@@ -1,4 +1,4 @@
-import { CleanedEnv, cleanEnv, port, str, ValidatorSpec } from 'envalid';
+import { bool, num, CleanedEnv, cleanEnv, port, str, ValidatorSpec } from 'envalid';
 import { StringifyEnv } from '@novu/shared';
 
 export function validateEnv() {
@@ -8,8 +8,15 @@ export function validateEnv() {
 export type ValidatedEnv = StringifyEnv<CleanedEnv<typeof envValidators>>;
 
 export const envValidators = {
-  TZ: str({ default: 'UTC' }),
+  /** @deprecated - use `MONGO_AUTO_CREATE_INDEXES` instead */
+  AUTO_CREATE_INDEXES: bool({ default: true }),
+  MONGO_AUTO_CREATE_INDEXES: bool({ default: true }),
+  MONGO_MAX_IDLE_TIME_IN_MS: num({ default: 1000 * 30 }),
+  MONGO_MAX_POOL_SIZE: num({ default: 50 }),
+  MONGO_MIN_POOL_SIZE: num({ default: 10 }),
+  MONGO_URL: str(),
   NODE_ENV: str({ choices: ['dev', 'test', 'production', 'ci', 'local'], default: 'local' }),
   PORT: port(),
   SENTRY_DSN: str({ default: undefined }),
+  TZ: str({ default: 'UTC' }),
 } satisfies Record<string, ValidatorSpec<unknown>>;
diff --git a/apps/worker/src/config/env.validators.ts b/apps/worker/src/config/env.validators.ts
@@ -51,9 +51,13 @@ export const envValidators = {
   REDIS_CACHE_KEEP_ALIVE: str({ default: undefined }),
   REDIS_CACHE_FAMILY: str({ default: undefined }),
   REDIS_CACHE_KEY_PREFIX: str({ default: undefined }),
-  MONGO_URL: str(),
+  /** @deprecated - use `MONGO_AUTO_CREATE_INDEXES` instead */
+  AUTO_CREATE_INDEXES: bool({ default: true }),
+  MONGO_AUTO_CREATE_INDEXES: bool({ default: true }),
+  MONGO_MAX_IDLE_TIME_IN_MS: num({ default: 1000 * 30 }),
+  MONGO_MAX_POOL_SIZE: num({ default: 50 }),
   MONGO_MIN_POOL_SIZE: num({ default: 10 }),
-  MONGO_MAX_POOL_SIZE: num({ default: 500 }),
+  MONGO_URL: str(),
   SEGMENT_TOKEN: str({ default: undefined }),
   LAUNCH_DARKLY_SDK_KEY: str({ default: undefined }),
   STRIPE_API_KEY: str({ default: undefined }),

diff --git a/apps/ws/src/config/env.validators.ts b/apps/ws/src/config/env.validators.ts
@@ -1,4 +1,4 @@
-import { json, num, str, port, ValidatorSpec, cleanEnv, CleanedEnv } from 'envalid';
+import { json, bool, num, str, port, ValidatorSpec, cleanEnv, CleanedEnv } from 'envalid';
 import { StringifyEnv } from '@novu/shared';
 
 export function validateEnv() {
@@ -8,14 +8,21 @@ export function validateEnv() {
 export type ValidatedEnv = StringifyEnv<CleanedEnv<typeof envValidators>>;
 
 export const envValidators = {
-  TZ: str({ default: 'UTC' }),
+  JWT_SECRET: str(),
+  /** @deprecated - use `MONGO_AUTO_CREATE_INDEXES` instead */
+  AUTO_CREATE_INDEXES: bool({ default: true }),
+  MONGO_AUTO_CREATE_INDEXES: bool({ default: true }),
+  MONGO_MAX_IDLE_TIME_IN_MS: num({ default: 1000 * 30 }),
+  MONGO_MAX_POOL_SIZE: num({ default: 50 }),
+  MONGO_MIN_POOL_SIZE: num({ default: 10 }),
+  MONGO_URL: str(),
   NODE_ENV: str({ choices: ['dev', 'test', 'production', 'ci', 'local'], default: 'local' }),
   PORT: port(),
   REDIS_HOST: str(),
   REDIS_PORT: port(),
   REDIS_TLS: json({ default: undefined }),
-  JWT_SECRET: str(),
+  SENTRY_DSN: str({ default: undefined }),
+  TZ: str({ default: 'UTC' }),
   WORKER_DEFAULT_CONCURRENCY: num({ default: undefined }),
   WORKER_DEFAULT_LOCK_DURATION: num({ default: undefined }),
-  SENTRY_DSN: str({ default: undefined }),
 } satisfies Record<string, ValidatorSpec<unknown>>;
diff --git a/libs/dal/src/dal.service.ts b/libs/dal/src/dal.service.ts
@@ -1,16 +1,17 @@
 import mongoose, { Connection, ConnectOptions } from 'mongoose';
 
+export const baseConfig: ConnectOptions = {
+  // AUTO_CREATE_INDEXES is deprecated, use MONGO_AUTO_CREATE_INDEXES
+  autoIndex: process.env.AUTO_CREATE_INDEXES === 'true' || process.env.MONGO_AUTO_CREATE_INDEXES === 'true',
+  maxIdleTimeMS: process.env.MONGO_MAX_IDLE_TIME_IN_MS ? Number(process.env.MONGO_MAX_IDLE_TIME_IN_MS) : 1000 * 30,
+  maxPoolSize: process.env.MONGO_MAX_POOL_SIZE ? Number(process.env.MONGO_MAX_POOL_SIZE) : 50,
+  minPoolSize: process.env.MONGO_MIN_POOL_SIZE ? Number(process.env.MONGO_MIN_POOL_SIZE) : 10,
+};
+
 export class DalService {
   connection: Connection;
 
   async connect(url: string, config: ConnectOptions = {}) {
-    const baseConfig: ConnectOptions = {
-      maxPoolSize: +process.env.MONGO_MAX_POOL_SIZE || 500,
-      minPoolSize: +process.env.MONGO_MIN_POOL_SIZE || 10,
-      autoIndex: process.env.AUTO_CREATE_INDEXES === 'true',
-      maxIdleTimeMS: 1000 * 60 * 10,
-    };
-
     const instance = await mongoose.connect(url, {
       ...baseConfig,
       ...config,