Ed 4224 config split (#53)

* making configs module global
making configs import as dynamic aws
removing configs imports everywhere else

* support multi aws config sources, add tests

* updating config split for the rest of the envs

* linter

config/default.ts

@@ -1,8 +1,14 @@
 export default {
-  aws: {
-    region: 'us-east-2',
-    secretID: 'eco-solver-secrets',
-  },
+  aws: [
+    {
+      region: 'us-east-2',
+      secretID: 'eco-solver-secrets-dev',
+    },
+    {
+      region: 'us-east-2',
+      secretID: 'eco-solver-configs-dev',
+    },
+  ],
   redis: {
     options: {
       single: {

config/preproduction.ts

@@ -1,6 +1,13 @@
 export default {
-  aws: {
-    region: 'us-east-2',
-    secretID: 'eco-solver-secrets-pre-prod',
-  },
+  aws: [
+    {
+      region: 'us-east-2',
+      secretID: 'eco-solver-secrets-pre-prod',
+    },
+    {
+      region: 'us-east-2',
+      secretID: 'eco-solver-configs-pre-prod',
+    },
+  ],
+  //don't add anything else here
 }

config/production.ts

@@ -1,7 +1,13 @@
 export default {
-  aws: {
-    region: 'us-east-2',
-    secretID: 'eco-solver-secrets-prod',
-  },
+  aws: [
+    {
+      region: 'us-east-2',
+      secretID: 'eco-solver-secrets-prod',
+    },
+    {
+      region: 'us-east-2',
+      secretID: 'eco-solver-configs-prod',
+    },
+  ],
   //don't add anything else here
 }

config/staging.ts

@@ -1,6 +1,13 @@
 export default {
-  aws: {
-    region: 'us-east-2',
-    secretID: 'eco-solver-secrets-staging',
-  },
+  aws: [
+    {
+      region: 'us-east-2',
+      secretID: 'eco-solver-secrets-staging',
+    },
+    {
+      region: 'us-east-2',
+      secretID: 'eco-solver-configs-staging',
+    },
+  ],
+  //don't add anything else here
 }

src/app.module.ts

@@ -18,7 +18,7 @@ import { SolverModule } from './solver/solver.module'
   imports: [
     BalanceModule,
     ChainMonitorModule,
-    EcoConfigModule,
+    EcoConfigModule.withAWS(),
     EventEmitterModule.forRoot({
       // the delimiter used to segment namespaces
       delimiter: '.',
@@ -29,7 +29,6 @@ import { SolverModule } from './solver/solver.module'
     SourceIntentModel,
     ProcessorModule,
     MongooseModule.forRootAsync({
-      imports: [EcoConfigModule],
       inject: [EcoConfigService],
       useFactory: async (configService: EcoConfigService) => {
         const uri = configService.getMongooseUri()
@@ -54,7 +53,6 @@ function getPino() {
   return EcoConfigService.getStaticConfig().logger.usePino
     ? [
         LoggerModule.forRootAsync({
-          imports: [EcoConfigModule],
           inject: [EcoConfigService],
           useFactory: async (configService: EcoConfigService) => {
             const loggerConfig = configService.getLoggerConfig()

src/balance/balance.module.ts

@@ -1,13 +1,12 @@
 import { Module } from '@nestjs/common'
-import { EcoConfigModule } from '../eco-configs/eco-config.module'
 import { BalanceService } from './balance.service'
 import { initBullMQ } from '../bullmq/bullmq.helper'
 import { QUEUES } from '../common/redis/constants'
 import { BalanceWebsocketService } from './balance.ws.service'
 import { TransactionModule } from '../transaction/transaction.module'
 
 @Module({
-  imports: [EcoConfigModule, TransactionModule, initBullMQ(QUEUES.ETH_SOCKET)],
+  imports: [TransactionModule, initBullMQ(QUEUES.ETH_SOCKET)],
   providers: [BalanceService, BalanceWebsocketService],
   exports: [BalanceService],
 })

src/bullmq/bullmq.helper.ts

@@ -1,6 +1,5 @@
 import { BullModule } from '@nestjs/bullmq'
 import { DynamicModule } from '@nestjs/common'
-import { EcoConfigModule } from '../eco-configs/eco-config.module'
 import { EcoConfigService } from '../eco-configs/eco-config.service'
 import { RedisConnectionUtils } from '../common/redis/redis-connection-utils'
 import { QueueInterface } from '../common/redis/constants'
@@ -13,7 +12,6 @@ import { QueueInterface } from '../common/redis/constants'
 export function initBullMQ(queueInterface: QueueInterface): DynamicModule {
   return BullModule.registerQueueAsync({
     name: queueInterface.queue,
-    imports: [EcoConfigModule],
     useFactory: async (configService: EcoConfigService) => {
       return RedisConnectionUtils.getQueueOptions(queueInterface, configService.getRedis())
     },

src/chain-monitor/chain-monitor.module.ts

@@ -1,11 +1,10 @@
 import { Module } from '@nestjs/common'
-import { EcoConfigModule } from '../eco-configs/eco-config.module'
 import { IntentModule } from '../intent/intent.module'
 import { ChainSyncService } from './chain-sync.service'
 import { TransactionModule } from '../transaction/transaction.module'
 
 @Module({
-  imports: [EcoConfigModule, IntentModule, TransactionModule],
+  imports: [IntentModule, TransactionModule],
   providers: [ChainSyncService],
   exports: [ChainSyncService],
 })

src/eco-configs/aws-config.service.ts

@@ -1,10 +1,11 @@
 import { SecretsManager } from '@aws-sdk/client-secrets-manager'
-import { AwsCredentials } from './eco-config.types'
+import { AwsCredential } from './eco-config.types'
 import { Injectable, Logger, OnModuleInit } from '@nestjs/common'
 import * as config from 'config'
 import { EcoLogMessage } from '../common/logging/eco-log-message'
 import { ConfigSource } from './interfaces/config-source.interface'
 import { EcoError } from '../common/errors/eco-error'
+import { merge } from 'lodash'
 
 /**
  * Service to retrieve AWS secrets from AWS Secrets Manager
@@ -35,8 +36,16 @@ export class AwsConfigService implements OnModuleInit, ConfigSource {
         message: `Initializing aws configs`,
       }),
     )
-    const awsCreds = config.get('aws') as AwsCredentials
-    this._awsConfigs = await this.getAwsSecrets(awsCreds)
+    let awsCreds = config.get('aws')
+    if (!Array.isArray(awsCreds)) {
+      awsCreds = [awsCreds]
+    }
+    const creds = await Promise.all(
+      awsCreds.map(async (cred: AwsCredential) => {
+        return await this.getAwsSecrets(cred)
+      }),
+    )
+    merge(this._awsConfigs, ...creds)
   }
 
   get awsConfigs(): Record<string, string> {
@@ -48,7 +57,7 @@ export class AwsConfigService implements OnModuleInit, ConfigSource {
    * @param awsCreds the aws credentials
    * @returns
    */
-  private async getAwsSecrets(awsCreds: AwsCredentials): Promise<Record<string, string>> {
+  private async getAwsSecrets(awsCreds: AwsCredential): Promise<Record<string, string>> {
     const secretsManager = new SecretsManager({
       region: awsCreds.region,
     })

src/eco-configs/eco-config.module.ts

@@ -1,19 +1,44 @@
-import { Module } from '@nestjs/common'
+import { DynamicModule, FactoryProvider, Global, Module, Provider } from '@nestjs/common'
 import { EcoConfigService } from './eco-config.service'
 import { AwsConfigService } from './aws-config.service'
 
+@Global()
 @Module({
-  providers: [
-    {
+  providers: [EcoConfigModule.createBaseProvider(), AwsConfigService],
+  exports: [EcoConfigService, AwsConfigService],
+})
+export class EcoConfigModule {
+  static withAWS(): DynamicModule {
+    return {
+      global: true,
+      module: EcoConfigModule,
+      providers: [EcoConfigModule.createAwsProvider()],
+      exports: [EcoConfigService],
+    }
+  }
+
+  static base(): DynamicModule {
+    return {
+      global: true,
+      module: EcoConfigModule,
+      providers: [EcoConfigModule.createBaseProvider()],
+      exports: [EcoConfigService],
+    }
+  }
+
+  static createAwsProvider(): Provider {
+    const dynamicConfig: FactoryProvider<EcoConfigService> = {
       provide: EcoConfigService,
       useFactory: async (awsConfigService: AwsConfigService) => {
         await awsConfigService.initConfigs()
-        return new EcoConfigService(awsConfigService)
+        return new EcoConfigService([awsConfigService])
       },
       inject: [AwsConfigService],
-    },
-    AwsConfigService,
-  ],
-  exports: [EcoConfigService, AwsConfigService],
-})
-export class EcoConfigModule {}
+    }
+    return dynamicConfig
+  }
+
+  static createBaseProvider(): Provider {
+    return EcoConfigService
+  }
+}

src/eco-configs/eco-config.service.ts

@@ -13,11 +13,14 @@ import { addressKeys } from '../common/viem/utils'
 @Injectable()
 export class EcoConfigService implements OnModuleInit {
   private logger = new Logger(EcoConfigService.name)
-  private externalConfigs: any
+  private externalConfigs: any = {}
   private ecoConfig: config.IConfig
 
-  constructor(private readonly configSource: ConfigSource) {
-    this.externalConfigs = configSource.getConfig()
+  constructor(private readonly sources: ConfigSource[]) {
+    this.sources.reduce((prev, curr) => {
+      return config.util.extendDeep(prev, curr.getConfig())
+    }, this.externalConfigs)
+
     this.ecoConfig = config
     this.initConfigs()
   }

src/eco-configs/eco-config.types.ts

@@ -38,7 +38,7 @@ export type EcoConfigType = {
     }
     pollingInterval: number
   }
-  aws: AwsCredentials
+  aws: AwsCredential[]
   database: {
     auth: MongoAuthType
     uriPrefix: string
@@ -75,7 +75,7 @@ export type RedisConfig = {
 /**
  * The config type for the aws credentials
  */
-export type AwsCredentials = {
+export type AwsCredential = {
   region: string
   secretID: string
 }

src/eco-configs/tests/aws-config.service.spec.ts

@@ -0,0 +1,52 @@
+const mockConfigGet = jest.fn()
+import { Test, TestingModule } from '@nestjs/testing'
+import { AwsConfigService } from '../aws-config.service'
+import { merge } from 'lodash'
+
+jest.mock('config', () => {
+  return {
+    get: mockConfigGet,
+  }
+})
+
+describe('Aws Config Helper Tests', () => {
+  let awsConfigService: AwsConfigService
+  let mockLog: jest.Mock
+  let mockAwsGet: jest.Mock
+  const awsConfigSingle = { region: 'us-florida', secretID: 'secrets' }
+  const awsConfigSingleData = { key: 1, value: 'value1' }
+  const awsConfigArray = [awsConfigSingle, { region: 'us-california', secretID: 'configs' }]
+  const awsConfigArrayData = [awsConfigSingleData, { key: 3, value_second: 'value3' }]
+  beforeEach(async () => {
+    const configMod: TestingModule = await Test.createTestingModule({
+      providers: [AwsConfigService],
+    }).compile()
+
+    awsConfigService = configMod.get<AwsConfigService>(AwsConfigService)
+    mockLog = jest.fn()
+    mockAwsGet = jest.fn()
+  })
+
+  it('should read single aws credential', async () => {
+    mockConfigGet.mockReturnValue(awsConfigSingle)
+    awsConfigService['getAwsSecrets'] = mockAwsGet.mockResolvedValue(awsConfigSingleData)
+    await awsConfigService.initConfigs()
+    expect(awsConfigService.getConfig()).toEqual(awsConfigSingleData)
+  })
+
+  it('should read array of aws credentials', async () => {
+    mockConfigGet.mockReturnValue(awsConfigArray)
+    let index = 0
+    awsConfigService['getAwsSecrets'] = mockAwsGet.mockImplementation(() => {
+      return awsConfigArrayData[index++]
+    })
+    const me = {}
+    awsConfigArrayData.forEach((data) => {
+      merge(me, data)
+    })
+    merge({}, ...awsConfigArrayData)
+
+    await awsConfigService.initConfigs()
+    expect(awsConfigService.getConfig()).toEqual(me)
+  })
+})

src/eco-configs/tests/eco-config.service.spec.ts

@@ -17,7 +17,7 @@ describe('Eco Config Helper Tests', () => {
           provide: EcoConfigService,
           useFactory: async (awsConfigService: AwsConfigService) => {
             await awsConfigService.initConfigs()
-            return new EcoConfigService(awsConfigService)
+            return new EcoConfigService([awsConfigService])
           },
           inject: [AwsConfigService],
         },

src/health/health.module.ts

@@ -6,13 +6,12 @@ import { BalanceHealthIndicator } from './indicators/balance.indicator'
 import { RedisHealthModule } from '@liaoliaots/nestjs-redis-health'
 import { EcoRedisHealthIndicator } from './indicators/eco-redis.indicator'
 import { MongoDBHealthIndicator } from './indicators/mongodb.indicator'
-import { EcoConfigModule } from '../eco-configs/eco-config.module'
 import { PermissionHealthIndicator } from './indicators/permission.indicator'
 import { TransactionModule } from '../transaction/transaction.module'
 import { GitCommitHealthIndicator } from './indicators/git-commit.indicator'
 
 @Module({
-  imports: [EcoConfigModule, TransactionModule, RedisHealthModule, TerminusModule],
+  imports: [TransactionModule, RedisHealthModule, TerminusModule],
   controllers: [HealthController],
   providers: [
     HealthService,

src/intent/intent.module.ts

@@ -1,5 +1,4 @@
 import { Module } from '@nestjs/common'
-import { EcoConfigModule } from '../eco-configs/eco-config.module'
 import { initBullMQ } from '../bullmq/bullmq.helper'
 import { QUEUES } from '../common/redis/constants'
 import { SourceIntentModel, SourceIntentSchema } from './schemas/source-intent.schema'
@@ -18,7 +17,6 @@ import { SolverModule } from '../solver/solver.module'
 @Module({
   imports: [
     BalanceModule,
-    EcoConfigModule,
     MongooseModule.forFeature([{ name: SourceIntentModel.name, schema: SourceIntentSchema }]),
     ProverModule,
     SolverModule,

src/prover/prover.module.ts

@@ -1,10 +1,9 @@
 import { Module } from '@nestjs/common'
 import { ProofService } from './proof.service'
 import { TransactionModule } from '../transaction/transaction.module'
-import { EcoConfigModule } from '../eco-configs/eco-config.module'
 
 @Module({
-  imports: [TransactionModule, EcoConfigModule],
+  imports: [TransactionModule],
   providers: [ProofService],
   exports: [ProofService],
 })

src/sign/sign.module.ts

@@ -3,14 +3,12 @@ import { MongooseModule } from '@nestjs/mongoose'
 import { Nonce, NonceSchema } from './schemas/nonce.schema'
 import { initBullMQ } from '../bullmq/bullmq.helper'
 import { QUEUES } from '../common/redis/constants'
-import { EcoConfigModule } from '../eco-configs/eco-config.module'
 import { SignerService } from './signer.service'
 import { AtomicSignerService } from './atomic-signer.service'
 import { NonceService } from './nonce.service'
 
 @Module({
   imports: [
-    EcoConfigModule,
     MongooseModule.forFeature([{ name: Nonce.name, schema: NonceSchema }]),
     initBullMQ(QUEUES.SIGNER),
   ],