Add Azure Redis private endpoint; config geoprocessing for deployment…

…; refactor redis config loading
Vizzuality · Feb 24, 2022 · 56f61d4 · 56f61d4
1 parent 034a4da
commit 56f61d4
Show file tree

Hide file tree

Showing 44 changed files with 372 additions and 331 deletions.
diff --git a/api/apps/api/config/custom-environment-variables.json b/api/apps/api/config/custom-environment-variables.json
@@ -36,12 +36,11 @@
   "geoprocessing": {
     "url": "GEOPROCESSING_URL"
   },
-  "redisApi": {
-    "connection": {
-      "host": "REDIS_HOST",
-      "port": "REDIS_PORT",
-      "password": "REDIS_PASSWORD"
-    }
+  "redis": {
+    "host": "REDIS_HOST",
+    "port": "REDIS_PORT",
+    "password": "REDIS_PASSWORD",
+    "useTLS": "REDIS_USE_TLS"
   },
   "api": {
     "url": "API_SERVICE_URL",

diff --git a/api/apps/api/config/default.json b/api/apps/api/config/default.json
@@ -9,13 +9,12 @@
       ]
     }
   },
-  "redisApi": {
-    "connection": {
-      "host": "marxan-redis",
-      "port": 6379,
-      "password": null
-    },
-    "concurrency": 50
+  "redis": {
+    "host": "marxan-redis",
+    "port": 6379,
+    "password": null,
+    "useTLS": false,
+    "concurrency" : 50
   },
   "postgresApi": {
     "url": null,

diff --git a/api/apps/api/src/modules/queue/queue-options.provider.ts b/api/apps/api/src/modules/queue/queue-options.provider.ts
@@ -1,13 +1,14 @@
 import { FactoryProvider } from '@nestjs/common';
 import { QueueOptions } from 'bullmq';
 import * as config from 'config';
+import { getRedisConfig } from '@marxan-api/utils/redisConfig.utils';
 
 export const queueOptionsToken = Symbol('queue options token');
 export const queueOptionsProvider: FactoryProvider<QueueOptions> = {
   provide: queueOptionsToken,
   useFactory: () => {
     return {
-      ...config.get('redisApi'),
+      ...getRedisConfig(),
       defaultJobOptions: config.get('jobOptions'),
     };
   },

diff --git a/api/apps/api/src/utils/redisConfig.utils.ts b/api/apps/api/src/utils/redisConfig.utils.ts
@@ -0,0 +1,23 @@
+import * as config from 'config';
+import { QueueBaseOptions } from 'bullmq/dist/interfaces/queue-options';
+
+export function getRedisConfig() {
+  const redisConfig: Record<string, any> = config.get('redis');
+  const useTLS: boolean = `${redisConfig.useTLS}`.toLowerCase() === 'true';
+
+  const redisSettings: QueueBaseOptions = {
+    connection: {
+      host: redisConfig.host,
+      port: redisConfig.port,
+      password: redisConfig.password,
+      tls: useTLS
+        ? {
+            host: redisConfig.host,
+            port: redisConfig.port,
+          }
+        : undefined,
+    },
+  };
+
+  return redisSettings;
+}
diff --git a/api/apps/api/test/scenario-cost-template/bull-mq-integration.e2e-spec.ts b/api/apps/api/test/scenario-cost-template/bull-mq-integration.e2e-spec.ts
@@ -10,6 +10,7 @@ import { Queue, Worker } from 'bullmq';
 import * as config from 'config';
 import waitForExpect from 'wait-for-expect';
 import { QueueModule } from '@marxan-api/modules/queue/queue.module';
+import { getRedisConfig } from '@marxan-api/utils/redisConfig.utils';
 
 const queueName = baseQueueName + '_test_' + Date.now();
 
@@ -124,7 +125,10 @@ const getFixtures = async () => {
         async () => {
           /**/
         },
-        config.get('redisApi'),
+        {
+          ...getRedisConfig(),
+          concurrency: config.get('redis.concurrency'),
+        },
       );
       workers.push(worker);
     },

diff --git a/api/apps/api/test/utils/queues.ts b/api/apps/api/test/utils/queues.ts
@@ -5,6 +5,7 @@ import { assertDefined, FieldsOf } from '@marxan/utils';
 import { QueueBuilder } from '@marxan-api/modules/queue/queue.builder';
 import { QueueNameToken } from '@marxan-api/modules/queue/queue.tokens';
 import * as config from 'config';
+import { getRedisConfig } from '@marxan-api/utils/redisConfig.utils';
 
 @Injectable({
   scope: Scope.TRANSIENT,
@@ -34,7 +35,7 @@ export class FakeQueue implements Partial<Queue> {
   private static instanceKeys: Record<string, Key | undefined> = {};
   public jobs: Record<string, Job> = {};
   private readonly queueBase = new QueueBase(v4(), {
-    ...config.get('redisApi'),
+    ...getRedisConfig(),
   });
 
   constructor(

diff --git a/api/apps/geoprocessing/config/custom-environment-variables.json b/api/apps/geoprocessing/config/custom-environment-variables.json
@@ -30,12 +30,11 @@
       "secret": "API_AUTH_X_API_KEY"
     }
   },
-  "redisApi": {
-    "connection": {
-      "host": "REDIS_HOST",
-      "port": "REDIS_PORT",
-      "password": "REDIS_PASSWORD"
-    }
+  "redis": {
+    "host": "REDIS_HOST",
+    "port": "REDIS_PORT",
+    "password": "REDIS_PASSWORD",
+    "useTLS": "REDIS_USE_TLS"
   },
   "api": {
     "url": "API_SERVICE_URL"

diff --git a/api/apps/geoprocessing/config/default.json b/api/apps/geoprocessing/config/default.json
@@ -1,11 +1,10 @@
 {
-  "redisApi": {
-    "concurrency": 50,
-    "connection": {
-      "host": "marxan-redis",
-      "port": 6379,
-      "password": null
-    }
+  "redis": {
+    "useTLS": false,
+    "host": "marxan-redis",
+    "port": 6379,
+    "password": null,
+    "concurrency" : 50
   },
   "postgresApi": {
     "url": null,

diff --git a/api/apps/geoprocessing/config/test.json b/api/apps/geoprocessing/config/test.json
@@ -1,8 +1,6 @@
 {
-  "redisApi": {
-    "connection": {
-      "host": "test-e2e-redis",
-      "port": "6379"
-    }
+  "redis": {
+    "host": "test-e2e-redis",
+    "port": "6379"
   }
 }
diff --git a/api/apps/geoprocessing/src/modules/worker/config.ts b/api/apps/geoprocessing/src/modules/worker/config.ts
@@ -1,12 +1,16 @@
 import * as config from 'config';
 import { Injectable } from '@nestjs/common';
 import { WorkerOptions } from 'bullmq';
+import { getRedisConfig } from '@marxan-geoprocessing/utils/redisConfig.utils';
 
 @Injectable()
 export class Config {
   public readonly redis: WorkerOptions;
 
   constructor() {
-    this.redis = config.get('redisApi');
+    this.redis = {
+      ...getRedisConfig(),
+      concurrency: config.get('redis.concurrency'),
+    };
   }
 }
diff --git a/api/apps/geoprocessing/src/utils/redisConfig.utils.ts b/api/apps/geoprocessing/src/utils/redisConfig.utils.ts
@@ -0,0 +1,23 @@
+import * as config from 'config';
+import { QueueBaseOptions } from 'bullmq/dist/interfaces/queue-options';
+
+export function getRedisConfig() {
+  const redisConfig: Record<string, any> = config.get('redis');
+  const useTLS: boolean = `${redisConfig.useTLS}`.toLowerCase() === 'true';
+
+  const redisSettings: QueueBaseOptions = {
+    connection: {
+      host: redisConfig.host,
+      port: redisConfig.port,
+      password: redisConfig.password,
+      tls: useTLS
+        ? {
+            host: redisConfig.host,
+            port: redisConfig.port,
+          }
+        : undefined,
+    },
+  };
+
+  return redisSettings;
+}
diff --git a/api/apps/geoprocessing/test/cost-template/cost-template-worker.e2e-spec.ts b/api/apps/geoprocessing/test/cost-template/cost-template-worker.e2e-spec.ts
@@ -20,6 +20,7 @@ import {
   ScenarioCostSurfaceRepository,
 } from '@marxan/scenario-cost-surface';
 import { AppConfig } from '@marxan-geoprocessing/utils/config.utils';
+import { getRedisConfig } from '@marxan-geoprocessing/utils/redisConfig.utils';
 
 jest.setTimeout(35000);
 
@@ -110,7 +111,7 @@ async function getFixtures() {
   const fileRepository = application.get(ScenarioCostSurfaceRepository);
 
   const queue = new Queue(`cost-surface-template-creation`, {
-    ...config.get('redisApi'),
+    ...getRedisConfig(),
   });
 
   let processedScenario: string | undefined;

diff --git a/api/apps/geoprocessing/test/integration/marxan-run/run-worker.e2e-spec.ts b/api/apps/geoprocessing/test/integration/marxan-run/run-worker.e2e-spec.ts
@@ -12,6 +12,7 @@ import {
   runWorkerQueueNameToken,
   sandboxRunnerToken,
 } from '@marxan-geoprocessing/modules/scenarios/runs/tokens';
+import { getRedisConfig } from '@marxan-geoprocessing/utils/redisConfig.utils';
 
 let fixtures: PromiseType<ReturnType<typeof getFixtures>>;
 
@@ -100,9 +101,10 @@ async function getFixtures() {
   }).compile();
   await testingModule.enableShutdownHooks().init();
 
-  const queue = new Queue(testingModule.get(runWorkerQueueNameToken), {
-    ...config.get('redisApi'),
-  });
+  const queue = new Queue(
+    testingModule.get(runWorkerQueueNameToken),
+    getRedisConfig(),
+  );
   const fakeMarxanRunner = testingModule.get(FakeMarxanRunner);
 
   return {

diff --git a/api/apps/geoprocessing/test/worker-module/worker-module-di-usage.e2e-spec.ts b/api/apps/geoprocessing/test/worker-module/worker-module-di-usage.e2e-spec.ts
@@ -7,6 +7,7 @@ import {
   WorkerModule,
   WorkerProcessor,
 } from '../../src/modules/worker';
+import { getRedisConfig } from '@marxan-geoprocessing/utils/redisConfig.utils';
 
 let app: TestingModule;
 let queue: Queue;
@@ -27,9 +28,7 @@ beforeAll(async () => {
   app = await sandbox.init();
   processor = app.get(ExampleProcessingService);
 
-  queue = new Queue(queueName, {
-    ...config.get('redisApi'),
-  });
+  queue = new Queue(queueName, getRedisConfig());
 });
 
 afterAll(async () => {

diff --git a/api/apps/geoprocessing/test/worker-module/worker-module-usage.e2e-spec.ts b/api/apps/geoprocessing/test/worker-module/worker-module-usage.e2e-spec.ts
@@ -5,6 +5,7 @@ import * as config from 'config';
 
 import { ExampleWorkerJobProcessor } from './bullmq-worker-code';
 import { WorkerModule, WorkerBuilder } from '../../src/modules/worker';
+import { getRedisConfig } from '@marxan-geoprocessing/utils/redisConfig.utils';
 
 let app: TestingModule;
 let queue: Queue;
@@ -23,9 +24,7 @@ beforeAll(async () => {
   app = await sandbox.init();
   processor = app.get(ExampleProcessingService);
 
-  queue = new Queue(queueName, {
-    ...config.get('redisApi'),
-  });
+  queue = new Queue(queueName, getRedisConfig());
 });
 
 afterAll(async () => {