feat: Adds Redis Sentinel support. (#74)

ryasmi · web-flow · commit 6682760e54b9 · 2018-02-28T14:50:10.000Z
diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -37,7 +37,7 @@
     "@learninglocker/xapi-activities": "^4.0.2",
     "@learninglocker/xapi-agents": "^4.0.3",
     "@learninglocker/xapi-state": "^4.0.2",
-    "@learninglocker/xapi-statements": "^6.0.4",
+    "@learninglocker/xapi-statements": "^7.0.0",
     "boolean": "^0.1.2",
     "dotenv": "^5.0.0",
     "express": "^4.14.1",
@@ -54,6 +54,7 @@
     "@types/dotenv": "4.0.2",
     "@types/express": "4.11.1",
     "@types/google-cloud__storage": "1.1.7",
+    "@types/ioredis": "3.2.5",
     "@types/lodash": "4.14.104",
     "@types/mongodb": "3.0.5",
     "@types/node": "9.4.6",
diff --git a/src/apps/AppConfig.ts b/src/apps/AppConfig.ts
@@ -1,4 +1,5 @@
 import { S3 } from 'aws-sdk';
+import { Redis } from 'ioredis';
 import Tracker from 'jscommons/dist/tracker/Tracker';
 import { Db } from 'mongodb';
 import { LoggerInstance } from 'winston';
@@ -35,7 +36,11 @@ export default interface AppConfig {
     };
     readonly redis: {
       readonly prefix: string;
-      readonly url: string;
+      readonly client: () => Promise<Redis>;
+    };
+    readonly sentinel: {
+      readonly prefix: string;
+      readonly client: () => Promise<Redis>;
     };
   };
   readonly service: {
diff --git a/src/apps/app.ts b/src/apps/app.ts
@@ -78,6 +78,7 @@ export default (appConfig: AppConfig): Router => {
       mongo: appConfig.repo.mongo,
       redis: appConfig.repo.redis,
       s3: appConfig.repo.s3,
+      sentinel: appConfig.repo.sentinel,
       storageSubFolder: appConfig.repo.storageSubFolders.statements,
     },
     service: appConfig.service.statements,
diff --git a/src/apps/statements/AppConfig.ts b/src/apps/statements/AppConfig.ts
@@ -1,4 +1,5 @@
 import * as S3 from 'aws-sdk/clients/s3';
+import { Redis } from 'ioredis';
 import Tracker from 'jscommons/dist/tracker/Tracker';
 import { Db } from 'mongodb';
 import { LoggerInstance } from 'winston';
@@ -51,7 +52,11 @@ export default interface AppConfig {
     };
     readonly redis: {
       readonly prefix: string;
-      readonly url: string;
+      readonly client: () => Promise<Redis>;
+    };
+    readonly sentinel: {
+      readonly prefix: string;
+      readonly client: () => Promise<Redis>;
     };
   };
 }
diff --git a/src/apps/statements/app.ts b/src/apps/statements/app.ts
@@ -18,8 +18,12 @@ export default (appConfig: AppConfig): Result => {
     events: {
       facade: appConfig.repo.factory.eventsRepoName,
       redis: {
+        client: appConfig.repo.redis.client,
         prefix: appConfig.repo.redis.prefix,
-        url: appConfig.repo.redis.url,
+      },
+      sentinel: {
+        client: appConfig.repo.sentinel.client,
+        prefix: appConfig.repo.sentinel.prefix,
       },
     },
     models: {
diff --git a/src/config.ts b/src/config.ts
@@ -10,6 +10,7 @@ import getDbFromUrl from 'jscommons/dist/mongoRepo/utils/getDbFromUrl';
 import { defaultTo } from 'lodash';
 import * as os from 'os';
 
+const DEFAULT_REDIS_PORT = 6379;
 const DEFAULT_EXPRESS_PORT = 8081;
 const DEFAULT_TIMEOUT_MS = 300000; // 5 minutes.
 
@@ -71,6 +72,16 @@ export default {
     } as S3.ClientConfiguration,
     bucketName: getStringOption(process.env.FS_S3_BUCKET, 'xapi-service'),
   },
+  sentinel: {
+    name: getStringOption(process.env.SENTINEL_NAME, 'mymaster'),
+    prefix: getStringOption(process.env.SENTINEL_PREFIX, 'LEARNINGLOCKER'),
+    sentinels: (
+      getStringOption(process.env.SENTINEL_CONNECTIONS, '127.0.0.1:6379').split(' ').map((conn) => {
+        const [host, port] = conn.split(':');
+        return { host, port: getNumberOption(port, DEFAULT_REDIS_PORT) };
+      })
+    ),
+  },
   statementsService: {
     awaitUpdates: getBooleanOption(defaultTo<any>(
       process.env.SERVICE_AWAIT_UPDATES,
diff --git a/src/server.ts b/src/server.ts
@@ -8,6 +8,8 @@ import app from './apps/app';
 import config from './config';
 import logger from './logger';
 import connectToMongoDb from './utils/connectToMongoDb';
+import connectToRedis from './utils/connectToRedis';
+import connectToSentinel from './utils/connectToSentinel';
 
 const expressApp = express();
 
@@ -20,9 +22,16 @@ expressApp.use(app({
     google: config.googleStorageRepo,
     local: config.localStorageRepo,
     mongo: { db: connectToMongoDb() },
-    redis: config.redis,
+    redis: {
+      client: connectToRedis(),
+      prefix: config.redis.prefix,
+    },
     repoFactory: config.repoFactory,
     s3: config.s3StorageRepo,
+    sentinel: {
+      client: connectToSentinel(),
+      prefix: config.redis.prefix,
+    },
     storageSubFolders: config.storageSubFolders,
   },
   service: {
diff --git a/src/utils/connectToRedis.ts b/src/utils/connectToRedis.ts
@@ -0,0 +1,13 @@
+import * as Ioredis from 'ioredis';
+import { defaultTo, once } from 'lodash';
+import config from '../config';
+import logger from '../logger';
+
+export default once((): () => Promise<Ioredis.Redis> => {
+  return once(async () => {
+    logger.info('Creating redis connection');
+    return new Ioredis(
+      defaultTo(config.redis.url, 'redis://127.0.0.1:6379/0'),
+    );
+  });
+});
diff --git a/src/utils/connectToSentinel.ts b/src/utils/connectToSentinel.ts
@@ -0,0 +1,14 @@
+import * as Ioredis from 'ioredis';
+import { defaultTo, once } from 'lodash';
+import config from '../config';
+import logger from '../logger';
+
+export default once((): () => Promise<Ioredis.Redis> => {
+  return once(async () => {
+    logger.info('Creating sentinel connection');
+    return new Ioredis({
+      name: defaultTo(config.sentinel.name, 'mymaster'),
+      sentinels: defaultTo(config.sentinel.sentinels, [{ host: '127.0.0.1', port: 6379 }]),
+    });
+  });
+});
diff --git a/yarn.lock b/yarn.lock
@@ -152,9 +152,9 @@
     string-to-stream "^1.1.0"
     uuid "^3.0.1"
 
-"@learninglocker/xapi-statements@^6.0.4":
-  version "6.0.4"
-  resolved "https://registry.yarnpkg.com/@learninglocker/xapi-statements/-/xapi-statements-6.0.4.tgz#5750d3e21a307e6ecfb1669749ade5c6cd0cd4eb"
+"@learninglocker/xapi-statements@^7.0.0":
+  version "7.0.0"
+  resolved "https://registry.yarnpkg.com/@learninglocker/xapi-statements/-/xapi-statements-7.0.0.tgz#08fd8f02561d7f20921c4faf85b2493bb7ca78de"
   dependencies:
     "@google-cloud/storage" "^1.5.2"
     "@learninglocker/xapi-validation" "^2.1.10"
@@ -171,6 +171,7 @@
     file-stream-rotator "^0.2.0"
     fs-extra "^5.0.0"
     helmet "^3.5.0"
+    ioredis "^3.2.2"
     jscommons "^2.3.0"
     jsonwebtoken "^8.0.1"
     lodash "^4.17.4"
@@ -327,6 +328,10 @@
   version "0.7.0"
   resolved "https://registry.yarnpkg.com/@sindresorhus/is/-/is-0.7.0.tgz#9a06f4f137ee84d7df0460c1fdb1135ffa6c50fd"
 
+"@types/bluebird@*":
+  version "3.5.20"
+  resolved "https://registry.yarnpkg.com/@types/bluebird/-/bluebird-3.5.20.tgz#f6363172add6f4eabb8cada53ca9af2781e8d6a1"
+
 "@types/body-parser@*":
   version "1.16.8"
   resolved "https://registry.yarnpkg.com/@types/body-parser/-/body-parser-1.16.8.tgz#687ec34140624a3bec2b1a8ea9268478ae8f3be3"
@@ -371,6 +376,13 @@
   dependencies:
     "@types/node" "*"
 
+"@types/ioredis@3.2.5":
+  version "3.2.5"
+  resolved "https://registry.yarnpkg.com/@types/ioredis/-/ioredis-3.2.5.tgz#81889bed41ccaf175a46fb788ab7eee688bab1c6"
+  dependencies:
+    "@types/bluebird" "*"
+    "@types/node" "*"
+
 "@types/lodash@4.14.104":
   version "4.14.104"
   resolved "https://registry.yarnpkg.com/@types/lodash/-/lodash-4.14.104.tgz#53ee2357fa2e6e68379341d92eb2ecea4b11bb80"
@@ -1148,7 +1160,7 @@ blamer@^0.1.9:
     bluebird "~2.3.x"
     xml2js "~0.4.x"
 
-bluebird@^3.0.5, bluebird@^3.4.6, bluebird@^3.5.0, bluebird@^3.5.1:
+bluebird@^3.0.5, bluebird@^3.3.4, bluebird@^3.4.6, bluebird@^3.5.0, bluebird@^3.5.1:
   version "3.5.1"
   resolved "https://registry.yarnpkg.com/bluebird/-/bluebird-3.5.1.tgz#d9551f9de98f1fcda1e683d17ee91a0602ee2eb9"
 
@@ -1443,6 +1455,10 @@ clone-response@1.0.2:
   dependencies:
     mimic-response "^1.0.0"
 
+cluster-key-slot@^1.0.6:
+  version "1.0.8"
+  resolved "https://registry.yarnpkg.com/cluster-key-slot/-/cluster-key-slot-1.0.8.tgz#7654556085a65330932a2e8b5976f8e2d0b3e414"
+
 co@^4.6.0:
   version "4.6.0"
   resolved "https://registry.yarnpkg.com/co/-/co-4.6.0.tgz#6ea6bdf3d853ae54ccb8e47bfa0bf3f9031fb184"
@@ -1726,7 +1742,7 @@ dateformat@^1.0.11:
     get-stdin "^4.0.1"
     meow "^3.3.0"
 
-debug@2, debug@2.6.9, debug@^2.2.0, debug@^2.3.3, debug@^2.6.8:
+debug@2, debug@2.6.9, debug@^2.2.0, debug@^2.3.3, debug@^2.6.8, debug@^2.6.9:
   version "2.6.9"
   resolved "https://registry.yarnpkg.com/debug/-/debug-2.6.9.tgz#5d128515df134ff327e90a4c93f4e077a536341f"
   dependencies:
@@ -1783,6 +1799,10 @@ delayed-stream@~1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/delayed-stream/-/delayed-stream-1.0.0.tgz#df3ae199acadfb7d440aaae0b29e2272b24ec619"
 
+denque@^1.1.0:
+  version "1.2.3"
+  resolved "https://registry.yarnpkg.com/denque/-/denque-1.2.3.tgz#98c50c8dd8cdfae318cc5859cc8ee3da0f9b0cc2"
+
 depd@1.1.1:
   version "1.1.1"
   resolved "https://registry.yarnpkg.com/depd/-/depd-1.1.1.tgz#5783b4e1c459f06fa5ca27f991f3d06e7a310359"
@@ -2255,6 +2275,10 @@ flagged-respawn@^0.3.2:
   version "0.3.2"
   resolved "https://registry.yarnpkg.com/flagged-respawn/-/flagged-respawn-0.3.2.tgz#ff191eddcd7088a675b2610fffc976be9b8074b5"
 
+flexbuffer@0.0.6:
+  version "0.0.6"
+  resolved "https://registry.yarnpkg.com/flexbuffer/-/flexbuffer-0.0.6.tgz#039fdf23f8823e440c38f3277e6fef1174215b30"
+
 for-in@^1.0.1, for-in@^1.0.2:
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/for-in/-/for-in-1.0.2.tgz#81068d295a8142ec0ac726c6e2200c30fb6d5e80"
@@ -2884,6 +2908,34 @@ invert-kv@^1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/invert-kv/-/invert-kv-1.0.0.tgz#104a8e4aaca6d3d8cd157a8ef8bfab2d7a3ffdb6"
 
+ioredis@^3.2.2:
+  version "3.2.2"
+  resolved "https://registry.yarnpkg.com/ioredis/-/ioredis-3.2.2.tgz#b7d5ff3afd77bb9718bb2821329b894b9a44c00b"
+  dependencies:
+    bluebird "^3.3.4"
+    cluster-key-slot "^1.0.6"
+    debug "^2.6.9"
+    denque "^1.1.0"
+    flexbuffer "0.0.6"
+    lodash.assign "^4.2.0"
+    lodash.bind "^4.2.1"
+    lodash.clone "^4.5.0"
+    lodash.clonedeep "^4.5.0"
+    lodash.defaults "^4.2.0"
+    lodash.difference "^4.5.0"
+    lodash.flatten "^4.4.0"
+    lodash.foreach "^4.5.0"
+    lodash.isempty "^4.4.0"
+    lodash.keys "^4.2.0"
+    lodash.noop "^3.0.1"
+    lodash.partial "^4.2.1"
+    lodash.pick "^4.4.0"
+    lodash.sample "^4.2.1"
+    lodash.shuffle "^4.2.0"
+    lodash.values "^4.3.0"
+    redis-commands "^1.2.0"
+    redis-parser "^2.4.0"
+
 ipaddr.js@1.5.2:
   version "1.5.2"
   resolved "https://registry.yarnpkg.com/ipaddr.js/-/ipaddr.js-1.5.2.tgz#d4b505bde9946987ccf0fc58d9010ff9607e3fa0"
@@ -3483,6 +3535,34 @@ lodash.assign@^4.2.0:
   version "4.2.0"
   resolved "https://registry.yarnpkg.com/lodash.assign/-/lodash.assign-4.2.0.tgz#0d99f3ccd7a6d261d19bdaeb9245005d285808e7"
 
+lodash.bind@^4.2.1:
+  version "4.2.1"
+  resolved "https://registry.yarnpkg.com/lodash.bind/-/lodash.bind-4.2.1.tgz#7ae3017e939622ac31b7d7d7dcb1b34db1690d35"
+
+lodash.clone@^4.5.0:
+  version "4.5.0"
+  resolved "https://registry.yarnpkg.com/lodash.clone/-/lodash.clone-4.5.0.tgz#195870450f5a13192478df4bc3d23d2dea1907b6"
+
+lodash.clonedeep@^4.5.0:
+  version "4.5.0"
+  resolved "https://registry.yarnpkg.com/lodash.clonedeep/-/lodash.clonedeep-4.5.0.tgz#e23f3f9c4f8fbdde872529c1071857a086e5ccef"
+
+lodash.defaults@^4.2.0:
+  version "4.2.0"
+  resolved "https://registry.yarnpkg.com/lodash.defaults/-/lodash.defaults-4.2.0.tgz#d09178716ffea4dde9e5fb7b37f6f0802274580c"
+
+lodash.difference@^4.5.0:
+  version "4.5.0"
+  resolved "https://registry.yarnpkg.com/lodash.difference/-/lodash.difference-4.5.0.tgz#9ccb4e505d486b91651345772885a2df27fd017c"
+
+lodash.flatten@^4.4.0:
+  version "4.4.0"
+  resolved "https://registry.yarnpkg.com/lodash.flatten/-/lodash.flatten-4.4.0.tgz#f31c22225a9632d2bbf8e4addbef240aa765a61f"
+
+lodash.foreach@^4.5.0:
+  version "4.5.0"
+  resolved "https://registry.yarnpkg.com/lodash.foreach/-/lodash.foreach-4.5.0.tgz#1a6a35eace401280c7f06dddec35165ab27e3e53"
+
 lodash.includes@^4.3.0:
   version "4.3.0"
   resolved "https://registry.yarnpkg.com/lodash.includes/-/lodash.includes-4.3.0.tgz#60bb98a87cb923c68ca1e51325483314849f553f"
@@ -3491,7 +3571,7 @@ lodash.isboolean@^3.0.3:
   version "3.0.3"
   resolved "https://registry.yarnpkg.com/lodash.isboolean/-/lodash.isboolean-3.0.3.tgz#6c2e171db2a257cd96802fd43b01b20d5f5870f6"
 
-lodash.isempty@^4.2.1:
+lodash.isempty@^4.2.1, lodash.isempty@^4.4.0:
   version "4.4.0"
   resolved "https://registry.yarnpkg.com/lodash.isempty/-/lodash.isempty-4.4.0.tgz#6f86cbedd8be4ec987be9aaf33c9684db1b31e7e"
 
@@ -3511,6 +3591,10 @@ lodash.isstring@^4.0.1:
   version "4.0.1"
   resolved "https://registry.yarnpkg.com/lodash.isstring/-/lodash.isstring-4.0.1.tgz#d527dfb5456eca7cc9bb95d5daeaf88ba54a5451"
 
+lodash.keys@^4.2.0:
+  version "4.2.0"
+  resolved "https://registry.yarnpkg.com/lodash.keys/-/lodash.keys-4.2.0.tgz#a08602ac12e4fb83f91fc1fb7a360a4d9ba35205"
+
 lodash.map@^4.5.1:
   version "4.6.0"
   resolved "https://registry.yarnpkg.com/lodash.map/-/lodash.map-4.6.0.tgz#771ec7839e3473d9c4cde28b19394c3562f4f6d3"
@@ -3531,14 +3615,34 @@ lodash.once@^4.0.0:
   version "4.1.1"
   resolved "https://registry.yarnpkg.com/lodash.once/-/lodash.once-4.1.1.tgz#0dd3971213c7c56df880977d504c88fb471a97ac"
 
+lodash.partial@^4.2.1:
+  version "4.2.1"
+  resolved "https://registry.yarnpkg.com/lodash.partial/-/lodash.partial-4.2.1.tgz#49f3d8cfdaa3bff8b3a91d127e923245418961d4"
+
+lodash.pick@^4.4.0:
+  version "4.4.0"
+  resolved "https://registry.yarnpkg.com/lodash.pick/-/lodash.pick-4.4.0.tgz#52f05610fff9ded422611441ed1fc123a03001b3"
+
 lodash.reduce@4.6.0:
   version "4.6.0"
   resolved "https://registry.yarnpkg.com/lodash.reduce/-/lodash.reduce-4.6.0.tgz#f1ab6b839299ad48f784abbf476596f03b914d3b"
 
+lodash.sample@^4.2.1:
+  version "4.2.1"
+  resolved "https://registry.yarnpkg.com/lodash.sample/-/lodash.sample-4.2.1.tgz#5e4291b0c753fa1abeb0aab8fb29df1b66f07f6d"
+
+lodash.shuffle@^4.2.0:
+  version "4.2.0"
+  resolved "https://registry.yarnpkg.com/lodash.shuffle/-/lodash.shuffle-4.2.0.tgz#145b5053cf875f6f5c2a33f48b6e9948c6ec7b4b"
+
 lodash.toarray@^4.4.0:
   version "4.4.0"
   resolved "https://registry.yarnpkg.com/lodash.toarray/-/lodash.toarray-4.4.0.tgz#24c4bfcd6b2fba38bfd0594db1179d8e9b656561"
 
+lodash.values@^4.3.0:
+  version "4.3.0"
+  resolved "https://registry.yarnpkg.com/lodash.values/-/lodash.values-4.3.0.tgz#a3a6c2b0ebecc5c2cba1c17e6e620fe81b53d347"
+
 lodash@4.17.2:
   version "4.17.2"
   resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.2.tgz#34a3055babe04ce42467b607d700072c7ff6bf42"
@@ -4398,7 +4502,7 @@ redis-commands@^1.2.0:
   version "1.3.1"
   resolved "https://registry.yarnpkg.com/redis-commands/-/redis-commands-1.3.1.tgz#81d826f45fa9c8b2011f4cd7a0fe597d241d442b"
 
-redis-parser@^2.6.0:
+redis-parser@^2.4.0, redis-parser@^2.6.0:
   version "2.6.0"
   resolved "https://registry.yarnpkg.com/redis-parser/-/redis-parser-2.6.0.tgz#52ed09dacac108f1a631c07e9b69941e7a19504b"
 
