feat: Adds Redis Sentinel support. (#74)

package.json

@@ -37,7 +37,7 @@
  "@learninglocker/xapi-activities": "^4.0.2",
  "@learninglocker/xapi-agents": "^4.0.3",
  "@learninglocker/xapi-state": "^4.0.2",
- "@learninglocker/xapi-statements": "^6.0.4",
+ "@learninglocker/xapi-statements": "^7.0.0",
  "boolean": "^0.1.2",
  "dotenv": "^5.0.0",
  "express": "^4.14.1",
@@ -54,6 +54,7 @@
  "@types/dotenv": "4.0.2",
  "@types/express": "4.11.1",
  "@types/google-cloud__storage": "1.1.7",
+ "@types/ioredis": "3.2.5",
  "@types/lodash": "4.14.104",
  "@types/mongodb": "3.0.5",
  "@types/node": "9.4.6",

src/apps/AppConfig.ts

@@ -1,4 +1,5 @@
 import { S3 } from 'aws-sdk';
+import { Redis } from 'ioredis';
 import Tracker from 'jscommons/dist/tracker/Tracker';
 import { Db } from 'mongodb';
 import { LoggerInstance } from 'winston';
@@ -35,7 +36,11 @@ export default interface AppConfig {
  };
  readonly redis: {
  readonly prefix: string;
- readonly url: string;
+ readonly client: () => Promise<Redis>;
+ };
+ readonly sentinel: {
+ readonly prefix: string;
+ readonly client: () => Promise<Redis>;
  };
  };
  readonly service: {

src/apps/app.ts

@@ -78,6 +78,7 @@ export default (appConfig: AppConfig): Router => {
  mongo: appConfig.repo.mongo,
  redis: appConfig.repo.redis,
  s3: appConfig.repo.s3,
+ sentinel: appConfig.repo.sentinel,
  storageSubFolder: appConfig.repo.storageSubFolders.statements,
  },
  service: appConfig.service.statements,

src/apps/statements/AppConfig.ts

@@ -1,4 +1,5 @@
 import * as S3 from 'aws-sdk/clients/s3';
+import { Redis } from 'ioredis';
 import Tracker from 'jscommons/dist/tracker/Tracker';
 import { Db } from 'mongodb';
 import { LoggerInstance } from 'winston';
@@ -51,7 +52,11 @@ export default interface AppConfig {
  };
  readonly redis: {
  readonly prefix: string;
- readonly url: string;
+ readonly client: () => Promise<Redis>;
+ };
+ readonly sentinel: {
+ readonly prefix: string;
+ readonly client: () => Promise<Redis>;
  };
  };
 }

src/apps/statements/app.ts

@@ -18,8 +18,12 @@ export default (appConfig: AppConfig): Result => {
  events: {
  facade: appConfig.repo.factory.eventsRepoName,
  redis: {
+ client: appConfig.repo.redis.client,
  prefix: appConfig.repo.redis.prefix,
- url: appConfig.repo.redis.url,
+ },
+ sentinel: {
+ client: appConfig.repo.sentinel.client,
+ prefix: appConfig.repo.sentinel.prefix,
  },
  },
  models: {

src/config.ts

@@ -10,6 +10,7 @@ import getDbFromUrl from 'jscommons/dist/mongoRepo/utils/getDbFromUrl';
 import { defaultTo } from 'lodash';
 import * as os from 'os';
 
+const DEFAULT_REDIS_PORT = 6379;
 const DEFAULT_EXPRESS_PORT = 8081;
 const DEFAULT_TIMEOUT_MS = 300000; // 5 minutes.
 
@@ -71,6 +72,16 @@ export default {
  } as S3.ClientConfiguration,
  bucketName: getStringOption(process.env.FS_S3_BUCKET, 'xapi-service'),
  },
+ sentinel: {
+ name: getStringOption(process.env.SENTINEL_NAME, 'mymaster'),
+ prefix: getStringOption(process.env.SENTINEL_PREFIX, 'LEARNINGLOCKER'),
+ sentinels: (
+ getStringOption(process.env.SENTINEL_CONNECTIONS, '127.0.0.1:6379').split(' ').map((conn) => {
+ const [host, port] = conn.split(':');
+ return { host, port: getNumberOption(port, DEFAULT_REDIS_PORT) };
+ })
+ ),
+ },
  statementsService: {
  awaitUpdates: getBooleanOption(defaultTo<any>(
  process.env.SERVICE_AWAIT_UPDATES,

src/server.ts

@@ -8,6 +8,8 @@ import app from './apps/app';
 import config from './config';
 import logger from './logger';
 import connectToMongoDb from './utils/connectToMongoDb';
+import connectToRedis from './utils/connectToRedis';
+import connectToSentinel from './utils/connectToSentinel';
 
 const expressApp = express();
 
@@ -20,9 +22,16 @@ expressApp.use(app({
  google: config.googleStorageRepo,
  local: config.localStorageRepo,
  mongo: { db: connectToMongoDb() },
- redis: config.redis,
+ redis: {
+ client: connectToRedis(),
+ prefix: config.redis.prefix,
+ },
  repoFactory: config.repoFactory,
  s3: config.s3StorageRepo,
+ sentinel: {
+ client: connectToSentinel(),
+ prefix: config.redis.prefix,
+ },
  storageSubFolders: config.storageSubFolders,
  },
  service: {

src/utils/connectToRedis.ts

@@ -0,0 +1,13 @@
+import * as Ioredis from 'ioredis';
+import { defaultTo, once } from 'lodash';
+import config from '../config';
+import logger from '../logger';
+
+export default once((): () => Promise<Ioredis.Redis> => {
+ return once(async () => {
+ logger.info('Creating redis connection');
+ return new Ioredis(
+ defaultTo(config.redis.url, 'redis://127.0.0.1:6379/0'),
+ );
+ });
+});

src/utils/connectToSentinel.ts

@@ -0,0 +1,14 @@
+import * as Ioredis from 'ioredis';
+import { defaultTo, once } from 'lodash';
+import config from '../config';
+import logger from '../logger';
+
+export default once((): () => Promise<Ioredis.Redis> => {
+ return once(async () => {
+ logger.info('Creating sentinel connection');
+ return new Ioredis({
+ name: defaultTo(config.sentinel.name, 'mymaster'),
+ sentinels: defaultTo(config.sentinel.sentinels, [{ host: '127.0.0.1', port: 6379 }]),
+ });
+ });
+});

yarn.lock

@@ -152,9 +152,9 @@
  string-to-stream "^1.1.0"
  uuid "^3.0.1"
 
-"@learninglocker/xapi-statements@^6.0.4":
- version "6.0.4"
- resolved "https://registry.yarnpkg.com/@learninglocker/xapi-statements/-/xapi-statements-6.0.4.tgz#5750d3e21a307e6ecfb1669749ade5c6cd0cd4eb"
+"@learninglocker/xapi-statements@^7.0.0":
+ version "7.0.0"
+ resolved "https://registry.yarnpkg.com/@learninglocker/xapi-statements/-/xapi-statements-7.0.0.tgz#08fd8f02561d7f20921c4faf85b2493bb7ca78de"
  dependencies:
  "@google-cloud/storage" "^1.5.2"
  "@learninglocker/xapi-validation" "^2.1.10"
@@ -171,6 +171,7 @@
  file-stream-rotator "^0.2.0"
  fs-extra "^5.0.0"
  helmet "^3.5.0"
+ ioredis "^3.2.2"
  jscommons "^2.3.0"
  jsonwebtoken "^8.0.1"
  lodash "^4.17.4"
@@ -327,6 +328,10 @@
  version "0.7.0"
  resolved "https://registry.yarnpkg.com/@sindresorhus/is/-/is-0.7.0.tgz#9a06f4f137ee84d7df0460c1fdb1135ffa6c50fd"
 
+"@types/bluebird@*":
+ version "3.5.20"
+ resolved "https://registry.yarnpkg.com/@types/bluebird/-/bluebird-3.5.20.tgz#f6363172add6f4eabb8cada53ca9af2781e8d6a1"
+
 "@types/body-parser@*":
  version "1.16.8"
  resolved "https://registry.yarnpkg.com/@types/body-parser/-/body-parser-1.16.8.tgz#687ec34140624a3bec2b1a8ea9268478ae8f3be3"
@@ -371,6 +376,13 @@
  dependencies:
  "@types/node" "*"
 
+"@types/ioredis@3.2.5":
+ version "3.2.5"
+ resolved "https://registry.yarnpkg.com/@types/ioredis/-/ioredis-3.2.5.tgz#81889bed41ccaf175a46fb788ab7eee688bab1c6"
+ dependencies:
+ "@types/bluebird" "*"
+ "@types/node" "*"
+
 "@types/lodash@4.14.104":
  version "4.14.104"
  resolved "https://registry.yarnpkg.com/@types/lodash/-/lodash-4.14.104.tgz#53ee2357fa2e6e68379341d92eb2ecea4b11bb80"
@@ -1148,7 +1160,7 @@ blamer@^0.1.9:
  bluebird "~2.3.x"
  xml2js "~0.4.x"
 
-bluebird@^3.0.5, bluebird@^3.4.6, bluebird@^3.5.0, bluebird@^3.5.1:
+bluebird@^3.0.5, bluebird@^3.3.4, bluebird@^3.4.6, bluebird@^3.5.0, bluebird@^3.5.1:
  version "3.5.1"
  resolved "https://registry.yarnpkg.com/bluebird/-/bluebird-3.5.1.tgz#d9551f9de98f1fcda1e683d17ee91a0602ee2eb9"
 
@@ -1443,6 +1455,10 @@ clone-response@1.0.2:
  dependencies:
  mimic-response "^1.0.0"
 
+cluster-key-slot@^1.0.6:
+ version "1.0.8"
+ resolved "https://registry.yarnpkg.com/cluster-key-slot/-/cluster-key-slot-1.0.8.tgz#7654556085a65330932a2e8b5976f8e2d0b3e414"
+
 co@^4.6.0:
  version "4.6.0"
  resolved "https://registry.yarnpkg.com/co/-/co-4.6.0.tgz#6ea6bdf3d853ae54ccb8e47bfa0bf3f9031fb184"
@@ -1726,7 +1742,7 @@ dateformat@^1.0.11:
  get-stdin "^4.0.1"
  meow "^3.3.0"
 
-debug@2, debug@2.6.9, debug@^2.2.0, debug@^2.3.3, debug@^2.6.8:
+debug@2, debug@2.6.9, debug@^2.2.0, debug@^2.3.3, debug@^2.6.8, debug@^2.6.9:
  version "2.6.9"
  resolved "https://registry.yarnpkg.com/debug/-/debug-2.6.9.tgz#5d128515df134ff327e90a4c93f4e077a536341f"
  dependencies:
@@ -1783,6 +1799,10 @@ delayed-stream@~1.0.0:
  version "1.0.0"
  resolved "https://registry.yarnpkg.com/delayed-stream/-/delayed-stream-1.0.0.tgz#df3ae199acadfb7d440aaae0b29e2272b24ec619"
 
+denque@^1.1.0:
+ version "1.2.3"
+ resolved "https://registry.yarnpkg.com/denque/-/denque-1.2.3.tgz#98c50c8dd8cdfae318cc5859cc8ee3da0f9b0cc2"
+
 depd@1.1.1:
  version "1.1.1"
  resolved "https://registry.yarnpkg.com/depd/-/depd-1.1.1.tgz#5783b4e1c459f06fa5ca27f991f3d06e7a310359"
@@ -2255,6 +2275,10 @@ flagged-respawn@^0.3.2:
  version "0.3.2"
  resolved "https://registry.yarnpkg.com/flagged-respawn/-/flagged-respawn-0.3.2.tgz#ff191eddcd7088a675b2610fffc976be9b8074b5"
 
+flexbuffer@0.0.6:
+ version "0.0.6"
+ resolved "https://registry.yarnpkg.com/flexbuffer/-/flexbuffer-0.0.6.tgz#039fdf23f8823e440c38f3277e6fef1174215b30"
+
 for-in@^1.0.1, for-in@^1.0.2:
  version "1.0.2"
  resolved "https://registry.yarnpkg.com/for-in/-/for-in-1.0.2.tgz#81068d295a8142ec0ac726c6e2200c30fb6d5e80"
@@ -2884,6 +2908,34 @@ invert-kv@^1.0.0:
  version "1.0.0"
  resolved "https://registry.yarnpkg.com/invert-kv/-/invert-kv-1.0.0.tgz#104a8e4aaca6d3d8cd157a8ef8bfab2d7a3ffdb6"
 
+ioredis@^3.2.2:
+ version "3.2.2"
+ resolved "https://registry.yarnpkg.com/ioredis/-/ioredis-3.2.2.tgz#b7d5ff3afd77bb9718bb2821329b894b9a44c00b"
+ dependencies:
+ bluebird "^3.3.4"
+ cluster-key-slot "^1.0.6"
+ debug "^2.6.9"
+ denque "^1.1.0"
+ flexbuffer "0.0.6"
+ lodash.assign "^4.2.0"
+ lodash.bind "^4.2.1"
+ lodash.clone "^4.5.0"
+ lodash.clonedeep "^4.5.0"
+ lodash.defaults "^4.2.0"
+ lodash.difference "^4.5.0"
+ lodash.flatten "^4.4.0"
+ lodash.foreach "^4.5.0"
+ lodash.isempty "^4.4.0"
+ lodash.keys "^4.2.0"
+ lodash.noop "^3.0.1"
+ lodash.partial "^4.2.1"
+ lodash.pick "^4.4.0"
+ lodash.sample "^4.2.1"
+ lodash.shuffle "^4.2.0"
+ lodash.values "^4.3.0"
+ redis-commands "^1.2.0"
+ redis-parser "^2.4.0"
+
 ipaddr.js@1.5.2:
  version "1.5.2"
  resolved "https://registry.yarnpkg.com/ipaddr.js/-/ipaddr.js-1.5.2.tgz#d4b505bde9946987ccf0fc58d9010ff9607e3fa0"
@@ -3483,6 +3535,34 @@ lodash.assign@^4.2.0:
  version "4.2.0"
  resolved "https://registry.yarnpkg.com/lodash.assign/-/lodash.assign-4.2.0.tgz#0d99f3ccd7a6d261d19bdaeb9245005d285808e7"
 
+lodash.bind@^4.2.1:
+ version "4.2.1"
+ resolved "https://registry.yarnpkg.com/lodash.bind/-/lodash.bind-4.2.1.tgz#7ae3017e939622ac31b7d7d7dcb1b34db1690d35"
+
+lodash.clone@^4.5.0:
+ version "4.5.0"
+ resolved "https://registry.yarnpkg.com/lodash.clone/-/lodash.clone-4.5.0.tgz#195870450f5a13192478df4bc3d23d2dea1907b6"
+
+lodash.clonedeep@^4.5.0:
+ version "4.5.0"
+ resolved "https://registry.yarnpkg.com/lodash.clonedeep/-/lodash.clonedeep-4.5.0.tgz#e23f3f9c4f8fbdde872529c1071857a086e5ccef"
+
+lodash.defaults@^4.2.0:
+ version "4.2.0"
+ resolved "https://registry.yarnpkg.com/lodash.defaults/-/lodash.defaults-4.2.0.tgz#d09178716ffea4dde9e5fb7b37f6f0802274580c"
+
+lodash.difference@^4.5.0:
+ version "4.5.0"
+ resolved "https://registry.yarnpkg.com/lodash.difference/-/lodash.difference-4.5.0.tgz#9ccb4e505d486b91651345772885a2df27fd017c"
+
+lodash.flatten@^4.4.0:
+ version "4.4.0"
+ resolved "https://registry.yarnpkg.com/lodash.flatten/-/lodash.flatten-4.4.0.tgz#f31c22225a9632d2bbf8e4addbef240aa765a61f"
+
+lodash.foreach@^4.5.0:
+ version "4.5.0"
+ resolved "https://registry.yarnpkg.com/lodash.foreach/-/lodash.foreach-4.5.0.tgz#1a6a35eace401280c7f06dddec35165ab27e3e53"
+
 lodash.includes@^4.3.0:
  version "4.3.0"
  resolved "https://registry.yarnpkg.com/lodash.includes/-/lodash.includes-4.3.0.tgz#60bb98a87cb923c68ca1e51325483314849f553f"
@@ -3491,7 +3571,7 @@ lodash.isboolean@^3.0.3:
  version "3.0.3"
  resolved "https://registry.yarnpkg.com/lodash.isboolean/-/lodash.isboolean-3.0.3.tgz#6c2e171db2a257cd96802fd43b01b20d5f5870f6"
 
-lodash.isempty@^4.2.1:
+lodash.isempty@^4.2.1, lodash.isempty@^4.4.0:
  version "4.4.0"
  resolved "https://registry.yarnpkg.com/lodash.isempty/-/lodash.isempty-4.4.0.tgz#6f86cbedd8be4ec987be9aaf33c9684db1b31e7e"
 
@@ -3511,6 +3591,10 @@ lodash.isstring@^4.0.1:
  version "4.0.1"
  resolved "https://registry.yarnpkg.com/lodash.isstring/-/lodash.isstring-4.0.1.tgz#d527dfb5456eca7cc9bb95d5daeaf88ba54a5451"
 
+lodash.keys@^4.2.0:
+ version "4.2.0"
+ resolved "https://registry.yarnpkg.com/lodash.keys/-/lodash.keys-4.2.0.tgz#a08602ac12e4fb83f91fc1fb7a360a4d9ba35205"
+
 lodash.map@^4.5.1:
  version "4.6.0"
  resolved "https://registry.yarnpkg.com/lodash.map/-/lodash.map-4.6.0.tgz#771ec7839e3473d9c4cde28b19394c3562f4f6d3"
@@ -3531,14 +3615,34 @@ lodash.once@^4.0.0:
  version "4.1.1"
  resolved "https://registry.yarnpkg.com/lodash.once/-/lodash.once-4.1.1.tgz#0dd3971213c7c56df880977d504c88fb471a97ac"
 
+lodash.partial@^4.2.1:
+ version "4.2.1"
+ resolved "https://registry.yarnpkg.com/lodash.partial/-/lodash.partial-4.2.1.tgz#49f3d8cfdaa3bff8b3a91d127e923245418961d4"
+
+lodash.pick@^4.4.0:
+ version "4.4.0"
+ resolved "https://registry.yarnpkg.com/lodash.pick/-/lodash.pick-4.4.0.tgz#52f05610fff9ded422611441ed1fc123a03001b3"
+
 lodash.reduce@4.6.0:
  version "4.6.0"
  resolved "https://registry.yarnpkg.com/lodash.reduce/-/lodash.reduce-4.6.0.tgz#f1ab6b839299ad48f784abbf476596f03b914d3b"
 
+lodash.sample@^4.2.1:
+ version "4.2.1"
+ resolved "https://registry.yarnpkg.com/lodash.sample/-/lodash.sample-4.2.1.tgz#5e4291b0c753fa1abeb0aab8fb29df1b66f07f6d"
+
+lodash.shuffle@^4.2.0:
+ version "4.2.0"
+ resolved "https://registry.yarnpkg.com/lodash.shuffle/-/lodash.shuffle-4.2.0.tgz#145b5053cf875f6f5c2a33f48b6e9948c6ec7b4b"
+
 lodash.toarray@^4.4.0:
  version "4.4.0"
  resolved "https://registry.yarnpkg.com/lodash.toarray/-/lodash.toarray-4.4.0.tgz#24c4bfcd6b2fba38bfd0594db1179d8e9b656561"
 
+lodash.values@^4.3.0:
+ version "4.3.0"
+ resolved "https://registry.yarnpkg.com/lodash.values/-/lodash.values-4.3.0.tgz#a3a6c2b0ebecc5c2cba1c17e6e620fe81b53d347"
+
 lodash@4.17.2:
  version "4.17.2"
  resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.2.tgz#34a3055babe04ce42467b607d700072c7ff6bf42"
@@ -4398,7 +4502,7 @@ redis-commands@^1.2.0:
  version "1.3.1"
  resolved "https://registry.yarnpkg.com/redis-commands/-/redis-commands-1.3.1.tgz#81d826f45fa9c8b2011f4cd7a0fe597d241d442b"
 
-redis-parser@^2.6.0:
+redis-parser@^2.4.0, redis-parser@^2.6.0:
  version "2.6.0"
  resolved "https://registry.yarnpkg.com/redis-parser/-/redis-parser-2.6.0.tgz#52ed09dacac108f1a631c07e9b69941e7a19504b"