Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(csi-369): added support for both: redis and redis-cluster #8

Merged
merged 2 commits into from
Jul 29, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion .env
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
PROXY_CACHE_LOG_LEVEL=debug

REDIS_HOST=localhost
REDIS_PORT=6379
REDIS_STANDALONE_PORT=6379
REDIS_CLUSTER_PORT=16379
17 changes: 12 additions & 5 deletions docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -22,19 +22,26 @@ services:
- redis-node-4
- redis-node-5
ports:
- "6379:6379"
- "${REDIS_CLUSTER_PORT}:6379"

redis-node-1:
<<: *REDIS_NODE

redis-node-2:
<<: *REDIS_NODE

redis-node-3:
<<: *REDIS_NODE

redis-node-4:
<<: *REDIS_NODE

redis-node-5:
<<: *REDIS_NODE


redis:
image: redis:6.2.4-alpine
restart: "unless-stopped"
environment:
- ALLOW_EMPTY_PASSWORD=yes
- REDIS_PORT=6379
- REDIS_REPLICATION_MODE=master
ports:
- "${REDIS_STANDALONE_PORT}:6379"
28 changes: 14 additions & 14 deletions package-lock.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

8 changes: 4 additions & 4 deletions package.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "@mojaloop/inter-scheme-proxy-cache-lib",
"version": "2.0.0-snapshot.1",
"version": "2.2.0-snapshot.0",
"description": "Common component, that provides scheme proxy caching mapping (ISPC)",
"author": "Eugen Klymniuk (geka-evk)",
"contributors": [
Expand Down Expand Up @@ -35,7 +35,7 @@
"prepublishOnly": "npm run build"
},
"dependencies": {
"@mojaloop/central-services-logger": "11.4.5",
"@mojaloop/central-services-logger": "11.5.0",
"ajv": "^8.17.1",
"convict": "^6.2.4",
"fast-safe-stringify": "^2.1.1",
Expand All @@ -53,7 +53,7 @@
"eslint": "8.56.0",
"eslint-config-prettier": "9.1.0",
"eslint-plugin-import": "2.29.1",
"husky": "^9.1.1",
"husky": "^9.1.2",
"ioredis-mock": "^8.9.0",
"jest": "^29.7.0",
"jest-junit": "^16.0.0",
Expand All @@ -65,7 +65,7 @@
"standard-version": "^9.5.0",
"ts-jest": "29.2.3",
"ts-node": "10.9.2",
"tsup": "^8.2.2",
"tsup": "^8.2.3",
"typedoc": "^0.26.5",
"typedoc-theme-hierarchy": "^5.0.3",
"typescript": "5.5.4"
Expand Down
1 change: 1 addition & 0 deletions src/constants.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
export const STORAGE_TYPES = {
redis: 'redis',
redisCluster: 'redis-cluster',
mysql: 'mysql',
} as const;

Expand Down
5 changes: 4 additions & 1 deletion src/lib/createProxyCache.ts
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@
**********/

import { ProxyCacheFactory, StorageType, ProxyCacheConfig } from '../types';
import { validateRedisProxyCacheConfig } from '../validation';
import { validateRedisClusterProxyCacheConfig, validateRedisProxyCacheConfig } from '../validation';
import { logger } from '../utils';
import { STORAGE_TYPES } from '../constants';
import { ProxyCacheError } from './errors';
Expand All @@ -35,6 +35,9 @@ export const createProxyCache: ProxyCacheFactory = (type: StorageType, proxyConf
case STORAGE_TYPES.redis: {
return new storages.RedisProxyCache(validateRedisProxyCacheConfig(proxyConfig));
}
case STORAGE_TYPES.redisCluster: {
return new storages.RedisProxyCache(validateRedisClusterProxyCacheConfig(proxyConfig));
}
case STORAGE_TYPES.mysql:
throw new Error('Mysql storage is not implemented yet');
default: {
Expand Down
37 changes: 27 additions & 10 deletions src/lib/storages/RedisProxyCache.ts
Original file line number Diff line number Diff line change
@@ -1,17 +1,29 @@
import { Cluster } from 'ioredis';
import Redis, { Cluster } from 'ioredis';

import * as validation from '../../validation';
import config from '../../config';
import { createLogger } from '../../utils';
import { IProxyCache, RedisProxyCacheConfig, IsLastFailure, AlsRequestDetails, ILogger } from '../../types';
import {
IProxyCache,
RedisProxyCacheConfig,
RedisClusterProxyCacheConfig,
IsLastFailure,
AlsRequestDetails,
ILogger,
} from '../../types';
import { REDIS_KEYS_PREFIXES, REDIS_SUCCESS, REDIS_IS_CONNECTED_STATUSES } from './constants';

type RedisClient = Redis | Cluster;
type RedisConfig = RedisProxyCacheConfig | RedisClusterProxyCacheConfig;

const isClusterConfig = (config: RedisConfig): config is RedisClusterProxyCacheConfig => 'cluster' in config;

export class RedisProxyCache implements IProxyCache {
private readonly redisClient: Cluster;
private readonly redisClient: RedisClient;
private readonly log: ILogger;
private readonly defaultTtlSec = config.get('defaultTtlSec');

constructor(private readonly proxyConfig: RedisProxyCacheConfig) {
constructor(private readonly proxyConfig: RedisConfig) {
this.log = createLogger(this.constructor.name);
this.redisClient = this.createRedisClient();
}
Expand Down Expand Up @@ -118,11 +130,18 @@ export class RedisProxyCache implements IProxyCache {
}

private createRedisClient() {
const { log } = this;
const { cluster, ...redisOptions } = this.proxyConfig;
const { lazyConnect = true } = redisOptions;
this.proxyConfig.lazyConnect ??= true;
// prettier-ignore
const redisClient = isClusterConfig(this.proxyConfig)
? new Cluster(this.proxyConfig.cluster, this.proxyConfig)
: new Redis(this.proxyConfig);

this.addEventListeners(redisClient);
return redisClient;
}

const redisClient = new Cluster(cluster, { ...redisOptions, lazyConnect });
private addEventListeners(redisClient: RedisClient) {
const { log } = this;
// prettier-ignore
redisClient
.on('error', (err) => { log.error('redis connection error', err); })
Expand All @@ -131,8 +150,6 @@ export class RedisProxyCache implements IProxyCache {
.on('reconnecting', (ms: number) => { log.info('redis connection reconnecting', { ms }); })
.on('connect', () => { log.verbose('redis connection is established'); })
.on('ready', () => { log.verbose('redis connection is ready'); });

return redisClient;
}

private async executePipeline(commands: [string, ...any[]][]): Promise<unknown[]> {
Expand Down
11 changes: 7 additions & 4 deletions src/types/lib.ts
Original file line number Diff line number Diff line change
Expand Up @@ -39,13 +39,14 @@ export type IsLastFailure = boolean;
export type ProxyCacheFactory = (type: StorageType, proxyConfig: ProxyCacheConfig) => IProxyCache;
// todo: think about making proxyConfig optional, and assemble it using env vars if it wasn't passed

export type ProxyCacheConfig = RedisProxyCacheConfig | MySqlProxyCacheConfig;
export type ProxyCacheConfig = RedisProxyCacheConfig | RedisClusterProxyCacheConfig | MySqlProxyCacheConfig;

export type RedisProxyCacheConfig = Prettify<RedisConnectionConfig & RedisOptions>;
export type RedisProxyCacheConfig = Prettify<BasicConnectionConfig & RedisOptions>;

export type RedisConnectionConfig = {
export type RedisClusterProxyCacheConfig = Prettify<RedisClusterConnectionConfig & RedisClusterOptions>;

export type RedisClusterConnectionConfig = {
cluster: BasicConnectionConfig[];
// todo: think, if it's better to add also { host, port } options for standalone redis
};

export type RedisOptions = {
Expand All @@ -57,6 +58,8 @@ export type RedisOptions = {
// define all needed options here
};

export type RedisClusterOptions = RedisOptions;

/** **(!)** _MySqlProxyCacheConfig_ is not supported yet */
// prettier-ignore
export type MySqlProxyCacheConfig = Prettify<BasicConnectionConfig & {
Expand Down
43 changes: 36 additions & 7 deletions src/validation.ts
Original file line number Diff line number Diff line change
Expand Up @@ -27,22 +27,48 @@
* We need a separate validation, coz the library might be used in JS projects, where TS type checks are not available
*/
import Ajv, { JSONSchemaType } from 'ajv';
import { RedisProxyCacheConfig, BasicConnectionConfig, AlsRequestDetails } from './types';
import { RedisProxyCacheConfig, BasicConnectionConfig, AlsRequestDetails, RedisClusterProxyCacheConfig } from './types';
import { ValidationError } from '../src/lib';

const ajv = new Ajv();

const RedisProxyCacheConfigSchema: JSONSchemaType<RedisProxyCacheConfig> = {
type: 'object',
properties: {
host: { type: 'string', minLength: 1 },
port: { type: 'integer' },
// todo: think, how to avoid duplication of the same fields for both redis schemas
username: { type: 'string', nullable: true },
password: { type: 'string', nullable: true },
lazyConnect: { type: 'boolean', nullable: true },
db: { type: 'number', nullable: true },
// find a better way to define optional params (without nullable: true)
},
required: ['host', 'port'],
additionalProperties: true,
};
const redisProxyCacheConfigValidatingFn = ajv.compile<RedisProxyCacheConfig>(RedisProxyCacheConfigSchema);

export const validateRedisProxyCacheConfig = (cacheConfig: unknown): RedisProxyCacheConfig => {
const isValid = redisProxyCacheConfigValidatingFn(cacheConfig);
if (!isValid) {
const errDetails = `redisProxyCacheConfig error: ${redisProxyCacheConfigValidatingFn.errors![0]!.message}`;
throw ValidationError.invalidFormat(errDetails);
}
return cacheConfig;
};

const ClusterSchema: JSONSchemaType<BasicConnectionConfig> = {
type: 'object',
properties: {
host: { type: 'string' },
host: { type: 'string', minLength: 1 },
port: { type: 'integer' },
},
required: ['host', 'port'],
additionalProperties: false,
};

const RedisProxyCacheConfigSchema: JSONSchemaType<RedisProxyCacheConfig> = {
const RedisClusterProxyCacheConfigSchema: JSONSchemaType<RedisClusterProxyCacheConfig> = {
type: 'object',
properties: {
cluster: { type: 'array', items: ClusterSchema, minItems: 1 },
Expand All @@ -55,12 +81,15 @@ const RedisProxyCacheConfigSchema: JSONSchemaType<RedisProxyCacheConfig> = {
required: ['cluster'],
additionalProperties: true,
};
const redisProxyCacheConfigValidatingFn = ajv.compile<RedisProxyCacheConfig>(RedisProxyCacheConfigSchema);

export const validateRedisProxyCacheConfig = (cacheConfig: unknown): RedisProxyCacheConfig => {
const isValid = redisProxyCacheConfigValidatingFn(cacheConfig);
const redisClusterProxyCacheConfigValidatingFn = ajv.compile<RedisClusterProxyCacheConfig>(
RedisClusterProxyCacheConfigSchema,
);

export const validateRedisClusterProxyCacheConfig = (cacheConfig: unknown): RedisClusterProxyCacheConfig => {
const isValid = redisClusterProxyCacheConfigValidatingFn(cacheConfig);
if (!isValid) {
const errDetails = `redisProxyCacheConfig: ${redisProxyCacheConfigValidatingFn.errors![0]!.message}`;
const errDetails = `redisClusterProxyCacheConfig error: ${redisClusterProxyCacheConfigValidatingFn.errors![0]!.message}`;
throw ValidationError.invalidFormat(errDetails);
}
return cacheConfig;
Expand Down
19 changes: 15 additions & 4 deletions test/fixtures.ts
Original file line number Diff line number Diff line change
@@ -1,17 +1,28 @@
import { RedisProxyCacheConfig, AlsRequestDetails } from '#src/types';
import { randomUUID } from 'node:crypto';
import { RedisProxyCacheConfig, RedisClusterProxyCacheConfig, AlsRequestDetails } from '#src/types';

export const redisProxyConfigDto = ({
cluster = [{ host: '127.0.0.1', port: 16379 }],
host = '127.0.0.1',
port = 26379,
lazyConnect = true,
} = {}): RedisProxyCacheConfig => ({
host,
port,
lazyConnect,
});

export const redisClusterProxyConfigDto = ({
cluster = [{ host: '127.0.0.1', port: 56379 }],
lazyConnect = true,
} = {}): RedisClusterProxyCacheConfig => ({
cluster,
lazyConnect,
});

export const alsRequestDetailsDto = ({
sourceId = 'test-source',
sourceId = `test-source-${Date.now()}`,
type = 'MSISDN', // todo: use enum for type
partyId = `${Date.now()}`,
partyId = `party-${randomUUID()}`,
} = {}): AlsRequestDetails => ({
sourceId,
type,
Expand Down
Loading