Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

CLDSRV-547 [HOTFIX 9.3.0.5] cherry-pick #5606

Merged
merged 2 commits into from
Jun 27, 2024
Merged

CLDSRV-547 [HOTFIX 9.3.0.5] cherry-pick #5606

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
6084921
bf: CLDSRV-547 update redis config for utapi reindex
jonathan-gramain Jun 25, 2024
3f8dcf9
CLDSRV-547 [hotfix 7.70] bump version to 7.70.45-3
jonathan-gramain Jun 27, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
186 changes: 62 additions & 124 deletions lib/Config.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,47 @@ function parseSproxydConfig(configSproxyd) {
return joi.attempt(configSproxyd, joiSchema, 'bad config');
}

function parseRedisConfig(redisConfig) {
const joiSchema = joi.object({
password: joi.string().allow(''),
host: joi.string(),
port: joi.number(),
retry: joi.object({
connectBackoff: joi.object({
min: joi.number().required(),
max: joi.number().required(),
jitter: joi.number().required(),
factor: joi.number().required(),
deadline: joi.number().required(),
}),
}),
// sentinel config
sentinels: joi.alternatives().try(
joi.string()
.pattern(/^[a-zA-Z0-9.-]+:[0-9]+(,[a-zA-Z0-9.-]+:[0-9]+)*$/)
.custom(hosts => hosts.split(',').map(item => {
const [host, port] = item.split(':');
return { host, port: Number.parseInt(port, 10) };
})),
joi.array().items(
joi.object({
host: joi.string().required(),
port: joi.number().required(),
})
).min(1),
),
name: joi.string(),
sentinelPassword: joi.string().allow(''),
})
.and('host', 'port')
.and('sentinels', 'name')
.xor('host', 'sentinels')
.without('sentinels', ['host', 'port'])
.without('host', ['sentinels', 'sentinelPassword']);

return joi.attempt(redisConfig, joiSchema, 'bad config');
}

function restEndpointsAssert(restEndpoints, locationConstraints) {
assert(typeof restEndpoints === 'object',
'bad config: restEndpoints must be an object of endpoints');
Expand Down Expand Up @@ -294,18 +335,17 @@ function parseUtapiReindex(config) {
const {
enabled,
schedule,
sentinel,
redis,
bucketd,
onlyCountLatestWhenObjectLocked,
} = config;
assert(typeof enabled === 'boolean',
'bad config: utapi.reindex.enabled must be a boolean');
assert(typeof sentinel === 'object',
'bad config: utapi.reindex.sentinel must be an object');
assert(typeof sentinel.port === 'number',
'bad config: utapi.reindex.sentinel.port must be a number');
assert(typeof sentinel.name === 'string',
'bad config: utapi.reindex.sentinel.name must be a string');
'bad config: utapi.reindex.enabled must be a boolean');

const parsedRedis = parseRedisConfig(redis);
assert(Array.isArray(parsedRedis.sentinels),
'bad config: utapi reindex redis config requires a list of sentinels');

assert(typeof bucketd === 'object',
'bad config: utapi.reindex.bucketd must be an object');
assert(typeof bucketd.port === 'number',
Expand All @@ -323,6 +363,13 @@ function parseUtapiReindex(config) {
'bad config: utapi.reindex.schedule must be a valid ' +
`cron schedule. ${e.message}.`);
}
return {
enabled,
schedule,
redis: parsedRedis,
bucketd,
onlyCountLatestWhenObjectLocked,
};
}

function requestsConfigAssert(requestsConfig) {
Expand Down Expand Up @@ -761,8 +808,7 @@ class Config extends EventEmitter {
assert(typeof config.localCache.port === 'number',
'config: invalid port for localCache. port must be a number');
if (config.localCache.password !== undefined) {
assert(
this._verifyRedisPassword(config.localCache.password),
assert(typeof config.localCache.password === 'string',
'config: invalid password for localCache. password must' +
' be a string');
}
Expand All @@ -774,55 +820,7 @@ class Config extends EventEmitter {
}

if (config.redis) {
if (config.redis.sentinels) {
this.redis = { sentinels: [], name: null };

assert(typeof config.redis.name === 'string',
'bad config: redis sentinel name must be a string');
this.redis.name = config.redis.name;
assert(Array.isArray(config.redis.sentinels) ||
typeof config.redis.sentinels === 'string',
'bad config: redis sentinels must be an array or string');

if (typeof config.redis.sentinels === 'string') {
config.redis.sentinels.split(',').forEach(item => {
const [host, port] = item.split(':');
this.redis.sentinels.push({ host,
port: Number.parseInt(port, 10) });
});
} else if (Array.isArray(config.redis.sentinels)) {
config.redis.sentinels.forEach(item => {
const { host, port } = item;
assert(typeof host === 'string',
'bad config: redis sentinel host must be a string');
assert(typeof port === 'number',
'bad config: redis sentinel port must be a number');
this.redis.sentinels.push({ host, port });
});
}

if (config.redis.sentinelPassword !== undefined) {
assert(
this._verifyRedisPassword(config.redis.sentinelPassword));
this.redis.sentinelPassword = config.redis.sentinelPassword;
}
} else {
// check for standalone configuration
this.redis = {};
assert(typeof config.redis.host === 'string',
'bad config: redis.host must be a string');
assert(typeof config.redis.port === 'number',
'bad config: redis.port must be a number');
this.redis.host = config.redis.host;
this.redis.port = config.redis.port;
}
if (config.redis.password !== undefined) {
assert(
this._verifyRedisPassword(config.redis.password),
'bad config: invalid password for redis. password must ' +
'be a string');
this.redis.password = config.redis.password;
}
this.redis = parseRedisConfig(config.redis);
}
if (config.utapi) {
this.utapi = { component: 's3' };
Expand Down Expand Up @@ -851,65 +849,8 @@ class Config extends EventEmitter {
this.utapi.localCache = config.localCache;
assert(config.utapi.redis, 'missing required property of utapi ' +
'configuration: redis');
if (config.utapi.redis.sentinels) {
this.utapi.redis = { sentinels: [], name: null };

assert(typeof config.utapi.redis.name === 'string',
'bad config: redis sentinel name must be a string');
this.utapi.redis.name = config.utapi.redis.name;

assert(Array.isArray(config.utapi.redis.sentinels),
'bad config: redis sentinels must be an array');
config.utapi.redis.sentinels.forEach(item => {
const { host, port } = item;
assert(typeof host === 'string',
'bad config: redis sentinel host must be a string');
assert(typeof port === 'number',
'bad config: redis sentinel port must be a number');
this.utapi.redis.sentinels.push({ host, port });
});
} else {
// check for standalone configuration
this.utapi.redis = {};
assert(typeof config.utapi.redis.host === 'string',
'bad config: redis.host must be a string');
assert(typeof config.utapi.redis.port === 'number',
'bad config: redis.port must be a number');
this.utapi.redis.host = config.utapi.redis.host;
this.utapi.redis.port = config.utapi.redis.port;
}
if (config.utapi.redis.password !== undefined) {
assert(
this._verifyRedisPassword(config.utapi.redis.password),
'config: invalid password for utapi redis. password' +
' must be a string');
this.utapi.redis.password = config.utapi.redis.password;
}
if (config.utapi.redis.sentinelPassword !== undefined) {
assert(
this._verifyRedisPassword(config.utapi.redis.sentinelPassword),
'config: invalid password for utapi redis. password' +
' must be a string');
this.utapi.redis.sentinelPassword =
config.utapi.redis.sentinelPassword;
}
if (config.utapi.redis.retry !== undefined) {
if (config.utapi.redis.retry.connectBackoff !== undefined) {
const { min, max, jitter, factor, deadline } = config.utapi.redis.retry.connectBackoff;
assert.strictEqual(typeof min, 'number',
'utapi.redis.retry.connectBackoff: min must be a number');
assert.strictEqual(typeof max, 'number',
'utapi.redis.retry.connectBackoff: max must be a number');
assert.strictEqual(typeof jitter, 'number',
'utapi.redis.retry.connectBackoff: jitter must be a number');
assert.strictEqual(typeof factor, 'number',
'utapi.redis.retry.connectBackoff: factor must be a number');
assert.strictEqual(typeof deadline, 'number',
'utapi.redis.retry.connectBackoff: deadline must be a number');
}

this.utapi.redis.retry = config.utapi.redis.retry;
} else {
this.utapi.redis = parseRedisConfig(config.utapi.redis);
if (this.utapi.redis.retry === undefined) {
this.utapi.redis.retry = {
connectBackoff: {
min: 10,
Expand All @@ -920,6 +861,7 @@ class Config extends EventEmitter {
},
};
}

if (config.utapi.metrics) {
this.utapi.metrics = config.utapi.metrics;
}
Expand Down Expand Up @@ -988,8 +930,7 @@ class Config extends EventEmitter {
}

if (config.utapi && config.utapi.reindex) {
parseUtapiReindex(config.utapi.reindex);
this.utapi.reindex = config.utapi.reindex;
this.utapi.reindex = parseUtapiReindex(config.utapi.reindex);
}
}

Expand Down Expand Up @@ -1430,10 +1371,6 @@ class Config extends EventEmitter {
};
}

_verifyRedisPassword(password) {
return typeof password === 'string';
}

setAuthDataAccounts(accounts) {
this.authData.accounts = accounts;
this.emit('authdata-update');
Expand Down Expand Up @@ -1547,6 +1484,7 @@ class Config extends EventEmitter {

module.exports = {
parseSproxydConfig,
parseRedisConfig,
locationConstraintAssert,
ConfigObject: Config,
config: new Config(),
Expand Down
4 changes: 2 additions & 2 deletions package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "s3",
"version": "7.70.45-2",
"version": "7.70.45-3",
"description": "S3 connector",
"main": "index.js",
"engines": {
Expand Down Expand Up @@ -35,7 +35,7 @@
"moment": "^2.26.0",
"npm-run-all": "~4.1.5",
"prom-client": "14.2.0",
"utapi": "git+https://github.com/scality/utapi#7.70.4",
"utapi": "git+https://github.com/scality/utapi#7.70.5",
"utf8": "~2.1.1",
"uuid": "^3.0.1",
"vaultclient": "scality/vaultclient#7.10.13",
Expand Down
Loading
Loading