diff --git a/Dockerfile-poller b/Dockerfile-poller index 0d7491cf..36dc4e34 100644 --- a/Dockerfile-poller +++ b/Dockerfile-poller @@ -19,6 +19,7 @@ WORKDIR /usr/src/app COPY src/autoscaler-common/ src/autoscaler-common/ COPY src/poller/ src/poller/ COPY package*.json ./ +COPY autoscaler-config.schema.json ./ RUN npm config set update-notifier false RUN npm install --omit=dev diff --git a/Dockerfile-unified b/Dockerfile-unified index 34949e85..82825f1d 100644 --- a/Dockerfile-unified +++ b/Dockerfile-unified @@ -21,6 +21,7 @@ COPY src/scaler/scaler-core/ src/scaler/scaler-core/ COPY src/poller/poller-core/ src/poller/poller-core/ COPY src/unifiedScaler.js src/ COPY package*.json ./ +COPY autoscaler-config.schema.json ./ RUN npm config set update-notifier false RUN npm install --omit=dev diff --git a/autoscaler-config.schema.json b/autoscaler-config.schema.json new file mode 100644 index 00000000..12c7228f --- /dev/null +++ b/autoscaler-config.schema.json @@ -0,0 +1,245 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://github.com/cloudspannerecosystem/autoscaler/autoscaler-config.schema.json", + "title": "Cloud Spanner Autoscaler configuration", + "description": "JSON schema for the Cloud Spanner autoscaler configuration, specifying one or more Spanner instances to monitor and automatically scale", + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/$defs/spannerInstance" + }, + "$comment": "Any changes to this file also need to be reflected in src/poller/README.md, and in autoscaler-common/types.js.", + "$defs": { + "spannerInstance": { + "type": "object", + "title": "Spanner Instance", + "description": "Specification of a Cloud Spanner instance to be managed by the autoscaler.", + "additionalProperties": false, + "required": ["projectId", "instanceId"], + "properties": { + "$comment": { + "type": "string" + }, + "projectId": { + "type": "string", + "minLength": 2, + "description": "Project ID of the Cloud Spanner to be monitored." + }, + "instanceId": { + "type": "string", + "minLength": 2, + "description": "Instance ID of the Cloud Spanner to be monitored." + }, + "units": { + "enum": ["NODES", "PROCESSING_UNITS"], + "description": "Specifies the units how the spanner capacity will be measured.", + "default": "NODES" + }, + "minSize": { + "type": "number", + "minimum": 1, + "description": "Minimum number of Cloud Spanner `NODES` or `PROCESSING_UNITS` that the instance can be scaled IN to.", + "default": "1 NODE or 100 PROCESSING_UNITS" + }, + "maxSize": { + "type": "number", + "minimum": 1, + "description": "Maximum number of Cloud Spanner `NODES` or `PROCESSING_UNITS` that the instance can be scaled OUT to.", + "default": "3 NODES or 2000 PROCESSING_UNITS" + }, + "scalingMethod": { + "type": "string", + "minLength": 2, + "description": "Scaling method that should be used. See the [scaling methods](https://github.com/cloudspannerecosystem/autoscaler/blob/main/src/scaler/README.md#scaling-methods) for more information.", + "default": "STEPWISE" + }, + "stepSize": { + "type": "number", + "minimum": 1, + "description": "Amount of capacity that should be added or removed when scaling with the STEPWISE method.\nWhen the Spanner instance size is over 1000 `PROCESSING_UNITS`, scaling will be done in steps of 1000 `PROCESSING_UNITS`.\n For more information see the [Spanner compute capacity documentation](https://cloud.google.com/spanner/docs/compute-capacity#compute_capacity).", + "default": "2 NODES or 200 PROCESSING_UNITS" + }, + "overloadStepSize": { + "type": "number", + "minimum": 1, + "description": "Amount of capacity that should be added when the Cloud Spanner instance is overloaded, and the `STEPWISE` method is used.", + "default": "5 NODES or 500 PROCESSING_UNITS" + }, + "scaleInLimit": { + "type": "number", + "minimum": 1, + "maximum": 100, + "description": "Percentage (integer) of the total instance size that can be removed in a scale in event when using the `LINEAR` scaling method.\nFor example if set to `20`, only 20% of the instance size can be removed in a single scaling event. When `scaleInLimit` is not defined a limit is not enforced.", + "default": 100 + }, + "scaleOutCoolingMinutes": { + "type": "number", + "minimum": 1, + "description": "Minutes to wait after scaling IN or OUT before a scale OUT event can be processed.", + "default": 5 + }, + "scaleInCoolingMinutes": { + "type": "number", + "minimum": 1, + "description": "Minutes to wait after scaling IN or OUT before a scale IN event can be processed.", + "default": 30 + }, + "overloadCoolingMinutes": { + "type": "number", + "minimum": 1, + "description": "Minutes to wait after scaling IN or OUT before a scale OUT event can be processed, when the Spanner instance is overloaded.\nAn instance is overloaded if its High Priority CPU utilization is over 90%.", + "default": 5 + }, + "stateProjectId": { + "type": "string", + "minLength": 2, + "description": "The project ID where the Autoscaler state will be persisted.\nBy default it is persisted using Cloud Firestore in the same project as the Spanner instance being scaled - see `stateDatabase`.", + "default": "${projectId}" + }, + "stateDatabase": { + "type": "object", + "description": "Object defining the database for managing the state of the Autoscaler.", + "default": "firestore", + "additionalProperties": false, + "properties": { + "name": { + "enum": ["firestore", "spanner"], + "description": "Type of the database for storing the persistent state of the Autoscaler.", + "default": "firestore" + }, + "instanceId": { + "type": "string", + "minLength": 2, + "description": "The instance id of Cloud Spanner in which you want to persist the state. Required if name=spanner." + }, + "databaseId": { + "type": "string", + "minLength": 2, + "description": "The instance id of Cloud Spanner in which you want to persist the state. Required if name=spanner." + } + } + }, + "scalerPubSubTopic": { + "type": "string", + "minLength": 2, + "pattern": "^projects/[^/]+/topics/[^/]+$", + "description": "PubSub topic (in the form `projects/${projectId}/topics/scaler-topic`) for the Poller function to publish messages for the Scaler function (Required for Cloud Functions deployments)" + }, + "scalerURL": { + "type": "string", + "minLength": 2, + "pattern": "^https?://.+", + "description": "URL where the scaler service receives HTTP requests (Required for non-unified GKE deployments)", + "default": "http://scaler" + }, + "downstreamPubSubTopic": { + "type": "string", + "minLength": 2, + "pattern": "^projects/[^/]+/topics/[^/]+$", + "description": "Set this parameter to point to a pubsub topic (in the form `projects/${projectId}/topics/downstream-topic-name`) to make the Autoscaler publish events that can be consumed by downstream applications.\nSee [Downstream messaging](https://github.com/cloudspannerecosystem/autoscaler/blob/main/src/scaler/README.md#downstream-messaging) for more information." + }, + "metrics": { + "type": "array", + "description": "An array of custom metric definitions.\nThese can be provided in the configuration objects to customize the metrics used to autoscale your Cloud Spanner instances\n", + "items": { + "$ref": "#/$defs/metricDefinition" + } + } + } + }, + "metricDefinition": { + "title": "Custom Metric Definition", + "description": "To specify a custom threshold specify the name of the metrics to customize followed by the parameter values you wish to change.\nThe updated parameters will be merged with the default metric parameters.", + "type": "object", + "additionalProperties": false, + "required": ["name"], + "properties": { + "name": { + "type": "string", + "minLength": 2, + "description": "A unique name of the for the metric to be evaulated.\nIf you want to override the default metrics, their names are: `high_priority_cpu`, `rolling_24_hr` and `storage`." + }, + "filter": { + "type": "string", + "minLength": 2, + "description": "The Cloud Spanner metric and filter that should be used when querying for data.\nThe Autoscaler will automatically add the filter expressions for Spanner instance resources, instance id and project id." + }, + "reducer": { + "$comment": "from https://monitoring.googleapis.com/$discovery/rest?version=v3", + "enum": [ + "REDUCE_NONE", + "REDUCE_MEAN", + "REDUCE_MIN", + "REDUCE_MAX", + "REDUCE_SUM", + "REDUCE_STDDEV", + "REDUCE_COUNT", + "REDUCE_COUNT_TRUE", + "REDUCE_COUNT_FALSE", + "REDUCE_FRACTION_TRUE", + "REDUCE_PERCENTILE_99", + "REDUCE_PERCENTILE_95", + "REDUCE_PERCENTILE_50", + "REDUCE_PERCENTILE_05" + ], + "description": "The reducer specifies how the data points should be aggregated when querying for metrics, typically `REDUCE_SUM`.\nFor more details please refer to [Alert Policies - Reducer](https://cloud.google.com/monitoring/api/ref_v3/rest/v3/projects.alertPolicies#reducer) documentation.", + "default": "REDUCE_SUM" + }, + "aligner": { + "$comment": "Values from https://monitoring.googleapis.com/$discovery/rest?version=v3", + "enum": [ + "ALIGN_NONE", + "ALIGN_DELTA", + "ALIGN_RATE", + "ALIGN_INTERPOLATE", + "ALIGN_NEXT_OLDER", + "ALIGN_MIN", + "ALIGN_MAX", + "ALIGN_MEAN", + "ALIGN_COUNT", + "ALIGN_SUM", + "ALIGN_STDDEV", + "ALIGN_COUNT_TRUE", + "ALIGN_COUNT_FALSE", + "ALIGN_FRACTION_TRUE", + "ALIGN_PERCENTILE_99", + "ALIGN_PERCENTILE_95", + "ALIGN_PERCENTILE_50", + "ALIGN_PERCENTILE_05", + "ALIGN_PERCENT_CHANGE" + ], + "description": "The aligner specifies how the data points should be aligned in the time series, typically `ALIGN_MAX`.\nFor more details please refer to [Alert Policies - Aligner](https://cloud.google.com/monitoring/api/ref_v3/rest/v3/projects.alertPolicies#aligner) documentation.", + "default": "ALIGN_MAX" + }, + "period": { + "type": "number", + "minimum": 1, + "description": "Defines the period of time in units of seconds at which aggregation takes place. Typically the period should be 60.", + "default": 60 + }, + "regional_threshold": { + "type": "number", + "minimum": 1, + "description": "Threshold used to evaluate if a regional instance needs to be scaled in or out." + }, + "multi_regional_threshold": { + "type": "number", + "minimum": 1, + "description": "Threshold used to evaluate if a multi-regional instance needs to be scaled in or out." + }, + "regional_margin": { + "type": "number", + "minimum": 1, + "description": "Margin above and below the threshold where the metric value is allowed.\nIf the metric falls outside of the range `[threshold - margin, threshold + margin]`, then the regional instance needs to be scaled in or out.", + "default": 5 + }, + "multi_regional_margin": { + "type": "number", + "minimum": 1, + "description": "Margin above and below the threshold where the metric value is allowed.\nIf the metric falls outside of the range `[threshold - margin, threshold + margin]`, then the multi regional instance needs to be scaled in or out.", + "default": 5 + } + } + } + } +} diff --git a/package-lock.json b/package-lock.json index a213c34c..0cbd756b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -20,6 +20,7 @@ "@opentelemetry/sdk-metrics": "^1.25.1", "@opentelemetry/sdk-node": "^0.52.1", "@opentelemetry/semantic-conventions": "^1.25.1", + "ajv": "^8.17.1", "axios": "^1.7.2", "eventid": "^2.0.1", "express": "^4.19.2", diff --git a/package.json b/package.json index 244e19c6..00a38b99 100644 --- a/package.json +++ b/package.json @@ -28,7 +28,8 @@ "test-e2e": "pushd terraform/cloud-functions/per-project/test && go test -run . -timeout 60m --tags=e2e && popd", "typecheck": "tsc --project jsconfig.json --maxNodeModuleJsDepth 0 --noEmit", "unified-job": "node -e \"require('./src/unifiedScaler').main()\"", - "update-all": "npm update -S" + "update-all": "npm update -S", + "validate-config-file": "node -e \"require('./src/poller/poller-core/config-validator').main()\" -- " }, "dependencies": { "@google-cloud/firestore": "^7.9.0", @@ -42,6 +43,7 @@ "@opentelemetry/sdk-metrics": "^1.25.1", "@opentelemetry/sdk-node": "^0.52.1", "@opentelemetry/semantic-conventions": "^1.25.1", + "ajv": "^8.17.1", "axios": "^1.7.2", "eventid": "^2.0.1", "express": "^4.19.2", diff --git a/src/poller/README.md b/src/poller/README.md index d87d0347..b9a3da73 100644 --- a/src/poller/README.md +++ b/src/poller/README.md @@ -69,6 +69,14 @@ configuration parameters are defined in YAML in a [Kubernetes ConfigMap][configm See the [configuration section][autoscaler-home-config] in the home page for instructions on how to change the payload. +The Autoscaler JSON (for Cloud functions) or YAML (for GKE) configuration can be +validated by running the command: + +```shell +npm install +npm run validate-config-file -- path/to/config_file +``` + ### Required | Key | Description | diff --git a/src/poller/poller-core/config-validator.js b/src/poller/poller-core/config-validator.js new file mode 100644 index 00000000..66432b05 --- /dev/null +++ b/src/poller/poller-core/config-validator.js @@ -0,0 +1,241 @@ +/* Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License + */ + +/** + * @fileoverview Validates a given configuration against the JSON schema + */ + +const Ajv = require('ajv').default; +const fs = require('fs/promises'); +const yaml = require('js-yaml'); + +/** + * @typedef {import('../../autoscaler-common/types').AutoscalerSpanner + * } AutoscalerSpanner + * + * @typedef {import('ajv').ValidateFunction} ValidateFunction + */ + +/** + * Error thrown when validation fails. + */ +class ValidationError extends Error { + /** + * @param {string} errors Human readable string with all errors listed. + */ + constructor(errors) { + super(errors); + } +} + +/** + * Encapsulates the Ajv validator initialzation and checks. + */ +class ConfigValidator { + /** Creates the class launches async initialization. */ + constructor() { + /** @type {ValidateFunction} */ + this.ajvConfigValidator; + + /** @type {Ajv} */ + this.ajv; + + this.pendingInit = this.initAsync(); + } + + /** + * Performs asynchronous initialization. + * + * @return {Promise} + */ + async initAsync() { + const tmpAjv = new Ajv({allErrors: true}); + + const configSchema = await fs.readFile( + 'autoscaler-config.schema.json', + 'utf-8', + ); + + const schema = JSON.parse(configSchema); + this.ajvConfigValidator = tmpAjv.compile(schema); + this.ajv = tmpAjv; + } + + /** + * Validates the given object against the Spanner Autoscaler Config schema. + * + * @param {AutoscalerSpanner[]} clusters + */ + async assertValidConfig(clusters) { + await this.pendingInit; + const valid = this.ajvConfigValidator(clusters); + if (!valid) { + throw new ValidationError( + 'Invalid Autoscaler Configuration parameters:\n' + + this.ajv.errorsText(this.ajvConfigValidator.errors, { + separator: '\n', + dataVar: 'SpannerConfig', + }), + ); + } + } + + /** + * Parses the given string as JSON and validate it against the SannerConfig + * schema. Throws an Error if the config is not valid. + * + * @param {string} jsonString + * @return {Promise} + */ + async parseAndAssertValidConfig(jsonString) { + let configJson; + try { + configJson = JSON.parse(jsonString); + } catch (e) { + throw new Error(`Invalid JSON in Autoscaler configuration: ${e}`); + } + await this.assertValidConfig(configJson); + return configJson; + } +} + +/** + * Validates the specified Spanner Autoscaler JSON configuration file. + * Throws an Error and reports to console if the config is not valid. + * + * @param {ConfigValidator} configValidator + * @param {string} filename + */ +async function assertValidJsonFile(configValidator, filename) { + try { + const configText = await fs.readFile(filename, 'utf-8'); + await configValidator.parseAndAssertValidConfig(configText); + } catch (e) { + if (e instanceof ValidationError) { + console.error( + `Validation of config in file ${filename} failed:\n${e.message}`, + ); + } else { + console.error(`Processing of config in file ${filename} failed: ${e}`); + } + throw new Error(`${filename} Failed validation`); + } +} + +/** + * Validates all the Spanner Autoscaler YAML config files specified in the + * GKE configMap. Throws an Error and reports + * to console if any of the configmaps do not pass validation. + * + * @param {ConfigValidator} configValidator + * @param {string} filename + */ +async function assertValidGkeConfigMapFile(configValidator, filename) { + let configMap; + + try { + const configText = await fs.readFile(filename, 'utf-8'); + configMap = /** @type {any} */ (yaml.load(configText)); + } catch (e) { + console.error(`Could not parse YAML from ${filename}: ${e}`); + throw e; + } + + if (configMap.kind !== 'ConfigMap') { + console.error(`${filename} is not a GKE ConfigMap`); + throw new Error(`${filename} is not a GKE ConfigMap`); + } + + let success = true; + for (const configMapFile of Object.keys(configMap.data)) { + const configMapData = configMap.data[configMapFile]; + try { + const spannerConfig = yaml.load(configMapData); + await configValidator.parseAndAssertValidConfig( + JSON.stringify(spannerConfig), + ); + } catch (e) { + if (e instanceof ValidationError) { + console.error( + `Validation of configMap entry data.${configMapFile} in file ${filename} failed:\n${e.message}`, + ); + } else if (e instanceof yaml.YAMLException) { + console.error( + `Could not parse YAML from value data.${configMapFile} in ${filename}: ${e}`, + ); + } else { + console.error( + `Processing of configMap entry data.${configMapFile} in file ${filename} failed: ${e}`, + ); + } + success = false; + } + } + if (!success) { + throw new Error(`${filename} Failed validation`); + } +} + +/** + * Validates a configuration file passed in on the command line. + */ +function main() { + if ( + process.argv.length <= 1 || + process.argv[1] === '-h' || + process.argv[1] === '--help' + ) { + console.log('Usage: validate-config-file CONFIG_FILE_NAME'); + console.log( + 'Validates that the specified Autoscaler JSON config is defined correctly', + ); + process.exit(1); + } + + const configValidator = new ConfigValidator(); + + if (process.argv[1].toLowerCase().endsWith('.yaml')) { + assertValidGkeConfigMapFile(configValidator, process.argv[1]).then( + () => process.exit(0), + (e) => { + console.error(e); + process.exit(1); + }, + ); + } else if (process.argv[1].toLowerCase().endsWith('.json')) { + assertValidJsonFile(configValidator, process.argv[1]).then( + () => process.exit(0), + (e) => { + console.error(e); + process.exit(1); + }, + ); + } else { + console.log( + `filename ${process.argv[1]} must either be JSON (.json) or a YAML configmap (.yaml) file`, + ); + process.exit(1); + } +} + +module.exports = { + ConfigValidator, + ValidationError, + main, + TEST_ONLY: { + assertValidGkeConfigMapFile, + assertValidJsonFile, + }, +}; diff --git a/src/poller/poller-core/index.js b/src/poller/poller-core/index.js index 2b871111..dd90c7da 100644 --- a/src/poller/poller-core/index.js +++ b/src/poller/poller-core/index.js @@ -31,6 +31,7 @@ const Counters = require('./counters.js'); const {AutoscalerUnits} = require('../../autoscaler-common/types'); const assertDefined = require('../../autoscaler-common/assertDefined'); const {version: packageVersion} = require('../../../package.json'); +const {ConfigValidator} = require('./config-validator'); /** * @typedef {import('../../autoscaler-common/types').AutoscalerSpanner @@ -48,6 +49,9 @@ const {version: packageVersion} = require('../../../package.json'); // GCP service clients const metricsClient = new monitoring.MetricServiceClient(); const pubSub = new PubSub(); + +const configValidator = new ConfigValidator(); + const baseDefaults = { units: AutoscalerUnits.NODES, scaleOutCoolingMinutes: 5, @@ -397,8 +401,8 @@ async function callScalerHTTP(spanner, metrics) { * @return {Promise} enriched payload */ async function parseAndEnrichPayload(payload) { + const spanners = await configValidator.parseAndAssertValidConfig(payload); /** @type {AutoscalerSpanner[]} */ - const spanners = JSON.parse(payload); const spannersFound = []; for (let sIdx = 0; sIdx < spanners.length; sIdx++) { @@ -410,70 +414,13 @@ async function parseAndEnrichPayload(payload) { // merge in the defaults spanners[sIdx] = {...baseDefaults, ...spanners[sIdx]}; - // handle processing units and deprecation of minNodes/maxNodes - if (spanners[sIdx].units.toUpperCase() == 'PROCESSING_UNITS') { - spanners[sIdx].units = spanners[sIdx].units.toUpperCase(); + spanners[sIdx].units = spanners[sIdx].units?.toUpperCase(); + // handle processing units/nodes defaults + if (spanners[sIdx].units == 'PROCESSING_UNITS') { // merge in the processing units defaults spanners[sIdx] = {...processingUnitsDefaults, ...spanners[sIdx]}; - - // minNodes and maxNodes should not be used with processing units. If - // they are set the config is invalid. - if (spanners[sIdx].minNodes || spanners[sIdx].maxNodes) { - throw new Error( - 'INVALID CONFIG: units is set to PROCESSING_UNITS, ' + - 'however, minNodes or maxNodes is set, ' + - 'remove minNodes and maxNodes from your configuration.', - ); - } - } else if (spanners[sIdx].units.toUpperCase() == 'NODES') { - spanners[sIdx].units = spanners[sIdx].units.toUpperCase(); - - // if minNodes or minSize are provided set the other, and if both are set - // and not match throw an error - if (spanners[sIdx].minNodes && !spanners[sIdx].minSize) { - logger.warn({ - message: `DEPRECATION: minNodes is deprecated, ' + - 'remove minNodes from your config and instead use: ' + - 'units = 'NODES' and minSize = ${spanners[sIdx].minNodes}`, - projectId: spanners[sIdx].projectId, - instanceId: spanners[sIdx].instanceId, - }); - spanners[sIdx].minSize = assertDefined(spanners[sIdx].minNodes); - } else if ( - spanners[sIdx].minSize && - spanners[sIdx].minNodes && - spanners[sIdx].minSize != spanners[sIdx].minNodes - ) { - throw new Error( - 'INVALID CONFIG: minSize and minNodes are both set ' + - 'but do not match, make them match or only set minSize', - ); - } - - // if maxNodes or maxSize are provided set the other, and if both are set - // and not match throw an error - if (spanners[sIdx].maxNodes && !spanners[sIdx].maxSize) { - logger.warn({ - message: `DEPRECATION: maxNodes is deprecated, remove maxSize ' + - 'from your config and instead use: ' + - 'units = 'NODES' and maxSize = ${spanners[sIdx].maxNodes}`, - projectId: spanners[sIdx].projectId, - instanceId: spanners[sIdx].instanceId, - }); - spanners[sIdx].maxSize = assertDefined(spanners[sIdx].maxNodes); - } else if ( - spanners[sIdx].maxSize && - spanners[sIdx].maxNodes && - spanners[sIdx].maxSize != spanners[sIdx].maxNodes - ) { - throw new Error( - 'INVALID CONFIG: maxSize and maxNodes are both set ' + - 'but do not match, make them match or only set maxSize', - ); - } - - // at this point both minNodes/minSize and maxNodes/maxSize are matching - // or are both not set so we can merge in defaults + } else if (spanners[sIdx].units == 'NODES') { + // merge in the nodes defaults spanners[sIdx] = {...nodesDefaults, ...spanners[sIdx]}; } else { throw new Error( diff --git a/src/poller/poller-core/test/config-validator.test.js b/src/poller/poller-core/test/config-validator.test.js new file mode 100644 index 00000000..dd790509 --- /dev/null +++ b/src/poller/poller-core/test/config-validator.test.js @@ -0,0 +1,182 @@ +/* Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License + */ + +/* + * ESLINT: Ignore max line length errors on lines starting with 'it(' + * (test descriptions) + */ +/* eslint max-len: ["error", { "ignorePattern": "^\\s*it\\(" }] */ + +// eslint-disable-next-line no-unused-vars +const should = require('should'); +const { + ConfigValidator, + ValidationError, + TEST_ONLY, +} = require('../config-validator'); +const fs = require('node:fs'); +const path = require('node:path'); + +const configValidator = new ConfigValidator(); + +/** + * @typedef {import('../../../autoscaler-common/types').AutoscalerSpanner + * } AutoscalerSpanner + */ + +describe('configValidator', () => { + describe('#parseAndAssertValidConfig', () => { + it('fails when given an empty config', async () => { + await configValidator + .parseAndAssertValidConfig('') + .should.be.rejectedWith( + new Error( + 'Invalid JSON in Autoscaler configuration:' + + ' SyntaxError: Unexpected end of JSON input', + ), + ); + }); + it('fails when not given an array', async () => { + await configValidator + .parseAndAssertValidConfig('{}') + .should.be.rejectedWith( + new ValidationError( + 'Invalid Autoscaler Configuration parameters:\n' + + 'SpannerConfig must be array', + ), + ); + }); + + it('fails when given an empty array', async () => { + await configValidator + .parseAndAssertValidConfig('[]') + .should.be.rejectedWith( + new ValidationError( + 'Invalid Autoscaler Configuration parameters:\n' + + 'SpannerConfig must NOT have fewer than 1 items', + ), + ); + }); + it('fails when config does not contain required params', async () => { + await configValidator + .parseAndAssertValidConfig('[{}]') + .should.be.rejectedWith( + new ValidationError( + 'Invalid Autoscaler Configuration parameters:\n' + + "SpannerConfig/0 must have required property 'projectId'\n" + + "SpannerConfig/0 must have required property 'instanceId'", + ), + ); + }); + it('fails with an invalid property ', async () => { + await configValidator + .parseAndAssertValidConfig( + `[{ + "projectId": "my-project", + "instanceId": "my-instance", + "invalidProp": "nothing" + }]`, + ) + .should.be.rejectedWith( + new ValidationError( + 'Invalid Autoscaler Configuration parameters:\n' + + 'SpannerConfig/0 must NOT have additional properties', + ), + ); + }); + it('fails when a property is not valid', async () => { + await configValidator + .parseAndAssertValidConfig( + `[{ + "projectId": "my-project", + "instanceId": "my-instance", + "minSize": "1" + }]`, + ) + .should.be.rejectedWith( + new ValidationError( + 'Invalid Autoscaler Configuration parameters:\n' + + 'SpannerConfig/0/minSize must be number', + ), + ); + }); + it('passes with valid config', async () => { + const config = [ + { + '$comment': 'Sample autoscaler config', + 'projectId': 'my-project', + 'instanceId': 'my-instance', + 'scalerPubSubTopic': 'projects/my-project/topics/scaler-topic', + 'units': 'NODES', + 'minSize': 1, + 'maxSize': 3, + }, + ]; + const parsedConfig = await configValidator.parseAndAssertValidConfig( + JSON.stringify(config), + ); + parsedConfig.should.deepEqual(config); + }); + }); + describe('#validateTestFiles', async () => { + const dir = 'src/poller/poller-core/test/resources'; + const files = fs.readdirSync(dir, { + withFileTypes: true, + }); + + const yamlFiles = files.filter((f) => f.name.endsWith('.yaml')); + const goodYamlFiles = yamlFiles.filter((f) => f.name.startsWith('good-')); + const badYamlFiles = yamlFiles.filter((f) => f.name.startsWith('bad-')); + const jsonFiles = files.filter((f) => f.name.endsWith('.json')); + const goodJsonFiles = jsonFiles.filter((f) => f.name.startsWith('good-')); + const badJsonFiles = jsonFiles.filter((f) => f.name.startsWith('bad-')); + + goodYamlFiles.forEach((file) => { + it(`validates file ${file.name} successfully`, async () => { + await TEST_ONLY.assertValidGkeConfigMapFile( + configValidator, + path.join(dir, file.name), + ).should.be.resolved(); + }); + }); + + badYamlFiles.forEach((file) => { + it(`invalid file ${file.name} correctly fails validation`, async () => { + await TEST_ONLY.assertValidGkeConfigMapFile( + configValidator, + path.join(dir, file.name), + ).should.be.rejected(); + }); + }); + + goodJsonFiles.forEach((file) => { + it(`validates file ${file.name} successfully`, async () => { + await TEST_ONLY.assertValidJsonFile( + configValidator, + path.join(dir, file.name), + ).should.be.resolved(); + }); + }); + + badJsonFiles.forEach((file) => { + it(`invalid file ${file.name} correctly fails validation`, async () => { + await TEST_ONLY.assertValidJsonFile( + configValidator, + path.join(dir, file.name), + ).should.be.rejected(); + }); + }); + }); +}); diff --git a/src/poller/poller-core/test/index.test.js b/src/poller/poller-core/test/index.test.js index ba63598d..5ec78799 100644 --- a/src/poller/poller-core/test/index.test.js +++ b/src/poller/poller-core/test/index.test.js @@ -23,6 +23,7 @@ const rewire = require('rewire'); // eslint-disable-next-line no-unused-vars const should = require('should'); const sinon = require('sinon'); +const {ValidationError} = require('../config-validator'); const app = rewire('../index.js'); @@ -100,13 +101,14 @@ describe('#validateCustomMetric', () => { describe('#parseAndEnrichPayload', () => { it('should return the default for stepSize', async () => { - const payload = - '[{' + - ' "projectId": "my-spanner-project", ' + - ' "instanceId": "spanner1", ' + - ' "scalerPubSubTopic": "spanner-scaling", ' + - ' "minNodes": 10' + - '}]'; + const payload = JSON.stringify([ + { + projectId: 'my-spanner-project', + instanceId: 'spanner1', + scalerPubSubTopic: 'projects/my-project/topics/spanner-scaling', + minSize: 10, + }, + ]); const stub = sinon.stub().resolves({currentNode: 5, regional: true}); const unset = app.__set__('getSpannerMetadata', stub); @@ -117,34 +119,16 @@ describe('#parseAndEnrichPayload', () => { unset(); }); - it('should override the default for minNodes', async () => { - const payload = - '[{' + - ' "projectId": "my-spanner-project",' + - ' "instanceId": "spanner1",' + - ' "scalerPubSubTopic": "spanner-scaling",' + - ' "minNodes": 10 ' + - '}]'; - - const stub = sinon.stub().resolves({currentNode: 5, regional: true}); - const unset = app.__set__('getSpannerMetadata', stub); - - const mergedConfig = await parseAndEnrichPayload(payload); - mergedConfig[0].units.should.equal('NODES'); - should(mergedConfig[0].minSize).equal(10); - - unset(); - }); - it('should merge in defaults for processing units', async () => { - const payload = - '[{' + - ' "projectId": "my-spanner-project", ' + - ' "instanceId": "spanner1", ' + - ' "scalerPubSubTopic": "spanner-scaling", ' + - ' "units": "PROCESSING_UNITS", ' + - ' "minSize": 200' + - '}]'; + const payload = JSON.stringify([ + { + projectId: 'my-spanner-project', + instanceId: 'spanner1', + scalerPubSubTopic: 'projects/my-project/topics/spanner-scaling', + units: 'PROCESSING_UNITS', + minSize: 200, + }, + ]); const stub = sinon.stub().resolves({currentSize: 500, regional: true}); const unset = app.__set__('getSpannerMetadata', stub); @@ -160,15 +144,16 @@ describe('#parseAndEnrichPayload', () => { }); it('should use the value of minSize/maxSize for minNodes/maxNodes instead of overriding with the defaults, Github Issue 61', async () => { - const payload = - '[{' + - ' "projectId": "my-spanner-project", ' + - ' "instanceId": "spanner1", ' + - ' "scalerPubSubTopic": "spanner-scaling", ' + - ' "units": "NODES", ' + - ' "minSize": 20, ' + - ' "maxSize": 50 ' + - '}]'; + const payload = JSON.stringify([ + { + projectId: 'my-spanner-project', + instanceId: 'spanner1', + scalerPubSubTopic: 'projects/my-project/topics/spanner-scaling', + units: 'NODES', + minSize: 20, + maxSize: 50, + }, + ]); const stub = sinon.stub().resolves({currentSize: 50, regional: true}); const unset = app.__set__('getSpannerMetadata', stub); @@ -180,87 +165,22 @@ describe('#parseAndEnrichPayload', () => { unset(); }); - it('should throw if the nodes are specified when units is set to processing units', async () => { - const payload = - '[{' + - ' "projectId": "my-spanner-project", ' + - ' "instanceId": "spanner1", ' + - ' "scalerPubSubTopic": "spanner-scaling", ' + - ' "units": "PROCESSING_UNITS", ' + - ' "minNodes": 200' + - '}]'; - - const stub = sinon.stub().resolves({currentSize: 500, regional: true}); - const unset = app.__set__('getSpannerMetadata', stub); - - await parseAndEnrichPayload(payload).should.be.rejectedWith( - new Error( - 'INVALID CONFIG: units is set to PROCESSING_UNITS, ' + - 'however, minNodes or maxNodes is set, ' + - 'remove minNodes and maxNodes from your configuration.', - ), - ); - - unset(); - }); - - it('should throw if the nodes are specified when but minSize and minNodes are both provided but not matching', async () => { - const payload = - '[{' + - ' "projectId": "my-spanner-project", ' + - ' "instanceId": "spanner1", ' + - ' "scalerPubSubTopic": "spanner-scaling", ' + - ' "units": "NODES", ' + - ' "minNodes": 20, ' + - ' "minSize": 5' + - '}]'; - - const stub = sinon.stub().resolves({currentSize: 50, regional: true}); - const unset = app.__set__('getSpannerMetadata', stub); - - await parseAndEnrichPayload(payload).should.be.rejectedWith( - new Error( - 'INVALID CONFIG: minSize and minNodes are both set ' + - 'but do not match, make them match or only set minSize', - ), - ); - - unset(); - }); - - it('should throw if the nodes are specified when but maxSize and maxNodes are both provided but not matching', async () => { - const payload = - '[{' + - ' "projectId": "my-spanner-project", ' + - ' "instanceId": "spanner1", ' + - ' "scalerPubSubTopic": "spanner-scaling", ' + - ' "units": "NODES", ' + - ' "maxNodes": 20, ' + - ' "maxSize": 5' + - '}]'; - - const stub = sinon.stub().resolves({currentSize: 50, regional: true}); - const unset = app.__set__('getSpannerMetadata', stub); - - await parseAndEnrichPayload(payload).should.be.rejectedWith( - new Error( - 'INVALID CONFIG: maxSize and maxNodes are both set ' + - 'but do not match, make them match or only set maxSize', - ), - ); - - unset(); - }); - it('should override the regional threshold for storage but not high_priority_cpu', async () => { - const payload = - '[{' + - ' "projectId": "my-spanner-project", ' + - ' "instanceId": "spanner1", ' + - ' "scalerPubSubTopic": "spanner-scaling", ' + - ' "minNodes": 10, ' + - ' "metrics": [{"name": "storage", "regional_threshold":10}]' + - '}]'; + const payload = JSON.stringify([ + { + projectId: 'my-spanner-project', + instanceId: 'spanner1', + scalerPubSubTopic: 'projects/my-project/topics/spanner-scaling', + minSize: 10, + metrics: [ + { + name: 'storage', + regional_threshold: 10, + multi_regional_threshold: 10, + }, + ], + }, + ]); const stub = sinon.stub().resolves({currentNode: 5, regional: true}); const unset = app.__set__('getSpannerMetadata', stub); @@ -271,6 +191,7 @@ describe('#parseAndEnrichPayload', () => { let metric = /** @type {SpannerMetric} */ (mergedConfig[0].metrics[idx]); metric.regional_threshold.should.equal(10); + metric.multi_regional_threshold.should.equal(10); idx = mergedConfig[0].metrics.findIndex( (x) => x.name === 'high_priority_cpu', ); @@ -281,17 +202,26 @@ describe('#parseAndEnrichPayload', () => { }); it('should override the multiple thresholds', async () => { - const payload = - '[{' + - ' "projectId": "my-spanner-project", ' + - ' "instanceId": "spanner1", ' + - ' "scalerPubSubTopic": "spanner-scaling", ' + - ' "minNodes": 10, ' + - ' "metrics": [' + - ' {"name": "high_priority_cpu", "multi_regional_threshold":20}, ' + - ' {"name": "storage", "regional_threshold":10}' + - ' ]' + - '}]'; + const payload = JSON.stringify([ + { + projectId: 'my-spanner-project', + instanceId: 'spanner1', + scalerPubSubTopic: 'projects/my-project/topics/spanner-scaling', + minSize: 10, + metrics: [ + { + name: 'high_priority_cpu', + regional_threshold: 20, + multi_regional_threshold: 20, + }, + { + name: 'storage', + regional_threshold: 10, + multi_regional_threshold: 10, + }, + ], + }, + ]); const stub = sinon.stub().resolves({currentNode: 5, regional: true}); const unset = app.__set__('getSpannerMetadata', stub); @@ -311,20 +241,22 @@ describe('#parseAndEnrichPayload', () => { }); it('should add a custom metric to the list if metric name is a default metric', async () => { - const payload = - '[{' + - ' "projectId": "my-spanner-project", ' + - ' "instanceId": "spanner1", ' + - ' "scalerPubSubTopic": "spanner-scaling", ' + - ' "minNodes": 10, ' + - ' "metrics": [' + - ' {' + - ' "filter": "my super cool filter", ' + - ' "name": "bogus", ' + - ' "multi_regional_threshold":20' + - ' }' + - ' ]' + - '}]'; + const payload = JSON.stringify([ + { + projectId: 'my-spanner-project', + instanceId: 'spanner1', + scalerPubSubTopic: 'projects/my-project/topics/spanner-scaling', + minSize: 10, + metrics: [ + { + filter: 'my super cool filter', + name: 'bogus', + multi_regional_threshold: 20, + regional_threshold: 20, + }, + ], + }, + ]); const stub = sinon.stub().resolves({currentNode: 5, regional: true}); const unset = app.__set__('getSpannerMetadata', stub); @@ -337,16 +269,21 @@ describe('#parseAndEnrichPayload', () => { }); it('should not add a custom metric to the list if the provided metric is not valid', async () => { - const payload = - '[{' + - ' "projectId": "my-spanner-project",' + - ' "instanceId": "spanner1",' + - ' "scalerPubSubTopic": "spanner-scaling", ' + - ' "minNodes": 10, ' + - ' "metrics": [' + - ' {"filter": "my super cool filter", "name": "bogus"}' + - ' ]' + - '}]'; + const payload = JSON.stringify([ + { + projectId: 'my-spanner-project', + instanceId: 'spanner1', + scalerPubSubTopic: 'projects/my-project/topics/spanner-scaling', + minSize: 10, + metrics: [ + { + name: 'bogus', + regional_threshold: 10, + multi_regional_threshold: 20, + }, + ], + }, + ]); const stub = sinon.stub().resolves({currentNode: 5, regional: true}); const unset = app.__set__('getSpannerMetadata', stub); @@ -358,22 +295,69 @@ describe('#parseAndEnrichPayload', () => { }); it('should throw if the nodes are specified if units is set something other than nodes or processing units', async () => { - const payload = - '[{' + - ' "projectId": "my-spanner-project",' + - ' "instanceId": "spanner1",' + - ' "scalerPubSubTopic": "spanner-scaling", ' + - ' "units": "BOGUS", ' + - ' "minNodes": 200' + - '}]'; + const payload = JSON.stringify([ + { + projectId: 'my-spanner-project', + instanceId: 'spanner1', + scalerPubSubTopic: 'projects/my-project/topics/spanner-scaling', + units: 'BOGUS', + minSize: 200, + }, + ]); + + const stub = sinon.stub().resolves({currentSize: 500, regional: true}); + const unset = app.__set__('getSpannerMetadata', stub); + + await parseAndEnrichPayload(payload).should.be.rejectedWith( + new ValidationError( + 'Invalid Autoscaler Configuration parameters:\n' + + 'SpannerConfig/0/units must be equal to one of the allowed values', + ), + ); + + unset(); + }); + + it('should throw if the sizes are specified as strings', async () => { + const payload = JSON.stringify([ + { + projectId: 'my-spanner-project', + instanceId: 'spanner1', + scalerPubSubTopic: 'projects/my-project/topics/spanner-scaling', + units: 'NODES', + minSize: '300', + }, + ]); + + const stub = sinon.stub().resolves({currentSize: 500, regional: true}); + const unset = app.__set__('getSpannerMetadata', stub); + + await parseAndEnrichPayload(payload).should.be.rejectedWith( + new ValidationError( + 'Invalid Autoscaler Configuration parameters:\n' + + 'SpannerConfig/0/minSize must be number', + ), + ); + + unset(); + }); + + it('should throw if the config is not an array', async () => { + const payload = JSON.stringify({ + projectId: 'my-spanner-project', + instanceId: 'spanner1', + scalerPubSubTopic: 'projects/my-project/topics/spanner-scaling', + units: 'NODES', + minSize: '300', + }); const stub = sinon.stub().resolves({currentSize: 500, regional: true}); const unset = app.__set__('getSpannerMetadata', stub); await parseAndEnrichPayload(payload).should.be.rejectedWith( - new Error( - 'INVALID CONFIG: BOGUS is invalid. ' + - 'Valid values are NODES or PROCESSING_UNITS', + new ValidationError( + 'Invalid Autoscaler Configuration parameters:\n' + + 'SpannerConfig must be array', ), ); diff --git a/src/poller/poller-core/test/resources/bad-data-contents.yaml b/src/poller/poller-core/test/resources/bad-data-contents.yaml new file mode 100644 index 00000000..4da386c5 --- /dev/null +++ b/src/poller/poller-core/test/resources/bad-data-contents.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: dsdssfdfdsa +metadata: + name: autoscaler-config + namespace: spanner-autoscaler +data: + autoscaler-config.yaml: | + hello world diff --git a/src/poller/poller-core/test/resources/bad-empty-array.json b/src/poller/poller-core/test/resources/bad-empty-array.json new file mode 100644 index 00000000..fe51488c --- /dev/null +++ b/src/poller/poller-core/test/resources/bad-empty-array.json @@ -0,0 +1 @@ +[] diff --git a/src/poller/poller-core/test/resources/bad-empty.json b/src/poller/poller-core/test/resources/bad-empty.json new file mode 100644 index 00000000..e69de29b diff --git a/src/poller/poller-core/test/resources/bad-empty.yaml b/src/poller/poller-core/test/resources/bad-empty.yaml new file mode 100644 index 00000000..bd9bfb6d --- /dev/null +++ b/src/poller/poller-core/test/resources/bad-empty.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: dsdssfdfdsa +metadata: + name: autoscaler-config + namespace: spanner-autoscaler +data: + autoscaler-config.yaml: "" diff --git a/src/poller/poller-core/test/resources/bad-invalid-props.json b/src/poller/poller-core/test/resources/bad-invalid-props.json new file mode 100644 index 00000000..89a0048a --- /dev/null +++ b/src/poller/poller-core/test/resources/bad-invalid-props.json @@ -0,0 +1,12 @@ +[ + { + "projectId": "basic-configuration", + "instanceId": "another-spanner1", + "scalerPubSubTopic": "projects/my-spanner-project/topics/spanner-scaling", + "units": "NODES", + "minSize": 5, + "maxSize": 30, + "scalingMethod": "DIRECT", + "garbage": "value" + } +] diff --git a/src/poller/poller-core/test/resources/bad-invalid-props.yaml b/src/poller/poller-core/test/resources/bad-invalid-props.yaml new file mode 100644 index 00000000..b36853ae --- /dev/null +++ b/src/poller/poller-core/test/resources/bad-invalid-props.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: autoscaler-config + namespace: spanner-autoscaler +data: + autoscaler-config.yaml: | + --- + - projectId: spanner-autoscaler-test + instanceId: spanner-scaling-direct + units: NODES + minSize: 5 + maxSize: 30 + scalingMethod: DIRECT + garbage:value diff --git a/src/poller/poller-core/test/resources/bad-invalid-value.json b/src/poller/poller-core/test/resources/bad-invalid-value.json new file mode 100644 index 00000000..fd04dfad --- /dev/null +++ b/src/poller/poller-core/test/resources/bad-invalid-value.json @@ -0,0 +1,11 @@ +[ + { + "projectId": "basic-configuration", + "instanceId": "another-spanner1", + "scalerPubSubTopic": "projects/my-spanner-project/topics/spanner-scaling", + "units": "NODES", + "minSize": 5, + "maxSize": "30", + "scalingMethod": "DIRECT" + } +] diff --git a/src/poller/poller-core/test/resources/bad-invalid-value.yaml b/src/poller/poller-core/test/resources/bad-invalid-value.yaml new file mode 100644 index 00000000..81b8aa53 --- /dev/null +++ b/src/poller/poller-core/test/resources/bad-invalid-value.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: autoscaler-config + namespace: spanner-autoscaler +data: + autoscaler-config.yaml: | + --- + - projectId: spanner-autoscaler-test + instanceId: spanner-scaling-direct + units: NODES + minSize: 5 + maxSize: rubbish + scalingMethod: DIRECT diff --git a/src/poller/poller-core/test/resources/bad-missing-props.json b/src/poller/poller-core/test/resources/bad-missing-props.json new file mode 100644 index 00000000..93d51406 --- /dev/null +++ b/src/poller/poller-core/test/resources/bad-missing-props.json @@ -0,0 +1 @@ +[{}] diff --git a/src/poller/poller-core/test/resources/bad-missing-props.yaml b/src/poller/poller-core/test/resources/bad-missing-props.yaml new file mode 100644 index 00000000..c0e64338 --- /dev/null +++ b/src/poller/poller-core/test/resources/bad-missing-props.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: autoscaler-config + namespace: spanner-autoscaler +data: + autoscaler-config.yaml: | + --- + - projectId: spanner-autoscaler-test diff --git a/src/poller/poller-core/test/resources/bad-not-configmap.yaml b/src/poller/poller-core/test/resources/bad-not-configmap.yaml new file mode 100644 index 00000000..563eb5e4 --- /dev/null +++ b/src/poller/poller-core/test/resources/bad-not-configmap.yaml @@ -0,0 +1,35 @@ +apiVersion: v1 +kind: dsdssfdfdsa +metadata: + name: autoscaler-config + namespace: spanner-autoscaler +data: + autoscaler-config.yaml: | + --- + - projectId: spanner-autoscaler-test + instanceId: spanner-scaling-direct + units: NODES + minSize: 5 + maxSize: 30 + scalingMethod: DIRECT + - projectId: spanner-autoscaler-test + instanceId: spanner-scaling-threshold + units: PROCESSING_UNITS + minSize: 100 + maxSize: 3000 + metrics: + - name: high_priority_cpu + regional_threshold: 40 + regional_margin: 3 + - projectId: spanner-autoscaler-test + instanceId: spanner-scaling-custom + units: NODES + minSize: 5 + maxSize: 30 + scalingMethod: LINEAR + scaleInLimit: 25 + metrics: + - name: my_custom_metric + filter: metric.type="spanner.googleapis.com/instance/resource/metric" + regional_threshold: 40 + multi_regional_threshold: 30 diff --git a/src/poller/poller-core/test/resources/bad-not-yaml.yaml b/src/poller/poller-core/test/resources/bad-not-yaml.yaml new file mode 100644 index 00000000..78f6cd18 --- /dev/null +++ b/src/poller/poller-core/test/resources/bad-not-yaml.yaml @@ -0,0 +1 @@ +some garbage... diff --git a/src/poller/poller-core/test/resources/good-multi-config.yaml b/src/poller/poller-core/test/resources/good-multi-config.yaml new file mode 100644 index 00000000..e903ab3d --- /dev/null +++ b/src/poller/poller-core/test/resources/good-multi-config.yaml @@ -0,0 +1,37 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: autoscaler-config + namespace: spanner-autoscaler +data: + autoscaler-config.yaml: | + --- + - projectId: spanner-autoscaler-test + instanceId: spanner-scaling-threshold + units: PROCESSING_UNITS + minSize: 100 + maxSize: 3000 + metrics: + - name: high_priority_cpu + regional_threshold: 40 + regional_margin: 3 + - projectId: spanner-autoscaler-test + instanceId: spanner-scaling-custom + units: NODES + minSize: 5 + maxSize: 30 + scalingMethod: LINEAR + scaleInLimit: 25 + metrics: + - name: my_custom_metric + filter: metric.type="spanner.googleapis.com/instance/resource/metric" + regional_threshold: 40 + multi_regional_threshold: 30 + autoscaler-config-direct.yaml: | + --- + - projectId: spanner-autoscaler-test + instanceId: spanner-scaling-direct + units: NODES + minSize: 5 + maxSize: 30 + scalingMethod: DIRECT diff --git a/terraform/cloud-functions/per-project/test/per_project_e2e_test.go b/terraform/cloud-functions/per-project/test/per_project_e2e_test.go index b44b6ebd..dd07cf69 100644 --- a/terraform/cloud-functions/per-project/test/per_project_e2e_test.go +++ b/terraform/cloud-functions/per-project/test/per_project_e2e_test.go @@ -22,7 +22,6 @@ import ( "context" "encoding/json" "fmt" - "strconv" "testing" "time" @@ -64,7 +63,7 @@ func setAutoscalerConfigMinProcessingUnits(t *testing.T, schedulerClient *schedu t.Fatal() } - schedulerJobBody[0]["minSize"] = strconv.Itoa(units) + schedulerJobBody[0]["minSize"] = units schedulerJobBodyUpdate, err := json.Marshal(schedulerJobBody) if err != nil { logger.Log(t, err) diff --git a/terraform/gke/README.md b/terraform/gke/README.md index d598fd6c..6f190d8c 100644 --- a/terraform/gke/README.md +++ b/terraform/gke/README.md @@ -697,6 +697,13 @@ following the instructions above. cat ${AUTOSCALER_ROOT}/autoscaler-config/autoscaler-config.yaml ``` +3. Validate the contents of the YAML configuraration file: + + ```sh + npm install + npm run validate-config-file -- ${AUTOSCALER_ROOT}/autoscaler-config/autoscaler-config.yaml + ``` + [architecture-gke]: ../../resources/architecture-gke.png [autoscaler-poller]: ../../src/poller/README.md