From a7915ef2746542e79faf13b1d346fda6fbba3b4c Mon Sep 17 00:00:00 2001 From: nielm Date: Thu, 14 Mar 2024 12:50:34 +0100 Subject: [PATCH] fix: Apply prettier reformatting --- .github/dependabot.yaml | 2 - .github/release-please.yml | 7 +- .github/workflows/codehealth.yaml | 77 ++- .github/workflows/codeql.yaml | 59 ++- .github/workflows/linters.yaml | 13 +- .github/workflows/unit_tests.yaml | 61 ++- .mdl.json | 6 +- .../autoscaler-pkg/networkpolicy.yaml | 2 +- .../otel-collector/otel-collector.yaml | 22 +- .../autoscaler-pkg/poller/poller-hourly.yaml | 46 +- .../autoscaler-pkg/poller/poller.yaml | 42 +- .../autoscaler-pkg/scaler/scaler.yaml | 34 +- .../unified/autoscaler-pkg/networkpolicy.yaml | 2 +- .../otel-collector/otel-collector.yaml | 22 +- .../autoscaler-pkg/scaler/scaler-hourly.yaml | 46 +- .../unified/autoscaler-pkg/scaler/scaler.yaml | 42 +- src/autoscaler-common/counters_base.js | 128 ++--- src/autoscaler-common/logger.js | 4 +- src/autoscaler-common/package.json | 26 +- src/autoscaler-common/types.js | 36 +- src/cloudbuild-poller.yaml | 13 +- src/cloudbuild-scaler.yaml | 13 +- src/cloudbuild-unified.yaml | 13 +- src/forwarder/index.js | 9 +- src/forwarder/package.json | 28 +- src/index.js | 8 +- src/poller/index.js | 8 +- src/poller/package.json | 24 +- src/poller/poller-core/counters.js | 16 +- src/poller/poller-core/index.js | 308 ++++++----- src/poller/poller-core/package.json | 50 +- src/poller/poller-core/test/index.test.js | 476 +++++++++-------- src/scaler/index.js | 3 +- src/scaler/package.json | 24 +- src/scaler/scaler-core/counters.js | 31 +- src/scaler/scaler-core/index.js | 186 +++---- .../scaler-core/scaling-methods/base.js | 67 +-- .../scaler-core/scaling-methods/direct.js | 18 +- .../scaler-core/scaling-methods/linear.js | 37 +- .../scaler-core/scaling-methods/stepwise.js | 22 +- src/scaler/scaler-core/state.js | 32 +- src/scaler/scaler-core/test/index.test.js | 216 ++++---- .../test/samples/downstream-msg.json | 52 +- .../scaler-core/test/samples/parameters.json | 62 +-- .../test/scaling-methods/base.test.js | 261 +++++----- .../test/scaling-methods/direct.test.js | 7 +- .../test/scaling-methods/linear.test.js | 207 +++++--- .../test/scaling-methods/stepwise.test.js | 166 +++--- src/scaler/scaler-core/test/state.test.js | 489 +++++++++--------- src/scaler/scaler-core/test/test-utils.js | 5 +- src/scaler/scaler-core/test/utils.test.js | 57 +- src/scaler/scaler-core/utils.js | 41 +- 52 files changed, 1907 insertions(+), 1719 deletions(-) diff --git a/.github/dependabot.yaml b/.github/dependabot.yaml index 11f12573..81c55fff 100644 --- a/.github/dependabot.yaml +++ b/.github/dependabot.yaml @@ -8,7 +8,6 @@ updates: commit-message: prefix: "fix" - # NPM dependancies -- only prompt to update minor versions. - directory: "/" package-ecosystem: "npm" @@ -120,4 +119,3 @@ updates: prefix: "fix" version: 2 -... diff --git a/.github/release-please.yml b/.github/release-please.yml index 291dd49e..7074a79f 100644 --- a/.github/release-please.yml +++ b/.github/release-please.yml @@ -1,7 +1,6 @@ handleGHRelease: true manifest: true branches: -- branch: version_2 - handleGHRelease: true - manifest: true - + - branch: version_2 + handleGHRelease: true + manifest: true diff --git a/.github/workflows/codehealth.yaml b/.github/workflows/codehealth.yaml index 4c880ce2..713bdf5a 100644 --- a/.github/workflows/codehealth.yaml +++ b/.github/workflows/codehealth.yaml @@ -14,55 +14,54 @@ jobs: strategy: fail-fast: false - steps: - - name: Checkout repository - uses: actions/checkout@v4 + - name: Checkout repository + uses: actions/checkout@v4 - - name: Use Node.js - uses: actions/setup-node@v4 - with: - node-version: 20 - check-latest: true + - name: Use Node.js + uses: actions/setup-node@v4 + with: + node-version: 20 + check-latest: true - - name: Install eslint and typescript compiler - run: npm install + - name: Install eslint and typescript compiler + run: npm install - - name: Install node modules in src/ - working-directory: src - run: npm install + - name: Install node modules in src/ + working-directory: src + run: npm install - - name: Execute "npm run typecheck" - working-directory: . - run: npm run typecheck + - name: Execute "npm run typecheck" + working-directory: . + run: npm run typecheck - - name: Execute "npm run eslint" - run: npm run eslint + - name: Execute "npm run eslint" + run: npm run eslint - - name: NPM Audit src/ - working-directory: src - run: npm audit + - name: NPM Audit src/ + working-directory: src + run: npm audit - - name: NPM Audit src/forwarder - working-directory: src/forwarder - run: npm audit + - name: NPM Audit src/forwarder + working-directory: src/forwarder + run: npm audit - - name: NPM Audit src/poller - working-directory: src/poller - run: npm audit + - name: NPM Audit src/poller + working-directory: src/poller + run: npm audit - - name: NPM Audit src/poller/poller-core - working-directory: src/poller/poller-core - run: npm audit + - name: NPM Audit src/poller/poller-core + working-directory: src/poller/poller-core + run: npm audit - - name: NPM Audit src/scaler - working-directory: src/scaler - run: npm audit + - name: NPM Audit src/scaler + working-directory: src/scaler + run: npm audit - - name: NPM Audit src/scaler/scaler-core - working-directory: src/scaler/scaler-core - run: npm audit + - name: NPM Audit src/scaler/scaler-core + working-directory: src/scaler/scaler-core + run: npm audit - - name: NPM Audit root - working-directory: . - run: npm audit + - name: NPM Audit root + working-directory: . + run: npm audit diff --git a/.github/workflows/codeql.yaml b/.github/workflows/codeql.yaml index 1a82b7d0..628875d5 100644 --- a/.github/workflows/codeql.yaml +++ b/.github/workflows/codeql.yaml @@ -15,7 +15,7 @@ on: push: pull_request: schedule: - - cron: '27 22 * * 5' + - cron: "27 22 * * 5" jobs: analyze: @@ -29,43 +29,42 @@ jobs: strategy: fail-fast: false matrix: - language: [ 'javascript' ] + language: ["javascript"] # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support steps: - - name: Checkout repository - uses: actions/checkout@v4 + - name: Checkout repository + uses: actions/checkout@v4 - # Initializes the CodeQL tools for scanning. - - name: Initialize CodeQL - uses: github/codeql-action/init@v3 - with: - languages: ${{ matrix.language }} - # If you wish to specify custom queries, you can do so here or in a config file. - # By default, queries listed here will override any specified in a config file. - # Prefix the list here with "+" to use these queries and those in the config file. + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v3 + with: + languages: ${{ matrix.language }} + # If you wish to specify custom queries, you can do so here or in a config file. + # By default, queries listed here will override any specified in a config file. + # Prefix the list here with "+" to use these queries and those in the config file. - # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs - # queries: security-extended,security-and-quality + # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs + # queries: security-extended,security-and-quality + # Autobuild attempts to build any compiled languages (C/C++, C#, Go, or Java). + # If this step fails, then you should remove it and run the build manually (see below) + - name: Autobuild + uses: github/codeql-action/autobuild@v3 - # Autobuild attempts to build any compiled languages (C/C++, C#, Go, or Java). - # If this step fails, then you should remove it and run the build manually (see below) - - name: Autobuild - uses: github/codeql-action/autobuild@v3 + # ℹī¸ Command-line programs to run using the OS shell. + # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun - # ℹī¸ Command-line programs to run using the OS shell. - # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun + # If the Autobuild fails above, remove it and uncomment the following three lines. + # modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance. - # If the Autobuild fails above, remove it and uncomment the following three lines. - # modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance. + # - run: | + # echo "Run, Build Application using script" + # ./location_of_script_within_repo/buildscript.sh - # - run: | - # echo "Run, Build Application using script" - # ./location_of_script_within_repo/buildscript.sh - - - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v3 - with: - category: "/language:${{matrix.language}}" + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v3 + with: + category: "/language:${{matrix.language}}" diff --git a/.github/workflows/linters.yaml b/.github/workflows/linters.yaml index e2635cdf..caba0250 100644 --- a/.github/workflows/linters.yaml +++ b/.github/workflows/linters.yaml @@ -18,13 +18,12 @@ on: [push, pull_request] jobs: delivery: - runs-on: ubuntu-latest steps: - - name: Check out code - uses: actions/checkout@v4 - - name: Run mdl - uses: actionshub/markdownlint@main - with: - filesToIgnoreRegex: code\-of\-conduct\.md|CHANGELOG.md + - name: Check out code + uses: actions/checkout@v4 + - name: Run mdl + uses: actionshub/markdownlint@main + with: + filesToIgnoreRegex: code\-of\-conduct\.md|CHANGELOG.md diff --git a/.github/workflows/unit_tests.yaml b/.github/workflows/unit_tests.yaml index ef43a91c..22dfae97 100644 --- a/.github/workflows/unit_tests.yaml +++ b/.github/workflows/unit_tests.yaml @@ -18,7 +18,6 @@ on: [push, pull_request] jobs: unit-tests: - runs-on: ubuntu-latest defaults: @@ -30,33 +29,33 @@ jobs: node-version: [18.x, 20.x] steps: - - uses: actions/checkout@v4 - - - name: Use Node.js ${{ matrix.node-version }} - uses: actions/setup-node@v4 - with: - node-version: ${{ matrix.node-version }} - - - name: npm install in autoscaler-common - working-directory: src/autoscaler-common - run: npm install - - - name: npm install in poller-core - working-directory: src/poller/poller-core - run: npm install - - - name: poller-core unit tests - working-directory: src/poller/poller-core - run: npm test - env: - CI: true - - - name: npm install in scaler-core - working-directory: src/scaler/scaler-core - run: npm install - - - name: scaler-core unit tests - working-directory: src/scaler/scaler-core - run: npm test - env: - CI: true + - uses: actions/checkout@v4 + + - name: Use Node.js ${{ matrix.node-version }} + uses: actions/setup-node@v4 + with: + node-version: ${{ matrix.node-version }} + + - name: npm install in autoscaler-common + working-directory: src/autoscaler-common + run: npm install + + - name: npm install in poller-core + working-directory: src/poller/poller-core + run: npm install + + - name: poller-core unit tests + working-directory: src/poller/poller-core + run: npm test + env: + CI: true + + - name: npm install in scaler-core + working-directory: src/scaler/scaler-core + run: npm install + + - name: scaler-core unit tests + working-directory: src/scaler/scaler-core + run: npm test + env: + CI: true diff --git a/.mdl.json b/.mdl.json index 53cd1820..923106fe 100644 --- a/.mdl.json +++ b/.mdl.json @@ -5,7 +5,11 @@ "MD002": false, "MD004": { "style": "asterisk" }, "MD007": { "indent": 4 }, - "MD013": { "ignore_code_blocks": true, "code_blocks": false, "tables": false }, + "MD013": { + "ignore_code_blocks": true, + "code_blocks": false, + "tables": false + }, "MD029": { "style": "ordered" }, "MD030": { "ul_single": 3, "ul_multi": 3, "ol_single": 2, "ol_multi": 2 } } diff --git a/kubernetes/decoupled/autoscaler-pkg/networkpolicy.yaml b/kubernetes/decoupled/autoscaler-pkg/networkpolicy.yaml index c308ed06..f9856b56 100644 --- a/kubernetes/decoupled/autoscaler-pkg/networkpolicy.yaml +++ b/kubernetes/decoupled/autoscaler-pkg/networkpolicy.yaml @@ -19,7 +19,7 @@ metadata: spec: podSelector: {} policyTypes: - - Ingress + - Ingress --- apiVersion: networking.k8s.io/v1 kind: NetworkPolicy diff --git a/kubernetes/decoupled/autoscaler-pkg/otel-collector/otel-collector.yaml b/kubernetes/decoupled/autoscaler-pkg/otel-collector/otel-collector.yaml index ea861f4f..30f05c21 100644 --- a/kubernetes/decoupled/autoscaler-pkg/otel-collector/otel-collector.yaml +++ b/kubernetes/decoupled/autoscaler-pkg/otel-collector/otel-collector.yaml @@ -16,18 +16,18 @@ spec: app: otel-collector spec: containers: - - name: otel-collector - image: otel/opentelemetry-collector-contrib:0.93.0 - args: - - --config - - /etc/otel/config.yaml - volumeMounts: - - mountPath: /etc/otel/ - name: otel-config + - name: otel-collector + image: otel/opentelemetry-collector-contrib:0.93.0 + args: + - --config + - /etc/otel/config.yaml + volumeMounts: + - mountPath: /etc/otel/ + name: otel-config volumes: - - name: otel-config - configMap: - name: otel-config + - name: otel-config + configMap: + name: otel-config nodeSelector: iam.gke.io/gke-metadata-server-enabled: "true" serviceAccountName: otel-collector-sa diff --git a/kubernetes/decoupled/autoscaler-pkg/poller/poller-hourly.yaml b/kubernetes/decoupled/autoscaler-pkg/poller/poller-hourly.yaml index 8490456a..fa77c6a6 100644 --- a/kubernetes/decoupled/autoscaler-pkg/poller/poller-hourly.yaml +++ b/kubernetes/decoupled/autoscaler-pkg/poller/poller-hourly.yaml @@ -28,30 +28,30 @@ spec: otel-submitter: "true" spec: containers: - - name: poller - image: poller-image # kpt-set: ${poller_image} - resources: - requests: - memory: "256Mi" - cpu: "250m" - limits: - memory: "256Mi" - env: - - name: AUTOSCALER_CONFIG - value: "/etc/autoscaler-config/autoscaler-config-hourly.yaml" - - name: K8S_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: OTLP_COLLECTOR_URL - value: "http://otel-collector:4317/" - volumeMounts: - - name: config-volume - mountPath: /etc/autoscaler-config + - name: poller + image: poller-image # kpt-set: ${poller_image} + resources: + requests: + memory: "256Mi" + cpu: "250m" + limits: + memory: "256Mi" + env: + - name: AUTOSCALER_CONFIG + value: "/etc/autoscaler-config/autoscaler-config-hourly.yaml" + - name: K8S_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: OTLP_COLLECTOR_URL + value: "http://otel-collector:4317/" + volumeMounts: + - name: config-volume + mountPath: /etc/autoscaler-config volumes: - - name: config-volume - configMap: - name: autoscaler-config-hourly + - name: config-volume + configMap: + name: autoscaler-config-hourly nodeSelector: iam.gke.io/gke-metadata-server-enabled: "true" restartPolicy: Never diff --git a/kubernetes/decoupled/autoscaler-pkg/poller/poller.yaml b/kubernetes/decoupled/autoscaler-pkg/poller/poller.yaml index 49bc978c..be470279 100644 --- a/kubernetes/decoupled/autoscaler-pkg/poller/poller.yaml +++ b/kubernetes/decoupled/autoscaler-pkg/poller/poller.yaml @@ -28,28 +28,28 @@ spec: otel-submitter: "true" spec: containers: - - name: poller - image: poller-image # kpt-set: ${poller_image} - resources: - requests: - memory: "256Mi" - cpu: "250m" - limits: - memory: "256Mi" - env: - - name: K8S_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: OTLP_COLLECTOR_URL - value: "http://otel-collector:4317/" - volumeMounts: - - name: config-volume - mountPath: /etc/autoscaler-config + - name: poller + image: poller-image # kpt-set: ${poller_image} + resources: + requests: + memory: "256Mi" + cpu: "250m" + limits: + memory: "256Mi" + env: + - name: K8S_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: OTLP_COLLECTOR_URL + value: "http://otel-collector:4317/" + volumeMounts: + - name: config-volume + mountPath: /etc/autoscaler-config volumes: - - name: config-volume - configMap: - name: autoscaler-config + - name: config-volume + configMap: + name: autoscaler-config nodeSelector: iam.gke.io/gke-metadata-server-enabled: "true" restartPolicy: Never diff --git a/kubernetes/decoupled/autoscaler-pkg/scaler/scaler.yaml b/kubernetes/decoupled/autoscaler-pkg/scaler/scaler.yaml index aa984260..95895bb7 100644 --- a/kubernetes/decoupled/autoscaler-pkg/scaler/scaler.yaml +++ b/kubernetes/decoupled/autoscaler-pkg/scaler/scaler.yaml @@ -32,23 +32,23 @@ spec: otel-submitter: "true" spec: containers: - - name: scaler - image: scaler-image # kpt-set: ${scaler_image} - ports: - - containerPort: 3000 - resources: - requests: - memory: "128Mi" - cpu: "100m" - limits: - memory: "256Mi" - env: - - name: K8S_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: OTLP_COLLECTOR_URL - value: "http://otel-collector:4317/" + - name: scaler + image: scaler-image # kpt-set: ${scaler_image} + ports: + - containerPort: 3000 + resources: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "256Mi" + env: + - name: K8S_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: OTLP_COLLECTOR_URL + value: "http://otel-collector:4317/" nodeSelector: iam.gke.io/gke-metadata-server-enabled: "true" serviceAccountName: scaler-sa diff --git a/kubernetes/unified/autoscaler-pkg/networkpolicy.yaml b/kubernetes/unified/autoscaler-pkg/networkpolicy.yaml index a7e3dd3c..56d0fd9d 100644 --- a/kubernetes/unified/autoscaler-pkg/networkpolicy.yaml +++ b/kubernetes/unified/autoscaler-pkg/networkpolicy.yaml @@ -20,7 +20,7 @@ metadata: spec: podSelector: {} policyTypes: - - Ingress + - Ingress --- apiVersion: networking.k8s.io/v1 kind: NetworkPolicy diff --git a/kubernetes/unified/autoscaler-pkg/otel-collector/otel-collector.yaml b/kubernetes/unified/autoscaler-pkg/otel-collector/otel-collector.yaml index ea861f4f..30f05c21 100644 --- a/kubernetes/unified/autoscaler-pkg/otel-collector/otel-collector.yaml +++ b/kubernetes/unified/autoscaler-pkg/otel-collector/otel-collector.yaml @@ -16,18 +16,18 @@ spec: app: otel-collector spec: containers: - - name: otel-collector - image: otel/opentelemetry-collector-contrib:0.93.0 - args: - - --config - - /etc/otel/config.yaml - volumeMounts: - - mountPath: /etc/otel/ - name: otel-config + - name: otel-collector + image: otel/opentelemetry-collector-contrib:0.93.0 + args: + - --config + - /etc/otel/config.yaml + volumeMounts: + - mountPath: /etc/otel/ + name: otel-config volumes: - - name: otel-config - configMap: - name: otel-config + - name: otel-config + configMap: + name: otel-config nodeSelector: iam.gke.io/gke-metadata-server-enabled: "true" serviceAccountName: otel-collector-sa diff --git a/kubernetes/unified/autoscaler-pkg/scaler/scaler-hourly.yaml b/kubernetes/unified/autoscaler-pkg/scaler/scaler-hourly.yaml index 95b16b29..2d8a2ea3 100644 --- a/kubernetes/unified/autoscaler-pkg/scaler/scaler-hourly.yaml +++ b/kubernetes/unified/autoscaler-pkg/scaler/scaler-hourly.yaml @@ -28,30 +28,30 @@ spec: otel-submitter: "true" spec: containers: - - name: scaler - image: scaler-image # kpt-set: ${scaler_image} - resources: - requests: - memory: "256Mi" - cpu: "250m" - limits: - memory: "256Mi" - env: - - name: AUTOSCALER_CONFIG - value: "/etc/autoscaler-config/autoscaler-config-hourly.yaml" - - name: K8S_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: OTLP_COLLECTOR_URL - value: "http://otel-collector:4317/" - volumeMounts: - - name: config-volume - mountPath: /etc/autoscaler-config + - name: scaler + image: scaler-image # kpt-set: ${scaler_image} + resources: + requests: + memory: "256Mi" + cpu: "250m" + limits: + memory: "256Mi" + env: + - name: AUTOSCALER_CONFIG + value: "/etc/autoscaler-config/autoscaler-config-hourly.yaml" + - name: K8S_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: OTLP_COLLECTOR_URL + value: "http://otel-collector:4317/" + volumeMounts: + - name: config-volume + mountPath: /etc/autoscaler-config volumes: - - name: config-volume - configMap: - name: autoscaler-config-hourly + - name: config-volume + configMap: + name: autoscaler-config-hourly nodeSelector: iam.gke.io/gke-metadata-server-enabled: "true" restartPolicy: Never diff --git a/kubernetes/unified/autoscaler-pkg/scaler/scaler.yaml b/kubernetes/unified/autoscaler-pkg/scaler/scaler.yaml index 7fe923f7..3f751cab 100644 --- a/kubernetes/unified/autoscaler-pkg/scaler/scaler.yaml +++ b/kubernetes/unified/autoscaler-pkg/scaler/scaler.yaml @@ -28,28 +28,28 @@ spec: otel-submitter: "true" spec: containers: - - name: scaler - image: scaler-image # kpt-set: ${scaler_image} - resources: - requests: - memory: "256Mi" - cpu: "250m" - limits: - memory: "256Mi" - env: - - name: K8S_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: OTLP_COLLECTOR_URL - value: "http://otel-collector:4317/" - volumeMounts: - - name: config-volume - mountPath: /etc/autoscaler-config + - name: scaler + image: scaler-image # kpt-set: ${scaler_image} + resources: + requests: + memory: "256Mi" + cpu: "250m" + limits: + memory: "256Mi" + env: + - name: K8S_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: OTLP_COLLECTOR_URL + value: "http://otel-collector:4317/" + volumeMounts: + - name: config-volume + mountPath: /etc/autoscaler-config volumes: - - name: config-volume - configMap: - name: autoscaler-config + - name: config-volume + configMap: + name: autoscaler-config nodeSelector: iam.gke.io/gke-metadata-server-enabled: "true" restartPolicy: Never diff --git a/src/autoscaler-common/counters_base.js b/src/autoscaler-common/counters_base.js index 91f6b249..e8a91b2b 100644 --- a/src/autoscaler-common/counters_base.js +++ b/src/autoscaler-common/counters_base.js @@ -20,17 +20,21 @@ * packages * */ -const {MeterProvider, PeriodicExportingMetricReader} = - require('@opentelemetry/sdk-metrics'); +const { + MeterProvider, + PeriodicExportingMetricReader, +} = require('@opentelemetry/sdk-metrics'); const {Resource} = require('@opentelemetry/resources'); -const {MetricExporter: GcpMetricExporter} = - require('@google-cloud/opentelemetry-cloud-monitoring-exporter'); -const {OTLPMetricExporter} = - require('@opentelemetry/exporter-metrics-otlp-grpc'); -const {GcpDetectorSync} = - require('@google-cloud/opentelemetry-resource-util'); -const {SemanticResourceAttributes: Semconv} = - require('@opentelemetry/semantic-conventions'); +const { + MetricExporter: GcpMetricExporter, +} = require('@google-cloud/opentelemetry-cloud-monitoring-exporter'); +const { + OTLPMetricExporter, +} = require('@opentelemetry/exporter-metrics-otlp-grpc'); +const {GcpDetectorSync} = require('@google-cloud/opentelemetry-resource-util'); +const { + SemanticResourceAttributes: Semconv, +} = require('@opentelemetry/semantic-conventions'); const OpenTelemetryApi = require('@opentelemetry/api'); const OpenTelemetryCore = require('@opentelemetry/core'); const {setTimeout} = require('timers/promises'); @@ -45,10 +49,9 @@ const {logger} = require('./logger.js'); /** * @typedef {{ -* [x: string]: string, -* }} CounterAttributes -*/ - + * [x: string]: string, + * }} CounterAttributes + */ const RESOURCE_ATTRIBUTES = { [Semconv.SERVICE_NAMESPACE]: 'cloudspannerecosystem', @@ -70,7 +73,6 @@ const COUNTERS_PREFIX = RESOURCE_ATTRIBUTES[Semconv.SERVICE_NAME] + '/'; - /** @enum{String} */ const ExporterMode = { GCM: 'GCM', @@ -85,7 +87,7 @@ const EXPORTER_PARAMETERS = { // // However the only Metrics pusher is a periodic one :/ [ExporterMode.GCM]: { - PERIODIC_FLUSH_INTERVAL: 12*60*60*1_000, // 12hr + PERIODIC_FLUSH_INTERVAL: 12 * 60 * 60 * 1_000, // 12hr FLUSH_MIN_INTERVAL: 10_000, FLUSH_MAX_ATTEMPTS: 6, }, @@ -132,27 +134,27 @@ class DiagToBunyanLogger { // eslint-disable-next-line require-jsdoc verbose(message, ...args) { - logger.trace('otel: '+message, args); + logger.trace('otel: ' + message, args); } // eslint-disable-next-line require-jsdoc debug(message, ...args) { - logger.debug('otel: '+message, args); + logger.debug('otel: ' + message, args); } // eslint-disable-next-line require-jsdoc info(message, ...args) { - logger.info('otel: '+message, args); + logger.info('otel: ' + message, args); } // eslint-disable-next-line require-jsdoc warn(message, ...args) { - logger.warn('otel: '+message, args); + logger.warn('otel: ' + message, args); } // eslint-disable-next-line require-jsdoc error(message, ...args) { - if ( ! this.suppressErrors ) { - logger.error('otel: '+message, args); + if (!this.suppressErrors) { + logger.error('otel: ' + message, args); } } -}; +} const DIAG_BUNYAN_LOGGER = new DiagToBunyanLogger(); @@ -174,17 +176,13 @@ function openTelemetryGlobalErrorHandler(err) { OpenTelemetryCore.loggingErrorHandler()(err); } - // Setup OpenTelemetry client libraries logging. -OpenTelemetryApi.default.diag.setLogger( - DIAG_BUNYAN_LOGGER, - { - logLevel: OpenTelemetryApi.DiagLogLevel.INFO, - suppressOverrideMessage: true, - }); +OpenTelemetryApi.default.diag.setLogger(DIAG_BUNYAN_LOGGER, { + logLevel: OpenTelemetryApi.DiagLogLevel.INFO, + suppressOverrideMessage: true, +}); OpenTelemetryCore.setGlobalErrorHandler(openTelemetryGlobalErrorHandler); - /** * Initialize the OpenTelemetry metrics, set up logging * @@ -208,7 +206,6 @@ async function initMetrics() { rejectPendingInit = rej; }); - try { logger.debug('initializing metrics'); @@ -216,13 +213,13 @@ async function initMetrics() { // In K8s. We need to set the Pod Name to prevent duplicate // timeseries errors. if (process.env.K8S_POD_NAME) { - RESOURCE_ATTRIBUTES[ - Semconv.K8S_POD_NAME] = - process.env.K8S_POD_NAME; + RESOURCE_ATTRIBUTES[Semconv.K8S_POD_NAME] = process.env.K8S_POD_NAME; } else { - logger.warn('WARNING: running under Kubernetes, but K8S_POD_NAME ' + - 'environment variable is not set. ' + - 'This may lead to Send TimeSeries errors'); + logger.warn( + 'WARNING: running under Kubernetes, but K8S_POD_NAME ' + + 'environment variable is not set. ' + + 'This may lead to Send TimeSeries errors', + ); } } @@ -239,11 +236,13 @@ async function initMetrics() { if (gcpResources.attributes[Semconv.FAAS_ID]?.toString()) { RESOURCE_ATTRIBUTES[Semconv.SERVICE_INSTANCE_ID] = - gcpResources.attributes[Semconv.FAAS_ID].toString(); + gcpResources.attributes[Semconv.FAAS_ID].toString(); } else { - logger.warn('WARNING: running under Cloud Functions, but FAAS_ID ' + - 'resource attribute is not set. ' + - 'This may lead to Send TimeSeries errors'); + logger.warn( + 'WARNING: running under Cloud Functions, but FAAS_ID ' + + 'resource attribute is not set. ' + + 'This may lead to Send TimeSeries errors', + ); } } @@ -253,8 +252,9 @@ async function initMetrics() { let exporter; if (process.env.OTLP_COLLECTOR_URL) { exporterMode = ExporterMode.OTEL; - logger.info(`Counters sent using OTLP to ${ - process.env.OTLP_COLLECTOR_URL}`); + logger.info( + `Counters sent using OTLP to ${process.env.OTLP_COLLECTOR_URL}`, + ); exporter = new OTLPMetricExporter({url: process.env.OTLP_COLLECTOR_URL}); } else { exporterMode = ExporterMode.GCM; @@ -266,10 +266,10 @@ async function initMetrics() { resource: resources, readers: [ new PeriodicExportingMetricReader({ - exportIntervalMillis: EXPORTER_PARAMETERS[exporterMode] - .PERIODIC_FLUSH_INTERVAL, - exportTimeoutMillis: EXPORTER_PARAMETERS[exporterMode] - .PERIODIC_FLUSH_INTERVAL, + exportIntervalMillis: + EXPORTER_PARAMETERS[exporterMode].PERIODIC_FLUSH_INTERVAL, + exportTimeoutMillis: + EXPORTER_PARAMETERS[exporterMode].PERIODIC_FLUSH_INTERVAL, exporter: exporter, }), ], @@ -277,12 +277,11 @@ async function initMetrics() { } catch (e) { // report failures to other waiters. rejectPendingInit(e); - throw (e); + throw e; } resolvePendingInit(); } - /** * Initialize metrics with cloud monitoring * @@ -299,15 +298,19 @@ async function createCounters(counterDefinitions) { for (const counterDef of counterDefinitions) { if (!counterDef.counterName || !counterDef.counterDesc) { - throw new Error('invalid counter definition: ' + - JSON.stringify(counterDef)); + throw new Error( + 'invalid counter definition: ' + JSON.stringify(counterDef), + ); } if (COUNTERS.get(counterDef.counterName)) { throw new Error('Counter already created: ' + counterDef.counterName); } - COUNTERS.set(counterDef.counterName, - meter.createCounter(COUNTERS_PREFIX + counterDef.counterName, - {description: counterDef.counterDesc})); + COUNTERS.set( + counterDef.counterName, + meter.createCounter(COUNTERS_PREFIX + counterDef.counterName, { + description: counterDef.counterDesc, + }), + ); } } @@ -325,7 +328,6 @@ function incCounter(counterName, counterAttributes) { counter.add(1, counterAttributes); } - let lastForceFlushTime = 0; let flushInProgress; let tryFlushEnabled = true; @@ -364,7 +366,8 @@ async function tryFlush() { try { // If flushed recently, wait for the min interval to pass. - const millisUntilNextForceFlush = lastForceFlushTime + + const millisUntilNextForceFlush = + lastForceFlushTime + EXPORTER_PARAMETERS[exporterMode].FLUSH_MIN_INTERVAL - Date.now(); @@ -374,7 +377,6 @@ async function tryFlush() { await setTimeout(millisUntilNextForceFlush); } - // OpenTelemetry's forceFlush() will always succeed, even if the backend // fails and reports an error... // @@ -394,7 +396,7 @@ async function tryFlush() { const oldOpenTelemetryErrorCount = openTelemetryErrorCount; // Suppress OTEL Diag error messages on all but the last flush attempt. - DIAG_BUNYAN_LOGGER.suppressErrors = (attempts > 1); + DIAG_BUNYAN_LOGGER.suppressErrors = attempts > 1; await meterProvider.forceFlush(); DIAG_BUNYAN_LOGGER.suppressErrors = false; @@ -410,9 +412,9 @@ async function tryFlush() { attempts--; } if (attempts <= 0) { - logger.error(`Failed to flush counters after ${ - EXPORTER_PARAMETERS[exporterMode].FLUSH_MAX_ATTEMPTS - } attempts - see OpenTelemetry logging`); + logger.error( + `Failed to flush counters after ${EXPORTER_PARAMETERS[exporterMode].FLUSH_MAX_ATTEMPTS} attempts - see OpenTelemetry logging`, + ); } } catch (e) { logger.error('Error while flushing counters', e); @@ -425,7 +427,7 @@ async function tryFlush() { /** * Specify whether the tryFlush function should try to flush or not. -* + * * In long-running processes, disabling flushing will give better results * while in short-lived processes, without flushing, counters may not * be reported to cloud monitoring. diff --git a/src/autoscaler-common/logger.js b/src/autoscaler-common/logger.js index 8a097082..c4e8353a 100644 --- a/src/autoscaler-common/logger.js +++ b/src/autoscaler-common/logger.js @@ -43,9 +43,7 @@ const loggingBunyan = new LoggingBunyan({ const logger = bunyan.createLogger({ name: 'cloud-spanner-autoscaler', - streams: [ - loggingBunyan.stream(getLogLevel()), - ], + streams: [loggingBunyan.stream(getLogLevel())], }); module.exports = { diff --git a/src/autoscaler-common/package.json b/src/autoscaler-common/package.json index 1581f602..7624a27f 100644 --- a/src/autoscaler-common/package.json +++ b/src/autoscaler-common/package.json @@ -1,15 +1,15 @@ { - "name": "autoscaler-common", - "description": "Common code for Spanner autoscater", - "license": "Apache-2.0", - "author": "Google Inc.", - "dependencies": { - "@google-cloud/logging-bunyan": "^5.0.1", - "@google-cloud/opentelemetry-cloud-monitoring-exporter": "^0.17.0", - "@opentelemetry/api": "^1.8.0", - "@opentelemetry/exporter-metrics-otlp-grpc": "^0.49.1", - "@opentelemetry/sdk-metrics": "^1.22.0", - "@opentelemetry/sdk-node": "^0.49.1", - "bunyan": "^1.8.15" - } + "name": "autoscaler-common", + "description": "Common code for Spanner autoscater", + "license": "Apache-2.0", + "author": "Google Inc.", + "dependencies": { + "@google-cloud/logging-bunyan": "^5.0.1", + "@google-cloud/opentelemetry-cloud-monitoring-exporter": "^0.17.0", + "@opentelemetry/api": "^1.8.0", + "@opentelemetry/exporter-metrics-otlp-grpc": "^0.49.1", + "@opentelemetry/sdk-metrics": "^1.22.0", + "@opentelemetry/sdk-node": "^0.49.1", + "bunyan": "^1.8.15" + } } diff --git a/src/autoscaler-common/types.js b/src/autoscaler-common/types.js index 39aad440..8952c550 100644 --- a/src/autoscaler-common/types.js +++ b/src/autoscaler-common/types.js @@ -30,17 +30,17 @@ const AutoscalerUnits = { /** * @typedef {{ -* name: string, -* filter: string, -* reducer: string, -* aligner: string, -* period: number, -* regional_threshold: number, -* regional_margin?: number, -* multi_regional_threshold: number, -* multi_regional_margin?: number, -* }} SpannerMetric -*/ + * name: string, + * filter: string, + * reducer: string, + * aligner: string, + * period: number, + * regional_threshold: number, + * regional_margin?: number, + * multi_regional_threshold: number, + * multi_regional_margin?: number, + * }} SpannerMetric + */ /** * @typedef {{ @@ -78,20 +78,18 @@ const AutoscalerUnits = { /** * @typedef {{ -* name: string, -* threshold: number, -* margin: number, -* value: number -* }} SpannerMetricValue -*/ - + * name: string, + * threshold: number, + * margin: number, + * value: number + * }} SpannerMetricValue + */ /** * @typedef {SpannerConfig & SpannerMetadata & SpannerData * } AutoscalerSpanner */ - module.exports = { AutoscalerUnits, }; diff --git a/src/cloudbuild-poller.yaml b/src/cloudbuild-poller.yaml index f777a462..bf6aadeb 100644 --- a/src/cloudbuild-poller.yaml +++ b/src/cloudbuild-poller.yaml @@ -13,6 +13,13 @@ # limitations under the License. steps: -- name: 'gcr.io/cloud-builders/docker' - args: ['build', '--tag=$LOCATION-docker.pkg.dev/$PROJECT_ID/spanner-autoscaler/poller', '-f', 'Dockerfile-poller', '.'] -images: ['$LOCATION-docker.pkg.dev/$PROJECT_ID/spanner-autoscaler/poller'] + - name: "gcr.io/cloud-builders/docker" + args: + [ + "build", + "--tag=$LOCATION-docker.pkg.dev/$PROJECT_ID/spanner-autoscaler/poller", + "-f", + "Dockerfile-poller", + ".", + ] +images: ["$LOCATION-docker.pkg.dev/$PROJECT_ID/spanner-autoscaler/poller"] diff --git a/src/cloudbuild-scaler.yaml b/src/cloudbuild-scaler.yaml index ca5602cc..9987c4ef 100644 --- a/src/cloudbuild-scaler.yaml +++ b/src/cloudbuild-scaler.yaml @@ -13,6 +13,13 @@ # limitations under the License. steps: -- name: 'gcr.io/cloud-builders/docker' - args: ['build', '--tag=$LOCATION-docker.pkg.dev/$PROJECT_ID/spanner-autoscaler/scaler', '-f', 'Dockerfile-scaler', '.'] -images: ['$LOCATION-docker.pkg.dev/$PROJECT_ID/spanner-autoscaler/scaler'] + - name: "gcr.io/cloud-builders/docker" + args: + [ + "build", + "--tag=$LOCATION-docker.pkg.dev/$PROJECT_ID/spanner-autoscaler/scaler", + "-f", + "Dockerfile-scaler", + ".", + ] +images: ["$LOCATION-docker.pkg.dev/$PROJECT_ID/spanner-autoscaler/scaler"] diff --git a/src/cloudbuild-unified.yaml b/src/cloudbuild-unified.yaml index d7397917..2ec33e58 100644 --- a/src/cloudbuild-unified.yaml +++ b/src/cloudbuild-unified.yaml @@ -13,6 +13,13 @@ # limitations under the License. steps: -- name: 'gcr.io/cloud-builders/docker' - args: ['build', '--tag=$LOCATION-docker.pkg.dev/$PROJECT_ID/spanner-autoscaler/scaler', '-f', 'Dockerfile-unified', '.'] -images: ['$LOCATION-docker.pkg.dev/$PROJECT_ID/spanner-autoscaler/scaler'] + - name: "gcr.io/cloud-builders/docker" + args: + [ + "build", + "--tag=$LOCATION-docker.pkg.dev/$PROJECT_ID/spanner-autoscaler/scaler", + "-f", + "Dockerfile-unified", + ".", + ] +images: ["$LOCATION-docker.pkg.dev/$PROJECT_ID/spanner-autoscaler/scaler"] diff --git a/src/forwarder/index.js b/src/forwarder/index.js index 15dae14b..77cf0b6d 100644 --- a/src/forwarder/index.js +++ b/src/forwarder/index.js @@ -60,7 +60,8 @@ async function forwardFromHTTP(req, res) { } catch (err) { logger.error({ message: `An error occurred in the Autoscaler forwarder (HTTP)`, - err: err}); + err: err, + }); logger.error({message: `JSON payload`, payload: payloadString}); res.status(500).end(err.toString()); } @@ -87,9 +88,9 @@ async function forwardFromPubSub(pubSubEvent, context) { }); } catch (err) { logger.error({ - message: - `An error occurred in the Autoscaler forwarder (PubSub)`, - err: err}); + message: `An error occurred in the Autoscaler forwarder (PubSub)`, + err: err, + }); logger.error({message: `JSON payload`, payload: payload}); } } diff --git a/src/forwarder/package.json b/src/forwarder/package.json index fc4d70a8..d7a03e13 100644 --- a/src/forwarder/package.json +++ b/src/forwarder/package.json @@ -1,16 +1,16 @@ { - "name": "pubsub-fowarder", - "description": "Function to fowarder messages from one PubSub Topic to another PubSub Topic", - "license": "Apache-2.0", - "author": "Google Inc.", - "dependencies": { - "@google-cloud/functions-framework": "^3.3.0", - "@google-cloud/pubsub": "^4.3.3", - "express": "^4.18.3", - "autoscaler-common": "file:../autoscaler-common" - }, - "scripts": { - "start": "functions-framework --target=forwardFromHTTP", - "debug": "node --inspect node_modules/@google-cloud/functions-framework --target=forwardFromHTTP" - } + "name": "pubsub-fowarder", + "description": "Function to fowarder messages from one PubSub Topic to another PubSub Topic", + "license": "Apache-2.0", + "author": "Google Inc.", + "dependencies": { + "@google-cloud/functions-framework": "^3.3.0", + "@google-cloud/pubsub": "^4.3.3", + "express": "^4.18.3", + "autoscaler-common": "file:../autoscaler-common" + }, + "scripts": { + "start": "functions-framework --target=forwardFromHTTP", + "debug": "node --inspect node_modules/@google-cloud/functions-framework --target=forwardFromHTTP" + } } diff --git a/src/index.js b/src/index.js index 09491182..8d11ade7 100644 --- a/src/index.js +++ b/src/index.js @@ -26,7 +26,7 @@ const CountersBase = require('./autoscaler-common/counters_base'); */ async function main() { const DEFAULT_CONFIG_LOCATION = - '/etc/autoscaler-config/autoscaler-config.yaml'; + '/etc/autoscaler-config/autoscaler-config.yaml'; logger.info(`Autoscaler unified poller+scaler job started`); @@ -54,14 +54,16 @@ async function main() { try { const config = await fs.readFile(configLocation, {encoding: 'utf8'}); const spanners = await pollerCore.checkSpannerScaleMetricsLocal( - JSON.stringify(yaml.load(config))); + JSON.stringify(yaml.load(config)), + ); for (const spanner of spanners) { await scalerCore.scaleSpannerInstanceLocal(spanner); } } catch (err) { logger.error({ message: 'Error in unified poller/scaler wrapper:', - err: err}); + err: err, + }); } finally { CountersBase.setTryFlushEnabled(true); await CountersBase.tryFlush(); diff --git a/src/poller/index.js b/src/poller/index.js index 106a8839..570dfc0d 100644 --- a/src/poller/index.js +++ b/src/poller/index.js @@ -23,7 +23,7 @@ const {logger} = require('../autoscaler-common/logger'); */ async function main() { const DEFAULT_CONFIG_LOCATION = - '/etc/autoscaler-config/autoscaler-config.yaml'; + '/etc/autoscaler-config/autoscaler-config.yaml'; logger.info(`Autoscaler Poller job started`); @@ -46,11 +46,13 @@ async function main() { try { const config = await fs.readFile(configLocation, {encoding: 'utf8'}); await pollerCore.checkSpannerScaleMetricsJSON( - JSON.stringify(yaml.load(config))); + JSON.stringify(yaml.load(config)), + ); } catch (err) { logger.error({ message: 'Error in Poller wrapper:', - err: err}); + err: err, + }); } } diff --git a/src/poller/package.json b/src/poller/package.json index 55132c63..2519b230 100644 --- a/src/poller/package.json +++ b/src/poller/package.json @@ -1,14 +1,14 @@ { - "name": "poller", - "description": "Spanner metrics poller cron job", - "license": "Apache-2.0", - "author": "Google Inc.", - "dependencies": { - "autoscaler-common": "file:../autoscaler-common", - "js-yaml": "^4.1.0", - "poller-core": "file:poller-core" - }, - "scripts": { - "start": "node -e \"require('./index').main()\"" - } + "name": "poller", + "description": "Spanner metrics poller cron job", + "license": "Apache-2.0", + "author": "Google Inc.", + "dependencies": { + "autoscaler-common": "file:../autoscaler-common", + "js-yaml": "^4.1.0", + "poller-core": "file:poller-core" + }, + "scripts": { + "start": "node -e \"require('./index').main()\"" + } } diff --git a/src/poller/poller-core/counters.js b/src/poller/poller-core/counters.js index 68ce5805..e3b7ac3f 100644 --- a/src/poller/poller-core/counters.js +++ b/src/poller/poller-core/counters.js @@ -73,9 +73,9 @@ const pendingInit = CountersBase.createCounters(COUNTERS); function _getCounterAttributes(spanner) { return { [CountersBase.COUNTER_ATTRIBUTE_NAMES.SPANNER_PROJECT_ID]: - spanner.projectId, + spanner.projectId, [CountersBase.COUNTER_ATTRIBUTE_NAMES.SPANNER_INSTANCE_ID]: - spanner.instanceId, + spanner.instanceId, }; } @@ -86,8 +86,10 @@ function _getCounterAttributes(spanner) { */ async function incPollingSuccessCounter(spanner) { await pendingInit; - CountersBase.incCounter(COUNTER_NAMES.POLLING_SUCCESS, - _getCounterAttributes(spanner)); + CountersBase.incCounter( + COUNTER_NAMES.POLLING_SUCCESS, + _getCounterAttributes(spanner), + ); } /** @@ -97,8 +99,10 @@ async function incPollingSuccessCounter(spanner) { */ async function incPollingFailedCounter(spanner) { await pendingInit; - CountersBase.incCounter(COUNTER_NAMES.POLLING_FAILED, - _getCounterAttributes(spanner)); + CountersBase.incCounter( + COUNTER_NAMES.POLLING_FAILED, + _getCounterAttributes(spanner), + ); } /** diff --git a/src/poller/poller-core/index.js b/src/poller/poller-core/index.js index 154c8034..a83fe6fe 100644 --- a/src/poller/poller-core/index.js +++ b/src/poller/poller-core/index.js @@ -87,10 +87,11 @@ function buildMetrics(projectId, instanceId) { const metrics = [ { name: 'high_priority_cpu', - filter: createBaseFilter(projectId, instanceId) + - 'metric.type=' + - '"spanner.googleapis.com/instance/cpu/utilization_by_priority" ' + - 'AND metric.label.priority="high"', + filter: + createBaseFilter(projectId, instanceId) + + 'metric.type=' + + '"spanner.googleapis.com/instance/cpu/utilization_by_priority" ' + + 'AND metric.label.priority="high"', reducer: 'REDUCE_SUM', aligner: 'ALIGN_MAX', period: 60, @@ -99,9 +100,10 @@ function buildMetrics(projectId, instanceId) { }, { name: 'rolling_24_hr', - filter: createBaseFilter(projectId, instanceId) + - 'metric.type='+ - '"spanner.googleapis.com/instance/cpu/smoothed_utilization"', + filter: + createBaseFilter(projectId, instanceId) + + 'metric.type=' + + '"spanner.googleapis.com/instance/cpu/smoothed_utilization"', reducer: 'REDUCE_SUM', aligner: 'ALIGN_MAX', period: 60, @@ -110,8 +112,9 @@ function buildMetrics(projectId, instanceId) { }, { name: 'storage', - filter: createBaseFilter(projectId, instanceId) + - 'metric.type="spanner.googleapis.com/instance/storage/utilization"', + filter: + createBaseFilter(projectId, instanceId) + + 'metric.type="spanner.googleapis.com/instance/storage/utilization"', reducer: 'REDUCE_SUM', aligner: 'ALIGN_MAX', period: 60, @@ -130,9 +133,15 @@ function buildMetrics(projectId, instanceId) { * @return {string} filter */ function createBaseFilter(projectId, instanceId) { - return 'resource.labels.instance_id="' + instanceId + '" AND ' + - 'resource.type="spanner_instance" AND ' + - 'project="' + projectId + '" AND '; + return ( + 'resource.labels.instance_id="' + + instanceId + + '" AND ' + + 'resource.type="spanner_instance" AND ' + + 'project="' + + projectId + + '" AND ' + ); } /** @@ -147,22 +156,29 @@ function validateCustomMetric(metric, projectId, instanceId) { if (!metric.name) { logger.info({ message: 'Missing name parameter for custom metric.', - projectId: projectId, instanceId: instanceId}); + projectId: projectId, + instanceId: instanceId, + }); return false; } if (!metric.filter) { logger.info({ message: 'Missing filter parameter for custom metric.', - projectId: projectId, instanceId: instanceId}); + projectId: projectId, + instanceId: instanceId, + }); return false; } if (!(metric.regional_threshold > 0 || metric.multi_regional_threshold > 0)) { logger.info({ - message: 'Missing regional_threshold or multi_multi_regional_threshold ' + - 'parameter for custom metric.', - projectId: projectId, instanceId: instanceId}); + message: + 'Missing regional_threshold or multi_multi_regional_threshold ' + + 'parameter for custom metric.', + projectId: projectId, + instanceId: instanceId, + }); return false; } @@ -180,9 +196,10 @@ function validateCustomMetric(metric, projectId, instanceId) { function getMaxMetricValue(projectId, spannerInstanceId, metric) { const metricWindow = 5; logger.debug({ - message: `Get max ${metric.name} from ${projectId}/${ - spannerInstanceId} over ${metricWindow} minutes.`, - projectId: projectId, instanceId: spannerInstanceId}); + message: `Get max ${metric.name} from ${projectId}/${spannerInstanceId} over ${metricWindow} minutes.`, + projectId: projectId, + instanceId: spannerInstanceId, + }); /** @type {monitoring.protos.google.monitoring.v3.IListTimeSeriesRequest} */ const request = { @@ -241,7 +258,9 @@ function getMaxMetricValue(projectId, spannerInstanceId, metric) { function getSpannerMetadata(projectId, spannerInstanceId, units) { logger.info({ message: `----- ${projectId}/${spannerInstanceId}: Getting Metadata -----`, - projectId: projectId, instanceId: spannerInstanceId}); + projectId: projectId, + instanceId: spannerInstanceId, + }); const spanner = new Spanner({ projectId: projectId, @@ -254,20 +273,30 @@ function getSpannerMetadata(projectId, spannerInstanceId, units) { const metadata = data[0]; logger.debug({ message: `DisplayName: ${metadata['displayName']}`, - projectId: projectId, instanceId: spannerInstanceId}); + projectId: projectId, + instanceId: spannerInstanceId, + }); logger.debug({ message: `NodeCount: ${metadata['nodeCount']}`, - projectId: projectId, instanceId: spannerInstanceId}); + projectId: projectId, + instanceId: spannerInstanceId, + }); logger.debug({ message: `ProcessingUnits: ${metadata['processingUnits']}`, - projectId: projectId, instanceId: spannerInstanceId}); + projectId: projectId, + instanceId: spannerInstanceId, + }); logger.debug({ message: `Config: ${metadata['config'].split('/').pop()}`, - projectId: projectId, instanceId: spannerInstanceId}); + projectId: projectId, + instanceId: spannerInstanceId, + }); const spannerMetadata = { - currentSize: (units === AutoscalerUnits.NODES) ? metadata['nodeCount'] : - metadata['processingUnits'], + currentSize: + units === AutoscalerUnits.NODES + ? metadata['nodeCount'] + : metadata['processingUnits'], regional: metadata['config'].split('/').pop().startsWith('regional'), // DEPRECATED currentNodes: metadata['nodeCount'], @@ -291,24 +320,25 @@ async function postPubSubMessage(spanner, metrics) { spanner.metrics = metrics; const messageBuffer = Buffer.from(JSON.stringify(spanner), 'utf8'); - return topic.publishMessage({data: messageBuffer}) - .then(() => logger.info({ - message: - `----- Published message to topic: ${spanner.scalerPubSubTopic}`, + return topic + .publishMessage({data: messageBuffer}) + .then(() => + logger.info({ + message: `----- Published message to topic: ${spanner.scalerPubSubTopic}`, projectId: spanner.projectId, instanceId: spanner.instanceId, payload: spanner, - })) - .catch((err) => { - logger.error({ - message: `An error occurred when publishing the message to ${ - spanner.scalerPubSubTopic}`, - projectId: spanner.projectId, - instanceId: spanner.instanceId, - payload: err, - err: err, - }); + }), + ) + .catch((err) => { + logger.error({ + message: `An error occurred when publishing the message to ${spanner.scalerPubSubTopic}`, + projectId: spanner.projectId, + instanceId: spanner.instanceId, + payload: err, + err: err, }); + }); } /** @@ -324,26 +354,25 @@ async function callScalerHTTP(spanner, metrics) { spanner.metrics = metrics; - return axios.post(url.toString(), spanner) - .then( - (response) => { - logger.info({ - message: `----- Published message to scaler, response ${ - response.statusText}`, - projectId: spanner.projectId, - instanceId: spanner.instanceId, - payload: spanner, - }); - }) - .catch((err) => { - logger.error({ - message: `An error occurred when calling the scaler`, - projectId: spanner.projectId, - instanceId: spanner.instanceId, - payload: err, - err: err, - }); + return axios + .post(url.toString(), spanner) + .then((response) => { + logger.info({ + message: `----- Published message to scaler, response ${response.statusText}`, + projectId: spanner.projectId, + instanceId: spanner.instanceId, + payload: spanner, }); + }) + .catch((err) => { + logger.error({ + message: `An error occurred when calling the scaler`, + projectId: spanner.projectId, + instanceId: spanner.instanceId, + payload: err, + err: err, + }); + }); } /** @@ -358,8 +387,9 @@ async function parseAndEnrichPayload(payload) { const spannersFound = []; for (let sIdx = 0; sIdx < spanners.length; sIdx++) { - const metricOverrides = /** @type {SpannerMetric[]} */ - (spanners[sIdx].metrics); + const metricOverrides = + /** @type {SpannerMetric[]} */ + (spanners[sIdx].metrics); // assemble the config // merge in the defaults @@ -375,9 +405,10 @@ async function parseAndEnrichPayload(payload) { // they are set the config is invalid. if (spanners[sIdx].minNodes || spanners[sIdx].maxNodes) { throw new Error( - 'INVALID CONFIG: units is set to PROCESSING_UNITS, ' + + 'INVALID CONFIG: units is set to PROCESSING_UNITS, ' + 'however, minNodes or maxNodes is set, ' + - 'remove minNodes and maxNodes from your configuration.'); + 'remove minNodes and maxNodes from your configuration.', + ); } } else if (spanners[sIdx].units.toUpperCase() == 'NODES') { spanners[sIdx].units = spanners[sIdx].units.toUpperCase(); @@ -394,11 +425,14 @@ async function parseAndEnrichPayload(payload) { }); spanners[sIdx].minSize = spanners[sIdx].minNodes; } else if ( - spanners[sIdx].minSize && spanners[sIdx].minNodes && - spanners[sIdx].minSize != spanners[sIdx].minNodes) { + spanners[sIdx].minSize && + spanners[sIdx].minNodes && + spanners[sIdx].minSize != spanners[sIdx].minNodes + ) { throw new Error( - 'INVALID CONFIG: minSize and minNodes are both set ' + - 'but do not match, make them match or only set minSize'); + 'INVALID CONFIG: minSize and minNodes are both set ' + + 'but do not match, make them match or only set minSize', + ); } // if maxNodes or maxSize are provided set the other, and if both are set @@ -413,30 +447,36 @@ async function parseAndEnrichPayload(payload) { }); spanners[sIdx].maxSize = spanners[sIdx].maxNodes; } else if ( - spanners[sIdx].maxSize && spanners[sIdx].maxNodes && - spanners[sIdx].maxSize != spanners[sIdx].maxNodes) { + spanners[sIdx].maxSize && + spanners[sIdx].maxNodes && + spanners[sIdx].maxSize != spanners[sIdx].maxNodes + ) { throw new Error( - 'INVALID CONFIG: maxSize and maxNodes are both set ' + - 'but do not match, make them match or only set maxSize'); + 'INVALID CONFIG: maxSize and maxNodes are both set ' + + 'but do not match, make them match or only set maxSize', + ); } // at this point both minNodes/minSize and maxNodes/maxSize are matching // or are both not set so we can merge in defaults spanners[sIdx] = {...nodesDefaults, ...spanners[sIdx]}; } else { - throw new Error(`INVALID CONFIG: ${ - spanners[sIdx] - .units} is invalid. Valid values are NODES or PROCESSING_UNITS`); + throw new Error( + `INVALID CONFIG: ${spanners[sIdx].units} is invalid. Valid values are NODES or PROCESSING_UNITS`, + ); } // assemble the metrics - spanners[sIdx].metrics = - buildMetrics(spanners[sIdx].projectId, spanners[sIdx].instanceId); + spanners[sIdx].metrics = buildMetrics( + spanners[sIdx].projectId, + spanners[sIdx].instanceId, + ); // merge in custom thresholds if (metricOverrides != null) { for (let oIdx = 0; oIdx < metricOverrides.length; oIdx++) { const mIdx = spanners[sIdx].metrics.findIndex( - (x) => x.name === metricOverrides[oIdx].name); + (x) => x.name === metricOverrides[oIdx].name, + ); if (mIdx != -1) { spanners[sIdx].metrics[mIdx] = { ...spanners[sIdx].metrics[mIdx], @@ -445,13 +485,18 @@ async function parseAndEnrichPayload(payload) { } else { /** @type {SpannerMetric} */ const metric = {...metricDefaults, ...metricOverrides[oIdx]}; - if (validateCustomMetric( - metric, spanners[sIdx].projectId, - spanners[sIdx].instanceId)) { + if ( + validateCustomMetric( + metric, + spanners[sIdx].projectId, + spanners[sIdx].instanceId, + ) + ) { metric.filter = - createBaseFilter( - spanners[sIdx].projectId, spanners[sIdx].instanceId) + - metric.filter; + createBaseFilter( + spanners[sIdx].projectId, + spanners[sIdx].instanceId, + ) + metric.filter; spanners[sIdx].metrics.push(metric); } } @@ -462,15 +507,16 @@ async function parseAndEnrichPayload(payload) { try { spanners[sIdx] = { ...spanners[sIdx], - ...await getSpannerMetadata( - spanners[sIdx].projectId, spanners[sIdx].instanceId, - spanners[sIdx].units.toUpperCase()), + ...(await getSpannerMetadata( + spanners[sIdx].projectId, + spanners[sIdx].instanceId, + spanners[sIdx].units.toUpperCase(), + )), }; spannersFound.push(spanners[sIdx]); } catch (err) { logger.error({ - message: `Unable to retrieve Spanner metadata for ${ - spanners[sIdx].projectId}/${spanners[sIdx].instanceId}`, + message: `Unable to retrieve Spanner metadata for ${spanners[sIdx].projectId}/${spanners[sIdx].instanceId}`, projectId: spanners[sIdx].projectId, instanceId: spanners[sIdx].instanceId, payload: err, @@ -489,8 +535,7 @@ async function parseAndEnrichPayload(payload) { */ async function getMetrics(spanner) { logger.info({ - message: - `----- ${spanner.projectId}/${spanner.instanceId}: Getting Metrics -----`, + message: `----- ${spanner.projectId}/${spanner.instanceId}: Getting Metrics -----`, projectId: spanner.projectId, instanceId: spanner.instanceId, }); @@ -498,8 +543,11 @@ async function getMetrics(spanner) { const metrics = []; for (const m of spanner.metrics) { const metric = /** @type {SpannerMetric} */ (m); - const [maxMetricValue, maxLocation] = - await getMaxMetricValue(spanner.projectId, spanner.instanceId, metric); + const [maxMetricValue, maxLocation] = await getMaxMetricValue( + spanner.projectId, + spanner.instanceId, + metric, + ); let threshold; let margin; @@ -518,9 +566,10 @@ async function getMetrics(spanner) { } logger.debug({ - message: ` ${metric.name} = ${maxMetricValue}, threshold = ${ - threshold}, margin = ${margin}, location = ${maxLocation}`, - projectId: spanner.projectId, instanceId: spanner.instanceId}); + message: ` ${metric.name} = ${maxMetricValue}, threshold = ${threshold}, margin = ${margin}, location = ${maxLocation}`, + projectId: spanner.projectId, + instanceId: spanner.instanceId, + }); /** @type {SpannerMetricValue} */ const metricsObject = { @@ -550,8 +599,7 @@ async function forwardMetrics(forwarderFunction, spanners) { await Counters.incPollingSuccessCounter(spanner); } catch (err) { logger.error({ - message: `Unable to retrieve metrics for ${spanner.projectId}/${ - spanner.instanceId}`, + message: `Unable to retrieve metrics for ${spanner.projectId}/${spanner.instanceId}`, projectId: spanner.projectId, instanceId: spanner.instanceId, err: err, @@ -559,7 +607,7 @@ async function forwardMetrics(forwarderFunction, spanners) { await Counters.incPollingFailedCounter(spanner); } } -}; +} /** * Aggregate metrics for a List of spanner config @@ -576,8 +624,7 @@ async function aggregateMetrics(spanners) { await Counters.incPollingSuccessCounter(spanner); } catch (err) { logger.error({ - message: `Unable to retrieve metrics for ${spanner.projectId}/${ - spanner.instanceId}`, + message: `Unable to retrieve metrics for ${spanner.projectId}/${spanner.instanceId}`, projectId: spanner.projectId, instanceId: spanner.instanceId, err: err, @@ -586,8 +633,7 @@ async function aggregateMetrics(spanners) { } } return aggregatedMetrics; -}; - +} /** * Handle a PubSub message and check if scaling is required @@ -602,26 +648,29 @@ async function checkSpannerScaleMetricsPubSub(pubSubEvent, context) { const spanners = await parseAndEnrichPayload(payload); logger.debug({ message: 'Autoscaler poller started (PubSub).', - payload: spanners}); + payload: spanners, + }); await forwardMetrics(postPubSubMessage, spanners); await Counters.incRequestsSuccessCounter(); } catch (err) { logger.error({ message: `An error occurred in the Autoscaler poller function (PubSub)`, - err: err}); + err: err, + }); logger.error({message: `JSON payload`, payload: payload}); await Counters.incRequestsFailedCounter(); } } catch (err) { logger.error({ message: `An error occurred in the Autoscaler poller function (PubSub)`, - err: err}); + err: err, + }); logger.error({message: `Pubsub data`, payload: pubSubEvent.data}); await Counters.incRequestsFailedCounter(); } finally { await Counters.tryFlush(); } -}; +} /** * For testing with: https://cloud.google.com/functions/docs/functions-framework @@ -630,15 +679,15 @@ async function checkSpannerScaleMetricsPubSub(pubSubEvent, context) { */ async function checkSpannerScaleMetricsHTTP(req, res) { const payload = - '[{ '+ - ' "projectId": "spanner-scaler", '+ - ' "instanceId": "autoscale-test", '+ - ' "scalerPubSubTopic": '+ - ' "projects/spanner-scaler/topics/test-scaling", '+ - ' "minNodes": 1, '+ - ' "maxNodes": 3, '+ - ' "stateProjectId" : "spanner-scaler"'+ - '}]'; + '[{ ' + + ' "projectId": "spanner-scaler", ' + + ' "instanceId": "autoscale-test", ' + + ' "scalerPubSubTopic": ' + + ' "projects/spanner-scaler/topics/test-scaling", ' + + ' "minNodes": 1, ' + + ' "maxNodes": 3, ' + + ' "stateProjectId" : "spanner-scaler"' + + '}]'; try { const spanners = await parseAndEnrichPayload(payload); await forwardMetrics(postPubSubMessage, spanners); @@ -647,7 +696,8 @@ async function checkSpannerScaleMetricsHTTP(req, res) { } catch (err) { logger.error({ message: `An error occurred in the Autoscaler poller function (HTTP)`, - err: err}); + err: err, + }); logger.error({message: `JSON payload`, payload: payload}); res.status(500).end(err.toString()); res.end(err.toString()); @@ -655,7 +705,7 @@ async function checkSpannerScaleMetricsHTTP(req, res) { } finally { await Counters.tryFlush(); } -}; +} /** * HTTP test @@ -667,20 +717,21 @@ async function checkSpannerScaleMetricsJSON(payload) { const spanners = await parseAndEnrichPayload(payload); logger.debug({ message: 'Autoscaler poller started (JSON/HTTP).', - payload: spanners}); + payload: spanners, + }); await forwardMetrics(callScalerHTTP, spanners); await Counters.incRequestsSuccessCounter(); } catch (err) { logger.error({ - message: - `An error occurred in the Autoscaler poller function (JSON/HTTP)`, - err: err}); + message: `An error occurred in the Autoscaler poller function (JSON/HTTP)`, + err: err, + }); logger.error({message: `JSON payload`, payload: payload}); await Counters.incRequestsFailedCounter(); } finally { await Counters.tryFlush(); } -}; +} /** * Entrypoint for Local config. @@ -693,21 +744,22 @@ async function checkSpannerScaleMetricsLocal(payload) { const spanners = await parseAndEnrichPayload(payload); logger.debug({ message: 'Autoscaler poller started (JSON/local).', - payload: spanners}); + payload: spanners, + }); const metrics = await aggregateMetrics(spanners); await Counters.incRequestsSuccessCounter(); return metrics; } catch (err) { logger.error({ - message: - `An error occurred in the Autoscaler poller function (JSON/Local)`, - err: err}); + message: `An error occurred in the Autoscaler poller function (JSON/Local)`, + err: err, + }); logger.error({message: `JSON payload`, payload: payload}); await Counters.incRequestsFailedCounter(); } finally { await Counters.tryFlush(); } -}; +} module.exports = { checkSpannerScaleMetricsPubSub, diff --git a/src/poller/poller-core/package.json b/src/poller/poller-core/package.json index 4737ca4f..7622671b 100644 --- a/src/poller/poller-core/package.json +++ b/src/poller/poller-core/package.json @@ -1,27 +1,27 @@ { - "name": "poller-core", - "description": "Polls Spanner metrics with Cloud Monitoring", - "license": "Apache-2.0", - "author": "Google Inc.", - "dependencies": { - "@google-cloud/functions-framework": "^3.3.0", - "@google-cloud/monitoring": "^4.0.0", - "@google-cloud/pubsub": "^4.3.3", - "@google-cloud/spanner": "^6.16.0", - "autoscaler-common": "file:../../autoscaler-common", - "axios": "^1.6.7", - "express": "^4.18.3" - }, - "scripts": { - "start": "functions-framework --target=checkSpannerScaleMetricsHTTP", - "debug": "node --inspect node_modules/.bin/functions-framework --target=checkSpannerScaleMetricsHTTP", - "test": "NODE_ENV=test nyc --reporter=text mocha --recursive" - }, - "devDependencies": { - "mocha": "^10.3.0", - "nyc": "^15.1.0", - "rewire": "^7.0.0", - "should": "^13.2.3", - "sinon": "^17.0.1" - } + "name": "poller-core", + "description": "Polls Spanner metrics with Cloud Monitoring", + "license": "Apache-2.0", + "author": "Google Inc.", + "dependencies": { + "@google-cloud/functions-framework": "^3.3.0", + "@google-cloud/monitoring": "^4.0.0", + "@google-cloud/pubsub": "^4.3.3", + "@google-cloud/spanner": "^6.16.0", + "autoscaler-common": "file:../../autoscaler-common", + "axios": "^1.6.7", + "express": "^4.18.3" + }, + "scripts": { + "start": "functions-framework --target=checkSpannerScaleMetricsHTTP", + "debug": "node --inspect node_modules/.bin/functions-framework --target=checkSpannerScaleMetricsHTTP", + "test": "NODE_ENV=test nyc --reporter=text mocha --recursive" + }, + "devDependencies": { + "mocha": "^10.3.0", + "nyc": "^15.1.0", + "rewire": "^7.0.0", + "should": "^13.2.3", + "sinon": "^17.0.1" + } } diff --git a/src/poller/poller-core/test/index.test.js b/src/poller/poller-core/test/index.test.js index 2460c876..612ff546 100644 --- a/src/poller/poller-core/test/index.test.js +++ b/src/poller/poller-core/test/index.test.js @@ -37,36 +37,46 @@ describe('#buildMetrics', () => { it('should insert the projectId', () => { buildMetrics('fakeProjectId', 'fakeInstanceId')[0].filter.should.have.match( - /fakeProjectId/); + /fakeProjectId/, + ); }); it('should insert the instanceId', () => { buildMetrics('fakeProjectId', 'fakeInstanceId')[2].filter.should.have.match( - /fakeInstanceId/); + /fakeInstanceId/, + ); }); }); describe('#validateCustomMetric', () => { it('should return false if name is missing', () => { - validateCustomMetric({filter: 'my filter', regional_threshold: 10}) - .should.be.false(); + validateCustomMetric({ + filter: 'my filter', + regional_threshold: 10, + }).should.be.false(); }); it('should return false if filter is blank', () => { - validateCustomMetric( - {name: 'custom_filter', filter: '', regional_threshold: 10}) - .should.be.false(); + validateCustomMetric({ + name: 'custom_filter', + filter: '', + regional_threshold: 10, + }).should.be.false(); }); it('should return false if thresholds are missing', () => { - validateCustomMetric({name: 'custom_filter', filter: 'my filter'}) - .should.be.false(); + validateCustomMetric({ + name: 'custom_filter', + filter: 'my filter', + }).should.be.false(); }); it('should return false if thresholds are less than equal to 0', () => { - validateCustomMetric( - {name: 'custom_filter', filter: 'my filter', regional_threshold: 0}) - .should.be.false(); + validateCustomMetric({ + name: 'custom_filter', + filter: 'my filter', + regional_threshold: 0, + }).should.be.false(); }); it('should return true all fields are present and valid', () => { @@ -80,185 +90,192 @@ describe('#validateCustomMetric', () => { describe('#parseAndEnrichPayload', () => { it('should return the default for stepSize', async () => { - const payload = '[{' + - ' "projectId": "my-spanner-project", ' + - ' "instanceId": "spanner1", ' + - ' "scalerPubSubTopic": "spanner-scaling", ' + - ' "minNodes": 10' + - '}]'; + const payload = + '[{' + + ' "projectId": "my-spanner-project", ' + + ' "instanceId": "spanner1", ' + + ' "scalerPubSubTopic": "spanner-scaling", ' + + ' "minNodes": 10' + + '}]'; const stub = sinon.stub().resolves({currentNode: 5, regional: true}); const unset = app.__set__('getSpannerMetadata', stub); const mergedConfig = await parseAndEnrichPayload(payload); - (mergedConfig[0].stepSize).should.equal(2); + mergedConfig[0].stepSize.should.equal(2); unset(); }); it('should override the default for minNodes', async () => { - const payload = '[{' + - ' "projectId": "my-spanner-project",' + - ' "instanceId": "spanner1",' + - ' "scalerPubSubTopic": "spanner-scaling",' + - ' "minNodes": 10 ' + - '}]'; + const payload = + '[{' + + ' "projectId": "my-spanner-project",' + + ' "instanceId": "spanner1",' + + ' "scalerPubSubTopic": "spanner-scaling",' + + ' "minNodes": 10 ' + + '}]'; const stub = sinon.stub().resolves({currentNode: 5, regional: true}); const unset = app.__set__('getSpannerMetadata', stub); const mergedConfig = await parseAndEnrichPayload(payload); - (mergedConfig[0].units).should.equal('NODES'); - (mergedConfig[0].minSize).should.equal(10); + mergedConfig[0].units.should.equal('NODES'); + mergedConfig[0].minSize.should.equal(10); unset(); }); it('should merge in defaults for processing units', async () => { - const payload = '[{' + - ' "projectId": "my-spanner-project", ' + - ' "instanceId": "spanner1", ' + - ' "scalerPubSubTopic": "spanner-scaling", ' + - ' "units": "PROCESSING_UNITS", ' + - ' "minSize": 200' + - '}]'; + const payload = + '[{' + + ' "projectId": "my-spanner-project", ' + + ' "instanceId": "spanner1", ' + + ' "scalerPubSubTopic": "spanner-scaling", ' + + ' "units": "PROCESSING_UNITS", ' + + ' "minSize": 200' + + '}]'; const stub = sinon.stub().resolves({currentSize: 500, regional: true}); const unset = app.__set__('getSpannerMetadata', stub); const mergedConfig = await parseAndEnrichPayload(payload); - (mergedConfig[0].minSize).should.equal(200); - (mergedConfig[0].maxSize).should.equal(2000); - (mergedConfig[0].stepSize).should.equal(200); + mergedConfig[0].minSize.should.equal(200); + mergedConfig[0].maxSize.should.equal(2000); + mergedConfig[0].stepSize.should.equal(200); const idx = mergedConfig[0].metrics.findIndex((x) => x.name === 'minNodes'); idx.should.equal(-1); unset(); }); - it('should use the value of minSize/maxSize for minNodes/maxNodes instead of overriding with the defaults, Github Issue 61', - async () => { - const payload = '[{' + - ' "projectId": "my-spanner-project", ' + - ' "instanceId": "spanner1", ' + - ' "scalerPubSubTopic": "spanner-scaling", ' + - ' "units": "NODES", ' + - ' "minSize": 20, ' + - ' "maxSize": 50 ' + - '}]'; - - const stub = sinon.stub().resolves({currentSize: 50, regional: true}); - const unset = app.__set__('getSpannerMetadata', stub); - - const mergedConfig = await parseAndEnrichPayload(payload); - (mergedConfig[0].minSize).should.equal(20); - (mergedConfig[0].maxSize).should.equal(50); - - unset(); - }); - - it('should throw if the nodes are specified when units is set to processing units', - async () => { - const payload = '[{' + - ' "projectId": "my-spanner-project", ' + - ' "instanceId": "spanner1", ' + - ' "scalerPubSubTopic": "spanner-scaling", ' + - ' "units": "PROCESSING_UNITS", ' + - ' "minNodes": 200' + - '}]'; - - const stub = sinon.stub().resolves({currentSize: 500, regional: true}); - const unset = app.__set__('getSpannerMetadata', stub); - - await parseAndEnrichPayload(payload).should.be.rejectedWith(Error, { - message: 'INVALID CONFIG: units is set to PROCESSING_UNITS, ' + - 'however, minNodes or maxNodes is set, ' + - 'remove minNodes and maxNodes from your configuration.', - }); - - unset(); - }); - - it('should throw if the nodes are specified when but minSize and minNodes are both provided but not matching', - async () => { - const payload = '[{' + - ' "projectId": "my-spanner-project", ' + - ' "instanceId": "spanner1", ' + - ' "scalerPubSubTopic": "spanner-scaling", ' + - ' "units": "NODES", ' + - ' "minNodes": 20, ' + - ' "minSize": 5' + - '}]'; - - const stub = sinon.stub().resolves({currentSize: 50, regional: true}); - const unset = app.__set__('getSpannerMetadata', stub); - - await parseAndEnrichPayload(payload).should.be.rejectedWith(Error, { - message: 'INVALID CONFIG: minSize and minNodes are both set ' + - 'but do not match, make them match or only set minSize', - }); - - unset(); - }); - - it('should throw if the nodes are specified when but maxSize and maxNodes are both provided but not matching', - async () => { - const payload = '[{' + - ' "projectId": "my-spanner-project", ' + - ' "instanceId": "spanner1", ' + - ' "scalerPubSubTopic": "spanner-scaling", ' + - ' "units": "NODES", ' + - ' "maxNodes": 20, ' + - ' "maxSize": 5' + - '}]'; - - const stub = sinon.stub().resolves({currentSize: 50, regional: true}); - const unset = app.__set__('getSpannerMetadata', stub); - - await parseAndEnrichPayload(payload).should.be.rejectedWith(Error, { - message: 'INVALID CONFIG: maxSize and maxNodes are both set ' + - 'but do not match, make them match or only set maxSize', - }); - - unset(); - }); - - it('should override the regional threshold for storage but not high_priority_cpu', - async () => { - const payload = '[{' + - ' "projectId": "my-spanner-project", ' + - ' "instanceId": "spanner1", ' + - ' "scalerPubSubTopic": "spanner-scaling", ' + - ' "minNodes": 10, ' + - ' "metrics": [{"name": "storage", "regional_threshold":10}]' + - '}]'; - - const stub = sinon.stub().resolves({currentNode: 5, regional: true}); - const unset = app.__set__('getSpannerMetadata', stub); - - const mergedConfig = await parseAndEnrichPayload(payload); - - let idx = mergedConfig[0].metrics.findIndex( - (x) => x.name === 'storage'); - (mergedConfig[0].metrics[idx].regional_threshold).should.equal(10); - idx = mergedConfig[0].metrics.findIndex( - (x) => x.name === 'high_priority_cpu'); - (mergedConfig[0].metrics[idx].regional_threshold).should.equal(65); - - unset(); - }); + it('should use the value of minSize/maxSize for minNodes/maxNodes instead of overriding with the defaults, Github Issue 61', async () => { + const payload = + '[{' + + ' "projectId": "my-spanner-project", ' + + ' "instanceId": "spanner1", ' + + ' "scalerPubSubTopic": "spanner-scaling", ' + + ' "units": "NODES", ' + + ' "minSize": 20, ' + + ' "maxSize": 50 ' + + '}]'; + + const stub = sinon.stub().resolves({currentSize: 50, regional: true}); + const unset = app.__set__('getSpannerMetadata', stub); + + const mergedConfig = await parseAndEnrichPayload(payload); + mergedConfig[0].minSize.should.equal(20); + mergedConfig[0].maxSize.should.equal(50); + + unset(); + }); + + it('should throw if the nodes are specified when units is set to processing units', async () => { + const payload = + '[{' + + ' "projectId": "my-spanner-project", ' + + ' "instanceId": "spanner1", ' + + ' "scalerPubSubTopic": "spanner-scaling", ' + + ' "units": "PROCESSING_UNITS", ' + + ' "minNodes": 200' + + '}]'; + + const stub = sinon.stub().resolves({currentSize: 500, regional: true}); + const unset = app.__set__('getSpannerMetadata', stub); + + await parseAndEnrichPayload(payload).should.be.rejectedWith(Error, { + message: + 'INVALID CONFIG: units is set to PROCESSING_UNITS, ' + + 'however, minNodes or maxNodes is set, ' + + 'remove minNodes and maxNodes from your configuration.', + }); + + unset(); + }); + + it('should throw if the nodes are specified when but minSize and minNodes are both provided but not matching', async () => { + const payload = + '[{' + + ' "projectId": "my-spanner-project", ' + + ' "instanceId": "spanner1", ' + + ' "scalerPubSubTopic": "spanner-scaling", ' + + ' "units": "NODES", ' + + ' "minNodes": 20, ' + + ' "minSize": 5' + + '}]'; + + const stub = sinon.stub().resolves({currentSize: 50, regional: true}); + const unset = app.__set__('getSpannerMetadata', stub); + + await parseAndEnrichPayload(payload).should.be.rejectedWith(Error, { + message: + 'INVALID CONFIG: minSize and minNodes are both set ' + + 'but do not match, make them match or only set minSize', + }); + + unset(); + }); + + it('should throw if the nodes are specified when but maxSize and maxNodes are both provided but not matching', async () => { + const payload = + '[{' + + ' "projectId": "my-spanner-project", ' + + ' "instanceId": "spanner1", ' + + ' "scalerPubSubTopic": "spanner-scaling", ' + + ' "units": "NODES", ' + + ' "maxNodes": 20, ' + + ' "maxSize": 5' + + '}]'; + + const stub = sinon.stub().resolves({currentSize: 50, regional: true}); + const unset = app.__set__('getSpannerMetadata', stub); + + await parseAndEnrichPayload(payload).should.be.rejectedWith(Error, { + message: + 'INVALID CONFIG: maxSize and maxNodes are both set ' + + 'but do not match, make them match or only set maxSize', + }); + + unset(); + }); + + it('should override the regional threshold for storage but not high_priority_cpu', async () => { + const payload = + '[{' + + ' "projectId": "my-spanner-project", ' + + ' "instanceId": "spanner1", ' + + ' "scalerPubSubTopic": "spanner-scaling", ' + + ' "minNodes": 10, ' + + ' "metrics": [{"name": "storage", "regional_threshold":10}]' + + '}]'; + + const stub = sinon.stub().resolves({currentNode: 5, regional: true}); + const unset = app.__set__('getSpannerMetadata', stub); + + const mergedConfig = await parseAndEnrichPayload(payload); + + let idx = mergedConfig[0].metrics.findIndex((x) => x.name === 'storage'); + mergedConfig[0].metrics[idx].regional_threshold.should.equal(10); + idx = mergedConfig[0].metrics.findIndex( + (x) => x.name === 'high_priority_cpu', + ); + mergedConfig[0].metrics[idx].regional_threshold.should.equal(65); + + unset(); + }); it('should override the multiple thresholds', async () => { - const payload = '[{' + - ' "projectId": "my-spanner-project", ' + - ' "instanceId": "spanner1", ' + - ' "scalerPubSubTopic": "spanner-scaling", ' + - ' "minNodes": 10, ' + - ' "metrics": [' + - ' {"name": "high_priority_cpu", "multi_regional_threshold":20}, ' + - ' {"name": "storage", "regional_threshold":10}' + - ' ]' + - '}]'; + const payload = + '[{' + + ' "projectId": "my-spanner-project", ' + + ' "instanceId": "spanner1", ' + + ' "scalerPubSubTopic": "spanner-scaling", ' + + ' "minNodes": 10, ' + + ' "metrics": [' + + ' {"name": "high_priority_cpu", "multi_regional_threshold":20}, ' + + ' {"name": "storage", "regional_threshold":10}' + + ' ]' + + '}]'; const stub = sinon.stub().resolves({currentNode: 5, regional: true}); const unset = app.__set__('getSpannerMetadata', stub); @@ -266,81 +283,80 @@ describe('#parseAndEnrichPayload', () => { const mergedConfig = await parseAndEnrichPayload(payload); let idx = mergedConfig[0].metrics.findIndex((x) => x.name === 'storage'); - (mergedConfig[0].metrics[idx].regional_threshold).should.equal(10); + mergedConfig[0].metrics[idx].regional_threshold.should.equal(10); idx = mergedConfig[0].metrics.findIndex( - (x) => x.name === 'high_priority_cpu'); - (mergedConfig[0].metrics[idx].multi_regional_threshold).should.equal(20); + (x) => x.name === 'high_priority_cpu', + ); + mergedConfig[0].metrics[idx].multi_regional_threshold.should.equal(20); + + unset(); + }); + + it('should add a custom metric to the list if metric name is a default metric', async () => { + const payload = + '[{' + + ' "projectId": "my-spanner-project", ' + + ' "instanceId": "spanner1", ' + + ' "scalerPubSubTopic": "spanner-scaling", ' + + ' "minNodes": 10, ' + + ' "metrics": [' + + ' {' + + ' "filter": "my super cool filter", ' + + ' "name": "bogus", ' + + ' "multi_regional_threshold":20' + + ' }' + + ' ]' + + '}]'; + + const stub = sinon.stub().resolves({currentNode: 5, regional: true}); + const unset = app.__set__('getSpannerMetadata', stub); + + const mergedConfig = await parseAndEnrichPayload(payload); + const idx = mergedConfig[0].metrics.findIndex((x) => x.name === 'bogus'); + mergedConfig[0].metrics[idx].multi_regional_threshold.should.equal(20); + unset(); + }); + + it('should not add a custom metric to the list if the provided metric is not valid', async () => { + const payload = + '[{' + + ' "projectId": "my-spanner-project",' + + ' "instanceId": "spanner1",' + + ' "scalerPubSubTopic": "spanner-scaling", ' + + ' "minNodes": 10, ' + + ' "metrics": [' + + ' {"filter": "my super cool filter", "name": "bogus"}' + + ' ]' + + '}]'; + + const stub = sinon.stub().resolves({currentNode: 5, regional: true}); + const unset = app.__set__('getSpannerMetadata', stub); + const mergedConfig = await parseAndEnrichPayload(payload); + const idx = mergedConfig[0].metrics.findIndex((x) => x.name === 'bogus'); + idx.should.equal(-1); unset(); }); - it('should add a custom metric to the list if metric name is a default metric', - async () => { - const payload = '[{' + - ' "projectId": "my-spanner-project", ' + - ' "instanceId": "spanner1", ' + - ' "scalerPubSubTopic": "spanner-scaling", ' + - ' "minNodes": 10, ' + - ' "metrics": [' + - ' {' + - ' "filter": "my super cool filter", ' + - ' "name": "bogus", ' + - ' "multi_regional_threshold":20' + - ' }' + - ' ]' + - '}]'; - - const stub = sinon.stub().resolves({currentNode: 5, regional: true}); - const unset = app.__set__('getSpannerMetadata', stub); - - const mergedConfig = await parseAndEnrichPayload(payload); - const idx = mergedConfig[0].metrics.findIndex( - (x) => x.name === 'bogus'); - (mergedConfig[0].metrics[idx].multi_regional_threshold) - .should.equal(20); - unset(); - }); - - it('should not add a custom metric to the list if the provided metric is not valid', - async () => { - const payload = '[{' + - ' "projectId": "my-spanner-project",' + - ' "instanceId": "spanner1",' + - ' "scalerPubSubTopic": "spanner-scaling", ' + - ' "minNodes": 10, ' + - ' "metrics": [' + - ' {"filter": "my super cool filter", "name": "bogus"}' + - ' ]' + - '}]'; - - const stub = sinon.stub().resolves({currentNode: 5, regional: true}); - const unset = app.__set__('getSpannerMetadata', stub); - - const mergedConfig = await parseAndEnrichPayload(payload); - const idx = mergedConfig[0].metrics.findIndex( - (x) => x.name === 'bogus'); - idx.should.equal(-1); - unset(); - }); - - it('should throw if the nodes are specified if units is set something other than nodes or processing units', - async () => { - const payload = '[{' + - ' "projectId": "my-spanner-project",' + - ' "instanceId": "spanner1",' + - ' "scalerPubSubTopic": "spanner-scaling", ' + - ' "units": "BOGUS", ' + - ' "minNodes": 200' + - '}]'; - - const stub = sinon.stub().resolves({currentSize: 500, regional: true}); - const unset = app.__set__('getSpannerMetadata', stub); - - await parseAndEnrichPayload(payload).should.be.rejectedWith(Error, { - message: 'INVALID CONFIG: BOGUS is invalid. ' + - 'Valid values are NODES or PROCESSING_UNITS', - }); - - unset(); - }); + it('should throw if the nodes are specified if units is set something other than nodes or processing units', async () => { + const payload = + '[{' + + ' "projectId": "my-spanner-project",' + + ' "instanceId": "spanner1",' + + ' "scalerPubSubTopic": "spanner-scaling", ' + + ' "units": "BOGUS", ' + + ' "minNodes": 200' + + '}]'; + + const stub = sinon.stub().resolves({currentSize: 500, regional: true}); + const unset = app.__set__('getSpannerMetadata', stub); + + await parseAndEnrichPayload(payload).should.be.rejectedWith(Error, { + message: + 'INVALID CONFIG: BOGUS is invalid. ' + + 'Valid values are NODES or PROCESSING_UNITS', + }); + + unset(); + }); }); diff --git a/src/scaler/index.js b/src/scaler/index.js index cf28a6f2..2b7df6df 100644 --- a/src/scaler/index.js +++ b/src/scaler/index.js @@ -41,7 +41,8 @@ function main() { } catch (err) { logger.error({ message: 'Error in Scaler wrapper:', - err: err}); + err: err, + }); } } diff --git a/src/scaler/package.json b/src/scaler/package.json index 4dd6e7b6..eeeff090 100644 --- a/src/scaler/package.json +++ b/src/scaler/package.json @@ -1,14 +1,14 @@ { - "name": "scaler", - "description": "Spanner instance scaler", - "license": "Apache-2.0", - "author": "Google Inc.", - "dependencies": { - "autoscaler-common": "file:../autoscaler-common", - "express": "^4.18.2", - "scaler-core": "file:scaler-core" - }, - "scripts": { - "start": "node -e \"require('./index').main()\"" - } + "name": "scaler", + "description": "Spanner instance scaler", + "license": "Apache-2.0", + "author": "Google Inc.", + "dependencies": { + "autoscaler-common": "file:../autoscaler-common", + "express": "^4.18.2", + "scaler-core": "file:scaler-core" + }, + "scripts": { + "start": "node -e \"require('./index').main()\"" + } } diff --git a/src/scaler/scaler-core/counters.js b/src/scaler/scaler-core/counters.js index 955edb71..12fda612 100644 --- a/src/scaler/scaler-core/counters.js +++ b/src/scaler/scaler-core/counters.js @@ -40,16 +40,16 @@ const ATTRIBUTE_NAMES = { /** * @typedef {import('../../autoscaler-common/types.js') -* .AutoscalerSpanner} AutoscalerSpanner -*/ + * .AutoscalerSpanner} AutoscalerSpanner + */ /** * @typedef {import('@opentelemetry/api').Attributes} Attributes */ /** -* @type {import('../../autoscaler-common/counters_base.js') -* .CounterDefinition[]} -*/ + * @type {import('../../autoscaler-common/counters_base.js') + * .CounterDefinition[]} + */ const COUNTERS = [ { counterName: COUNTER_NAMES.SCALING_SUCCESS, @@ -89,12 +89,14 @@ function _getCounterAttributes(spanner, requestedSize) { [ATTRIBUTE_NAMES.SPANNER_INSTANCE_ID]: spanner.instanceId, [ATTRIBUTE_NAMES.SCALING_METHOD]: spanner.scalingMethod, [ATTRIBUTE_NAMES.SCALING_DIRECTION]: - requestedSize > spanner.currentSize ? 'SCALE_UP' : - requestedSize < spanner.currentSize ? 'SCALE_DOWN' : 'SCALE_SAME', + requestedSize > spanner.currentSize + ? 'SCALE_UP' + : requestedSize < spanner.currentSize + ? 'SCALE_DOWN' + : 'SCALE_SAME', }; } - /** * Increment scaling success counter * @@ -103,8 +105,10 @@ function _getCounterAttributes(spanner, requestedSize) { */ async function incScalingSuccessCounter(spanner, requestedSize) { await pendingInit; - CountersBase.incCounter(COUNTER_NAMES.SCALING_SUCCESS, - _getCounterAttributes(spanner, requestedSize)); + CountersBase.incCounter( + COUNTER_NAMES.SCALING_SUCCESS, + _getCounterAttributes(spanner, requestedSize), + ); } /** @@ -115,8 +119,10 @@ async function incScalingSuccessCounter(spanner, requestedSize) { */ async function incScalingFailedCounter(spanner, requestedSize) { await pendingInit; - CountersBase.incCounter(COUNTER_NAMES.SCALING_FAILED, - _getCounterAttributes(spanner, requestedSize)); + CountersBase.incCounter( + COUNTER_NAMES.SCALING_FAILED, + _getCounterAttributes(spanner, requestedSize), + ); } /** @@ -150,7 +156,6 @@ async function incRequestsFailedCounter() { CountersBase.incCounter(COUNTER_NAMES.REQUESTS_FAILED); } - module.exports = { incScalingSuccessCounter, incScalingFailedCounter, diff --git a/src/scaler/scaler-core/index.js b/src/scaler/scaler-core/index.js index 24c25e6d..6f2483d7 100644 --- a/src/scaler/scaler-core/index.js +++ b/src/scaler/scaler-core/index.js @@ -41,7 +41,6 @@ const {AutoscalerUnits} = require('../../autoscaler-common/types'); * @typedef {import('./state.js').StateData} StateData */ - /** * Get scaling method function by name. * @@ -66,14 +65,19 @@ function getScalingMethod(methodName, projectId, instanceId) { } catch (err) { logger.warn({ message: `Unknown scaling method '${methodName}'`, - projectId: projectId, instanceId: instanceId}); - scalingMethod = - require(SCALING_METHODS_FOLDER + DEFAULT_METHOD_NAME.toLowerCase()); + projectId: projectId, + instanceId: instanceId, + }); + scalingMethod = require( + SCALING_METHODS_FOLDER + DEFAULT_METHOD_NAME.toLowerCase(), + ); methodName = DEFAULT_METHOD_NAME; } logger.info({ message: `Using scaling method: ${methodName}`, - projectId: projectId, instanceId: instanceId}); + projectId: projectId, + instanceId: instanceId, + }); return scalingMethod; } @@ -85,9 +89,10 @@ function getScalingMethod(methodName, projectId, instanceId) { * @return {spannerProtos.google.spanner.admin.instance.v1.IInstance}} */ function getNewMetadata(suggestedSize, units) { - const metadata = (units === AutoscalerUnits.NODES) ? - {nodeCount: suggestedSize} : - {processingUnits: suggestedSize}; + const metadata = + units === AutoscalerUnits.NODES + ? {nodeCount: suggestedSize} + : {processingUnits: suggestedSize}; // For testing: // metadata = { displayName : 'a' + Math.floor(Math.random() * 100) + '_' + @@ -104,9 +109,7 @@ function getNewMetadata(suggestedSize, units) { */ async function scaleSpannerInstance(spanner, suggestedSize) { logger.info({ - message: `----- ${spanner.projectId}/${ - spanner.instanceId}: Scaling Spanner instance to ${suggestedSize} ${ - spanner.units} -----`, + message: `----- ${spanner.projectId}/${spanner.instanceId}: Scaling Spanner instance to ${suggestedSize} ${spanner.units} -----`, projectId: spanner.projectId, instanceId: spanner.instanceId, }); @@ -117,15 +120,17 @@ async function scaleSpannerInstance(spanner, suggestedSize) { userAgent: 'cloud-solutions/spanner-autoscaler-scaler-usage-v1.0', }); - return spannerClient.instance(spanner.instanceId) - .setMetadata(getNewMetadata(suggestedSize, spanner.units)) - .then(function(data) { - const operation = data[0]; - logger.debug({ - message: - `Cloud Spanner started the scaling operation: ${operation.name}`, - projectId: spanner.projectId, instanceId: spanner.instanceId}); + return spannerClient + .instance(spanner.instanceId) + .setMetadata(getNewMetadata(suggestedSize, spanner.units)) + .then(function (data) { + const operation = data[0]; + logger.debug({ + message: `Cloud Spanner started the scaling operation: ${operation.name}`, + projectId: spanner.projectId, + instanceId: spanner.instanceId, }); + }); } /** @@ -147,7 +152,10 @@ async function publishDownstreamEvent(eventName, spanner, suggestedSize) { }; return publishProtoMsgDownstream( - eventName, message, spanner.downstreamPubSubTopic); + eventName, + message, + spanner.downstreamPubSubTopic, + ); } /** @@ -161,33 +169,33 @@ async function publishDownstreamEvent(eventName, spanner, suggestedSize) { */ function withinCooldownPeriod(spanner, suggestedSize, autoscalerState, now) { const MS_IN_1_MIN = 60000; - const scaleOutSuggested = (suggestedSize - spanner.currentSize > 0); + const scaleOutSuggested = suggestedSize - spanner.currentSize > 0; let cooldownPeriodOver; let duringOverload = ''; logger.debug({ - message: `----- ${spanner.projectId}/${ - spanner.instanceId}: Verifying if scaling is allowed -----`, + message: `----- ${spanner.projectId}/${spanner.instanceId}: Verifying if scaling is allowed -----`, projectId: spanner.projectId, - instanceId: spanner.instanceId}); - const operation = - (scaleOutSuggested ? - { - description: 'scale out', - lastScalingMillisec: autoscalerState.lastScalingTimestamp, - coolingMillisec: spanner.scaleOutCoolingMinutes * MS_IN_1_MIN, - } : - { - description: 'scale in', - lastScalingMillisec: autoscalerState.lastScalingTimestamp, - coolingMillisec: spanner.scaleInCoolingMinutes * MS_IN_1_MIN, - }); + instanceId: spanner.instanceId, + }); + const operation = scaleOutSuggested + ? { + description: 'scale out', + lastScalingMillisec: autoscalerState.lastScalingTimestamp, + coolingMillisec: spanner.scaleOutCoolingMinutes * MS_IN_1_MIN, + } + : { + description: 'scale in', + lastScalingMillisec: autoscalerState.lastScalingTimestamp, + coolingMillisec: spanner.scaleInCoolingMinutes * MS_IN_1_MIN, + }; if (spanner.isOverloaded) { if (spanner.overloadCoolingMinutes == null) { spanner.overloadCoolingMinutes = spanner.scaleOutCoolingMinutes; logger.info({ - message: '\tNo cooldown period defined for overload situations. ' + + message: + '\tNo cooldown period defined for overload situations. ' + `Using default: ${spanner.scaleOutCoolingMinutes} minutes`, projectId: spanner.projectId, instanceId: spanner.instanceId, @@ -200,25 +208,27 @@ function withinCooldownPeriod(spanner, suggestedSize, autoscalerState, now) { if (operation.lastScalingMillisec == 0) { cooldownPeriodOver = true; logger.debug({ - message: - `\tNo previous scaling operation found for this Spanner instance`, + message: `\tNo previous scaling operation found for this Spanner instance`, projectId: spanner.projectId, instanceId: spanner.instanceId, }); } else { const elapsedMillisec = now - operation.lastScalingMillisec; - cooldownPeriodOver = (elapsedMillisec >= operation.coolingMillisec); + cooldownPeriodOver = elapsedMillisec >= operation.coolingMillisec; logger.debug({ - message: `\tLast scaling operation was ${ - convertMillisecToHumanReadable( - now - operation.lastScalingMillisec)} ago.`, + message: `\tLast scaling operation was ${convertMillisecToHumanReadable( + now - operation.lastScalingMillisec, + )} ago.`, projectId: spanner.projectId, - instanceId: spanner.instanceId}); + instanceId: spanner.instanceId, + }); logger.debug({ - message: `\tCooldown period for ${operation.description}${ - duringOverload} is ${ - convertMillisecToHumanReadable(operation.coolingMillisec)}.`, - projectId: spanner.projectId, instanceId: spanner.instanceId}); + message: `\tCooldown period for ${operation.description}${duringOverload} is ${convertMillisecToHumanReadable( + operation.coolingMillisec, + )}.`, + projectId: spanner.projectId, + instanceId: spanner.instanceId, + }); } if (cooldownPeriodOver) { @@ -245,16 +255,21 @@ function withinCooldownPeriod(spanner, suggestedSize, autoscalerState, now) { */ function getSuggestedSize(spanner) { const scalingMethod = getScalingMethod( - spanner.scalingMethod, spanner.projectId, spanner.instanceId); + spanner.scalingMethod, + spanner.projectId, + spanner.instanceId, + ); if (scalingMethod.calculateSize) { return scalingMethod.calculateSize(spanner); } else if (scalingMethod.calculateNumNodes) { - logger.warn(`scaling method ${ - spanner.scalingMethod} uses deprecated calculateNumNodes function`); + logger.warn( + `scaling method ${spanner.scalingMethod} uses deprecated calculateNumNodes function`, + ); return scalingMethod.calculateNumNodes(spanner); } else { - throw new Error(`no calculateSize() in scaling method ${ - spanner.scalingMethod}`); + throw new Error( + `no calculateSize() in scaling method ${spanner.scalingMethod}`, + ); } } @@ -266,91 +281,84 @@ function getSuggestedSize(spanner) { */ async function processScalingRequest(spanner, autoscalerState) { logger.info({ - message: `----- ${spanner.projectId}/${ - spanner.instanceId}: Scaling request received`, + message: `----- ${spanner.projectId}/${spanner.instanceId}: Scaling request received`, projectId: spanner.projectId, instanceId: spanner.instanceId, payload: spanner, }); const suggestedSize = getSuggestedSize(spanner); - if (suggestedSize === spanner.currentSize && - spanner.currentSize === spanner.maxSize) { + if ( + suggestedSize === spanner.currentSize && + spanner.currentSize === spanner.maxSize + ) { logger.info({ - message: `----- ${spanner.projectId}/${spanner.instanceId}: has ${ - spanner.currentSize} ${ - spanner.units}, no scaling possible - at maxSize`, + message: `----- ${spanner.projectId}/${spanner.instanceId}: has ${spanner.currentSize} ${spanner.units}, no scaling possible - at maxSize`, projectId: spanner.projectId, instanceId: spanner.instanceId, payload: spanner, }); - await Counters.incScalingDeniedCounter( - spanner, - suggestedSize, - 'MAX_SIZE'); + await Counters.incScalingDeniedCounter(spanner, suggestedSize, 'MAX_SIZE'); return; } else if (suggestedSize === spanner.currentSize) { logger.info({ - message: `----- ${spanner.projectId}/${spanner.instanceId}: has ${ - spanner.currentSize} ${ - spanner.units}, no scaling needed - at current size`, + message: `----- ${spanner.projectId}/${spanner.instanceId}: has ${spanner.currentSize} ${spanner.units}, no scaling needed - at current size`, projectId: spanner.projectId, instanceId: spanner.instanceId, payload: spanner, }); await Counters.incScalingDeniedCounter( - spanner, - suggestedSize, - 'CURRENT_SIZE'); + spanner, + suggestedSize, + 'CURRENT_SIZE', + ); return; } - if (!withinCooldownPeriod( - spanner, suggestedSize, await autoscalerState.get(), - autoscalerState.now)) { + if ( + !withinCooldownPeriod( + spanner, + suggestedSize, + await autoscalerState.get(), + autoscalerState.now, + ) + ) { let eventType; try { await scaleSpannerInstance(spanner, suggestedSize); await autoscalerState.set(); eventType = 'SCALING'; - await Counters.incScalingSuccessCounter( - spanner, - suggestedSize); + await Counters.incScalingSuccessCounter(spanner, suggestedSize); } catch (err) { logger.error({ - message: `----- ${spanner.projectId}/${ - spanner.instanceId}: Unsuccessful scaling attempt.`, + message: `----- ${spanner.projectId}/${spanner.instanceId}: Unsuccessful scaling attempt.`, projectId: spanner.projectId, instanceId: spanner.instanceId, payload: err, err: err, }); logger.error({ - message: - `----- ${spanner.projectId}/${spanner.instanceId}: Spanner payload:`, + message: `----- ${spanner.projectId}/${spanner.instanceId}: Spanner payload:`, projectId: spanner.projectId, instanceId: spanner.instanceId, payload: spanner, }); eventType = 'SCALING_FAILURE'; - await Counters.incScalingFailedCounter( - spanner, - suggestedSize); + await Counters.incScalingFailedCounter(spanner, suggestedSize); } await publishDownstreamEvent(eventType, spanner, suggestedSize); } else { logger.info({ - message: `----- ${spanner.projectId}/${spanner.instanceId}: has ${ - spanner.currentSize} ${ - spanner.units}, no scaling possible - within cooldown period`, + message: `----- ${spanner.projectId}/${spanner.instanceId}: has ${spanner.currentSize} ${spanner.units}, no scaling possible - within cooldown period`, projectId: spanner.projectId, instanceId: spanner.instanceId, payload: spanner, }); await Counters.incScalingDeniedCounter( - spanner, - suggestedSize, - 'WITHIN_COOLDOWN'); + spanner, + suggestedSize, + 'WITHIN_COOLDOWN', + ); } } diff --git a/src/scaler/scaler-core/scaling-methods/base.js b/src/scaler/scaler-core/scaling-methods/base.js index d31c6418..08f7a290 100644 --- a/src/scaler/scaler-core/scaling-methods/base.js +++ b/src/scaler/scaler-core/scaling-methods/base.js @@ -58,22 +58,18 @@ function getScaleSuggestionMessage(spanner, suggestedSize, relativeToRange) { if (relativeToRange == RelativeToRange.WITHIN) { return `no change suggested`; } else if ( - suggestedSize <= spanner.maxSize && suggestedSize >= spanner.minSize) { + suggestedSize <= spanner.maxSize && + suggestedSize >= spanner.minSize + ) { if (suggestedSize == spanner.currentSize) { - return `the suggested size is equal to the current size: ${ - spanner.currentSize} ${spanner.units}`; + return `the suggested size is equal to the current size: ${spanner.currentSize} ${spanner.units}`; } else { - return `suggesting to scale from ${spanner.currentSize} to ${ - suggestedSize} ${spanner.units}.`; + return `suggesting to scale from ${spanner.currentSize} to ${suggestedSize} ${spanner.units}.`; } } else if (suggestedSize > spanner.maxSize) { - return `however, cannot scale to ${ - suggestedSize} because it is higher than MAX ${spanner.maxSize} ${ - spanner.units}`; + return `however, cannot scale to ${suggestedSize} because it is higher than MAX ${spanner.maxSize} ${spanner.units}`; } else if (suggestedSize < spanner.minSize) { - return `however, cannot scale to ${ - suggestedSize} because it is lower than MIN ${spanner.minSize} ${ - spanner.units}`; + return `however, cannot scale to ${suggestedSize} because it is lower than MIN ${spanner.minSize} ${spanner.units}`; } } @@ -130,22 +126,28 @@ function logSuggestion(spanner, metric, suggestedSize) { const relativeToRange = compareMetricValueWithRange(metric); const range = getRange(metric.threshold, metric.margin); - const rangeDetails = - `${relativeToRange} the range [${range.min}%-${range.max}%]`; + const rangeDetails = `${relativeToRange} the range [${range.min}%-${range.max}%]`; if (metric.name === OVERLOAD_METRIC && spanner.isOverloaded) { logger.debug({ - message: `${metricDetails} ABOVE the ${ - OVERLOAD_THRESHOLD} overload threshold => ${ - getScaleSuggestionMessage( - spanner, suggestedSize, RelativeToRange.ABOVE)}`, - projectId: spanner.projectId, instanceId: spanner.instanceId}); + message: `${metricDetails} ABOVE the ${OVERLOAD_THRESHOLD} overload threshold => ${getScaleSuggestionMessage( + spanner, + suggestedSize, + RelativeToRange.ABOVE, + )}`, + projectId: spanner.projectId, + instanceId: spanner.instanceId, + }); } else { logger.debug({ - message: `${metricDetails} ${rangeDetails} => ${ - getScaleSuggestionMessage( - spanner, suggestedSize, relativeToRange)}`, - projectId: spanner.projectId, instanceId: spanner.instanceId}); + message: `${metricDetails} ${rangeDetails} => ${getScaleSuggestionMessage( + spanner, + suggestedSize, + relativeToRange, + )}`, + projectId: spanner.projectId, + instanceId: spanner.instanceId, + }); } } @@ -160,13 +162,15 @@ function logSuggestion(spanner, metric, suggestedSize) { */ function loopThroughSpannerMetrics(spanner, getSuggestedSize) { logger.debug({ - message: `---- ${spanner.projectId}/${spanner.instanceId}: ${ - spanner.scalingMethod} size suggestions----`, - projectId: spanner.projectId, instanceId: spanner.instanceId}); + message: `---- ${spanner.projectId}/${spanner.instanceId}: ${spanner.scalingMethod} size suggestions----`, + projectId: spanner.projectId, + instanceId: spanner.instanceId, + }); logger.debug({ - message: `\tMin=${spanner.minSize}, Current=${spanner.currentSize}, Max=${ - spanner.maxSize} ${spanner.units}`, - projectId: spanner.projectId, instanceId: spanner.instanceId}); + message: `\tMin=${spanner.minSize}, Current=${spanner.currentSize}, Max=${spanner.maxSize} ${spanner.units}`, + projectId: spanner.projectId, + instanceId: spanner.instanceId, + }); let maxSuggestedSize = spanner.minSize; spanner.isOverloaded = false; @@ -188,9 +192,10 @@ function loopThroughSpannerMetrics(spanner, getSuggestedSize) { maxSuggestedSize = Math.min(maxSuggestedSize, spanner.maxSize); logger.debug({ - message: `\t=> Final ${spanner.scalingMethod} suggestion: ${ - maxSuggestedSize} ${spanner.units}`, - projectId: spanner.projectId, instanceId: spanner.instanceId}); + message: `\t=> Final ${spanner.scalingMethod} suggestion: ${maxSuggestedSize} ${spanner.units}`, + projectId: spanner.projectId, + instanceId: spanner.instanceId, + }); return maxSuggestedSize; } diff --git a/src/scaler/scaler-core/scaling-methods/direct.js b/src/scaler/scaler-core/scaling-methods/direct.js index e9b58730..cc3996db 100644 --- a/src/scaler/scaler-core/scaling-methods/direct.js +++ b/src/scaler/scaler-core/scaling-methods/direct.js @@ -24,8 +24,8 @@ const {logger} = require('../../../autoscaler-common/logger'); /** * @typedef {import('../../../autoscaler-common/types').AutoscalerSpanner -* } AutoscalerSpanner -*/ + * } AutoscalerSpanner + */ /** * Size calculation for Direct scaling method @@ -34,13 +34,15 @@ const {logger} = require('../../../autoscaler-common/logger'); */ function calculateSize(spanner) { logger.debug({ - message: `---- DIRECT size suggestions for ${spanner.projectId}/${ - spanner.instanceId}----`, - projectId: spanner.projectId, instanceId: spanner.instanceId}); + message: `---- DIRECT size suggestions for ${spanner.projectId}/${spanner.instanceId}----`, + projectId: spanner.projectId, + instanceId: spanner.instanceId, + }); logger.debug({ - message: - `\tFinal DIRECT suggestion: ${spanner.maxSize} + ${spanner.units}}`, - projectId: spanner.projectId, instanceId: spanner.instanceId}); + message: `\tFinal DIRECT suggestion: ${spanner.maxSize} + ${spanner.units}}`, + projectId: spanner.projectId, + instanceId: spanner.instanceId, + }); return spanner.maxSize; } diff --git a/src/scaler/scaler-core/scaling-methods/linear.js b/src/scaler/scaler-core/scaling-methods/linear.js index 45915427..54dc8ae7 100644 --- a/src/scaler/scaler-core/scaling-methods/linear.js +++ b/src/scaler/scaler-core/scaling-methods/linear.js @@ -32,7 +32,6 @@ const {logger} = require('../../../autoscaler-common/logger'); * } SpannerMetricValue */ - /** * Is suggested size less than current size * @@ -54,39 +53,45 @@ function calculateSize(spanner) { if (baseModule.metricValueWithinRange(metric)) { return spanner.currentSize; } else { - let suggestedSize = - Math.ceil(spanner.currentSize * metric.value / metric.threshold); + let suggestedSize = Math.ceil( + (spanner.currentSize * metric.value) / metric.threshold, + ); - if (isScaleIn(suggestedSize, spanner.currentSize) && - spanner.scaleInLimit) { + if ( + isScaleIn(suggestedSize, spanner.currentSize) && + spanner.scaleInLimit + ) { const limit = spanner.currentSize * (spanner.scaleInLimit / 100); logger.debug({ - message: `\tscaleInLimit = ${spanner.scaleInLimit}%, ` + - `so the maximum scale-in allowed for current size of ${ - spanner.currentSize} is ${limit} ${spanner.units}.`, + message: + `\tscaleInLimit = ${spanner.scaleInLimit}%, ` + + `so the maximum scale-in allowed for current size of ${spanner.currentSize} is ${limit} ${spanner.units}.`, projectId: spanner.projectId, instanceId: spanner.instanceId, }); const originalSuggestedSize = suggestedSize; - suggestedSize = - Math.max(suggestedSize, Math.ceil(spanner.currentSize - limit)); + suggestedSize = Math.max( + suggestedSize, + Math.ceil(spanner.currentSize - limit), + ); if (suggestedSize != originalSuggestedSize) { logger.debug({ - message: `\tscaleInLimit exceeded. Original suggested size was ${ - originalSuggestedSize} ${ - spanner.units}, new suggested size is ${suggestedSize} ${ - spanner.units}.`, + message: `\tscaleInLimit exceeded. Original suggested size was ${originalSuggestedSize} ${spanner.units}, new suggested size is ${suggestedSize} ${spanner.units}.`, projectId: spanner.projectId, instanceId: spanner.instanceId, }); } } return maybeRound( - suggestedSize, spanner.units, metric.name, spanner.projectId, - spanner.instanceId); + suggestedSize, + spanner.units, + metric.name, + spanner.projectId, + spanner.instanceId, + ); } }); } diff --git a/src/scaler/scaler-core/scaling-methods/stepwise.js b/src/scaler/scaler-core/scaling-methods/stepwise.js index e8d1a607..a2560c11 100644 --- a/src/scaler/scaler-core/scaling-methods/stepwise.js +++ b/src/scaler/scaler-core/scaling-methods/stepwise.js @@ -47,27 +47,31 @@ function calculateSize(spanner) { let stepSize = spanner.stepSize; // After 1000 PUs, scaling can only be done in steps of 1000 PUs - if (spanner.units.toUpperCase() == 'PROCESSING_UNITS' && - spanner.currentSize > 1000 && stepSize < 1000) { + if ( + spanner.units.toUpperCase() == 'PROCESSING_UNITS' && + spanner.currentSize > 1000 && + stepSize < 1000 + ) { stepSize = 1000; logger.debug({ - message: `\tCurrent=${spanner.currentSize} ${ - spanner.units} (> 1000) => overriding stepSize from ${ - spanner.stepSize} to 1000`, + message: `\tCurrent=${spanner.currentSize} ${spanner.units} (> 1000) => overriding stepSize from ${spanner.stepSize} to 1000`, projectId: spanner.projectId, instanceId: spanner.instanceId, }); } - let suggestedStep = - (metric.value > metric.threshold ? stepSize : -stepSize); + let suggestedStep = metric.value > metric.threshold ? stepSize : -stepSize; if (metric.name === baseModule.OVERLOAD_METRIC && spanner.isOverloaded) { suggestedStep = spanner.overloadStepSize; } return maybeRound( - Math.max(spanner.currentSize + suggestedStep, spanner.minSize), - spanner.units, metric.name, spanner.projectId, spanner.instanceId); + Math.max(spanner.currentSize + suggestedStep, spanner.minSize), + spanner.units, + metric.name, + spanner.projectId, + spanner.instanceId, + ); }); } diff --git a/src/scaler/scaler-core/state.js b/src/scaler/scaler-core/state.js index 6ee0cfbf..8378ed5e 100644 --- a/src/scaler/scaler-core/state.js +++ b/src/scaler/scaler-core/state.js @@ -41,7 +41,6 @@ const {logger} = require('../../autoscaler-common/logger'); * }} StateData */ - /** * Used to store state of a Spanner instance */ @@ -53,7 +52,7 @@ class State { * @return {State} */ static buildFor(spanner) { - if (! spanner) { + if (!spanner) { throw new Error('spanner should not be null'); } switch (spanner?.stateDatabase?.name) { @@ -73,9 +72,10 @@ class State { */ constructor(spanner) { /** @type {string} */ - this.stateProjectId = (spanner.stateProjectId != null) ? - spanner.stateProjectId : - spanner.projectId; + this.stateProjectId = + spanner.stateProjectId != null + ? spanner.stateProjectId + : spanner.projectId; this.projectId = spanner.projectId; this.instanceId = spanner.instanceId; } @@ -156,8 +156,9 @@ class StateSpanner extends State { if (!spanner.stateDatabase) { throw new Error('stateDatabase is not defined in Spanner config'); } - this.db = this.client.instance(spanner.stateDatabase.instanceId) - .database(spanner.stateDatabase.databaseId); + this.db = this.client + .instance(spanner.stateDatabase.instanceId) + .database(spanner.stateDatabase.databaseId); this.table = this.db.table('spannerAutoscaler'); } @@ -266,8 +267,9 @@ class StateFirestore extends State { */ get docRef() { if (this._docRef == null) { - this._docRef = this.firestore - .doc(`spannerAutoscaler/state/${this.getSpannerId()}`); + this._docRef = this.firestore.doc( + `spannerAutoscaler/state/${this.getSpannerId()}`, + ); } return this._docRef; } @@ -325,12 +327,16 @@ class StateFirestore extends State { */ async checkAndReplaceOldDocRef() { try { - const oldDocRef = - this.firestore.doc(`spannerAutoscaler/${this.instanceId}`); + const oldDocRef = this.firestore.doc( + `spannerAutoscaler/${this.instanceId}`, + ); const snapshot = await oldDocRef.get(); if (snapshot.exists) { - logger.info(`Migrating firestore doc path from spannerAutoscaler/${ - this.instanceId} to spannerAutoscaler/state/${this.getSpannerId()}`); + logger.info( + `Migrating firestore doc path from spannerAutoscaler/${ + this.instanceId + } to spannerAutoscaler/state/${this.getSpannerId()}`, + ); await this.docRef.set(snapshot.data()); await oldDocRef.delete(); } diff --git a/src/scaler/scaler-core/test/index.test.js b/src/scaler/scaler-core/test/index.test.js index de71c9f8..6f8fe4e7 100644 --- a/src/scaler/scaler-core/test/index.test.js +++ b/src/scaler/scaler-core/test/index.test.js @@ -36,20 +36,19 @@ const getNewMetadata = app.__get__('getNewMetadata'); describe('#getNewMetadata', () => { it('should return an object with the nodeCount property set', () => { getNewMetadata(99, 'NODES') - .should.have.property('nodeCount') - .which.is.a.Number() - .and.equal(99); + .should.have.property('nodeCount') + .which.is.a.Number() + .and.equal(99); }); it('should return an object with the processingUnits property set', () => { getNewMetadata(88, 'PROCESSING_UNITS') - .should.have.property('processingUnits') - .which.is.a.Number() - .and.equal(88); + .should.have.property('processingUnits') + .which.is.a.Number() + .and.equal(88); }); }); - const processScalingRequest = app.__get__('processScalingRequest'); const countersStub = { incScalingSuccessCounter: sinon.stub(), @@ -78,106 +77,105 @@ describe('#processScalingRequest', () => { withinCooldownPeriod.reset(); }); - it('should not autoscale if suggested size is equal to current size', - async function() { - const spanner = createSpannerParameters(); - getSuggestedSizeStub.returns(spanner.currentSize); - - await processScalingRequest(spanner, createStubState()); - - assert.equals(stubScaleSpannerInstance.callCount, 0); - - assert.equals(countersStub.incScalingSuccessCounter.callCount, 0); - assert.equals(countersStub.incScalingDeniedCounter.callCount, 1); - assert.equals( - countersStub.incScalingDeniedCounter.getCall(0).args[1], - spanner.currentSize); - assert.equals( - countersStub.incScalingDeniedCounter.getCall(0).args[2], - 'CURRENT_SIZE'); - assert.equals(countersStub.incScalingFailedCounter.callCount, 0); - }); - - it('should not autoscale if suggested size is equal to max size', - async function() { - const spanner = createSpannerParameters(); - spanner.currentSize = spanner.maxSize; - getSuggestedSizeStub.returns(spanner.maxSize); - - await processScalingRequest(spanner, createStubState()); - - assert.equals(stubScaleSpannerInstance.callCount, 0); - - assert.equals(countersStub.incScalingSuccessCounter.callCount, 0); - assert.equals(countersStub.incScalingDeniedCounter.callCount, 1); - assert.equals( - countersStub.incScalingDeniedCounter.getCall(0).args[1], - spanner.maxSize); - assert.equals( - countersStub.incScalingDeniedCounter.getCall(0).args[2], - 'MAX_SIZE'); - assert.equals(countersStub.incScalingFailedCounter.callCount, 0); - }); - - it('should autoscale if suggested size is not equal to current size', - async function() { - const spanner = createSpannerParameters(); - const suggestedSize = spanner.currentSize + 100; - getSuggestedSizeStub.returns(suggestedSize); - - await processScalingRequest(spanner, createStubState()); - - assert.equals(stubScaleSpannerInstance.callCount, 1); - assert.equals( - stubScaleSpannerInstance.getCall(0).args[1], - suggestedSize); - assert.equals(countersStub.incScalingSuccessCounter.callCount, 1); - assert.equals( - countersStub.incScalingSuccessCounter.getCall(0).args[1], - suggestedSize); - assert.equals(countersStub.incScalingDeniedCounter.callCount, 0); - assert.equals(countersStub.incScalingFailedCounter.callCount, 0); - }); - - it('should not autoscale if in cooldown period', - async function() { - const spanner = createSpannerParameters(); - const suggestedSize = spanner.currentSize + 100; - getSuggestedSizeStub.returns(suggestedSize); - withinCooldownPeriod.returns(true); - - await processScalingRequest(spanner, createStubState()); - - assert.equals(stubScaleSpannerInstance.callCount, 0); - assert.equals(countersStub.incScalingSuccessCounter.callCount, 0); - assert.equals(countersStub.incScalingDeniedCounter.callCount, 1); - assert.equals( - countersStub.incScalingDeniedCounter.getCall(0).args[1], - suggestedSize); - assert.equals( - countersStub.incScalingDeniedCounter.getCall(0).args[2], - 'WITHIN_COOLDOWN'); - assert.equals(countersStub.incScalingFailedCounter.callCount, 0); - }); - - it('Scaling failures increment counter', - async function() { - const spanner = createSpannerParameters(); - const suggestedSize = spanner.currentSize + 100; - getSuggestedSizeStub.returns(suggestedSize); - stubScaleSpannerInstance.rejects('Error'); - - await processScalingRequest(spanner, createStubState()); - - assert.equals(stubScaleSpannerInstance.callCount, 1); - assert.equals( - stubScaleSpannerInstance.getCall(0).args[1], - suggestedSize); - assert.equals(countersStub.incScalingSuccessCounter.callCount, 0); - assert.equals(countersStub.incScalingDeniedCounter.callCount, 0); - assert.equals(countersStub.incScalingFailedCounter.callCount, 1); - assert.equals( - countersStub.incScalingFailedCounter.getCall(0).args[1], - suggestedSize); - }); + it('should not autoscale if suggested size is equal to current size', async function () { + const spanner = createSpannerParameters(); + getSuggestedSizeStub.returns(spanner.currentSize); + + await processScalingRequest(spanner, createStubState()); + + assert.equals(stubScaleSpannerInstance.callCount, 0); + + assert.equals(countersStub.incScalingSuccessCounter.callCount, 0); + assert.equals(countersStub.incScalingDeniedCounter.callCount, 1); + assert.equals( + countersStub.incScalingDeniedCounter.getCall(0).args[1], + spanner.currentSize, + ); + assert.equals( + countersStub.incScalingDeniedCounter.getCall(0).args[2], + 'CURRENT_SIZE', + ); + assert.equals(countersStub.incScalingFailedCounter.callCount, 0); + }); + + it('should not autoscale if suggested size is equal to max size', async function () { + const spanner = createSpannerParameters(); + spanner.currentSize = spanner.maxSize; + getSuggestedSizeStub.returns(spanner.maxSize); + + await processScalingRequest(spanner, createStubState()); + + assert.equals(stubScaleSpannerInstance.callCount, 0); + + assert.equals(countersStub.incScalingSuccessCounter.callCount, 0); + assert.equals(countersStub.incScalingDeniedCounter.callCount, 1); + assert.equals( + countersStub.incScalingDeniedCounter.getCall(0).args[1], + spanner.maxSize, + ); + assert.equals( + countersStub.incScalingDeniedCounter.getCall(0).args[2], + 'MAX_SIZE', + ); + assert.equals(countersStub.incScalingFailedCounter.callCount, 0); + }); + + it('should autoscale if suggested size is not equal to current size', async function () { + const spanner = createSpannerParameters(); + const suggestedSize = spanner.currentSize + 100; + getSuggestedSizeStub.returns(suggestedSize); + + await processScalingRequest(spanner, createStubState()); + + assert.equals(stubScaleSpannerInstance.callCount, 1); + assert.equals(stubScaleSpannerInstance.getCall(0).args[1], suggestedSize); + assert.equals(countersStub.incScalingSuccessCounter.callCount, 1); + assert.equals( + countersStub.incScalingSuccessCounter.getCall(0).args[1], + suggestedSize, + ); + assert.equals(countersStub.incScalingDeniedCounter.callCount, 0); + assert.equals(countersStub.incScalingFailedCounter.callCount, 0); + }); + + it('should not autoscale if in cooldown period', async function () { + const spanner = createSpannerParameters(); + const suggestedSize = spanner.currentSize + 100; + getSuggestedSizeStub.returns(suggestedSize); + withinCooldownPeriod.returns(true); + + await processScalingRequest(spanner, createStubState()); + + assert.equals(stubScaleSpannerInstance.callCount, 0); + assert.equals(countersStub.incScalingSuccessCounter.callCount, 0); + assert.equals(countersStub.incScalingDeniedCounter.callCount, 1); + assert.equals( + countersStub.incScalingDeniedCounter.getCall(0).args[1], + suggestedSize, + ); + assert.equals( + countersStub.incScalingDeniedCounter.getCall(0).args[2], + 'WITHIN_COOLDOWN', + ); + assert.equals(countersStub.incScalingFailedCounter.callCount, 0); + }); + + it('Scaling failures increment counter', async function () { + const spanner = createSpannerParameters(); + const suggestedSize = spanner.currentSize + 100; + getSuggestedSizeStub.returns(suggestedSize); + stubScaleSpannerInstance.rejects('Error'); + + await processScalingRequest(spanner, createStubState()); + + assert.equals(stubScaleSpannerInstance.callCount, 1); + assert.equals(stubScaleSpannerInstance.getCall(0).args[1], suggestedSize); + assert.equals(countersStub.incScalingSuccessCounter.callCount, 0); + assert.equals(countersStub.incScalingDeniedCounter.callCount, 0); + assert.equals(countersStub.incScalingFailedCounter.callCount, 1); + assert.equals( + countersStub.incScalingFailedCounter.getCall(0).args[1], + suggestedSize, + ); + }); }); diff --git a/src/scaler/scaler-core/test/samples/downstream-msg.json b/src/scaler/scaler-core/test/samples/downstream-msg.json index ba9f355f..190a9653 100644 --- a/src/scaler/scaler-core/test/samples/downstream-msg.json +++ b/src/scaler/scaler-core/test/samples/downstream-msg.json @@ -1,27 +1,27 @@ { - "projectId":"my-spanner-project", - "instanceId":"autoscale-test", - "currentSize":100, - "suggestedSize":300, - "units":"PROCESSING_UNITS", - "metrics":[ - { - "name":"high_priority_cpu", - "threshold":65, - "value":85.764282783144476, - "margin":15 - }, - { - "name":"rolling_24_hr", - "threshold":90, - "value":70.105833476200979, - "margin":5 - }, - { - "name":"storage", - "threshold":75, - "value":21.980773510634042, - "margin":5 - } - ] - } + "projectId": "my-spanner-project", + "instanceId": "autoscale-test", + "currentSize": 100, + "suggestedSize": 300, + "units": "PROCESSING_UNITS", + "metrics": [ + { + "name": "high_priority_cpu", + "threshold": 65, + "value": 85.764282783144476, + "margin": 15 + }, + { + "name": "rolling_24_hr", + "threshold": 90, + "value": 70.105833476200979, + "margin": 5 + }, + { + "name": "storage", + "threshold": 75, + "value": 21.980773510634042, + "margin": 5 + } + ] +} diff --git a/src/scaler/scaler-core/test/samples/parameters.json b/src/scaler/scaler-core/test/samples/parameters.json index 0697c566..9bcd7df4 100644 --- a/src/scaler/scaler-core/test/samples/parameters.json +++ b/src/scaler/scaler-core/test/samples/parameters.json @@ -1,33 +1,33 @@ { - "units":"PROCESSING_UNITS", - "minSize":100, - "maxSize":2000, - "stepSize":200, - "overloadStepSize":500, - "scaleOutCoolingMinutes":5, - "scaleInCoolingMinutes":30, - "scalingMethod":"STEPWISE", - "projectId":"my-spanner-project", - "instanceId":"autoscale-test", - "metrics":[ - { - "name":"high_priority_cpu", - "threshold":65, - "value":85.764282783144476, - "margin":15 - }, - { - "name":"rolling_24_hr", - "threshold":90, - "value":70.105833476200979 - }, - { - "name":"storage", - "threshold":75, - "value":21.980773510634042 - } + "units": "PROCESSING_UNITS", + "minSize": 100, + "maxSize": 2000, + "stepSize": 200, + "overloadStepSize": 500, + "scaleOutCoolingMinutes": 5, + "scaleInCoolingMinutes": 30, + "scalingMethod": "STEPWISE", + "projectId": "my-spanner-project", + "instanceId": "autoscale-test", + "metrics": [ + { + "name": "high_priority_cpu", + "threshold": 65, + "value": 85.764282783144476, + "margin": 15 + }, + { + "name": "rolling_24_hr", + "threshold": 90, + "value": 70.105833476200979 + }, + { + "name": "storage", + "threshold": 75, + "value": 21.980773510634042 + } ], - "currentNodes":0, - "currentSize":100, - "regional":true -} \ No newline at end of file + "currentNodes": 0, + "currentSize": 100, + "regional": true +} diff --git a/src/scaler/scaler-core/test/scaling-methods/base.test.js b/src/scaler/scaler-core/test/scaling-methods/base.test.js index c72a18a2..ca11df23 100644 --- a/src/scaler/scaler-core/test/scaling-methods/base.test.js +++ b/src/scaler/scaler-core/test/scaling-methods/base.test.js @@ -28,40 +28,55 @@ const app = rewire('../../scaling-methods/base.js'); const compareMetricValueWithRange = app.__get__('compareMetricValueWithRange'); describe('#compareMetricValueWithRange', () => { - it('should return WITHIN when value is within range', - () => { - compareMetricValueWithRange({value: 70, threshold: 65, margin: 5}) - .should.equal('WITHIN'); - }); - - it('should return ABOVE when value is above range', - () => { - compareMetricValueWithRange({value: 80, threshold: 65, margin: 5}) - .should.equal('ABOVE'); - }); - - it('should return BELOW when value is below range', - () => { - compareMetricValueWithRange({value: 20, threshold: 65, margin: 5}) - .should.equal('BELOW'); - }); + it('should return WITHIN when value is within range', () => { + compareMetricValueWithRange({ + value: 70, + threshold: 65, + margin: 5, + }).should.equal('WITHIN'); + }); + + it('should return ABOVE when value is above range', () => { + compareMetricValueWithRange({ + value: 80, + threshold: 65, + margin: 5, + }).should.equal('ABOVE'); + }); + + it('should return BELOW when value is below range', () => { + compareMetricValueWithRange({ + value: 20, + threshold: 65, + margin: 5, + }).should.equal('BELOW'); + }); }); const metricValueWithinRange = app.__get__('metricValueWithinRange'); describe('#metricValueWithinRange', () => { it('should return true when metric falls within margins', () => { - metricValueWithinRange({value: 63, threshold: 65, margin: 5}) - .should.be.true(); + metricValueWithinRange({ + value: 63, + threshold: 65, + margin: 5, + }).should.be.true(); }); it('should return false when metric falls outside of the margins', () => { - metricValueWithinRange({value: 15, threshold: 45, margin: 10}) - .should.be.false(); + metricValueWithinRange({ + value: 15, + threshold: 45, + margin: 10, + }).should.be.false(); }); it('should return true when metric falls right at the edge', () => { - metricValueWithinRange({value: 70, threshold: 65, margin: 5}) - .should.be.true(); + metricValueWithinRange({ + value: 70, + threshold: 65, + margin: 5, + }).should.be.true(); }); }); @@ -93,102 +108,114 @@ describe('#getScaleSuggestionMessage', () => { }); // NODES -------------------------------------------------- - it('should not suggest scaling when nodes suggestion is equal to current', - () => { - const msg = getScaleSuggestionMessage( - {units: 'NODES', currentSize: 3, minSize: 2, maxSize: 8}, 3, ''); - msg.should.containEql('size is equal to the current size'); - msg.should.containEql('NODES'); - msg.should.not.containEql('PROCESSING_UNITS'); - }); - - it('should suggest scaling when nodes suggestion is not equal to current', - () => { - const msg = getScaleSuggestionMessage( - {units: 'NODES', currentSize: 3, minSize: 2, maxSize: 8}, 5, ''); - msg.should.containEql('suggesting to scale'); - msg.should.containEql('NODES'); - msg.should.not.containEql('PROCESSING_UNITS'); - }); - - it('should indicate scaling is not possible if nodes suggestion is above max', - () => { - const msg = getScaleSuggestionMessage( - {units: 'NODES', currentSize: 3, minSize: 2, maxSize: 8}, 9, ''); - msg.should.containEql('higher than MAX'); - msg.should.containEql('NODES'); - msg.should.not.containEql('PROCESSING_UNITS'); - }); - - it('should indicate scaling is not possible if nodes suggestion is below min', - () => { - const msg = getScaleSuggestionMessage( - {units: 'NODES', currentSize: 3, minSize: 2, maxSize: 8}, 1, ''); - msg.should.containEql('lower than MIN'); - msg.should.containEql('NODES'); - msg.should.not.containEql('PROCESSING_UNITS'); - }); + it('should not suggest scaling when nodes suggestion is equal to current', () => { + const msg = getScaleSuggestionMessage( + {units: 'NODES', currentSize: 3, minSize: 2, maxSize: 8}, + 3, + '', + ); + msg.should.containEql('size is equal to the current size'); + msg.should.containEql('NODES'); + msg.should.not.containEql('PROCESSING_UNITS'); + }); + + it('should suggest scaling when nodes suggestion is not equal to current', () => { + const msg = getScaleSuggestionMessage( + {units: 'NODES', currentSize: 3, minSize: 2, maxSize: 8}, + 5, + '', + ); + msg.should.containEql('suggesting to scale'); + msg.should.containEql('NODES'); + msg.should.not.containEql('PROCESSING_UNITS'); + }); + + it('should indicate scaling is not possible if nodes suggestion is above max', () => { + const msg = getScaleSuggestionMessage( + {units: 'NODES', currentSize: 3, minSize: 2, maxSize: 8}, + 9, + '', + ); + msg.should.containEql('higher than MAX'); + msg.should.containEql('NODES'); + msg.should.not.containEql('PROCESSING_UNITS'); + }); + + it('should indicate scaling is not possible if nodes suggestion is below min', () => { + const msg = getScaleSuggestionMessage( + {units: 'NODES', currentSize: 3, minSize: 2, maxSize: 8}, + 1, + '', + ); + msg.should.containEql('lower than MIN'); + msg.should.containEql('NODES'); + msg.should.not.containEql('PROCESSING_UNITS'); + }); // PROCESSING_UNITS --------------------------------------- - it('should not suggest scaling when processing units suggestion is equal to current', - () => { - const msg = getScaleSuggestionMessage( - { - units: 'PROCESSING_UNITS', - currentSize: 300, - minSize: 200, - maxSize: 800, - }, - 300, ''); - msg.should.containEql('size is equal to the current size'); - msg.should.containEql('PROCESSING_UNITS'); - msg.should.not.containEql('NODES'); - }); - - it('should suggest scaling when processing units suggestion is not equal to current', - () => { - const msg = getScaleSuggestionMessage( - { - units: 'PROCESSING_UNITS', - currentSize: 300, - minSize: 200, - maxSize: 800, - }, - 500, ''); - msg.should.containEql('suggesting to scale'); - msg.should.containEql('PROCESSING_UNITS'); - msg.should.not.containEql('NODES'); - }); - - it('should indicate scaling is not possible if processing units suggestion is above max', - () => { - const msg = getScaleSuggestionMessage( - { - units: 'PROCESSING_UNITS', - currentSize: 300, - minSize: 200, - maxSize: 800, - }, - 900, ''); - msg.should.containEql('higher than MAX'); - msg.should.containEql('PROCESSING_UNITS'); - msg.should.not.containEql('NODES'); - }); - - it('should indicate scaling is not possible if processing units suggestion is below min', - () => { - const msg = getScaleSuggestionMessage( - { - units: 'PROCESSING_UNITS', - currentSize: 300, - minSize: 200, - maxSize: 800, - }, - 100, ''); - msg.should.containEql('lower than MIN'); - msg.should.containEql('PROCESSING_UNITS'); - msg.should.not.containEql('NODES'); - }); + it('should not suggest scaling when processing units suggestion is equal to current', () => { + const msg = getScaleSuggestionMessage( + { + units: 'PROCESSING_UNITS', + currentSize: 300, + minSize: 200, + maxSize: 800, + }, + 300, + '', + ); + msg.should.containEql('size is equal to the current size'); + msg.should.containEql('PROCESSING_UNITS'); + msg.should.not.containEql('NODES'); + }); + + it('should suggest scaling when processing units suggestion is not equal to current', () => { + const msg = getScaleSuggestionMessage( + { + units: 'PROCESSING_UNITS', + currentSize: 300, + minSize: 200, + maxSize: 800, + }, + 500, + '', + ); + msg.should.containEql('suggesting to scale'); + msg.should.containEql('PROCESSING_UNITS'); + msg.should.not.containEql('NODES'); + }); + + it('should indicate scaling is not possible if processing units suggestion is above max', () => { + const msg = getScaleSuggestionMessage( + { + units: 'PROCESSING_UNITS', + currentSize: 300, + minSize: 200, + maxSize: 800, + }, + 900, + '', + ); + msg.should.containEql('higher than MAX'); + msg.should.containEql('PROCESSING_UNITS'); + msg.should.not.containEql('NODES'); + }); + + it('should indicate scaling is not possible if processing units suggestion is below min', () => { + const msg = getScaleSuggestionMessage( + { + units: 'PROCESSING_UNITS', + currentSize: 300, + minSize: 200, + maxSize: 800, + }, + 100, + '', + ); + msg.should.containEql('lower than MIN'); + msg.should.containEql('PROCESSING_UNITS'); + msg.should.not.containEql('NODES'); + }); }); /** diff --git a/src/scaler/scaler-core/test/scaling-methods/direct.test.js b/src/scaler/scaler-core/test/scaling-methods/direct.test.js index 2b509b99..afe8acdb 100644 --- a/src/scaler/scaler-core/test/scaling-methods/direct.test.js +++ b/src/scaler/scaler-core/test/scaling-methods/direct.test.js @@ -34,8 +34,11 @@ describe('#direct.calculateSize', () => { }); it('should ignore deprecated parameter maxNodes', () => { - const spanner = createSpannerParameters( - {units: 'NODES', maxSize: 8, maxNodes: 9}); + const spanner = createSpannerParameters({ + units: 'NODES', + maxSize: 8, + maxNodes: 9, + }); calculateSize(spanner).should.equal(8); }); diff --git a/src/scaler/scaler-core/test/scaling-methods/linear.test.js b/src/scaler/scaler-core/test/scaling-methods/linear.test.js index e4592699..fa263adc 100644 --- a/src/scaler/scaler-core/test/scaling-methods/linear.test.js +++ b/src/scaler/scaler-core/test/scaling-methods/linear.test.js @@ -45,8 +45,9 @@ function stubBaseModule(spanner, metric, metricValueWithinRange) { const callbackStub = sinon.stub().callsArgWith(1, spanner, metric); app.__set__('baseModule.loopThroughSpannerMetrics', callbackStub); app.__set__( - 'baseModule.metricValueWithinRange', - sinon.stub().returns(metricValueWithinRange)); + 'baseModule.metricValueWithinRange', + sinon.stub().returns(metricValueWithinRange), + ); return callbackStub; } @@ -60,105 +61,141 @@ describe('#linear.calculateSize', () => { assert.equals(callbackStub.callCount, 1); }); - it('should return higher value of processing units if the metric is above range', - () => { - const spanner = createSpannerParameters({currentSize: 700}); - const callbackStub = - stubBaseModule(spanner, {value: 75, threshold: 65}, false); + it('should return higher value of processing units if the metric is above range', () => { + const spanner = createSpannerParameters({currentSize: 700}); + const callbackStub = stubBaseModule( + spanner, + {value: 75, threshold: 65}, + false, + ); - calculateSize(spanner).should.equal(900); - assert.equals(callbackStub.callCount, 1); - }); + calculateSize(spanner).should.equal(900); + assert.equals(callbackStub.callCount, 1); + }); it('should return higher value of nodes if the metric above range', () => { - const spanner = - createSpannerParameters({units: 'NODES', currentSize: 7}); - const callbackStub = - stubBaseModule(spanner, {value: 75, threshold: 65}, false); + const spanner = createSpannerParameters({units: 'NODES', currentSize: 7}); + const callbackStub = stubBaseModule( + spanner, + {value: 75, threshold: 65}, + false, + ); calculateSize(spanner).should.equal(9); assert.equals(callbackStub.callCount, 1); }); - it('should return lower value of processing units if the metric is below range', - () => { - const spanner = createSpannerParameters({currentSize: 700}); - const callbackStub = - stubBaseModule(spanner, {value: 55, threshold: 65}, false); - - calculateSize(spanner).should.equal(600); - assert.equals(callbackStub.callCount, 1); - }); - - it('should return the number of processing units rounded to next 1000 if over 1000', - () => { - const spanner = createSpannerParameters({currentSize: 900}); - const callbackStub = - stubBaseModule(spanner, {value: 85, threshold: 65}, false); - - calculateSize(spanner).should.equal(2000); - assert.equals(callbackStub.callCount, 1); - }); - - it('should return the higher instance size if a scaleInLimit is specified', - () => { - const spanner = createSpannerParameters( - {units: 'NODES', currentSize: 20, scaleInLimit: 10}); - const callbackStub = - stubBaseModule(spanner, {value: 30, threshold: 65}, false); - - calculateSize(spanner).should.equal(18); - assert.equals(callbackStub.callCount, 1); - }); - - it('should not scale in if the scaleInLimit would allow for scaling less than one node', - () => { - const spanner = createSpannerParameters( - {units: 'NODES', currentSize: 10, scaleInLimit: 5}); - const callbackStub = - stubBaseModule(spanner, {value: 40, threshold: 65}, false); - - calculateSize(spanner).should.equal(10); - assert.equals(callbackStub.callCount, 1); - }); - - it('should not scale in if the scaleInLimit would allow for scaling less than a valid processing unit step size (100 PU)', - () => { - const spanner = createSpannerParameters( - {units: 'PROCESSING_UNITS', currentSize: 1000, scaleInLimit: 5}); - const callbackStub = - stubBaseModule(spanner, {value: 30, threshold: 65}, false); - - calculateSize(spanner).should.equal(1000); - assert.equals(callbackStub.callCount, 1); - }); - - it('should produce a valid capacity size when using scaleInLimit with processing units', - () => { - const spanner = createSpannerParameters( - {units: 'PROCESSING_UNITS', currentSize: 1000, scaleInLimit: 50}); - const callbackStub = - stubBaseModule(spanner, {value: 30, threshold: 65}, false); - - calculateSize(spanner).should.equal(500); - assert.equals(callbackStub.callCount, 1); - }); + it('should return lower value of processing units if the metric is below range', () => { + const spanner = createSpannerParameters({currentSize: 700}); + const callbackStub = stubBaseModule( + spanner, + {value: 55, threshold: 65}, + false, + ); + + calculateSize(spanner).should.equal(600); + assert.equals(callbackStub.callCount, 1); + }); + + it('should return the number of processing units rounded to next 1000 if over 1000', () => { + const spanner = createSpannerParameters({currentSize: 900}); + const callbackStub = stubBaseModule( + spanner, + {value: 85, threshold: 65}, + false, + ); + + calculateSize(spanner).should.equal(2000); + assert.equals(callbackStub.callCount, 1); + }); + + it('should return the higher instance size if a scaleInLimit is specified', () => { + const spanner = createSpannerParameters({ + units: 'NODES', + currentSize: 20, + scaleInLimit: 10, + }); + const callbackStub = stubBaseModule( + spanner, + {value: 30, threshold: 65}, + false, + ); + + calculateSize(spanner).should.equal(18); + assert.equals(callbackStub.callCount, 1); + }); + + it('should not scale in if the scaleInLimit would allow for scaling less than one node', () => { + const spanner = createSpannerParameters({ + units: 'NODES', + currentSize: 10, + scaleInLimit: 5, + }); + const callbackStub = stubBaseModule( + spanner, + {value: 40, threshold: 65}, + false, + ); + + calculateSize(spanner).should.equal(10); + assert.equals(callbackStub.callCount, 1); + }); + + it('should not scale in if the scaleInLimit would allow for scaling less than a valid processing unit step size (100 PU)', () => { + const spanner = createSpannerParameters({ + units: 'PROCESSING_UNITS', + currentSize: 1000, + scaleInLimit: 5, + }); + const callbackStub = stubBaseModule( + spanner, + {value: 30, threshold: 65}, + false, + ); + + calculateSize(spanner).should.equal(1000); + assert.equals(callbackStub.callCount, 1); + }); + + it('should produce a valid capacity size when using scaleInLimit with processing units', () => { + const spanner = createSpannerParameters({ + units: 'PROCESSING_UNITS', + currentSize: 1000, + scaleInLimit: 50, + }); + const callbackStub = stubBaseModule( + spanner, + {value: 30, threshold: 65}, + false, + ); + + calculateSize(spanner).should.equal(500); + assert.equals(callbackStub.callCount, 1); + }); it('should scaleIn even when a scaleInLimit is not specified', () => { - const spanner = - createSpannerParameters({units: 'NODES', currentSize: 20}); - const callbackStub = - stubBaseModule(spanner, {value: 30, threshold: 65}, false); + const spanner = createSpannerParameters({units: 'NODES', currentSize: 20}); + const callbackStub = stubBaseModule( + spanner, + {value: 30, threshold: 65}, + false, + ); calculateSize(spanner).should.equal(10); assert.equals(callbackStub.callCount, 1); }); it('should ignore scaleInLimit when scaling out', () => { - const spanner = createSpannerParameters( - {units: 'NODES', currentSize: 20, scaleInLimit: 10}); - const callbackStub = - stubBaseModule(spanner, {value: 80, threshold: 65}, false); + const spanner = createSpannerParameters({ + units: 'NODES', + currentSize: 20, + scaleInLimit: 10, + }); + const callbackStub = stubBaseModule( + spanner, + {value: 80, threshold: 65}, + false, + ); calculateSize(spanner).should.equal(25); assert.equals(callbackStub.callCount, 1); diff --git a/src/scaler/scaler-core/test/scaling-methods/stepwise.test.js b/src/scaler/scaler-core/test/scaling-methods/stepwise.test.js index 99a9a0e0..2af64779 100644 --- a/src/scaler/scaler-core/test/scaling-methods/stepwise.test.js +++ b/src/scaler/scaler-core/test/scaling-methods/stepwise.test.js @@ -26,8 +26,10 @@ const sinon = require('sinon'); const referee = require('@sinonjs/referee'); const assert = referee.assert; const {createSpannerParameters} = require('../test-utils.js'); -const {OVERLOAD_METRIC, OVERLOAD_THRESHOLD} = - require('../../scaling-methods/base.js'); +const { + OVERLOAD_METRIC, + OVERLOAD_THRESHOLD, +} = require('../../scaling-methods/base.js'); const app = rewire('../../scaling-methods/stepwise.js'); @@ -47,8 +49,9 @@ function stubBaseModule(spanner, metric, metricValueWithinRange) { const callbackStub = sinon.stub().callsArgWith(1, spanner, metric); app.__set__('baseModule.loopThroughSpannerMetrics', callbackStub); app.__set__( - 'baseModule.metricValueWithinRange', - sinon.stub().returns(metricValueWithinRange)); + 'baseModule.metricValueWithinRange', + sinon.stub().returns(metricValueWithinRange), + ); return callbackStub; } @@ -62,79 +65,84 @@ describe('#stepwise.calculateSize', () => { assert.equals(callbackStub.callCount, 1); }); - it('should return number of processing units increased by stepSize if the metric is above range', - () => { - const spanner = - createSpannerParameters({currentSize: 700, stepSize: 100}); - const callbackStub = - stubBaseModule(spanner, {value: 85, threshold: 65}, false); - - calculateSize(spanner).should.equal(800); - assert.equals(callbackStub.callCount, 1); - }); - - it('should return number of processing units decreased by stepSize if the metric is below range', - () => { - const spanner = - createSpannerParameters({currentSize: 700, stepSize: 100}); - const callbackStub = - stubBaseModule(spanner, {value: 15, threshold: 65}, false); - - calculateSize(spanner).should.equal(600); - assert.equals(callbackStub.callCount, 1); - }); - - it('should return the number of processing units increased by overloadStepSize if the instance is overloaded', - () => { - const spanner = createSpannerParameters( - { - currentSize: 300, - stepSize: 100, - overloadStepSize: 600, - isOverloaded: true, - }); - const callbackStub = stubBaseModule( - spanner, { - value: OVERLOAD_THRESHOLD + 1, - threshold: 65, - name: OVERLOAD_METRIC, - }, - false); - - calculateSize(spanner).should.equal(900); - assert.equals(callbackStub.callCount, 1); - }); - - it('should return the number of processing units rounded to next 1000 if over 1000', - () => { - const spanner = - createSpannerParameters({currentSize: 800, stepSize: 400}); - const callbackStub = - stubBaseModule(spanner, {value: 85, threshold: 65}, false); - - calculateSize(spanner).should.equal(2000); - assert.equals(callbackStub.callCount, 1); - }); - - it('should return PUs decreased by 1000 when scaling-in, and the size is >1000 PUs, and stepSize is <1000 PUs', - () => { - const spanner = - createSpannerParameters({currentSize: 4000, stepSize: 100}); - const callbackStub = - stubBaseModule(spanner, {value: 5, threshold: 65}, false); - - calculateSize(spanner).should.equal(3000); - assert.equals(callbackStub.callCount, 1); - }); - - it('should return PUs increased by 1000 when scaling-out, and the size is >1000 PUs, and stepSize is <1000 PUs', - () => { - const spanner = - createSpannerParameters({currentSize: 4000, stepSize: 100}); - const callbackStub = - stubBaseModule(spanner, {value: 85, threshold: 65}, false); - - calculateSize(spanner).should.equal(5000); - assert.equals(callbackStub.callCount, 1); - }); + it('should return number of processing units increased by stepSize if the metric is above range', () => { + const spanner = createSpannerParameters({currentSize: 700, stepSize: 100}); + const callbackStub = stubBaseModule( + spanner, + {value: 85, threshold: 65}, + false, + ); + + calculateSize(spanner).should.equal(800); + assert.equals(callbackStub.callCount, 1); + }); + + it('should return number of processing units decreased by stepSize if the metric is below range', () => { + const spanner = createSpannerParameters({currentSize: 700, stepSize: 100}); + const callbackStub = stubBaseModule( + spanner, + {value: 15, threshold: 65}, + false, + ); + + calculateSize(spanner).should.equal(600); + assert.equals(callbackStub.callCount, 1); + }); + + it('should return the number of processing units increased by overloadStepSize if the instance is overloaded', () => { + const spanner = createSpannerParameters({ + currentSize: 300, + stepSize: 100, + overloadStepSize: 600, + isOverloaded: true, + }); + const callbackStub = stubBaseModule( + spanner, + { + value: OVERLOAD_THRESHOLD + 1, + threshold: 65, + name: OVERLOAD_METRIC, + }, + false, + ); + + calculateSize(spanner).should.equal(900); + assert.equals(callbackStub.callCount, 1); + }); + + it('should return the number of processing units rounded to next 1000 if over 1000', () => { + const spanner = createSpannerParameters({currentSize: 800, stepSize: 400}); + const callbackStub = stubBaseModule( + spanner, + {value: 85, threshold: 65}, + false, + ); + + calculateSize(spanner).should.equal(2000); + assert.equals(callbackStub.callCount, 1); + }); + + it('should return PUs decreased by 1000 when scaling-in, and the size is >1000 PUs, and stepSize is <1000 PUs', () => { + const spanner = createSpannerParameters({currentSize: 4000, stepSize: 100}); + const callbackStub = stubBaseModule( + spanner, + {value: 5, threshold: 65}, + false, + ); + + calculateSize(spanner).should.equal(3000); + assert.equals(callbackStub.callCount, 1); + }); + + it('should return PUs increased by 1000 when scaling-out, and the size is >1000 PUs, and stepSize is <1000 PUs', () => { + const spanner = createSpannerParameters({currentSize: 4000, stepSize: 100}); + const callbackStub = stubBaseModule( + spanner, + {value: 85, threshold: 65}, + false, + ); + + calculateSize(spanner).should.equal(5000); + assert.equals(callbackStub.callCount, 1); + }); }); diff --git a/src/scaler/scaler-core/test/state.test.js b/src/scaler/scaler-core/test/state.test.js index c842c38c..7d859b11 100644 --- a/src/scaler/scaler-core/test/state.test.js +++ b/src/scaler/scaler-core/test/state.test.js @@ -26,8 +26,8 @@ const assert = referee.assert; /** * @typedef {import('../../../autoscaler-common/types').AutoscalerSpanner -* } AutoscalerSpanner -*/ + * } AutoscalerSpanner + */ // Create a dummy Firestore module with a dummy class constructor // that returns a stub instance. @@ -85,13 +85,12 @@ describe('stateFirestoreTests', () => { stateProjectId: 'stateProject', }; - const DUMMY_FIRESTORE_TIMESTAMP = firestore.Timestamp - .fromMillis(DUMMY_TIMESTAMP); + const DUMMY_FIRESTORE_TIMESTAMP = + firestore.Timestamp.fromMillis(DUMMY_TIMESTAMP); const NEW_DOC_PATH = - 'spannerAutoscaler/state/projects/myProject/instances/myInstance'; - const OLD_DOC_PATH = - 'spannerAutoscaler/myInstance'; + 'spannerAutoscaler/state/projects/myProject/instances/myInstance'; + const OLD_DOC_PATH = 'spannerAutoscaler/myInstance'; /** @type {firestore.DocumentSnapshot} */ // @ts-ignore @@ -101,8 +100,7 @@ describe('stateFirestoreTests', () => { return { createdOn: DUMMY_FIRESTORE_TIMESTAMP, updatedOn: DUMMY_FIRESTORE_TIMESTAMP, - lastScalingTimestamp: - DUMMY_FIRESTORE_TIMESTAMP, + lastScalingTimestamp: DUMMY_FIRESTORE_TIMESTAMP, }; }, }; @@ -114,153 +112,133 @@ describe('stateFirestoreTests', () => { data: () => null, }; - beforeEach(() => { // stub instances need to be recreated before each test. stubFirestoreInstance = sinon.createStubInstance(firestore.Firestore); stubFirestoreConstructor.returns(stubFirestoreInstance); newDocRef = sinon.createStubInstance(firestore.DocumentReference); oldDocRef = sinon.createStubInstance(firestore.DocumentReference); - stubFirestoreInstance.doc - .withArgs(NEW_DOC_PATH) - .returns(newDocRef); - stubFirestoreInstance.doc - .withArgs(OLD_DOC_PATH) - .returns(oldDocRef); + stubFirestoreInstance.doc.withArgs(NEW_DOC_PATH).returns(newDocRef); + stubFirestoreInstance.doc.withArgs(OLD_DOC_PATH).returns(oldDocRef); }); - it('should create a StateFirestore object on spanner projectId', - function() { - const config = { - ...autoscalerConfig, - stateProjectId: null, - }; - const state = State.buildFor(config); - assert.equals(state.constructor.name, 'StateFirestore'); - sinon.assert.calledWith(stubFirestoreConstructor, - {projectId: 'myProject'}); - }); - - it('should create a StateFirestore object connecting to stateProjectId', - function() { - const state = State.buildFor(autoscalerConfig); - assert.equals(state.constructor.name, 'StateFirestore'); - sinon.assert.calledWith(stubFirestoreConstructor, - {projectId: 'stateProject'}); - }); - - it('get() should read document from collection when exists', - async function() { - // @ts-ignore - newDocRef.get.returns(Promise.resolve(EXISTING_DOC)); - - const state = State.buildFor(autoscalerConfig); - const data = await state.get(); - - sinon.assert.calledOnce(newDocRef.get); - sinon.assert.calledWith(stubFirestoreInstance.doc, NEW_DOC_PATH); - - // timestamp was converted... - assert.equals(data, - { - createdOn: DUMMY_TIMESTAMP, - updatedOn: DUMMY_TIMESTAMP, - lastScalingTimestamp: DUMMY_TIMESTAMP, - }); - }); - - it('get() should create a document when it does not exist', - async function() { - newDocRef.get.returns(Promise.resolve(NON_EXISTING_DOC)); - oldDocRef.get.returns(Promise.resolve(NON_EXISTING_DOC)); - - - const state = State.buildFor(autoscalerConfig); - const data = await state.get(); - - const expected = { - lastScalingTimestamp: 0, - createdOn: firestore.FieldValue.serverTimestamp(), - }; - - sinon.assert.calledTwice(stubFirestoreInstance.doc); - // first call to create docref is for the "new" path - assert.equals(stubFirestoreInstance.doc.getCall(0).args[0], - NEW_DOC_PATH); - // second call to create docref is for the "old" path - assert.equals(stubFirestoreInstance.doc.getCall(1).args[0], - OLD_DOC_PATH); - - sinon.assert.calledOnce(newDocRef.get); - sinon.assert.calledOnce(oldDocRef.get); - - sinon.assert.calledOnce(newDocRef.set); - assert.equals(newDocRef.set.getCall(0).args[0], expected); - assert.equals(data, expected); - }); - - it('get() should copy document from old location to new if missing in new', - async function() { - /** - * Due to [issue 213](https://github.com/cloudspannerecosystem/autoscaler/issues/213) - * the docRef had to be changed, so check for an old doc at the old - * docref, if it exists, copy it to the new docref, delete it and - * return it. - */ - newDocRef.get.returns(Promise.resolve(NON_EXISTING_DOC)); - oldDocRef.get.returns(Promise.resolve(EXISTING_DOC)); - - const state = State.buildFor(autoscalerConfig); - const data = await state.get(); - - // Expected value set and returned is the old doc. - const expected = { - lastScalingTimestamp: DUMMY_TIMESTAMP, - createdOn: DUMMY_TIMESTAMP, - updatedOn: DUMMY_TIMESTAMP, - }; - - sinon.assert.calledTwice(stubFirestoreInstance.doc); - // first call to create docref is for the "new" path - assert.equals(stubFirestoreInstance.doc.getCall(0).args[0], - NEW_DOC_PATH); - // second call to create docref is for the "old" path - assert.equals(stubFirestoreInstance.doc.getCall(1).args[0], - OLD_DOC_PATH); - - sinon.assert.calledOnce(newDocRef.get); - sinon.assert.calledOnce(oldDocRef.get); - - // Copy data from existing doc in old location to new location. - sinon.assert.calledOnce(newDocRef.set); - assert.equals(newDocRef.set.getCall(0).args[0], EXISTING_DOC.data()); - sinon.assert.calledOnce(oldDocRef.delete); - - // return data from existing doc. - assert.equals(data, expected); - }); - - - it('set() should write document to collection', - async function() { - // set calls get(), so give it a doc to return... - newDocRef.get.returns(Promise.resolve(EXISTING_DOC)); - - const state = State.buildFor(autoscalerConfig); - await state.set(); - - sinon.assert.calledOnce(stubFirestoreInstance.doc); - assert.equals(stubFirestoreInstance.doc.getCall(0).args[0], - NEW_DOC_PATH); - - sinon.assert.calledOnce(newDocRef.update); - assert.equals(newDocRef.update.getCall(0).args[0], { - updatedOn: firestore.FieldValue.serverTimestamp(), - lastScalingTimestamp: firestore.FieldValue.serverTimestamp(), - }); - }); -}); + it('should create a StateFirestore object on spanner projectId', function () { + const config = { + ...autoscalerConfig, + stateProjectId: null, + }; + const state = State.buildFor(config); + assert.equals(state.constructor.name, 'StateFirestore'); + sinon.assert.calledWith(stubFirestoreConstructor, {projectId: 'myProject'}); + }); + + it('should create a StateFirestore object connecting to stateProjectId', function () { + const state = State.buildFor(autoscalerConfig); + assert.equals(state.constructor.name, 'StateFirestore'); + sinon.assert.calledWith(stubFirestoreConstructor, { + projectId: 'stateProject', + }); + }); + + it('get() should read document from collection when exists', async function () { + // @ts-ignore + newDocRef.get.returns(Promise.resolve(EXISTING_DOC)); + + const state = State.buildFor(autoscalerConfig); + const data = await state.get(); + + sinon.assert.calledOnce(newDocRef.get); + sinon.assert.calledWith(stubFirestoreInstance.doc, NEW_DOC_PATH); + + // timestamp was converted... + assert.equals(data, { + createdOn: DUMMY_TIMESTAMP, + updatedOn: DUMMY_TIMESTAMP, + lastScalingTimestamp: DUMMY_TIMESTAMP, + }); + }); + + it('get() should create a document when it does not exist', async function () { + newDocRef.get.returns(Promise.resolve(NON_EXISTING_DOC)); + oldDocRef.get.returns(Promise.resolve(NON_EXISTING_DOC)); + + const state = State.buildFor(autoscalerConfig); + const data = await state.get(); + + const expected = { + lastScalingTimestamp: 0, + createdOn: firestore.FieldValue.serverTimestamp(), + }; + + sinon.assert.calledTwice(stubFirestoreInstance.doc); + // first call to create docref is for the "new" path + assert.equals(stubFirestoreInstance.doc.getCall(0).args[0], NEW_DOC_PATH); + // second call to create docref is for the "old" path + assert.equals(stubFirestoreInstance.doc.getCall(1).args[0], OLD_DOC_PATH); + + sinon.assert.calledOnce(newDocRef.get); + sinon.assert.calledOnce(oldDocRef.get); + + sinon.assert.calledOnce(newDocRef.set); + assert.equals(newDocRef.set.getCall(0).args[0], expected); + assert.equals(data, expected); + }); + + it('get() should copy document from old location to new if missing in new', async function () { + /** + * Due to [issue 213](https://github.com/cloudspannerecosystem/autoscaler/issues/213) + * the docRef had to be changed, so check for an old doc at the old + * docref, if it exists, copy it to the new docref, delete it and + * return it. + */ + newDocRef.get.returns(Promise.resolve(NON_EXISTING_DOC)); + oldDocRef.get.returns(Promise.resolve(EXISTING_DOC)); + + const state = State.buildFor(autoscalerConfig); + const data = await state.get(); + + // Expected value set and returned is the old doc. + const expected = { + lastScalingTimestamp: DUMMY_TIMESTAMP, + createdOn: DUMMY_TIMESTAMP, + updatedOn: DUMMY_TIMESTAMP, + }; + + sinon.assert.calledTwice(stubFirestoreInstance.doc); + // first call to create docref is for the "new" path + assert.equals(stubFirestoreInstance.doc.getCall(0).args[0], NEW_DOC_PATH); + // second call to create docref is for the "old" path + assert.equals(stubFirestoreInstance.doc.getCall(1).args[0], OLD_DOC_PATH); + + sinon.assert.calledOnce(newDocRef.get); + sinon.assert.calledOnce(oldDocRef.get); + + // Copy data from existing doc in old location to new location. + sinon.assert.calledOnce(newDocRef.set); + assert.equals(newDocRef.set.getCall(0).args[0], EXISTING_DOC.data()); + sinon.assert.calledOnce(oldDocRef.delete); + + // return data from existing doc. + assert.equals(data, expected); + }); + + it('set() should write document to collection', async function () { + // set calls get(), so give it a doc to return... + newDocRef.get.returns(Promise.resolve(EXISTING_DOC)); + const state = State.buildFor(autoscalerConfig); + await state.set(); + + sinon.assert.calledOnce(stubFirestoreInstance.doc); + assert.equals(stubFirestoreInstance.doc.getCall(0).args[0], NEW_DOC_PATH); + + sinon.assert.calledOnce(newDocRef.update); + assert.equals(newDocRef.update.getCall(0).args[0], { + updatedOn: firestore.FieldValue.serverTimestamp(), + lastScalingTimestamp: firestore.FieldValue.serverTimestamp(), + }); + }); +}); describe('stateSpannerTests', () => { /** @type {sinon.SinonStubbedInstance} */ @@ -287,11 +265,15 @@ describe('stateSpannerTests', () => { const expectedQuery = { columns: ['lastScalingTimestamp', 'createdOn'], keySet: { - keys: [{ - values: [{ - stringValue: expectedRowId, - }], - }], + keys: [ + { + values: [ + { + stringValue: expectedRowId, + }, + ], + }, + ], }, }; @@ -304,8 +286,8 @@ describe('stateSpannerTests', () => { }, }; - const DUMMY_SPANNER_ISO_TIME = spanner.Spanner.timestamp(DUMMY_TIMESTAMP) - .toISOString(); + const DUMMY_SPANNER_ISO_TIME = + spanner.Spanner.timestamp(DUMMY_TIMESTAMP).toISOString(); beforeEach(() => { stubSpannerClient = sinon.createStubInstance(spanner.Spanner); @@ -315,109 +297,110 @@ describe('stateSpannerTests', () => { stubSpannerConstructor.returns(stubSpannerClient); stubSpannerClient.instance - .withArgs(autoscalerConfig.stateDatabase.instanceId) - .returns(stubSpannerInstance); + .withArgs(autoscalerConfig.stateDatabase.instanceId) + .returns(stubSpannerInstance); stubSpannerInstance.database - .withArgs(autoscalerConfig.stateDatabase.databaseId) - .returns(stubSpannerDatabase); + .withArgs(autoscalerConfig.stateDatabase.databaseId) + .returns(stubSpannerDatabase); stubSpannerDatabase.table - .withArgs('spannerAutoscaler') - .returns(stubSpannerTable); + .withArgs('spannerAutoscaler') + .returns(stubSpannerTable); }); + it('should create a StateSpanner object connecting to spanner projectId', function () { + const config = { + ...autoscalerConfig, + stateProjectId: null, + }; + const state = State.buildFor(config); + assert.equals(state.constructor.name, 'StateSpanner'); + sinon.assert.calledWith(stubSpannerConstructor, { + projectId: autoscalerConfig.projectId, + }); + sinon.assert.calledWith( + stubSpannerClient.instance, + autoscalerConfig.stateDatabase.instanceId, + ); + sinon.assert.calledWith( + stubSpannerInstance.database, + autoscalerConfig.stateDatabase.databaseId, + ); + sinon.assert.calledWith(stubSpannerDatabase.table, 'spannerAutoscaler'); + }); - it('should create a StateSpanner object connecting to spanner projectId', - function() { - const config = { - ...autoscalerConfig, - stateProjectId: null, - }; - const state = State.buildFor(config); - assert.equals(state.constructor.name, 'StateSpanner'); - sinon.assert.calledWith(stubSpannerConstructor, - {projectId: autoscalerConfig.projectId}); - sinon.assert.calledWith(stubSpannerClient.instance, - autoscalerConfig.stateDatabase.instanceId); - sinon.assert.calledWith(stubSpannerInstance.database, - autoscalerConfig.stateDatabase.databaseId); - sinon.assert.calledWith(stubSpannerDatabase.table, - 'spannerAutoscaler'); - }); - - it('should create a StateSpanner object connecting to stateProjectId', - function() { - const state = State.buildFor(autoscalerConfig); - assert.equals(state.constructor.name, 'StateSpanner'); - sinon.assert.calledWith(stubSpannerConstructor, - {projectId: autoscalerConfig.stateProjectId}); - sinon.assert.calledWith(stubSpannerClient.instance, - autoscalerConfig.stateDatabase.instanceId); - sinon.assert.calledWith(stubSpannerInstance.database, - autoscalerConfig.stateDatabase.databaseId); - sinon.assert.calledWith(stubSpannerDatabase.table, - 'spannerAutoscaler'); - }); - - it('get() should read document from table when exists', - async function() { - // @ts-ignore - stubSpannerTable.read.returns(Promise.resolve([[VALID_ROW]])); - - const state = State.buildFor(autoscalerConfig); - const data = await state.get(); - - sinon.assert.calledWith(stubSpannerTable.read, expectedQuery); - // timestamp was converted... - assert.equals(data, { - createdOn: DUMMY_TIMESTAMP, - lastScalingTimestamp: DUMMY_TIMESTAMP}); - }); - - it('get() should create a document when it does not exist', - async function() { - // @ts-ignore - stubSpannerTable.read.returns(Promise.resolve([[]])); - - const state = State.buildFor(autoscalerConfig); - // make state.now return a fixed value - const nowfunc = sinon.stub(); - sinon.replaceGetter(state, 'now', nowfunc); - nowfunc.returns(DUMMY_TIMESTAMP); - - const data = await state.get(); - - sinon.assert.calledWith(stubSpannerTable.upsert, { - id: expectedRowId, - createdOn: DUMMY_SPANNER_ISO_TIME, - lastScalingTimestamp: '1970-01-01T00:00:00.000000000Z', - }); - - assert.equals(data, { - lastScalingTimestamp: 0, - createdOn: DUMMY_TIMESTAMP, - }); - }); - - it('set() should write document to table', - async function() { - // set calls get(), so give it a doc to return... - // @ts-ignore - stubSpannerTable.read.returns(Promise.resolve([[VALID_ROW]])); - - const state = State.buildFor(autoscalerConfig); - - // make state.now return a fixed value - const nowfunc = sinon.stub(); - sinon.replaceGetter(state, 'now', nowfunc); - nowfunc.returns(DUMMY_TIMESTAMP); - await state.set(); - - sinon.assert.calledWith(stubSpannerTable.upsert, { - id: expectedRowId, - updatedOn: DUMMY_SPANNER_ISO_TIME, - lastScalingTimestamp: DUMMY_SPANNER_ISO_TIME, - }); - }); -}); + it('should create a StateSpanner object connecting to stateProjectId', function () { + const state = State.buildFor(autoscalerConfig); + assert.equals(state.constructor.name, 'StateSpanner'); + sinon.assert.calledWith(stubSpannerConstructor, { + projectId: autoscalerConfig.stateProjectId, + }); + sinon.assert.calledWith( + stubSpannerClient.instance, + autoscalerConfig.stateDatabase.instanceId, + ); + sinon.assert.calledWith( + stubSpannerInstance.database, + autoscalerConfig.stateDatabase.databaseId, + ); + sinon.assert.calledWith(stubSpannerDatabase.table, 'spannerAutoscaler'); + }); + + it('get() should read document from table when exists', async function () { + // @ts-ignore + stubSpannerTable.read.returns(Promise.resolve([[VALID_ROW]])); + + const state = State.buildFor(autoscalerConfig); + const data = await state.get(); + + sinon.assert.calledWith(stubSpannerTable.read, expectedQuery); + // timestamp was converted... + assert.equals(data, { + createdOn: DUMMY_TIMESTAMP, + lastScalingTimestamp: DUMMY_TIMESTAMP, + }); + }); + + it('get() should create a document when it does not exist', async function () { + // @ts-ignore + stubSpannerTable.read.returns(Promise.resolve([[]])); + + const state = State.buildFor(autoscalerConfig); + // make state.now return a fixed value + const nowfunc = sinon.stub(); + sinon.replaceGetter(state, 'now', nowfunc); + nowfunc.returns(DUMMY_TIMESTAMP); + const data = await state.get(); + + sinon.assert.calledWith(stubSpannerTable.upsert, { + id: expectedRowId, + createdOn: DUMMY_SPANNER_ISO_TIME, + lastScalingTimestamp: '1970-01-01T00:00:00.000000000Z', + }); + + assert.equals(data, { + lastScalingTimestamp: 0, + createdOn: DUMMY_TIMESTAMP, + }); + }); + it('set() should write document to table', async function () { + // set calls get(), so give it a doc to return... + // @ts-ignore + stubSpannerTable.read.returns(Promise.resolve([[VALID_ROW]])); + + const state = State.buildFor(autoscalerConfig); + + // make state.now return a fixed value + const nowfunc = sinon.stub(); + sinon.replaceGetter(state, 'now', nowfunc); + nowfunc.returns(DUMMY_TIMESTAMP); + await state.set(); + + sinon.assert.calledWith(stubSpannerTable.upsert, { + id: expectedRowId, + updatedOn: DUMMY_SPANNER_ISO_TIME, + lastScalingTimestamp: DUMMY_SPANNER_ISO_TIME, + }); + }); +}); diff --git a/src/scaler/scaler-core/test/test-utils.js b/src/scaler/scaler-core/test/test-utils.js index 77c38586..eea7c8f5 100644 --- a/src/scaler/scaler-core/test/test-utils.js +++ b/src/scaler/scaler-core/test/test-utils.js @@ -54,8 +54,9 @@ function createStubState() { * @return {string} downstream message */ function createDownstreamMsg() { - return JSON.parse(fs.readFileSync('./test/samples/downstream-msg.json') - .toString()); + return JSON.parse( + fs.readFileSync('./test/samples/downstream-msg.json').toString(), + ); } module.exports = { diff --git a/src/scaler/scaler-core/test/utils.test.js b/src/scaler/scaler-core/test/utils.test.js index 3c43822b..e089eb62 100644 --- a/src/scaler/scaler-core/test/utils.test.js +++ b/src/scaler/scaler-core/test/utils.test.js @@ -33,36 +33,33 @@ describe('#maybeRound', () => { maybeRound(7, 'NODES').should.equal(7); }); - it('should round to nearest 100 processing units when suggestion < 1000 PU', - () => { - maybeRound(567, 'PROCESSING_UNITS').should.equal(600); - }); - - it('should round to nearest 1000 processing units when suggestion > 1000 PU', - () => { - maybeRound(1001, 'PROCESSING_UNITS').should.equal(2000); - }); + it('should round to nearest 100 processing units when suggestion < 1000 PU', () => { + maybeRound(567, 'PROCESSING_UNITS').should.equal(600); + }); + + it('should round to nearest 1000 processing units when suggestion > 1000 PU', () => { + maybeRound(1001, 'PROCESSING_UNITS').should.equal(2000); + }); }); const publishProtoMsgDownstream = app.__get__('publishProtoMsgDownstream'); describe('#publishProtoMsgDownstream', () => { - beforeEach(function() { + beforeEach(function () { sinon.restore(); }); - it('should not instantiate downstream topic if not defined in config', - async function() { - const stubPubSub = sinon.stub(pubsub); - app.__set__('pubsub', stubPubSub); + it('should not instantiate downstream topic if not defined in config', async function () { + const stubPubSub = sinon.stub(pubsub); + app.__set__('pubsub', stubPubSub); - await publishProtoMsgDownstream('EVENT', '', undefined); + await publishProtoMsgDownstream('EVENT', '', undefined); - assert(stubPubSub.topic.notCalled); - }); + assert(stubPubSub.topic.notCalled); + }); - it('should publish downstream message', async function() { + it('should publish downstream message', async function () { const mockTopic = { - publishMessage: async function() { + publishMessage: async function () { return Promise.resolve(); }, }; @@ -73,23 +70,23 @@ describe('#publishProtoMsgDownstream', () => { app.__set__('pubsub', stubPubSub); app.__set__( - 'createProtobufMessage', sinon.stub().returns(Buffer.from('{}'))); + 'createProtobufMessage', + sinon.stub().returns(Buffer.from('{}')), + ); await publishProtoMsgDownstream('EVENT', '', 'the/topic'); assert(spyTopic.publishMessage.calledOnce); }); }); - const createProtobufMessage = app.__get__('createProtobufMessage'); describe('#createProtobufMessage', () => { - it('should create a Protobuf message that can be validated', - async function() { - const message = await createProtobufMessage(createDownstreamMsg()); - const result = message.toJSON(); - - const root = await protobuf.load('downstream.schema.proto'); - const DownstreamEvent = root.lookupType('DownstreamEvent'); - assert.equals(DownstreamEvent.verify(result), null); - }); + it('should create a Protobuf message that can be validated', async function () { + const message = await createProtobufMessage(createDownstreamMsg()); + const result = message.toJSON(); + + const root = await protobuf.load('downstream.schema.proto'); + const DownstreamEvent = root.lookupType('DownstreamEvent'); + assert.equals(DownstreamEvent.verify(result), null); + }); }); diff --git a/src/scaler/scaler-core/utils.js b/src/scaler/scaler-core/utils.js index ea8c56c5..0252f13c 100644 --- a/src/scaler/scaler-core/utils.js +++ b/src/scaler/scaler-core/utils.js @@ -31,10 +31,10 @@ const {logger} = require('../../autoscaler-common/logger'); */ function convertMillisecToHumanReadable(millisec) { // By Nofi @ https://stackoverflow.com/a/32180863 - const seconds = (millisec / 1000); - const minutes = (millisec / (1000 * 60)); - const hours = (millisec / (1000 * 60 * 60)); - const days = (millisec / (1000 * 60 * 60 * 24)); + const seconds = millisec / 1000; + const minutes = millisec / (1000 * 60); + const hours = millisec / (1000 * 60 * 60); + const days = millisec / (1000 * 60 * 60 * 24); if (seconds < 60) { return seconds.toFixed(1) + ' Sec'; @@ -61,14 +61,14 @@ function maybeRound(suggestedSize, units, label = '', projectId, instanceId) { if (units == 'NODES') { return suggestedSize; } else { - const roundTo = (suggestedSize < 1000) ? 100 : 1000; + const roundTo = suggestedSize < 1000 ? 100 : 1000; const roundedSize = Math.ceil(suggestedSize / roundTo) * roundTo; if (roundedSize != suggestedSize) { logger.debug({ - message: `\t${label}: Suggested ${suggestedSize}, rounded to ${ - roundedSize} ${units}`, + message: `\t${label}: Suggested ${suggestedSize}, rounded to ${roundedSize} ${units}`, projectId: projectId, - instanceId: instanceId}); + instanceId: instanceId, + }); } return roundedSize; } @@ -96,8 +96,9 @@ async function createProtobufMessage(jsonData) { async function publishProtoMsgDownstream(eventName, jsonData, topicId) { if (!topicId) { logger.debug( - `If you want ${eventName} messages published downstream then specify ` + - 'downstreamPubSubTopic in your config.'); + `If you want ${eventName} messages published downstream then specify ` + + 'downstreamPubSubTopic in your config.', + ); return Promise.resolve(); } @@ -106,15 +107,19 @@ async function publishProtoMsgDownstream(eventName, jsonData, topicId) { const data = Buffer.from(JSON.stringify(message.toJSON())); const attributes = {event: eventName}; - return topic.publishMessage({data: data, attributes: attributes}) - .then( () => logger.info( - `Published ${eventName} message downstream to topic: ${topicId}`)) - .catch((err) => { - logger.error({ - message: `An error occurred publishing ${ - eventName} message downstream to topic: ${topicId}`, - err: err}); + return topic + .publishMessage({data: data, attributes: attributes}) + .then(() => + logger.info( + `Published ${eventName} message downstream to topic: ${topicId}`, + ), + ) + .catch((err) => { + logger.error({ + message: `An error occurred publishing ${eventName} message downstream to topic: ${topicId}`, + err: err, }); + }); } module.exports = {