📈 PaaS-friendly metrics

config/custom-environment-variables.yml

@@ -6,6 +6,13 @@ public:
   metrics:
     prometheus:
       enabled: 'METRICS_PROMETHEUS_ENABLED'
+      endpointEnabled: 'METRICS_PROMETHEUS_ENDPOINT_ENABLED'
+    influx:
+      enabled: 'METRICS_INFLUX_ENABLED'
+      url: 'METRICS_INFLUX_URL'
+      timeoutMilliseconds: 'METRICS_INFLUX_TIMEOUT_MILLISECONDS'
+      intervalSeconds: 'METRICS_INFLUX_INTERVAL_SECONDS'
+      hostnameAsAnInstanceId: 'METRICS_INFLUX_HOSTNAME_AS_AN_INSTANCE_ID'
 
   ssl:
     isSecure: 'HTTPS'
@@ -85,3 +92,5 @@ private:
   twitch_client_id: 'TWITCH_CLIENT_ID'
   twitch_client_secret: 'TWITCH_CLIENT_SECRET'
   wheelmap_token: 'WHEELMAP_TOKEN'
+  influx_username: 'INFLUX_USERNAME'
+  influx_password: 'INFLUX_PASSWORD'

config/default.yml

@@ -5,7 +5,11 @@ public:
   metrics:
     prometheus:
       enabled: false
-
+      endpointEnabled: false
+    influx:
+      enabled: false
+      timeoutMilliseconds: 1000
+      intervalSeconds: 15
   ssl:
     isSecure: false
 

config/shields-io-production.yml

@@ -2,6 +2,7 @@ public:
   metrics:
     prometheus:
       enabled: true
+      endpointEnabled: true
 
   ssl:
     isSecure: true

core/server/in-process-server-test-helpers.js

@@ -1,21 +1,21 @@
 'use strict'
 
+const merge = require('deepmerge')
 const config = require('config').util.toObject()
+const portfinder = require('portfinder')
 const Server = require('./server')
 
-function createTestServer({ port }) {
-  const serverConfig = {
-    ...config,
-    public: {
-      ...config.public,
-      bind: {
-        ...config.public.bind,
-        port,
-      },
-    },
+async function createTestServer(customConfig = {}) {
+  const mergedConfig = merge(config, customConfig)
+  if (!mergedConfig.public.bind.port) {
+    mergedConfig.public.bind.port = await portfinder.getPortPromise()
+  }
+  const requiredInstanceMetadata = {
+    env: 'testing',
+    hostname: 'localhost',
   }
 
-  return new Server(serverConfig)
+  return new Server(mergedConfig, requiredInstanceMetadata)
 }
 
 module.exports = {

core/server/influx-metrics.js

@@ -0,0 +1,72 @@
+'use strict'
+
+const { promisify } = require('util')
+const { post } = require('request')
+const postAsync = promisify(post)
+const { promClientJsonToInfluxV2 } = require('./metrics/format-converters')
+const log = require('./log')
+
+module.exports = class InfluxMetrics {
+  constructor(metricInstance, instanceMetadata, config) {
+    this._metricInstance = metricInstance
+    this._instanceMetadata = instanceMetadata
+    this._config = config
+  }
+
+  async sendMetrics() {
+    const auth = {
+      user: this._config.username,
+      pass: this._config.password,
+    }
+    const request = {
+      uri: this._config.url,
+      headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
+      body: this.metrics(),
+      timeout: this._config.timeoutMillseconds,
+      auth,
+    }
+
+    let response
+    try {
+      response = await postAsync(request)
+    } catch (error) {
+      log.error(
+        new Error(`Cannot push metrics. Cause: ${error.name}: ${error.message}`)
+      )
+    }
+    if (response && response.statusCode >= 300) {
+      log.error(
+        new Error(
+          `Cannot push metrics. ${response.request.href} responded with status code ${response.statusCode}`
+        )
+      )
+    }
+  }
+
+  startPushingMetrics() {
+    this._intervalId = setInterval(
+      () => this.sendMetrics(),
+      this._config.intervalSeconds * 1000
+    )
+  }
+
+  metrics() {
+    const { env, hostname, id } = this._instanceMetadata
+    const { hostnameAliases = {}, hostnameAsAnInstanceId } = this._config
+    const instance = hostnameAsAnInstanceId
+      ? hostnameAliases[hostname] || hostname
+      : id
+    return promClientJsonToInfluxV2(this._metricInstance.metrics(), {
+      env,
+      application: 'shields',
+      instance,
+    })
+  }
+
+  stopPushingMetrics() {
+    if (this._intervalId) {
+      clearInterval(this._intervalId)
+      this._intervalId = undefined
+    }
+  }
+}

core/server/influx-metrics.spec.js

@@ -0,0 +1,177 @@
+'use strict'
+
+const nock = require('nock')
+const sinon = require('sinon')
+const waitForExpect = require('wait-for-expect')
+const { expect } = require('chai')
+const log = require('./log')
+const InfluxMetrics = require('./influx-metrics')
+require('../register-chai-plugins.spec')
+describe('Influx metrics', function() {
+  const metricInstance = {
+    metrics() {
+      return [
+        {
+          help: 'counter 1 help',
+          name: 'counter1',
+          type: 'counter',
+          values: [{ value: 11, labels: {} }],
+          aggregator: 'sum',
+        },
+      ]
+    },
+  }
+  const instanceMetadata = {
+    id: 'instance2',
+    env: 'test-env',
+  }
+  const config = {}
+  describe('"metrics" function', function() {
+    it('should add instance id as an instance label', async function() {
+      const influxMetrics = new InfluxMetrics(
+        metricInstance,
+        instanceMetadata,
+        config
+      )
+
+      expect(influxMetrics.metrics()).to.contain('instance=instance2')
+    })
+
+    it('should add hostname as an instance label when hostnameAsAnInstanceId is enabled', async function() {
+      const instanceMetadata = {
+        id: 'instance2',
+        env: 'test-env',
+        hostname: 'test-hostname',
+      }
+      const customConfig = {
+        hostnameAsAnInstanceId: true,
+      }
+      const influxMetrics = new InfluxMetrics(
+        metricInstance,
+        instanceMetadata,
+        customConfig
+      )
+
+      expect(influxMetrics.metrics()).to.be.contain('instance=test-hostname')
+    })
+
+    it('should use a hostname alias as an instance label', async function() {
+      const instanceMetadata = {
+        id: 'instance2',
+        env: 'test-env',
+        hostname: 'test-hostname',
+      }
+      const customConfig = {
+        hostnameAsAnInstanceId: true,
+        hostnameAliases: { 'test-hostname': 'test-hostname-aliass' },
+      }
+      const influxMetrics = new InfluxMetrics(
+        metricInstance,
+        instanceMetadata,
+        customConfig
+      )
+
+      expect(influxMetrics.metrics()).to.be.contain(
+        'instance=test-hostname-alias'
+      )
+    })
+  })
+
+  describe('startPushingMetrics', function() {
+    let influxMetrics
+    afterEach(function() {
+      influxMetrics.stopPushingMetrics()
+      nock.cleanAll()
+      nock.enableNetConnect()
+    })
+    it('should send metrics', async function() {
+      const scope = nock('http://shields-metrics.io/', {
+        reqheaders: {
+          'Content-Type': 'application/x-www-form-urlencoded',
+        },
+      })
+        .persist()
+        .post(
+          '/metrics',
+          'prometheus,application=shields,env=test-env,instance=instance2 counter1=11'
+        )
+        .basicAuth({ user: 'metrics-username', pass: 'metrics-password' })
+        .reply(200)
+      influxMetrics = new InfluxMetrics(metricInstance, instanceMetadata, {
+        url: 'http://shields-metrics.io/metrics',
+        timeoutMillseconds: 100,
+        intervalSeconds: 0,
+        username: 'metrics-username',
+        password: 'metrics-password',
+      })
+
+      influxMetrics.startPushingMetrics()
+
+      await waitForExpect(
+        () => {
+          expect(scope.isDone()).to.be.equal(
+            true,
+            `pending mocks: ${scope.pendingMocks()}`
+          )
+        },
+        1000,
+        1
+      )
+    })
+  })
+
+  describe('sendMetrics', function() {
+    beforeEach(function() {
+      sinon.spy(log, 'error')
+    })
+    afterEach(function() {
+      log.error.restore()
+      nock.cleanAll()
+      nock.enableNetConnect()
+    })
+
+    const influxMetrics = new InfluxMetrics(metricInstance, instanceMetadata, {
+      url: 'http://shields-metrics.io/metrics',
+      timeoutMillseconds: 50,
+      intervalSeconds: 0,
+      username: 'metrics-username',
+      password: 'metrics-password',
+    })
+    it('should log errors', async function() {
+      nock.disableNetConnect()
+
+      await influxMetrics.sendMetrics()
+
+      expect(log.error).to.be.calledWith(
+        sinon.match
+          .instanceOf(Error)
+          .and(
+            sinon.match.has(
+              'message',
+              'Cannot push metrics. Cause: NetConnectNotAllowedError: Nock: Disallowed net connect for "shields-metrics.io:80/metrics"'
+            )
+          )
+      )
+    })
+
+    it('should log error responses', async function() {
+      nock('http://shields-metrics.io/')
+        .persist()
+        .post('/metrics')
+        .reply(400)
+
+      await influxMetrics.sendMetrics()
+
+      expect(log.error).to.be.calledWith(
+        sinon.match
+          .instanceOf(Error)
+          .and(
+            sinon.match.has(
+              'message',
+              'Cannot push metrics. http://shields-metrics.io/metrics responded with status code 400'
+            )
+          )
+      )
+    })
+  })
+})

core/server/instance-id-generator.js

@@ -0,0 +1,10 @@
+'use strict'
+
+function generateInstanceId() {
+  // from https://gist.github.com/gordonbrander/2230317
+  return Math.random()
+    .toString(36)
+    .substr(2, 9)
+}
+
+module.exports = generateInstanceId

core/server/metrics/format-converters.js

@@ -0,0 +1,27 @@
+'use strict'
+const groupBy = require('lodash.groupby')
+
+function promClientJsonToInfluxV2(metrics, extraLabels = {}) {
+  // TODO Replace with Array.prototype.flatMap() after migrating to Node.js >= 11
+  const flatMap = (f, arr) => arr.reduce((acc, x) => acc.concat(f(x)), [])
+  return flatMap(metric => {
+    const valuesByLabels = groupBy(metric.values, value =>
+      JSON.stringify(Object.entries(value.labels).sort())
+    )
+    return Object.values(valuesByLabels).map(metricsWithSameLabel => {
+      const labels = Object.entries(metricsWithSameLabel[0].labels)
+        .concat(Object.entries(extraLabels))
+        .sort((a, b) => a[0].localeCompare(b[0]))
+        .map(labelEntry => `${labelEntry[0]}=${labelEntry[1]}`)
+        .join(',')
+      const labelsFormatted = labels ? `,${labels}` : ''
+      const values = metricsWithSameLabel
+        .sort((a, b) => a.metricName.localeCompare(b.metricName))
+        .map(value => `${value.metricName || metric.name}=${value.value}`)
+        .join(',')
+      return `prometheus${labelsFormatted} ${values}`
+    })
+  }, metrics).join('\n')
+}
+
+module.exports = { promClientJsonToInfluxV2 }