feature: Calibre Webhook

Adds a webhook that accepts metrics from Calibre and forwards them to
DataDog.

.dockerignore

@@ -20,3 +20,5 @@ node_modules
 npm-debug.log
 .vscode
 .circleci
+
+dist/

.env.test

@@ -1,2 +1,2 @@
 NODE_ENV=test
-DATADOG_AGENT_HOSTNAME=test-hostname
+DATADOG_AGENT_HOSTNAME=test-hostname

.gitignore

@@ -55,3 +55,5 @@ typings/
 # dotenv environment variables file
 .env
 
+# Dist folder
+dist/

.vscode/tasks.json

@@ -1,28 +1,44 @@
 {
   // See https://go.microsoft.com/fwlink/?LinkId=733558
   // for the documentation about the tasks.json format
-  "version": "0.1.0",
+  "version": "2.0.0",
   "command": "yarn",
-  "isShellCommand": true,
-  "showOutput": "always",
-  "suppressTaskName": true,
   "tasks": [
     {
-      "taskName": "install",
-      "args": ["install"]
+      "label": "install",
+      "type": "shell",
+      "args": [
+        "install"
+      ],
+      "problemMatcher": []
     },
     {
-      "taskName": "update",
-      "args": ["update"]
+      "label": "update",
+      "type": "shell",
+      "args": [
+        "update"
+      ],
+      "problemMatcher": []
     },
     {
-      "taskName": "test",
-      "args": ["run", "test"]
+      "label": "test",
+      "type": "shell",
+      "args": [
+        "run",
+        "test"
+      ],
+      "problemMatcher": [],
+      "group": "test"
     },
     {
-      "taskName": "build",
-      "isBuildCommand": true,
-      "args": ["run", "build"]
+      "label": "build",
+      "type": "shell",
+      "args": [
+        "run",
+        "build"
+      ],
+      "problemMatcher": [],
+      "group": "build"
     }
   ]
 }

Dockerfile

@@ -1,28 +1,30 @@
 FROM node:12.18-alpine
 
-WORKDIR /app
-
 # Install dumb-init
 # Set up deploy user
 RUN apk --no-cache --quiet add \
     dumb-init && \
     adduser -D -g '' deploy
 
+WORKDIR /app
+RUN chown deploy:deploy $(pwd)
+
+# Switch to deploy user
+USER deploy
+
 # Set up node modules
-COPY package.json yarn.lock ./
+COPY --chown=deploy:deploy package.json yarn.lock ./
 RUN yarn install --frozen-lockfile && yarn cache clean
 
+ENV PORT 8080
+EXPOSE 8080
+
 # Finally, add the rest of our app's code
 # (this is done at the end so that changes to our app's code
 # don't bust Docker's cache)
-COPY . ./
-RUN chown -R deploy:deploy ./
+COPY --chown=deploy:deploy . ./
 
-# Switch to deploy user
-USER deploy
-
-ENV PORT 8080
-EXPOSE 8080
+RUN yarn build
 
 ENTRYPOINT ["/usr/bin/dumb-init", "--"]
-CMD ["node", "index.js"]
+CMD ["node", "dist/index.js"]

hokusai/development.yml

@@ -2,7 +2,7 @@
 version: '2'
 services:
   volley:
-    command: node index.js
+    command: yarn dev
     environment:
       - NODE_ENV=development
       - PORT=5500

package.json

@@ -6,26 +6,17 @@
   "repository": "https://github.com/artsy/volley",
   "author": "Alan Johnson",
   "scripts": {
-    "start": "node index.js",
+    "build": "tsc",
+    "start": "node dist/index.js",
+    "start:debug": "node --inspect dist/index.js",
+    "dev": "yarn build && yarn start",
     "test": "jest"
   },
   "license": "MIT",
   "private": true,
   "engines": {
     "node": "12"
   },
-  "dependencies": {
-    "@koa/cors": "2.2.1",
-    "@sentry/node": "5.11.1",
-    "dd-trace": "0.16.3",
-    "dotenv": "8.2.0",
-    "ip-address": "6.1.0",
-    "koa": "2.5.1",
-    "koa-body": "2.5.0",
-    "koa-router": "7.4.0",
-    "koa-sslify": "4.0.3",
-    "node-dogstatsd": "0.0.7"
-  },
   "prettier": {
     "bracketSpacing": true,
     "semi": false,
@@ -38,12 +29,33 @@
       "git add"
     ]
   },
+  "dependencies": {
+    "@koa/cors": "2.2.1",
+    "@koa/router": "10.0.0",
+    "@sentry/node": "5.11.1",
+    "dd-trace": "0.16.3",
+    "dotenv": "8.2.0",
+    "ip-address": "6.1.0",
+    "koa": "2.5.1",
+    "koa-body": "2.5.0",
+    "koa-sslify": "5.0.0",
+    "node-dogstatsd": "0.0.7"
+  },
   "devDependencies": {
-    "jest": "24.9.0",
+    "@types/jest": "26.0.23",
+    "@types/koa": "2.13.1",
+    "@types/koa-sslify": "4.0.1",
+    "@types/koa__cors": "3.0.2",
+    "@types/koa__router": "8.0.4",
+    "@types/supertest": "2.0.11",
+    "jest": "26.6.3",
     "jest-junit": "10.0.0",
-    "supertest": "4.0.2"
+    "supertest": "4.0.2",
+    "ts-jest": "26.5.6",
+    "typescript": "4.2.4"
   },
   "jest": {
+    "preset": "ts-jest",
     "testEnvironment": "node",
     "setupFiles": [
       "dotenv/config",
@@ -60,6 +72,9 @@
     "reporters": [
       "default",
       "jest-junit"
+    ],
+    "testPathIgnorePatterns": [
+      "<rootDir>/dist"
     ]
   },
   "jest-junit": {

__test__/index.test.js → src/__test__/index.test.ts

@@ -1,14 +1,14 @@
-const request = require('supertest')
-const server = require('../index')
+import request from 'supertest'
+import { server } from '../index'
 
 describe("the server's reporting capabilities", () => {
   beforeEach(() => {
     console.error = jest.fn()
   })
 
   afterEach(() => {
-    server.close()
-    console.error.mockClear()
+    server.close(() => {})
+    ;(console.error as any).mockClear()
   })
 
   it('responds that the server is healthy', async () => {

src/__test__/postMetric.test.js → src/app/__test__/postMetric.test.ts

@@ -1,8 +1,8 @@
-const initializePostMetric = require('../postMetric')
+import { initialize as initializePostMetric } from '../postMetric'
 
 describe('postMetric', () => {
-  let statsdClient
-  let postMetric
+  let statsdClient: any
+  let postMetric: any
 
   beforeEach(() => {
     statsdClient = {
@@ -19,7 +19,7 @@ describe('postMetric', () => {
   })
 
   afterEach(() => {
-    console.error.mockClear()
+    (console.error as any).mockClear()
   })
 
   describe('with no whitelists', () => {

src/__test__/tracer.test.js → src/app/__test__/tracer.test.ts

@@ -1,5 +1,5 @@
 const tracer = require('dd-trace')
-const initDataDogTracer = require('../tracer')
+const { initDataDogTracer } = require('../tracer')
 
 jest.mock('dd-trace', () => ({
   init: jest.fn(),

src/app/calibreWebhookRoute.ts

@@ -0,0 +1,40 @@
+import { CalibrePage } from '../common/calibre'
+import { VolleyApi } from '../common/volley'
+import Koa from 'koa'
+import Router from '@koa/router'
+import type { postMetric } from './postMetric'
+
+const CALIBRE_SERVICE_PREFIX = process.env.CALIBRE_SERVICE_PREFIX || 'calibre'
+
+const CALIBRE_PRODUCTION_SITE = process.env.CALIBRE_PRODUCTION_SITE ? process.env.CALIBRE_PRODUCTION_SITE.split('/') : [ 'artsy_net' ]
+
+function sanitize(data: string): string {
+  return data.replace(/[- ]/g, "_")
+}
+
+export const calibreWebhookRoute = (postMetric: postMetric) => {
+  return (ctx: Koa.ParameterizedContext<any, Router.RouterParamContext<any, {}>, any>) => {
+    const siteId = sanitize(ctx.request.body.site_id)
+    const environment = (CALIBRE_PRODUCTION_SITE.indexOf(siteId) !== -1) ? 'production' : 'staging'
+
+    const pages = ctx.request.body.pages as CalibrePage[]
+    for (let page of pages) {
+      const pageId = sanitize(page.id)
+      const pageDevice = sanitize(page.profile).toLowerCase()
+      const deviceType = (pageDevice === 'chrome_desktop') ? 'desktop' : 'mobile'
+
+      for (let metric of page.metrics) {
+        const prefix = `${CALIBRE_SERVICE_PREFIX}.${siteId}`
+
+        const volley = VolleyApi.fromCalibreApi(metric, [
+          `env:${environment}`, `form:${deviceType}`, `mode:${pageDevice}`, `page:${pageId}`
+        ])
+
+        postMetric(prefix, volley)
+      }
+    }
+
+    ctx.status = 202
+    ctx.body = 'OK'
+  }
+}

src/postMetric.js → src/app/postMetric.ts

@@ -1,15 +1,17 @@
-module.exports = function initialize(
-  statsdClient,
-  metricNameWhitelist,
-  metricTagWhitelist
-) {
-  return function postMetric(serviceName, metricData) {
+export type postMetric = (serviceName: string, metricData: object) => void
+
+export const initialize = (
+  statsdClient: any,
+  metricNameWhitelist: any,
+  metricTagWhitelist: any
+): postMetric => {
+  return function postMetric(serviceName: any, metricData: any) {
     if (metricNameWhitelist.length && !metricNameWhitelist.includes(metricData.name)) {
       console.error(`Metric name "${metricData.name}" not in white list.`)
     } else if (
       metricTagWhitelist.length &&
       metricData.tags &&
-      metricData.tags.some(tag => !metricTagWhitelist.includes(tag))
+      metricData.tags.some((tag: string) => !metricTagWhitelist.includes(tag))
     ) {
       console.error(
         `Metric tags "${metricData.tags}" has tags not in white list.`

src/tracer.js → src/app/tracer.ts

@@ -1,6 +1,6 @@
-const tracer = require('dd-trace')
+import tracer from 'dd-trace'
 
-module.exports = function init() {
+export const initDataDogTracer = () => {
   const ddServicePrefix = 'volley'
 
   tracer.init({

src/common/calibre.ts

@@ -0,0 +1,29 @@
+/**
+ * The shape of a calibre page report
+ */
+export type CalibrePage = {
+  id: string
+  uuid: string
+  name: string
+  status: string
+  endpoint: string
+  canonical: string
+  profile: string
+  profile_uuid: string
+  metrics: CalibreMetric[]
+}
+
+export type CalibreMetric = {
+  name: string
+  value: number
+  /**
+   * Accepted values
+   *
+   * fileSize
+   * gradeScore
+   * humanDuration
+   * milliunit
+   * trust
+   */
+  formatter: string
+}

src/common/volley.ts

@@ -0,0 +1,32 @@
+import { CalibreMetric } from './calibre'
+
+interface VolleyMetric {
+    type: 'timing' | 'increment' | 'incrementBy' | 'decrement' | 'decrementBy' | 'gauge' | 'histogram' | 'set'
+    name: string,
+    value: number
+}
+
+interface VolleySampledMetric extends VolleyMetric {
+    sampleRate: number
+}
+
+interface VolleyTaggedMetric extends VolleyMetric {
+    tags: string[]
+}
+
+interface VolleyTaggedSampledMetric extends VolleySampledMetric {
+    tags: string[]
+}
+
+export type VolleyMetrics = VolleyMetric | VolleySampledMetric | VolleyTaggedMetric | VolleyTaggedSampledMetric
+
+export class VolleyApi {
+  public static fromCalibreApi(calibreMetric: CalibreMetric, tags: string[]): VolleyMetrics {
+    return {
+      type: 'gauge',
+      name: calibreMetric.name.replace(/[- ]/g, "_"),
+      value: calibreMetric.value,
+      tags: tags,
+    }
+  }
+}