Add end-point to collect brave-core usage data

  * Add collection endpoint
  * Updated failing tests
  * Add tests for new collection end point

package.json

@@ -11,7 +11,7 @@
     "start": "./node_modules/.bin/babel-node src/index.js",
     "lint": "standard",
     "verify": "node tools/verify.js",
-    "test": "tap test/*.js"
+    "test": "BEHIND_FASTLY=1 tap test/*.js"
   },
   "author": "Brave",
   "license": "MPL-2.0",

src/controllers/braveCore.js

@@ -0,0 +1,65 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+let Joi = require('joi')
+
+let platforms = ['osx-bc', 'winia32-bc', 'winx64-bc', 'linux-bc']
+let channels = ['dev', 'release', 'nightly', 'beta']
+let booleanString = ['true', 'false']
+
+let validator = {
+  query: {
+    platform: Joi.valid(platforms).required(),
+    channel: Joi.valid(channels).required(),
+    version: Joi.string().required(),
+    daily: Joi.valid(booleanString).required(),
+    weekly: Joi.valid(booleanString).required(),
+    monthly: Joi.valid(booleanString).required(),
+    first: Joi.valid(booleanString).required(),
+    woi: Joi.string(),
+    ref: Joi.string()
+  }
+}
+
+// Build a usage object if query parameters passed in
+let buildUsage = (request) => {
+  if (request.query.daily) {
+    return {
+      daily: request.query.daily === 'true',
+      weekly: request.query.weekly === 'true',
+      monthly: request.query.monthly === 'true',
+      platform: request.query.platform || 'unknown',
+      version: request.query.version || 'unknown',
+      first: request.query.first === 'true',
+      channel: request.query.channel || 'unknown',
+      woi: request.query.woi || '2016-01-04',
+      ref: request.query.ref || 'none'
+    }
+  } else {
+    return null
+  }
+}
+
+exports.setup = (runtime) => {
+  const get = {
+    method: 'GET',
+    path: '/1/usage/brave-core',
+    config: {
+      handler: function (request, reply) {
+        var usage = buildUsage(request)
+        runtime.mongo.models.insertBraveCoreUsage(usage, (err, results) => {
+          if (err) {
+            console.log(err.toString())
+            reply({ ts: (new Date()).getTime(), status: 'error', message: err }).code(500)
+          } else {
+            reply({ ts: (new Date()).getTime(), status: 'ok' })
+          }
+        })
+      },
+      validate: validator
+    }
+  }
+
+  return [get]
+}

src/db.js

@@ -33,6 +33,7 @@ exports.setup = (amqpSender, done) => {
     const usageCollection = connection.collection('usage')
     const androidUsageCollection = connection.collection('android_usage')
     const iosUsageCollection = connection.collection('ios_usage')
+    const braveCoreUsageCollection = connection.collection('brave_core_usage')
     const crashesCollection = connection.collection('crashes')
 
     // install a series of model data handlers on connection
@@ -82,6 +83,26 @@ exports.setup = (amqpSender, done) => {
         }
       },
 
+      // insert brave core usage record
+      insertBraveCoreUsage: (usage, done) => {
+        if (usage) {
+          const invalid = Joi.validate(usage, usageSchema)
+          if (invalid.error) {
+            done(invalid, null)
+          } else {
+            // store the current timestamp in epoch seconds
+            usage.ts = (new Date()).getTime()
+            usage.year_month_day = moment().format('YYYY-MM-DD')
+            // store as a useful backup
+            console.log(JSON.stringify(usage))
+            braveCoreUsageCollection.insertOne(usage, done)
+          }
+        } else {
+          // Null usage indicates no values passed
+          done(null, {})
+        }
+      },
+
       // insert iOS usage record
       insertIOSUsage: (usage, done) => {
         if (usage) {

src/index.js

@@ -48,9 +48,10 @@ mq.setup((sender) => {
     let crashes = require('./controllers/crashes').setup(runtime)
     let monitoring = require('./controllers/monitoring').setup(runtime)
 
-    // GET /1/usage/[ios|android]
+    // GET /1/usage/[ios|android|brave-core]
     let androidRoutes = require('./controllers/android').setup(runtime)
     let iosRoutes = require('./controllers/ios').setup(runtime)
+    let braveCoreRoutes = require('./controllers/braveCore').setup(runtime)
 
     // promotional proxy
     let promoProxy = []
@@ -102,7 +103,7 @@ mq.setup((sender) => {
     server.route(
       [
         common.root
-      ].concat(releaseRoutes, extensionRoutes, crashes, monitoring, androidRoutes, iosRoutes, promoProxy)
+      ].concat(releaseRoutes, extensionRoutes, crashes, monitoring, androidRoutes, iosRoutes, braveCoreRoutes, promoProxy)
     )
 
     server.start((err) => {

test/androidController.js

@@ -22,7 +22,9 @@ var expected = {
   version: '1.2.3',
   first: true,
   channel: 'dev',
-  platform: 'android'
+  platform: 'android',
+  ref: 'none',
+  woi: '2016-01-04'
 }
 
 tap.test('Android Controller', function (t) {

test/braveCoreController.js

@@ -0,0 +1,57 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+var tap = require('tap')
+var _ = require('underscore')
+var ctrl = require('../src/controllers/braveCore')
+
+var query = {
+  daily: 'true',
+  weekly: 'true',
+  monthly: 'true',
+  version: '1.2.3',
+  first: 'true',
+  channel: 'dev',
+  platform: 'winia32-bc'
+}
+
+var expected = {
+  daily: true,
+  weekly: true,
+  monthly: true,
+  version: '1.2.3',
+  first: true,
+  channel: 'dev',
+  platform: 'winia32-bc',
+  ref: 'none',
+  woi: '2016-01-04'
+}
+
+tap.test('Brave Core Controller', function (t) {
+  var runtimeMock = {
+    mongo: {
+      models: {
+        insertBraveCoreUsage: function (usage, cb) {
+          t.ok(_.isObject(usage), 'usage is an object')
+          t.same(usage, expected, 'usage built correctly')
+          cb(null, 'ok')
+        }
+      }
+    }
+  }
+  var replyMock = function (obj) {
+    t.ok(obj.ts, 'timestamp returned')
+    t.ok(obj.status === 'ok', 'status ok')
+  }
+  var requestMock = {
+    query: query,
+    info: {
+      remoteAddress: '1.1.1.1'
+    },
+    headers: {}
+  }
+  var endpoints = ctrl.setup(runtimeMock)
+  endpoints[0].config.handler(requestMock, replyMock)
+  t.plan(4)
+})

test/extensions.js

@@ -57,43 +57,49 @@ const availableExtensions = [
 ]
 
 tap.test('Extracts extension information from requests', (test) => {
-  tap.same(getRequestedExtensions(onePasswordRequest('0.0.0.0')), [['aomjjhallfgjeglblehebfpbcfeobpgk', '0.0.0.0']])
-  tap.same(getRequestedExtensions(onePasswordRequest('4.5.9.90')), [['aomjjhallfgjeglblehebfpbcfeobpgk', '4.5.9.90']])
-  tap.same(getRequestedExtensions(onePasswordAndPDFJSRequest('4.5.9.90', '1.5.444')), [['aomjjhallfgjeglblehebfpbcfeobpgk', '4.5.9.90'], ['jdbefljfgobbmcidnmpjamcbhnbphjnb', '1.5.444']])
+  tap.same(getRequestedExtensions(onePasswordRequest('0.0.0.0')),
+    { requestedExtensions: [['aomjjhallfgjeglblehebfpbcfeobpgk', '0.0.0.0']], version: '3.0'}
+  )
+  tap.same(getRequestedExtensions(onePasswordRequest('4.5.9.90')),
+    { requestedExtensions: [['aomjjhallfgjeglblehebfpbcfeobpgk', '4.5.9.90']], version: '3.0'}
+  )
+  tap.same(getRequestedExtensions(onePasswordAndPDFJSRequest('4.5.9.90', '1.5.444')),
+    { requestedExtensions: [['aomjjhallfgjeglblehebfpbcfeobpgk', '4.5.9.90'], ['jdbefljfgobbmcidnmpjamcbhnbphjnb', '1.5.444']], version: '3.0' }
+  )
   tap.equal(getRequestedExtensions(unsupportedProtocolRequest), undefined)
   test.end()
 })
 
 tap.test('Initial update for an extension works', (test) => {
-  tap.same(getExtensionsWithUpdates(availableExtensions, getRequestedExtensions(onePasswordRequest('0.0.0.0'))),
+  tap.same(getExtensionsWithUpdates(availableExtensions, getRequestedExtensions(onePasswordRequest('0.0.0.0')).requestedExtensions),
     [
       ['aomjjhallfgjeglblehebfpbcfeobpgk', '4.5.9.90', 'f75d7808766429ec63ec41d948c1cb6a486407945d604961c6adf54fe3f459b7']
     ])
   test.end()
 })
 
 tap.test('No updates returned for same version', (test) => {
-  tap.same(getExtensionsWithUpdates(availableExtensions, getRequestedExtensions(onePasswordRequest('4.5.9.90'))), [])
+  tap.same(getExtensionsWithUpdates(availableExtensions, getRequestedExtensions(onePasswordRequest('4.5.9.90')).requestedExtensions), [])
   test.end()
 })
 
 tap.test('No updates returned for unknown extension ID', (test) => {
-  tap.same(getExtensionsWithUpdates(availableExtensions, getRequestedExtensions(unknownExtensionRequest('0.0.0.0'))), [])
+  tap.same(getExtensionsWithUpdates(availableExtensions, getRequestedExtensions(unknownExtensionRequest('0.0.0.0')).requestedExtensions), [])
   test.end()
 })
 
 tap.test('No updates returned for newer extension ID', (test) => {
-  tap.same(getExtensionsWithUpdates(availableExtensions, getRequestedExtensions(onePasswordRequest('9.5.9.90'))), [])
+  tap.same(getExtensionsWithUpdates(availableExtensions, getRequestedExtensions(onePasswordRequest('9.5.9.90')).requestedExtensions), [])
   test.end()
 })
 
 tap.test('Blank update request returns no updates', (test) => {
-  tap.same(getExtensionsWithUpdates(availableExtensions, getRequestedExtensions(noUpdatesRequest)), [])
+  tap.same(getExtensionsWithUpdates(availableExtensions, getRequestedExtensions(noUpdatesRequest, []).requestedExtensions), [])
   test.end()
 })
 
 tap.test('Update for multiple extensions works', (test) => {
-  tap.same(getExtensionsWithUpdates(availableExtensions, getRequestedExtensions(onePasswordAndPDFJSRequest('0.0.0.0', '0.0.0.0'))),
+  tap.same(getExtensionsWithUpdates(availableExtensions, getRequestedExtensions(onePasswordAndPDFJSRequest('0.0.0.0', '0.0.0.0')).requestedExtensions),
     [
       ['aomjjhallfgjeglblehebfpbcfeobpgk', '4.5.9.90', 'f75d7808766429ec63ec41d948c1cb6a486407945d604961c6adf54fe3f459b7'],
       ['jdbefljfgobbmcidnmpjamcbhnbphjnb', '1.5.444', '25689984431ca8a60f087c761f472e500a7fe8a9065a4a47e92559237bcd1d6d']

test/iosController.js

@@ -18,7 +18,9 @@ var expected = {
   version: '1.2.3',
   first: true,
   channel: 'dev',
-  platform: 'ios'
+  platform: 'ios',
+  ref: 'none',
+  woi: '2016-01-04'
 }
 
 tap.test('iOS Controller', function (t) {

test/ip.js

@@ -10,10 +10,10 @@ tap.test('retrieve IP address', function (t) {
   var requestMock
   requestMock = {
     headers: {
-      'X-Forwarded-For': '1.1.1.1,2.2.2.2'
+      'x-forwarded-for': '1.1.1.1,2.2.2.2'
     }
   }
-  t.ok(common.ipAddressFrom(requestMock) === '1.1.1.1', 'retrieve forwarded ip address')
+  t.equal(common.ipAddressFrom(requestMock), '1.1.1.1', 'retrieve forwarded ip address')
   requestMock = {
     headers: {},
     info: {