Use ledger-client@0.9.8 to support excludeOnly option

app/ledger.js

@@ -93,6 +93,7 @@ var client
 const clientOptions = {
   debugP: process.env.LEDGER_DEBUG,
   loggingP: process.env.LEDGER_LOGGING,
+  rulesTestP: process.env.LEDGER_RULES_TESTING,
   verboseP: process.env.LEDGER_VERBOSE,
   server: process.env.LEDGER_SERVER_URL
 }
@@ -1232,6 +1233,8 @@ var excludeP = (publisher, callback) => {
     if (callback) callback(err, result)
   }
 
+  if (!v2RulesetDB) return setTimeout(() => { excludeP(publisher, callback) }, 5 * msecs.second)
+
   inspectP(v2RulesetDB, v2RulesetPath, publisher, 'exclude', 'domain:' + publisher, (err, result) => {
     var props
 
@@ -1495,16 +1498,26 @@ var callback = (err, result, delayTime) => {
 
     entries = []
     results.forEach((entry) => {
-      entries.push({ type: 'put',
-        key: entry.facet + ':' + entry.publisher,
-        value: JSON.stringify(underscore.omit(entry, [ 'facet', 'publisher' ]))
-      })
+      var key = entry.facet + ':' + entry.publisher
+
+      if (entry.exclude !== false) {
+        entries.push({ type: 'put', key: key, value: JSON.stringify(underscore.omit(entry, [ 'facet', 'publisher' ])) })
+      } else {
+        entries.push({ type: 'del', key: key })
+      }
     })
 
     v2RulesetDB.batch(entries, (err) => {
       if (err) return console.log(v2RulesetPath + ' error: ' + JSON.stringify(err, null, 2))
 
-      underscore.keys(synopsis.publishers).forEach((publisher) => { excludeP(publisher) })
+      if (entries.length === 0) return
+
+      underscore.keys(synopsis.publishers).forEach((publisher) => {
+// be safe...
+        if (synopsis.publishers[publisher]) delete synopsis.publishers[publisher].options.exclude
+
+        excludeP(publisher)
+      })
     })
   }
   if (result.publishersV2) {
@@ -1701,6 +1714,31 @@ var run = (delayTime) => {
  * ledger client utilities
  */
 
+/* code that may never be needed...
+
+var rulesV2Reset = (callback) => {
+  if (clientOptions.verboseP) console.log('\n\nreset rulesets')
+  if (!v2RulesetDB) return
+
+  if (client) {
+    delete client.state.rulesV2Stamp
+    client.state.updatesStamp = underscore.now()
+  }
+  v2RulesetDB.close((err) => {
+    if (err) console.log(v2RulesetPath + ' close error: ' + err.toString())
+
+    v2RulesetDB = null
+    require('leveldown').destroy(pathName(v2RulesetPath), (err) => {
+      if (err) console.log(v2RulesetPath + ' destroy error: ' + err.toString())
+
+      v2RulesetDB = levelup(pathName(v2RulesetPath))
+      callback()
+    })
+  })
+}
+
+*/
+
 var getStateInfo = (state) => {
   var ballots, i, transaction
   var info = state.paymentInfo

package.json

@@ -104,7 +104,7 @@
     "keytar": "^3.0.0",
     "l20n": "^3.5.1",
     "ledger-balance": "^0.8.64",
-    "ledger-client": "^0.9.7",
+    "ledger-client": "^0.9.9",
     "ledger-geoip": "^0.8.75",
     "ledger-publisher": "^0.9.0",
     "lru-cache": "^1.0.0",

test/taxonomy/exclusionListTest.js

@@ -0,0 +1,125 @@
+/* global describe, it, before */
+
+/* jshint asi: true, node: true, laxbreak: true, laxcomma: true, undef: true, unused: true */
+const levelup = require('level')
+const assert = require('assert')
+
+const includes = (db, domain, done) => {
+  db.get('domain:' + domain, function (err, value) {
+    assert.equal(!!err, true)
+    done()
+  })
+}
+
+const excludes = (db, domain, done) => {
+  db.get('domain:' + domain, function (err, value) {
+    if (err) throw err
+    const parsedValue = JSON.parse(value)
+    assert.equal(parsedValue && parsedValue.exclude, true)
+    done()
+  })
+}
+
+const excludesSubdomain = (db, subdomain, done) => {
+  db.get('SLD:' + subdomain, function (err, value) {
+    if (err) throw err
+    const parsedValue = JSON.parse(value)
+    assert.equal(parsedValue && parsedValue.exclude, true)
+    done()
+  })
+}
+
+const excludesTLD = (db, tld, done) => {
+  db.get('TLD:' + tld, function (err, value) {
+    if (err) throw err
+    const parsedValue = JSON.parse(value)
+    assert.equal(parsedValue && parsedValue.exclude, true)
+    done()
+  })
+}
+
+const waitForDownload = (db, done) => {
+  const entries = []
+
+  db.createKeyStream()
+  .on('data', function (data) {
+    entries.push(data)
+  }).on('error', function (err) {
+    throw err
+  }).on('end', function () {
+    if (entries.length === 0) {
+      const err = 'No entries found in leveldb;\n' +
+        'manually run `LEDGER_VERBOSE=1 LEDGER_RULES_TESTING=1 npm start` and wait for all entries to download\n' +
+        '(takes about 6 minutes).\n' +
+        'If running in CI, make a test case that just sleeps for 10 minutes and run prior to this.'
+      throw err
+    }
+    done()
+  })
+}
+
+let dbHandle
+
+// https://github.com/Level/levelup
+
+describe('smoketest for our exclusion list', function () {
+  before(function (done) {
+    // TODO(bsclifton): only works on macOS at the moment
+    const filepath =
+      require('os').homedir() +
+      '/Library/Application Support' +
+      '/brave-development/ledger-rulesV2.leveldb'
+
+    levelup(filepath,
+      {},
+      function (err, db) {
+        if (err) throw err
+        dbHandle = db
+
+        // TODO: wait until the DB has entries
+        waitForDownload(db, done)
+      })
+  })
+
+  describe('includes', function () {
+    it('brianbondy.com', function (done) {
+      includes(dbHandle, 'brianbondy.com', done)
+    })
+    it('clifton.io', function (done) {
+      includes(dbHandle, 'clifton.io', done)
+    })
+    it('coindesk.com', function (done) {
+      includes(dbHandle, 'coindesk.com', done)
+    })
+    it('cnn.com', function (done) {
+      includes(dbHandle, 'cnn.com', done)
+    })
+    it('cnet.com', function (done) {
+      includes(dbHandle, 'cnet.com', done)
+    })
+    it('Hacker News', function (done) {
+      includes(dbHandle, 'news.ycombinator.com', done)
+    })
+  })
+
+  describe('excludes', function () {
+    it('bankofamerica.com', function (done) {
+      excludes(dbHandle, 'bankofamerica.com', done)
+    })
+    it('brave.com', function (done) {
+      excludes(dbHandle, 'brave.com', done)
+    })
+    it('Facebook', function (done) {
+      excludesSubdomain(dbHandle, 'facebook', done)
+    })
+    it('.gov TLD', function (done) {
+      excludesTLD(dbHandle, 'gov', done)
+    })
+    it('Google', function (done) {
+      excludesSubdomain(dbHandle, 'google', done)
+    })
+    it('walmart.com', function (done) {
+      excludes(dbHandle, 'walmart.com', done)
+    })
+  })
+})