From 7c8639b553ba80c00d323f46d11e46851b8dc38a Mon Sep 17 00:00:00 2001
From: Matt Simerson <matt@tnpi.net>
Date: Mon, 23 Jan 2017 17:57:35 -0800
Subject: [PATCH 1/5] progress

---
 README.md     |  5 +--
 index.js      | 92 +++++++++++++++++++++++++++++++++++++++++++++------
 package.json  |  5 ++-
 test/index.js | 56 +++++++++++++++++++++++++++++++
 4 files changed, 145 insertions(+), 13 deletions(-)
 create mode 100644 test/index.js

diff --git a/README.md b/README.md
index b7e2429..efbcb1f 100644
--- a/README.md
+++ b/README.md
@@ -5,8 +5,9 @@ Increase the reputation of domains you send email to.
 
 # How it works
 
-When you[r users] send emails, this plugin will increment counters for each recipient domain.
-When emails later arrive from domains your users have sent email to, reputation engines like
+This plugin looks at outgoing emails that have been successfully queued. Outgoing messages are determined by inspecting the `relaying` property of the connection. If `relaying=true`, then the connection has (via AUTH credentials or IP ACLs) been extended a form of trust.
+
+In those outbound emails, the sender domain and recipient domains are parsed out and a redis entry is inserted/updated. Later when emails arrive from domains your users have sent email to, reputation engines like
  [karma](https://github.com/haraka/haraka-plugin-karma) can observe this and boost their deliverability.
 
 
diff --git a/index.js b/index.js
index 360ed6d..8902b0e 100644
--- a/index.js
+++ b/index.js
@@ -3,28 +3,76 @@ var tlds = require('haraka-tld');
 exports.register = function () {
   var plugin = this;
 
-  plugin.register_hook('queue_ok', 'after_queue');
+  plugin.register_hook('mail',     'check_sender');
+  plugin.register_hook('rcpt_ok',  'check_recipient');
+  plugin.register_hook('queue_ok', 'update_sender');
 }
 
-exports.after_queue = function (next, connection, params) {
+exports.check_sender = function (next, connection, params) {
+  var plugin = this;
+
+  if (connection.relaying) return next();
+
+  var sender_od = plugin.get_sender_domain(connection.transaction);
+
+  if (plugin.has_fcrdns_match(sender_od, connection)) return next();
+  if (plugin.has_spf_match(sender_od, connection)) return next();
+
+  // no other auth mechanisms to test
+  return next();
+}
+
+exports.check_recipient = function (next, connection, rcpt) {
+  var plugin = this;
+  // a plugin has vouched that the rcpt is for a domain we accept mail for
+
+  function errNext (err) {
+    plugin.logerror(err);
+    next();
+  }
+
+  // if no validated sender domain, there's nothing to do here
+  if (!plugin.validated_sender_od) return errNext('no valid sender od');
+
+  if (!rcpt.host) return errNext('rcpt.host unset?');
+
+  var rcpt_od = tlds.get_organizational_domain(rcpt.host);
+  if (!rcpt_od) return errNext('no rcpt od');
+
+  // DO THE CHECK
+
+  return next();
+}
+
+exports.update_sender = function (next, connection, params) {
   var plugin = this;
   // queue_ok arguments: next, connection, msg
   // ok 1390590369 qp 634 (F82E2DD5-9238-41DC-BC95-9C3A02716AD2.1)
 
+  function errNext (err) {
+    plugin.logerror(err);
+    next();
+  }
+
   plugin.loginfo(plugin, params);
-  if (!connection) return next();
-  if (!connection.transaction) return next();
+  if (!connection) return errNext('no connection');
+  if (!connection.transaction) return errNext('no transaction');
+  if (!connection.relaying) return errNext('not relaying');
   var txn = connection.transaction;
 
   var sender_od = plugin.get_sender_domain(txn);
-  if (!sender_od) return next();
-  var rcpt_domains = plugin.get_recipient_domains(txn);
-  if (rcpt_domains.length === 0) return next();
-
+  if (!sender_od) return errNext('no sender domain');
   plugin.loginfo('sender domain: ' + sender_od);
-  plugin.loginfo('recip domains: ' + rcpt_domains.join(','));
 
-  next();
+  var rcpt_domains = plugin.get_recipient_domains(txn);
+  if (rcpt_domains.length === 0) {
+    plugin.logerror('no recipient domains');
+  }
+  else {
+    plugin.loginfo('recip domains: ' + rcpt_domains.join(','));
+  }
+
+  next(undefined, sender_od, rcpt_domains);
 }
 
 exports.get_recipient_domains = function (txn) {
@@ -40,6 +88,7 @@ exports.get_recipient_domains = function (txn) {
       plugin.loginfo('rcpt: ' + txn.rcpt_to[i].host + ' -> ' + rcpt_od);
     }
     if (rcpt_domains.indexOf(rcpt_od) === -1) {
+      // not a duplicate, add to the list
       rcpt_domains.push(rcpt_od);
     }
   }
@@ -56,4 +105,27 @@ exports.get_sender_domain = function (txn) {
     plugin.loginfo('sender: ' + txn.mail_from.host + ' -> ' + sender_od);
   }
   return sender_od;
+}
+
+exports.has_fcrdns_match = function (sender_od, connection) {
+  var plugin = this;
+  var fcrdns = connection.results.get('fcrdns');
+  if (!fcrdns) return false;
+  if (!fcrdns.fcrdns) return false;
+
+  var fcrdns_od = tlds.get_organizational_domain(fcrdns.fcrdns);
+  if (fcrdns_od !== sender_od) return false;
+
+  plugin.validated_sender_od = sender_od;
+  return true;
+}
+
+exports.has_spf_match = function (sender_od, connection) {
+  var plugin = this;
+
+  var spf = connection.results.get('spf');
+  if (!spf) return false;
+
+  plugin.loginfo(spf);
+  return false;
 }
\ No newline at end of file
diff --git a/package.json b/package.json
index 56c4731..d079a86 100644
--- a/package.json
+++ b/package.json
@@ -4,6 +4,7 @@
   "description": "Increase the reputation of domains you send email to.",
   "main": "index.js",
   "scripts": {
+    "lint": "./node_modules/.bin/eslint *.js test/*.js",
     "test": "mocha"
   },
   "repository": {
@@ -24,7 +25,9 @@
   },
   "homepage": "https://github.com/haraka/haraka-plugin-domain-counters#readme",
   "devDependencies": {
-    "haraka-test-fixtures": "^1.0.13"
+    "address-rfc2821": "*",
+    "eslint": "^3",
+    "haraka-test-fixtures": "*"
   },
   "dependencies": {
     "haraka-tld": "^1.0.12"
diff --git a/test/index.js b/test/index.js
new file mode 100644
index 0000000..0a0c3d1
--- /dev/null
+++ b/test/index.js
@@ -0,0 +1,56 @@
+
+var assert   = require('assert');
+
+var Address  = require('address-rfc2821').Address;
+var fixtures = require('haraka-test-fixtures');
+
+
+
+describe('after_queue', function () {
+
+  var plugin = fixtures.plugin('index');
+  var connection;
+
+  beforeEach(function (done) {
+    connection = fixtures.connection.createConnection();
+    connection.relaying = true;
+    connection.transaction = fixtures.transaction.createTransaction();
+    done();
+  });
+
+  it('gets the sender domain', function (done) {
+    connection.transaction.mail_from = new Address('<johndoe@example.com>');
+
+    plugin.after_queue(function (undefined, sender_dom, rcpt_doms) {
+      assert.equal(sender_dom, 'example.com');
+      assert.deepEqual(rcpt_doms, [])
+      done();
+    },
+    connection);
+  });
+
+  it('gets the recipient domain', function (done) {
+    connection.transaction.mail_from = new Address('<johndoe@example.com>');
+    connection.transaction.rcpt_to.push(new Address('<jane@test1.com>'));
+
+    plugin.after_queue(function (undefined, sender_dom, rcpt_doms) {
+      assert.equal(sender_dom, 'example.com');
+      assert.deepEqual(rcpt_doms, ['test1.com']);
+      done();
+    },
+    connection);
+  });
+
+  it('gets the recipient domains', function (done) {
+    connection.transaction.mail_from = new Address('<johndoe@example.com>');
+    connection.transaction.rcpt_to.push(new Address('<jane@test1.com>'));
+    connection.transaction.rcpt_to.push(new Address('<jane@test2.com>'));
+
+    plugin.after_queue(function (undefined, sender_dom, rcpt_doms) {
+      assert.equal(sender_dom, 'example.com');
+      assert.deepEqual(rcpt_doms, ['test1.com', 'test2.com']);
+      done();
+    },
+    connection);
+  });
+});
\ No newline at end of file

From 93318ff3e19be2b8db6a822d324625574b33b9ee Mon Sep 17 00:00:00 2001
From: Matt Simerson <matt@tnpi.net>
Date: Mon, 23 Jan 2017 20:30:51 -0800
Subject: [PATCH 2/5] added fcrdns & spf OD tests

---
 index.js      | 33 ++++++++++++++++-----
 test/index.js | 82 +++++++++++++++++++++++++++++++++++++++++++++++----
 2 files changed, 103 insertions(+), 12 deletions(-)

diff --git a/index.js b/index.js
index 8902b0e..d471103 100644
--- a/index.js
+++ b/index.js
@@ -1,22 +1,28 @@
+'use strict';
+
 var tlds = require('haraka-tld');
 
 exports.register = function () {
   var plugin = this;
 
-  plugin.register_hook('mail',     'check_sender');
+  plugin.register_hook('mail',     'check_sender_early');
   plugin.register_hook('rcpt_ok',  'check_recipient');
   plugin.register_hook('queue_ok', 'update_sender');
 }
 
-exports.check_sender = function (next, connection, params) {
+exports.check_sender_early = function (next, connection, params) {
   var plugin = this;
 
   if (connection.relaying) return next();
 
   var sender_od = plugin.get_sender_domain(connection.transaction);
 
-  if (plugin.has_fcrdns_match(sender_od, connection)) return next();
-  if (plugin.has_spf_match(sender_od, connection)) return next();
+  if (plugin.has_fcrdns_match(sender_od, connection)) {
+    return next(null, sender_od);
+  }
+  if (plugin.has_spf_match(sender_od, connection)) {
+    return next(null, sender_od);
+  }
 
   // no other auth mechanisms to test
   return next();
@@ -75,6 +81,7 @@ exports.update_sender = function (next, connection, params) {
   next(undefined, sender_od, rcpt_domains);
 }
 
+
 exports.get_recipient_domains = function (txn) {
   var plugin = this;
 
@@ -121,11 +128,23 @@ exports.has_fcrdns_match = function (sender_od, connection) {
 }
 
 exports.has_spf_match = function (sender_od, connection) {
-  var plugin = this;
 
   var spf = connection.results.get('spf');
-  if (!spf) return false;
+  if (spf && spf.domain && spf.result === 'pass') {
+    // scope=helo (HELO/EHLO)
+    if (tlds.get_organizational_domain(spf.domain) === sender_od) {
+      return true;
+    }
+  }
+
+  spf = connection.transaction.results.get('spf');
+  if (spf && spf.domain && spf.result === 'pass') {
+    // scope=mfrom (HELO/EHLO)
+    if (tlds.get_organizational_domain(spf.domain) === sender_od) {
+      return true;
+    }
+  }
 
-  plugin.loginfo(spf);
+  // this.loginfo(spf);
   return false;
 }
\ No newline at end of file
diff --git a/test/index.js b/test/index.js
index 0a0c3d1..63ea918 100644
--- a/test/index.js
+++ b/test/index.js
@@ -4,9 +4,80 @@ var assert   = require('assert');
 var Address  = require('address-rfc2821').Address;
 var fixtures = require('haraka-test-fixtures');
 
+describe('check_sender_early', function () {
 
+  var plugin = fixtures.plugin('index');
+  var connection;
+
+  beforeEach(function (done) {
+    connection = fixtures.connection.createConnection();
+    connection.results = new fixtures.result_store(connection);
+    connection.transaction = fixtures.transaction.createTransaction();
+    connection.transaction.results = new fixtures.result_store(connection);
+    done();
+  });
+
+  it('returns empty when no auth found', function (done) {
+    connection.transaction.mail_from = new Address('<johndoe@test.com>');
+    plugin.check_sender_early(function (undefined, sender_od) {
+      assert.equal(sender_od, undefined);
+      done();
+    },
+    connection);
+  });
+
+  it('finds OD from FCrDNS match', function (done) {
+    connection.transaction.mail_from = new Address('<johndoe@validated-test.com>');
+    connection.results.add({ name: 'fcrdns'}, {fcrdns: 'validated-test.com'});
+    plugin.check_sender_early(function (undefined, sender_od) {
+      assert.equal(sender_od, 'validated-test.com');
+      done();
+    },
+    connection);
+  });
 
-describe('after_queue', function () {
+  it('misses OD on FCrDNS miss', function (done) {
+    connection.transaction.mail_from = new Address('<johndoe@invalid-test.com>');
+    connection.results.add({ name: 'fcrdns'}, {fcrdns: 'valid-test.com'});
+    plugin.check_sender_early(function (undefined, sender_od) {
+      assert.equal(sender_od, undefined);
+      done();
+    },
+    connection);
+  });
+
+  it('finds OD on SPF mfrom match', function (done) {
+    connection.transaction.mail_from = new Address('<johndoe@spf-mfrom.com>');
+    connection.transaction.results.add({ name: 'spf'}, {
+      scope: 'mfrom',
+      result: 'pass',
+      domain: 'spf-mfrom.com',
+    });
+
+    plugin.check_sender_early(function (undefined, sender_od) {
+      assert.equal(sender_od, 'spf-mfrom.com');
+      done();
+    },
+    connection);
+  });
+
+  it('finds OD on SPF helo match', function (done) {
+    connection.transaction.mail_from = new Address('<johndoe@helo-pass.com>');
+    connection.results.add({ name: 'spf'}, {
+      scope: 'helo',
+      result: 'pass',
+      domain: 'helo-pass.com',
+    });
+
+    plugin.check_sender_early(function (undefined, sender_od) {
+      assert.equal(sender_od, 'helo-pass.com');
+      done();
+    },
+    connection);
+  });
+});
+
+describe('update_sender', function () {
 
   var plugin = fixtures.plugin('index');
   var connection;
@@ -21,7 +92,7 @@ describe('after_queue', function () {
   it('gets the sender domain', function (done) {
     connection.transaction.mail_from = new Address('<johndoe@example.com>');
 
-    plugin.after_queue(function (undefined, sender_dom, rcpt_doms) {
+    plugin.update_sender(function (undefined, sender_dom, rcpt_doms) {
       assert.equal(sender_dom, 'example.com');
       assert.deepEqual(rcpt_doms, [])
       done();
@@ -33,7 +104,7 @@ describe('after_queue', function () {
     connection.transaction.mail_from = new Address('<johndoe@example.com>');
     connection.transaction.rcpt_to.push(new Address('<jane@test1.com>'));
 
-    plugin.after_queue(function (undefined, sender_dom, rcpt_doms) {
+    plugin.update_sender(function (undefined, sender_dom, rcpt_doms) {
       assert.equal(sender_dom, 'example.com');
       assert.deepEqual(rcpt_doms, ['test1.com']);
       done();
@@ -46,11 +117,12 @@ describe('after_queue', function () {
     connection.transaction.rcpt_to.push(new Address('<jane@test1.com>'));
     connection.transaction.rcpt_to.push(new Address('<jane@test2.com>'));
 
-    plugin.after_queue(function (undefined, sender_dom, rcpt_doms) {
+    plugin.update_sender(function (undefined, sender_dom, rcpt_doms) {
       assert.equal(sender_dom, 'example.com');
       assert.deepEqual(rcpt_doms, ['test1.com', 'test2.com']);
       done();
     },
     connection);
   });
-});
\ No newline at end of file
+});
+

From 92605fd1b7219fd8545d870855b4920f2934cbcb Mon Sep 17 00:00:00 2001
From: Matt Simerson <matt@tnpi.net>
Date: Tue, 24 Jan 2017 09:12:58 -0800
Subject: [PATCH 3/5] start redis support

---
 index.js      | 21 +++++++++++++++++++--
 test/index.js | 12 ++++++++++--
 2 files changed, 29 insertions(+), 4 deletions(-)

diff --git a/index.js b/index.js
index d471103..69eac95 100644
--- a/index.js
+++ b/index.js
@@ -4,12 +4,23 @@ var tlds = require('haraka-tld');
 
 exports.register = function () {
   var plugin = this;
+  plugin.inherits('redis');
+
+  plugin.load_sender_ini();
+  plugin.load_redis_ini();
 
   plugin.register_hook('mail',     'check_sender_early');
   plugin.register_hook('rcpt_ok',  'check_recipient');
   plugin.register_hook('queue_ok', 'update_sender');
 }
 
+exports.load_sender_ini = function () {
+  var plugin = this;
+  plugin.config.get('sender.ini', function () {
+    plugin.load_sender_ini();
+  });
+}
+
 exports.check_sender_early = function (next, connection, params) {
   var plugin = this;
 
@@ -78,10 +89,13 @@ exports.update_sender = function (next, connection, params) {
     plugin.loginfo('recip domains: ' + rcpt_domains.join(','));
   }
 
+
+
   next(undefined, sender_od, rcpt_domains);
 }
 
 
+
 exports.get_recipient_domains = function (txn) {
   var plugin = this;
 
@@ -128,19 +142,22 @@ exports.has_fcrdns_match = function (sender_od, connection) {
 }
 
 exports.has_spf_match = function (sender_od, connection) {
+  var plugin = this;
 
   var spf = connection.results.get('spf');
-  if (spf && spf.domain && spf.result === 'pass') {
+  if (spf && spf.domain && spf.result === 'Pass') {
     // scope=helo (HELO/EHLO)
     if (tlds.get_organizational_domain(spf.domain) === sender_od) {
+      plugin.validated_sender_od = sender_od;
       return true;
     }
   }
 
   spf = connection.transaction.results.get('spf');
-  if (spf && spf.domain && spf.result === 'pass') {
+  if (spf && spf.domain && spf.result === 'Pass') {
     // scope=mfrom (HELO/EHLO)
     if (tlds.get_organizational_domain(spf.domain) === sender_od) {
+      plugin.validated_sender_od = sender_od;
       return true;
     }
   }
diff --git a/test/index.js b/test/index.js
index 63ea918..62d8f07 100644
--- a/test/index.js
+++ b/test/index.js
@@ -50,7 +50,7 @@ describe('check_sender_early', function () {
     connection.transaction.mail_from = new Address('<johndoe@spf-mfrom.com>');
     connection.transaction.results.add({ name: 'spf'}, {
       scope: 'mfrom',
-      result: 'pass',
+      result: 'Pass',
       domain: 'spf-mfrom.com',
     });
 
@@ -65,7 +65,7 @@ describe('check_sender_early', function () {
     connection.transaction.mail_from = new Address('<johndoe@helo-pass.com>');
     connection.results.add({ name: 'spf'}, {
       scope: 'helo',
-      result: 'pass',
+      result: 'Pass',
       domain: 'helo-pass.com',
     });
 
@@ -77,6 +77,14 @@ describe('check_sender_early', function () {
   });
 });
 
+describe('check_recipient', function () {
+  it('reduces domain to OD', function (done) {
+
+    assert.equal(true, false);
+    done();
+  });
+});
+
 describe('update_sender', function () {
 
   var plugin = fixtures.plugin('index');

From 8706c2e51d37c316fbe5c28297f80091709e4f65 Mon Sep 17 00:00:00 2001
From: Matt Simerson <matt@tnpi.net>
Date: Wed, 25 Jan 2017 15:26:14 -0800
Subject: [PATCH 4/5] * add config/known-senders.ini * add inherits('redis') *
 store authed sender & rcpt ODs in results * perform checks against redis &
 add pass results

---
 README.md                |  23 ++--
 config/known-senders.ini |   6 +
 index.js                 | 259 +++++++++++++++++++++++++++++----------
 package.json             |  11 +-
 test/index.js            |  37 ++++--
 5 files changed, 241 insertions(+), 95 deletions(-)
 create mode 100644 config/known-senders.ini

diff --git a/README.md b/README.md
index efbcb1f..b0ae521 100644
--- a/README.md
+++ b/README.md
@@ -1,23 +1,19 @@
-# haraka-plugin-domain-counters
+# haraka-plugin-known-senders
 
-Increase the reputation of domains you send email to.
+Boost the reputation of domains you send email to.
 
 
-# How it works
+## How it works
 
-This plugin looks at outgoing emails that have been successfully queued. Outgoing messages are determined by inspecting the `relaying` property of the connection. If `relaying=true`, then the connection has (via AUTH credentials or IP ACLs) been extended a form of trust.
+This plugin inspects outgoing emails and adds the destination domains to a known senders database. When emails arrive from those known senders, this plugin stores a result object noting that.
 
-In those outbound emails, the sender domain and recipient domains are parsed out and a redis entry is inserted/updated. Later when emails arrive from domains your users have sent email to, reputation engines like
- [karma](https://github.com/haraka/haraka-plugin-karma) can observe this and boost their deliverability.
+### TL;DR
 
+Outgoing messages are determined by inspecting the `relaying` property of the connection. If `relaying=true`, then the connection has been extended a form of trust, usually via AUTH credentials or IP ACLs. In those outbound emails, the sender domain and recipient domains are parsed and a redis entry is inserted/updated.
 
-# PLAN
+When emails later arrive from a domain your users have sent email to, the redis DB is checked and if a match is found, a result object is stored in the transaction results. That result can be scored by reputation engines like
+ [karma](https://github.com/haraka/haraka-plugin-karma) and used to affect the messages deliverability.
 
-- for domains we send to, increment when the message is queued
-- for domains we receive from, increment only when:
-    - message has been queued
-
-There's no attempt to validate outbound (sent) email domains.
 
 # MULTI-TENANCY
 
@@ -44,8 +40,7 @@ This plugin can operate in two contexts where the incoming sender is validated a
 
 ## Authentication
 
-Email has several authentication mechanisms that can validate that a sending host
-has authority to send on behalf of the [purported] domain:
+Email has several authentication mechanisms that can validate that a sending host has authority to send on behalf of the [purported] domain:
 
 - FCrDNS: [Forward Confirmed reverse DNS](https://en.wikipedia.org/wiki/Forward-confirmed_reverse_DNS)
 - SPF: [Sender Policy Framework](https://en.wikipedia.org/wiki/Sender_Policy_Framework)
diff --git a/config/known-senders.ini b/config/known-senders.ini
new file mode 100644
index 0000000..c64d2a3
--- /dev/null
+++ b/config/known-senders.ini
@@ -0,0 +1,6 @@
+
+[redis]
+host   = 127.0.0.1
+port   = 6379
+db     = 3
+
diff --git a/index.js b/index.js
index 69eac95..134fd6d 100644
--- a/index.js
+++ b/index.js
@@ -7,59 +7,34 @@ exports.register = function () {
   plugin.inherits('redis');
 
   plugin.load_sender_ini();
-  plugin.load_redis_ini();
 
-  plugin.register_hook('mail',     'check_sender_early');
-  plugin.register_hook('rcpt_ok',  'check_recipient');
-  plugin.register_hook('queue_ok', 'update_sender');
+  plugin.register_hook('init_master',  'init_redis_plugin');
+  plugin.register_hook('init_child',   'init_redis_plugin');
+
+  plugin.register_hook('mail',      'is_authenticated');
+  plugin.register_hook('rcpt_ok',   'check_recipient');
+  plugin.register_hook('queue_ok',  'update_sender');
+  plugin.register_hook('data_post', 'is_dkim_authenticated');
 }
 
 exports.load_sender_ini = function () {
   var plugin = this;
-  plugin.config.get('sender.ini', function () {
+
+  plugin.cfg = plugin.config.get('known-senders.ini', function () {
     plugin.load_sender_ini();
   });
-}
-
-exports.check_sender_early = function (next, connection, params) {
-  var plugin = this;
-
-  if (connection.relaying) return next();
-
-  var sender_od = plugin.get_sender_domain(connection.transaction);
-
-  if (plugin.has_fcrdns_match(sender_od, connection)) {
-    return next(null, sender_od);
-  }
-  if (plugin.has_spf_match(sender_od, connection)) {
-    return next(null, sender_od);
-  }
 
-  // no other auth mechanisms to test
-  return next();
+  plugin.merge_redis_ini();
 }
 
-exports.check_recipient = function (next, connection, rcpt) {
-  var plugin = this;
-  // a plugin has vouched that the rcpt is for a domain we accept mail for
-
-  function errNext (err) {
-    plugin.logerror(err);
-    next();
-  }
-
-  // if no validated sender domain, there's nothing to do here
-  if (!plugin.validated_sender_od) return errNext('no valid sender od');
-
-  if (!rcpt.host) return errNext('rcpt.host unset?');
-
-  var rcpt_od = tlds.get_organizational_domain(rcpt.host);
-  if (!rcpt_od) return errNext('no rcpt od');
-
-  // DO THE CHECK
-
-  return next();
-}
+/*
+ *                 Outbound Processing
+ *
+ * Identify and save to Redis domains the local users send email to
+ *
+ * Context: these functions run after a message has been queued.
+ *
+*/
 
 exports.update_sender = function (next, connection, params) {
   var plugin = this;
@@ -67,34 +42,55 @@ exports.update_sender = function (next, connection, params) {
   // ok 1390590369 qp 634 (F82E2DD5-9238-41DC-BC95-9C3A02716AD2.1)
 
   function errNext (err) {
-    plugin.logerror(err);
+    connection.logerror(plugin, 'update_sender: ' + err);
     next();
   }
 
-  plugin.loginfo(plugin, params);
+  // connection.loginfo(plugin, params);
   if (!connection) return errNext('no connection');
   if (!connection.transaction) return errNext('no transaction');
-  if (!connection.relaying) return errNext('not relaying');
+  if (!connection.relaying) return next();
   var txn = connection.transaction;
 
   var sender_od = plugin.get_sender_domain(txn);
   if (!sender_od) return errNext('no sender domain');
-  plugin.loginfo('sender domain: ' + sender_od);
 
   var rcpt_domains = plugin.get_recipient_domains(txn);
   if (rcpt_domains.length === 0) {
-    plugin.logerror('no recipient domains');
-  }
-  else {
-    plugin.loginfo('recip domains: ' + rcpt_domains.join(','));
+    connection.logerror(plugin, 'update_sender: no rcpt ODs for ' + sender_od);
+    return next();
   }
 
+  // within this function, the sender is a local domain
+  // and the recipient is an external domain
+  var multi = plugin.db.multi();
+  for (let i = 0; i < rcpt_domains.length; i++) {
+    multi.hincrby(sender_od, rcpt_domains[i], 1);
+  }
 
-
-  next(undefined, sender_od, rcpt_domains);
+  multi.exec(function (err, replies) {
+    if (err) {
+      connection.logerror(plugin, err);
+      return next();
+    }
+    for (let i = 0; i < rcpt_domains.length; i++) {
+      connection.loginfo(plugin, 'saved ' + sender_od + ' : ' + rcpt_domains[i] + ' : ' + replies[i]);
+    }
+    next(null, null, sender_od, rcpt_domains);
+  });
 }
 
+exports.get_sender_domain = function (txn) {
+  var plugin = this;
 
+  if (!txn.mail_from) return;
+  if (!txn.mail_from.host) return;
+  var sender_od = tlds.get_organizational_domain(txn.mail_from.host);
+  if (txn.mail_from.host !== sender_od) {
+    plugin.logdebug('sender: ' + txn.mail_from.host + ' -> ' + sender_od);
+  }
+  return sender_od;
+}
 
 exports.get_recipient_domains = function (txn) {
   var plugin = this;
@@ -106,7 +102,7 @@ exports.get_recipient_domains = function (txn) {
     if (!txn.rcpt_to[i].host) continue;
     var rcpt_od = tlds.get_organizational_domain(txn.rcpt_to[i].host);
     if (txn.rcpt_to[i].host !== rcpt_od) {
-      plugin.loginfo('rcpt: ' + txn.rcpt_to[i].host + ' -> ' + rcpt_od);
+      plugin.logdebug('rcpt: ' + txn.rcpt_to[i].host + ' -> ' + rcpt_od);
     }
     if (rcpt_domains.indexOf(rcpt_od) === -1) {
       // not a duplicate, add to the list
@@ -116,16 +112,147 @@ exports.get_recipient_domains = function (txn) {
   return rcpt_domains;
 }
 
-exports.get_sender_domain = function (txn) {
+/*
+ *                Inbound Processing
+ *
+ * Look for sender domains we can validate against something. Anything..
+ *   FCrDNS, SPF, DKIM, verified TLS host name, etc..
+ *
+ * When verified / validated sender domains are found, check to see if
+ * their recipients have ever sent mail to their domain.
+*/
+
+// early checks, on the mail hook
+exports.is_authenticated = function (next, connection, params) {
   var plugin = this;
 
-  if (!txn.mail_from) return;
-  if (!txn.mail_from.host) return;
-  var sender_od = tlds.get_organizational_domain(txn.mail_from.host);
-  if (txn.mail_from.host !== sender_od) {
-    plugin.loginfo('sender: ' + txn.mail_from.host + ' -> ' + sender_od);
+  // only validate inbound messages
+  if (connection.relaying) return next();
+
+  var sender_od = plugin.get_sender_domain(connection.transaction);
+
+  if (plugin.has_fcrdns_match(sender_od, connection)) {
+    connection.loginfo(plugin, '+fcrdns: ' + sender_od);
+    return next(null, null, sender_od);
   }
-  return sender_od;
+  if (plugin.has_spf_match(sender_od, connection)) {
+    connection.loginfo(plugin, '+spf: ' + sender_od);
+    return next(null, null, sender_od);
+  }
+
+  // TODO: TLS verified domain?
+
+  return next();
+}
+
+function get_sender_od (connection) {
+  if (!connection) return;
+  if (!connection.transaction) return;
+  var txn_res = connection.transaction.results.get('known-senders');
+  if (!txn_res) return;
+  return txn_res.sender;
+}
+
+function get_rcpt_ods (connection) {
+  if (!connection) return;
+  if (!connection.transaction) return;
+
+  var txn_r = connection.transaction.results.get('known-senders');
+  if (!txn_r) return;
+
+  return txn_r.rcpt_ods;
+}
+
+function already_matched (connection) {
+  var res = connection.transaction.results.get({ name: 'known-senders'});
+  if (!res) return false;
+  return (res.pass && res.pass.length) ? true : false;
+}
+
+exports.check_recipient = function (next, connection, rcpt) {
+  var plugin = this;
+  // rcpt is a valid local email address. Some rcpt_to.* plugin has
+  // accepted it.
+
+  // inbound only
+  if (connection.relaying) return next();
+
+  function errNext (err) {
+    connection.logerror(plugin, 'check_recipient: ' + err);
+    next();
+  }
+
+  if (!rcpt.host) return errNext('rcpt.host unset?');
+
+  // reduce the host portion of the email address to an OD
+  var rcpt_od = tlds.get_organizational_domain(rcpt.host);
+  if (!rcpt_od) return errNext('no rcpt od for ' + rcpt.host);
+
+  connection.transaction.results.push(plugin, { rcpt_ods: rcpt_od });
+
+  // if no validated sender domain, there's nothing to do...yet
+  var sender_od = get_sender_od(connection);
+  if (!sender_od) return next();
+
+  // The sender OD is validated, check Redis for a match
+  plugin.db.hget(rcpt_od, sender_od, function (err, reply) {
+    if (err) {
+      plugin.logerror(err);
+      return next();
+    }
+    connection.loginfo(plugin, rcpt_od + ' : ' + sender_od + ' : ' + reply);
+    if (reply) {
+      connection.transaction.results.add(plugin, { pass: rcpt_od, count: reply });
+    }
+    return next(null, null, rcpt_od);
+  });
+}
+
+exports.is_dkim_authenticated = function (next, connection) {
+  var plugin = this;
+  if (connection.relaying) return next();
+  if (already_matched(connection)) return next();
+
+  var sender_od = get_sender_od(connection);
+  if (!sender_od) return next();
+
+  var rcpt_ods = get_rcpt_ods(connection);
+  if (!rcpt_ods || ! rcpt_ods.length) return next();
+
+  var dkim = connection.results.get('dkim_verify');
+  if (!dkim) return next();
+  connection.loginfo(plugin, dkim);
+  if (!dkim.pass || !dkim.pass.length) return next();
+
+  var multi = plugin.db.multi();
+
+  for (let i = 0; i < dkim.pass.length; i++) {
+    var dkim_od = tlds.get_organizational_domain(dkim.pass[i]);
+    if (dkim_od === sender_od) {
+      connection.transaction.results.add(plugin, { sender: sender_od, auth: 'dkim' });
+      for (let j = 0; j < rcpt_ods.length; j++) {
+        multi.hget(rcpt_ods[j], sender_od);
+      }
+    }
+  }
+
+  multi.exec(function (err, replies) {
+    if (err) {
+      connection.logerror(plugin, err);
+      return next();
+    }
+
+    connection.loginfo(plugin, 'is_dkim_auth: ');
+    connection.loginfo(plugin, replies);
+
+    for (let j = 0; j < rcpt_ods.length; j++) {
+      connection.loginfo(plugin, rcpt_ods[j] + ' : ' + sender_od + ' : ' + replies[j]);
+      if (replies[j]) {
+        connection.transaction.results.add(plugin, { pass: rcpt_ods[j], count: replies[j] });
+      }
+    }
+    return next(null, null, rcpt_ods);
+  });
 }
 
 exports.has_fcrdns_match = function (sender_od, connection) {
@@ -134,10 +261,12 @@ exports.has_fcrdns_match = function (sender_od, connection) {
   if (!fcrdns) return false;
   if (!fcrdns.fcrdns) return false;
 
+  connection.loginfo(plugin, fcrdns.fcrdns);
+
   var fcrdns_od = tlds.get_organizational_domain(fcrdns.fcrdns);
   if (fcrdns_od !== sender_od) return false;
 
-  plugin.validated_sender_od = sender_od;
+  connection.transaction.results.add(plugin, {sender: sender_od, auth: 'fcrdns'});
   return true;
 }
 
@@ -148,7 +277,7 @@ exports.has_spf_match = function (sender_od, connection) {
   if (spf && spf.domain && spf.result === 'Pass') {
     // scope=helo (HELO/EHLO)
     if (tlds.get_organizational_domain(spf.domain) === sender_od) {
-      plugin.validated_sender_od = sender_od;
+      connection.transaction.results.add(plugin, {sender: sender_od});
       return true;
     }
   }
@@ -157,11 +286,11 @@ exports.has_spf_match = function (sender_od, connection) {
   if (spf && spf.domain && spf.result === 'Pass') {
     // scope=mfrom (HELO/EHLO)
     if (tlds.get_organizational_domain(spf.domain) === sender_od) {
-      plugin.validated_sender_od = sender_od;
+      connection.transaction.results.add(plugin, {sender: sender_od, auth: 'spf' });
       return true;
     }
   }
 
   // this.loginfo(spf);
   return false;
-}
\ No newline at end of file
+}
diff --git a/package.json b/package.json
index d079a86..593fa6d 100644
--- a/package.json
+++ b/package.json
@@ -1,7 +1,7 @@
 {
-  "name": "haraka-plugin-domain-counters",
+  "name": "haraka-plugin-known-senders",
   "version": "1.0.0",
-  "description": "Increase the reputation of domains you send email to.",
+  "description": "Increase the reputation of recognized sender domains.",
   "main": "index.js",
   "scripts": {
     "lint": "./node_modules/.bin/eslint *.js test/*.js",
@@ -9,21 +9,22 @@
   },
   "repository": {
     "type": "git",
-    "url": "git+https://github.com/haraka/haraka-plugin-domain-counters.git"
+    "url": "git+https://github.com/haraka/haraka-plugin-known-senders.git"
   },
   "keywords": [
     "Haraka",
     "domain",
     "reputation",
+    "well-known",
     "sender",
     "whitelist"
   ],
   "author": "Matt Simerson <matt@tnpi.net>",
   "license": "MIT",
   "bugs": {
-    "url": "https://github.com/haraka/haraka-plugin-domain-counters/issues"
+    "url": "https://github.com/haraka/haraka-plugin-known-senders/issues"
   },
-  "homepage": "https://github.com/haraka/haraka-plugin-domain-counters#readme",
+  "homepage": "https://github.com/haraka/haraka-plugin-known-senders#readme",
   "devDependencies": {
     "address-rfc2821": "*",
     "eslint": "^3",
diff --git a/test/index.js b/test/index.js
index 62d8f07..931c834 100644
--- a/test/index.js
+++ b/test/index.js
@@ -4,7 +4,7 @@ var assert   = require('assert');
 var Address  = require('address-rfc2821').Address;
 var fixtures = require('haraka-test-fixtures');
 
-describe('check_sender_early', function () {
+describe('is_authenticated', function () {
 
   var plugin = fixtures.plugin('index');
   var connection;
@@ -19,7 +19,7 @@ describe('check_sender_early', function () {
 
   it('returns empty when no auth found', function (done) {
     connection.transaction.mail_from = new Address('<johndoe@test.com>');
-    plugin.check_sender_early(function (undefined, sender_od) {
+    plugin.is_authenticated(function (null1, null2, sender_od) {
       assert.equal(sender_od, undefined);
       done();
     },
@@ -29,7 +29,7 @@ describe('check_sender_early', function () {
   it('finds OD from FCrDNS match', function (done) {
     connection.transaction.mail_from = new Address('<johndoe@validated-test.com>');
     connection.results.add({ name: 'fcrdns'}, {fcrdns: 'validated-test.com'});
-    plugin.check_sender_early(function (undefined, sender_od) {
+    plugin.is_authenticated(function (null1, null2, sender_od) {
       assert.equal(sender_od, 'validated-test.com');
       done();
     },
@@ -39,7 +39,7 @@ describe('check_sender_early', function () {
   it('misses OD on FCrDNS miss', function (done) {
     connection.transaction.mail_from = new Address('<johndoe@invalid-test.com>');
     connection.results.add({ name: 'fcrdns'}, {fcrdns: 'valid-test.com'});
-    plugin.check_sender_early(function (undefined, sender_od) {
+    plugin.is_authenticated(function (null1, null2, sender_od) {
       assert.equal(sender_od, undefined);
       done();
     },
@@ -54,7 +54,7 @@ describe('check_sender_early', function () {
       domain: 'spf-mfrom.com',
     });
 
-    plugin.check_sender_early(function (undefined, sender_od) {
+    plugin.is_authenticated(function (null1, null2, sender_od) {
       assert.equal(sender_od, 'spf-mfrom.com');
       done();
     },
@@ -69,7 +69,7 @@ describe('check_sender_early', function () {
       domain: 'helo-pass.com',
     });
 
-    plugin.check_sender_early(function (undefined, sender_od) {
+    plugin.is_authenticated(function (null1, null2, sender_od) {
       assert.equal(sender_od, 'helo-pass.com');
       done();
     },
@@ -78,11 +78,26 @@ describe('check_sender_early', function () {
 });
 
 describe('check_recipient', function () {
-  it('reduces domain to OD', function (done) {
+  var connection;
 
-    assert.equal(true, false);
+  beforeEach(function (done) {
+    connection = fixtures.connection.createConnection();
+    connection.relaying = true;
+    connection.transaction = fixtures.transaction.createTransaction();
     done();
   });
+
+  it('reduces domain to OD', function (done) {
+    var plugin = fixtures.plugin('index');
+    plugin.validated_sender_od = 'example.com';
+
+    plugin.check_recipient(function (null1, null2, reply) {
+      assert.equal(reply, 'example.com');
+      done();
+    },
+    connection,
+    new Address('<user@host.example.com>'));
+  });
 });
 
 describe('update_sender', function () {
@@ -100,7 +115,7 @@ describe('update_sender', function () {
   it('gets the sender domain', function (done) {
     connection.transaction.mail_from = new Address('<johndoe@example.com>');
 
-    plugin.update_sender(function (undefined, sender_dom, rcpt_doms) {
+    plugin.update_sender(function (null1, null2, sender_dom, rcpt_doms) {
       assert.equal(sender_dom, 'example.com');
       assert.deepEqual(rcpt_doms, [])
       done();
@@ -112,7 +127,7 @@ describe('update_sender', function () {
     connection.transaction.mail_from = new Address('<johndoe@example.com>');
     connection.transaction.rcpt_to.push(new Address('<jane@test1.com>'));
 
-    plugin.update_sender(function (undefined, sender_dom, rcpt_doms) {
+    plugin.update_sender(function (null1, null2, sender_dom, rcpt_doms) {
       assert.equal(sender_dom, 'example.com');
       assert.deepEqual(rcpt_doms, ['test1.com']);
       done();
@@ -125,7 +140,7 @@ describe('update_sender', function () {
     connection.transaction.rcpt_to.push(new Address('<jane@test1.com>'));
     connection.transaction.rcpt_to.push(new Address('<jane@test2.com>'));
 
-    plugin.update_sender(function (undefined, sender_dom, rcpt_doms) {
+    plugin.update_sender(function (null1, null2, sender_dom, rcpt_doms) {
       assert.equal(sender_dom, 'example.com');
       assert.deepEqual(rcpt_doms, ['test1.com', 'test2.com']);
       done();

From 575395c2921d0b94ca45274b7706adc99a9a1d4e Mon Sep 17 00:00:00 2001
From: Matt Simerson <matt@tnpi.net>
Date: Wed, 25 Jan 2017 17:01:42 -0800
Subject: [PATCH 5/5] slap on a mock redis so tests can pass

---
 index.js      | 43 +++++++++++++++++++++++--------------------
 test/index.js | 45 ++++++++++++++++++++++++++++++++++++++++++---
 2 files changed, 65 insertions(+), 23 deletions(-)

diff --git a/index.js b/index.js
index 134fd6d..a3a3a70 100644
--- a/index.js
+++ b/index.js
@@ -43,7 +43,7 @@ exports.update_sender = function (next, connection, params) {
 
   function errNext (err) {
     connection.logerror(plugin, 'update_sender: ' + err);
-    next();
+    next(null, null, sender_od, rcpt_domains);
   }
 
   // connection.loginfo(plugin, params);
@@ -57,8 +57,7 @@ exports.update_sender = function (next, connection, params) {
 
   var rcpt_domains = plugin.get_recipient_domains(txn);
   if (rcpt_domains.length === 0) {
-    connection.logerror(plugin, 'update_sender: no rcpt ODs for ' + sender_od);
-    return next();
+    return errNext('no rcpt ODs for ' + sender_od);
   }
 
   // within this function, the sender is a local domain
@@ -145,19 +144,21 @@ exports.is_authenticated = function (next, connection, params) {
   return next();
 }
 
-function get_sender_od (connection) {
+exports.get_sender_od = function (connection) {
+  var plugin = this;
   if (!connection) return;
   if (!connection.transaction) return;
-  var txn_res = connection.transaction.results.get('known-senders');
+  var txn_res = connection.transaction.results.get(plugin.name);
   if (!txn_res) return;
   return txn_res.sender;
 }
 
-function get_rcpt_ods (connection) {
+exports.get_rcpt_ods = function (connection) {
+  var plugin = this;
   if (!connection) return;
   if (!connection.transaction) return;
 
-  var txn_r = connection.transaction.results.get('known-senders');
+  var txn_r = connection.transaction.results.get(plugin.name);
   if (!txn_r) return;
 
   return txn_r.rcpt_ods;
@@ -191,7 +192,7 @@ exports.check_recipient = function (next, connection, rcpt) {
   connection.transaction.results.push(plugin, { rcpt_ods: rcpt_od });
 
   // if no validated sender domain, there's nothing to do...yet
-  var sender_od = get_sender_od(connection);
+  var sender_od = plugin.get_sender_od(connection);
   if (!sender_od) return next();
 
   // The sender OD is validated, check Redis for a match
@@ -211,18 +212,23 @@ exports.check_recipient = function (next, connection, rcpt) {
 exports.is_dkim_authenticated = function (next, connection) {
   var plugin = this;
   if (connection.relaying) return next();
-  if (already_matched(connection)) return next();
 
-  var sender_od = get_sender_od(connection);
-  if (!sender_od) return next();
+  function errNext (err) {
+    connection.logerror(plugin, 'is_dkim_authenticated: ' + err);
+    return next(null, null, rcpt_ods);
+  }
+
+  if (already_matched(connection)) return errNext('already matched');
 
-  var rcpt_ods = get_rcpt_ods(connection);
-  if (!rcpt_ods || ! rcpt_ods.length) return next();
+  var sender_od = plugin.get_sender_od(connection);
+  if (!sender_od) return errNext('no sender_od');
 
-  var dkim = connection.results.get('dkim_verify');
-  if (!dkim) return next();
-  connection.loginfo(plugin, dkim);
-  if (!dkim.pass || !dkim.pass.length) return next();
+  var rcpt_ods = plugin.get_rcpt_ods(connection);
+  if (!rcpt_ods || ! rcpt_ods.length) return errNext('no rcpt_ods');
+
+  var dkim = connection.transaction.results.get('dkim_verify');
+  if (!dkim) return errNext('no dkim_verify results');
+  if (!dkim.pass || !dkim.pass.length) return errNext('no dkim pass');
 
   var multi = plugin.db.multi();
 
@@ -242,9 +248,6 @@ exports.is_dkim_authenticated = function (next, connection) {
       return next();
     }
 
-    connection.loginfo(plugin, 'is_dkim_auth: ');
-    connection.loginfo(plugin, replies);
-
     for (let j = 0; j < rcpt_ods.length; j++) {
       connection.loginfo(plugin, rcpt_ods[j] + ' : ' + sender_od + ' : ' + replies[j]);
       if (replies[j]) {
diff --git a/test/index.js b/test/index.js
index 931c834..60c2193 100644
--- a/test/index.js
+++ b/test/index.js
@@ -82,8 +82,9 @@ describe('check_recipient', function () {
 
   beforeEach(function (done) {
     connection = fixtures.connection.createConnection();
-    connection.relaying = true;
+    // connection.relaying = true;
     connection.transaction = fixtures.transaction.createTransaction();
+    connection.transaction.results = new fixtures.result_store(connection);
     done();
   });
 
@@ -91,8 +92,9 @@ describe('check_recipient', function () {
     var plugin = fixtures.plugin('index');
     plugin.validated_sender_od = 'example.com';
 
-    plugin.check_recipient(function (null1, null2, reply) {
-      assert.equal(reply, 'example.com');
+    plugin.check_recipient(function () {
+      var res = connection.transaction.results.get(plugin.name);
+      assert.equal(res.rcpt_ods[0], 'example.com');
       done();
     },
     connection,
@@ -103,6 +105,12 @@ describe('check_recipient', function () {
 describe('update_sender', function () {
 
   var plugin = fixtures.plugin('index');
+  plugin.db = {
+    multi :function () { return plugin.db; },
+    hget : function () {},
+    hincrby : function () {},
+    exec: function (cb) { if (cb) cb(null, []); },
+  }
   var connection;
 
   beforeEach(function (done) {
@@ -149,3 +157,34 @@ describe('update_sender', function () {
   });
 });
 
+describe('is_dkim_authenticated', function () {
+
+  var plugin = fixtures.plugin('index');
+  plugin.db = {
+    multi : function () { return plugin.db; },
+    hget : function () {},
+    hincrby : function () {},
+    exec: function (cb) { cb(null, []); },
+  }
+  var connection;
+
+  beforeEach(function (done) {
+    connection = fixtures.connection.createConnection();
+    connection.transaction = fixtures.transaction.createTransaction();
+    connection.transaction.results = new fixtures.result_store(connection);
+    done();
+  });
+
+  it('finds dkim results', function (done) {
+    connection.transaction.results.add(plugin, { sender: 'sender.com' });
+    connection.transaction.results.push(plugin, { rcpt_ods: 'rcpt.com' });
+    connection.transaction.results.add({name: 'dkim_verify'}, { pass: 'sender.com'});
+
+    plugin.is_dkim_authenticated(function () {
+      var res = connection.transaction.results.get(plugin.name);
+      assert.equal('dkim', res.auth);
+      done();
+    },
+    connection);
+  });
+});