rec: Support exporting more record types via protobuf

pdns/dnsrecords.hh

@@ -211,6 +211,10 @@ class SPFRecordContent : public DNSRecordContent
 {
 public:
   includeboilerplate(SPF)
+  const std::string& getText() const
+  {
+    return d_text;
+  }
 
 private:
   string d_text;
@@ -240,6 +244,7 @@ class PTRRecordContent : public DNSRecordContent
 public:
   includeboilerplate(PTR)
   explicit PTRRecordContent(const DNSName& content) : d_content(content){}
+  const DNSName& getContent() const { return d_content; }
 private:
   DNSName d_content;
 };

pdns/lwres.cc

@@ -66,7 +66,7 @@ static void logOutgoingQuery(std::shared_ptr<RemoteLogger> outgoingLogger, boost
   outgoingLogger->queueData(str);
 }
 
-static void logIncomingResponse(std::shared_ptr<RemoteLogger> outgoingLogger, boost::optional<const boost::uuids::uuid&> initialRequestId, const boost::uuids::uuid& uuid, const ComboAddress& ip, const DNSName& domain, int type, uint16_t qid, bool doTCP, size_t bytes, int rcode, const std::vector<DNSRecord>& records, const struct timeval& queryTime)
+static void logIncomingResponse(std::shared_ptr<RemoteLogger> outgoingLogger, boost::optional<const boost::uuids::uuid&> initialRequestId, const boost::uuids::uuid& uuid, const ComboAddress& ip, const DNSName& domain, int type, uint16_t qid, bool doTCP, size_t bytes, int rcode, const std::vector<DNSRecord>& records, const struct timeval& queryTime, const std::set<uint16_t>& exportTypes)
 {
   if(!outgoingLogger)
     return;
@@ -77,7 +77,7 @@ static void logIncomingResponse(std::shared_ptr<RemoteLogger> outgoingLogger, bo
   }
   message.setQueryTime(queryTime.tv_sec, queryTime.tv_usec);
   message.setResponseCode(rcode);
-  message.addRRs(records);
+  message.addRRs(records, exportTypes);
 
 //  cerr <<message.toDebugString()<<endl;
   std::string str;
@@ -90,7 +90,7 @@ static void logIncomingResponse(std::shared_ptr<RemoteLogger> outgoingLogger, bo
 /** lwr is only filled out in case 1 was returned, and even when returning 1 for 'success', lwr might contain DNS errors
     Never throws! 
  */
-int asyncresolve(const ComboAddress& ip, const DNSName& domain, int type, bool doTCP, bool sendRDQuery, int EDNS0Level, struct timeval* now, boost::optional<Netmask>& srcmask, boost::optional<const ResolveContext&> context, const std::shared_ptr<RemoteLogger>& outgoingLogger, LWResult *lwr, bool* chained)
+int asyncresolve(const ComboAddress& ip, const DNSName& domain, int type, bool doTCP, bool sendRDQuery, int EDNS0Level, struct timeval* now, boost::optional<Netmask>& srcmask, boost::optional<const ResolveContext&> context, const std::shared_ptr<RemoteLogger>& outgoingLogger, const std::set<uint16_t>& exportTypes, LWResult *lwr, bool* chained)
 {
   size_t len;
   size_t bufsize=g_outgoingEDNSBufsize;
@@ -234,7 +234,7 @@ int asyncresolve(const ComboAddress& ip, const DNSName& domain, int type, bool d
     if(mdp.d_header.rcode == RCode::FormErr && mdp.d_qname.empty() && mdp.d_qtype == 0 && mdp.d_qclass == 0) {
 #ifdef HAVE_PROTOBUF
       if(outgoingLogger) {
-        logIncomingResponse(outgoingLogger, context ? context->d_initialRequestId : boost::none, uuid, ip, domain, type, qid, doTCP, len, lwr->d_rcode, lwr->d_records, queryTime);
+        logIncomingResponse(outgoingLogger, context ? context->d_initialRequestId : boost::none, uuid, ip, domain, type, qid, doTCP, len, lwr->d_rcode, lwr->d_records, queryTime, exportTypes);
       }
 #endif
       lwr->d_validpacket=true;
@@ -280,7 +280,7 @@ int asyncresolve(const ComboAddress& ip, const DNSName& domain, int type, bool d
 
 #ifdef HAVE_PROTOBUF
     if(outgoingLogger) {
-      logIncomingResponse(outgoingLogger, context ? context->d_initialRequestId : boost::none, uuid, ip, domain, type, qid, doTCP, len, lwr->d_rcode, lwr->d_records, queryTime);
+      logIncomingResponse(outgoingLogger, context ? context->d_initialRequestId : boost::none, uuid, ip, domain, type, qid, doTCP, len, lwr->d_rcode, lwr->d_records, queryTime, exportTypes);
     }
 #endif
     lwr->d_validpacket=true;
@@ -293,7 +293,7 @@ int asyncresolve(const ComboAddress& ip, const DNSName& domain, int type, bool d
     g_stats.serverParseError++;
 #ifdef HAVE_PROTOBUF
     if(outgoingLogger) {
-      logIncomingResponse(outgoingLogger, context ? context->d_initialRequestId : boost::none, uuid, ip, domain, type, qid, doTCP, len, lwr->d_rcode, lwr->d_records, queryTime);
+      logIncomingResponse(outgoingLogger, context ? context->d_initialRequestId : boost::none, uuid, ip, domain, type, qid, doTCP, len, lwr->d_rcode, lwr->d_records, queryTime, exportTypes);
     }
 #endif
     lwr->d_validpacket=false;

pdns/lwres.hh

@@ -68,5 +68,5 @@ public:
   bool d_haveEDNS{false};
 };
 
-int asyncresolve(const ComboAddress& ip, const DNSName& domain, int type, bool doTCP, bool sendRDQuery, int EDNS0Level, struct timeval* now, boost::optional<Netmask>& srcmask, boost::optional<const ResolveContext&> context, const std::shared_ptr<RemoteLogger>& outgoingLogger, LWResult* res, bool* chained);
+int asyncresolve(const ComboAddress& ip, const DNSName& domain, int type, bool doTCP, bool sendRDQuery, int EDNS0Level, struct timeval* now, boost::optional<Netmask>& srcmask, boost::optional<const ResolveContext&> context, const std::shared_ptr<RemoteLogger>& outgoingLogger, const std::set<uint16_t>& exportTypes, LWResult* res, bool* chained);
 #endif // PDNS_LWRES_HH

pdns/pdns_recursor.cc

@@ -1009,10 +1009,8 @@ static void startDoResolve(void *p)
       logResponse = t_protobufServer && luaconfsLocal->protobufExportConfig.logResponses;
       Netmask requestorNM(dc->d_source, dc->d_source.sin4.sin_family == AF_INET ? luaconfsLocal->protobufMaskV4 : luaconfsLocal->protobufMaskV6);
       const ComboAddress& requestor = requestorNM.getMaskedNetwork();
-      pbMessage = RecProtoBufMessage(RecProtoBufMessage::Response);
-      pbMessage->update(dc->d_uuid, &requestor, &dc->d_destination, dc->d_tcp, dc->d_mdp.d_header.id);
+      pbMessage = RecProtoBufMessage(RecProtoBufMessage::Response, dc->d_uuid, &requestor, &dc->d_destination, dc->d_mdp.d_qname, dc->d_mdp.d_qtype, dc->d_mdp.d_qclass, dc->d_mdp.d_header.id, dc->d_tcp, 0);
       pbMessage->setEDNSSubnet(dc->d_ednssubnet.source, dc->d_ednssubnet.source.isIpv4() ? luaconfsLocal->protobufMaskV4 : luaconfsLocal->protobufMaskV6);
-      pbMessage->setQuestion(dc->d_mdp.d_qname, dc->d_mdp.d_qtype, dc->d_mdp.d_qclass);
     }
 #endif /* HAVE_PROTOBUF */
 
@@ -1390,8 +1388,8 @@ static void startDoResolve(void *p)
         needCommit = true;
 
 #ifdef HAVE_PROTOBUF
-        if(t_protobufServer && (i->d_type == QType::A || i->d_type == QType::AAAA || i->d_type == QType::CNAME)) {
-          pbMessage->addRR(*i);
+        if(t_protobufServer) {
+          pbMessage->addRR(*i, luaconfsLocal->protobufExportConfig.exportTypes);
         }
 #endif
       }

pdns/rec-lua-conf.cc

@@ -82,7 +82,7 @@ static void parseRPZParameters(const std::unordered_map<string,boost::variant<ui
 }
 
 #if HAVE_PROTOBUF
-typedef std::unordered_map<std::string, boost::variant<bool, uint64_t, std::string> > protobufOptions_t;
+typedef std::unordered_map<std::string, boost::variant<bool, uint64_t, std::string, std::vector<std::pair<int,std::string> > > > protobufOptions_t;
 
 static void parseProtobufOptions(boost::optional<protobufOptions_t> vars, ProtobufExportConfig& config)
 {
@@ -117,6 +117,28 @@ static void parseProtobufOptions(boost::optional<protobufOptions_t> vars, Protob
   if (vars->count("logResponses")) {
     config.logResponses = boost::get<bool>((*vars)["logResponses"]);
   }
+
+  if (vars->count("exportTypes")) {
+    config.exportTypes.clear();
+
+    auto types =  boost::get<std::vector<std::pair<int, std::string>>>((*vars)["exportTypes"]);
+    for (const auto& pair : types) {
+      const auto type = pair.second;
+      bool found = false;
+
+      for (const auto& entry : QType::names) {
+        if (entry.first == type) {
+          found = true;
+          config.exportTypes.insert(entry.second);
+          break;
+        }
+      }
+
+      if (!found) {
+        throw std::runtime_error("Unknown QType '" + type + "' in protobuf's export types");
+      }
+    }
+  }
 }
 #endif /* HAVE_PROTOBUF */
 

pdns/rec-lua-conf.hh

@@ -20,13 +20,16 @@
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  */
 #pragma once
+#include <set>
+
 #include "sholder.hh"
 #include "sortlist.hh"
 #include "filterpo.hh"
 #include "validate.hh"
 
 struct ProtobufExportConfig
 {
+  std::set<uint16_t> exportTypes = { QType::A, QType::AAAA, QType::CNAME };
   ComboAddress server;
   uint64_t maxQueuedEntries{100};
   uint16_t timeout{2};

pdns/rec-protobuf.cc

@@ -2,52 +2,99 @@
 #include "config.h"
 #include "rec-protobuf.hh"
 
-void RecProtoBufMessage::addRR(const DNSRecord& record)
+void RecProtoBufMessage::addRR(const DNSRecord& record, const std::set<uint16_t>& exportTypes)
 {
 #ifdef HAVE_PROTOBUF
   PBDNSMessage_DNSResponse* response = d_message.mutable_response();
   if (!response) {
     return;
   }
 
-  if (record.d_place != DNSResourceRecord::ANSWER ||
-      record.d_class != QClass::IN ||
-      (record.d_type != QType::A &&
-       record.d_type != QType::AAAA &&
-       record.d_type != QType::CNAME)) {
+  if (record.d_place != DNSResourceRecord::ANSWER || record.d_class != QClass::IN) {
     return;
   }
 
-   PBDNSMessage_DNSResponse_DNSRR* pbRR = response->add_rrs();
-   if (!pbRR) {
-     return;
-   }
+  if (exportTypes.count(record.d_type) == 0) {
+    return;
+  }
+
+  PBDNSMessage_DNSResponse_DNSRR* pbRR = response->add_rrs();
+  if (!pbRR) {
+    return;
+  }
+
+  pbRR->set_name(record.d_name.toString());
+  pbRR->set_type(record.d_type);
+  pbRR->set_class_(record.d_class);
+  pbRR->set_ttl(record.d_ttl);
 
-   pbRR->set_name(record.d_name.toString());
-   pbRR->set_type(record.d_type);
-   pbRR->set_class_(record.d_class);
-   pbRR->set_ttl(record.d_ttl);
-   if (record.d_type == QType::A) {
-     const ARecordContent& arc = dynamic_cast<const ARecordContent&>(*(record.d_content));
-     ComboAddress data = arc.getCA();
-     pbRR->set_rdata(&data.sin4.sin_addr.s_addr, sizeof(data.sin4.sin_addr.s_addr));
-   }
-   else if (record.d_type == QType::AAAA) {
-     const AAAARecordContent& arc = dynamic_cast<const AAAARecordContent&>(*(record.d_content));
-     ComboAddress data = arc.getCA();
-     pbRR->set_rdata(&data.sin6.sin6_addr.s6_addr, sizeof(data.sin6.sin6_addr.s6_addr));
-   } else if (record.d_type == QType::CNAME) {
-     const CNAMERecordContent& crc = dynamic_cast<const CNAMERecordContent&>(*(record.d_content));
-     DNSName data = crc.getTarget();
-     pbRR->set_rdata(data.toString());
-   }
+  switch(record.d_type) {
+  case QType::A:
+  {
+    const auto& content = dynamic_cast<const ARecordContent&>(*(record.d_content));
+    ComboAddress data = content.getCA();
+    pbRR->set_rdata(&data.sin4.sin_addr.s_addr, sizeof(data.sin4.sin_addr.s_addr));
+    break;
+  }
+  case QType::AAAA:
+  {
+    const auto& content = dynamic_cast<const AAAARecordContent&>(*(record.d_content));
+    ComboAddress data = content.getCA();
+    pbRR->set_rdata(&data.sin6.sin6_addr.s6_addr, sizeof(data.sin6.sin6_addr.s6_addr));
+    break;
+  }
+  case QType::CNAME:
+  {
+    const auto& content = dynamic_cast<const CNAMERecordContent&>(*(record.d_content));
+    pbRR->set_rdata(content.getTarget().toString());
+    break;
+  }
+  case QType::TXT:
+  {
+    const auto& content = dynamic_cast<const TXTRecordContent&>(*(record.d_content));
+    pbRR->set_rdata(content.d_text);
+    break;
+  }
+  case QType::NS:
+  {
+    const auto& content = dynamic_cast<const NSRecordContent&>(*(record.d_content));
+    pbRR->set_rdata(content.getNS().toString());
+    break;
+  }
+  case QType::PTR:
+  {
+    const auto& content = dynamic_cast<const PTRRecordContent&>(*(record.d_content));
+    pbRR->set_rdata(content.getContent().toString());
+    break;
+  }
+  case QType::MX:
+  {
+    const auto& content = dynamic_cast<const MXRecordContent&>(*(record.d_content));
+    pbRR->set_rdata(content.d_mxname.toString());
+    break;
+  }
+  case QType::SPF:
+  {
+    const auto& content = dynamic_cast<const SPFRecordContent&>(*(record.d_content));
+    pbRR->set_rdata(content.getText());
+    break;
+  }
+  case QType::SRV:
+  {
+    const auto& content = dynamic_cast<const SRVRecordContent&>(*(record.d_content));
+    pbRR->set_rdata(content.d_target.toString());
+    break;
+  }
+  default:
+    break;
+  }
 #endif /* HAVE_PROTOBUF */
 }
 
-void RecProtoBufMessage::addRRs(const std::vector<DNSRecord>& records)
+void RecProtoBufMessage::addRRs(const std::vector<DNSRecord>& records, const std::set<uint16_t>& exportTypes)
 {
   for (const auto& record : records) {
-    addRR(record);
+    addRR(record, exportTypes);
   }
 }
 

pdns/rec-protobuf.hh

@@ -42,8 +42,8 @@ public:
   }
 #endif /* HAVE_PROTOBUF */
 
-  void addRRs(const std::vector<DNSRecord>& records);
-  void addRR(const DNSRecord& record);
+  void addRRs(const std::vector<DNSRecord>& records, const std::set<uint16_t>& exportTypes);
+  void addRR(const DNSRecord& record, const std::set<uint16_t>& exportTypes);
   void setAppliedPolicy(const std::string& policy);
   void setAppliedPolicyType(const DNSFilterEngine::PolicyType& policyType);
   void setPolicyTags(const std::vector<std::string>& policyTags);

pdns/recursordist/docs/lua-config/protobuf.rst

@@ -30,6 +30,7 @@ Protobuf export to a server is enabled using the ``protobufServer()`` directive:
   * ``asyncConnect``: bool - When set to false (default) the first connection to the server during startup will block up to ``timeout`` seconds, otherwise the connection is done in a separate thread, after the first message has been queued
   * ``logQueries=true``: bool - Whether to export queries
   * ``logResponses=true``: bool - Whether to export responses
+  * ``exportTypes={'A', 'AAAA', 'CNAME'}``: list of strings - The list of record types found in the answer section to export. Only A, AAAA, CNAME, MX, NS, PTR, SPF, SRV and TXT are currently supported
 
 .. function:: protobufServer(server [[[[[[[, timeout=2], maxQueuedEntries=100], reconnectWaitTime=1], maskV4=32], maskV6=128], asyncConnect=false], taggedOnly=false])
 
@@ -74,6 +75,7 @@ While :func:`protobufServer` only exports the queries sent to the recursor from
   * ``asyncConnect``: bool - When set to false (default) the first connection to the server during startup will block up to ``timeout`` seconds, otherwise the connection is done in a separate thread, after the first message has been queued
   * ``logQueries=true``: bool - Whether to export queries
   * ``logResponses=true``: bool - Whether to export responses
+  * ``exportTypes={'A', 'AAAA', 'CNAME'}``: list of strings - The list of record types found in the answer section to export. Only A, AAAA, CNAME, MX, NS, PTR, SPF, SRV and TXT are currently supported
 
 .. function:: outgoingProtobufServer(server [[[[, timeout=2], maxQueuedEntries=100], reconnectWaitTime=1], asyncConnect=false])