Merge pull request #685 from pi-hole/new/CNAME_inspection_details

Improvements to deep CNAME inspection data processing

src/FTL.h

@@ -107,7 +107,9 @@
 // FTLDNS enums
 enum { QUERIES, UPSTREAMS, CLIENTS, DOMAINS, OVERTIME, WILDCARD, DNS_CACHE };
 enum { DNSSEC_UNSPECIFIED, DNSSEC_SECURE, DNSSEC_INSECURE, DNSSEC_BOGUS, DNSSEC_ABANDONED, DNSSEC_UNKNOWN };
-enum { QUERY_UNKNOWN, QUERY_GRAVITY, QUERY_FORWARDED, QUERY_CACHE, QUERY_WILDCARD, QUERY_BLACKLIST, QUERY_EXTERNAL_BLOCKED_IP, QUERY_EXTERNAL_BLOCKED_NULL, QUERY_EXTERNAL_BLOCKED_NXRA };
+enum { QUERY_UNKNOWN, QUERY_GRAVITY, QUERY_FORWARDED, QUERY_CACHE, QUERY_REGEX, QUERY_BLACKLIST, \
+ QUERY_EXTERNAL_BLOCKED_IP, QUERY_EXTERNAL_BLOCKED_NULL, QUERY_EXTERNAL_BLOCKED_NXRA, \
+ QUERY_GRAVITY_CNAME, QUERY_REGEX_CNAME, QUERY_BLACKLIST_CNAME };
 enum { TYPE_A = 1, TYPE_AAAA, TYPE_ANY, TYPE_SRV, TYPE_SOA, TYPE_PTR, TYPE_TXT, TYPE_MAX };
 enum { REPLY_UNKNOWN, REPLY_NODATA, REPLY_NXDOMAIN, REPLY_CNAME, REPLY_IP, REPLY_DOMAIN, REPLY_RRNAME, REPLY_SERVFAIL, REPLY_REFUSED, REPLY_NOTIMP, REPLY_OTHER };
 enum { PRIVACY_SHOW_ALL = 0, PRIVACY_HIDE_DOMAINS, PRIVACY_HIDE_DOMAINS_CLIENTS, PRIVACY_MAXIMUM, PRIVACY_NOSTATS };

src/api/api.c

@@ -802,8 +802,11 @@ void getAllQueries(const char *client_message, const int *sock)
 
  // 1 = gravity.list, 4 = wildcard, 5 = black.list
  if((query->status == QUERY_GRAVITY ||
- query->status == QUERY_WILDCARD ||
- query->status == QUERY_BLACKLIST) && !showblocked)
+ query->status == QUERY_REGEX ||
+ query->status == QUERY_BLACKLIST ||
+ query->status == QUERY_GRAVITY_CNAME ||
+ query->status == QUERY_REGEX_CNAME ||
+ query->status == QUERY_BLACKLIST_CNAME) && !showblocked)
  continue;
  // 2 = forwarded, 3 = cached
  if((query->status == QUERY_FORWARDED ||
@@ -830,8 +833,11 @@ void getAllQueries(const char *client_message, const int *sock)
  {
  // Does the user want to see queries answered from blocking lists?
  if(forwarddestid == -2 && query->status != QUERY_GRAVITY
- && query->status != QUERY_WILDCARD
- && query->status != QUERY_BLACKLIST)
+ && query->status != QUERY_REGEX
+ && query->status != QUERY_BLACKLIST
+ && query->status != QUERY_GRAVITY_CNAME
+ && query->status != QUERY_REGEX_CNAME
+ && query->status != QUERY_BLACKLIST_CNAME)
  continue;
  // Does the user want to see queries answered from local cache?
  else if(forwarddestid == -1 && query->status != QUERY_CACHE)
@@ -843,7 +849,7 @@ void getAllQueries(const char *client_message, const int *sock)
 
  // Ask subroutine for domain. It may return "hidden" depending on
  // the privacy settings at the time the query was made
- const char *domain = getDomainString(queryID);
+ const char *domain = getDomainString(query);
 
  // Similarly for the client
  const char *clientIPName = NULL;
@@ -853,18 +859,45 @@ void getAllQueries(const char *client_message, const int *sock)
  continue;
 
  if(strlen(getstr(client->namepos)) > 0)
- clientIPName = getClientNameString(queryID);
+ clientIPName = getClientNameString(query);
  else
- clientIPName = getClientIPString(queryID);
+ clientIPName = getClientIPString(query);
 
  unsigned long delay = query->response;
  // Check if received (delay should be smaller than 30min)
  if(delay > 1.8e7)
  delay = 0;
 
+ // Get domain blocked during deep CNAME inspection, if applicable
+ const char *CNAME_domain = "N/A";
+ if(query->CNAME_domainID > -1)
+ {
+ CNAME_domain = getCNAMEDomainString(query);
+ }
+
+ // Get ID of blocking regex, if applicable
+ int regex_idx = -1;
+ if (query->status == QUERY_REGEX || query->status == QUERY_REGEX_CNAME)
+ {
+ unsigned int cacheID = findCacheID(query->domainID, query->clientID);
+ DNSCacheData *dns_cache = getDNSCache(cacheID, true);
+ if(dns_cache != NULL)
+ regex_idx = dns_cache->black_regex_idx;
+ }
+
  if(istelnet[*sock])
  {
- ssend(*sock,"%li %s %s %s %i %i %i %lu",query->timestamp,qtype,domain,clientIPName,query->status,query->dnssec,query->reply,delay);
+ ssend(*sock,"%li %s %s %s %i %i %i %lu %s %i",
+ query->timestamp,
+ qtype,
+ domain,
+ clientIPName,
+ query->status,
+ query->dnssec,
+ query->reply,
+ delay,
+ CNAME_domain,
+ regex_idx);
  if(config.debug & DEBUG_API)
  ssend(*sock, " %i", queryID);
  ssend(*sock, "\n");
@@ -917,14 +950,17 @@ void getRecentBlocked(const char *client_message, const int *sock)
  continue;
 
  if(query->status == QUERY_GRAVITY ||
- query->status == QUERY_WILDCARD ||
- query->status == QUERY_BLACKLIST)
+ query->status == QUERY_REGEX ||
+ query->status == QUERY_BLACKLIST ||
+ query->status == QUERY_GRAVITY_CNAME ||
+ query->status == QUERY_REGEX_CNAME ||
+ query->status == QUERY_BLACKLIST_CNAME)
  {
  found++;
 
  // Ask subroutine for domain. It may return "hidden" depending on
  // the privacy settings at the time the query was made
- const char *domain = getDomainString(queryID);
+ const char *domain = getDomainString(query);
  if(domain == NULL)
  continue;
 

src/database/gravity-db.c

@@ -561,7 +561,7 @@ inline bool in_whitelist(const char *domain, clientsData* client, const int clie
  // optimization as the database lookup will most likely hit (a) more domains and (b)
  // will be faster (given a sufficiently large number of regex whitelisting filters).
  return domain_in_list(domain, client->whitelist_stmt, "whitelist") ||
- match_regex(domain, clientID, REGEX_WHITELIST);
+ match_regex(domain, clientID, REGEX_WHITELIST) != -1;
 }
 
 inline bool in_gravity(const char *domain, clientsData* client)

src/database/query-table.c

@@ -113,11 +113,11 @@ void DB_save_queries(void)
  sqlite3_bind_int(stmt, 3, query->status);
 
  // DOMAIN
- const char *domain = getDomainString(queryID);
+ const char *domain = getDomainString(query);
  sqlite3_bind_text(stmt, 4, domain, -1, SQLITE_STATIC);
 
  // CLIENT
- const char *client = getClientIPString(queryID);
+ const char *client = getClientIPString(query);
  sqlite3_bind_text(stmt, 5, client, -1, SQLITE_STATIC);
 
  // FORWARD
@@ -161,10 +161,13 @@ void DB_save_queries(void)
  total++;
  if(query->status == QUERY_GRAVITY ||
  query->status == QUERY_BLACKLIST ||
- query->status == QUERY_WILDCARD ||
+ query->status == QUERY_REGEX ||
  query->status == QUERY_EXTERNAL_BLOCKED_IP ||
  query->status == QUERY_EXTERNAL_BLOCKED_NULL ||
- query->status == QUERY_EXTERNAL_BLOCKED_NXRA)
+ query->status == QUERY_EXTERNAL_BLOCKED_NXRA ||
+ query->status == QUERY_GRAVITY_CNAME ||
+ query->status == QUERY_REGEX_CNAME ||
+ query->status == QUERY_BLACKLIST_CNAME)
  blocked++;
 
  // Update lasttimestamp variable with timestamp of the latest stored query
@@ -371,6 +374,7 @@ void DB_read_queries(void)
  query->response = 0;
  query->dnssec = DNSSEC_UNKNOWN;
  query->reply = REPLY_UNKNOWN;
+ query->CNAME_domainID = -1;
 
  // Set lastQuery timer and add one query for network table
  clientsData* client = getClient(clientID, true);
@@ -399,12 +403,15 @@ void DB_read_queries(void)
  counters->unknown++;
  break;
 
- case QUERY_GRAVITY: // Blocked by gravity.list
- case QUERY_WILDCARD: // Blocked by regex filter
- case QUERY_BLACKLIST: // Blocked by black.list
+ case QUERY_GRAVITY: // Blocked by gravity
+ case QUERY_REGEX: // Blocked by regex blacklist
+ case QUERY_BLACKLIST: // Blocked by exact blacklist
  case QUERY_EXTERNAL_BLOCKED_IP: // Blocked by external provider
  case QUERY_EXTERNAL_BLOCKED_NULL: // Blocked by external provider
  case QUERY_EXTERNAL_BLOCKED_NXRA: // Blocked by external provider
+ case QUERY_GRAVITY_CNAME: // Blocked by gravity
+ case QUERY_REGEX_CNAME: // Blocked by regex blacklist
+ case QUERY_BLACKLIST_CNAME: // Blocked by exact blacklist
  counters->blocked++;
  // Get domain pointer
  domainsData* domain = getDomain(domainID, true);

src/datastructure.c

@@ -121,8 +121,9 @@ int findDomainID(const char *domainString, const bool count)
 
  // Set magic byte
  domain->magic = MAGICBYTE;
- // Set its counter to 1
- domain->count = 1;
+ // Set its counter to 1 only if this domain is to be counted
+ // Domains only encountered during CNAME inspection are NOT counted here
+ domain->count = count ? 1 : 0;
  // Set blocked counter to zero
  domain->blockedcount = 0;
  // Store domain name - no need to check for NULL here as it doesn't harm
@@ -269,10 +270,8 @@ bool isValidIPv6(const char *addr)
 
 // Privacy-level sensitive subroutine that returns the domain name
 // only when appropriate for the requested query
-const char *getDomainString(const int queryID)
+const char *getDomainString(const queriesData* query)
 {
- const queriesData* query = getQuery(queryID, true);
-
  // Check if the returned pointer is valid before trying to access it
  if(query == NULL)
  return "";
@@ -289,12 +288,30 @@ const char *getDomainString(const int queryID)
  return HIDDEN_DOMAIN;
 }
 
-// Privacy-level sensitive subroutine that returns the client IP
+// Privacy-level sensitive subroutine that returns the domain name
 // only when appropriate for the requested query
-const char *getClientIPString(const int queryID)
+const char *getCNAMEDomainString(const queriesData* query)
 {
- const queriesData* query = getQuery(queryID, false);
+ // Check if the returned pointer is valid before trying to access it
+ if(query == NULL)
+ return "";
+
+ if(query->privacylevel < PRIVACY_HIDE_DOMAINS)
+ {
+ // Get domain pointer
+ const domainsData* domain = getDomain(query->CNAME_domainID, true);
 
+ // Return string
+ return getstr(domain->domainpos);
+ }
+ else
+ return HIDDEN_DOMAIN;
+}
+
+// Privacy-level sensitive subroutine that returns the client IP
+// only when appropriate for the requested query
+const char *getClientIPString(const queriesData* query)
+{
  // Check if the returned pointer is valid before trying to access it
  if(query == NULL)
  return "";
@@ -313,10 +330,8 @@ const char *getClientIPString(const int queryID)
 
 // Privacy-level sensitive subroutine that returns the client host name
 // only when appropriate for the requested query
-const char *getClientNameString(const int queryID)
+const char *getClientNameString(const queriesData* query)
 {
- const queriesData* query = getQuery(queryID, true);
-
  // Check if the returned pointer is valid before trying to access it
  if(query == NULL)
  return "";

src/datastructure.h

@@ -20,9 +20,6 @@ int findClientID(const char *client, const bool count);
 int findCacheID(int domainID, int clientID);
 bool isValidIPv4(const char *addr);
 bool isValidIPv6(const char *addr);
-const char *getDomainString(const int queryID);
-const char *getClientIPString(const int queryID);
-const char *getClientNameString(const int queryID);
 
 void FTL_reload_all_domainlists(void);
 void FTL_reset_per_client_domain_data(void);
@@ -39,6 +36,7 @@ typedef struct {
  int clientID;
  int upstreamID;
  int id; // the ID is a (signed) int in dnsmasq, so no need for a long int here
+ int CNAME_domainID; // only valid if query has a CNAME blocking status
  unsigned long response; // saved in units of 1/10 milliseconds (1 = 0.1ms, 2 = 0.2ms, 2500 = 250.0ms, etc.)
  int64_t db;
  unsigned int timeidx;
@@ -82,8 +80,14 @@ typedef struct {
  unsigned char blocking_status;
  int domainID;
  int clientID;
+ int black_regex_idx;
 } DNSCacheData;
 
+const char *getDomainString(const queriesData* query);
+const char *getCNAMEDomainString(const queriesData* query);
+const char *getClientIPString(const queriesData* query);
+const char *getClientNameString(const queriesData* query);
+
 // Pointer getter functions
 #define getQuery(queryID, checkMagic) _getQuery(queryID, checkMagic, __LINE__, __FUNCTION__, __FILE__)
 queriesData* _getQuery(int queryID, bool checkMagic, int line, const char * function, const char * file);

src/dnsmasq_interface.c

@@ -145,7 +145,7 @@ static bool _FTL_check_blocking(int queryID, int domainID, int clientID, const c
  // as sometving along the CNAME path hit the whitelist
  if(!query->whitelisted)
  {
- query_blocked(query, domain, client, QUERY_WILDCARD);
+ query_blocked(query, domain, client, QUERY_REGEX);
  return true;
  }
  break;
@@ -223,16 +223,18 @@ static bool _FTL_check_blocking(int queryID, int domainID, int clientID, const c
 
  // Check domain against blacklist regex filters
  // Skipped when the domain is whitelisted or blocked by exact blacklist or gravity
+ int regex_idx = 0;
  if(!query->whitelisted && !blockDomain &&
- match_regex(domainString, clientID, REGEX_BLACKLIST))
+ (regex_idx = match_regex(domainString, clientID, REGEX_BLACKLIST)) > -1)
  {
  // We block this domain
  blockDomain = true;
- new_status = QUERY_WILDCARD;
+ new_status = QUERY_REGEX;
  *blockingreason = "regex blacklisted";
 
  // Mark domain as regex matched for this client
  dns_cache->blocking_status = REGEX_BLOCKED;
+ dns_cache->black_regex_idx = regex_idx;
  }
 
  // Common actions regardless what the possible blocking reason is
@@ -328,14 +330,23 @@ bool _FTL_CNAME(const char *domain, const struct crec *cpp, const int id, const
  head_domain->blockedcount++;
 
  // Store query response as CNAME type
- // The web interface will show this next to the blocking reason to highlight
- // that blocking hapend during deep CNAME inspection, i.e., the domains shown
- // does not necessarily need to be on any block- or blacklist itself.
  struct timeval response;
  gettimeofday(&response, 0);
  save_reply_type(F_CNAME, query, response);
+
+ // Store domain that was the reason for blocking the entire chain
+ query->CNAME_domainID = domainID;
+
+ // Change blocking reason into CNAME-caused blocking
+ if(query->status == QUERY_GRAVITY)
+ query->status = QUERY_GRAVITY_CNAME;
+ else if(query->status == QUERY_REGEX)
+ query->status = QUERY_REGEX_CNAME;
+ else if(query->status == QUERY_BLACKLIST)
+ query->status = QUERY_BLACKLIST_CNAME;
  }
 
+ // Debug logging for deep CNAME inspection (if enabled)
  if(config.debug & DEBUG_QUERIES)
  {
  if(src == NULL)
@@ -499,6 +510,7 @@ bool _FTL_new_query(const unsigned int flags, const char *name,
  query->reply = REPLY_UNKNOWN;
  // Store DNSSEC result for this domain
  query->dnssec = DNSSEC_UNSPECIFIED;
+ query->CNAME_domainID = -1;
 
  // Check and apply possible privacy level rules
  // The currently set privacy level (at the time the query is
@@ -851,7 +863,7 @@ void _FTL_reply(const unsigned short flags, const char *name, const struct all_a
  {
  // Mark query as blocked
  clientsData* client = getClient(query->clientID, true);
- query_blocked(query, domain, client, QUERY_WILDCARD);
+ query_blocked(query, domain, client, QUERY_REGEX);
  }
  else
  {

src/gc.c

@@ -115,10 +115,13 @@ void *GC_thread(void *val)
  break;
  case QUERY_GRAVITY: // Blocked by Pi-hole's blocking lists (fall through)
  case QUERY_BLACKLIST: // Exact blocked (fall through)
- case QUERY_WILDCARD: // Regex blocked (fall through)
+ case QUERY_REGEX: // Regex blocked (fall through)
  case QUERY_EXTERNAL_BLOCKED_IP: // Blocked by upstream provider (fall through)
  case QUERY_EXTERNAL_BLOCKED_NXRA: // Blocked by upstream provider (fall through)
  case QUERY_EXTERNAL_BLOCKED_NULL: // Blocked by upstream provider (fall through)
+ case QUERY_GRAVITY_CNAME: // Gravity domain in CNAME chain (fall through)
+ case QUERY_BLACKLIST_CNAME: // Exactly blacklisted domain in CNAME chain (fall through)
+ case QUERY_REGEX_CNAME: // Regex blacklisted domain in CNAME chain (fall through)
  counters->blocked--;
  overTime[timeidx].blocked--;
  if(domain != NULL)