Add whitelist regex support

src/FTL.h

@@ -112,7 +112,6 @@ enum { TYPE_A = 1, TYPE_AAAA, TYPE_ANY, TYPE_SRV, TYPE_SOA, TYPE_PTR, TYPE_TXT,
 enum { REPLY_UNKNOWN, REPLY_NODATA, REPLY_NXDOMAIN, REPLY_CNAME, REPLY_IP, REPLY_DOMAIN, REPLY_RRNAME, REPLY_SERVFAIL, REPLY_REFUSED, REPLY_NOTIMP, REPLY_OTHER };
 enum { PRIVACY_SHOW_ALL = 0, PRIVACY_HIDE_DOMAINS, PRIVACY_HIDE_DOMAINS_CLIENTS, PRIVACY_MAXIMUM, PRIVACY_NOSTATS };
 enum { MODE_IP, MODE_NX, MODE_NULL, MODE_IP_NODATA_AAAA, MODE_NODATA };
-enum { GRAVITY_LIST, BLACK_LIST, WHITE_LIST, REGEX_LIST, UNKNOWN_LIST };
 
 // Use out own memory handling functions that will detect possible errors
 // and report accordingly in the log. This will make debugging FTL crashs

src/api/request.c

@@ -172,14 +172,8 @@ void process_request(const char *client_message, int *sock)
  logg("Received API request to recompile regex");
  lock_shm();
  // Reread regex.list
- // Free regex list and array of whitelisted domains
- free_regex();
- // Start timer for regex compilation analysis
- timer_start(REGEX_TIMER);
  // Read and compile possible regex filters
  read_regex_from_database();
- // Log result
- log_regex(timer_elapsed_msec(REGEX_TIMER));
  unlock_shm();
  }
  else if(command(client_message, ">update-mac-vendor"))

src/database/gravity-db.c

@@ -15,6 +15,8 @@
 // global variable counters
 #include "memory.h"
 #include "sqlite3.h"
+// match_regex()
+#include "regex_r.h"
 
 // Private variables
 static sqlite3 *gravity_db = NULL;
@@ -23,6 +25,9 @@ static sqlite3_stmt* whitelist_stmt = NULL;
 static sqlite3_stmt* auditlist_stmt = NULL;
 bool gravity_database_avail = false;
 
+// Table names corresponding to the enum defined in gravity-db.h
+static const char* tablename[] = { "vw_gravity", "vw_blacklist", "vw_whitelist", "vw_regex_blacklist", "vw_regex_whitelist" , ""};
+
 // Prototypes from functions in dnsmasq's source
 void rehash(int size);
 
@@ -119,37 +124,37 @@ bool gravityDB_getTable(const unsigned char list)
 {
  if(!gravity_database_avail)
  {
- logg("gravityDB_getTable(%d): Gravity database not available", list);
+ logg("gravityDB_getTable(%u): Gravity database not available", list);
+ return false;
+ }
+
+ // Checking for smaller than GRAVITY_LIST is omitted due to list being unsigned
+ if(list >= UNKNOWN_TABLE)
+ {
+ logg("gravityDB_getTable(%u): Requested list is not known!", list);
  return false;
  }
 
- // Select correct query string to be used depending on list to be read
- const char *querystr = NULL;
- switch(list)
+ char *querystr = NULL;
+ // Build correct query string to be used depending on list to be read
+ if(asprintf(&querystr, "SELECT domain FROM %s", tablename[list]) < 18)
  {
- case GRAVITY_LIST:
- querystr = "SELECT domain FROM vw_gravity;";
- break;
- case BLACK_LIST:
- querystr = "SELECT domain FROM vw_blacklist;";
- break;
- case REGEX_LIST:
- querystr = "SELECT domain FROM vw_regex;";
- break;
- default:
- logg("gravityDB_getTable(%i): Requested list is not known!", list);
- return false;
+ logg("readGravity(%u) - asprintf() error", list);
+ return false;
  }
 
  // Prepare SQLite3 statement
  int rc = sqlite3_prepare_v2(gravity_db, querystr, -1, &table_stmt, NULL);
- if( rc )
+ if(rc != SQLITE_OK)
  {
  logg("readGravity(%s) - SQL error prepare (%i): %s", querystr, rc, sqlite3_errmsg(gravity_db));
  gravityDB_close();
+ free(querystr);
  return false;
  }
 
+ // Free allocated memory and return success
+ free(querystr);
  return true;
 }
 
@@ -179,7 +184,6 @@ inline const char* gravityDB_getDomain(void)
  if(rc != SQLITE_DONE)
  {
  logg("gravityDB_getDomain() - SQL error step (%i): %s", rc, sqlite3_errmsg(gravity_db));
- gravityDB_finalizeTable();
  return NULL;
  }
 
@@ -208,42 +212,41 @@ int gravityDB_count(const unsigned char list)
  return DB_FAILED;
  }
 
- // Select correct query string to be used depending on list to be read
- const char* querystr = NULL;
- switch(list)
+ // Checking for smaller than GRAVITY_LIST is omitted due to list being unsigned
+ if(list >= UNKNOWN_TABLE)
+ {
+ logg("gravityDB_getTable(%u): Requested list is not known!", list);
+ return false;
+ }
+
+ char *querystr = NULL;
+ // Build correct query string to be used depending on list to be read
+ if(asprintf(&querystr, "SELECT count(domain) FROM %s", tablename[list]) < 18)
  {
- case GRAVITY_LIST:
- querystr = "SELECT COUNT(*) FROM vw_gravity;";
- break;
- case BLACK_LIST:
- querystr = "SELECT COUNT(*) FROM vw_blacklist;";
- break;
- case WHITE_LIST:
- querystr = "SELECT COUNT(*) FROM vw_whitelist;";
- break;
- case REGEX_LIST:
- querystr = "SELECT COUNT(*) FROM vw_regex;";
- break;
- default:
- logg("gravityDB_count(%i): Requested list is not known!", list);
- return DB_FAILED;
+ logg("readGravity(%u) - asprintf() error", list);
+ return false;
  }
 
+ if(config.debug & DEBUG_DATABASE)
+ logg("Querying gravity database table %s", tablename[list]);
+
  // Prepare query
  int rc = sqlite3_prepare_v2(gravity_db, querystr, -1, &table_stmt, NULL);
- if( rc ){
+ if(rc != SQLITE_OK){
  logg("gravityDB_count(%s) - SQL error prepare (%i): %s", querystr, rc, sqlite3_errmsg(gravity_db));
  sqlite3_finalize(table_stmt);
  gravityDB_close();
+ free(querystr);
  return DB_FAILED;
  }
 
  // Perform query
  rc = sqlite3_step(table_stmt);
- if( rc != SQLITE_ROW ){
+ if(rc != SQLITE_ROW){
  logg("gravityDB_count(%s) - SQL error step (%i): %s", querystr, rc, sqlite3_errmsg(gravity_db));
  sqlite3_finalize(table_stmt);
  gravityDB_close();
+ free(querystr);
  return DB_FAILED;
  }
 
@@ -253,6 +256,8 @@ int gravityDB_count(const unsigned char list)
  // Finalize statement
  gravityDB_finalizeTable();
 
+ // Free allocated memory and return result
+ free(querystr);
  return result;
 }
 
@@ -309,18 +314,29 @@ static bool domain_in_list(const char *domain, sqlite3_stmt* stmt)
  // all host parameters to NULL.
  sqlite3_clear_bindings(stmt);
 
- // Return result.
+ // Return if domain was found in current table
  // SELECT EXISTS(...) either returns 0 (false) or 1 (true).
- return result == 1;
+ return (result == 1);
 }
 
 bool in_whitelist(const char *domain)
 {
- return domain_in_list(domain, whitelist_stmt);
+ if(config.debug & DEBUG_DATABASE)
+ logg("Querying whitelist for %s", domain);
+ // We have to check both the exact whitelist (using a prepared database statement)
+ // as well the compiled regex whitelist filters to check if the current domain is
+ // whitelisted. Due to short-circuit-evaluation in C, the regex evaluations is executed
+ // only if the exact whitelist lookup does not deliver a positive match. This is an
+ // optimization as the database lookup will most likely hit (a) more domains and (b)
+ // will be faster (given a sufficiently large number of regex whitelisting filters).
+ return domain_in_list(domain, whitelist_stmt) || match_regex(domain, REGEX_WHITELIST);
 }
 
 bool in_auditlist(const char *domain)
 {
+ if(config.debug & DEBUG_DATABASE)
+ logg("Querying audit list for %s", domain);
+ // We check the domain_audit table for the given domain
  return domain_in_list(domain, auditlist_stmt);
 }
 

src/database/gravity-db.h

@@ -10,6 +10,9 @@
 #ifndef GRAVITY_H
 #define GRAVITY_H
 
+// Table indices
+enum { GRAVITY_TABLE, EXACT_BLACKLIST_TABLE, EXACT_WHITELIST_TABLE, REGEX_BLACKLIST_TABLE, REGEX_WHITELIST_TABLE, UNKNOWN_TABLE };
+
 bool gravityDB_open(void);
 void gravityDB_close(void);
 bool gravityDB_getTable(unsigned char list);

src/dnsmasq_interface.c

@@ -225,7 +225,7 @@ void _FTL_new_query(const unsigned int flags, const char *name, const struct all
  // of a specific domain. The logic herein is:
  // If matched, then compare against whitelist
  // If in whitelist, negate matched so this function returns: not-to-be-blocked
- if(match_regex(domainString) && !in_whitelist(domainString))
+ if(match_regex(domainString, REGEX_BLACKLIST) && !in_whitelist(domainString))
  {
  // We have to block this domain
  block_single_domain_regex(domainString);
@@ -415,20 +415,13 @@ void FTL_dnsmasq_reload(void)
  // Reread pihole-FTL.conf to see which debugging flags are set
  read_debuging_settings(NULL);
 
- // Free regex list
- free_regex();
-
  // (Re-)open gravity database connection
  gravityDB_close();
  gravityDB_open();
 
- // Start timer for regex compilation analysis
- timer_start(REGEX_TIMER);
  // Read and compile possible regex filters
  // only after having called gravityDB_open()
  read_regex_from_database();
- // Log result
- log_regex(timer_elapsed_msec(REGEX_TIMER));
 
  // Print current set of capabilities if requested via debug flag
  if(config.debug & DEBUG_CAPS)
@@ -1430,6 +1423,11 @@ static int FTL_table_import(const char *tablename, const unsigned char list, con
  if(len == 0)
  continue;
 
+ // Do not add gravity or blacklist domains that match
+ // a regex-based whitelist filter
+ if(match_regex(domain, REGEX_WHITELIST))
+ continue;
+
  // As of here we assume the entry to be valid
  // Rehash every 1000 valid names
  if(rhash && ((name_count - cache_size) > 1000))
@@ -1487,10 +1485,10 @@ int FTL_database_import(int cache_size, struct crec **rhash, int hashsz)
  return cache_size;
  }
 
- // Import gravity and blacklist domains
+ // Import gravity and exact blacklisted domains
  int added;
- added = FTL_table_import("gravity", GRAVITY_LIST, SRC_GRAVITY, addr4, addr6, has_IPv4, has_IPv6, cache_size, rhash, hashsz);
- added += FTL_table_import("blacklist", BLACK_LIST, SRC_BLACK, addr4, addr6, has_IPv4, has_IPv6, cache_size, rhash, hashsz);
+ added = FTL_table_import("gravity", GRAVITY_TABLE, SRC_GRAVITY, addr4, addr6, has_IPv4, has_IPv6, cache_size, rhash, hashsz);
+ added += FTL_table_import("blacklist", EXACT_BLACKLIST_TABLE, SRC_BLACK, addr4, addr6, has_IPv4, has_IPv6, cache_size, rhash, hashsz);
 
  // Update counter of blocked domains
  counters->gravity = added;

src/main.c

@@ -102,9 +102,6 @@ int main (int argc, char* argv[])
  close_telnet_socket();
  close_unix_socket();
 
- // Free regex list and array of whitelisted domains
- free_regex();
-
  // Remove shared memory objects
  destroy_shmem();