diff --git a/src/FTL.h b/src/FTL.h index 2b87364b4..9839c9cf1 100644 --- a/src/FTL.h +++ b/src/FTL.h @@ -112,7 +112,6 @@ enum { TYPE_A = 1, TYPE_AAAA, TYPE_ANY, TYPE_SRV, TYPE_SOA, TYPE_PTR, TYPE_TXT, enum { REPLY_UNKNOWN, REPLY_NODATA, REPLY_NXDOMAIN, REPLY_CNAME, REPLY_IP, REPLY_DOMAIN, REPLY_RRNAME, REPLY_SERVFAIL, REPLY_REFUSED, REPLY_NOTIMP, REPLY_OTHER }; enum { PRIVACY_SHOW_ALL = 0, PRIVACY_HIDE_DOMAINS, PRIVACY_HIDE_DOMAINS_CLIENTS, PRIVACY_MAXIMUM, PRIVACY_NOSTATS }; enum { MODE_IP, MODE_NX, MODE_NULL, MODE_IP_NODATA_AAAA, MODE_NODATA }; -enum { GRAVITY_LIST, BLACK_LIST, WHITE_LIST, REGEX_LIST, UNKNOWN_LIST }; // Use out own memory handling functions that will detect possible errors // and report accordingly in the log. This will make debugging FTL crashs diff --git a/src/api/request.c b/src/api/request.c index 0265cdfec..a165770a9 100644 --- a/src/api/request.c +++ b/src/api/request.c @@ -172,14 +172,8 @@ void process_request(const char *client_message, int *sock) logg("Received API request to recompile regex"); lock_shm(); // Reread regex.list - // Free regex list and array of whitelisted domains - free_regex(); - // Start timer for regex compilation analysis - timer_start(REGEX_TIMER); // Read and compile possible regex filters read_regex_from_database(); - // Log result - log_regex(timer_elapsed_msec(REGEX_TIMER)); unlock_shm(); } else if(command(client_message, ">update-mac-vendor")) diff --git a/src/database/gravity-db.c b/src/database/gravity-db.c index 08ec04f65..9907df949 100644 --- a/src/database/gravity-db.c +++ b/src/database/gravity-db.c @@ -15,6 +15,8 @@ // global variable counters #include "memory.h" #include "sqlite3.h" +// match_regex() +#include "regex_r.h" // Private variables static sqlite3 *gravity_db = NULL; @@ -23,6 +25,9 @@ static sqlite3_stmt* whitelist_stmt = NULL; static sqlite3_stmt* auditlist_stmt = NULL; bool gravity_database_avail = false; +// Table names corresponding to the enum defined in gravity-db.h +static const char* tablename[] = { "vw_gravity", "vw_blacklist", "vw_whitelist", "vw_regex_blacklist", "vw_regex_whitelist" , ""}; + // Prototypes from functions in dnsmasq's source void rehash(int size); @@ -119,37 +124,37 @@ bool gravityDB_getTable(const unsigned char list) { if(!gravity_database_avail) { - logg("gravityDB_getTable(%d): Gravity database not available", list); + logg("gravityDB_getTable(%u): Gravity database not available", list); + return false; + } + + // Checking for smaller than GRAVITY_LIST is omitted due to list being unsigned + if(list >= UNKNOWN_TABLE) + { + logg("gravityDB_getTable(%u): Requested list is not known!", list); return false; } - // Select correct query string to be used depending on list to be read - const char *querystr = NULL; - switch(list) + char *querystr = NULL; + // Build correct query string to be used depending on list to be read + if(asprintf(&querystr, "SELECT domain FROM %s", tablename[list]) < 18) { - case GRAVITY_LIST: - querystr = "SELECT domain FROM vw_gravity;"; - break; - case BLACK_LIST: - querystr = "SELECT domain FROM vw_blacklist;"; - break; - case REGEX_LIST: - querystr = "SELECT domain FROM vw_regex;"; - break; - default: - logg("gravityDB_getTable(%i): Requested list is not known!", list); - return false; + logg("readGravity(%u) - asprintf() error", list); + return false; } // Prepare SQLite3 statement int rc = sqlite3_prepare_v2(gravity_db, querystr, -1, &table_stmt, NULL); - if( rc ) + if(rc != SQLITE_OK) { logg("readGravity(%s) - SQL error prepare (%i): %s", querystr, rc, sqlite3_errmsg(gravity_db)); gravityDB_close(); + free(querystr); return false; } + // Free allocated memory and return success + free(querystr); return true; } @@ -179,7 +184,6 @@ inline const char* gravityDB_getDomain(void) if(rc != SQLITE_DONE) { logg("gravityDB_getDomain() - SQL error step (%i): %s", rc, sqlite3_errmsg(gravity_db)); - gravityDB_finalizeTable(); return NULL; } @@ -208,42 +212,41 @@ int gravityDB_count(const unsigned char list) return DB_FAILED; } - // Select correct query string to be used depending on list to be read - const char* querystr = NULL; - switch(list) + // Checking for smaller than GRAVITY_LIST is omitted due to list being unsigned + if(list >= UNKNOWN_TABLE) + { + logg("gravityDB_getTable(%u): Requested list is not known!", list); + return false; + } + + char *querystr = NULL; + // Build correct query string to be used depending on list to be read + if(asprintf(&querystr, "SELECT count(domain) FROM %s", tablename[list]) < 18) { - case GRAVITY_LIST: - querystr = "SELECT COUNT(*) FROM vw_gravity;"; - break; - case BLACK_LIST: - querystr = "SELECT COUNT(*) FROM vw_blacklist;"; - break; - case WHITE_LIST: - querystr = "SELECT COUNT(*) FROM vw_whitelist;"; - break; - case REGEX_LIST: - querystr = "SELECT COUNT(*) FROM vw_regex;"; - break; - default: - logg("gravityDB_count(%i): Requested list is not known!", list); - return DB_FAILED; + logg("readGravity(%u) - asprintf() error", list); + return false; } + if(config.debug & DEBUG_DATABASE) + logg("Querying gravity database table %s", tablename[list]); + // Prepare query int rc = sqlite3_prepare_v2(gravity_db, querystr, -1, &table_stmt, NULL); - if( rc ){ + if(rc != SQLITE_OK){ logg("gravityDB_count(%s) - SQL error prepare (%i): %s", querystr, rc, sqlite3_errmsg(gravity_db)); sqlite3_finalize(table_stmt); gravityDB_close(); + free(querystr); return DB_FAILED; } // Perform query rc = sqlite3_step(table_stmt); - if( rc != SQLITE_ROW ){ + if(rc != SQLITE_ROW){ logg("gravityDB_count(%s) - SQL error step (%i): %s", querystr, rc, sqlite3_errmsg(gravity_db)); sqlite3_finalize(table_stmt); gravityDB_close(); + free(querystr); return DB_FAILED; } @@ -253,6 +256,8 @@ int gravityDB_count(const unsigned char list) // Finalize statement gravityDB_finalizeTable(); + // Free allocated memory and return result + free(querystr); return result; } @@ -309,18 +314,29 @@ static bool domain_in_list(const char *domain, sqlite3_stmt* stmt) // all host parameters to NULL. sqlite3_clear_bindings(stmt); - // Return result. + // Return if domain was found in current table // SELECT EXISTS(...) either returns 0 (false) or 1 (true). - return result == 1; + return (result == 1); } bool in_whitelist(const char *domain) { - return domain_in_list(domain, whitelist_stmt); + if(config.debug & DEBUG_DATABASE) + logg("Querying whitelist for %s", domain); + // We have to check both the exact whitelist (using a prepared database statement) + // as well the compiled regex whitelist filters to check if the current domain is + // whitelisted. Due to short-circuit-evaluation in C, the regex evaluations is executed + // only if the exact whitelist lookup does not deliver a positive match. This is an + // optimization as the database lookup will most likely hit (a) more domains and (b) + // will be faster (given a sufficiently large number of regex whitelisting filters). + return domain_in_list(domain, whitelist_stmt) || match_regex(domain, REGEX_WHITELIST); } bool in_auditlist(const char *domain) { + if(config.debug & DEBUG_DATABASE) + logg("Querying audit list for %s", domain); + // We check the domain_audit table for the given domain return domain_in_list(domain, auditlist_stmt); } diff --git a/src/database/gravity-db.h b/src/database/gravity-db.h index 5f4c1a94c..32df39130 100644 --- a/src/database/gravity-db.h +++ b/src/database/gravity-db.h @@ -10,6 +10,9 @@ #ifndef GRAVITY_H #define GRAVITY_H +// Table indices +enum { GRAVITY_TABLE, EXACT_BLACKLIST_TABLE, EXACT_WHITELIST_TABLE, REGEX_BLACKLIST_TABLE, REGEX_WHITELIST_TABLE, UNKNOWN_TABLE }; + bool gravityDB_open(void); void gravityDB_close(void); bool gravityDB_getTable(unsigned char list); diff --git a/src/dnsmasq_interface.c b/src/dnsmasq_interface.c index f071fa501..27ea76961 100644 --- a/src/dnsmasq_interface.c +++ b/src/dnsmasq_interface.c @@ -225,7 +225,7 @@ void _FTL_new_query(const unsigned int flags, const char *name, const struct all // of a specific domain. The logic herein is: // If matched, then compare against whitelist // If in whitelist, negate matched so this function returns: not-to-be-blocked - if(match_regex(domainString) && !in_whitelist(domainString)) + if(match_regex(domainString, REGEX_BLACKLIST) && !in_whitelist(domainString)) { // We have to block this domain block_single_domain_regex(domainString); @@ -415,20 +415,13 @@ void FTL_dnsmasq_reload(void) // Reread pihole-FTL.conf to see which debugging flags are set read_debuging_settings(NULL); - // Free regex list - free_regex(); - // (Re-)open gravity database connection gravityDB_close(); gravityDB_open(); - // Start timer for regex compilation analysis - timer_start(REGEX_TIMER); // Read and compile possible regex filters // only after having called gravityDB_open() read_regex_from_database(); - // Log result - log_regex(timer_elapsed_msec(REGEX_TIMER)); // Print current set of capabilities if requested via debug flag if(config.debug & DEBUG_CAPS) @@ -1430,6 +1423,11 @@ static int FTL_table_import(const char *tablename, const unsigned char list, con if(len == 0) continue; + // Do not add gravity or blacklist domains that match + // a regex-based whitelist filter + if(match_regex(domain, REGEX_WHITELIST)) + continue; + // As of here we assume the entry to be valid // Rehash every 1000 valid names if(rhash && ((name_count - cache_size) > 1000)) @@ -1487,10 +1485,10 @@ int FTL_database_import(int cache_size, struct crec **rhash, int hashsz) return cache_size; } - // Import gravity and blacklist domains + // Import gravity and exact blacklisted domains int added; - added = FTL_table_import("gravity", GRAVITY_LIST, SRC_GRAVITY, addr4, addr6, has_IPv4, has_IPv6, cache_size, rhash, hashsz); - added += FTL_table_import("blacklist", BLACK_LIST, SRC_BLACK, addr4, addr6, has_IPv4, has_IPv6, cache_size, rhash, hashsz); + added = FTL_table_import("gravity", GRAVITY_TABLE, SRC_GRAVITY, addr4, addr6, has_IPv4, has_IPv6, cache_size, rhash, hashsz); + added += FTL_table_import("blacklist", EXACT_BLACKLIST_TABLE, SRC_BLACK, addr4, addr6, has_IPv4, has_IPv6, cache_size, rhash, hashsz); // Update counter of blocked domains counters->gravity = added; diff --git a/src/main.c b/src/main.c index 8a37161a9..8d42ba85d 100644 --- a/src/main.c +++ b/src/main.c @@ -102,9 +102,6 @@ int main (int argc, char* argv[]) close_telnet_socket(); close_unix_socket(); - // Free regex list and array of whitelisted domains - free_regex(); - // Remove shared memory objects destroy_shmem(); diff --git a/src/regex.c b/src/regex.c index 63d7c21e0..f12cce688 100644 --- a/src/regex.c +++ b/src/regex.c @@ -19,58 +19,63 @@ #include "datastructure.h" #include -static int num_regex; -static regex_t *regex = NULL; -static bool *regexconfigured = NULL; -static char **regexbuffer = NULL; +static int num_regex[2] = { 0 }; +static regex_t *regex[2] = { NULL }; +static bool *regexconfigured[2] = { NULL }; +static char **regexbuffer[2] = { NULL }; -static void log_regex_error(const char *where, const int errcode, const int index) +static const char *regextype[] = { "blacklist", "whitelist" }; + +static void log_regex_error(const int errcode, const int index, const unsigned char regexid, const char *regexin) { // Regex failed for some reason (probably user syntax error) // Get error string and log it - const size_t length = regerror(errcode, ®ex[index], NULL, 0); + const size_t length = regerror(errcode, ®ex[regexid][index], NULL, 0); char *buffer = calloc(length,sizeof(char)); - (void) regerror (errcode, ®ex[index], buffer, length); - logg("ERROR %s regex on line %i: %s (%i)", where, index+1, buffer, errcode); + (void) regerror (errcode, ®ex[regexid][index], buffer, length); + logg("Warning: Invalid regex %s filter \"%s\": %s (error code %i)", regextype[regexid], regexin, buffer, errcode); free(buffer); } -static bool init_regex(const char *regexin, const int index) +static bool compile_regex(const char *regexin, const int index, const unsigned char regexid) { // compile regular expressions into data structures that // can be used with regexec to match against a string int regflags = REG_EXTENDED; if(config.regex_ignorecase) regflags |= REG_ICASE; - const int errcode = regcomp(®ex[index], regexin, regflags); + const int errcode = regcomp(®ex[regexid][index], regexin, regflags); if(errcode != 0) { - log_regex_error("compiling", errcode, index); + log_regex_error(errcode, index, regexid, regexin); return false; } // Store compiled regex string in buffer if in regex debug mode if(config.debug & DEBUG_REGEX) { - regexbuffer[index] = strdup(regexin); + regexbuffer[regexid][index] = strdup(regexin); } + return true; } -bool match_regex(const char *input) +bool match_regex(const char *input, const unsigned char regexid) { bool matched = false; // Start matching timer timer_start(REGEX_TIMER); - for(int index = 0; index < num_regex; index++) + for(int index = 0; index < num_regex[regexid]; index++) { // Only check regex which have been successfully compiled - if(!regexconfigured[index]) + if(!regexconfigured[regexid][index]) continue; // Try to match the compiled regular expression against input - int errcode = regexec(®ex[index], input, 0, NULL, 0); + int errcode = regexec(®ex[regexid][index], input, 0, NULL, 0); + // regexec() returns zero for a successful match or REG_NOMATCH for failure. + // We are only interested in the matching case here. if (errcode == 0) { // Match, return true @@ -78,13 +83,7 @@ bool match_regex(const char *input) // Print match message when in regex debug mode if(config.debug & DEBUG_REGEX) - logg("Regex in line %i \"%s\" matches \"%s\"", index+1, regexbuffer[index], input); - break; - } - else if (errcode != REG_NOMATCH) - { - // Error, return false afterwards - log_regex_error("matching", errcode, index); + logg("Regex %s in line %i \"%s\" matches \"%s\"", regextype[regexid], index+1, regexbuffer[regexid][index], input); break; } } @@ -93,13 +92,13 @@ bool match_regex(const char *input) // Only log evaluation times if they are longer than normal if(elapsed > 10.0) - logg("WARN: Regex evaluation took %.3f msec", elapsed); + logg("WARN: Regex %s evaluation took %.3f msec", regextype[regexid], elapsed); // No match, no error, return false return matched; } -void free_regex(void) +static void free_regex(void) { // Reset cached regex results for(int i = 0; i < counters->domains; i++) { @@ -110,65 +109,78 @@ void free_regex(void) domain->regexmatch = REGEX_UNKNOWN; } - // Return early if we don't use any regex - if(regex == NULL) + // Return early if we don't use any regex filters + if(regex[REGEX_WHITELIST] == NULL && + regex[REGEX_BLACKLIST] == NULL) return; - // Disable blocking regex checking and free regex datastructure - for(int index = 0; index < num_regex; index++) + // Free regex datastructure + for(int regexid = 0; regexid < 2; regexid++) { - if(regexconfigured[index]) + for(int index = 0; index < num_regex[regexid]; index++) { - regfree(®ex[index]); - - // Also free buffered regex strings if in regex debug mode - if(config.debug & DEBUG_REGEX) + if(regexconfigured[regexid][index]) { - free(regexbuffer[index]); - regexbuffer[index] = NULL; + regfree(®ex[regexid][index]); + + // Also free buffered regex strings if in regex debug mode + if(config.debug & DEBUG_REGEX) + { + free(regexbuffer[regexid][index]); + regexbuffer[regexid][index] = NULL; + } } } - } - // Free array with regex datastructure - free(regex); - regex = NULL; - free(regexconfigured); - regexconfigured = NULL; + // Free array with regex datastructure + if(regex[regexid] != NULL) + { + free(regex[regexid]); + regex[regexid] = NULL; + } + if(regexconfigured[regexid] != NULL) + { + free(regexconfigured[regexid]); + regexconfigured[regexid] = NULL; + } - // Reset counter for number of regex - num_regex = 0; + // Reset counter for number of regex + num_regex[regexid] = 0; + } } -void read_regex_from_database(void) +static void read_regex_table(const unsigned char regexid) { + // Get database ID + unsigned char databaseID = (regexid == REGEX_BLACKLIST) ? REGEX_BLACKLIST_TABLE : REGEX_WHITELIST_TABLE; + // Get number of lines in the regex table - num_regex = gravityDB_count(REGEX_LIST); + num_regex[regexid] = gravityDB_count(databaseID); - if(num_regex == 0) + if(num_regex[regexid] == 0) { - logg("INFO: No regex entries found"); + logg("INFO: No regex %s entries found", regextype[regexid]); return; } - else if(num_regex == DB_FAILED) + else if(num_regex[regexid] == DB_FAILED) { - logg("WARN: Database query failed, assuming there are no regex entries"); - num_regex = 0; + logg("WARN: Database query failed, assuming there are no regex %s entries", regextype[regexid]); + num_regex[regexid] = 0; return; } // Allocate memory for regex - regex = calloc(num_regex, sizeof(regex_t)); - regexconfigured = calloc(num_regex, sizeof(bool)); + regex[regexid] = calloc(num_regex[regexid], sizeof(regex_t)); + regexconfigured[regexid] = calloc(num_regex[regexid], sizeof(bool)); // Buffer strings if in regex debug mode if(config.debug & DEBUG_REGEX) - regexbuffer = calloc(num_regex, sizeof(char*)); + regexbuffer[regexid] = calloc(num_regex[regexid], sizeof(char*)); - // Connect to whitelist table - if(!gravityDB_getTable(REGEX_LIST)) + // Connect to regex table + if(!gravityDB_getTable(databaseID)) { - logg("read_regex_from_database(): Error getting table from database"); + logg("read_regex_from_database(): Error getting regex %s table from database", regextype[regexid]); return; } @@ -179,19 +191,20 @@ void read_regex_from_database(void) { // Avoid buffer overflow if database table changed // since we counted its entries - if(i >= num_regex) + if(i >= num_regex[regexid]) break; // Skip this entry if empty: an empty regex filter would match - // anything anywhere and hence match (and block) all incoming domains. - // A user can still achieve this with a filter such as ".*", however - // empty filters in the regex table are probably not expected to have such - // an effect and would immediately lead to "blocking the entire Internet" + // anything anywhere and hence match all incoming domains. A user + // can still achieve this with a filter such as ".*", however empty + // filters in the regex table are probably not expected to have such + // an effect and would immediately lead to "blocking or whitelisting + // the entire Internet" if(strlen(domain) < 1) continue; - // Copy this regex domain into memory - regexconfigured[i] = init_regex(domain, i); + // Compile this regex + regexconfigured[regexid][i] = compile_regex(domain, i, regexid); // Increase counter i++; @@ -201,7 +214,23 @@ void read_regex_from_database(void) gravityDB_finalizeTable(); } -void log_regex(const double time) +void read_regex_from_database(void) { - logg("Compiled %i Regex filters in %.1f msec", num_regex, time); + // Free regex filters + // This routine is safe to be called even when there + // are no regex filters at the moment + free_regex(); + + // Start timer for regex compilation analysis + timer_start(REGEX_TIMER); + + // Read and compile regex blacklist + read_regex_table(REGEX_BLACKLIST); + + // Read and compile regex whitelist + read_regex_table(REGEX_WHITELIST); + + // Print message to FTL's log after reloading regex filters + logg("Compiled %i whitelist and %i blacklist regex filters in %.1f msec", + num_regex[REGEX_WHITELIST], num_regex[REGEX_BLACKLIST], timer_elapsed_msec(REGEX_TIMER)); } diff --git a/src/regex_r.h b/src/regex_r.h index f3e68a8de..b6c8af5c3 100644 --- a/src/regex_r.h +++ b/src/regex_r.h @@ -10,11 +10,10 @@ #ifndef REGEX_H #define REGEX_H -bool match_regex(const char *input); -void free_regex(void); +bool match_regex(const char *input, const unsigned char regexid); void read_regex_from_database(void); -void log_regex(const double time); enum { REGEX_UNKNOWN, REGEX_BLOCKED, REGEX_NOTBLOCKED }; +enum { REGEX_BLACKLIST, REGEX_WHITELIST }; #endif //REGEX_H diff --git a/test/gravity.db.sql b/test/gravity.db.sql index 059478473..564e45908 100644 --- a/test/gravity.db.sql +++ b/test/gravity.db.sql @@ -43,7 +43,7 @@ CREATE TABLE blacklist_by_group PRIMARY KEY (blacklist_id, group_id) ); -CREATE TABLE regex +CREATE TABLE regex_blacklist ( id INTEGER PRIMARY KEY AUTOINCREMENT, domain TEXT UNIQUE NOT NULL, @@ -53,11 +53,28 @@ CREATE TABLE regex comment TEXT ); -CREATE TABLE regex_by_group +CREATE TABLE regex_blacklist_by_group ( - regex_id INTEGER NOT NULL REFERENCES regex (id), + regex_blacklist_id INTEGER NOT NULL REFERENCES regex_blacklist (id), group_id INTEGER NOT NULL REFERENCES "group" (id), - PRIMARY KEY (regex_id, group_id) + PRIMARY KEY (regex_blacklist_id, group_id) +); + +CREATE TABLE regex_whitelist +( + id INTEGER PRIMARY KEY AUTOINCREMENT, + domain TEXT UNIQUE NOT NULL, + enabled BOOLEAN NOT NULL DEFAULT 1, + date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), + date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), + comment TEXT +); + +CREATE TABLE regex_whitelist_by_group +( + regex_whitelist_id INTEGER NOT NULL REFERENCES regex_whitelist (id), + group_id INTEGER NOT NULL REFERENCES "group" (id), + PRIMARY KEY (regex_whitelist_id, group_id) ); CREATE TABLE adlist @@ -125,16 +142,28 @@ CREATE TRIGGER tr_blacklist_update AFTER UPDATE ON blacklist UPDATE blacklist SET date_modified = (cast(strftime('%s', 'now') as int)) WHERE domain = NEW.domain; END; -CREATE VIEW vw_regex AS SELECT DISTINCT domain - FROM regex - LEFT JOIN regex_by_group ON regex_by_group.regex_id = regex.id - LEFT JOIN "group" ON "group".id = regex_by_group.group_id - WHERE regex.enabled = 1 AND (regex_by_group.group_id IS NULL OR "group".enabled = 1) - ORDER BY regex.id; +CREATE VIEW vw_regex_blacklist AS SELECT DISTINCT domain + FROM regex_blacklist + LEFT JOIN regex_blacklist_by_group ON regex_blacklist_by_group.regex_blacklist_id = regex_blacklist.id + LEFT JOIN "group" ON "group".id = regex_blacklist_by_group.group_id + WHERE regex_blacklist.enabled = 1 AND (regex_blacklist_by_group.group_id IS NULL OR "group".enabled = 1) + ORDER BY regex_blacklist.id; + +CREATE TRIGGER tr_regex_blacklist_update AFTER UPDATE ON regex_blacklist + BEGIN + UPDATE regex_blacklist SET date_modified = (cast(strftime('%s', 'now') as int)) WHERE domain = NEW.domain; + END; + +CREATE VIEW vw_regex_whitelist AS SELECT DISTINCT domain + FROM regex_whitelist + LEFT JOIN regex_whitelist_by_group ON regex_whitelist_by_group.regex_whitelist_id = regex_whitelist.id + LEFT JOIN "group" ON "group".id = regex_whitelist_by_group.group_id + WHERE regex_whitelist.enabled = 1 AND (regex_whitelist_by_group.group_id IS NULL OR "group".enabled = 1) + ORDER BY regex_whitelist.id; -CREATE TRIGGER tr_regex_update AFTER UPDATE ON regex +CREATE TRIGGER tr_regex_whitelist_update AFTER UPDATE ON regex_whitelist BEGIN - UPDATE regex SET date_modified = (cast(strftime('%s', 'now') as int)) WHERE domain = NEW.domain; + UPDATE regex_whitelist SET date_modified = (cast(strftime('%s', 'now') as int)) WHERE domain = NEW.domain; END; CREATE VIEW vw_adlist AS SELECT DISTINCT address @@ -150,10 +179,18 @@ CREATE TRIGGER tr_adlist_update AFTER UPDATE ON adlist END; INSERT INTO whitelist VALUES(1,'whitelisted.com',1,1559928803,1559928803,'Migrated from /etc/pihole/whitelist.txt'); +INSERT INTO whitelist VALUES(2,'regex1.com',1,1559928803,1559928803,''); +INSERT INTO regex_whitelist VALUES(1,'regex2',1,1559928803,1559928803,''); +INSERT INTO regex_whitelist VALUES(2,'tse',1,1559928803,1559928803,''); + INSERT INTO blacklist VALUES(1,'blacklisted.com',1,1559928803,1559928803,'Migrated from /etc/pihole/blacklist.txt'); -INSERT INTO regex VALUES(1,'regex[0-9].com',1,1559928803,1559928803,'Migrated from /etc/pihole/regex.list'); +INSERT INTO regex_blacklist VALUES(1,'regex[0-9].com',1,1559928803,1559928803,'Migrated from /etc/pihole/regex.list'); + INSERT INTO adlist VALUES(1,'https://hosts-file.net/ad_servers.txt',1,1559928803,1559928803,'Migrated from /etc/pihole/adlists.list'); + +INSERT INTO gravity VALUES('whitelisted.com'); INSERT INTO gravity VALUES('0427d7.se'); +INSERT INTO gravity VALUES('01tse443.se'); INSERT INTO "group" VALUES(1,0,'Test group','A disabled test group'); INSERT INTO blacklist VALUES(2,'blacklisted-group-disabled.com',1,1559928803,1559928803,'Entry disabled by a group'); diff --git a/test/test_suite.bats b/test/test_suite.bats index 9044ad8f4..d38d2d587 100644 --- a/test/test_suite.bats +++ b/test/test_suite.bats @@ -11,6 +11,37 @@ [[ ${lines[6]} == "" ]] } +@test "Starting tests without prior history" { + run bash -c 'grep -c "Total DNS queries: 0" /var/log/pihole-FTL.log' + printf "%s\n" "${lines[@]}" + [[ ${lines[0]} == "1" ]] +} + +@test "Initial blocking status is enabled" { + run bash -c 'grep -c "Blocking status is enabled" /var/log/pihole-FTL.log' + printf "%s\n" "${lines[@]}" + [[ ${lines[0]} == "1" ]] +} + +@test "Number of imported gravity domains as expected" { + run bash -c 'grep -c "Database (gravity): imported 1 domains" /var/log/pihole-FTL.log' + printf "%s\n" "${lines[@]}" + [[ ${lines[0]} == "1" ]] +} + + +@test "Number of compiled regex filters as expected" { + run bash -c 'grep -c "Compiled 2 whitelist and 1 blacklist regex filters" /var/log/pihole-FTL.log' + printf "%s\n" "${lines[@]}" + [[ ${lines[0]} == "1" ]] +} + +@test "Number of imported blacklist domains as expected" { + run bash -c 'grep -c "Database (blacklist): imported 1 domains" /var/log/pihole-FTL.log' + printf "%s\n" "${lines[@]}" + [[ ${lines[0]} == "1" ]] +} + @test "Blacklisted domain is blocked" { run bash -c "dig blacklisted.com @127.0.0.1 +short" printf "%s\n" "${lines[@]}" @@ -25,27 +56,48 @@ [[ ${lines[1]} == "" ]] } -@test "Whitelisted domain is not blocked" { +@test "Gravity domain + whitelist exact match is not blocked" { run bash -c "dig whitelisted.com @127.0.0.1 +short" printf "%s\n" "${lines[@]}" [[ ${lines[0]} != "0.0.0.0" ]] [[ ${lines[1]} == "" ]] } -@test "Regex filter match is blocked" { +@test "Gravity domain + whitelist regex match is not blocked" { + run bash -c "dig 01tse443.se @127.0.0.1 +short" + printf "%s\n" "${lines[@]}" + [[ ${lines[0]} != "0.0.0.0" ]] + [[ ${lines[1]} == "" ]] +} + +@test "Regex blacklist match is blocked" { run bash -c "dig regex5.com @127.0.0.1 +short" printf "%s\n" "${lines[@]}" [[ ${lines[0]} == "0.0.0.0" ]] [[ ${lines[1]} == "" ]] } -@test "Regex filter mismatch is not blocked" { +@test "Regex blacklist mismatch is not blocked" { run bash -c "dig regexA.com @127.0.0.1 +short" printf "%s\n" "${lines[@]}" [[ ${lines[0]} != "0.0.0.0" ]] [[ ${lines[1]} == "" ]] } +@test "Regex blacklist match + whitelist exact match is not blocked" { + run bash -c "dig regex1.com @127.0.0.1 +short" + printf "%s\n" "${lines[@]}" + [[ ${lines[0]} != "0.0.0.0" ]] + [[ ${lines[1]} == "" ]] +} + +@test "Regex blacklist match + whitelist regex match is not blocked" { + run bash -c "dig regex2.com @127.0.0.1 +short" + printf "%s\n" "${lines[@]}" + [[ ${lines[0]} != "0.0.0.0" ]] + [[ ${lines[1]} == "" ]] +} + @test "Google.com (A) is not blocked" { run bash -c "dig A google.com @127.0.0.1 +short" printf "%s\n" "${lines[@]}" @@ -71,19 +123,19 @@ run bash -c 'echo ">stats >quit" | nc -v 127.0.0.1 4711' printf "%s\n" "${lines[@]}" [[ ${lines[1]} == "domains_being_blocked 2" ]] - [[ ${lines[2]} == "dns_queries_today 10" ]] + [[ ${lines[2]} == "dns_queries_today 13" ]] [[ ${lines[3]} == "ads_blocked_today 3" ]] - [[ ${lines[4]} == "ads_percentage_today 30.000000" ]] - [[ ${lines[5]} == "unique_domains 9" ]] - [[ ${lines[6]} == "queries_forwarded 5" ]] + [[ ${lines[4]} == "ads_percentage_today 23.076923" ]] + [[ ${lines[5]} == "unique_domains 12" ]] + [[ ${lines[6]} == "queries_forwarded 8" ]] [[ ${lines[7]} == "queries_cached 2" ]] [[ ${lines[8]} == "clients_ever_seen 1" ]] [[ ${lines[9]} == "unique_clients 1" ]] - [[ ${lines[10]} == "dns_queries_all_types 10" ]] + [[ ${lines[10]} == "dns_queries_all_types 13" ]] [[ ${lines[11]} == "reply_NODATA 0" ]] - [[ ${lines[12]} == "reply_NXDOMAIN 0" ]] + [[ ${lines[12]} == "reply_NXDOMAIN 2" ]] [[ ${lines[13]} == "reply_CNAME 0" ]] - [[ ${lines[14]} == "reply_IP 7" ]] + [[ ${lines[14]} == "reply_IP 8" ]] [[ ${lines[15]} == "privacy_level 0" ]] [[ ${lines[16]} == "status enabled" ]] [[ ${lines[17]} == "" ]] @@ -92,14 +144,14 @@ @test "Top Clients (descending, default)" { run bash -c 'echo ">top-clients >quit" | nc -v 127.0.0.1 4711' printf "%s\n" "${lines[@]}" - [[ ${lines[1]} == "0 10 127.0.0.1 "* ]] + [[ ${lines[1]} == "0 13 127.0.0.1 "* ]] [[ ${lines[2]} == "" ]] } @test "Top Clients (ascending)" { run bash -c 'echo ">top-clients asc >quit" | nc -v 127.0.0.1 4711' printf "%s\n" "${lines[@]}" - [[ ${lines[1]} == "0 10 127.0.0.1 "* ]] + [[ ${lines[1]} == "0 13 127.0.0.1 "* ]] [[ ${lines[2]} == "" ]] } @@ -112,13 +164,16 @@ @test "Top Domains (descending, default)" { run bash -c 'echo ">top-domains >quit" | nc -v 127.0.0.1 4711' printf "%s\n" "${lines[@]}" - [[ ${lines[1]} == "0 2 google.com" ]] + [[ "${lines[1]}" == *" 2 google.com"* ]] [[ "${lines[@]}" == *" 1 version.ftl"* ]] [[ "${lines[@]}" == *" 1 version.bind"* ]] [[ "${lines[@]}" == *" 1 whitelisted.com"* ]] + [[ "${lines[@]}" == *" 1 01tse443.se"* ]] [[ "${lines[@]}" == *" 1 regexa.com"* ]] + [[ "${lines[@]}" == *" 1 regex1.com"* ]] + [[ "${lines[@]}" == *" 1 regex2.com"* ]] [[ "${lines[@]}" == *" 1 ftl.pi-hole.net"* ]] - [[ ${lines[7]} == "" ]] + [[ "${lines[10]}" == "" ]] } @test "Top Domains (ascending)" { @@ -127,10 +182,13 @@ [[ "${lines[@]}" == *" 1 version.ftl"* ]] [[ "${lines[@]}" == *" 1 version.bind"* ]] [[ "${lines[@]}" == *" 1 whitelisted.com"* ]] + [[ "${lines[@]}" == *" 1 01tse443.se"* ]] [[ "${lines[@]}" == *" 1 regexa.com"* ]] + [[ "${lines[@]}" == *" 1 regex1.com"* ]] + [[ "${lines[@]}" == *" 1 regex2.com"* ]] [[ "${lines[@]}" == *" 1 ftl.pi-hole.net"* ]] - [[ ${lines[6]} == "5 2 google.com" ]] - [[ ${lines[7]} == "" ]] + [[ "${lines[9]}" == *" 2 google.com"* ]] + [[ "${lines[10]}" == "" ]] } @test "Top Ads (descending, default)" { @@ -160,31 +218,31 @@ @test "Forward Destinations" { run bash -c 'echo ">forward-dest >quit" | nc -v 127.0.0.1 4711' printf "%s\n" "${lines[@]}" - [[ ${lines[1]} == "-2 30.00 blocklist blocklist" ]] - [[ ${lines[2]} == "-1 20.00 cache cache" ]] - [[ ${lines[3]} == "0 50.00 "* ]] + [[ ${lines[1]} == "-2 23.08 blocklist blocklist" ]] + [[ ${lines[2]} == "-1 15.38 cache cache" ]] + [[ ${lines[3]} == "0 61.54 "* ]] [[ ${lines[4]} == "" ]] } @test "Forward Destinations (unsorted)" { run bash -c 'echo ">forward-dest unsorted >quit" | nc -v 127.0.0.1 4711' printf "%s\n" "${lines[@]}" - [[ ${lines[1]} == "-2 30.00 blocklist blocklist" ]] - [[ ${lines[2]} == "-1 20.00 cache cache" ]] - [[ ${lines[3]} == "0 50.00 "* ]] + [[ ${lines[1]} == "-2 23.08 blocklist blocklist" ]] + [[ ${lines[2]} == "-1 15.38 cache cache" ]] + [[ ${lines[3]} == "0 61.54 "* ]] [[ ${lines[4]} == "" ]] } @test "Query Types" { run bash -c 'echo ">querytypes >quit" | nc -v 127.0.0.1 4711' printf "%s\n" "${lines[@]}" - [[ ${lines[1]} == "A (IPv4): 70.00" ]] - [[ ${lines[2]} == "AAAA (IPv6): 10.00" ]] + [[ ${lines[1]} == "A (IPv4): 76.92" ]] + [[ ${lines[2]} == "AAAA (IPv6): 7.69" ]] [[ ${lines[3]} == "ANY: 0.00" ]] [[ ${lines[4]} == "SRV: 0.00" ]] [[ ${lines[5]} == "SOA: 0.00" ]] [[ ${lines[6]} == "PTR: 0.00" ]] - [[ ${lines[7]} == "TXT: 20.00" ]] + [[ ${lines[7]} == "TXT: 15.38" ]] [[ ${lines[8]} == "" ]] } @@ -199,12 +257,15 @@ [[ ${lines[3]} == *"A blacklisted.com "?*" 5 0 4"* ]] [[ ${lines[4]} == *"A 0427d7.se "?*" 1 0 4"* ]] [[ ${lines[5]} == *"A whitelisted.com "?*" 2 0 4"* ]] - [[ ${lines[6]} == *"A regex5.com "?*" 4 0 4"* ]] - [[ ${lines[7]} == *"A regexa.com "?*" 2 0 7"* ]] - [[ ${lines[8]} == *"A google.com "?*" 2 0 4"* ]] - [[ ${lines[9]} == *"AAAA google.com "?*" 2 0 4"* ]] - [[ ${lines[10]} == *"A ftl.pi-hole.net "?*" 2 0 4"* ]] - [[ ${lines[11]} == "" ]] + [[ ${lines[6]} == *"A 01tse443.se "?*" 2 0 2"* ]] + [[ ${lines[7]} == *"A regex5.com "?*" 4 0 4"* ]] + [[ ${lines[8]} == *"A regexa.com "?*" 2 0 7"* ]] + [[ ${lines[9]} == *"A regex1.com "?*" 2 0 4"* ]] + [[ ${lines[10]} == *"A regex2.com "?*" 2 0 2"* ]] + [[ ${lines[11]} == *"A google.com "?*" 2 0 4"* ]] + [[ ${lines[12]} == *"AAAA google.com "?*" 2 0 4"* ]] + [[ ${lines[13]} == *"A ftl.pi-hole.net "?*" 2 0 4"* ]] + [[ ${lines[14]} == "" ]] } @test "Get all queries (domain filtered)" { @@ -229,12 +290,15 @@ [[ ${lines[3]} == *"A blacklisted.com "?*" 5 0 4"* ]] [[ ${lines[4]} == *"A 0427d7.se "?*" 1 0 4"* ]] [[ ${lines[5]} == *"A whitelisted.com "?*" 2 0 4"* ]] - [[ ${lines[6]} == *"A regex5.com "?*" 4 0 4"* ]] - [[ ${lines[7]} == *"A regexa.com "?*" 2 0 7"* ]] - [[ ${lines[8]} == *"A google.com "?*" 2 0 4"* ]] - [[ ${lines[9]} == *"AAAA google.com "?*" 2 0 4"* ]] - [[ ${lines[10]} == *"A ftl.pi-hole.net "?*" 2 0 4"* ]] - [[ ${lines[11]} == "" ]] + [[ ${lines[6]} == *"A 01tse443.se "?*" 2 0 2"* ]] + [[ ${lines[7]} == *"A regex5.com "?*" 4 0 4"* ]] + [[ ${lines[8]} == *"A regexa.com "?*" 2 0 7"* ]] + [[ ${lines[9]} == *"A regex1.com "?*" 2 0 4"* ]] + [[ ${lines[10]} == *"A regex2.com "?*" 2 0 2"* ]] + [[ ${lines[11]} == *"A google.com "?*" 2 0 4"* ]] + [[ ${lines[12]} == *"AAAA google.com "?*" 2 0 4"* ]] + [[ ${lines[13]} == *"A ftl.pi-hole.net "?*" 2 0 4"* ]] + [[ ${lines[14]} == "" ]] } @test "Get all queries (client + number filtered)" {