Merge pull request #129 from pi-hole/development

FTL v2.11

README.md

@@ -117,6 +117,11 @@ domains_being_blocked 19977
 dns_queries_today 104749
 ads_blocked_today 279
 ads_percentage_today 1.396606
+unique_domains 6
+queries_forwarded 3
+queries_cached 2
+clients_ever_seen 3
+unique_clients 3
 ```
 
 - `>overTime` : over time data (10 min intervals)
@@ -156,26 +161,29 @@ ads_percentage_today 1.396606
 ```
  Variant: `>top-ads (14)` to show (up to) 14 entries
 
-- `top-clients` : get top clients (IP addresses + host names (if available))
+- `top-clients` : get recently active top clients (IP addresses + host names (if available))
  ```
 0 9373 192.168.2.1 router
 1 484 192.168.2.2 work-machine
 2 8 127.0.0.1 localhost
 ```
  Variant: `>top-clients (9)` to show (up to) 9 client entries
 
-- `>forward-dest` : get forward destinations (IP addresses + host names (if available))
+ Variant: `>top-clients withzero (15)` to show (up to) 15 clients even if they have not been active recently (see PR #124 for further details)
+
+- `>forward-dest` : get forward destinations (IP addresses + host names (if available)) along with the percentage
  ```
-0 12940 1.2.3.4 some.dns.de
-1 629 5.6.7.8 some.other.dns.com
+0 80.0 ::1 local
+1 10.0 1.2.3.4 some.dns.de
+2 10.0 5.6.7.8 some.other.dns.com
+```
+ Variant: `>forward-dest unsorted` to show forward destinations in unsorted order (equivalent to using `>forward-names`)
 ```
 
-- `>querytypes` : get collected query types
+- `>querytypes` : get collected query types percentage
  ```
-A (IPv4): 7729
-AAAA (IPv6): 5880
-PTR: 12
-SRV: 0
+A (IPv4): 60.5
+AAAA (IPv6): 39.5
 ```
 
 - `>getallqueries` : get all queries that FTL has in its database

gc.c

@@ -148,6 +148,10 @@ void *GC_thread(void *val)
  logg("Notice: GC removed %i queries", invalidated);
  }
 
+ // Run reresolveHostnames at the end of GC to account for
+ // formally unknown and/or changed host names on the network
+ reresolveHostnames();
+
  // Release thread lock
  disable_thread_lock("GC_thread");
 

main.c

@@ -90,6 +90,7 @@ int main (int argc, char* argv[]) {
  runDBthread = true;
 
  // Garbadge collect in regular interval, but don't do it if the threadlocks is set
+ // This will also run reresolveHostnames()
  if(runGCthread || needGC)
  {
  // Wait until we are allowed to work on the data

parser.c

@@ -13,7 +13,7 @@
 
 char *resolveHostname(const char *addr);
 void extracttimestamp(const char *readbuffer, int *querytimestamp, int *overTimetimestamp);
-int getforwardID(const char * str);
+int getforwardID(const char * str, bool count);
 int findDomain(const char *domain);
 int findClient(const char *client);
 int detectStatus(const char *domain);
@@ -245,19 +245,43 @@ void process_pihole_log(int file)
  }
  if(!found)
  {
- timeidx = counters.overTime;
- validate_access("overTime", timeidx, false, __LINE__, __FUNCTION__, __FILE__);
- // Set magic byte
- overTime[timeidx].magic = MAGICBYTE;
- overTime[timeidx].timestamp = overTimetimestamp;
- overTime[timeidx].total = 0;
- overTime[timeidx].blocked = 0;
- overTime[timeidx].cached = 0;
- overTime[timeidx].forwardnum = 0;
- overTime[timeidx].forwarddata = NULL;
- overTime[timeidx].querytypedata = calloc(2, sizeof(int));
- memory.querytypedata += 2*sizeof(int);
- counters.overTime++;
+ // We loop over this to fill potential data holes with zeros
+ int nexttimestamp = 0;
+ if(counters.overTime != 0)
+ {
+ validate_access("overTime", counters.overTime-1, false, __LINE__, __FUNCTION__, __FILE__);
+ nexttimestamp = overTime[counters.overTime-1].timestamp + 600;
+ }
+ else
+ {
+ nexttimestamp = overTimetimestamp;
+ }
+
+ while(overTimetimestamp >= nexttimestamp)
+ {
+ // Check struct size
+ memory_check(OVERTIME);
+ timeidx = counters.overTime;
+ validate_access("overTime", timeidx, false, __LINE__, __FUNCTION__, __FILE__);
+ // Set magic byte
+ overTime[timeidx].magic = MAGICBYTE;
+ overTime[timeidx].timestamp = nexttimestamp;
+ overTime[timeidx].total = 0;
+ overTime[timeidx].blocked = 0;
+ overTime[timeidx].cached = 0;
+ overTime[timeidx].forwardnum = 0;
+ overTime[timeidx].forwarddata = NULL;
+ overTime[timeidx].querytypedata = calloc(2, sizeof(int));
+ memory.querytypedata += 2*sizeof(int);
+ counters.overTime++;
+
+ // Update time stamp for next loop interation
+ if(counters.overTime != 0)
+ {
+ validate_access("overTime", counters.overTime-1, false, __LINE__, __FUNCTION__, __FILE__);
+ nexttimestamp = overTime[counters.overTime-1].timestamp + 600;
+ }
+ }
  }
 
  // Get domain
@@ -378,7 +402,7 @@ void process_pihole_log(int file)
  status = 2;
  // Get ID of forward destination, create new forward destination record
  // if not found in current data structure
- forwardID = getforwardID(readbuffer2);
+ forwardID = getforwardID(readbuffer2, false);
  if(forwardID == -2)
  continue;
  break;
@@ -580,9 +604,14 @@ void process_pihole_log(int file)
  continue;
  }
 
+ // Check if this is a PTR query
+ // if so: skip analysis of this log line
+ if(strstr(readbuffer,"in-addr.arpa") != NULL)
+ continue;
+
  // Get ID of forward destination, create new forward destination record
  // if not found in current data structure
- int forwardID = getforwardID(readbuffer);
+ int forwardID = getforwardID(readbuffer, true);
  if(forwardID == -2)
  continue;
 
@@ -821,7 +850,7 @@ void extracttimestamp(const char *readbuffer, int *querytimestamp, int *overTime
  *overTimetimestamp = *querytimestamp-(*querytimestamp%600)+300;
 }
 
-int getforwardID(const char * str)
+int getforwardID(const char * str, bool count)
 {
  // Get forward destination
  // forwardstart = pointer to | in "forwarded domain.name| to www.xxx.yyy.zzz\n"
@@ -866,7 +895,8 @@ int getforwardID(const char * str)
  if(strcmp(forwarded[i].ip, forward) == 0)
  {
  forwardID = i;
- forwarded[forwardID].count++;
+ if(count)
+ forwarded[forwardID].count++;
  processed = true;
  break;
  }
@@ -886,8 +916,11 @@ int getforwardID(const char * str)
  validate_access("forwarded", forwardID, false, __LINE__, __FUNCTION__, __FILE__);
  // Set magic byte
  forwarded[forwardID].magic = MAGICBYTE;
- // Set its counter to 1
- forwarded[forwardID].count = 1;
+ // Initialize its counter
+ if(count)
+ forwarded[forwardID].count = 1;
+ else
+ forwarded[forwardID].count = 0;
  // Save IP
  forwarded[forwardID].ip = strdup(forward);
  memory.forwardedips += (forwardlen + 1) * sizeof(char);
@@ -991,3 +1024,35 @@ void validate_access_oTfd(int timeidx, int pos, int line, const char * function,
  logg(" found in %s() (line %i) in %s", function, line, file);
  }
 }
+
+void reresolveHostnames(void)
+{
+ int clientID;
+ for(clientID = 0; clientID < counters.clients; clientID++)
+ {
+ // Memory validation
+ validate_access("clients", clientID, true, __LINE__, __FUNCTION__, __FILE__);
+
+ // Process this client only if it has at least one active query in the log
+ if(clients[clientID].count < 1)
+ continue;
+
+ // Get client hostname
+ char *hostname = resolveHostname(clients[clientID].ip);
+ if(strlen(hostname) > 0)
+ {
+ // Delete possibly already existing hostname pointer before storing new data
+ if(clients[clientID].name != NULL)
+ {
+ memory.clientnames -= (strlen(clients[clientID].name) + 1) * sizeof(char);
+ free(clients[clientID].name);
+ clients[clientID].name = NULL;
+ }
+
+ // Store client hostname
+ clients[clientID].name = strdup(hostname);
+ memory.clientnames += (strlen(hostname) + 1) * sizeof(char);
+ }
+ free(hostname);
+ }
+}