From 36d14998043f7c27eabaa0d6a0aec4e1dda6cfc5 Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Sun, 23 Dec 2018 12:40:39 +0100
Subject: [PATCH 01/27] Add network table. This will update the database to
 version 3. Furthermore, we make some database routines globally (add
 prototypes to routines.h) and mark some internal database variables as
 static. This commit also improves on the speed of the database routines as
 the main loop is changed to run from the last saved query to the most recent
 one instead of looping over all queries in memory. This ID is corrected when
 queries are removed in gc.c

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 FTL.h          |  6 +++++-
 Makefile       |  2 +-
 database.c     | 47 +++++++++++++++++++++++++++++++----------------
 gc.c           |  2 ++
 networktable.c | 31 +++++++++++++++++++++++++++++++
 routines.h     |  7 +++++++
 6 files changed, 77 insertions(+), 18 deletions(-)
 create mode 100644 networktable.c

diff --git a/FTL.h b/FTL.h
index 402f48efd..876910f7b 100644
--- a/FTL.h
+++ b/FTL.h
@@ -80,6 +80,11 @@ enum { MODE_IP, MODE_NX, MODE_NULL, MODE_IP_NODATA_AAAA, MODE_NODATA };
 enum { REGEX_UNKNOWN, REGEX_BLOCKED, REGEX_NOTBLOCKED };
 enum { BLOCKING_DISABLED, BLOCKING_ENABLED, BLOCKING_UNKNOWN };
 
+// Database table "ftl"
+enum { DB_VERSION, DB_LASTTIMESTAMP, DB_FIRSTCOUNTERTIMESTAMP };
+// Database table "counters"
+enum { DB_TOTALQUERIES, DB_BLOCKEDQUERIES };
+
 // Privacy mode constants
 #define HIDDEN_DOMAIN "hidden"
 #define HIDDEN_CLIENT "0.0.0.0"
@@ -250,7 +255,6 @@ extern long int lastdbindex;
 extern bool travis;
 extern bool DBdeleteoldqueries;
 extern bool rereadgravity;
-extern long int lastDBimportedtimestamp;
 extern bool ipv4telnet, ipv6telnet;
 extern bool istelnet[MAXCONNS];
 
diff --git a/Makefile b/Makefile
index 6e27a0e7e..c2f4b1dc3 100644
--- a/Makefile
+++ b/Makefile
@@ -14,7 +14,7 @@ DNSMASQOPTS = -DHAVE_DNSSEC -DHAVE_DNSSEC_STATIC
 # Flags for compiling with libidn2: -DHAVE_LIBIDN2 -DIDN2_VERSION_NUMBER=0x02000003
 
 FTLDEPS = FTL.h routines.h version.h api.h dnsmasq_interface.h shmem.h
-FTLOBJ = main.o memory.o log.o daemon.o datastructure.o signals.o socket.o request.o grep.o setupVars.o args.o gc.o config.o database.o msgpack.o api.o dnsmasq_interface.o resolve.o regex.o shmem.o
+FTLOBJ = main.o memory.o log.o daemon.o datastructure.o signals.o socket.o request.o grep.o setupVars.o args.o gc.o config.o database.o msgpack.o api.o dnsmasq_interface.o resolve.o regex.o shmem.o networktable.o
 
 DNSMASQDEPS = config.h dhcp-protocol.h dns-protocol.h radv-protocol.h dhcp6-protocol.h dnsmasq.h ip6addr.h metrics.h
 DNSMASQOBJ = arp.o dbus.o domain.o lease.o outpacket.o rrfilter.o auth.o dhcp6.o edns0.o log.o poll.o slaac.o blockdata.o dhcp.o forward.o loop.o radv.o tables.o bpf.o dhcp-common.o helper.o netlink.o rfc1035.o tftp.o cache.o dnsmasq.o inotify.o network.o rfc2131.o util.o conntrack.o dnssec.o ipset.o option.o rfc3315.o crypto.o dump.o ubus.o metrics.o
diff --git a/database.c b/database.c
index 7dcb6cbaf..bc930619a 100644
--- a/database.c
+++ b/database.c
@@ -11,21 +11,14 @@
 #include "FTL.h"
 #include "shmem.h"
 
-sqlite3 *db;
+static sqlite3 *db;
 bool database = false;
 bool DBdeleteoldqueries = false;
 long int lastdbindex = 0;
-long int lastDBimportedtimestamp = 0;
 
-pthread_mutex_t dblock;
-
-// TABLE ftl
-enum { DB_VERSION, DB_LASTTIMESTAMP, DB_FIRSTCOUNTERTIMESTAMP };
-// TABLE counters
-enum { DB_TOTALQUERIES, DB_BLOCKEDQUERIES };
+static pthread_mutex_t dblock;
 
 bool db_set_counter(unsigned int ID, int value);
-bool db_set_FTL_property(unsigned int ID, int value);
 int db_get_FTL_property(unsigned int ID);
 
 void check_database(int rc)
@@ -155,8 +148,8 @@ bool db_create(void)
 	ret = dbquery("CREATE TABLE ftl ( id INTEGER PRIMARY KEY NOT NULL, value BLOB NOT NULL );");
 	if(!ret){ dbclose(); return false; }
 
-	// DB version 2
-	ret = dbquery("INSERT INTO ftl (ID,VALUE) VALUES(%i,2);", DB_VERSION);
+	// Set DB version 1
+	ret = dbquery("INSERT INTO ftl (ID,VALUE) VALUES(%i,1);", DB_VERSION);
 	if(!ret){ dbclose(); return false; }
 
 	// Most recent timestamp initialized to 00:00 1 Jan 1970
@@ -164,9 +157,15 @@ bool db_create(void)
 	if(!ret){ dbclose(); return false; }
 
 	// Create counter table
+	// Will update DB version to 2
 	if(!create_counter_table())
 		return false;
 
+	// Create network table
+	// Will update DB version to 3
+	if(!create_network_table())
+		return false;
+
 	return true;
 }
 
@@ -203,16 +202,33 @@ void db_init(void)
 		database = false;
 		return;
 	}
-	else if(dbversion < 2)
+	// Update to version 2 if still version 1
+	if(dbversion < 2)
 	{
-		// Database is still in version 1
-		// Update to version 2 and create counters table
+		// Update to version 2: Create counters table
+		logg("Updating long-term database to version 2");
 		if (!create_counter_table())
 		{
 			logg("Counter table not initialized, database not available");
 			database = false;
 			return;
 		}
+		// Get updated version
+		dbversion = db_get_FTL_property(DB_VERSION);
+	}
+	// Update to version 2 if still version 1
+	if(dbversion < 3)
+	{
+		// Update to version 3: Create network table
+		logg("Updating long-term database to version 3");
+		if (!create_network_table())
+		{
+			logg("Network table not initialized, database not available");
+			database = false;
+			return;
+		}
+		// Get updated version
+		dbversion = db_get_FTL_property(DB_VERSION);
 	}
 
 	// Close database to prevent having it opened all time
@@ -406,7 +422,7 @@ void save_to_DB(void)
 	int total = 0, blocked = 0;
 	time_t currenttimestamp = time(NULL);
 	time_t newlasttimestamp = 0;
-	for(i = 0; i < counters->queries; i++)
+	for(i = lastdbindex; i < counters->queries; i++)
 	{
 		validate_access("queries", i, true, __LINE__, __FUNCTION__, __FILE__);
 		if(queries[i].db != 0)
@@ -741,7 +757,6 @@ void read_data_from_DB(void)
 		queries[queryIndex].complete = true; // Mark as all information is avaiable
 		queries[queryIndex].response = 0;
 		queries[queryIndex].AD = false;
-		lastDBimportedtimestamp = queryTimeStamp;
 
 		// Handle type counters
 		if(type >= TYPE_A && type < TYPE_MAX)
diff --git a/gc.c b/gc.c
index bc93aa588..813b96aa2 100644
--- a/gc.c
+++ b/gc.c
@@ -154,6 +154,8 @@ void *GC_thread(void *val)
 
 			// Update queries counter
 			counters->queries -= removed;
+			// Update DB index as total number of queries reduced
+			lastdbindex -= removed;
 
 			// Zero out remaining memory (marked as "F" in the above example)
 			memset(&queries[counters->queries], 0, (counters->queries_MAX - counters->queries)*sizeof(*queries));
diff --git a/networktable.c b/networktable.c
new file mode 100644
index 000000000..79cb7853e
--- /dev/null
+++ b/networktable.c
@@ -0,0 +1,31 @@
+/* Pi-hole: A black hole for Internet advertisements
+*  (c) 2017 Pi-hole, LLC (https://pi-hole.net)
+*  Network-wide ad blocking via your own hardware.
+*
+*  FTL Engine
+*  Network table routines
+*
+*  This file is copyright under the latest version of the EUPL.
+*  Please see LICENSE file for your rights under this license. */
+
+#include "FTL.h"
+
+bool create_network_table(void)
+{
+	bool ret;
+	// Create FTL table in the database (holds properties like database version, etc.)
+	ret = dbquery("CREATE TABLE network ( id        INTEGER PRIMARY KEY NOT NULL, \
+	                                      ip        TEXT NOT NULL, \
+	                                      mac       TEXT NOT NULL, \
+	                                      name      TEXT, \
+	                                      firstSeen INTEGER NOT NULL, \
+	                                      lastSeen  INTEGER NOT NULL, \
+	                                      PiholeDNS BOOLEAN NOT NULL );");
+	if(!ret){ dbclose(); return false; }
+
+	// Update database version to 3
+	ret = db_set_FTL_property(DB_VERSION, 3);
+	if(!ret){ dbclose(); return false; }
+
+	return true;
+}
diff --git a/routines.h b/routines.h
index b4ad082b4..c4a3a22db 100644
--- a/routines.h
+++ b/routines.h
@@ -84,6 +84,10 @@ int get_number_of_queries_in_DB(void);
 void save_to_DB(void);
 void read_data_from_DB(void);
 
+bool db_set_FTL_property(unsigned int ID, int value);
+bool dbquery(const char *format, ...);
+void dbclose(void);
+
 // memory.c
 void memory_check(int which);
 char *FTLstrdup(const char *src, const char *file, const char *function, int line);
@@ -126,3 +130,6 @@ void newOverTimeClient();
  * This also updates `overTimeClientData`.
  */
 void addOverTimeClientSlot();
+
+// networktable.c
+bool create_network_table(void);

From 40fd4d317e8d46ae92ceaaada2b64ee0d55be257 Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Mon, 24 Dec 2018 13:18:51 +0100
Subject: [PATCH 02/27] Add / update rows in table network depending on what we
 see in the ARP cache

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 database.c     |  46 +++++++++++++++++++++-
 networktable.c | 103 +++++++++++++++++++++++++++++++++++++++++++++----
 request.c      |   5 +++
 routines.h     |   4 +-
 4 files changed, 148 insertions(+), 10 deletions(-)

diff --git a/database.c b/database.c
index bc930619a..d05939935 100644
--- a/database.c
+++ b/database.c
@@ -32,6 +32,7 @@ void check_database(int rc)
 	   rc != SQLITE_ROW &&
 	   rc != SQLITE_BUSY)
 	{
+		logg("check_database(%i): Disabling database connection due to error", rc);
 		database = false;
 	}
 }
@@ -196,13 +197,14 @@ void db_init(void)
 
 	// Test DB version and see if we need to upgrade the database file
 	int dbversion = db_get_FTL_property(DB_VERSION);
+	logg("Database version is %i", dbversion);
 	if(dbversion < 1)
 	{
 		logg("Database version incorrect, database not available");
 		database = false;
 		return;
 	}
-	// Update to version 2 if still version 1
+	// Update to version 2 if lower
 	if(dbversion < 2)
 	{
 		// Update to version 2: Create counters table
@@ -216,7 +218,7 @@ void db_init(void)
 		// Get updated version
 		dbversion = db_get_FTL_property(DB_VERSION);
 	}
-	// Update to version 2 if still version 1
+	// Update to version 3 if lower
 	if(dbversion < 3)
 	{
 		// Update to version 3: Create network table
@@ -306,6 +308,46 @@ bool db_update_counters(int total, int blocked)
 	return true;
 }
 
+int db_query_int(const char* querystr)
+{
+	// Check if database is enabled
+	if(!database)
+		return -2;
+
+	sqlite3_stmt* stmt;
+	int rc = sqlite3_prepare_v2(db, querystr, -1, &stmt, NULL);
+	if( rc ){
+		logg("db_query_int(%s) - SQL error prepare (%i): %s", querystr, rc, sqlite3_errmsg(db));
+		dbclose();
+		check_database(rc);
+		return -2;
+	}
+
+	rc = sqlite3_step(stmt);
+	int result;
+
+	if( rc == SQLITE_ROW )
+	{
+		result = sqlite3_column_int(stmt, 0);
+	}
+	else if( rc == SQLITE_DONE )
+	{
+		// No rows available
+		result = -1;
+	}
+	else
+	{
+		logg("db_query_int(%s) - SQL error step (%i): %s", querystr, rc, sqlite3_errmsg(db));
+		dbclose();
+		check_database(rc);
+		return -2;
+	}
+
+	sqlite3_finalize(stmt);
+
+	return result;
+}
+
 int number_of_queries_in_DB(void)
 {
 	sqlite3_stmt* stmt;
diff --git a/networktable.c b/networktable.c
index 79cb7853e..7f94c4369 100644
--- a/networktable.c
+++ b/networktable.c
@@ -9,18 +9,20 @@
 *  Please see LICENSE file for your rights under this license. */
 
 #include "FTL.h"
+#define ARPCACHE "/proc/net/arp"
 
 bool create_network_table(void)
 {
 	bool ret;
 	// Create FTL table in the database (holds properties like database version, etc.)
-	ret = dbquery("CREATE TABLE network ( id        INTEGER PRIMARY KEY NOT NULL, \
-	                                      ip        TEXT NOT NULL, \
-	                                      mac       TEXT NOT NULL, \
-	                                      name      TEXT, \
-	                                      firstSeen INTEGER NOT NULL, \
-	                                      lastSeen  INTEGER NOT NULL, \
-	                                      PiholeDNS BOOLEAN NOT NULL );");
+	ret = dbquery("CREATE TABLE network ( id INTEGER PRIMARY KEY NOT NULL, " \
+	                                     "ip TEXT NOT NULL, " \
+	                                     "hwaddr TEXT NOT NULL, " \
+	                                     "interface TEXT NOT NULL, " \
+	                                     "name TEXT, " \
+	                                     "firstSeen INTEGER NOT NULL, " \
+	                                     "lastSeen INTEGER NOT NULL, " \
+	                                     "usesPihole BOOLEAN NOT NULL );");
 	if(!ret){ dbclose(); return false; }
 
 	// Update database version to 3
@@ -29,3 +31,90 @@ bool create_network_table(void)
 
 	return true;
 }
+
+// Read kernel's ARP cache using procfs
+void read_arp_cache(void)
+{
+	FILE* arpfp = NULL;
+	// Try to access the kernel's ARP cache
+	if((arpfp = fopen(ARPCACHE, "r")) == NULL)
+	{
+		logg("WARN: Opening of %s failed!", ARPCACHE);
+		logg("      Message: %s", strerror(errno));
+		return;
+	}
+
+	// Open database file
+	if(!dbopen())
+	{
+		logg("read_arp_cache() - Failed to open DB");
+		return;
+	}
+
+	// Prepare buffers
+	char * linebuffer = NULL;
+	size_t linebuffersize = 0;
+	char ip[100], mask[100], hwaddr[100], iface[100];
+	int type, flags, entries = 0;
+	time_t now = time(NULL);
+
+	// Read ARP cache line by line
+	while(getline(&linebuffer, &linebuffersize, arpfp) != -1)
+	{
+		int num = sscanf(linebuffer, "%99s 0x%x 0x%x %99s %99s %99s\n",
+		                 ip, &type, &flags, hwaddr, mask, iface);
+
+		// Skip header and empty lines
+		if (num < 4)
+			continue;
+
+		if (num == 5)
+		{
+			/*
+			* This happens for incomplete ARP entries for which there is
+			* no hardware address in the line. We don't use these
+			*/
+			//num = sscanf(linebuffer, "%s 0x%x 0x%x %99s %99s\n",
+			//             ip, &type, &flags, mask, iface);
+			//hwaddr[0] = '\0';
+		}
+
+		entries++;
+		if(debug) logg("ARP (%i): %i %i %s %s %s <-> %s", num, type, flags, mask, iface, hwaddr, ip);
+
+		// Get ID of this device in our network database. If it cannot be found, then this is a new device
+		char querystr[256];
+		sprintf(querystr, "SELECT id FROM network WHERE ip = \"%s\" AND hwaddr = \"%s\";", ip, hwaddr);
+		int dbID = db_query_int(querystr);
+
+		if(dbID == -2)
+		{
+			// SQLite error
+			break;
+		}
+		else if(dbID == -1)
+		{
+			// Device not in database, add new entry
+			dbquery("INSERT INTO network "\
+			        "(ip,hwaddr,interface,firstSeen,lastSeen,usesPihole) "\
+			        "VALUES "\
+			        "(\"%s\",\"%s\",\"%s\",%lu,%lu,false);",\
+			        ip, hwaddr, iface, now, now);
+		}
+		else
+		{
+			// Device already known, update lastSeen
+			dbquery("UPDATE network "\
+			        "SET lastSeen = %lu "\
+			        "WHERE id = %i;",\
+			        now, dbID);
+		}
+
+	}
+
+	// Close file handle
+	fclose(arpfp);
+
+	// Close database connection
+	dbclose();
+}
diff --git a/request.c b/request.c
index 82a791a21..1549e0e5c 100644
--- a/request.c
+++ b/request.c
@@ -132,6 +132,11 @@ void process_request(char *client_message, int *sock)
 		free_regex();
 		read_regex_from_file();
 	}
+	else if(command(client_message, ">arp"))
+	{
+		processed = true;
+		read_arp_cache();
+	}
 
 	// Test only at the end if we want to quit or kill
 	// so things can be processed before
diff --git a/routines.h b/routines.h
index c4a3a22db..c3485379d 100644
--- a/routines.h
+++ b/routines.h
@@ -83,10 +83,11 @@ void *DB_thread(void *val);
 int get_number_of_queries_in_DB(void);
 void save_to_DB(void);
 void read_data_from_DB(void);
-
 bool db_set_FTL_property(unsigned int ID, int value);
 bool dbquery(const char *format, ...);
+bool dbopen(void);
 void dbclose(void);
+int db_query_int(const char*);
 
 // memory.c
 void memory_check(int which);
@@ -133,3 +134,4 @@ void addOverTimeClientSlot();
 
 // networktable.c
 bool create_network_table(void);
+void read_arp_cache(void);

From c7bdf9bd60ef6e2484cae3678b908d92a1bb4134 Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Tue, 25 Dec 2018 13:44:39 +0100
Subject: [PATCH 03/27] Update if device uses Pi-hole and store host name if
 available Print executed SQL statements when in debug mode

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 database.c          |  4 +++-
 datastructure.c     |  6 +++++-
 dnsmasq_interface.c |  2 +-
 networktable.c      | 50 +++++++++++++++++++++++++++++++--------------
 routines.h          |  2 +-
 5 files changed, 45 insertions(+), 19 deletions(-)

diff --git a/database.c b/database.c
index d05939935..fe1dc64ac 100644
--- a/database.c
+++ b/database.c
@@ -88,6 +88,8 @@ bool dbquery(const char *format, ...)
 		return false;
 	}
 
+	if(debug) logg("dbquery: %s", query);
+
 	int rc = sqlite3_exec(db, query, NULL, NULL, &zErrMsg);
 
 	if( rc != SQLITE_OK ){
@@ -775,7 +777,7 @@ void read_data_from_DB(void)
 		int overTimeTimeStamp = queryTimeStamp - (queryTimeStamp % 600) + 300;
 		int timeidx = findOverTimeID(overTimeTimeStamp);
 		int domainID = findDomainID(domain);
-		int clientID = findClientID(client);
+		int clientID = findClientID(client, true);
 
 		// Ensure we have enough space in the queries struct
 		memory_check(QUERIES);
diff --git a/datastructure.c b/datastructure.c
index 18cb871ac..610d4cd9e 100644
--- a/datastructure.c
+++ b/datastructure.c
@@ -176,7 +176,7 @@ int findDomainID(const char *domain)
 	return domainID;
 }
 
-int findClientID(const char *client)
+int findClientID(const char *client, bool addNew)
 {
 	int i;
 	// Compare content of client against known client IP addresses
@@ -196,6 +196,10 @@ int findClientID(const char *client)
 		}
 	}
 
+	// Return -1 (= not found) if addNew is false
+	if(!addNew)
+		return -1;
+
 	// If we did not return until here, then this client is definitely new
 	// Store ID
 	int clientID = counters->clients;
diff --git a/dnsmasq_interface.c b/dnsmasq_interface.c
index 9aa4869d9..45d8bac90 100644
--- a/dnsmasq_interface.c
+++ b/dnsmasq_interface.c
@@ -137,7 +137,7 @@ void FTL_new_query(unsigned int flags, char *name, struct all_addr *addr, char *
 	int domainID = findDomainID(domain);
 
 	// Go through already knows clients and see if it is one of them
-	int clientID = findClientID(client);
+	int clientID = findClientID(client, true);
 
 	// Save everything
 	validate_access("queries", queryID, false, __LINE__, __FUNCTION__, __FILE__);
diff --git a/networktable.c b/networktable.c
index 7f94c4369..1ef169161 100644
--- a/networktable.c
+++ b/networktable.c
@@ -68,17 +68,6 @@ void read_arp_cache(void)
 		if (num < 4)
 			continue;
 
-		if (num == 5)
-		{
-			/*
-			* This happens for incomplete ARP entries for which there is
-			* no hardware address in the line. We don't use these
-			*/
-			//num = sscanf(linebuffer, "%s 0x%x 0x%x %99s %99s\n",
-			//             ip, &type, &flags, mask, iface);
-			//hwaddr[0] = '\0';
-		}
-
 		entries++;
 		if(debug) logg("ARP (%i): %i %i %s %s %s <-> %s", num, type, flags, mask, iface, hwaddr, ip);
 
@@ -92,22 +81,53 @@ void read_arp_cache(void)
 			// SQLite error
 			break;
 		}
-		else if(dbID == -1)
+
+		// If we reach this point, we can check if this client
+		// is known to pihole-FTL
+		// false = do not create a new record if the client is
+		//         unknown (only DNS requesting clients do this)
+		int clientID = findClientID(ip, false);
+		bool clientKnown = clientID >= 0;
+
+		if(dbID == -1)
 		{
 			// Device not in database, add new entry
 			dbquery("INSERT INTO network "\
 			        "(ip,hwaddr,interface,firstSeen,lastSeen,usesPihole) "\
 			        "VALUES "\
-			        "(\"%s\",\"%s\",\"%s\",%lu,%lu,false);",\
-			        ip, hwaddr, iface, now, now);
+			        "(\"%s\",\"%s\",\"%s\",%lu,%lu,%s);",\
+			        ip, hwaddr, iface, now, now,
+			        clientKnown ? "true" : "false");
 		}
 		else
 		{
-			// Device already known, update lastSeen
+			// Device already in database, update lastSeen
 			dbquery("UPDATE network "\
 			        "SET lastSeen = %lu "\
 			        "WHERE id = %i;",\
 			        now, dbID);
+
+			// Store if device uses Pi-hole
+			if(clientKnown)
+			{
+				// Device uses Pi-hole, update record
+				dbquery("UPDATE network "\
+				        "SET usesPihole = true "\
+				        "WHERE id = %i;",\
+				        dbID);
+			}
+		}
+
+		char *hostname = NULL;
+		if(clientKnown)
+			hostname = getstr(clients[clientID].namepos);
+		if(hostname != NULL && strlen(hostname) > 0)
+		{
+			// Store host name
+			dbquery("UPDATE network "\
+			        "SET name = \"%s\" "\
+			        "WHERE id = %i;",\
+			        hostname, dbID);
 		}
 
 	}
diff --git a/routines.h b/routines.h
index c3485379d..06c29ef98 100644
--- a/routines.h
+++ b/routines.h
@@ -29,7 +29,7 @@ void strtolower(char *str);
 int findOverTimeID(int overTimetimestamp);
 int findForwardID(const char * forward, bool count);
 int findDomainID(const char *domain);
-int findClientID(const char *client);
+int findClientID(const char *client, bool addNew);
 bool isValidIPv4(const char *addr);
 bool isValidIPv6(const char *addr);
 char *getDomainString(int queryID);

From 82b43c332ba0bc1e871b88ecc2607c0c36327287 Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Tue, 25 Dec 2018 17:56:44 +0100
Subject: [PATCH 04/27] Parse ARP cache after storing queries into long-term
 database(usen the dedicated database thread)

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 FTL.h          |  4 ++--
 database.c     |  3 +++
 networktable.c | 54 ++++++++++++++++++++++++++++++--------------------
 request.c      |  5 -----
 routines.h     |  2 +-
 5 files changed, 39 insertions(+), 29 deletions(-)

diff --git a/FTL.h b/FTL.h
index 876910f7b..e2a8f708e 100644
--- a/FTL.h
+++ b/FTL.h
@@ -69,7 +69,7 @@
 #define MAXITER 1000
 
 // FTLDNS enums
-enum { DATABASE_WRITE_TIMER, EXIT_TIMER, GC_TIMER, LISTS_TIMER, REGEX_TIMER };
+enum { DATABASE_WRITE_TIMER, EXIT_TIMER, GC_TIMER, LISTS_TIMER, REGEX_TIMER, ARP_TIMER, LAST_TIMER };
 enum { QUERIES, FORWARDED, CLIENTS, DOMAINS, OVERTIME, WILDCARD };
 enum { DNSSEC_UNSPECIFIED, DNSSEC_SECURE, DNSSEC_INSECURE, DNSSEC_BOGUS, DNSSEC_ABANDONED, DNSSEC_UNKNOWN };
 enum { QUERY_UNKNOWN, QUERY_GRAVITY, QUERY_FORWARDED, QUERY_CACHE, QUERY_WILDCARD, QUERY_BLACKLIST, QUERY_EXTERNAL_BLOCKED };
@@ -213,7 +213,7 @@ typedef struct {
 } whitelistStruct;
 
 // Prepare timers, used mainly for debugging purposes
-#define NUMTIMERS 5
+#define NUMTIMERS LAST_TIMER
 
 // Used to check memory integrity in various structs
 #define MAGICBYTE 0x57
diff --git a/database.c b/database.c
index fe1dc64ac..350cb3b76 100644
--- a/database.c
+++ b/database.c
@@ -656,6 +656,9 @@ void *DB_thread(void *val)
 				delete_old_queries_in_DB();
 				DBdeleteoldqueries = false;
 			}
+
+			// Parse ARP cache (fill network table)
+			parse_arp_cache();
 		}
 		sleepms(100);
 	}
diff --git a/networktable.c b/networktable.c
index 1ef169161..dd24963ad 100644
--- a/networktable.c
+++ b/networktable.c
@@ -9,6 +9,7 @@
 *  Please see LICENSE file for your rights under this license. */
 
 #include "FTL.h"
+#include "shmem.h"
 #define ARPCACHE "/proc/net/arp"
 
 bool create_network_table(void)
@@ -33,7 +34,7 @@ bool create_network_table(void)
 }
 
 // Read kernel's ARP cache using procfs
-void read_arp_cache(void)
+void parse_arp_cache(void)
 {
 	FILE* arpfp = NULL;
 	// Try to access the kernel's ARP cache
@@ -51,6 +52,9 @@ void read_arp_cache(void)
 		return;
 	}
 
+	// Start ARP timer
+	if(debug) timer_start(ARP_TIMER);
+
 	// Prepare buffers
 	char * linebuffer = NULL;
 	size_t linebuffersize = 0;
@@ -86,52 +90,60 @@ void read_arp_cache(void)
 		// is known to pihole-FTL
 		// false = do not create a new record if the client is
 		//         unknown (only DNS requesting clients do this)
+		lock_shm();
 		int clientID = findClientID(ip, false);
+		unlock_shm();
 		bool clientKnown = clientID >= 0;
 
+		char *hostname = NULL;
+		if(clientKnown)
+			hostname = getstr(clients[clientID].namepos);
+
 		if(dbID == -1)
 		{
 			// Device not in database, add new entry
 			dbquery("INSERT INTO network "\
-			        "(ip,hwaddr,interface,firstSeen,lastSeen,usesPihole) "\
+			        "(ip,hwaddr,interface,firstSeen,lastSeen,usesPihole,name) "\
 			        "VALUES "\
-			        "(\"%s\",\"%s\",\"%s\",%lu,%lu,%s);",\
+			        "(\"%s\",\"%s\",\"%s\",%lu, %lu, %s, \"%s\");",\
 			        ip, hwaddr, iface, now, now,
-			        clientKnown ? "true" : "false");
+			        clientKnown ? "true" : "false",
+			        hostname == NULL ? "" : hostname);
 		}
 		else
 		{
-			// Device already in database, update lastSeen
+			// Start collecting database commands
+			dbquery("BEGIN TRANSACTION");
+
+			// Update lastSeen
 			dbquery("UPDATE network "\
 			        "SET lastSeen = %lu "\
 			        "WHERE id = %i;",\
 			        now, dbID);
 
 			// Store if device uses Pi-hole
-			if(clientKnown)
+			dbquery("UPDATE network "\
+			        "SET usesPihole = %s "\
+			        "WHERE id = %i;",\
+			        clientKnown ? "true" : "false", dbID);
+
+			// Store hostname if available
+			if(hostname != NULL && strlen(hostname) > 0)
 			{
-				// Device uses Pi-hole, update record
+				// Store host name
 				dbquery("UPDATE network "\
-				        "SET usesPihole = true "\
+				        "SET name = \"%s\" "\
 				        "WHERE id = %i;",\
-				        dbID);
+				        hostname, dbID);
 			}
-		}
 
-		char *hostname = NULL;
-		if(clientKnown)
-			hostname = getstr(clients[clientID].namepos);
-		if(hostname != NULL && strlen(hostname) > 0)
-		{
-			// Store host name
-			dbquery("UPDATE network "\
-			        "SET name = \"%s\" "\
-			        "WHERE id = %i;",\
-			        hostname, dbID);
+			// Actually update the database
+			dbquery("COMMIT");
 		}
-
 	}
 
+	if(debug) logg("ARP table processing took %.1ems",timer_elapsed_msec(ARP_TIMER));
+
 	// Close file handle
 	fclose(arpfp);
 
diff --git a/request.c b/request.c
index 2e313b5ad..f83b7c5ed 100644
--- a/request.c
+++ b/request.c
@@ -168,11 +168,6 @@ void process_request(char *client_message, int *sock)
 		read_regex_from_file();
 		unlock_shm();
 	}
-	else if(command(client_message, ">arp"))
-	{
-		processed = true;
-		read_arp_cache();
-	}
 
 	// Test only at the end if we want to quit or kill
 	// so things can be processed before
diff --git a/routines.h b/routines.h
index 06c29ef98..c7630805e 100644
--- a/routines.h
+++ b/routines.h
@@ -134,4 +134,4 @@ void addOverTimeClientSlot();
 
 // networktable.c
 bool create_network_table(void);
-void read_arp_cache(void);
+void parse_arp_cache(void);

From 95d3c02c5f6066100358d0627575c7fdecd140b7 Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Tue, 25 Dec 2018 22:35:16 +0100
Subject: [PATCH 05/27] Store lastQuery property for known clients

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 FTL.h               |  1 +
 database.c          |  6 +++++-
 datastructure.c     |  2 ++
 dnsmasq_interface.c |  3 +++
 networktable.c      | 36 ++++++++++++++++--------------------
 5 files changed, 27 insertions(+), 21 deletions(-)

diff --git a/FTL.h b/FTL.h
index e2a8f708e..34135449e 100644
--- a/FTL.h
+++ b/FTL.h
@@ -187,6 +187,7 @@ typedef struct {
 	unsigned long long ippos;
 	unsigned long long namepos;
 	bool new;
+	time_t lastQuery;
 } clientsDataStruct;
 
 typedef struct {
diff --git a/database.c b/database.c
index 350cb3b76..2164bd6b4 100644
--- a/database.c
+++ b/database.c
@@ -708,7 +708,7 @@ void read_data_from_DB(void)
 	while((rc = sqlite3_step(stmt)) == SQLITE_ROW)
 	{
 		sqlite3_int64 dbid = sqlite3_column_int64(stmt, 0);
-		int queryTimeStamp = sqlite3_column_int(stmt, 1);
+		time_t queryTimeStamp = sqlite3_column_int(stmt, 1);
 		// 1483228800 = 01/01/2017 @ 12:00am (UTC)
 		if(queryTimeStamp < 1483228800)
 		{
@@ -791,6 +791,7 @@ void read_data_from_DB(void)
 		// Store this query in memory
 		validate_access("overTime", timeidx, true, __LINE__, __FUNCTION__, __FILE__);
 		validate_access("queries", queryIndex, false, __LINE__, __FUNCTION__, __FILE__);
+		validate_access("clients", clientID, true, __LINE__, __FUNCTION__, __FILE__);
 		queries[queryIndex].magic = MAGICBYTE;
 		queries[queryIndex].timestamp = queryTimeStamp;
 		queries[queryIndex].type = type;
@@ -805,6 +806,9 @@ void read_data_from_DB(void)
 		queries[queryIndex].response = 0;
 		queries[queryIndex].AD = false;
 
+		// Update lastQuery of corresponding client
+		clients[clientID].lastQuery = queryTimeStamp;
+
 		// Handle type counters
 		if(type >= TYPE_A && type < TYPE_MAX)
 		{
diff --git a/datastructure.c b/datastructure.c
index 610d4cd9e..4819947ae 100644
--- a/datastructure.c
+++ b/datastructure.c
@@ -222,6 +222,8 @@ int findClientID(const char *client, bool addNew)
 	// to be done separately to be non-blocking
 	clients[clientID].new = true;
 	clients[clientID].namepos = 0;
+	// No query seen so far
+	clients[clientID].lastQuery = 0;
 	// Increase counter by one
 	counters->clients++;
 
diff --git a/dnsmasq_interface.c b/dnsmasq_interface.c
index 45d8bac90..16bd40f97 100644
--- a/dnsmasq_interface.c
+++ b/dnsmasq_interface.c
@@ -179,6 +179,9 @@ void FTL_new_query(unsigned int flags, char *name, struct all_addr *addr, char *
 	// Update overTime data structure with the new client
 	overTimeClientData[clientID][timeidx]++;
 
+	// Set lastQuery timer for network table
+	clients[clientID].lastQuery = querytimestamp;
+
 	// Try blocking regex if configured
 	validate_access("domains", domainID, false, __LINE__, __FUNCTION__, __FILE__);
 	if(domains[domainID].regexmatch == REGEX_UNKNOWN && blockingstatus != BLOCKING_DISABLED)
diff --git a/networktable.c b/networktable.c
index dd24963ad..89eae08dc 100644
--- a/networktable.c
+++ b/networktable.c
@@ -22,8 +22,7 @@ bool create_network_table(void)
 	                                     "interface TEXT NOT NULL, " \
 	                                     "name TEXT, " \
 	                                     "firstSeen INTEGER NOT NULL, " \
-	                                     "lastSeen INTEGER NOT NULL, " \
-	                                     "usesPihole BOOLEAN NOT NULL );");
+	                                     "lastQuery INTEGER NOT NULL);");
 	if(!ret){ dbclose(); return false; }
 
 	// Update database version to 3
@@ -72,9 +71,6 @@ void parse_arp_cache(void)
 		if (num < 4)
 			continue;
 
-		entries++;
-		if(debug) logg("ARP (%i): %i %i %s %s %s <-> %s", num, type, flags, mask, iface, hwaddr, ip);
-
 		// Get ID of this device in our network database. If it cannot be found, then this is a new device
 		char querystr[256];
 		sprintf(querystr, "SELECT id FROM network WHERE ip = \"%s\" AND hwaddr = \"%s\";", ip, hwaddr);
@@ -97,35 +93,34 @@ void parse_arp_cache(void)
 
 		char *hostname = NULL;
 		if(clientKnown)
+		{
+			validate_access("clients", clientID, true, __LINE__, __FUNCTION__, __FILE__);
 			hostname = getstr(clients[clientID].namepos);
+		}
 
 		if(dbID == -1)
 		{
 			// Device not in database, add new entry
 			dbquery("INSERT INTO network "\
-			        "(ip,hwaddr,interface,firstSeen,lastSeen,usesPihole,name) "\
+			        "(ip,hwaddr,interface,firstSeen,lastQuery,name) "\
 			        "VALUES "\
-			        "(\"%s\",\"%s\",\"%s\",%lu, %lu, %s, \"%s\");",\
-			        ip, hwaddr, iface, now, now,
-			        clientKnown ? "true" : "false",
+			        "(\"%s\",\"%s\",\"%s\",%lu, 0, \"%s\");",\
+			        ip, hwaddr, iface, now,
 			        hostname == NULL ? "" : hostname);
 		}
-		else
+		else if(clientKnown)
 		{
 			// Start collecting database commands
 			dbquery("BEGIN TRANSACTION");
 
-			// Update lastSeen
+			// Update lastQuery, only use new value if larger
+			// clients[clientID].lastQuery may be zero if this
+			// client is only known from a database entry but has
+			// not been seen since then
 			dbquery("UPDATE network "\
-			        "SET lastSeen = %lu "\
+			        "SET lastQuery = MAX(lastQuery, %ld) "\
 			        "WHERE id = %i;",\
-			        now, dbID);
-
-			// Store if device uses Pi-hole
-			dbquery("UPDATE network "\
-			        "SET usesPihole = %s "\
-			        "WHERE id = %i;",\
-			        clientKnown ? "true" : "false", dbID);
+			        clients[clientID].lastQuery, dbID);
 
 			// Store hostname if available
 			if(hostname != NULL && strlen(hostname) > 0)
@@ -140,9 +135,10 @@ void parse_arp_cache(void)
 			// Actually update the database
 			dbquery("COMMIT");
 		}
+		entries++;
 	}
 
-	if(debug) logg("ARP table processing took %.1ems",timer_elapsed_msec(ARP_TIMER));
+	if(debug) logg("ARP table processing (%i entries) took %.1f ms", entries, timer_elapsed_msec(ARP_TIMER));
 
 	// Close file handle
 	fclose(arpfp);

From 68a584ad8bf05bfe119aca0b10d9848afc16dbea Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Wed, 26 Dec 2018 01:21:00 +0100
Subject: [PATCH 06/27] Reduce code duplication

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 database.c | 37 +++++++------------------------------
 1 file changed, 7 insertions(+), 30 deletions(-)

diff --git a/database.c b/database.c
index 2164bd6b4..23bd2e460 100644
--- a/database.c
+++ b/database.c
@@ -252,43 +252,20 @@ void db_init(void)
 
 int db_get_FTL_property(unsigned int ID)
 {
-	int rc, ret = 0;
-	sqlite3_stmt* dbstmt;
-	char *querystring = NULL;
-
 	// Prepare SQL statement
-	ret = asprintf(&querystring, "SELECT VALUE FROM ftl WHERE id = %u;",ID);
+	char* querystr = NULL;
+	int ret = asprintf(&querystr, "SELECT VALUE FROM ftl WHERE id = %u;", ID);
 
-	if(querystring == NULL || ret < 0)
+	if(querystr == NULL || ret < 0)
 	{
-		logg("Memory allocation failed in db_get_FTL_property, not saving query with ID = %u (%i)", ID, ret);
+		logg("Memory allocation failed in db_get_FTL_property with ID = %u (%i)", ID, ret);
 		return false;
 	}
 
-	rc = sqlite3_prepare(db, querystring, -1, &dbstmt, NULL);
-	if( rc ){
-		logg("db_get_FTL_property() - SQL error prepare (%i): %s", rc, sqlite3_errmsg(db));
-		logg("Query: \"%s\"", querystring);
-		dbclose();
-		check_database(rc);
-		return -1;
-	}
-	free(querystring);
+	int value = db_query_int(querystr);
+	free(querystr);
 
-	// Evaluate SQL statement
-	rc = sqlite3_step(dbstmt);
-	if( rc != SQLITE_ROW ){
-		logg("db_get_FTL_property() - SQL error step (%i): %s", rc, sqlite3_errmsg(db));
-		dbclose();
-		check_database(rc);
-		return -1;
-	}
-
-	int result = sqlite3_column_int(dbstmt, 0);
-
-	sqlite3_finalize(dbstmt);
-
-	return result;
+	return value;
 }
 
 bool db_set_FTL_property(unsigned int ID, int value)

From 465a4900eca6935559c0b82f12ee827f94ffbe48 Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Wed, 26 Dec 2018 01:21:31 +0100
Subject: [PATCH 07/27] Use one big transaction for changes to be written to
 the network table and skip incomplete entires when parsing the ARP cache

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 networktable.c | 34 ++++++++++++++++++++++++++--------
 1 file changed, 26 insertions(+), 8 deletions(-)

diff --git a/networktable.c b/networktable.c
index 89eae08dc..2c918047f 100644
--- a/networktable.c
+++ b/networktable.c
@@ -61,6 +61,9 @@ void parse_arp_cache(void)
 	int type, flags, entries = 0;
 	time_t now = time(NULL);
 
+	// Start collecting database commands
+	dbquery("BEGIN TRANSACTION");
+
 	// Read ARP cache line by line
 	while(getline(&linebuffer, &linebuffersize, arpfp) != -1)
 	{
@@ -71,10 +74,28 @@ void parse_arp_cache(void)
 		if (num < 4)
 			continue;
 
+		// Skip incomplete entires, i.e., entries without C (complete) flag
+		if(!(flags & 0x02))
+			continue;
+
 		// Get ID of this device in our network database. If it cannot be found, then this is a new device
-		char querystr[256];
-		sprintf(querystr, "SELECT id FROM network WHERE ip = \"%s\" AND hwaddr = \"%s\";", ip, hwaddr);
+		// We match both IP *and* MAC address
+		// Same MAC, two IPs: Non-deterministic DHCP server, treat as two entries
+		// Same IP, two MACs: Either non-deterministic DHCP server or (almost) full DHCP address pool
+		// We can run this SELECT inside the currently active transaction as only the
+		// changed to the database are collected for latter commitment. Read-only access
+		// such as this SELECT command will be executed immediately on the database.
+		char* querystr = NULL;
+		int ret = asprintf(&querystr, "SELECT id FROM network WHERE ip = \"%s\" AND hwaddr = \"%s\";", ip, hwaddr);
+
+		if(querystr == NULL || ret < 0)
+		{
+			logg("Memory allocation failed in parse_arp_cache (%i)", ret);
+			break;
+		}
+
 		int dbID = db_query_int(querystr);
+		free(querystr);
 
 		if(dbID == -2)
 		{
@@ -110,9 +131,6 @@ void parse_arp_cache(void)
 		}
 		else if(clientKnown)
 		{
-			// Start collecting database commands
-			dbquery("BEGIN TRANSACTION");
-
 			// Update lastQuery, only use new value if larger
 			// clients[clientID].lastQuery may be zero if this
 			// client is only known from a database entry but has
@@ -131,13 +149,13 @@ void parse_arp_cache(void)
 				        "WHERE id = %i;",\
 				        hostname, dbID);
 			}
-
-			// Actually update the database
-			dbquery("COMMIT");
 		}
 		entries++;
 	}
 
+	// Actually update the database
+	dbquery("COMMIT");
+
 	if(debug) logg("ARP table processing (%i entries) took %.1f ms", entries, timer_elapsed_msec(ARP_TIMER));
 
 	// Close file handle

From 3a8c87907a6a07989cf88e98b4ef371a21c0fb6c Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Wed, 26 Dec 2018 01:35:50 +0100
Subject: [PATCH 08/27] Initialize new devices with lastQuery property if
 available

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 database.c     | 6 +-----
 networktable.c | 3 ++-
 2 files changed, 3 insertions(+), 6 deletions(-)

diff --git a/database.c b/database.c
index 23bd2e460..e0ccd4daf 100644
--- a/database.c
+++ b/database.c
@@ -259,7 +259,7 @@ int db_get_FTL_property(unsigned int ID)
 	if(querystr == NULL || ret < 0)
 	{
 		logg("Memory allocation failed in db_get_FTL_property with ID = %u (%i)", ID, ret);
-		return false;
+		return -2;
 	}
 
 	int value = db_query_int(querystr);
@@ -289,10 +289,6 @@ bool db_update_counters(int total, int blocked)
 
 int db_query_int(const char* querystr)
 {
-	// Check if database is enabled
-	if(!database)
-		return -2;
-
 	sqlite3_stmt* stmt;
 	int rc = sqlite3_prepare_v2(db, querystr, -1, &stmt, NULL);
 	if( rc ){
diff --git a/networktable.c b/networktable.c
index 2c918047f..e3c0d89a9 100644
--- a/networktable.c
+++ b/networktable.c
@@ -125,8 +125,9 @@ void parse_arp_cache(void)
 			dbquery("INSERT INTO network "\
 			        "(ip,hwaddr,interface,firstSeen,lastQuery,name) "\
 			        "VALUES "\
-			        "(\"%s\",\"%s\",\"%s\",%lu, 0, \"%s\");",\
+			        "(\"%s\",\"%s\",\"%s\",%lu, %lu, \"%s\");",\
 			        ip, hwaddr, iface, now,
+			        clientKnown ? clients[clientID].lastQuery : 0L,
 			        hostname == NULL ? "" : hostname);
 		}
 		else if(clientKnown)

From 54253bd7d59cd7afc07eee51c2a72bcf10eab1c0 Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Wed, 26 Dec 2018 11:25:41 +0100
Subject: [PATCH 09/27] Added more comments

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 networktable.c | 17 +++++++++++++----
 1 file changed, 13 insertions(+), 4 deletions(-)

diff --git a/networktable.c b/networktable.c
index e3c0d89a9..ce8c1df49 100644
--- a/networktable.c
+++ b/networktable.c
@@ -87,13 +87,13 @@ void parse_arp_cache(void)
 		// such as this SELECT command will be executed immediately on the database.
 		char* querystr = NULL;
 		int ret = asprintf(&querystr, "SELECT id FROM network WHERE ip = \"%s\" AND hwaddr = \"%s\";", ip, hwaddr);
-
 		if(querystr == NULL || ret < 0)
 		{
 			logg("Memory allocation failed in parse_arp_cache (%i)", ret);
 			break;
 		}
 
+		// Perform SQL query
 		int dbID = db_query_int(querystr);
 		free(querystr);
 
@@ -110,8 +110,12 @@ void parse_arp_cache(void)
 		lock_shm();
 		int clientID = findClientID(ip, false);
 		unlock_shm();
+
+		// This client is known (by its IP address) to pihole-FTL if
+		// findClientID() returned a non-negative index
 		bool clientKnown = clientID >= 0;
 
+		// Get hostname of this client if the client is known
 		char *hostname = NULL;
 		if(clientKnown)
 		{
@@ -119,17 +123,17 @@ void parse_arp_cache(void)
 			hostname = getstr(clients[clientID].namepos);
 		}
 
+		// Device not in database, add new entry
 		if(dbID == -1)
 		{
-			// Device not in database, add new entry
 			dbquery("INSERT INTO network "\
 			        "(ip,hwaddr,interface,firstSeen,lastQuery,name) "\
-			        "VALUES "\
-			        "(\"%s\",\"%s\",\"%s\",%lu, %lu, \"%s\");",\
+			        "VALUES (\"%s\",\"%s\",\"%s\",%lu, %ld, \"%s\");",\
 			        ip, hwaddr, iface, now,
 			        clientKnown ? clients[clientID].lastQuery : 0L,
 			        hostname == NULL ? "" : hostname);
 		}
+		// Device in database AND client known to Pi-hole
 		else if(clientKnown)
 		{
 			// Update lastQuery, only use new value if larger
@@ -151,12 +155,17 @@ void parse_arp_cache(void)
 				        hostname, dbID);
 			}
 		}
+		// else:
+		// Device in database but not known to Pi-hole: No action required
+
+		// Count number of processed ARP cache entries
 		entries++;
 	}
 
 	// Actually update the database
 	dbquery("COMMIT");
 
+	// Debug logging
 	if(debug) logg("ARP table processing (%i entries) took %.1f ms", entries, timer_elapsed_msec(ARP_TIMER));
 
 	// Close file handle

From ac6ee9e1ef71a982a7e88ba0ae359c87896d6517 Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Wed, 26 Dec 2018 11:29:28 +0100
Subject: [PATCH 10/27] Add config option to disable ARP cache parsing

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 FTL.h      |  1 +
 config.c   | 13 +++++++++++++
 database.c |  5 +++--
 3 files changed, 17 insertions(+), 2 deletions(-)

diff --git a/FTL.h b/FTL.h
index 34135449e..8628ea20a 100644
--- a/FTL.h
+++ b/FTL.h
@@ -149,6 +149,7 @@ typedef struct {
 	bool regex_debugmode;
 	bool analyze_only_A_AAAA;
 	bool DBimport;
+	bool parse_arp_cache;
 } ConfigStruct;
 
 // Dynamic structs
diff --git a/config.c b/config.c
index 2ba5b03b6..fd000f039 100644
--- a/config.c
+++ b/config.c
@@ -319,6 +319,19 @@ void read_FTLconf(void)
 	// AUDITLISTFILE
 	getpath(fp, "AUDITLISTFILE", "/etc/pihole/auditlog.list", &files.auditlist);
 
+	// PARSE_ARP_CACHE
+	// defaults to: Yes
+	config.parse_arp_cache = true;
+	buffer = parse_FTLconf(fp, "PARSE_ARP_CACHE");
+
+	if(buffer != NULL && strcasecmp(buffer, "false") == 0)
+		config.parse_arp_cache = false;
+
+	if(config.parse_arp_cache)
+		logg("   PARSE_ARP_CACHE: Active");
+	else
+		logg("   PARSE_ARP_CACHE: Inactive");
+
 	logg("Finished config file parsing");
 
 	// Release memory
diff --git a/database.c b/database.c
index e0ccd4daf..1aed35b84 100644
--- a/database.c
+++ b/database.c
@@ -630,8 +630,9 @@ void *DB_thread(void *val)
 				DBdeleteoldqueries = false;
 			}
 
-			// Parse ARP cache (fill network table)
-			parse_arp_cache();
+			// Parse ARP cache (fill network table) if enabled
+			if (config.parse_arp_cache)
+				parse_arp_cache();
 		}
 		sleepms(100);
 	}

From d8fdf76cebb1e52c1e35348a0c53039762a828e4 Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Fri, 28 Dec 2018 12:18:35 +0100
Subject: [PATCH 11/27] Store number of queries per client in the database

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 FTL.h               |  1 +
 database.c          |  5 +++--
 datastructure.c     |  1 +
 dnsmasq_interface.c |  3 ++-
 networktable.c      | 18 ++++++++++++++----
 5 files changed, 21 insertions(+), 7 deletions(-)

diff --git a/FTL.h b/FTL.h
index 8628ea20a..015448462 100644
--- a/FTL.h
+++ b/FTL.h
@@ -189,6 +189,7 @@ typedef struct {
 	unsigned long long namepos;
 	bool new;
 	time_t lastQuery;
+	unsigned int numQueriesARP;
 } clientsDataStruct;
 
 typedef struct {
diff --git a/database.c b/database.c
index 1aed35b84..f4815fea8 100644
--- a/database.c
+++ b/database.c
@@ -776,12 +776,13 @@ void read_data_from_DB(void)
 		queries[queryIndex].timeidx = timeidx;
 		queries[queryIndex].db = dbid;
 		queries[queryIndex].id = 0;
-		queries[queryIndex].complete = true; // Mark as all information is avaiable
+		queries[queryIndex].complete = true; // Mark as all information is available
 		queries[queryIndex].response = 0;
 		queries[queryIndex].AD = false;
 
-		// Update lastQuery of corresponding client
+		// Set lastQuery timer and add one query for network table
 		clients[clientID].lastQuery = queryTimeStamp;
+		clients[clientID].numQueriesARP++;
 
 		// Handle type counters
 		if(type >= TYPE_A && type < TYPE_MAX)
diff --git a/datastructure.c b/datastructure.c
index 4819947ae..01fe403fb 100644
--- a/datastructure.c
+++ b/datastructure.c
@@ -224,6 +224,7 @@ int findClientID(const char *client, bool addNew)
 	clients[clientID].namepos = 0;
 	// No query seen so far
 	clients[clientID].lastQuery = 0;
+	clients[clientID].numQueriesARP = 0;
 	// Increase counter by one
 	counters->clients++;
 
diff --git a/dnsmasq_interface.c b/dnsmasq_interface.c
index 16bd40f97..7c83108d0 100644
--- a/dnsmasq_interface.c
+++ b/dnsmasq_interface.c
@@ -179,8 +179,9 @@ void FTL_new_query(unsigned int flags, char *name, struct all_addr *addr, char *
 	// Update overTime data structure with the new client
 	overTimeClientData[clientID][timeidx]++;
 
-	// Set lastQuery timer for network table
+	// Set lastQuery timer and add one query for network table
 	clients[clientID].lastQuery = querytimestamp;
+	clients[clientID].numQueriesARP++;
 
 	// Try blocking regex if configured
 	validate_access("domains", domainID, false, __LINE__, __FUNCTION__, __FILE__);
diff --git a/networktable.c b/networktable.c
index ce8c1df49..6d474d3e0 100644
--- a/networktable.c
+++ b/networktable.c
@@ -22,7 +22,8 @@ bool create_network_table(void)
 	                                     "interface TEXT NOT NULL, " \
 	                                     "name TEXT, " \
 	                                     "firstSeen INTEGER NOT NULL, " \
-	                                     "lastQuery INTEGER NOT NULL);");
+	                                     "lastQuery INTEGER NOT NULL, " \
+	                                     "numQueries INTEGER NOT NULL);");
 	if(!ret){ dbclose(); return false; }
 
 	// Update database version to 3
@@ -127,16 +128,17 @@ void parse_arp_cache(void)
 		if(dbID == -1)
 		{
 			dbquery("INSERT INTO network "\
-			        "(ip,hwaddr,interface,firstSeen,lastQuery,name) "\
-			        "VALUES (\"%s\",\"%s\",\"%s\",%lu, %ld, \"%s\");",\
+			        "(ip,hwaddr,interface,firstSeen,lastQuery,numQueries,name) "\
+			        "VALUES (\"%s\",\"%s\",\"%s\",%lu, %ld, %u, \"%s\");",\
 			        ip, hwaddr, iface, now,
 			        clientKnown ? clients[clientID].lastQuery : 0L,
+			        clientKnown ? clients[clientID].numQueriesARP : 0u,
 			        hostname == NULL ? "" : hostname);
 		}
 		// Device in database AND client known to Pi-hole
 		else if(clientKnown)
 		{
-			// Update lastQuery, only use new value if larger
+			// Update lastQuery. Only use new value if larger
 			// clients[clientID].lastQuery may be zero if this
 			// client is only known from a database entry but has
 			// not been seen since then
@@ -145,6 +147,14 @@ void parse_arp_cache(void)
 			        "WHERE id = %i;",\
 			        clients[clientID].lastQuery, dbID);
 
+			// Update numQueries. Add queries seen since last update
+			// and reset counter afterwards
+			dbquery("UPDATE network "\
+			        "SET numQueries = numQueries + %u "\
+			        "WHERE id = %i;",\
+			        clients[clientID].numQueriesARP, dbID);
+			clients[clientID].numQueriesARP = 0;
+
 			// Store hostname if available
 			if(hostname != NULL && strlen(hostname) > 0)
 			{

From f2f543c8d6b87f8da6f1b7e54693119004ff9352 Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Fri, 28 Dec 2018 20:39:03 +0100
Subject: [PATCH 12/27] Do not count in findClient() if search is triggered
 from parse_arp_cache()

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 datastructure.c | 10 ++++++----
 gc.c            |  1 -
 2 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/datastructure.c b/datastructure.c
index 01fe403fb..721dcf821 100644
--- a/datastructure.c
+++ b/datastructure.c
@@ -176,7 +176,7 @@ int findDomainID(const char *domain)
 	return domainID;
 }
 
-int findClientID(const char *client, bool addNew)
+int findClientID(const char *client, bool count)
 {
 	int i;
 	// Compare content of client against known client IP addresses
@@ -191,14 +191,16 @@ int findClientID(const char *client, bool addNew)
 		// If so, compare the full IP using strcmp
 		if(strcmp(getstr(clients[i].ippos), client) == 0)
 		{
-			clients[i].count++;
+			// Add one if count == true (do not add one, e.g., during ARP table processing)
+			if(count) clients[i].count++;
 			return i;
 		}
 	}
 
-	// Return -1 (= not found) if addNew is false
-	if(!addNew)
+	// Return -1 (= not found) if count is false ...
+	if(!count)
 		return -1;
+	// ... otherwise proceed with adding a new client entry
 
 	// If we did not return until here, then this client is definitely new
 	// Store ID
diff --git a/gc.c b/gc.c
index 813b96aa2..b5d3f9151 100644
--- a/gc.c
+++ b/gc.c
@@ -51,7 +51,6 @@ void *GC_thread(void *val)
 				if(queries[i].timestamp > mintime)
 					break;
 
-
 				// Adjust total counters and total over time data
 				// We cannot edit counters->queries directly as it is used
 				// as max ID for the queries[] struct

From 92d819899eb170031330c361b42da7229ecbddf4 Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Sun, 30 Dec 2018 23:34:58 +0100
Subject: [PATCH 13/27] Add MAC->Vendor database support. We will provide the
 database as an optional auxiliary file.

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 FTL.h          |  1 +
 config.c       |  3 ++
 memory.c       |  1 +
 networktable.c | 84 +++++++++++++++++++++++++++++++++++++++++++++++---
 4 files changed, 85 insertions(+), 4 deletions(-)

diff --git a/FTL.h b/FTL.h
index 015448462..a66ad50eb 100644
--- a/FTL.h
+++ b/FTL.h
@@ -98,6 +98,7 @@ typedef struct {
 	char* port;
 	char* db;
 	char* socketfile;
+	char* macvendordb;
 } FTLFileNamesStruct;
 
 typedef struct {
diff --git a/config.c b/config.c
index fd000f039..3f5f71b25 100644
--- a/config.c
+++ b/config.c
@@ -319,6 +319,9 @@ void read_FTLconf(void)
 	// AUDITLISTFILE
 	getpath(fp, "AUDITLISTFILE", "/etc/pihole/auditlog.list", &files.auditlist);
 
+	// MACVENDORDB
+	getpath(fp, "MACVENDORDB", "/etc/pihole/macvendor.db", &FTLfiles.macvendordb);
+
 	// PARSE_ARP_CACHE
 	// defaults to: Yes
 	config.parse_arp_cache = true;
diff --git a/memory.c b/memory.c
index a959a022d..30e8b2ddb 100644
--- a/memory.c
+++ b/memory.c
@@ -20,6 +20,7 @@ FTLFileNamesStruct FTLfiles = {
 	NULL,
 	NULL,
 	NULL,
+	NULL,
 	NULL
 };
 
diff --git a/networktable.c b/networktable.c
index 6d474d3e0..c29cd7fc6 100644
--- a/networktable.c
+++ b/networktable.c
@@ -12,6 +12,9 @@
 #include "shmem.h"
 #define ARPCACHE "/proc/net/arp"
 
+// Private prototypes
+char* getMACVendor(const char* hwaddr);
+
 bool create_network_table(void)
 {
 	bool ret;
@@ -23,7 +26,8 @@ bool create_network_table(void)
 	                                     "name TEXT, " \
 	                                     "firstSeen INTEGER NOT NULL, " \
 	                                     "lastQuery INTEGER NOT NULL, " \
-	                                     "numQueries INTEGER NOT NULL);");
+	                                     "numQueries INTEGER NOT NULL," \
+	                                     "macVendor TEXT);");
 	if(!ret){ dbclose(); return false; }
 
 	// Update database version to 3
@@ -127,13 +131,17 @@ void parse_arp_cache(void)
 		// Device not in database, add new entry
 		if(dbID == -1)
 		{
+			char* macVendor = getMACVendor(hwaddr);
 			dbquery("INSERT INTO network "\
-			        "(ip,hwaddr,interface,firstSeen,lastQuery,numQueries,name) "\
-			        "VALUES (\"%s\",\"%s\",\"%s\",%lu, %ld, %u, \"%s\");",\
+			        "(ip,hwaddr,interface,firstSeen,lastQuery,numQueries,name,macVendor) "\
+			        "VALUES (\"%s\",\"%s\",\"%s\",%lu, %ld, %u, \"%s\", \"%s\");",\
 			        ip, hwaddr, iface, now,
 			        clientKnown ? clients[clientID].lastQuery : 0L,
 			        clientKnown ? clients[clientID].numQueriesARP : 0u,
-			        hostname == NULL ? "" : hostname);
+			        hostname == NULL ? "" : hostname,
+			        macVendor);
+			if(strlen(macVendor) > 0)
+				free(macVendor);
 		}
 		// Device in database AND client known to Pi-hole
 		else if(clientKnown)
@@ -184,3 +192,71 @@ void parse_arp_cache(void)
 	// Close database connection
 	dbclose();
 }
+
+char* getMACVendor(const char* hwaddr)
+{
+	struct stat st;
+	if(stat(FTLfiles.macvendordb, &st) != 0 || strlen(hwaddr) != 17)
+	{
+		// File does not exist or MAC address is incomplete
+		if(debug) logg("getMACVenor(%s): %s does not exist or MAC invalid (length %lu)", hwaddr, FTLfiles.macvendordb, strlen(hwaddr));
+		return "";
+	}
+
+	sqlite3 *macdb;
+	int rc = sqlite3_open_v2(FTLfiles.macvendordb, &macdb, SQLITE_OPEN_READONLY, NULL);
+	if( rc ){
+		logg("getMACVendor(%s) - SQL error (%i): %s", hwaddr, rc, sqlite3_errmsg(macdb));
+		sqlite3_close(macdb);
+		return "";
+	}
+
+	char *querystr = NULL;
+	// Only keep "XX:YY:ZZ" (8 characters)
+	char * hwaddrshort = strdup(hwaddr);
+	hwaddrshort[8] = '\0';
+	rc = asprintf(&querystr, "SELECT vendor FROM macvendor WHERE mac LIKE \"%s\";", hwaddrshort);
+	if(rc < 1)
+	{
+		logg("getMACVendor(%s) - Allocation error (%i)", hwaddr, rc);
+		sqlite3_close(macdb);
+		return "";
+	}
+	free(hwaddrshort);
+
+	sqlite3_stmt* stmt;
+	rc = sqlite3_prepare_v2(macdb, querystr, -1, &stmt, NULL);
+	if( rc ){
+		logg("getMACVendor(%s) - SQL error prepare (%s, %i): %s", hwaddr, querystr, rc, sqlite3_errmsg(macdb));
+		sqlite3_close(macdb);
+		return "";
+	}
+	free(querystr);
+
+	char *vendor = NULL;
+	rc = sqlite3_step(stmt);
+	if(rc == SQLITE_ROW)
+	{
+		const unsigned char *result = sqlite3_column_text(stmt, 0);
+		// Need to use sprintf(%s) to convert unsigned char* to
+		// standard C string literals (which are char*)
+		if(asprintf(&vendor, "%s", result) < 1)
+			logg("getMACVendor(%s) - Allocation error 2");
+	}
+	else if(rc == SQLITE_DONE)
+	{
+		// Not found
+		vendor = "";
+	}
+	else
+	{
+		// Error
+		logg("getMACVendor(%s) - SQL error step (%i): %s", hwaddr, rc, sqlite3_errmsg(macdb));
+		vendor = "";
+	}
+
+	sqlite3_finalize(stmt);
+	sqlite3_close(macdb);
+
+	return vendor;
+}

From a2b7a241e225dcbabead6b01bbc8db2bf7b67dca Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Mon, 31 Dec 2018 00:18:23 +0100
Subject: [PATCH 14/27] Add routine to update all existing network table
 entries using the latest mac->vendor database

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 networktable.c | 77 +++++++++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 76 insertions(+), 1 deletion(-)

diff --git a/networktable.c b/networktable.c
index c29cd7fc6..e7a03c25c 100644
--- a/networktable.c
+++ b/networktable.c
@@ -241,7 +241,7 @@ char* getMACVendor(const char* hwaddr)
 		// Need to use sprintf(%s) to convert unsigned char* to
 		// standard C string literals (which are char*)
 		if(asprintf(&vendor, "%s", result) < 1)
-			logg("getMACVendor(%s) - Allocation error 2");
+			logg("getMACVendor(%s) - Allocation error 2", hwaddr);
 	}
 	else if(rc == SQLITE_DONE)
 	{
@@ -260,3 +260,78 @@ char* getMACVendor(const char* hwaddr)
 
 	return vendor;
 }
+
+void updateMACVendorRecords()
+{
+	struct stat st;
+	if(stat(FTLfiles.macvendordb, &st) != 0)
+	{
+		// File does not exist or MAC address is incomplete
+		if(debug) logg("updateMACVendorRecords(): %s does not exist", FTLfiles.macvendordb);
+		return;
+	}
+
+	sqlite3 *db;
+	int rc = sqlite3_open_v2(FTLfiles.db, &db, SQLITE_OPEN_READWRITE, NULL);
+	if( rc ){
+		logg("updateMACVendorRecords() - SQL error (%i): %s", rc, sqlite3_errmsg(db));
+		sqlite3_close(db);
+		return;
+	}
+
+	sqlite3_stmt* stmt;
+	const char* querystr = "SELECT id,hwaddr FROM network;";
+	rc = sqlite3_prepare_v2(db, querystr, -1, &stmt, NULL);
+	if( rc ){
+		logg("updateMACVendorRecords() - SQL error prepare (%s, %i): %s", querystr, rc, sqlite3_errmsg(db));
+		sqlite3_close(db);
+		return;
+	}
+
+	while((rc = sqlite3_step(stmt)) == SQLITE_ROW)
+	{
+		const int id = sqlite3_column_int(stmt, 0);
+		const unsigned char *hwaddr = sqlite3_column_text(stmt, 1);
+		// Need to use sprintf(%s) to convert unsigned char* to
+		// standard C string literals (which are char*)
+		char *querystr = NULL;
+		if(asprintf(&querystr, "%s", hwaddr) < 1)
+		{
+			logg("updateMACVendorRecords() - Allocation error 1");
+			break;
+		}
+
+		// Get vendor for MAC
+		char* vendor = getMACVendor(querystr);
+		free(querystr);
+
+		// Prepare UPDATE statement
+		if(asprintf(&querystr, "UPDATE network SET macVendor = \"%s\" WHERE id = %i", vendor, id) < 1)
+		{
+			logg("updateMACVendorRecords() - Allocation error 2");
+			break;
+		}
+
+		// Execute prepared statement
+		char *zErrMsg = NULL;
+		rc = sqlite3_exec(db, querystr, NULL, NULL, &zErrMsg);
+		if( rc != SQLITE_OK ){
+			logg("updateMACVendorRecords() - SQL exec error: %s (%i): %s", querystr, rc, zErrMsg);
+			sqlite3_free(zErrMsg);
+			break;
+		}
+
+		// Free allocated memory
+		free(querystr);
+		if(strlen(vendor) > 0)
+			free(vendor);
+	}
+	if(rc != SQLITE_DONE)
+	{
+		// Error
+		logg("updateMACVendorRecords() - SQL error step (%i): %s", rc, sqlite3_errmsg(db));
+	}
+
+	sqlite3_finalize(stmt);
+	sqlite3_close(db);
+}

From bd749f4a4c83114e77edf0afc0a7fdb721e49a7a Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Mon, 31 Dec 2018 10:33:33 +0100
Subject: [PATCH 15/27] Added python3 script to automatically generate the
 macvendor database used by FTL

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 aux/macvendor.py | 69 ++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 69 insertions(+)
 create mode 100644 aux/macvendor.py

diff --git a/aux/macvendor.py b/aux/macvendor.py
new file mode 100644
index 000000000..41cb97dc4
--- /dev/null
+++ b/aux/macvendor.py
@@ -0,0 +1,69 @@
+# Pi-hole: A black hole for Internet advertisements
+# (c) 2019 Pi-hole, LLC (https://pi-hole.net)
+# Network-wide ad blocking via your own hardware.
+#
+# FTL Engine - auxiliary files
+# MAC -> Vendor database generator
+#
+# This is a python3 script
+#
+# This file is copyright under the latest version of the EUPL.
+# Please see LICENSE file for your rights under this license.
+
+import os, re, urllib.request, sqlite3
+import unicodecsv as unicodecsv
+
+# Download raw data from Wireshark's website
+# We use the official URL recommended in the header of this file
+print("Downloading...")
+urllib.request.urlretrieve("https://code.wireshark.org/review/gitweb?p=wireshark.git;a=blob_plain;f=manuf", "manuf.data")
+print("...done")
+
+# Read file into memory and process lines
+file = open("manuf.data", "r")
+data = []
+print("Processing...")
+for line in file:
+	line = line.strip()
+
+	# Skip comments and empty lines
+	if line[:1] == "#" or line == "":
+		continue
+	# \s = Unicode whitespace characters, including [ \t\n\r\f\v]
+	cols = re.split("\s\s+|\t", line)
+	# Use try/except chain to catch empty/incomplete lines without failing hard
+	try:
+		# Strip whitespace and quotation marks (some entries are incomplete and cause errors with the CSV parser otherwise)
+		mac = cols[0].strip().strip("\"")
+	except:
+		continue
+	try:
+		desc_short = cols[1].strip().strip("\"")
+	except:
+		desc_short = ""
+	try:
+		desc_long = cols[2].strip().strip("\"")
+	except:
+		desc_long = ""
+
+	# Only add long description where available
+	# There are a few vendors for which only the
+	# short description field is used
+	if(len(desc_long) > 0):
+		data.append([mac, desc_long])
+	else:
+		data.append([mac, desc_short])
+print("...done")
+file.close()
+
+# Create database
+database = "macvendor.db"
+os.remove(database)
+print("Generating database...")
+con = sqlite3.connect(database)
+cur = con.cursor()
+cur.execute("CREATE TABLE macvendor (mac TEXT NOT NULL, vendor TEXT NOT NULL, PRIMARY KEY (mac))")
+cur.executemany("INSERT INTO macvendor (mac, vendor) VALUES (?, ?);", data)
+con.commit()
+print("...done.")
+print("Lines inserted into database:", cur.rowcount)

From 85e954f4338cc9d8171849467b8343c97fb76f6f Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Mon, 31 Dec 2018 11:48:05 +0100
Subject: [PATCH 16/27] Optimize database after creation (reduces filesize by
 about 10%) + don't fail if there was no previous database present

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 aux/macvendor.py | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/aux/macvendor.py b/aux/macvendor.py
index 41cb97dc4..a36bc46fb 100644
--- a/aux/macvendor.py
+++ b/aux/macvendor.py
@@ -58,7 +58,13 @@
 
 # Create database
 database = "macvendor.db"
-os.remove(database)
+
+# Try to delete old database file, pass if no old file exists
+try:
+	os.remove(database)
+except OSError:
+	pass
+
 print("Generating database...")
 con = sqlite3.connect(database)
 cur = con.cursor()
@@ -66,4 +72,7 @@
 cur.executemany("INSERT INTO macvendor (mac, vendor) VALUES (?, ?);", data)
 con.commit()
 print("...done.")
+print("Optimizing database...")
+con.execute("VACUUM")
+print("...done")
 print("Lines inserted into database:", cur.rowcount)

From 8ad22b283d81c3844d695d415173f5f37a4849c6 Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Tue, 1 Jan 2019 10:36:45 +0100
Subject: [PATCH 17/27] Mac vendor database generator: Remove quotation marks
 as these might interfere with later INSERT / UPDATE commands

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 aux/macvendor.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/aux/macvendor.py b/aux/macvendor.py
index a36bc46fb..ce2e640b1 100644
--- a/aux/macvendor.py
+++ b/aux/macvendor.py
@@ -29,6 +29,9 @@
 	# Skip comments and empty lines
 	if line[:1] == "#" or line == "":
 		continue
+
+	# Remove quotation marks as these might interfere with later INSERT / UPDATE commands
+	line = re.sub("\'|\"","", line)
 	# \s = Unicode whitespace characters, including [ \t\n\r\f\v]
 	cols = re.split("\s\s+|\t", line)
 	# Use try/except chain to catch empty/incomplete lines without failing hard

From 262fd8e440ee4fa008dd21a631bd75bd2e9713a9 Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Sun, 6 Jan 2019 20:36:20 +0100
Subject: [PATCH 18/27] Improve python aux script

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 aux/macvendor.py | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/aux/macvendor.py b/aux/macvendor.py
index ce2e640b1..b255247d8 100644
--- a/aux/macvendor.py
+++ b/aux/macvendor.py
@@ -10,7 +10,10 @@
 # This file is copyright under the latest version of the EUPL.
 # Please see LICENSE file for your rights under this license.
 
-import os, re, urllib.request, sqlite3
+import os
+import re
+import urllib.request
+import sqlite3
 import unicodecsv as unicodecsv
 
 # Download raw data from Wireshark's website
@@ -20,10 +23,10 @@
 print("...done")
 
 # Read file into memory and process lines
-file = open("manuf.data", "r")
+manuf = open("manuf.data", "r")
 data = []
 print("Processing...")
-for line in file:
+for line in manuf:
 	line = line.strip()
 
 	# Skip comments and empty lines
@@ -52,12 +55,12 @@
 	# Only add long description where available
 	# There are a few vendors for which only the
 	# short description field is used
-	if(len(desc_long) > 0):
+	if(desc_long):
 		data.append([mac, desc_long])
 	else:
 		data.append([mac, desc_short])
 print("...done")
-file.close()
+manuf.close()
 
 # Create database
 database = "macvendor.db"

From 0f6c52036fde30d9c2e5a0e50da65a7b8cad2c1e Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Mon, 7 Jan 2019 01:57:49 +0100
Subject: [PATCH 19/27] Remove obsolete dependency

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 aux/macvendor.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/aux/macvendor.py b/aux/macvendor.py
index b255247d8..12fd3fd18 100644
--- a/aux/macvendor.py
+++ b/aux/macvendor.py
@@ -14,7 +14,6 @@
 import re
 import urllib.request
 import sqlite3
-import unicodecsv as unicodecsv
 
 # Download raw data from Wireshark's website
 # We use the official URL recommended in the header of this file

From 0742d4c21a3f98c6e5c08fa0f12638e1f98d179e Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Mon, 7 Jan 2019 01:58:09 +0100
Subject: [PATCH 20/27] Use simple cast of sqlite3_column_text() instead of
 using asprintf()

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 networktable.c | 22 ++++++----------------
 1 file changed, 6 insertions(+), 16 deletions(-)

diff --git a/networktable.c b/networktable.c
index e7a03c25c..1c0de87b9 100644
--- a/networktable.c
+++ b/networktable.c
@@ -237,11 +237,7 @@ char* getMACVendor(const char* hwaddr)
 	rc = sqlite3_step(stmt);
 	if(rc == SQLITE_ROW)
 	{
-		const unsigned char *result = sqlite3_column_text(stmt, 0);
-		// Need to use sprintf(%s) to convert unsigned char* to
-		// standard C string literals (which are char*)
-		if(asprintf(&vendor, "%s", result) < 1)
-			logg("getMACVendor(%s) - Allocation error 2", hwaddr);
+		vendor = strdup((char*)sqlite3_column_text(stmt, 0));
 	}
 	else if(rc == SQLITE_DONE)
 	{
@@ -291,21 +287,15 @@ void updateMACVendorRecords()
 	while((rc = sqlite3_step(stmt)) == SQLITE_ROW)
 	{
 		const int id = sqlite3_column_int(stmt, 0);
-		const unsigned char *hwaddr = sqlite3_column_text(stmt, 1);
-		// Need to use sprintf(%s) to convert unsigned char* to
-		// standard C string literals (which are char*)
-		char *querystr = NULL;
-		if(asprintf(&querystr, "%s", hwaddr) < 1)
-		{
-			logg("updateMACVendorRecords() - Allocation error 1");
-			break;
-		}
+		char* hwaddr = strdup((char*)sqlite3_column_text(stmt, 1));
 
 		// Get vendor for MAC
-		char* vendor = getMACVendor(querystr);
-		free(querystr);
+		char* vendor = getMACVendor(hwaddr);
+		free(hwaddr);
+		hwaddr = NULL;
 
 		// Prepare UPDATE statement
+		char *querystr = NULL;
 		if(asprintf(&querystr, "UPDATE network SET macVendor = \"%s\" WHERE id = %i", vendor, id) < 1)
 		{
 			logg("updateMACVendorRecords() - Allocation error 2");

From 078b2ff26d901ce950c6a84fccaba43711fe9a5d Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Mon, 7 Jan 2019 02:08:52 +0100
Subject: [PATCH 21/27] Free memory when breaking

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 networktable.c | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/networktable.c b/networktable.c
index 1c0de87b9..62f186da3 100644
--- a/networktable.c
+++ b/networktable.c
@@ -262,7 +262,7 @@ void updateMACVendorRecords()
 	struct stat st;
 	if(stat(FTLfiles.macvendordb, &st) != 0)
 	{
-		// File does not exist or MAC address is incomplete
+		// File does not exist
 		if(debug) logg("updateMACVendorRecords(): %s does not exist", FTLfiles.macvendordb);
 		return;
 	}
@@ -299,6 +299,10 @@ void updateMACVendorRecords()
 		if(asprintf(&querystr, "UPDATE network SET macVendor = \"%s\" WHERE id = %i", vendor, id) < 1)
 		{
 			logg("updateMACVendorRecords() - Allocation error 2");
+
+			if(strlen(vendor) > 0)
+				free(vendor);
+
 			break;
 		}
 
@@ -308,6 +312,11 @@ void updateMACVendorRecords()
 		if( rc != SQLITE_OK ){
 			logg("updateMACVendorRecords() - SQL exec error: %s (%i): %s", querystr, rc, zErrMsg);
 			sqlite3_free(zErrMsg);
+
+			free(querystr);
+			if(strlen(vendor) > 0)
+				free(vendor);
+
 			break;
 		}
 

From d01d660bc588137bfb0e5749eba5f51700239ef4 Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Mon, 7 Jan 2019 02:09:10 +0100
Subject: [PATCH 22/27] Separate error messages into two if-statements

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 networktable.c | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/networktable.c b/networktable.c
index 62f186da3..e1ad6c4dc 100644
--- a/networktable.c
+++ b/networktable.c
@@ -196,10 +196,16 @@ void parse_arp_cache(void)
 char* getMACVendor(const char* hwaddr)
 {
 	struct stat st;
-	if(stat(FTLfiles.macvendordb, &st) != 0 || strlen(hwaddr) != 17)
+	if(stat(FTLfiles.macvendordb, &st) != 0)
+	{
+		// File does not exist
+		if(debug) logg("getMACVenor(%s): %s does not exist", hwaddr, FTLfiles.macvendordb);
+		return "";
+	}
+	else if(strlen(hwaddr) != 17)
 	{
-		// File does not exist or MAC address is incomplete
-		if(debug) logg("getMACVenor(%s): %s does not exist or MAC invalid (length %lu)", hwaddr, FTLfiles.macvendordb, strlen(hwaddr));
+		// MAC address is incomplete
+		if(debug) logg("getMACVenor(%s): MAC invalid (length %lu)", hwaddr, strlen(hwaddr));
 		return "";
 	}
 

From 9e6696575c75bbbd31cfed892cef6b0bed9306f0 Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Mon, 7 Jan 2019 02:10:06 +0100
Subject: [PATCH 23/27] Change code to ensure hostname is always set

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 networktable.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/networktable.c b/networktable.c
index e1ad6c4dc..8b5ad3683 100644
--- a/networktable.c
+++ b/networktable.c
@@ -121,7 +121,7 @@ void parse_arp_cache(void)
 		bool clientKnown = clientID >= 0;
 
 		// Get hostname of this client if the client is known
-		char *hostname = NULL;
+		char *hostname = "";
 		if(clientKnown)
 		{
 			validate_access("clients", clientID, true, __LINE__, __FUNCTION__, __FILE__);
@@ -138,7 +138,7 @@ void parse_arp_cache(void)
 			        ip, hwaddr, iface, now,
 			        clientKnown ? clients[clientID].lastQuery : 0L,
 			        clientKnown ? clients[clientID].numQueriesARP : 0u,
-			        hostname == NULL ? "" : hostname,
+			        hostname,
 			        macVendor);
 			if(strlen(macVendor) > 0)
 				free(macVendor);

From 486e497d8a634ee780cea5516fa0b5e56513941b Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Mon, 7 Jan 2019 02:17:22 +0100
Subject: [PATCH 24/27] Review comments

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 FTL.h          |  4 ++++
 config.c       |  2 +-
 database.c     | 20 ++++++++++----------
 networktable.c |  7 ++++---
 4 files changed, 19 insertions(+), 14 deletions(-)

diff --git a/FTL.h b/FTL.h
index a66ad50eb..a4a1095c8 100644
--- a/FTL.h
+++ b/FTL.h
@@ -222,6 +222,10 @@ typedef struct {
 // Used to check memory integrity in various structs
 #define MAGICBYTE 0x57
 
+// Some magic database constants constants
+#define DB_FAILED -2
+#define DB_NODATA -1
+
 extern logFileNamesStruct files;
 extern FTLFileNamesStruct FTLfiles;
 extern countersStruct *counters;
diff --git a/config.c b/config.c
index 3f5f71b25..95bf42fbc 100644
--- a/config.c
+++ b/config.c
@@ -323,7 +323,7 @@ void read_FTLconf(void)
 	getpath(fp, "MACVENDORDB", "/etc/pihole/macvendor.db", &FTLfiles.macvendordb);
 
 	// PARSE_ARP_CACHE
-	// defaults to: Yes
+	// defaults to: true
 	config.parse_arp_cache = true;
 	buffer = parse_FTLconf(fp, "PARSE_ARP_CACHE");
 
diff --git a/database.c b/database.c
index f4815fea8..8de6983c6 100644
--- a/database.c
+++ b/database.c
@@ -259,7 +259,7 @@ int db_get_FTL_property(unsigned int ID)
 	if(querystr == NULL || ret < 0)
 	{
 		logg("Memory allocation failed in db_get_FTL_property with ID = %u (%i)", ID, ret);
-		return -2;
+		return DB_FAILED;
 	}
 
 	int value = db_query_int(querystr);
@@ -295,7 +295,7 @@ int db_query_int(const char* querystr)
 		logg("db_query_int(%s) - SQL error prepare (%i): %s", querystr, rc, sqlite3_errmsg(db));
 		dbclose();
 		check_database(rc);
-		return -2;
+		return DB_FAILED;
 	}
 
 	rc = sqlite3_step(stmt);
@@ -308,14 +308,14 @@ int db_query_int(const char* querystr)
 	else if( rc == SQLITE_DONE )
 	{
 		// No rows available
-		result = -1;
+		result = DB_NODATA;
 	}
 	else
 	{
 		logg("db_query_int(%s) - SQL error step (%i): %s", querystr, rc, sqlite3_errmsg(db));
 		dbclose();
 		check_database(rc);
-		return -2;
+		return DB_FAILED;
 	}
 
 	sqlite3_finalize(stmt);
@@ -333,7 +333,7 @@ int number_of_queries_in_DB(void)
 		logg("number_of_queries_in_DB() - SQL error prepare (%i): %s", rc, sqlite3_errmsg(db));
 		dbclose();
 		check_database(rc);
-		return -1;
+		return DB_FAILED;
 	}
 
 	rc = sqlite3_step(stmt);
@@ -341,7 +341,7 @@ int number_of_queries_in_DB(void)
 		logg("number_of_queries_in_DB() - SQL error step (%i): %s", rc, sqlite3_errmsg(db));
 		dbclose();
 		check_database(rc);
-		return -1;
+		return DB_FAILED;
 	}
 
 	int result = sqlite3_column_int(stmt, 0);
@@ -360,7 +360,7 @@ static sqlite3_int64 last_ID_in_DB(void)
 		logg("last_ID_in_DB() - SQL error prepare (%i): %s", rc, sqlite3_errmsg(db));
 		dbclose();
 		check_database(rc);
-		return -1;
+		return DB_FAILED;
 	}
 
 	rc = sqlite3_step(stmt);
@@ -368,7 +368,7 @@ static sqlite3_int64 last_ID_in_DB(void)
 		logg("last_ID_in_DB() - SQL error step (%i): %s", rc, sqlite3_errmsg(db));
 		dbclose();
 		check_database(rc);
-		return -1;
+		return DB_FAILED;
 	}
 
 	sqlite3_int64 result = sqlite3_column_int64(stmt, 0);
@@ -380,12 +380,12 @@ static sqlite3_int64 last_ID_in_DB(void)
 
 int get_number_of_queries_in_DB(void)
 {
-	int result = -1;
+	int result = DB_NODATA;
 
 	if(!dbopen())
 	{
 		logg("Failed to open DB in get_number_of_queries_in_DB()");
-		return -2;
+		return DB_FAILED;
 	}
 
 	result = number_of_queries_in_DB();
diff --git a/networktable.c b/networktable.c
index 8b5ad3683..e8385ee19 100644
--- a/networktable.c
+++ b/networktable.c
@@ -18,7 +18,7 @@ char* getMACVendor(const char* hwaddr);
 bool create_network_table(void)
 {
 	bool ret;
-	// Create FTL table in the database (holds properties like database version, etc.)
+	// Create network table in the database
 	ret = dbquery("CREATE TABLE network ( id INTEGER PRIMARY KEY NOT NULL, " \
 	                                     "ip TEXT NOT NULL, " \
 	                                     "hwaddr TEXT NOT NULL, " \
@@ -53,6 +53,7 @@ void parse_arp_cache(void)
 	if(!dbopen())
 	{
 		logg("read_arp_cache() - Failed to open DB");
+		fclose(arpfp);
 		return;
 	}
 
@@ -102,7 +103,7 @@ void parse_arp_cache(void)
 		int dbID = db_query_int(querystr);
 		free(querystr);
 
-		if(dbID == -2)
+		if(dbID == DB_FAILED)
 		{
 			// SQLite error
 			break;
@@ -129,7 +130,7 @@ void parse_arp_cache(void)
 		}
 
 		// Device not in database, add new entry
-		if(dbID == -1)
+		if(dbID == DB_NODATA)
 		{
 			char* macVendor = getMACVendor(hwaddr);
 			dbquery("INSERT INTO network "\

From 427f8ad8e169c0f545a30cf064133fad14298b7f Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Mon, 7 Jan 2019 18:46:57 +0100
Subject: [PATCH 25/27] Further review comments. Always allocate vendor so we
 can always free this pointer. Add a callback for the update subroutine.

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 FTL.h            |  2 +-
 aux/macvendor.py |  2 +-
 networktable.c   | 25 +++++++++----------------
 request.c        |  6 ++++++
 routines.h       |  1 +
 5 files changed, 18 insertions(+), 18 deletions(-)

diff --git a/FTL.h b/FTL.h
index a4a1095c8..eda21af63 100644
--- a/FTL.h
+++ b/FTL.h
@@ -222,7 +222,7 @@ typedef struct {
 // Used to check memory integrity in various structs
 #define MAGICBYTE 0x57
 
-// Some magic database constants constants
+// Some magic database constants
 #define DB_FAILED -2
 #define DB_NODATA -1
 
diff --git a/aux/macvendor.py b/aux/macvendor.py
index 12fd3fd18..344dab3e7 100644
--- a/aux/macvendor.py
+++ b/aux/macvendor.py
@@ -29,7 +29,7 @@
 	line = line.strip()
 
 	# Skip comments and empty lines
-	if line[:1] == "#" or line == "":
+	if line[1] == "#" or line == "":
 		continue
 
 	# Remove quotation marks as these might interfere with later INSERT / UPDATE commands
diff --git a/networktable.c b/networktable.c
index e8385ee19..738e3cce7 100644
--- a/networktable.c
+++ b/networktable.c
@@ -13,7 +13,7 @@
 #define ARPCACHE "/proc/net/arp"
 
 // Private prototypes
-char* getMACVendor(const char* hwaddr);
+static char* getMACVendor(const char* hwaddr);
 
 bool create_network_table(void)
 {
@@ -194,7 +194,7 @@ void parse_arp_cache(void)
 	dbclose();
 }
 
-char* getMACVendor(const char* hwaddr)
+static char* getMACVendor(const char* hwaddr)
 {
 	struct stat st;
 	if(stat(FTLfiles.macvendordb, &st) != 0)
@@ -246,16 +246,16 @@ char* getMACVendor(const char* hwaddr)
 	{
 		vendor = strdup((char*)sqlite3_column_text(stmt, 0));
 	}
-	else if(rc == SQLITE_DONE)
+	else
 	{
 		// Not found
-		vendor = "";
+		vendor = strdup("");
 	}
-	else
+
+	if(rc != SQLITE_DONE && rc != SQLITE_ROW)
 	{
 		// Error
 		logg("getMACVendor(%s) - SQL error step (%i): %s", hwaddr, rc, sqlite3_errmsg(macdb));
-		vendor = "";
 	}
 
 	sqlite3_finalize(stmt);
@@ -306,10 +306,7 @@ void updateMACVendorRecords()
 		if(asprintf(&querystr, "UPDATE network SET macVendor = \"%s\" WHERE id = %i", vendor, id) < 1)
 		{
 			logg("updateMACVendorRecords() - Allocation error 2");
-
-			if(strlen(vendor) > 0)
-				free(vendor);
-
+			free(vendor);
 			break;
 		}
 
@@ -319,18 +316,14 @@ void updateMACVendorRecords()
 		if( rc != SQLITE_OK ){
 			logg("updateMACVendorRecords() - SQL exec error: %s (%i): %s", querystr, rc, zErrMsg);
 			sqlite3_free(zErrMsg);
-
 			free(querystr);
-			if(strlen(vendor) > 0)
-				free(vendor);
-
+			free(vendor);
 			break;
 		}
 
 		// Free allocated memory
 		free(querystr);
-		if(strlen(vendor) > 0)
-			free(vendor);
+		free(vendor);
 	}
 	if(rc != SQLITE_DONE)
 	{
diff --git a/request.c b/request.c
index f83b7c5ed..0245cbb97 100644
--- a/request.c
+++ b/request.c
@@ -168,6 +168,12 @@ void process_request(char *client_message, int *sock)
 		read_regex_from_file();
 		unlock_shm();
 	}
+	else if(command(client_message, ">update-mac-vendor"))
+	{
+		processed = true;
+		logg("Received API request to update vendors in network table");
+		updateMACVendorRecords();
+	}
 
 	// Test only at the end if we want to quit or kill
 	// so things can be processed before
diff --git a/routines.h b/routines.h
index f293c70fa..109b92c75 100644
--- a/routines.h
+++ b/routines.h
@@ -138,3 +138,4 @@ bool check_capabilities(void);
 // networktable.c
 bool create_network_table(void);
 void parse_arp_cache(void);
+void updateMACVendorRecords(void);

From 054dfce2037140b6b3ddc7888f7b9af6d9f4f288 Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Mon, 7 Jan 2019 19:22:49 +0100
Subject: [PATCH 26/27] Check for hostname != NULL is obsolete

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 networktable.c | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/networktable.c b/networktable.c
index 738e3cce7..bb5a785e8 100644
--- a/networktable.c
+++ b/networktable.c
@@ -141,8 +141,7 @@ void parse_arp_cache(void)
 			        clientKnown ? clients[clientID].numQueriesARP : 0u,
 			        hostname,
 			        macVendor);
-			if(strlen(macVendor) > 0)
-				free(macVendor);
+			free(macVendor);
 		}
 		// Device in database AND client known to Pi-hole
 		else if(clientKnown)
@@ -165,7 +164,7 @@ void parse_arp_cache(void)
 			clients[clientID].numQueriesARP = 0;
 
 			// Store hostname if available
-			if(hostname != NULL && strlen(hostname) > 0)
+			if(strlen(hostname) > 0)
 			{
 				// Store host name
 				dbquery("UPDATE network "\

From 787cf113007fd1797a8c03e442d59d59feb4cd67 Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Tue, 8 Jan 2019 21:07:32 +0100
Subject: [PATCH 27/27] Add generated aux files to .gitignore

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 .gitignore | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/.gitignore b/.gitignore
index a952a2bb2..005afc238 100644
--- a/.gitignore
+++ b/.gitignore
@@ -19,3 +19,7 @@ version*
 /pihole-FTL.conf
 /pihole-FTL.db
 /pihole-FTL.log
+
+# aux files
+aux/manuf.data
+aux/macvendor.db