diff --git a/src/database/common.c b/src/database/common.c index f9e1917d5..f35cde5ab 100644 --- a/src/database/common.c +++ b/src/database/common.c @@ -422,6 +422,11 @@ int db_query_int(const char* querystr) return DB_FAILED; } + if(config.debug & DEBUG_DATABASE) + { + logg("dbquery: \"%s\"", querystr); + } + sqlite3_stmt* stmt; int rc = sqlite3_prepare_v2(FTL_db, querystr, -1, &stmt, NULL); if( rc != SQLITE_OK ){ diff --git a/src/database/network-table.c b/src/database/network-table.c index 03bdb5720..8bfc35c84 100644 --- a/src/database/network-table.c +++ b/src/database/network-table.c @@ -9,8 +9,8 @@ * Please see LICENSE file for your rights under this license. */ #include "FTL.h" -#include "network-table.h" -#include "common.h" +#include "database/network-table.h" +#include "database/common.h" #include "shmem.h" #include "memory.h" #include "log.h" @@ -31,7 +31,7 @@ bool create_network_table(void) "name TEXT, " \ "firstSeen INTEGER NOT NULL, " \ "lastQuery INTEGER NOT NULL, " \ - "numQueries INTEGER NOT NULL," \ + "numQueries INTEGER NOT NULL, " \ "macVendor TEXT);"); // Update database version to 3 @@ -106,6 +106,130 @@ bool create_network_addresses_table(void) return true; } +// Try to find device by recent usage of this IP address +static int find_device_by_recent_ip(const char *ipaddr) +{ + char* querystr = NULL; + int ret = asprintf(&querystr, + "SELECT network_id FROM network_addresses " + "WHERE ip = \'%s\' AND " + "lastSeen > (cast(strftime('%%s', 'now') as int)-86400) " + "ORDER BY lastSeen DESC LIMIT 1;", + ipaddr); + if(querystr == NULL || ret < 0) + { + logg("Memory allocation failed in find_device_by_recent_ip(\"%s\"): %i", + ipaddr, ret); + return -1; + } + + // Perform SQL query + int network_id = db_query_int(querystr); + free(querystr); + + if(network_id == DB_FAILED) + { + // SQLite error + return -1; + } + else if(network_id == DB_NODATA) + { + // No result found + return -1; + } + + if(config.debug & DEBUG_ARP) + logg("APR: Identified device %s using most recently used IP address", ipaddr); + + // Found network_id + return network_id; +} + +// Try to find device by mock hardware address (generated from IP address) +static int find_device_by_mock_hwaddr(const char *ipaddr) +{ + char* querystr = NULL; + int ret = asprintf(&querystr, "SELECT id FROM network WHERE hwaddr = \'ip-%s\';", ipaddr); + if(querystr == NULL || ret < 0) + { + logg("Memory allocation failed in find_device_by_mock_hwaddr(\"%s\"): %i", + ipaddr, ret); + return -1; + } + + // Perform SQL query + int network_id = db_query_int(querystr); + free(querystr); + + return network_id; +} + +// Try to find device by RECENT mock hardware address (generated from IP address) +static int find_recent_device_by_mock_hwaddr(const char *ipaddr) +{ + char* querystr = NULL; + int ret = asprintf(&querystr, + "SELECT id FROM network WHERE " + "hwaddr = \'ip-%s\' AND " + "firstSeen > (cast(strftime('%%s', 'now') as int)-3600);", + ipaddr); + if(querystr == NULL || ret < 0) + { + logg("Memory allocation failed in find_device_by_recent_mock_hwaddr(\"%s\"): %i", + ipaddr, ret); + return -1; + } + + // Perform SQL query + int network_id = db_query_int(querystr); + free(querystr); + + return network_id; +} + +// Store hostname of device identified by dbID if neither NULL nor empty +static void update_hostname(const int dbID, const char *hostname) +{ + if(hostname == NULL || strlen(hostname) < 1) + return; + + sqlite3_stmt *query_stmt; + const char *querystr = "UPDATE network SET name = ? WHERE id = ?;"; + + int rc = sqlite3_prepare_v2(FTL_db, querystr, -1, &query_stmt, NULL); + if(rc != SQLITE_OK){ + logg("update_hostname(%i, %s) - SQL error prepare (%i): %s", + dbID, hostname, rc, sqlite3_errmsg(FTL_db)); + return; + } + if(config.debug & DEBUG_DATABASE) + { + logg("dbquery: \"%s\" with arguments 1 = \"%s\" and 2 = %i", querystr, hostname, dbID); + } + + // Bind hostname to prepared statement + // SQLITE_STATIC: Use the string without first duplicating it internally. + // We can do this as hostname has dynamic scope that exceeds that of the binding. + if((rc = sqlite3_bind_text(query_stmt, 1, hostname, -1, SQLITE_STATIC)) != SQLITE_OK) + { + logg("update_hostname(%i, %s): Failed to bind hostname (error %d) - %s", + dbID, hostname, rc, sqlite3_errmsg(FTL_db)); + sqlite3_reset(query_stmt); + return; + } + if((rc = sqlite3_bind_int(query_stmt, 2, dbID)) != SQLITE_OK) + { + logg("update_hostname(%i, %s): Failed to bind dbID (error %d) - %s", + dbID, hostname, rc, sqlite3_errmsg(FTL_db)); + sqlite3_reset(query_stmt); + return; + } + + // Perform step + sqlite3_step(query_stmt); + sqlite3_finalize(query_stmt); +} + // Parse kernel's neighbor cache void parse_neighbor_cache(void) { @@ -119,22 +243,23 @@ void parse_neighbor_cache(void) // Try to access the kernel's neighbor cache // We are only interested in entries which are in either STALE or REACHABLE state FILE* arpfp = NULL; - if((arpfp = popen("ip neigh show nud stale nud reachable", "r")) == NULL) + const char *neigh_command = "ip neigh show"; + if((arpfp = popen(neigh_command, "r")) == NULL) { - logg("WARN: Command \"ip neigh show nud stale nud reachable\" failed!"); + logg("WARN: Command \"%s\" failed!", neigh_command); logg(" Message: %s", strerror(errno)); dbclose(); return; } // Start ARP timer - if(config.debug & DEBUG_ARP) timer_start(ARP_TIMER); + if(config.debug & DEBUG_ARP) + timer_start(ARP_TIMER); // Prepare buffers char * linebuffer = NULL; - size_t linebuffersize = 0; - char ip[100], hwaddr[100], iface[100]; - unsigned int entries = 0; + size_t linebuffersize = 0u; + unsigned int entries = 0u, additional_entries = 0u; time_t now = time(NULL); // Start collecting database commands @@ -152,15 +277,38 @@ void parse_neighbor_cache(void) return; } + // Used to remember the status of a client already seen in the neigh cache + enum arp_status { CLIENT_NOT_HANDLED, CLIENT_ARP_COMPLETE, CLIENT_ARP_INCOMPLETE }; + // Initialize status + enum arp_status client_status[counters->clients]; + for(int i = 0; i < counters->clients; i++) + { + client_status[i] = CLIENT_NOT_HANDLED; + } + // Read ARP cache line by line while(getline(&linebuffer, &linebuffersize, arpfp) != -1) { + char ip[100], hwaddr[100], iface[100]; int num = sscanf(linebuffer, "%99s dev %99s lladdr %99s", ip, iface, hwaddr); // Check if we want to process the line we just read if(num != 3) + { + if(num == 2) + { + // This line is incomplete, remember this to skip + // mock-device creation after ARP processing + int clientID = findClientID(ip, false); + if(clientID >= 0) + client_status[clientID] = CLIENT_ARP_INCOMPLETE; + } + + // Skip to the next row in the neigh cache rather when + // marking as incomplete client continue; + } // Get ID of this device in our network database. If it cannot be // found, then this is a new device. We only use the hardware address @@ -177,7 +325,7 @@ void parse_neighbor_cache(void) ret = asprintf(&querystr, "SELECT id FROM network WHERE hwaddr = \'%s\';", hwaddr); if(querystr == NULL || ret < 0) { - logg("Memory allocation failed in parse_arp_cache(): %i", ret); + logg("Memory allocation failed in parse_arp_cache() [1]: %i", ret); break; } @@ -206,6 +354,7 @@ void parse_neighbor_cache(void) // findClientID() returned a non-negative index if(clientID >= 0) { + client_status[clientID] = CLIENT_ARP_COMPLETE; client = getClient(clientID, true); hostname = getstr(client->namepos); } @@ -213,22 +362,176 @@ void parse_neighbor_cache(void) // Device not in database, add new entry if(dbID == DB_NODATA) { + // Check if we recently added a mock-device with the same IP address + // and the ARP entry just came a bit delayed (reported by at least one user) + dbID = find_recent_device_by_mock_hwaddr(ip); + char* macVendor = getMACVendor(hwaddr); + if(dbID == DB_NODATA) + { + // Device not known AND no recent mock-device found ---> create new device record + if(config.debug & DEBUG_ARP) + { + logg("Device with IP %s not known and " + "no recent mock-device found ---> creating new record", ip); + } + + // Create new record (INSERT) + dbquery("INSERT INTO network " + "(hwaddr,interface,firstSeen,lastQuery,numQueries,name,macVendor) " + "VALUES (\'%s\',\'%s\',%lu, %ld, %u, \'%s\', \'%s\');", + hwaddr, iface, now, + client != NULL ? client->lastQuery : 0L, + client != NULL ? client->numQueriesARP : 0u, + hostname, + macVendor); + + // Reset client ARP counter (we stored the entry in the database) + if(client != NULL) + { + client->numQueriesARP = 0; + } + + // Obtain ID which was given to this new entry + dbID = get_lastID(); + } + else + { + // Device is ALREADY KNOWN ---> convert mock-device to a "real" one + if(config.debug & DEBUG_ARP) + { + logg("Device with IP %s already known (mock-device) " + "---> converting mock-record to real record", ip); + } + + // Update important device properties + dbquery("UPDATE network SET " + "hwaddr = '%s', " + "interface = '%s', " + "macVendor = '%s' " + "WHERE id = %i;", + hwaddr, iface, macVendor, dbID); + // Host name, count and last query timestamp will be set in the next + // loop interation for the sake of simplicity + } + + // Free allocated mememory + free(macVendor); + } + // Device in database AND client known to Pi-hole + else if(client != NULL) + { + // Update lastQuery. Only use new value if larger + // client->lastQuery may be zero if this + // client is only known from a database entry but has + // not been seen since then + if(client->lastQuery > 0) + { + dbquery("UPDATE network "\ + "SET lastQuery = MAX(lastQuery, %ld) "\ + "WHERE id = %i;", + client->lastQuery, dbID); + } + + // Update numQueries. Add queries seen since last update + // and reset counter afterwards + dbquery("UPDATE network "\ + "SET numQueries = numQueries + %u "\ + "WHERE id = %i;", + client->numQueriesARP, dbID); + client->numQueriesARP = 0; + + // Update hostname if available + update_hostname(dbID, hostname); + } + // else: + // Device in database but not known to Pi-hole: No action required + + // Add unique pair of ID (corresponds to one particular hardware + // address) and IP address if it does not exist (INSERT). In case + // this pair already exists, replace it + dbquery("INSERT OR REPLACE INTO network_addresses "\ + "(network_id,ip,lastSeen) VALUES(%i,\'%s\',(cast(strftime('%%s', 'now') as int)));", + dbID, ip); + + // Count number of processed ARP cache entries + entries++; + } + + // Close file handle + pclose(arpfp); + + // Finally, loop over all clients known to FTL and ensure we add them + // all to the database + for(int clientID = 0; clientID < counters->clients; clientID++) + { + + // Get client pointer + clientsData* client = getClient(clientID, true); + if(client == NULL) + { + if(config.debug & DEBUG_ARP) + logg("Network table: Client %d returned NULL pointer", clientID); + continue; + } + + // Get hostname and IP address of this client + const char *hostname, *ipaddr; + ipaddr = getstr(client->ippos); + hostname = getstr(client->namepos); + + // Skip if this client was inactive (last query may be older than 24 hours) + // This also reduces database I/O when nothing would change anyways + if(client->count < 1 || client->numQueriesARP < 1) + { + if(config.debug & DEBUG_ARP) + logg("Network table: Client %s has zero new queries (count: %d, ARPcount: %d)", + ipaddr, client->count, client->numQueriesARP); + continue; + } + // Skip if already handled above + else if(client_status[clientID] != CLIENT_NOT_HANDLED) + { + if(config.debug & DEBUG_ARP) + logg("Network table: Client %s known through ARP/neigh cache", + ipaddr); + continue; + } + else if(config.debug & DEBUG_ARP) + { + logg("Network table: %s NOT known through ARP/neigh cache", ipaddr); + } + + // + // Variant 1: Try to find a device using the same IP address within the last 24 hours + // + int dbID = find_device_by_recent_ip(ipaddr); + + // + // Variant 2: Try to find a device with mock IP address + // + if(dbID < 0) + dbID = find_device_by_mock_hwaddr(ipaddr); + + if(dbID == DB_FAILED) + { + // SQLite error + break; + } + // Device not in database, add new entry + else if(dbID == DB_NODATA) + { dbquery("INSERT INTO network "\ "(hwaddr,interface,firstSeen,lastQuery,numQueries,name,macVendor) "\ - "VALUES (\'%s\',\'%s\',%lu, %ld, %u, \'%s\', \'%s\');",\ - hwaddr, iface, now, - client != NULL ? client->lastQuery : 0L, - client != NULL ? client->numQueriesARP : 0u, - hostname, - macVendor); - free(macVendor); + "VALUES (\'ip-%s\',\'N/A\',%lu, %ld, %u, \'%s\', \'\');",\ + ipaddr, now, client->lastQuery, client->numQueriesARP, hostname); + client->numQueriesARP = 0; // Obtain ID which was given to this new entry dbID = get_lastID(); } - // Device in database AND client known to Pi-hole - else if(client != NULL) + // Device already in database + else { // Update lastQuery. Only use new value if larger // client->lastQuery may be zero if this @@ -247,49 +550,35 @@ void parse_neighbor_cache(void) client->numQueriesARP, dbID); client->numQueriesARP = 0; - // Store hostname if available - if(strlen(hostname) > 0) - { - // Store host name - dbquery("UPDATE network "\ - "SET name = \'%s\' "\ - "WHERE id = %i;",\ - hostname, dbID); - } + // Update host name + update_hostname(dbID, hostname); } - // else: - // Device in database but not known to Pi-hole: No action required - // Add unique pair of ID (corresponds to one particular hardware - // address) and IP address if it does not exist (INSERT). In case - // this pair already exists, the UNIQUE(network_id,ip) trigger - // becomes active and the line is instead REPLACEd, causing the - // lastQuery timestamp to be updated + // Add/replace IP/mock-MAC pair to address database dbquery("INSERT OR REPLACE INTO network_addresses "\ - "(network_id,ip) VALUES(%i,\'%s\');", dbID, ip); + "(network_id,ip,lastSeen) VALUES(%i,\'%s\',(cast(strftime('%%s', 'now') as int)));", + dbID, ipaddr); - // Count number of processed ARP cache entries - entries++; + // Add to number of processed ARP cache entries + additional_entries++; } // Actually update the database - if(dbquery("COMMIT") != SQLITE_OK) { + if(dbquery("COMMIT") != SQLITE_OK) logg("ERROR: parse_neighbor_cache() failed!"); - unlock_shm(); - return; - } + + // Close database connection + // We opened the connection in this function + dbclose(); unlock_shm(); // Debug logging if(config.debug & DEBUG_ARP) - logg("ARP table processing (%i entries) took %.1f ms", entries, timer_elapsed_msec(ARP_TIMER)); - - // Close file handle - pclose(arpfp); - - // Close database connection - dbclose(); + { + logg("ARP table processing (%i entries from ARP, %i from FTL's cache) took %.1f ms", + entries, additional_entries, timer_elapsed_msec(ARP_TIMER)); + } } // Loop over all entries in network table and unify entries by their hwaddr @@ -374,9 +663,9 @@ static char* getMACVendor(const char* hwaddr) logg("getMACVenor(%s): %s does not exist", hwaddr, FTLfiles.macvendor_db); return strdup(""); } - else if(strlen(hwaddr) != 17) + else if(strlen(hwaddr) != 17 || strstr(hwaddr, "ip-") != NULL) { - // MAC address is incomplete + // MAC address is incomplete or mock address (for distant clients) if(config.debug & DEBUG_ARP) logg("getMACVenor(%s): MAC invalid (length %zu)", hwaddr, strlen(hwaddr)); return strdup(""); diff --git a/src/database/query-table.c b/src/database/query-table.c index 32a0de5e8..83eebf419 100644 --- a/src/database/query-table.c +++ b/src/database/query-table.c @@ -376,10 +376,9 @@ void DB_read_queries(void) query->reply = REPLY_UNKNOWN; query->CNAME_domainID = -1; - // Set lastQuery timer and add one query for network table + // Set lastQuery timer for network table clientsData* client = getClient(clientID, true); client->lastQuery = queryTimeStamp; - client->numQueriesARP++; // Handle type counters if(type >= TYPE_A && type < TYPE_MAX) diff --git a/src/datastructure.c b/src/datastructure.c index 168f4c5ef..58b2ab4cc 100644 --- a/src/datastructure.c +++ b/src/datastructure.c @@ -159,10 +159,7 @@ int findClientID(const char *clientIP, const bool count) } } - // Return -1 (= not found) if count is false ... - if(!count) - return -1; - // ... otherwise proceed with adding a new client entry + // Proceed with adding a new client entry // If we did not return until here, then this client is definitely new // Store ID @@ -182,7 +179,7 @@ int findClientID(const char *clientIP, const bool count) // Set magic byte client->magic = MAGICBYTE; // Set its counter to 1 - client->count = 1; + client->count = count ? 1 : 0; // Initialize blocked count to zero client->blockedcount = 0; // Store client IP - no need to check for NULL here as it doesn't harm @@ -195,7 +192,7 @@ int findClientID(const char *clientIP, const bool count) client->namepos = 0; // No query seen so far client->lastQuery = 0; - client->numQueriesARP = 0; + client->numQueriesARP = client->count; // Initialize client-specific overTime data for(int i = 0; i < OVERTIME_SLOTS; i++) diff --git a/src/resolve.c b/src/resolve.c index 4d67b8998..5e40df8ed 100644 --- a/src/resolve.c +++ b/src/resolve.c @@ -23,6 +23,46 @@ // struct _res #include +// Validate given hostname +static bool valid_hostname(char* name, const char* clientip) +{ + // Check for validity of input + if(name == NULL) + return false; + + // Check for maximum length of hostname + // Truncate if too long (MAXHOSTNAMELEN defaults to 64, see asm-generic/param.h) + if(strlen(name) > MAXHOSTNAMELEN) + { + logg("WARN: Hostname of client %s too long, truncating to %d chars!", + clientip, MAXHOSTNAMELEN); + // We can modify the string in-place as the target is + // shorter than the source + name[MAXHOSTNAMELEN] = '\0'; + } + + // Iterate over characters in hostname + // to check for legal char: A-Z a-z 0-9 - _ . + for (char c; (c = *name); name++) + { + if ((c >= 'A' && c <= 'Z') || + (c >= 'a' && c <= 'z') || + (c >= '0' && c <= '9') || + c == '-' || + c == '_' || + c == '.' ) + continue; + + // Invalid character found, log and return hostname being invalid + logg("WARN: Hostname of client %s contains invalid character: %c (char code %d)", + clientip, (unsigned char)c, (unsigned char)c); + return false; + } + + // No invalid characters found + return true; +} + static char *resolveHostname(const char *addr) { // Get host name @@ -30,6 +70,9 @@ static char *resolveHostname(const char *addr) char *hostname = NULL;; bool IPv6 = false; + if(config.debug & DEBUG_API) + logg("Trying to resolve %s", addr); + // Check if this is a hidden client // if so, return "hidden" as hostname if(strcmp(addr, "0.0.0.0") == 0) @@ -65,13 +108,8 @@ static char *resolveHostname(const char *addr) he = gethostbyaddr(&ipaddr, sizeof ipaddr, AF_INET); } - if(he == NULL) - { - // No hostname found - hostname = strdup(""); - //if(hostname == NULL) return NULL; - } - else + // First check for he not being NULL before trying to dereference it + if(he != NULL && valid_hostname(he->h_name, addr)) { // Return hostname copied to new memory location hostname = strdup(he->h_name); @@ -80,6 +118,11 @@ static char *resolveHostname(const char *addr) if(hostname != NULL) strtolower(hostname); } + else + { + // No (he == NULL) or invalid (valid_hostname returned false) hostname found + hostname = strdup(""); + } // Restore ns record in _res _res.nsaddr_list[MAXNS-1].sin_addr = nsbck; diff --git a/src/resolve.h b/src/resolve.h index b1f0361f5..b0e01afd0 100644 --- a/src/resolve.h +++ b/src/resolve.h @@ -14,4 +14,11 @@ void *DNSclient_thread(void *val); void resolveClients(const bool onlynew); void resolveForwardDestinations(const bool onlynew); +// musl does not define MAXHOSTNAMELEN +// If it is not defined, we set the value +// found on a x86_64 glibc instance +#ifndef MAXHOSTNAMELEN +#define MAXHOSTNAMELEN 64 +#endif + #endif //RESOLVE_H