From 3e8849049ee787f2826b2ef34c9556c39417f620 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 1 Aug 2018 17:21:02 +0200 Subject: [PATCH 01/48] Store forward destination IP address and host name in contiguous string buffer Signed-off-by: DL6ER --- FTL.h | 4 ++-- api.c | 10 ++++------ database.c | 2 +- datastructure.c | 6 +++--- main.c | 1 + memory.c | 44 ++++++++++++++++++++++++++++++++++++++++++++ resolve.c | 5 ++++- routines.h | 3 +++ 8 files changed, 62 insertions(+), 13 deletions(-) diff --git a/FTL.h b/FTL.h index c5267aa14..7aa1cac5d 100644 --- a/FTL.h +++ b/FTL.h @@ -174,8 +174,8 @@ typedef struct { unsigned char magic; int count; int failed; - char *ip; - char *name; + unsigned int ippos; + unsigned int namepos; bool new; } forwardedDataStruct; diff --git a/api.c b/api.c index 8a9f946d2..50683ada8 100644 --- a/api.c +++ b/api.c @@ -504,13 +504,11 @@ void getForwardDestinations(char *client_message, int *sock) else j = i; validate_access("forwarded", j, true, __LINE__, __FUNCTION__, __FILE__); - ip = forwarded[j].ip; - // Only return name if available - if(forwarded[j].name != NULL) - name = forwarded[j].name; - else - name = ""; + // Get IP and host name of forward destination if available + ip = getstr(forwarded[j].ippos); + name = getstr(forwarded[j].namepos); + logg("\"%s\" -> \"%s\"",ip,name); // Math explanation: // A single query may result in requests being forwarded to multiple destinations diff --git a/database.c b/database.c index 6700c94d1..7e45203d0 100644 --- a/database.c +++ b/database.c @@ -418,7 +418,7 @@ void save_to_DB(void) if(queries[i].status == QUERY_FORWARDED && queries[i].forwardID > -1) { validate_access("forwarded", queries[i].forwardID, true, __LINE__, __FUNCTION__, __FILE__); - sqlite3_bind_text(stmt, 6, forwarded[queries[i].forwardID].ip, -1, SQLITE_TRANSIENT); + sqlite3_bind_text(stmt, 6, getstr(forwarded[queries[i].forwardID].ippos), -1, SQLITE_TRANSIENT); } else { diff --git a/datastructure.c b/datastructure.c index fd1c1c0cf..80e37c5d0 100644 --- a/datastructure.c +++ b/datastructure.c @@ -88,7 +88,7 @@ int findForwardID(const char * forward, bool count) // Go through already knows forward servers and see if we used one of those for(i=0; i < counters.forwarded; i++) { - if(strcmp(forwarded[i].ip, forward) == 0) + if(strcmp(getstr(forwarded[i].ippos), forward) == 0) { forwardID = i; if(count) forwarded[forwardID].count++; @@ -112,14 +112,14 @@ int findForwardID(const char * forward, bool count) else forwarded[forwardID].count = 0; // Save forward destination IP address - forwarded[forwardID].ip = strdup(forward); + forwarded[forwardID].ippos = addstr(forward); forwarded[forwardID].failed = 0; // Initialize forward hostname // Due to the nature of us being the resolver, // the actual resolving of the host name has // to be done separately to be non-blocking forwarded[forwardID].new = true; - forwarded[forwardID].name = NULL; + forwarded[forwardID].namepos = 0; // 0 -> string with length zero // Increase counter by one counters.forwarded++; diff --git a/main.c b/main.c index 9c9c7a488..9dccbc94a 100644 --- a/main.c +++ b/main.c @@ -35,6 +35,7 @@ int main (int argc, char* argv[]) logg("########## FTL started! ##########"); log_FTL_version(); init_thread_lock(); + initstrbuffer(); // pihole-FTL should really be run as user "pihole" to not mess up with file permissions // print warning otherwise diff --git a/memory.c b/memory.c index b611d3fd8..82a864bab 100644 --- a/memory.c +++ b/memory.c @@ -189,6 +189,50 @@ void validate_access_oTcl(int timeidx, int clientID, int line, const char * func } } +static unsigned int last_pos = 0; +static char* strbuffer = NULL; + +void initstrbuffer(void) +{ + strbuffer = calloc(1, 1); + strbuffer[0] = '\0'; + last_pos = 1; +} + +unsigned int addstr(const char *str) +{ + if(str == NULL) + { + logg("WARN: Called addstr() with NULL pointer"); + return 0; + } + + // Get string length + int len = strlen(str); + + // Reserve memory (will later be replaced for shmem) + strbuffer = realloc(strbuffer, last_pos + len + 1); + + // Copies the C string pointed by str into the array + // pointed by &strbuffer[last_pos], including the + // terminating null character (and stopping at that point) + strncpy(&strbuffer[last_pos], str, len); + strbuffer[last_pos+len+1] = '\0'; + + if(debug) logg("Added string \"%s\" at pos [%u,%u]", str, last_pos, last_pos+len+1); + + // Increment string length counter + last_pos += len+2; + + // Return start of stored string + return (last_pos - (len+2)); +} + +char *getstr(unsigned int pos) +{ + return &strbuffer[pos]; +} + // The special memory handling routines have to be the last ones in this source file // as we restore the original definition of the strdup, free, calloc, and realloc // functions in here, i.e. if anything extra would come below these lines, it would diff --git a/resolve.c b/resolve.c index 9d5a1ff7c..c0e072d69 100644 --- a/resolve.c +++ b/resolve.c @@ -121,7 +121,10 @@ void resolveNewClients(void) // Only try to resolve new forward destinations if(forwarded[i].new) { - forwarded[i].name = resolveHostname(forwarded[i].ip); + char *ipaddr = getstr(forwarded[i].ippos); + char *hostname = resolveHostname(ipaddr); + forwarded[i].namepos = addstr(hostname); + free(hostname); forwarded[i].new = false; } } diff --git a/routines.h b/routines.h index 00040e922..6d6ee20fb 100644 --- a/routines.h +++ b/routines.h @@ -93,6 +93,9 @@ void *FTLrealloc(void *ptr_in, size_t size, const char *file, const char *functi void FTLfree(void *ptr, const char* file, const char *function, int line); void validate_access(const char * name, int pos, bool testmagic, int line, const char * function, const char * file); void validate_access_oTcl(int timeidx, int clientID, int line, const char * function, const char * file); +void initstrbuffer(void); +unsigned int addstr(const char *str); +char *getstr(unsigned int pos); int main_dnsmasq(int argc, char **argv); From b7da12b600ceee64bd2d460c0b63524edee59adb Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 1 Aug 2018 18:37:48 +0200 Subject: [PATCH 02/48] Use strbuffer also for client IPs and host names Signed-off-by: DL6ER --- FTL.h | 4 ++-- api.c | 45 ++++++++++++++++++++------------------------- database.c | 2 +- datastructure.c | 8 ++++---- memory.c | 4 +--- resolve.c | 18 ++++++------------ 6 files changed, 34 insertions(+), 47 deletions(-) diff --git a/FTL.h b/FTL.h index 7aa1cac5d..7f3a6b1d8 100644 --- a/FTL.h +++ b/FTL.h @@ -183,8 +183,8 @@ typedef struct { unsigned char magic; int count; int blockedcount; - char *ip; - char *name; + unsigned int ippos; + unsigned int namepos; bool new; } clientsDataStruct; diff --git a/api.c b/api.c index 50683ada8..5b1901933 100644 --- a/api.c +++ b/api.c @@ -399,19 +399,15 @@ void getTopClients(char *client_message, int *sock) // Skip this client if there is a filter on it if(excludeclients != NULL && - (insetupVarsArray(clients[j].ip) || insetupVarsArray(clients[j].name))) + (insetupVarsArray(getstr(clients[j].ippos)) || insetupVarsArray(getstr(clients[j].namepos)))) continue; // Hidden client, probably due to privacy level. Skip this in the top lists - if(strcmp(clients[j].ip, "0.0.0.0") == 0) + if(strcmp(getstr(clients[j].ippos), "0.0.0.0") == 0) continue; - // Only return name if available - char *name; - if(clients[j].name != NULL) - name = clients[j].name; - else - name = ""; + char *client_ip = getstr(clients[j].ippos); + char *client_name = getstr(clients[j].namepos); // Return this client if either // - "withzero" option is set, and/or @@ -419,10 +415,10 @@ void getTopClients(char *client_message, int *sock) if(includezeroclients || ccount > 0) { if(istelnet[*sock]) - ssend(*sock,"%i %i %s %s\n", n, ccount, clients[j].ip, name); + ssend(*sock,"%i %i %s %s\n", n, ccount, client_ip, client_name); else { - if(!pack_str32(*sock, "") || !pack_str32(*sock, clients[j].ip)) + if(!pack_str32(*sock, "") || !pack_str32(*sock, client_ip)) return; pack_int32(*sock, ccount); @@ -685,19 +681,17 @@ void getAllQueries(char *client_message, int *sock) if(filterclientname) { // Skip if client name and IP are not identical with what the user wants to see - if(strcmp(clients[queries[i].clientID].ip, clientname) != 0 && - (clients[queries[i].clientID].name != NULL && - strcmp(clients[queries[i].clientID].name, clientname) != 0)) + if(strcmp(getstr(clients[queries[i].clientID].ippos), clientname) != 0 && + (strcmp(getstr(clients[queries[i].clientID].namepos), clientname) != 0)) continue; } char *domain = domains[queries[i].domainID].domain; char *client; - if(clients[queries[i].clientID].name != NULL && - strlen(clients[queries[i].clientID].name) > 0) - client = clients[queries[i].clientID].name; + if(strlen(getstr(clients[queries[i].clientID].namepos)) > 0) + client = getstr(clients[queries[i].clientID].namepos); else - client = clients[queries[i].clientID].ip; + client = getstr(clients[queries[i].clientID].ippos); unsigned long delay = queries[i].response; // Check if received (delay should be smaller than 30min) @@ -933,8 +927,8 @@ void getClientsOverTime(int *sock) { validate_access("clients", i, true, __LINE__, __FUNCTION__, __FILE__); // Check if this client should be skipped - if(insetupVarsArray(clients[i].ip) || - insetupVarsArray(clients[i].name)) + if(insetupVarsArray(getstr(clients[i].ippos)) || + insetupVarsArray(getstr(clients[i].namepos))) skipclient[i] = true; } } @@ -1006,8 +1000,8 @@ void getClientNames(int *sock) { validate_access("clients", i, true, __LINE__, __FUNCTION__, __FILE__); // Check if this client should be skipped - if(insetupVarsArray(clients[i].ip) || - insetupVarsArray(clients[i].name)) + if(insetupVarsArray(getstr(clients[i].ippos)) || + insetupVarsArray(getstr(clients[i].namepos))) skipclient[i] = true; } } @@ -1019,13 +1013,14 @@ void getClientNames(int *sock) if(skipclient[i]) continue; - char *client_name = clients[i].name != NULL ? clients[i].name : ""; + char *client_ip = getstr(clients[i].ippos); + char *client_name = getstr(clients[i].namepos); if(istelnet[*sock]) - ssend(*sock, "%s %s\n", client_name, clients[i].ip); + ssend(*sock, "%s %s\n", client_name, client_ip); else { pack_str32(*sock, client_name); - pack_str32(*sock, clients[i].ip); + pack_str32(*sock, client_ip); } } @@ -1060,7 +1055,7 @@ void getUnknownQueries(int *sock) validate_access("clients", queries[i].clientID, true, __LINE__, __FUNCTION__, __FILE__); - char *client = clients[queries[i].clientID].ip; + char *client = getstr(clients[queries[i].clientID].ippos); if(istelnet[*sock]) ssend(*sock, "%i %i %i %s %s %s %i %s\n", queries[i].timestamp, i, queries[i].id, type, domains[queries[i].domainID].domain, client, queries[i].status, queries[i].complete ? "true" : "false"); diff --git a/database.c b/database.c index 7e45203d0..00bd2488f 100644 --- a/database.c +++ b/database.c @@ -412,7 +412,7 @@ void save_to_DB(void) sqlite3_bind_text(stmt, 4, domains[queries[i].domainID].domain, -1, SQLITE_TRANSIENT); // CLIENT - sqlite3_bind_text(stmt, 5, clients[queries[i].clientID].ip, -1, SQLITE_TRANSIENT); + sqlite3_bind_text(stmt, 5, getstr(clients[queries[i].clientID].ippos), -1, SQLITE_TRANSIENT); // FORWARD if(queries[i].status == QUERY_FORWARDED && queries[i].forwardID > -1) diff --git a/datastructure.c b/datastructure.c index 80e37c5d0..4bddc15e0 100644 --- a/datastructure.c +++ b/datastructure.c @@ -178,11 +178,11 @@ int findClientID(const char *client) for(i=0; i < counters.clients; i++) { // Quick test: Does the clients IP start with the same character? - if(clients[i].ip[0] != client[0]) + if(getstr(clients[i].ippos)[0] != client[0]) continue; // If so, compare the full IP using strcmp - if(strcmp(clients[i].ip, client) == 0) + if(strcmp(getstr(clients[i].ippos), client) == 0) { clients[i].count++; return i; @@ -204,13 +204,13 @@ int findClientID(const char *client) // Initialize blocked count to zero clients[clientID].blockedcount = 0; // Store client IP - no need to check for NULL here as it doesn't harm - clients[clientID].ip = strdup(client); + clients[clientID].ippos = addstr(client); // Initialize client hostname // Due to the nature of us being the resolver, // the actual resolving of the host name has // to be done separately to be non-blocking clients[clientID].new = true; - clients[clientID].name = NULL; + clients[clientID].namepos = 0; // Increase counter by one counters.clients++; diff --git a/memory.c b/memory.c index 82a864bab..4aac2d557 100644 --- a/memory.c +++ b/memory.c @@ -217,9 +217,7 @@ unsigned int addstr(const char *str) // pointed by &strbuffer[last_pos], including the // terminating null character (and stopping at that point) strncpy(&strbuffer[last_pos], str, len); - strbuffer[last_pos+len+1] = '\0'; - - if(debug) logg("Added string \"%s\" at pos [%u,%u]", str, last_pos, last_pos+len+1); + strbuffer[last_pos+len] = '\0'; // Increment string length counter last_pos += len+2; diff --git a/resolve.c b/resolve.c index c0e072d69..01b9670b6 100644 --- a/resolve.c +++ b/resolve.c @@ -77,20 +77,14 @@ void reresolveHostnames(void) continue; // Get client hostname - char *hostname = resolveHostname(clients[clientID].ip); - if(strlen(hostname) > 0) + char *oldhostname = getstr(clients[clientID].namepos); + char *newhostname = resolveHostname(getstr(clients[clientID].ippos)); + if(strlen(newhostname) > 0 && strcmp(newhostname,oldhostname) != 0) { - // Delete possibly already existing hostname pointer before storing new data - if(clients[clientID].name != NULL) - { - free(clients[clientID].name); - clients[clientID].name = NULL; - } - // Store client hostname - clients[clientID].name = strdup(hostname); + clients[clientID].namepos = addstr(newhostname); } - free(hostname); + free(newhostname); } } @@ -109,7 +103,7 @@ void resolveNewClients(void) // want to try to resolve them every minute in this case. if(clients[i].new) { - clients[i].name = resolveHostname(clients[i].ip); + clients[i].namepos = addstr(resolveHostname(getstr(clients[i].ippos))); clients[i].new = false; } } From 96928c73ef463e88cec9fc97e67bb733162b26af Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 1 Aug 2018 19:04:02 +0200 Subject: [PATCH 03/48] Use strbuffer also for domains Signed-off-by: DL6ER --- FTL.h | 2 +- api.c | 35 +++++++++++++++++------------------ database.c | 2 +- datastructure.c | 6 +++--- dnsmasq_interface.c | 4 ++-- 5 files changed, 24 insertions(+), 25 deletions(-) diff --git a/FTL.h b/FTL.h index 7f3a6b1d8..952d9cea5 100644 --- a/FTL.h +++ b/FTL.h @@ -192,7 +192,7 @@ typedef struct { unsigned char magic; int count; int blockedcount; - char *domain; + unsigned int domainpos; unsigned char regexmatch; } domainsDataStruct; diff --git a/api.c b/api.c index 5b1901933..3c3cc3b03 100644 --- a/api.c +++ b/api.c @@ -256,15 +256,15 @@ void getTopDomains(char *client_message, int *sock) validate_access("domains", j, true, __LINE__, __FUNCTION__, __FILE__); // Skip this domain if there is a filter on it - if(excludedomains != NULL && insetupVarsArray(domains[j].domain)) + if(excludedomains != NULL && insetupVarsArray(getstr(domains[j].domainpos))) continue; // Skip this domain if already included in audit - if(audit && countlineswith(domains[j].domain, files.auditlist) > 0) + if(audit && countlineswith(getstr(domains[j].domainpos), files.auditlist) > 0) continue; // Hidden domain, probably due to privacy level. Skip this in the top lists - if(strcmp(domains[j].domain, "hidden") == 0) + if(strcmp(getstr(domains[j].domainpos), "hidden") == 0) continue; if(blocked && showblocked && domains[j].blockedcount > 0) @@ -272,11 +272,11 @@ void getTopDomains(char *client_message, int *sock) if(audit && domains[j].regexmatch == REGEX_BLOCKED) { if(istelnet[*sock]) - ssend(*sock, "%i %i %s wildcard\n", n, domains[j].blockedcount, domains[j].domain); + ssend(*sock, "%i %i %s wildcard\n", n, domains[j].blockedcount, getstr(domains[j].domainpos)); else { - char *fancyWildcard = calloc(3 + strlen(domains[j].domain), sizeof(char)); + char *fancyWildcard = calloc(3 + strlen(getstr(domains[j].domainpos)), sizeof(char)); if(fancyWildcard == NULL) return; - sprintf(fancyWildcard, "*.%s", domains[j].domain); + sprintf(fancyWildcard, "*.%s", getstr(domains[j].domainpos)); if(!pack_str32(*sock, fancyWildcard)) return; @@ -288,9 +288,9 @@ void getTopDomains(char *client_message, int *sock) else { if(istelnet[*sock]) - ssend(*sock, "%i %i %s\n", n, domains[j].blockedcount, domains[j].domain); + ssend(*sock, "%i %i %s\n", n, domains[j].blockedcount, getstr(domains[j].domainpos)); else { - if(!pack_str32(*sock, domains[j].domain)) + if(!pack_str32(*sock, getstr(domains[j].domainpos))) return; pack_int32(*sock, domains[j].blockedcount); @@ -301,10 +301,10 @@ void getTopDomains(char *client_message, int *sock) else if(!blocked && showpermitted && (domains[j].count - domains[j].blockedcount) > 0) { if(istelnet[*sock]) - ssend(*sock,"%i %i %s\n",n,(domains[j].count - domains[j].blockedcount),domains[j].domain); + ssend(*sock,"%i %i %s\n",n,(domains[j].count - domains[j].blockedcount),getstr(domains[j].domainpos)); else { - if(!pack_str32(*sock, domains[j].domain)) + if(!pack_str32(*sock, getstr(domains[j].domainpos))) return; pack_int32(*sock, domains[j].count - domains[j].blockedcount); @@ -504,7 +504,6 @@ void getForwardDestinations(char *client_message, int *sock) // Get IP and host name of forward destination if available ip = getstr(forwarded[j].ippos); name = getstr(forwarded[j].namepos); - logg("\"%s\" -> \"%s\"",ip,name); // Math explanation: // A single query may result in requests being forwarded to multiple destinations @@ -674,7 +673,7 @@ void getAllQueries(char *client_message, int *sock) if(filterdomainname) { // Skip if domain name is not identical with what the user wants to see - if(strcmp(domains[queries[i].domainID].domain, domainname) != 0) + if(strcmp(getstr(domains[queries[i].domainID].domainpos), domainname) != 0) continue; } @@ -686,7 +685,7 @@ void getAllQueries(char *client_message, int *sock) continue; } - char *domain = domains[queries[i].domainID].domain; + char *domain = getstr(domains[queries[i].domainID].domainpos); char *client; if(strlen(getstr(clients[queries[i].clientID].namepos)) > 0) client = getstr(clients[queries[i].clientID].namepos); @@ -756,8 +755,8 @@ void getRecentBlocked(char *client_message, int *sock) found++; if(istelnet[*sock]) - ssend(*sock,"%s\n", domains[queries[i].domainID].domain); - else if(!pack_str32(*sock, domains[queries[i].domainID].domain)) + ssend(*sock,"%s\n", getstr(domains[queries[i].domainID].domainpos)); + else if(!pack_str32(*sock, getstr(domains[queries[i].domainID].domainpos))) return; } @@ -1058,7 +1057,7 @@ void getUnknownQueries(int *sock) char *client = getstr(clients[queries[i].clientID].ippos); if(istelnet[*sock]) - ssend(*sock, "%i %i %i %s %s %s %i %s\n", queries[i].timestamp, i, queries[i].id, type, domains[queries[i].domainID].domain, client, queries[i].status, queries[i].complete ? "true" : "false"); + ssend(*sock, "%i %i %i %s %s %s %i %s\n", queries[i].timestamp, i, queries[i].id, type, getstr(domains[queries[i].domainID].domainpos), client, queries[i].status, queries[i].complete ? "true" : "false"); else { pack_int32(*sock, queries[i].timestamp); pack_int32(*sock, queries[i].id); @@ -1068,7 +1067,7 @@ void getUnknownQueries(int *sock) return; // Use str32 for domain and client because we have no idea how long they will be (max is 4294967295 for str32) - if(!pack_str32(*sock, domains[queries[i].domainID].domain) || !pack_str32(*sock, client)) + if(!pack_str32(*sock, getstr(domains[queries[i].domainID].domainpos)) || !pack_str32(*sock, client)) return; pack_uint8(*sock, queries[i].status); @@ -1091,7 +1090,7 @@ void getDomainDetails(char *client_message, int *sock) for(i = 0; i < counters.domains; i++) { validate_access("domains", i, true, __LINE__, __FUNCTION__, __FILE__); - if(strcmp(domains[i].domain, domain) == 0) + if(strcmp(getstr(domains[i].domainpos), domain) == 0) { ssend(*sock,"Domain \"%s\", ID: %i\n", domain, i); ssend(*sock,"Total: %i\n", domains[i].count); diff --git a/database.c b/database.c index 00bd2488f..b2e6c46c6 100644 --- a/database.c +++ b/database.c @@ -409,7 +409,7 @@ void save_to_DB(void) sqlite3_bind_int(stmt, 3, queries[i].status); // DOMAIN - sqlite3_bind_text(stmt, 4, domains[queries[i].domainID].domain, -1, SQLITE_TRANSIENT); + sqlite3_bind_text(stmt, 4, getstr(domains[queries[i].domainID].domainpos), -1, SQLITE_TRANSIENT); // CLIENT sqlite3_bind_text(stmt, 5, getstr(clients[queries[i].clientID].ippos), -1, SQLITE_TRANSIENT); diff --git a/datastructure.c b/datastructure.c index 4bddc15e0..e3d6b0a5d 100644 --- a/datastructure.c +++ b/datastructure.c @@ -134,11 +134,11 @@ int findDomainID(const char *domain) for(i=0; i < counters.domains; i++) { // Quick test: Does the domain start with the same character? - if(domains[i].domain[0] != domain[0]) + if(getstr(domains[i].domainpos)[0] != domain[0]) continue; // If so, compare the full domain using strcmp - if(strcmp(domains[i].domain, domain) == 0) + if(strcmp(getstr(domains[i].domainpos), domain) == 0) { domains[i].count++; return i; @@ -160,7 +160,7 @@ int findDomainID(const char *domain) // Set blocked counter to zero domains[domainID].blockedcount = 0; // Store domain name - no need to check for NULL here as it doesn't harm - domains[domainID].domain = strdup(domain); + domains[domainID].domainpos = addstr(domain); // RegEx needs to be evaluated for this new domain domains[domainID].regexmatch = REGEX_UNKNOWN; // Increase counter by one diff --git a/dnsmasq_interface.c b/dnsmasq_interface.c index 5ec36fede..b706d7ed7 100644 --- a/dnsmasq_interface.c +++ b/dnsmasq_interface.c @@ -473,7 +473,7 @@ void FTL_reply(unsigned short flags, char *name, struct all_addr *addr, int id) int domainID = queries[i].domainID; validate_access("domains", domainID, true, __LINE__, __FUNCTION__, __FILE__); - if(strcmp(domains[domainID].domain, name) == 0) + if(strcmp(getstr(domains[domainID].domainpos), name) == 0) { // Save reply type and update individual reply counters save_reply_type(flags, i, response); @@ -666,7 +666,7 @@ void FTL_dnssec(int status, int id) { int domainID = queries[i].domainID; validate_access("domains", domainID, true, __LINE__, __FUNCTION__, __FILE__); - logg("**** got DNSSEC details for %s: %i (ID %i)", domains[domainID].domain, status, id); + logg("**** got DNSSEC details for %s: %i (ID %i)", getstr(domains[domainID].domainpos), status, id); } // Iterate through possible values From 1a508fc40b4b4942de6d932fe5f65d124a8384db Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 2 Aug 2018 12:08:21 +0200 Subject: [PATCH 04/48] Rename last_pos to next_pos and use struct for string buffer Signed-off-by: DL6ER --- memory.c | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/memory.c b/memory.c index 4aac2d557..ade08de1a 100644 --- a/memory.c +++ b/memory.c @@ -189,14 +189,18 @@ void validate_access_oTcl(int timeidx, int clientID, int line, const char * func } } -static unsigned int last_pos = 0; -static char* strbuffer = NULL; +static struct { + unsigned int size; + char* buffer; +} strbuffer; +static unsigned int next_pos = 0; void initstrbuffer(void) { - strbuffer = calloc(1, 1); - strbuffer[0] = '\0'; - last_pos = 1; + strbuffer.buffer = calloc(1, 1); + strbuffer.size = 1; + strbuffer.buffer[0] = '\0'; + next_pos = 1; } unsigned int addstr(const char *str) @@ -211,24 +215,24 @@ unsigned int addstr(const char *str) int len = strlen(str); // Reserve memory (will later be replaced for shmem) - strbuffer = realloc(strbuffer, last_pos + len + 1); + strbuffer.buffer = realloc(strbuffer.buffer, next_pos + len + 1); // Copies the C string pointed by str into the array // pointed by &strbuffer[last_pos], including the // terminating null character (and stopping at that point) - strncpy(&strbuffer[last_pos], str, len); - strbuffer[last_pos+len] = '\0'; + strncpy(&strbuffer.buffer[next_pos], str, len); + strbuffer.buffer[next_pos+len] = '\0'; // Increment string length counter - last_pos += len+2; + next_pos += len+2; // Return start of stored string - return (last_pos - (len+2)); + return (next_pos - (len+2)); } char *getstr(unsigned int pos) { - return &strbuffer[pos]; + return &strbuffer.buffer[pos]; } // The special memory handling routines have to be the last ones in this source file From d7d851860a78d2b4c8ec5c156618083e51cadffd Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 2 Aug 2018 12:18:01 +0200 Subject: [PATCH 05/48] Store size in string buffer + use long long for string position indices Signed-off-by: DL6ER --- FTL.h | 10 +++++----- memory.c | 1 + 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/FTL.h b/FTL.h index 952d9cea5..2eb449aed 100644 --- a/FTL.h +++ b/FTL.h @@ -174,8 +174,8 @@ typedef struct { unsigned char magic; int count; int failed; - unsigned int ippos; - unsigned int namepos; + long long ippos; + long long namepos; bool new; } forwardedDataStruct; @@ -183,8 +183,8 @@ typedef struct { unsigned char magic; int count; int blockedcount; - unsigned int ippos; - unsigned int namepos; + long long ippos; + long long namepos; bool new; } clientsDataStruct; @@ -192,7 +192,7 @@ typedef struct { unsigned char magic; int count; int blockedcount; - unsigned int domainpos; + long long domainpos; unsigned char regexmatch; } domainsDataStruct; diff --git a/memory.c b/memory.c index ade08de1a..90dd50dc7 100644 --- a/memory.c +++ b/memory.c @@ -216,6 +216,7 @@ unsigned int addstr(const char *str) // Reserve memory (will later be replaced for shmem) strbuffer.buffer = realloc(strbuffer.buffer, next_pos + len + 1); + strbuffer.size = next_pos + len + 1; // Copies the C string pointed by str into the array // pointed by &strbuffer[last_pos], including the From 1b7e08419cb6d1277ab6e9c48e80be2ae03123fd Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 2 Aug 2018 13:23:12 +0200 Subject: [PATCH 06/48] Add shared memory components to store strings in shared memory Signed-off-by: DL6ER --- Makefile | 4 +- main.c | 8 ++- memory.c | 47 ---------------- routines.h | 8 +-- shmem.c | 154 +++++++++++++++++++++++++++++++++++++++++++++++++++++ shmem.h | 43 +++++++++++++++ 6 files changed, 211 insertions(+), 53 deletions(-) create mode 100644 shmem.c create mode 100644 shmem.h diff --git a/Makefile b/Makefile index cd25f0429..c798bf178 100644 --- a/Makefile +++ b/Makefile @@ -14,7 +14,7 @@ DNSMASQOPTS = -DHAVE_DNSSEC -DHAVE_DNSSEC_STATIC # Flags for compiling with libidn2: -DHAVE_LIBIDN2 -DIDN2_VERSION_NUMBER=0x02000003 FTLDEPS = FTL.h routines.h version.h api.h dnsmasq_interface.h -FTLOBJ = main.o memory.o log.o daemon.o datastructure.o signals.o socket.o request.o grep.o setupVars.o args.o threads.o gc.o config.o database.o msgpack.o api.o dnsmasq_interface.o resolve.o regex.o +FTLOBJ = main.o memory.o log.o daemon.o datastructure.o signals.o socket.o request.o grep.o setupVars.o args.o threads.o gc.o config.o database.o msgpack.o api.o dnsmasq_interface.o resolve.o regex.o shmem.o DNSMASQDEPS = config.h dhcp-protocol.h dns-protocol.h radv-protocol.h dhcp6-protocol.h dnsmasq.h ip6addr.h DNSMASQOBJ = arp.o dbus.o domain.o lease.o outpacket.o rrfilter.o auth.o dhcp6.o edns0.o log.o poll.o slaac.o blockdata.o dhcp.o forward.o loop.o radv.o tables.o bpf.o dhcp-common.o helper.o netlink.o rfc1035.o tftp.o cache.o dnsmasq.o inotify.o network.o rfc2131.o util.o conntrack.o dnssec.o ipset.o option.o rfc3315.o crypto.o @@ -51,7 +51,7 @@ CCFLAGS=-std=gnu11 -I$(IDIR) -Wall -Wextra -Wno-unused-parameter -D_FILE_OFFSET_ # for dnsmasq we need the nettle crypto library and the gmp maths library # We link the two libraries statically. Althougth this increases the binary file size by about 1 MB, it saves about 5 MB of shared libraries and makes deployment easier #LIBS=-pthread -lnettle -lgmp -lhogweed -LIBS=-pthread -Wl,-Bstatic -L/usr/local/lib -lhogweed -lgmp -lnettle -Wl,-Bdynamic +LIBS=-pthread -Wl,-Bstatic -L/usr/local/lib -lhogweed -lgmp -lnettle -Wl,-Bdynamic -lrt # Flags for compiling with libidn : -lidn # Flags for compiling with libidn2: -lidn2 diff --git a/main.c b/main.c index 9dccbc94a..baecf420d 100644 --- a/main.c +++ b/main.c @@ -35,7 +35,13 @@ int main (int argc, char* argv[]) logg("########## FTL started! ##########"); log_FTL_version(); init_thread_lock(); - initstrbuffer(); + + // Initialize shared memory + if(!init_shmem()) + { + logg("Initialization of shared memory failed."); + return EXIT_FAILURE; + } // pihole-FTL should really be run as user "pihole" to not mess up with file permissions // print warning otherwise diff --git a/memory.c b/memory.c index 90dd50dc7..b611d3fd8 100644 --- a/memory.c +++ b/memory.c @@ -189,53 +189,6 @@ void validate_access_oTcl(int timeidx, int clientID, int line, const char * func } } -static struct { - unsigned int size; - char* buffer; -} strbuffer; -static unsigned int next_pos = 0; - -void initstrbuffer(void) -{ - strbuffer.buffer = calloc(1, 1); - strbuffer.size = 1; - strbuffer.buffer[0] = '\0'; - next_pos = 1; -} - -unsigned int addstr(const char *str) -{ - if(str == NULL) - { - logg("WARN: Called addstr() with NULL pointer"); - return 0; - } - - // Get string length - int len = strlen(str); - - // Reserve memory (will later be replaced for shmem) - strbuffer.buffer = realloc(strbuffer.buffer, next_pos + len + 1); - strbuffer.size = next_pos + len + 1; - - // Copies the C string pointed by str into the array - // pointed by &strbuffer[last_pos], including the - // terminating null character (and stopping at that point) - strncpy(&strbuffer.buffer[next_pos], str, len); - strbuffer.buffer[next_pos+len] = '\0'; - - // Increment string length counter - next_pos += len+2; - - // Return start of stored string - return (next_pos - (len+2)); -} - -char *getstr(unsigned int pos) -{ - return &strbuffer.buffer[pos]; -} - // The special memory handling routines have to be the last ones in this source file // as we restore the original definition of the strdup, free, calloc, and realloc // functions in here, i.e. if anything extra would come below these lines, it would diff --git a/routines.h b/routines.h index 6d6ee20fb..f1bbb43ba 100644 --- a/routines.h +++ b/routines.h @@ -93,9 +93,6 @@ void *FTLrealloc(void *ptr_in, size_t size, const char *file, const char *functi void FTLfree(void *ptr, const char* file, const char *function, int line); void validate_access(const char * name, int pos, bool testmagic, int line, const char * function, const char * file); void validate_access_oTcl(int timeidx, int clientID, int line, const char * function, const char * file); -void initstrbuffer(void); -unsigned int addstr(const char *str); -char *getstr(unsigned int pos); int main_dnsmasq(int argc, char **argv); @@ -111,3 +108,8 @@ bool match_regex(char *input); void free_regex(void); void read_regex_from_file(void); bool in_whitelist(char *domain); + +// shmem.c +bool init_shmem(void); +unsigned int addstr(const char *str); +char *getstr(unsigned int pos); diff --git a/shmem.c b/shmem.c new file mode 100644 index 000000000..0fbf25152 --- /dev/null +++ b/shmem.c @@ -0,0 +1,154 @@ +/* Pi-hole: A black hole for Internet advertisements +* (c) 2018 Pi-hole, LLC (https://pi-hole.net) +* Network-wide ad blocking via your own hardware. +* +* FTL Engine +* Shared memory subroutines +* +* This file is copyright under the latest version of the EUPL. +* Please see LICENSE file for your rights under this license. */ + +#include "FTL.h" +#include "shmem.h" + +/// The name of the shared memory. Use this when connecting to the shared memory. +#define SHARED_STRINGS_NAME "FTL-strings" + +/// The pointer in shared memory to the shared string buffer +static SharedMemory strBuffer = { 0 }; + +static unsigned int next_pos = 0; + +unsigned int addstr(const char *str) +{ + if(str == NULL) + { + logg("WARN: Called addstr() with NULL pointer"); + return 0; + } + + // Get string length + int len = strlen(str); + + // Reserve memory (will later be replaced for shmem) + if(!realloc_shm(&strBuffer, next_pos + len + 1)) + return 0; + strBuffer.size = next_pos + len + 1; + + // Copy the C string pointed by str into the shared string buffer + char *buffer = strBuffer.ptr + next_pos; + strncpy(buffer, str, len); + buffer[len] = '\0'; + + // Increment string length counter + next_pos += len+2; + + // Return start of stored string + return (next_pos - (len+2)); +} + +char *getstr(unsigned int pos) +{ + return strBuffer.ptr + pos; +} + +bool init_shmem(void) +{ + // Try unlinking the shared memory object before creating a new one + // If the object is still existing, e.g., due to a past unclean exit + // of FTL, shm_open() would fail with error "File exists" + shm_unlink(SHARED_STRINGS_NAME); + + // Try to create shared memory object + strBuffer = create_shm(SHARED_STRINGS_NAME, 1); + if(strBuffer.ptr == NULL) + return false; + + logg("Created shared memory with name \"%s\" (%i)", strBuffer.name, strBuffer.fd); + + // Initialize shared string object with an empty string at position zero + char *buffer = strBuffer.ptr; + buffer[0] = '\0'; + next_pos = 1; + + return true; +} + +SharedMemory create_shm(char *name, size_t size) +{ + if(debug) logg("Creating shared memory with name \"%s\" and size %zu", name, size); + + SharedMemory sharedMemory = { + .fd = 0, + .name = name, + .size = size, + .ptr = NULL + }; + + // Create the shared memory file in read/write mode with 600 permissions + sharedMemory.fd = shm_open(name, O_CREAT | O_EXCL | O_TRUNC | O_RDWR, S_IRUSR | S_IWUSR); + + // Check for `shm_open` error + if(sharedMemory.fd == -1) + { + if(debug) logg("Failed to create_shm shared memory: %s", strerror(errno)); + return sharedMemory; + } + + // Resize shared memory file + int result = ftruncate(sharedMemory.fd, size); + + // Check for `ftruncate` error + if(result == -1) + { + if(debug) logg("Failed to resize shared memory: %s", strerror(errno)); + return sharedMemory; + } + + // Create shared memory mapping + void *shm = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED, sharedMemory.fd, 0); + + // Check for `mmap` error + if(shm == MAP_FAILED) + { + if(debug) logg("Failed to map shared memory: %s", strerror(errno)); + return sharedMemory; + } + + sharedMemory.ptr = shm; + return sharedMemory; +} + +bool realloc_shm(SharedMemory *sharedMemory, size_t size) { + if(debug) logg("Resizing \"%s\" from %zu to %zu", sharedMemory->name, sharedMemory->size, size); + + int result = ftruncate(sharedMemory->fd, size); + + if(result == -1) { + if(debug) logg("Failed to resize \"%s\" (%i): %s", sharedMemory->name, sharedMemory->fd, strerror(errno)); + return false; + } + + void *new_ptr = mremap(sharedMemory->ptr, sharedMemory->size, size, MREMAP_MAYMOVE); + + if(new_ptr == MAP_FAILED) + { + if(debug) logg("Failed to reallocate \"%s\" (%i): %s", sharedMemory->name, sharedMemory->fd, strerror(errno)); + return false; + } + + sharedMemory->ptr = new_ptr; + sharedMemory->size = size; + + return true; +} + +void delete_shm(SharedMemory *sharedMemory) +{ + // Unmap shared memory + munmap(sharedMemory->ptr, sharedMemory->size); + + // Now you can no longer `shm_open` the memory, + // and once all others unlink, it will be destroyed. + shm_unlink(sharedMemory->name); +} diff --git a/shmem.h b/shmem.h new file mode 100644 index 000000000..f6acb88d8 --- /dev/null +++ b/shmem.h @@ -0,0 +1,43 @@ +/* Pi-hole: A black hole for Internet advertisements +* (c) 2018 Pi-hole, LLC (https://pi-hole.net) +* Network-wide ad blocking via your own hardware. +* +* FTL Engine +* Shared memory header +* +* This file is copyright under the latest version of the EUPL. +* Please see LICENSE file for your rights under this license. */ + +#ifndef SHARED_MEMORY_SERVER_H +#define SHARED_MEMORY_SERVER_H +#include /* For shm_* functions */ +#include /* For mode constants */ +#include /* For O_* constants */ + +typedef struct { + int fd; + char *name; + size_t size; + void *ptr; +} SharedMemory; + +/// Create shared memory +/// +/// \param name the name of the shared memory +/// \param size the size to allocate +/// \return a structure with a pointer to the mounted shared memory. The pointer will be NULL if it failed +SharedMemory create_shm(char *name, size_t size); + +/// Reallocate shared memory +/// +/// \param sharedMemory the shared memory struct +/// \param size the new size +/// \return if reallocation was successful +bool realloc_shm(SharedMemory *sharedMemory, size_t size); + +/// Disconnect from shared memory. If there are no other connections to shared memory, it will be deleted. +/// +/// \param sharedMemory the shared memory struct +void delete_shm(SharedMemory *sharedMemory); + +#endif //SHARED_MEMORY_SERVER_H From bb51f4ea31c7780c3fac9f2bfba4440038d84517 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 2 Aug 2018 13:46:01 +0200 Subject: [PATCH 07/48] Improve shmem error outputs Signed-off-by: DL6ER --- shmem.c | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/shmem.c b/shmem.c index 0fbf25152..5653b3dbe 100644 --- a/shmem.c +++ b/shmem.c @@ -64,8 +64,6 @@ bool init_shmem(void) if(strBuffer.ptr == NULL) return false; - logg("Created shared memory with name \"%s\" (%i)", strBuffer.name, strBuffer.fd); - // Initialize shared string object with an empty string at position zero char *buffer = strBuffer.ptr; buffer[0] = '\0'; @@ -91,7 +89,8 @@ SharedMemory create_shm(char *name, size_t size) // Check for `shm_open` error if(sharedMemory.fd == -1) { - if(debug) logg("Failed to create_shm shared memory: %s", strerror(errno)); + logg("create_shm(): Failed to create_shm shared memory object \"%s\": %s", + name, strerror(errno)); return sharedMemory; } @@ -101,7 +100,8 @@ SharedMemory create_shm(char *name, size_t size) // Check for `ftruncate` error if(result == -1) { - if(debug) logg("Failed to resize shared memory: %s", strerror(errno)); + logg("create_shm(): ftruncate(%i, %zu): Failed to resize shared memory object \"%s\": %s", + sharedMemory.fd, size, sharedMemory.name, strerror(errno)); return sharedMemory; } @@ -111,7 +111,8 @@ SharedMemory create_shm(char *name, size_t size) // Check for `mmap` error if(shm == MAP_FAILED) { - if(debug) logg("Failed to map shared memory: %s", strerror(errno)); + logg("create_shm(): Failed to map shared memory object \"%s\" (%i): %s", + sharedMemory.name, sharedMemory.fd, strerror(errno)); return sharedMemory; } @@ -125,7 +126,8 @@ bool realloc_shm(SharedMemory *sharedMemory, size_t size) { int result = ftruncate(sharedMemory->fd, size); if(result == -1) { - if(debug) logg("Failed to resize \"%s\" (%i): %s", sharedMemory->name, sharedMemory->fd, strerror(errno)); + logg("realloc_shm(%i, %zu): ftruncate(%i, %zu): Failed to resize \"%s\": %s", + sharedMemory->fd, size, sharedMemory->name, strerror(errno)); return false; } @@ -133,7 +135,9 @@ bool realloc_shm(SharedMemory *sharedMemory, size_t size) { if(new_ptr == MAP_FAILED) { - if(debug) logg("Failed to reallocate \"%s\" (%i): %s", sharedMemory->name, sharedMemory->fd, strerror(errno)); + logg("realloc_shm(): mremap(%p, %zu, %zu, MREMAP_MAYMOVE): Failed to reallocate \"%s\" (%i): %s", + sharedMemory->ptr, sharedMemory->size, size, sharedMemory->name, sharedMemory->fd, + strerror(errno)); return false; } @@ -146,9 +150,14 @@ bool realloc_shm(SharedMemory *sharedMemory, size_t size) { void delete_shm(SharedMemory *sharedMemory) { // Unmap shared memory - munmap(sharedMemory->ptr, sharedMemory->size); + int ret; + ret = munmap(sharedMemory->ptr, sharedMemory->size); + if(ret != 0) + logg("delete_shm(): munmap(%p, %zu) failed: %s", sharedMemory->ptr, sharedMemory->size, strerror(errno)); // Now you can no longer `shm_open` the memory, // and once all others unlink, it will be destroyed. - shm_unlink(sharedMemory->name); + ret = shm_unlink(sharedMemory->name); + if(ret != 0) + logg("delete_shm(): munmap(%s) failed: %s", sharedMemory->name, strerror(errno)); } From 2e12558f0432150adb0e077d8d2ec8ba98e91a1c Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 2 Aug 2018 17:40:26 +0200 Subject: [PATCH 08/48] Allocate shared memory in units of pages Signed-off-by: DL6ER --- Makefile | 2 +- memory.c | 2 ++ shmem.c | 54 ++++++++++++++++++++++++++++++++++++++++-------------- shmem.h | 2 +- 4 files changed, 44 insertions(+), 16 deletions(-) diff --git a/Makefile b/Makefile index c798bf178..d55f5c94a 100644 --- a/Makefile +++ b/Makefile @@ -13,7 +13,7 @@ DNSMASQOPTS = -DHAVE_DNSSEC -DHAVE_DNSSEC_STATIC # Flags for compiling with libidn : -DHAVE_IDN # Flags for compiling with libidn2: -DHAVE_LIBIDN2 -DIDN2_VERSION_NUMBER=0x02000003 -FTLDEPS = FTL.h routines.h version.h api.h dnsmasq_interface.h +FTLDEPS = FTL.h routines.h version.h api.h dnsmasq_interface.h shmem.h FTLOBJ = main.o memory.o log.o daemon.o datastructure.o signals.o socket.o request.o grep.o setupVars.o args.o threads.o gc.o config.o database.o msgpack.o api.o dnsmasq_interface.o resolve.o regex.o shmem.o DNSMASQDEPS = config.h dhcp-protocol.h dns-protocol.h radv-protocol.h dhcp6-protocol.h dnsmasq.h ip6addr.h diff --git a/memory.c b/memory.c index b611d3fd8..30bee70fe 100644 --- a/memory.c +++ b/memory.c @@ -9,6 +9,7 @@ * Please see LICENSE file for your rights under this license. */ #include "FTL.h" +#include "shmem.h" FTLFileNamesStruct FTLfiles = { "/etc/pihole/pihole-FTL.conf", @@ -94,6 +95,7 @@ void memory_check(int which) // Have to reallocate memory counters.domains_MAX += DOMAINSALLOCSTEP; logg_struct_resize("domains",counters.domains_MAX,DOMAINSALLOCSTEP); + //realloc_shm(); domains = realloc(domains, counters.domains_MAX*sizeof(domainsDataStruct)); if(domains == NULL) { diff --git a/shmem.c b/shmem.c index 5653b3dbe..014f13cd7 100644 --- a/shmem.c +++ b/shmem.c @@ -13,10 +13,13 @@ /// The name of the shared memory. Use this when connecting to the shared memory. #define SHARED_STRINGS_NAME "FTL-strings" +#define SHARED_DOMAINS_NAME "FTL-domains" /// The pointer in shared memory to the shared string buffer -static SharedMemory strBuffer = { 0 }; +static SharedMemory shm_strings = { 0 }; +static SharedMemory shm_domains = { 0 }; +static int pagesize; static unsigned int next_pos = 0; unsigned int addstr(const char *str) @@ -30,15 +33,20 @@ unsigned int addstr(const char *str) // Get string length int len = strlen(str); - // Reserve memory (will later be replaced for shmem) - if(!realloc_shm(&strBuffer, next_pos + len + 1)) - return 0; - strBuffer.size = next_pos + len + 1; + if(debug) logg("Adding \"%s\" (len %i) to buffer. next_pos is %i", str, len, next_pos); + + // Reserve memory + size_t newsize = next_pos + len + 1; + if((long long)newsize-(long long)shm_strings.size > pagesize) + { + if(!realloc_shm(&shm_strings, shm_strings.size + pagesize)) + return 0; + shm_strings.size += pagesize; + } // Copy the C string pointed by str into the shared string buffer - char *buffer = strBuffer.ptr + next_pos; - strncpy(buffer, str, len); - buffer[len] = '\0'; + strncpy(&shm_strings.ptr[next_pos], str, len); + shm_strings.ptr[next_pos + len] = '\0'; // Increment string length counter next_pos += len+2; @@ -49,26 +57,38 @@ unsigned int addstr(const char *str) char *getstr(unsigned int pos) { - return strBuffer.ptr + pos; + return shm_strings.ptr + pos; } bool init_shmem(void) { + // Get kernel's page size + pagesize = getpagesize(); + + /****************************** shared strings buffer ******************************/ // Try unlinking the shared memory object before creating a new one // If the object is still existing, e.g., due to a past unclean exit // of FTL, shm_open() would fail with error "File exists" shm_unlink(SHARED_STRINGS_NAME); + // Try to create shared memory object - strBuffer = create_shm(SHARED_STRINGS_NAME, 1); - if(strBuffer.ptr == NULL) + shm_strings = create_shm(SHARED_STRINGS_NAME, 1); + if(shm_strings.ptr == NULL) return false; // Initialize shared string object with an empty string at position zero - char *buffer = strBuffer.ptr; - buffer[0] = '\0'; + shm_strings.ptr[0] = '\0'; next_pos = 1; + /****************************** shared domains struct ******************************/ + shm_unlink(SHARED_DOMAINS_NAME); + + // Try to create shared memory object + shm_domains = create_shm(SHARED_DOMAINS_NAME, 1); + if(shm_domains.ptr == NULL) + return false; + return true; } @@ -124,15 +144,21 @@ bool realloc_shm(SharedMemory *sharedMemory, size_t size) { if(debug) logg("Resizing \"%s\" from %zu to %zu", sharedMemory->name, sharedMemory->size, size); int result = ftruncate(sharedMemory->fd, size); - if(result == -1) { logg("realloc_shm(%i, %zu): ftruncate(%i, %zu): Failed to resize \"%s\": %s", sharedMemory->fd, size, sharedMemory->name, strerror(errno)); return false; } +#if 1 void *new_ptr = mremap(sharedMemory->ptr, sharedMemory->size, size, MREMAP_MAYMOVE); +#else + result = munmap(sharedMemory->ptr, sharedMemory->size); + if(result != 0) + logg("realloc_shm(): munmap(%p, %zu) failed: %s", sharedMemory->ptr, sharedMemory->size, strerror(errno)); + void *new_ptr = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED, sharedMemory->fd, 0); +#endif if(new_ptr == MAP_FAILED) { logg("realloc_shm(): mremap(%p, %zu, %zu, MREMAP_MAYMOVE): Failed to reallocate \"%s\" (%i): %s", diff --git a/shmem.h b/shmem.h index f6acb88d8..9c668c470 100644 --- a/shmem.h +++ b/shmem.h @@ -18,7 +18,7 @@ typedef struct { int fd; char *name; size_t size; - void *ptr; + char *ptr; } SharedMemory; /// Create shared memory From f1ba1391bcd214e27b5abd5557ec9d158822eefa Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 2 Aug 2018 18:03:56 +0200 Subject: [PATCH 09/48] Unify two if statements into one + allocate already one page size initially Signed-off-by: DL6ER --- shmem.c | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/shmem.c b/shmem.c index 014f13cd7..ded89502f 100644 --- a/shmem.c +++ b/shmem.c @@ -35,14 +35,11 @@ unsigned int addstr(const char *str) if(debug) logg("Adding \"%s\" (len %i) to buffer. next_pos is %i", str, len, next_pos); - // Reserve memory - size_t newsize = next_pos + len + 1; - if((long long)newsize-(long long)shm_strings.size > pagesize) - { - if(!realloc_shm(&shm_strings, shm_strings.size + pagesize)) - return 0; - shm_strings.size += pagesize; - } + // Reserve additional memory if necessary + size_t required_size = next_pos + len + 1; + // Need to cast to long long because size_t calculations cannot be negative + if((long long)required_size-(long long)shm_strings.size > 0 && !realloc_shm(&shm_strings, shm_strings.size + pagesize)) + return 0; // Copy the C string pointed by str into the shared string buffer strncpy(&shm_strings.ptr[next_pos], str, len); @@ -71,9 +68,8 @@ bool init_shmem(void) // of FTL, shm_open() would fail with error "File exists" shm_unlink(SHARED_STRINGS_NAME); - // Try to create shared memory object - shm_strings = create_shm(SHARED_STRINGS_NAME, 1); + shm_strings = create_shm(SHARED_STRINGS_NAME, pagesize); if(shm_strings.ptr == NULL) return false; From 0ced77c93bd4041afa325cc45ce0ed8dcb621391 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 2 Aug 2018 19:51:36 +0200 Subject: [PATCH 10/48] Also store domains struct in shared memory object Signed-off-by: DL6ER --- main.c | 3 +++ memory.c | 11 ++++++----- routines.h | 2 ++ shmem.c | 44 +++++++++++++++++++++++++++++++++++++++++++- 4 files changed, 54 insertions(+), 6 deletions(-) diff --git a/main.c b/main.c index baecf420d..fc7c200d9 100644 --- a/main.c +++ b/main.c @@ -93,6 +93,9 @@ int main (int argc, char* argv[]) // Invalidate blocking regex if compiled free_regex(); + // Remove shared memory objects + destroy_shmem(); + //Remove PID file removepid(); logg("########## FTL terminated after %.1f ms! ##########", timer_elapsed_msec(EXIT_TIMER)); diff --git a/memory.c b/memory.c index 30bee70fe..a77bc2e02 100644 --- a/memory.c +++ b/memory.c @@ -92,11 +92,12 @@ void memory_check(int which) case DOMAINS: if(counters.domains >= counters.domains_MAX) { - // Have to reallocate memory - counters.domains_MAX += DOMAINSALLOCSTEP; - logg_struct_resize("domains",counters.domains_MAX,DOMAINSALLOCSTEP); - //realloc_shm(); - domains = realloc(domains, counters.domains_MAX*sizeof(domainsDataStruct)); + // Have to reallocate shared memory + domains = enlarge_shmem_struct('d'); + // logg_struct_resize("domains",counters.domains_MAX,DOMAINSALLOCSTEP); + // //realloc_shm(); + // domains = realloc(domains, counters.domains_MAX*sizeof(domainsDataStruct)); + if(domains == NULL) { logg("FATAL: Memory allocation failed! Exiting"); diff --git a/routines.h b/routines.h index f1bbb43ba..9ef17fc08 100644 --- a/routines.h +++ b/routines.h @@ -111,5 +111,7 @@ bool in_whitelist(char *domain); // shmem.c bool init_shmem(void); +void destroy_shmem(void); unsigned int addstr(const char *str); char *getstr(unsigned int pos); +void *enlarge_shmem_struct(char type); diff --git a/shmem.c b/shmem.c index ded89502f..c3eca1153 100644 --- a/shmem.c +++ b/shmem.c @@ -81,13 +81,19 @@ bool init_shmem(void) shm_unlink(SHARED_DOMAINS_NAME); // Try to create shared memory object - shm_domains = create_shm(SHARED_DOMAINS_NAME, 1); + shm_domains = create_shm(SHARED_DOMAINS_NAME, pagesize*sizeof(domainsDataStruct)); if(shm_domains.ptr == NULL) return false; return true; } +void destroy_shmem(void) +{ + delete_shm(&shm_strings); + delete_shm(&shm_domains); +} + SharedMemory create_shm(char *name, size_t size) { if(debug) logg("Creating shared memory with name \"%s\" and size %zu", name, size); @@ -136,6 +142,42 @@ SharedMemory create_shm(char *name, size_t size) return sharedMemory; } +void *enlarge_shmem_struct(char type) +{ + SharedMemory sharedMemory; + size_t sizeofobj; + int *counter; + switch(type) + { +/* case 'c': + sharedMemory = shm_clients; + sizeofobj = sizeof(clientsDataStruct); + counter = &counters.clients_MAX; + break;*/ + case 'd': + sharedMemory = shm_domains; + sizeofobj = sizeof(domainsDataStruct); + counter = &counters.domains_MAX; + break; +/* case 'f': + sharedMemory = shm_forwarded; + sizeofobj = sizeof(forwardedDataStruct); + counter = &counters.forwarded_MAX; + break;*/ + default: + logg("Invalid argument in enlarge_shmem_struct(): %c (%i)", type, type); + return 0; + } + + // Reallocate enough space for 4096 instances of requested object + realloc_shm(&sharedMemory, sharedMemory.size + pagesize*sizeofobj); + + // Add allocated memory to corresponding counter + *counter += pagesize; + + return sharedMemory.ptr; +} + bool realloc_shm(SharedMemory *sharedMemory, size_t size) { if(debug) logg("Resizing \"%s\" from %zu to %zu", sharedMemory->name, sharedMemory->size, size); From 249d23667332de23437075c4fd1e0a908fbf4a1e Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 2 Aug 2018 20:00:05 +0200 Subject: [PATCH 11/48] Also store clients struct in shared memory object Signed-off-by: DL6ER --- memory.c | 11 ++++------- shmem.c | 19 +++++++++++++++++-- 2 files changed, 21 insertions(+), 9 deletions(-) diff --git a/memory.c b/memory.c index a77bc2e02..7c91c3ccc 100644 --- a/memory.c +++ b/memory.c @@ -78,10 +78,9 @@ void memory_check(int which) case CLIENTS: if(counters.clients >= counters.clients_MAX) { - // Have to reallocate memory - counters.clients_MAX += CLIENTSALLOCSTEP; - logg_struct_resize("clients",counters.clients_MAX,CLIENTSALLOCSTEP); - clients = realloc(clients, counters.clients_MAX*sizeof(clientsDataStruct)); + // Have to reallocate shared memory + clients = enlarge_shmem_struct('c'); + logg_struct_resize("clients", counters.clients_MAX, CLIENTSALLOCSTEP); if(clients == NULL) { logg("FATAL: Memory allocation failed! Exiting"); @@ -94,9 +93,7 @@ void memory_check(int which) { // Have to reallocate shared memory domains = enlarge_shmem_struct('d'); - // logg_struct_resize("domains",counters.domains_MAX,DOMAINSALLOCSTEP); - // //realloc_shm(); - // domains = realloc(domains, counters.domains_MAX*sizeof(domainsDataStruct)); + logg_struct_resize("clients", counters.domains_MAX, DOMAINSALLOCSTEP); if(domains == NULL) { diff --git a/shmem.c b/shmem.c index c3eca1153..c4bdf8655 100644 --- a/shmem.c +++ b/shmem.c @@ -14,10 +14,12 @@ /// The name of the shared memory. Use this when connecting to the shared memory. #define SHARED_STRINGS_NAME "FTL-strings" #define SHARED_DOMAINS_NAME "FTL-domains" +#define SHARED_CLIENTS_NAME "FTL-clients" /// The pointer in shared memory to the shared string buffer static SharedMemory shm_strings = { 0 }; static SharedMemory shm_domains = { 0 }; +static SharedMemory shm_clients = { 0 }; static int pagesize; static unsigned int next_pos = 0; @@ -84,6 +86,18 @@ bool init_shmem(void) shm_domains = create_shm(SHARED_DOMAINS_NAME, pagesize*sizeof(domainsDataStruct)); if(shm_domains.ptr == NULL) return false; + domains = (domainsDataStruct*)shm_domains.ptr; + counters.domains_MAX = pagesize; + + /****************************** shared clients struct ******************************/ + shm_unlink(SHARED_CLIENTS_NAME); + + // Try to create shared memory object + shm_clients = create_shm(SHARED_CLIENTS_NAME, pagesize*sizeof(clientsDataStruct)); + if(shm_clients.ptr == NULL) + return false; + clients = (clientsDataStruct*)shm_clients.ptr; + counters.clients_MAX = pagesize; return true; } @@ -92,6 +106,7 @@ void destroy_shmem(void) { delete_shm(&shm_strings); delete_shm(&shm_domains); + delete_shm(&shm_clients); } SharedMemory create_shm(char *name, size_t size) @@ -149,11 +164,11 @@ void *enlarge_shmem_struct(char type) int *counter; switch(type) { -/* case 'c': + case 'c': sharedMemory = shm_clients; sizeofobj = sizeof(clientsDataStruct); counter = &counters.clients_MAX; - break;*/ + break; case 'd': sharedMemory = shm_domains; sizeofobj = sizeof(domainsDataStruct); From 584c23dc26ff1471afa36c13deaafe82ecfcb2a1 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 2 Aug 2018 20:04:47 +0200 Subject: [PATCH 12/48] Also store forwarded struct in shared memory object Signed-off-by: DL6ER --- memory.c | 9 ++------- shmem.c | 24 +++++++++++++++++++----- 2 files changed, 21 insertions(+), 12 deletions(-) diff --git a/memory.c b/memory.c index 7c91c3ccc..8bad0e3e9 100644 --- a/memory.c +++ b/memory.c @@ -64,10 +64,8 @@ void memory_check(int which) case FORWARDED: if(counters.forwarded >= counters.forwarded_MAX) { - // Have to reallocate memory - counters.forwarded_MAX += FORWARDEDALLOCSTEP; - logg_struct_resize("forwarded",counters.forwarded_MAX,FORWARDEDALLOCSTEP); - forwarded = realloc(forwarded, counters.forwarded_MAX*sizeof(forwardedDataStruct)); + // Have to reallocate shared memory + clients = enlarge_shmem_struct('f'); if(forwarded == NULL) { logg("FATAL: Memory allocation failed! Exiting"); @@ -80,7 +78,6 @@ void memory_check(int which) { // Have to reallocate shared memory clients = enlarge_shmem_struct('c'); - logg_struct_resize("clients", counters.clients_MAX, CLIENTSALLOCSTEP); if(clients == NULL) { logg("FATAL: Memory allocation failed! Exiting"); @@ -93,8 +90,6 @@ void memory_check(int which) { // Have to reallocate shared memory domains = enlarge_shmem_struct('d'); - logg_struct_resize("clients", counters.domains_MAX, DOMAINSALLOCSTEP); - if(domains == NULL) { logg("FATAL: Memory allocation failed! Exiting"); diff --git a/shmem.c b/shmem.c index c4bdf8655..304dc8443 100644 --- a/shmem.c +++ b/shmem.c @@ -15,11 +15,13 @@ #define SHARED_STRINGS_NAME "FTL-strings" #define SHARED_DOMAINS_NAME "FTL-domains" #define SHARED_CLIENTS_NAME "FTL-clients" +#define SHARED_FORWARDED_NAME "FTL-forwarded" /// The pointer in shared memory to the shared string buffer static SharedMemory shm_strings = { 0 }; static SharedMemory shm_domains = { 0 }; static SharedMemory shm_clients = { 0 }; +static SharedMemory shm_forwarded = { 0 }; static int pagesize; static unsigned int next_pos = 0; @@ -69,7 +71,6 @@ bool init_shmem(void) // If the object is still existing, e.g., due to a past unclean exit // of FTL, shm_open() would fail with error "File exists" shm_unlink(SHARED_STRINGS_NAME); - // Try to create shared memory object shm_strings = create_shm(SHARED_STRINGS_NAME, pagesize); if(shm_strings.ptr == NULL) @@ -81,7 +82,6 @@ bool init_shmem(void) /****************************** shared domains struct ******************************/ shm_unlink(SHARED_DOMAINS_NAME); - // Try to create shared memory object shm_domains = create_shm(SHARED_DOMAINS_NAME, pagesize*sizeof(domainsDataStruct)); if(shm_domains.ptr == NULL) @@ -91,7 +91,6 @@ bool init_shmem(void) /****************************** shared clients struct ******************************/ shm_unlink(SHARED_CLIENTS_NAME); - // Try to create shared memory object shm_clients = create_shm(SHARED_CLIENTS_NAME, pagesize*sizeof(clientsDataStruct)); if(shm_clients.ptr == NULL) @@ -99,6 +98,15 @@ bool init_shmem(void) clients = (clientsDataStruct*)shm_clients.ptr; counters.clients_MAX = pagesize; + /****************************** shared forwarded struct ******************************/ + shm_unlink(SHARED_FORWARDED_NAME); + // Try to create shared memory object + shm_forwarded = create_shm(SHARED_FORWARDED_NAME, pagesize*sizeof(forwardedDataStruct)); + if(shm_forwarded.ptr == NULL) + return false; + forwarded = (forwardedDataStruct*)shm_forwarded.ptr; + counters.forwarded_MAX = pagesize; + return true; } @@ -162,28 +170,34 @@ void *enlarge_shmem_struct(char type) SharedMemory sharedMemory; size_t sizeofobj; int *counter; + char *typ; switch(type) { case 'c': + typ = "clients"; sharedMemory = shm_clients; sizeofobj = sizeof(clientsDataStruct); counter = &counters.clients_MAX; break; case 'd': + typ = "domains"; sharedMemory = shm_domains; sizeofobj = sizeof(domainsDataStruct); counter = &counters.domains_MAX; break; -/* case 'f': + case 'f': + typ = "forwarded"; sharedMemory = shm_forwarded; sizeofobj = sizeof(forwardedDataStruct); counter = &counters.forwarded_MAX; - break;*/ + break; default: logg("Invalid argument in enlarge_shmem_struct(): %c (%i)", type, type); return 0; } + logg("Reallocating %s struct (increasing %zu by %zu)", typ, sharedMemory.size, pagesize*sizeofobj); + // Reallocate enough space for 4096 instances of requested object realloc_shm(&sharedMemory, sharedMemory.size + pagesize*sizeofobj); From 5d22aa774e13c56c7ae3b8ed66c0dc4c9e6da4d7 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 2 Aug 2018 20:09:39 +0200 Subject: [PATCH 13/48] Also store queries struct in shared memory object Signed-off-by: DL6ER --- memory.c | 6 ++---- shmem.c | 19 +++++++++++++++++++ 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/memory.c b/memory.c index 8bad0e3e9..9b05aa450 100644 --- a/memory.c +++ b/memory.c @@ -50,10 +50,8 @@ void memory_check(int which) case QUERIES: if(counters.queries >= counters.queries_MAX) { - // Have to reallocate memory - counters.queries_MAX += QUERIESALLOCSTEP; - logg_struct_resize("queries",counters.queries_MAX,QUERIESALLOCSTEP); - queries = realloc(queries, counters.queries_MAX*sizeof(queriesDataStruct)); + // Have to reallocate shared memory + queries = enlarge_shmem_struct('q'); if(queries == NULL) { logg("FATAL: Memory allocation failed! Exiting"); diff --git a/shmem.c b/shmem.c index 304dc8443..d766a0b2e 100644 --- a/shmem.c +++ b/shmem.c @@ -15,12 +15,14 @@ #define SHARED_STRINGS_NAME "FTL-strings" #define SHARED_DOMAINS_NAME "FTL-domains" #define SHARED_CLIENTS_NAME "FTL-clients" +#define SHARED_QUERIES_NAME "FTL-queries" #define SHARED_FORWARDED_NAME "FTL-forwarded" /// The pointer in shared memory to the shared string buffer static SharedMemory shm_strings = { 0 }; static SharedMemory shm_domains = { 0 }; static SharedMemory shm_clients = { 0 }; +static SharedMemory shm_queries = { 0 }; static SharedMemory shm_forwarded = { 0 }; static int pagesize; @@ -107,6 +109,15 @@ bool init_shmem(void) forwarded = (forwardedDataStruct*)shm_forwarded.ptr; counters.forwarded_MAX = pagesize; + /****************************** shared queries struct ******************************/ + shm_unlink(SHARED_QUERIES_NAME); + // Try to create shared memory object + shm_queries = create_shm(SHARED_QUERIES_NAME, pagesize*sizeof(queriesDataStruct)); + if(shm_queries.ptr == NULL) + return false; + queries = (queriesDataStruct*)shm_queries.ptr; + counters.queries_MAX = pagesize; + return true; } @@ -115,6 +126,8 @@ void destroy_shmem(void) delete_shm(&shm_strings); delete_shm(&shm_domains); delete_shm(&shm_clients); + delete_shm(&shm_queries); + delete_shm(&shm_forwarded); } SharedMemory create_shm(char *name, size_t size) @@ -173,6 +186,12 @@ void *enlarge_shmem_struct(char type) char *typ; switch(type) { + case 'q': + typ = "queries"; + sharedMemory = shm_queries; + sizeofobj = sizeof(queriesDataStruct); + counter = &counters.queries_MAX; + break; case 'c': typ = "clients"; sharedMemory = shm_clients; From edf74617ae34e600b94082b886e684531ec001b5 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 2 Aug 2018 23:00:00 +0100 Subject: [PATCH 14/48] Fix wrong pointer reference Signed-off-by: DL6ER --- memory.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/memory.c b/memory.c index 9b05aa450..2c04fa58d 100644 --- a/memory.c +++ b/memory.c @@ -63,7 +63,7 @@ void memory_check(int which) if(counters.forwarded >= counters.forwarded_MAX) { // Have to reallocate shared memory - clients = enlarge_shmem_struct('f'); + forwarded = enlarge_shmem_struct('f'); if(forwarded == NULL) { logg("FATAL: Memory allocation failed! Exiting"); From 74d25d16a77082c1d96253dd175037914cfa8522 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 3 Aug 2018 09:15:32 +0200 Subject: [PATCH 15/48] Fix wrong error message, remove duplicated resized message, and don't assume char* pointer type (use void* and cast at the right places if needed) Signed-off-by: DL6ER --- shmem.c | 19 +++++++------------ shmem.h | 2 +- 2 files changed, 8 insertions(+), 13 deletions(-) diff --git a/shmem.c b/shmem.c index d766a0b2e..4af11c25d 100644 --- a/shmem.c +++ b/shmem.c @@ -48,8 +48,8 @@ unsigned int addstr(const char *str) return 0; // Copy the C string pointed by str into the shared string buffer - strncpy(&shm_strings.ptr[next_pos], str, len); - shm_strings.ptr[next_pos + len] = '\0'; + strncpy(&((char*)shm_strings.ptr)[next_pos], str, len); + ((char*)shm_strings.ptr)[next_pos + len] = '\0'; // Increment string length counter next_pos += len+2; @@ -79,7 +79,7 @@ bool init_shmem(void) return false; // Initialize shared string object with an empty string at position zero - shm_strings.ptr[0] = '\0'; + ((char*)shm_strings.ptr)[0] = '\0'; next_pos = 1; /****************************** shared domains struct ******************************/ @@ -183,29 +183,26 @@ void *enlarge_shmem_struct(char type) SharedMemory sharedMemory; size_t sizeofobj; int *counter; - char *typ; + + // Select type of struct that should be enlarged switch(type) { case 'q': - typ = "queries"; sharedMemory = shm_queries; sizeofobj = sizeof(queriesDataStruct); counter = &counters.queries_MAX; break; case 'c': - typ = "clients"; sharedMemory = shm_clients; sizeofobj = sizeof(clientsDataStruct); counter = &counters.clients_MAX; break; case 'd': - typ = "domains"; sharedMemory = shm_domains; sizeofobj = sizeof(domainsDataStruct); counter = &counters.domains_MAX; break; case 'f': - typ = "forwarded"; sharedMemory = shm_forwarded; sizeofobj = sizeof(forwardedDataStruct); counter = &counters.forwarded_MAX; @@ -215,8 +212,6 @@ void *enlarge_shmem_struct(char type) return 0; } - logg("Reallocating %s struct (increasing %zu by %zu)", typ, sharedMemory.size, pagesize*sizeofobj); - // Reallocate enough space for 4096 instances of requested object realloc_shm(&sharedMemory, sharedMemory.size + pagesize*sizeofobj); @@ -227,11 +222,11 @@ void *enlarge_shmem_struct(char type) } bool realloc_shm(SharedMemory *sharedMemory, size_t size) { - if(debug) logg("Resizing \"%s\" from %zu to %zu", sharedMemory->name, sharedMemory->size, size); + logg("Resizing \"%s\" from %zu to %zu", sharedMemory->name, sharedMemory->size, size); int result = ftruncate(sharedMemory->fd, size); if(result == -1) { - logg("realloc_shm(%i, %zu): ftruncate(%i, %zu): Failed to resize \"%s\": %s", + logg("realloc_shm(): ftruncate(%i,%zu): Failed to resize \"%s\": %s", sharedMemory->fd, size, sharedMemory->name, strerror(errno)); return false; } diff --git a/shmem.h b/shmem.h index 9c668c470..f6acb88d8 100644 --- a/shmem.h +++ b/shmem.h @@ -18,7 +18,7 @@ typedef struct { int fd; char *name; size_t size; - char *ptr; + void *ptr; } SharedMemory; /// Create shared memory From 0003a41a82e886f4785e76ce29657d00ebb9a227 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 3 Aug 2018 11:29:39 +0200 Subject: [PATCH 16/48] Use munmap + mmap instead of mremap when enlarging shared memory objects Signed-off-by: DL6ER --- shmem.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/shmem.c b/shmem.c index 4af11c25d..262f7d851 100644 --- a/shmem.c +++ b/shmem.c @@ -44,7 +44,8 @@ unsigned int addstr(const char *str) // Reserve additional memory if necessary size_t required_size = next_pos + len + 1; // Need to cast to long long because size_t calculations cannot be negative - if((long long)required_size-(long long)shm_strings.size > 0 && !realloc_shm(&shm_strings, shm_strings.size + pagesize)) + if((long long)required_size-(long long)shm_strings.size > 0 && + !realloc_shm(&shm_strings, shm_strings.size + pagesize)) return 0; // Copy the C string pointed by str into the shared string buffer @@ -231,7 +232,7 @@ bool realloc_shm(SharedMemory *sharedMemory, size_t size) { return false; } -#if 1 +#if 0 void *new_ptr = mremap(sharedMemory->ptr, sharedMemory->size, size, MREMAP_MAYMOVE); #else result = munmap(sharedMemory->ptr, sharedMemory->size); From 77f932be47a0bcb1531199bc4c752803e73e41fa Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 3 Aug 2018 15:42:15 +0200 Subject: [PATCH 17/48] Move query type determination further up Signed-off-by: DL6ER --- FTL.h | 4 ---- dnsmasq_interface.c | 52 ++++++++++++++++++++++----------------------- memory.c | 10 ++++----- shmem.c | 4 ++-- 4 files changed, 32 insertions(+), 38 deletions(-) diff --git a/FTL.h b/FTL.h index 2eb449aed..22cb16f1b 100644 --- a/FTL.h +++ b/FTL.h @@ -55,10 +55,6 @@ // Next we define the step size in which the struct arrays are reallocated if they // grow too large. This number should be large enough so that reallocation does not // have to run very often, but should be as small as possible to avoid wasting memory -#define QUERIESALLOCSTEP 10000 -#define FORWARDEDALLOCSTEP 4 -#define CLIENTSALLOCSTEP 10 -#define DOMAINSALLOCSTEP 1000 #define OVERTIMEALLOCSTEP 100 #define WILDCARDALLOCSTEP 100 diff --git a/dnsmasq_interface.c b/dnsmasq_interface.c index b706d7ed7..26957b2c1 100644 --- a/dnsmasq_interface.c +++ b/dnsmasq_interface.c @@ -32,8 +32,32 @@ void FTL_new_query(unsigned int flags, char *name, struct all_addr *addr, char * struct timeval request; gettimeofday(&request, 0); + // Determine query type + unsigned char querytype = 0; + if(strcmp(types,"query[A]") == 0) + querytype = TYPE_A; + else if(strcmp(types,"query[AAAA]") == 0) + querytype = TYPE_AAAA; + else if(strcmp(types,"query[ANY]") == 0) + querytype = TYPE_ANY; + else if(strcmp(types,"query[SRV]") == 0) + querytype = TYPE_SRV; + else if(strcmp(types,"query[SOA]") == 0) + querytype = TYPE_SOA; + else if(strcmp(types,"query[PTR]") == 0) + querytype = TYPE_PTR; + else if(strcmp(types,"query[TXT]") == 0) + querytype = TYPE_TXT; + else + { + // Return early if query type is not known + if(debug) logg("Notice: Skipping unknown query type: %s (%i)", types, id); + disable_thread_lock(); + return; + } + // Skip AAAA queries if user doesn't want to have them analyzed - if(!config.analyze_AAAA && strcmp(types,"query[AAAA]") == 0) + if(!config.analyze_AAAA && querytype == TYPE_AAAA) { if(debug) logg("Not analyzing AAAA query"); disable_thread_lock(); @@ -94,32 +118,6 @@ void FTL_new_query(unsigned int flags, char *name, struct all_addr *addr, char * char *proto = (type == UDP) ? "UDP" : "TCP"; if(debug) logg("**** new %s %s \"%s\" from %s (ID %i)", proto, types, domain, client, id); - // Determine query type - unsigned char querytype = 0; - if(strcmp(types,"query[A]") == 0) - querytype = TYPE_A; - else if(strcmp(types,"query[AAAA]") == 0) - querytype = TYPE_AAAA; - else if(strcmp(types,"query[ANY]") == 0) - querytype = TYPE_ANY; - else if(strcmp(types,"query[SRV]") == 0) - querytype = TYPE_SRV; - else if(strcmp(types,"query[SOA]") == 0) - querytype = TYPE_SOA; - else if(strcmp(types,"query[PTR]") == 0) - querytype = TYPE_PTR; - else if(strcmp(types,"query[TXT]") == 0) - querytype = TYPE_TXT; - else - { - // Return early to avoid accessing querytypedata out of bounds - if(debug) logg("Notice: Skipping unknown query type: %s (%i)", types, id); - free(domain); - free(client); - disable_thread_lock(); - return; - } - // Update counters int timeidx = findOverTimeID(overTimetimestamp); validate_access("overTime", timeidx, true, __LINE__, __FUNCTION__, __FILE__); diff --git a/memory.c b/memory.c index 2c04fa58d..525cb465a 100644 --- a/memory.c +++ b/memory.c @@ -37,11 +37,11 @@ countersStruct counters = { 0 }; ConfigStruct config; // Variable size array structs -queriesDataStruct *queries; -forwardedDataStruct *forwarded; -clientsDataStruct *clients; -domainsDataStruct *domains; -overTimeDataStruct *overTime; +queriesDataStruct *queries = NULL; +forwardedDataStruct *forwarded = NULL; +clientsDataStruct *clients = NULL; +domainsDataStruct *domains = NULL; +overTimeDataStruct *overTime = NULL; void memory_check(int which) { diff --git a/shmem.c b/shmem.c index 262f7d851..16518b561 100644 --- a/shmem.c +++ b/shmem.c @@ -42,7 +42,7 @@ unsigned int addstr(const char *str) if(debug) logg("Adding \"%s\" (len %i) to buffer. next_pos is %i", str, len, next_pos); // Reserve additional memory if necessary - size_t required_size = next_pos + len + 1; + size_t required_size = next_pos + len + 2; // Need to cast to long long because size_t calculations cannot be negative if((long long)required_size-(long long)shm_strings.size > 0 && !realloc_shm(&shm_strings, shm_strings.size + pagesize)) @@ -61,7 +61,7 @@ unsigned int addstr(const char *str) char *getstr(unsigned int pos) { - return shm_strings.ptr + pos; + return &((char*)shm_strings.ptr)[pos]; } bool init_shmem(void) From f40d8b1170a35dbc6fd59376ebae30a4f71e63e2 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 3 Aug 2018 16:15:57 +0200 Subject: [PATCH 18/48] First munmap() the shared memory object, thereafter resize it (using ftruncate), and only then mmap() it again Signed-off-by: DL6ER --- shmem.c | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/shmem.c b/shmem.c index 16518b561..7f939abde 100644 --- a/shmem.c +++ b/shmem.c @@ -225,22 +225,19 @@ void *enlarge_shmem_struct(char type) bool realloc_shm(SharedMemory *sharedMemory, size_t size) { logg("Resizing \"%s\" from %zu to %zu", sharedMemory->name, sharedMemory->size, size); - int result = ftruncate(sharedMemory->fd, size); + int result = munmap(sharedMemory->ptr, sharedMemory->size); + if(result != 0) + logg("realloc_shm(): munmap(%p, %zu) failed: %s", sharedMemory->ptr, sharedMemory->size, strerror(errno)); + + result = ftruncate(sharedMemory->fd, size); if(result == -1) { logg("realloc_shm(): ftruncate(%i,%zu): Failed to resize \"%s\": %s", sharedMemory->fd, size, sharedMemory->name, strerror(errno)); return false; } -#if 0 - void *new_ptr = mremap(sharedMemory->ptr, sharedMemory->size, size, MREMAP_MAYMOVE); -#else - result = munmap(sharedMemory->ptr, sharedMemory->size); - if(result != 0) - logg("realloc_shm(): munmap(%p, %zu) failed: %s", sharedMemory->ptr, sharedMemory->size, strerror(errno)); - +// void *new_ptr = mremap(sharedMemory->ptr, sharedMemory->size, size, MREMAP_MAYMOVE); void *new_ptr = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED, sharedMemory->fd, 0); -#endif if(new_ptr == MAP_FAILED) { logg("realloc_shm(): mremap(%p, %zu, %zu, MREMAP_MAYMOVE): Failed to reallocate \"%s\" (%i): %s", From f72a4cd522565718a6eb0c4cbbc628891fb380e9 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 3 Aug 2018 20:23:13 +0200 Subject: [PATCH 19/48] man shm_open recommends to use shared memory object names starting with "/" for portability Signed-off-by: DL6ER --- shmem.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/shmem.c b/shmem.c index 7f939abde..c81d8f7a8 100644 --- a/shmem.c +++ b/shmem.c @@ -12,11 +12,11 @@ #include "shmem.h" /// The name of the shared memory. Use this when connecting to the shared memory. -#define SHARED_STRINGS_NAME "FTL-strings" -#define SHARED_DOMAINS_NAME "FTL-domains" -#define SHARED_CLIENTS_NAME "FTL-clients" -#define SHARED_QUERIES_NAME "FTL-queries" -#define SHARED_FORWARDED_NAME "FTL-forwarded" +#define SHARED_STRINGS_NAME "/FTL-strings" +#define SHARED_DOMAINS_NAME "/FTL-domains" +#define SHARED_CLIENTS_NAME "/FTL-clients" +#define SHARED_QUERIES_NAME "/FTL-queries" +#define SHARED_FORWARDED_NAME "/FTL-forwarded" /// The pointer in shared memory to the shared string buffer static SharedMemory shm_strings = { 0 }; From 9f275a08aa05691cf995918d781692eb42577d87 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 3 Aug 2018 20:45:59 +0200 Subject: [PATCH 20/48] Re-open shared memory object when resizing the object using ftruncate(). shm_open() automatically sets FD_CLOEXEC which closes the file handler automatically whenever something is calling system() to avoid leaking the file descriptors to random programs. The file descriptor becomes invalid and later mmap operations fail. We always use shm_open() (without the O_CREAT attribute) every time before we run mmap to ensure we always have a valid file descriptor. Signed-off-by: DL6ER --- shmem.c | 39 ++++++++++++++++++++++++++++----------- shmem.h | 1 - 2 files changed, 28 insertions(+), 12 deletions(-) diff --git a/shmem.c b/shmem.c index c81d8f7a8..ad4d9ac89 100644 --- a/shmem.c +++ b/shmem.c @@ -136,17 +136,16 @@ SharedMemory create_shm(char *name, size_t size) if(debug) logg("Creating shared memory with name \"%s\" and size %zu", name, size); SharedMemory sharedMemory = { - .fd = 0, .name = name, .size = size, .ptr = NULL }; // Create the shared memory file in read/write mode with 600 permissions - sharedMemory.fd = shm_open(name, O_CREAT | O_EXCL | O_TRUNC | O_RDWR, S_IRUSR | S_IWUSR); + int fd = shm_open(sharedMemory.name, O_CREAT | O_EXCL | O_TRUNC | O_RDWR, S_IRUSR | S_IWUSR); // Check for `shm_open` error - if(sharedMemory.fd == -1) + if(fd == -1) { logg("create_shm(): Failed to create_shm shared memory object \"%s\": %s", name, strerror(errno)); @@ -154,27 +153,31 @@ SharedMemory create_shm(char *name, size_t size) } // Resize shared memory file - int result = ftruncate(sharedMemory.fd, size); + int result = ftruncate(fd, size); // Check for `ftruncate` error if(result == -1) { logg("create_shm(): ftruncate(%i, %zu): Failed to resize shared memory object \"%s\": %s", - sharedMemory.fd, size, sharedMemory.name, strerror(errno)); + fd, size, sharedMemory.name, strerror(errno)); return sharedMemory; } // Create shared memory mapping - void *shm = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED, sharedMemory.fd, 0); + void *shm = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); // Check for `mmap` error if(shm == MAP_FAILED) { logg("create_shm(): Failed to map shared memory object \"%s\" (%i): %s", - sharedMemory.name, sharedMemory.fd, strerror(errno)); + sharedMemory.name, fd, strerror(errno)); return sharedMemory; } + // Close shared memory object file descriptor as it is no longer + // needed after having called mmap() + close(fd); + sharedMemory.ptr = shm; return sharedMemory; } @@ -229,23 +232,37 @@ bool realloc_shm(SharedMemory *sharedMemory, size_t size) { if(result != 0) logg("realloc_shm(): munmap(%p, %zu) failed: %s", sharedMemory->ptr, sharedMemory->size, strerror(errno)); - result = ftruncate(sharedMemory->fd, size); + // Open shared memory object + int fd = shm_open(sharedMemory->name, O_RDWR, S_IRUSR | S_IWUSR); + if(fd == -1) + { + logg("realloc_shm(): Failed to open shared memory object \"%s\": %s", + sharedMemory->name, strerror(errno)); + return false; + } + + // Resize shard memory object to requested size + result = ftruncate(fd, size); if(result == -1) { logg("realloc_shm(): ftruncate(%i,%zu): Failed to resize \"%s\": %s", - sharedMemory->fd, size, sharedMemory->name, strerror(errno)); + fd, size, sharedMemory->name, strerror(errno)); return false; } // void *new_ptr = mremap(sharedMemory->ptr, sharedMemory->size, size, MREMAP_MAYMOVE); - void *new_ptr = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED, sharedMemory->fd, 0); + void *new_ptr = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); if(new_ptr == MAP_FAILED) { logg("realloc_shm(): mremap(%p, %zu, %zu, MREMAP_MAYMOVE): Failed to reallocate \"%s\" (%i): %s", - sharedMemory->ptr, sharedMemory->size, size, sharedMemory->name, sharedMemory->fd, + sharedMemory->ptr, sharedMemory->size, size, sharedMemory->name, fd, strerror(errno)); return false; } + // Close shared memory object file descriptor as it is no longer + // needed after having called mmap() + close(fd); + sharedMemory->ptr = new_ptr; sharedMemory->size = size; diff --git a/shmem.h b/shmem.h index f6acb88d8..d4911a633 100644 --- a/shmem.h +++ b/shmem.h @@ -15,7 +15,6 @@ #include /* For O_* constants */ typedef struct { - int fd; char *name; size_t size; void *ptr; From eb1d851ed2c61a4cbbaf36581e73402096f7df10 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 4 Aug 2018 09:26:37 +0200 Subject: [PATCH 21/48] Allocate more struct memory earlier Signed-off-by: DL6ER --- memory.c | 10 +++++----- shmem.c | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/memory.c b/memory.c index 525cb465a..81265f49f 100644 --- a/memory.c +++ b/memory.c @@ -48,7 +48,7 @@ void memory_check(int which) switch(which) { case QUERIES: - if(counters.queries >= counters.queries_MAX) + if(counters.queries >= counters.queries_MAX-1) { // Have to reallocate shared memory queries = enlarge_shmem_struct('q'); @@ -60,7 +60,7 @@ void memory_check(int which) } break; case FORWARDED: - if(counters.forwarded >= counters.forwarded_MAX) + if(counters.forwarded >= counters.forwarded_MAX-1) { // Have to reallocate shared memory forwarded = enlarge_shmem_struct('f'); @@ -72,7 +72,7 @@ void memory_check(int which) } break; case CLIENTS: - if(counters.clients >= counters.clients_MAX) + if(counters.clients >= counters.clients_MAX-1) { // Have to reallocate shared memory clients = enlarge_shmem_struct('c'); @@ -84,7 +84,7 @@ void memory_check(int which) } break; case DOMAINS: - if(counters.domains >= counters.domains_MAX) + if(counters.domains >= counters.domains_MAX-1) { // Have to reallocate shared memory domains = enlarge_shmem_struct('d'); @@ -96,7 +96,7 @@ void memory_check(int which) } break; case OVERTIME: - if(counters.overTime >= counters.overTime_MAX) + if(counters.overTime >= counters.overTime_MAX-1) { // Have to reallocate memory counters.overTime_MAX += OVERTIMEALLOCSTEP; diff --git a/shmem.c b/shmem.c index ad4d9ac89..8f55852bc 100644 --- a/shmem.c +++ b/shmem.c @@ -244,7 +244,7 @@ bool realloc_shm(SharedMemory *sharedMemory, size_t size) { // Resize shard memory object to requested size result = ftruncate(fd, size); if(result == -1) { - logg("realloc_shm(): ftruncate(%i,%zu): Failed to resize \"%s\": %s", + logg("realloc_shm(): ftruncate(%i, %zu): Failed to resize \"%s\": %s", fd, size, sharedMemory->name, strerror(errno)); return false; } From 992726d9293669b833f7b28751d990f7a2927717 Mon Sep 17 00:00:00 2001 From: Mcat12 Date: Sat, 1 Sep 2018 22:41:49 -0400 Subject: [PATCH 22/48] Fix compile error due to merge The merged in code tried to use normal strings instead of shared memory strings. Signed-off-by: Mcat12 --- api.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/api.c b/api.c index 4a432ba79..4bb946954 100644 --- a/api.c +++ b/api.c @@ -642,9 +642,9 @@ void getAllQueries(char *client_message, int *sock) { // Try to match the requested string against their IP addresses and // (if available) their host names - if(strcmp(forwarded[i].ip, forwarddest) == 0 || - (forwarded[i].name != NULL && - strcmp(forwarded[i].name, forwarddest) == 0)) + if(strcmp(getstr(forwarded[i].ippos), forwarddest) == 0 || + (forwarded[i].namepos != 0 && + strcmp(getstr(forwarded[i].namepos), forwarddest) == 0)) { forwarddestid = i; break; @@ -673,7 +673,7 @@ void getAllQueries(char *client_message, int *sock) for(i = 0; i < counters.domains; i++) { // Try to match the requested string - if(strcmp(domains[i].domain, domainname) == 0) + if(strcmp(getstr(domains[i].domainpos), domainname) == 0) { domainid = i; break; @@ -701,9 +701,9 @@ void getAllQueries(char *client_message, int *sock) for(i = 0; i < counters.clients; i++) { // Try to match the requested string - if(strcmp(clients[i].ip, clientname) == 0 || - (clients[i].name != NULL && - strcmp(clients[i].name, clientname) == 0)) + if(strcmp(getstr(clients[i].ippos), clientname) == 0 || + (clients[i].namepos != 0 && + strcmp(getstr(clients[i].namepos), clientname) == 0)) { clientid = i; break; From 99c4e40ee52b129d8e55f27012afd29b1c458555 Mon Sep 17 00:00:00 2001 From: Mcat12 Date: Sat, 1 Sep 2018 22:44:26 -0400 Subject: [PATCH 23/48] Fix getstr parameter warning unsigned int is smaller than long long, which all of the string position data was using. This generated a lot of warnings. Signed-off-by: Mcat12 --- routines.h | 2 +- shmem.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/routines.h b/routines.h index 9ef17fc08..d52eee441 100644 --- a/routines.h +++ b/routines.h @@ -113,5 +113,5 @@ bool in_whitelist(char *domain); bool init_shmem(void); void destroy_shmem(void); unsigned int addstr(const char *str); -char *getstr(unsigned int pos); +char *getstr(long long pos); void *enlarge_shmem_struct(char type); diff --git a/shmem.c b/shmem.c index 8f55852bc..06faec528 100644 --- a/shmem.c +++ b/shmem.c @@ -59,7 +59,7 @@ unsigned int addstr(const char *str) return (next_pos - (len+2)); } -char *getstr(unsigned int pos) +char *getstr(long long pos) { return &((char*)shm_strings.ptr)[pos]; } From 358504350bdb6ea886bbcb72b997302f88901e22 Mon Sep 17 00:00:00 2001 From: Mcat12 Date: Sat, 1 Sep 2018 22:57:29 -0400 Subject: [PATCH 24/48] Use unsigned long long for string positions Signed-off-by: Mcat12 --- FTL.h | 10 +++++----- routines.h | 4 ++-- shmem.c | 8 ++++---- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/FTL.h b/FTL.h index 5b526f3a4..883be8bb2 100644 --- a/FTL.h +++ b/FTL.h @@ -174,8 +174,8 @@ typedef struct { unsigned char magic; int count; int failed; - long long ippos; - long long namepos; + unsigned long long ippos; + unsigned long long namepos; bool new; } forwardedDataStruct; @@ -183,8 +183,8 @@ typedef struct { unsigned char magic; int count; int blockedcount; - long long ippos; - long long namepos; + unsigned long long ippos; + unsigned long long namepos; bool new; } clientsDataStruct; @@ -192,7 +192,7 @@ typedef struct { unsigned char magic; int count; int blockedcount; - long long domainpos; + unsigned long long domainpos; unsigned char regexmatch; } domainsDataStruct; diff --git a/routines.h b/routines.h index d52eee441..e36cf0d2b 100644 --- a/routines.h +++ b/routines.h @@ -112,6 +112,6 @@ bool in_whitelist(char *domain); // shmem.c bool init_shmem(void); void destroy_shmem(void); -unsigned int addstr(const char *str); -char *getstr(long long pos); +unsigned long long addstr(const char *str); +char *getstr(unsigned long long pos); void *enlarge_shmem_struct(char type); diff --git a/shmem.c b/shmem.c index 06faec528..9837287ff 100644 --- a/shmem.c +++ b/shmem.c @@ -28,7 +28,7 @@ static SharedMemory shm_forwarded = { 0 }; static int pagesize; static unsigned int next_pos = 0; -unsigned int addstr(const char *str) +unsigned long long addstr(const char *str) { if(str == NULL) { @@ -37,7 +37,7 @@ unsigned int addstr(const char *str) } // Get string length - int len = strlen(str); + size_t len = strlen(str); if(debug) logg("Adding \"%s\" (len %i) to buffer. next_pos is %i", str, len, next_pos); @@ -56,10 +56,10 @@ unsigned int addstr(const char *str) next_pos += len+2; // Return start of stored string - return (next_pos - (len+2)); + return (next_pos - (len + 2)); } -char *getstr(long long pos) +char *getstr(unsigned long long pos) { return &((char*)shm_strings.ptr)[pos]; } From d96c0096b83c484d1d0e65eb0f7e111d02c746b5 Mon Sep 17 00:00:00 2001 From: Mcat12 Date: Sat, 1 Sep 2018 23:48:13 -0400 Subject: [PATCH 25/48] Only use a single NULL terminator in strings shared memory Previously two NULL bytes were used between strings, which was not intended. Signed-off-by: Mcat12 --- shmem.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/shmem.c b/shmem.c index 9837287ff..1f624e53f 100644 --- a/shmem.c +++ b/shmem.c @@ -42,7 +42,7 @@ unsigned long long addstr(const char *str) if(debug) logg("Adding \"%s\" (len %i) to buffer. next_pos is %i", str, len, next_pos); // Reserve additional memory if necessary - size_t required_size = next_pos + len + 2; + size_t required_size = next_pos + len + 1; // Need to cast to long long because size_t calculations cannot be negative if((long long)required_size-(long long)shm_strings.size > 0 && !realloc_shm(&shm_strings, shm_strings.size + pagesize)) @@ -53,10 +53,10 @@ unsigned long long addstr(const char *str) ((char*)shm_strings.ptr)[next_pos + len] = '\0'; // Increment string length counter - next_pos += len+2; + next_pos += len+1; // Return start of stored string - return (next_pos - (len + 2)); + return (next_pos - (len + 1)); } char *getstr(unsigned long long pos) From e91369f6aa32dad95593903be004a523b271394b Mon Sep 17 00:00:00 2001 From: Mcat12 Date: Sun, 2 Sep 2018 00:01:44 -0400 Subject: [PATCH 26/48] Fix shared memory resize not being remembered The new size was not remembered after resizing, causing it to never resize larger than two 4k blocks of items. Signed-off-by: Mcat12 --- shmem.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/shmem.c b/shmem.c index 1f624e53f..a71b20109 100644 --- a/shmem.c +++ b/shmem.c @@ -184,7 +184,7 @@ SharedMemory create_shm(char *name, size_t size) void *enlarge_shmem_struct(char type) { - SharedMemory sharedMemory; + SharedMemory *sharedMemory; size_t sizeofobj; int *counter; @@ -192,22 +192,22 @@ void *enlarge_shmem_struct(char type) switch(type) { case 'q': - sharedMemory = shm_queries; + sharedMemory = &shm_queries; sizeofobj = sizeof(queriesDataStruct); counter = &counters.queries_MAX; break; case 'c': - sharedMemory = shm_clients; + sharedMemory = &shm_clients; sizeofobj = sizeof(clientsDataStruct); counter = &counters.clients_MAX; break; case 'd': - sharedMemory = shm_domains; + sharedMemory = &shm_domains; sizeofobj = sizeof(domainsDataStruct); counter = &counters.domains_MAX; break; case 'f': - sharedMemory = shm_forwarded; + sharedMemory = &shm_forwarded; sizeofobj = sizeof(forwardedDataStruct); counter = &counters.forwarded_MAX; break; @@ -217,12 +217,12 @@ void *enlarge_shmem_struct(char type) } // Reallocate enough space for 4096 instances of requested object - realloc_shm(&sharedMemory, sharedMemory.size + pagesize*sizeofobj); + realloc_shm(sharedMemory, sharedMemory->size + pagesize*sizeofobj); // Add allocated memory to corresponding counter *counter += pagesize; - return sharedMemory.ptr; + return sharedMemory->ptr; } bool realloc_shm(SharedMemory *sharedMemory, size_t size) { From 8401c9baad1f65dcc24b776e01af839dc371ab76 Mon Sep 17 00:00:00 2001 From: Mcat12 Date: Sun, 2 Sep 2018 13:11:25 -0400 Subject: [PATCH 27/48] Add stdbool include to shmem.h Signed-off-by: Mcat12 --- shmem.h | 1 + 1 file changed, 1 insertion(+) diff --git a/shmem.h b/shmem.h index d4911a633..908c742f4 100644 --- a/shmem.h +++ b/shmem.h @@ -13,6 +13,7 @@ #include /* For shm_* functions */ #include /* For mode constants */ #include /* For O_* constants */ +#include typedef struct { char *name; From ae1bb5b71a9ce3ef90b5b8dbb292fc88b86f3031 Mon Sep 17 00:00:00 2001 From: Mcat12 Date: Mon, 3 Sep 2018 14:25:16 -0400 Subject: [PATCH 28/48] Move counters to shared memory This requires that it be accessed through a pointer, so most of the changes are just changing `counters.` to `counters->` Signed-off-by: Mcat12 --- FTL.h | 2 +- api.c | 138 ++++++++++++++++++++++---------------------- database.c | 18 +++--- datastructure.c | 54 ++++++++--------- dnsmasq_interface.c | 48 +++++++-------- gc.c | 32 +++++----- log.c | 16 ++--- memory.c | 28 ++++----- regex.c | 6 +- resolve.c | 6 +- shmem.c | 27 ++++++--- 11 files changed, 193 insertions(+), 182 deletions(-) diff --git a/FTL.h b/FTL.h index 883be8bb2..7c7c096f8 100644 --- a/FTL.h +++ b/FTL.h @@ -221,7 +221,7 @@ typedef struct { extern logFileNamesStruct files; extern FTLFileNamesStruct FTLfiles; -extern countersStruct counters; +extern countersStruct *counters; extern ConfigStruct config; extern queriesDataStruct *queries; diff --git a/api.c b/api.c index 4bb946954..377193c35 100644 --- a/api.c +++ b/api.c @@ -44,8 +44,8 @@ int cmpdesc(const void *a, const void *b) void getStats(int *sock) { - int blocked = counters.blocked; - int total = counters.queries; + int blocked = counters->blocked; + int total = counters->queries; float percentage = 0.0f; // Avoid 1/0 condition @@ -54,14 +54,14 @@ void getStats(int *sock) // Send domains being blocked if(istelnet[*sock]) { - ssend(*sock, "domains_being_blocked %i\n", counters.gravity); + ssend(*sock, "domains_being_blocked %i\n", counters->gravity); } else - pack_int32(*sock, counters.gravity); + pack_int32(*sock, counters->gravity); // unique_clients: count only clients that have been active within the most recent 24 hours int i, activeclients = 0; - for(i=0; i < counters.clients; i++) + for(i=0; i < counters->clients; i++) { validate_access("clients", i, true, __LINE__, __FUNCTION__, __FILE__); if(clients[i].count > 0) @@ -72,37 +72,37 @@ void getStats(int *sock) ssend(*sock, "dns_queries_today %i\nads_blocked_today %i\nads_percentage_today %f\n", total, blocked, percentage); ssend(*sock, "unique_domains %i\nqueries_forwarded %i\nqueries_cached %i\n", - counters.domains, counters.forwardedqueries, counters.cached); - ssend(*sock, "clients_ever_seen %i\n", counters.clients); + counters->domains, counters->forwardedqueries, counters->cached); + ssend(*sock, "clients_ever_seen %i\n", counters->clients); ssend(*sock, "unique_clients %i\n", activeclients); // Sum up all query types (A, AAAA, ANY, SRV, SOA, ...) int sumalltypes = 0; for(i=0; i < TYPE_MAX-1; i++) { - sumalltypes += counters.querytype[i]; + sumalltypes += counters->querytype[i]; } ssend(*sock, "dns_queries_all_types %i\n", sumalltypes); // Send individual reply type counters ssend(*sock, "reply_NODATA %i\nreply_NXDOMAIN %i\nreply_CNAME %i\nreply_IP %i\n", - counters.reply_NODATA, counters.reply_NXDOMAIN, counters.reply_CNAME, counters.reply_IP); + counters->reply_NODATA, counters->reply_NXDOMAIN, counters->reply_CNAME, counters->reply_IP); } else { pack_int32(*sock, total); pack_int32(*sock, blocked); pack_float(*sock, percentage); - pack_int32(*sock, counters.domains); - pack_int32(*sock, counters.forwardedqueries); - pack_int32(*sock, counters.cached); - pack_int32(*sock, counters.clients); + pack_int32(*sock, counters->domains); + pack_int32(*sock, counters->forwardedqueries); + pack_int32(*sock, counters->cached); + pack_int32(*sock, counters->clients); pack_int32(*sock, activeclients); } // Send status if(istelnet[*sock]) { - ssend(*sock, "status %s\n", counters.gravity > 0 ? "enabled" : "disabled"); + ssend(*sock, "status %s\n", counters->gravity > 0 ? "enabled" : "disabled"); } else pack_uint8(*sock, blockingstatus); @@ -115,7 +115,7 @@ void getOverTime(int *sock) time_t mintime = time(NULL) - config.maxlogage; // Start with the first non-empty overTime slot - for(i=0; i < counters.overTime; i++) + for(i=0; i < counters->overTime; i++) { validate_access("overTime", i, true, __LINE__, __FUNCTION__, __FILE__); if((overTime[i].total > 0 || overTime[i].blocked > 0) && @@ -133,7 +133,7 @@ void getOverTime(int *sock) if(istelnet[*sock]) { - for(i = j; i < counters.overTime; i++) + for(i = j; i < counters->overTime; i++) { ssend(*sock,"%i %i %i\n",overTime[i].timestamp,overTime[i].total,overTime[i].blocked); } @@ -144,15 +144,15 @@ void getOverTime(int *sock) // and map16 can hold up to (2^16)-1 = 65535 pairs // Send domains over time - pack_map16_start(*sock, (uint16_t) (counters.overTime - j)); - for(i = j; i < counters.overTime; i++) { + pack_map16_start(*sock, (uint16_t) (counters->overTime - j)); + for(i = j; i < counters->overTime; i++) { pack_int32(*sock, overTime[i].timestamp); pack_int32(*sock, overTime[i].total); } // Send ads over time - pack_map16_start(*sock, (uint16_t) (counters.overTime - j)); - for(i = j; i < counters.overTime; i++) { + pack_map16_start(*sock, (uint16_t) (counters->overTime - j)); + for(i = j; i < counters->overTime; i++) { pack_int32(*sock, overTime[i].timestamp); pack_int32(*sock, overTime[i].blocked); } @@ -161,7 +161,7 @@ void getOverTime(int *sock) void getTopDomains(char *client_message, int *sock) { - int i, temparray[counters.domains][2], count=10, num; + int i, temparray[counters->domains][2], count=10, num; bool blocked, audit = false, asc = false; blocked = command(client_message, ">top-ads"); @@ -193,7 +193,7 @@ void getTopDomains(char *client_message, int *sock) if(command(client_message, " asc")) asc = true; - for(i=0; i < counters.domains; i++) + for(i=0; i < counters->domains; i++) { validate_access("domains", i, true, __LINE__, __FUNCTION__, __FILE__); temparray[i][0] = i; @@ -206,9 +206,9 @@ void getTopDomains(char *client_message, int *sock) // Sort temporary array if(asc) - qsort(temparray, counters.domains, sizeof(int[2]), cmpasc); + qsort(temparray, counters->domains, sizeof(int[2]), cmpasc); else - qsort(temparray, counters.domains, sizeof(int[2]), cmpdesc); + qsort(temparray, counters->domains, sizeof(int[2]), cmpdesc); // Get filter @@ -243,13 +243,13 @@ void getTopDomains(char *client_message, int *sock) { // Send the data required to get the percentage each domain has been blocked / queried if(blocked) - pack_int32(*sock, counters.blocked); + pack_int32(*sock, counters->blocked); else - pack_int32(*sock, counters.queries); + pack_int32(*sock, counters->queries); } int n = 0; - for(i=0; i < counters.domains; i++) + for(i=0; i < counters->domains; i++) { // Get sorted indices int j = temparray[i][0]; @@ -323,7 +323,7 @@ void getTopDomains(char *client_message, int *sock) void getTopClients(char *client_message, int *sock) { - int i, temparray[counters.clients][2], count=10, num; + int i, temparray[counters->clients][2], count=10, num; // Exit before processing any data if requested via config setting get_privacy_level(NULL); @@ -356,7 +356,7 @@ void getTopClients(char *client_message, int *sock) if(command(client_message, " blocked")) blockedonly = true; - for(i=0; i < counters.clients; i++) + for(i=0; i < counters->clients; i++) { validate_access("clients", i, true, __LINE__, __FUNCTION__, __FILE__); temparray[i][0] = i; @@ -372,9 +372,9 @@ void getTopClients(char *client_message, int *sock) // Sort temporary array if(asc) - qsort(temparray, counters.clients, sizeof(int[2]), cmpasc); + qsort(temparray, counters->clients, sizeof(int[2]), cmpasc); else - qsort(temparray, counters.clients, sizeof(int[2]), cmpdesc); + qsort(temparray, counters->clients, sizeof(int[2]), cmpdesc); // Get clients which the user doesn't want to see char * excludeclients = read_setupVarsconf("API_EXCLUDE_CLIENTS"); @@ -386,11 +386,11 @@ void getTopClients(char *client_message, int *sock) if(!istelnet[*sock]) { // Send the total queries so they can make percentages from this data - pack_int32(*sock, counters.queries); + pack_int32(*sock, counters->queries); } int n = 0; - for(i=0; i < counters.clients; i++) + for(i=0; i < counters->clients; i++) { // Get sorted indices and counter values (may be either total or blocked count) int j = temparray[i][0]; @@ -438,12 +438,12 @@ void getTopClients(char *client_message, int *sock) void getForwardDestinations(char *client_message, int *sock) { bool sort = true; - int i, temparray[counters.forwarded][2], forwardedsum = 0, totalqueries = 0; + int i, temparray[counters->forwarded][2], forwardedsum = 0, totalqueries = 0; if(command(client_message, "unsorted")) sort = false; - for(i=0; i < counters.forwarded; i++) { + for(i=0; i < counters->forwarded; i++) { validate_access("forwarded", i, true, __LINE__, __FUNCTION__, __FILE__); // Compute forwardedsum forwardedsum += forwarded[i].count; @@ -459,13 +459,13 @@ void getForwardDestinations(char *client_message, int *sock) if(sort) { // Sort temporary array in descending order - qsort(temparray, counters.forwarded, sizeof(int[2]), cmpdesc); + qsort(temparray, counters->forwarded, sizeof(int[2]), cmpdesc); } - totalqueries = counters.forwardedqueries + counters.cached + counters.blocked; + totalqueries = counters->forwardedqueries + counters->cached + counters->blocked; // Loop over available forward destinations - for(i=-2; i < min(counters.forwarded, 8); i++) + for(i=-2; i < min(counters->forwarded, 8); i++) { char *ip, *name; float percentage = 0.0f; @@ -478,7 +478,7 @@ void getForwardDestinations(char *client_message, int *sock) if(totalqueries > 0) // Whats the percentage of locked queries on the total amount of queries? - percentage = 1e2f * counters.blocked / totalqueries; + percentage = 1e2f * counters->blocked / totalqueries; } else if(i == -1) { @@ -488,7 +488,7 @@ void getForwardDestinations(char *client_message, int *sock) if(totalqueries > 0) // Whats the percentage of cached queries on the total amount of queries? - percentage = 1e2f * counters.cached / totalqueries; + percentage = 1e2f * counters->cached / totalqueries; } else { @@ -515,14 +515,14 @@ void getForwardDestinations(char *client_message, int *sock) // The fraction a describes now how much share an individual forward destination // has on the total sum of sent requests. // We also know the share of forwarded queries on the total number of queries - // b = counters.forwardedqueries / c + // b = counters->forwardedqueries / c // where c is the number of valid queries, - // c = counters.forwardedqueries + counters.cached + counters.blocked + // c = counters->forwardedqueries + counters->cached + counters->blocked // // To get the total percentage of a specific query on the total number of queries, // we simply have to scale b by a which is what we do in the following. if(forwardedsum > 0 && totalqueries > 0) - percentage = 1e2f * forwarded[j].count / forwardedsum * counters.forwardedqueries / totalqueries; + percentage = 1e2f * forwarded[j].count / forwardedsum * counters->forwardedqueries / totalqueries; } // Send data if count > 0 @@ -546,14 +546,14 @@ void getQueryTypes(int *sock) { int i,total = 0; for(i=0; i < TYPE_MAX-1; i++) - total += counters.querytype[i]; + total += counters->querytype[i]; float percentage[TYPE_MAX-1] = { 0.0 }; // Prevent floating point exceptions by checking if the divisor is != 0 if(total > 0) for(i=0; i < TYPE_MAX-1; i++) - percentage[i] = 1e2f*counters.querytype[i]/total; + percentage[i] = 1e2f*counters->querytype[i]/total; if(istelnet[*sock]) { ssend(*sock, "A (IPv4): %.2f\nAAAA (IPv6): %.2f\nANY: %.2f\nSRV: %.2f\nSOA: %.2f\nPTR: %.2f\nTXT: %.2f\n", @@ -636,9 +636,9 @@ void getAllQueries(char *client_message, int *sock) { // Iterate through all known forward destinations int i; - validate_access("forwards", MAX(0,counters.forwarded-1), true, __LINE__, __FUNCTION__, __FILE__); + validate_access("forwards", MAX(0,counters->forwarded-1), true, __LINE__, __FUNCTION__, __FILE__); forwarddestid = -3; - for(i = 0; i < counters.forwarded; i++) + for(i = 0; i < counters->forwarded; i++) { // Try to match the requested string against their IP addresses and // (if available) their host names @@ -669,8 +669,8 @@ void getAllQueries(char *client_message, int *sock) filterdomainname = true; // Iterate through all known domains int i; - validate_access("domains", MAX(0,counters.domains-1), true, __LINE__, __FUNCTION__, __FILE__); - for(i = 0; i < counters.domains; i++) + validate_access("domains", MAX(0,counters->domains-1), true, __LINE__, __FUNCTION__, __FILE__); + for(i = 0; i < counters->domains; i++) { // Try to match the requested string if(strcmp(getstr(domains[i].domainpos), domainname) == 0) @@ -697,8 +697,8 @@ void getAllQueries(char *client_message, int *sock) filterclientname = true; // Iterate through all known clients int i; - validate_access("clients", MAX(0,counters.clients-1), true, __LINE__, __FUNCTION__, __FILE__); - for(i = 0; i < counters.clients; i++) + validate_access("clients", MAX(0,counters->clients-1), true, __LINE__, __FUNCTION__, __FILE__); + for(i = 0; i < counters->clients; i++) { // Try to match the requested string if(strcmp(getstr(clients[i].ippos), clientname) == 0 || @@ -724,7 +724,7 @@ void getAllQueries(char *client_message, int *sock) { // User wants a different number of requests // Don't allow a start index that is smaller than zero - ibeg = counters.queries-num; + ibeg = counters->queries-num; if(ibeg < 0) ibeg = 0; } @@ -747,7 +747,7 @@ void getAllQueries(char *client_message, int *sock) clearSetupVarsArray(); int i; - for(i=ibeg; i < counters.queries; i++) + for(i=ibeg; i < counters->queries; i++) { validate_access("queries", i, true, __LINE__, __FUNCTION__, __FILE__); // Check if this query has been create while in maximum privacy mode @@ -855,13 +855,13 @@ void getRecentBlocked(char *client_message, int *sock) // Test for integer that specifies number of entries to be shown if(sscanf(client_message, "%*[^(](%i)", &num) > 0) { // User wants a different number of requests - if(num >= counters.queries) + if(num >= counters->queries) num = 0; } // Find most recently blocked query int found = 0; - for(i = counters.queries - 1; i > 0 ; i--) + for(i = counters->queries - 1; i > 0 ; i--) { validate_access("queries", i, true, __LINE__, __FUNCTION__, __FILE__); @@ -894,7 +894,7 @@ void getQueryTypesOverTime(int *sock) { int i, sendit = -1; time_t mintime = time(NULL) - config.maxlogage; - for(i = 0; i < counters.overTime; i++) + for(i = 0; i < counters->overTime; i++) { validate_access("overTime", i, true, __LINE__, __FUNCTION__, __FILE__); if((overTime[i].total > 0 || overTime[i].blocked > 0) && overTime[i].timestamp >= mintime) @@ -906,7 +906,7 @@ void getQueryTypesOverTime(int *sock) if(sendit > -1) { - for(i = sendit; i < counters.overTime; i++) + for(i = sendit; i < counters->overTime; i++) { validate_access("overTime", i, true, __LINE__, __FUNCTION__, __FILE__); @@ -1014,7 +1014,7 @@ void getClientsOverTime(int *sock) if(config.privacylevel >= PRIVACY_HIDE_DOMAINS_CLIENTS) return; - for(i = 0; i < counters.overTime; i++) + for(i = 0; i < counters->overTime; i++) { validate_access("overTime", i, true, __LINE__, __FUNCTION__, __FILE__); if((overTime[i].total > 0 || overTime[i].blocked > 0) && @@ -1032,14 +1032,14 @@ void getClientsOverTime(int *sock) // Array of clients to be skipped in the output // if skipclient[i] == true then this client should be hidden from // returned data. We initialize it with false - bool skipclient[counters.clients]; - memset(skipclient, false, counters.clients*sizeof(bool)); + bool skipclient[counters->clients]; + memset(skipclient, false, counters->clients*sizeof(bool)); if(excludeclients != NULL) { getSetupVarsArray(excludeclients); - for(i=0; i < counters.clients; i++) + for(i=0; i < counters->clients; i++) { validate_access("clients", i, true, __LINE__, __FUNCTION__, __FILE__); // Check if this client should be skipped @@ -1050,7 +1050,7 @@ void getClientsOverTime(int *sock) } // Main return loop - for(i = sendit; i < counters.overTime; i++) + for(i = sendit; i < counters->overTime; i++) { validate_access("overTime", i, true, __LINE__, __FUNCTION__, __FILE__); @@ -1061,7 +1061,7 @@ void getClientsOverTime(int *sock) // Loop over forward destinations to generate output to be sent to the client int j; - for(j = 0; j < counters.clients; j++) + for(j = 0; j < counters->clients; j++) { int thisclient = 0; @@ -1105,14 +1105,14 @@ void getClientNames(int *sock) // Array of clients to be skipped in the output // if skipclient[i] == true then this client should be hidden from // returned data. We initialize it with false - bool skipclient[counters.clients]; - memset(skipclient, false, counters.clients*sizeof(bool)); + bool skipclient[counters->clients]; + memset(skipclient, false, counters->clients*sizeof(bool)); if(excludeclients != NULL) { getSetupVarsArray(excludeclients); - for(i=0; i < counters.clients; i++) + for(i=0; i < counters->clients; i++) { validate_access("clients", i, true, __LINE__, __FUNCTION__, __FILE__); // Check if this client should be skipped @@ -1123,7 +1123,7 @@ void getClientNames(int *sock) } // Loop over clients to generate output to be sent to the client - for(i = 0; i < counters.clients; i++) + for(i = 0; i < counters->clients; i++) { validate_access("clients", i, true, __LINE__, __FUNCTION__, __FILE__); if(skipclient[i]) @@ -1152,7 +1152,7 @@ void getUnknownQueries(int *sock) return; int i; - for(i=0; i < counters.queries; i++) + for(i=0; i < counters->queries; i++) { validate_access("queries", i, true, __LINE__, __FUNCTION__, __FILE__); if(queries[i].status != QUERY_UNKNOWN && queries[i].complete) continue; @@ -1204,7 +1204,7 @@ void getDomainDetails(char *client_message, int *sock) } int i; - for(i = 0; i < counters.domains; i++) + for(i = 0; i < counters->domains; i++) { validate_access("domains", i, true, __LINE__, __FUNCTION__, __FILE__); if(strcmp(getstr(domains[i].domainpos), domain) == 0) diff --git a/database.c b/database.c index 59cefa3e4..73b292691 100644 --- a/database.c +++ b/database.c @@ -375,7 +375,7 @@ void save_to_DB(void) int total = 0, blocked = 0; time_t currenttimestamp = time(NULL); time_t newlasttimestamp = 0; - for(i = 0; i < counters.queries; i++) + for(i = 0; i < counters->queries; i++) { validate_access("queries", i, true, __LINE__, __FUNCTION__, __FILE__); if(queries[i].db) @@ -621,7 +621,7 @@ void read_data_from_DB(void) memory_check(QUERIES); // Set ID for this query - int queryID = counters.queries; + int queryID = counters->queries; int queryTimeStamp = sqlite3_column_int(stmt, 1); // 1483228800 = 01/01/2017 @ 12:00am (UTC) @@ -719,7 +719,7 @@ void read_data_from_DB(void) // Handle type counters if(type >= TYPE_A && type < TYPE_MAX) { - counters.querytype[type-1]++; + counters->querytype[type-1]++; overTime[timeidx].querytypedata[type-1]++; } @@ -731,32 +731,32 @@ void read_data_from_DB(void) overTime[timeidx].clientdata[clientID]++; // Increase DNS queries counter - counters.queries++; + counters->queries++; // Increment status counters switch(status) { case QUERY_UNKNOWN: // Unknown - counters.unknown++; + counters->unknown++; break; case QUERY_GRAVITY: // Blocked by gravity.list case QUERY_WILDCARD: // Blocked by regex filter case QUERY_BLACKLIST: // Blocked by black.list case QUERY_EXTERNAL_BLOCKED: // Blocked by external provider - counters.blocked++; + counters->blocked++; overTime[timeidx].blocked++; domains[domainID].blockedcount++; clients[clientID].blockedcount++; break; case QUERY_FORWARDED: // Forwarded - counters.forwardedqueries++; + counters->forwardedqueries++; // Update overTime data structure break; case QUERY_CACHE: // Cached or local config - counters.cached++; + counters->cached++; // Update overTime data structure overTime[timeidx].cached++; break; @@ -768,7 +768,7 @@ void read_data_from_DB(void) break; } } - logg("Imported %i queries from the long-term database", counters.queries); + logg("Imported %i queries from the long-term database", counters->queries); if( rc != SQLITE_DONE ){ logg("read_data_from_DB() - SQL error step (%i): %s", rc, sqlite3_errmsg(db)); diff --git a/datastructure.c b/datastructure.c index e3d6b0a5d..1528f8545 100644 --- a/datastructure.c +++ b/datastructure.c @@ -32,19 +32,19 @@ int findOverTimeID(int overTimetimestamp) int timeidx = -1, i; // Check struct size memory_check(OVERTIME); - if(counters.overTime > 0) - validate_access("overTime", counters.overTime-1, true, __LINE__, __FUNCTION__, __FILE__); - for(i=0; i < counters.overTime; i++) + if(counters->overTime > 0) + validate_access("overTime", counters->overTime-1, true, __LINE__, __FUNCTION__, __FILE__); + for(i=0; i < counters->overTime; i++) { if(overTime[i].timestamp == overTimetimestamp) return i; } // We loop over this to fill potential data holes with zeros int nexttimestamp = 0; - if(counters.overTime != 0) + if(counters->overTime != 0) { - validate_access("overTime", counters.overTime-1, false, __LINE__, __FUNCTION__, __FILE__); - nexttimestamp = overTime[counters.overTime-1].timestamp + 600; + validate_access("overTime", counters->overTime-1, false, __LINE__, __FUNCTION__, __FILE__); + nexttimestamp = overTime[counters->overTime-1].timestamp + 600; } else { @@ -57,7 +57,7 @@ int findOverTimeID(int overTimetimestamp) { // Check struct size memory_check(OVERTIME); - timeidx = counters.overTime; + timeidx = counters->overTime; validate_access("overTime", timeidx, false, __LINE__, __FUNCTION__, __FILE__); // Set magic byte overTime[timeidx].magic = MAGICBYTE; @@ -68,13 +68,13 @@ int findOverTimeID(int overTimetimestamp) // overTime[timeidx].querytypedata is static overTime[timeidx].clientnum = 0; overTime[timeidx].clientdata = NULL; - counters.overTime++; + counters->overTime++; // Update time stamp for next loop interation - if(counters.overTime != 0) + if(counters->overTime != 0) { - validate_access("overTime", counters.overTime-1, false, __LINE__, __FUNCTION__, __FILE__); - nexttimestamp = overTime[counters.overTime-1].timestamp + 600; + validate_access("overTime", counters->overTime-1, false, __LINE__, __FUNCTION__, __FILE__); + nexttimestamp = overTime[counters->overTime-1].timestamp + 600; } } return timeidx; @@ -83,10 +83,10 @@ int findOverTimeID(int overTimetimestamp) int findForwardID(const char * forward, bool count) { int i, forwardID = -1; - if(counters.forwarded > 0) - validate_access("forwarded", counters.forwarded-1, true, __LINE__, __FUNCTION__, __FILE__); + if(counters->forwarded > 0) + validate_access("forwarded", counters->forwarded-1, true, __LINE__, __FUNCTION__, __FILE__); // Go through already knows forward servers and see if we used one of those - for(i=0; i < counters.forwarded; i++) + for(i=0; i < counters->forwarded; i++) { if(strcmp(getstr(forwarded[i].ippos), forward) == 0) { @@ -97,8 +97,8 @@ int findForwardID(const char * forward, bool count) } // This forward server is not known // Store ID - forwardID = counters.forwarded; - logg("New forward server: %s (%i/%u)", forward, forwardID, counters.forwarded_MAX); + forwardID = counters->forwarded; + logg("New forward server: %s (%i/%u)", forward, forwardID, counters->forwarded_MAX); // Check struct size memory_check(FORWARDED); @@ -121,7 +121,7 @@ int findForwardID(const char * forward, bool count) forwarded[forwardID].new = true; forwarded[forwardID].namepos = 0; // 0 -> string with length zero // Increase counter by one - counters.forwarded++; + counters->forwarded++; return forwardID; } @@ -129,9 +129,9 @@ int findForwardID(const char * forward, bool count) int findDomainID(const char *domain) { int i; - if(counters.domains > 0) - validate_access("domains", counters.domains-1, true, __LINE__, __FUNCTION__, __FILE__); - for(i=0; i < counters.domains; i++) + if(counters->domains > 0) + validate_access("domains", counters->domains-1, true, __LINE__, __FUNCTION__, __FILE__); + for(i=0; i < counters->domains; i++) { // Quick test: Does the domain start with the same character? if(getstr(domains[i].domainpos)[0] != domain[0]) @@ -147,7 +147,7 @@ int findDomainID(const char *domain) // If we did not return until here, then this domain is not known // Store ID - int domainID = counters.domains; + int domainID = counters->domains; // Check struct size memory_check(DOMAINS); @@ -164,7 +164,7 @@ int findDomainID(const char *domain) // RegEx needs to be evaluated for this new domain domains[domainID].regexmatch = REGEX_UNKNOWN; // Increase counter by one - counters.domains++; + counters->domains++; return domainID; } @@ -173,9 +173,9 @@ int findClientID(const char *client) { int i; // Compare content of client against known client IP addresses - if(counters.clients > 0) - validate_access("clients", counters.clients-1, true, __LINE__, __FUNCTION__, __FILE__); - for(i=0; i < counters.clients; i++) + if(counters->clients > 0) + validate_access("clients", counters->clients-1, true, __LINE__, __FUNCTION__, __FILE__); + for(i=0; i < counters->clients; i++) { // Quick test: Does the clients IP start with the same character? if(getstr(clients[i].ippos)[0] != client[0]) @@ -191,7 +191,7 @@ int findClientID(const char *client) // If we did not return until here, then this client is definitely new // Store ID - int clientID = counters.clients; + int clientID = counters->clients; // Check struct size memory_check(CLIENTS); @@ -212,7 +212,7 @@ int findClientID(const char *client) clients[clientID].new = true; clients[clientID].namepos = 0; // Increase counter by one - counters.clients++; + counters->clients++; return clientID; } diff --git a/dnsmasq_interface.c b/dnsmasq_interface.c index 580229503..5658b7724 100644 --- a/dnsmasq_interface.c +++ b/dnsmasq_interface.c @@ -73,7 +73,7 @@ void FTL_new_query(unsigned int flags, char *name, struct all_addr *addr, char * // Ensure we have enough space in the queries struct memory_check(QUERIES); - int queryID = counters.queries; + int queryID = counters->queries; // Convert domain to lower case char *domain = strdup(name); @@ -133,7 +133,7 @@ void FTL_new_query(unsigned int flags, char *name, struct all_addr *addr, char * int timeidx = findOverTimeID(overTimetimestamp); validate_access("overTime", timeidx, true, __LINE__, __FUNCTION__, __FILE__); overTime[timeidx].querytypedata[querytype-1]++; - counters.querytype[querytype-1]++; + counters->querytype[querytype-1]++; // Skip rest of the analysis if this query is not of type A or AAAA // but user wants to see only A and AAAA queries (pre-v4.1 behavior) @@ -176,10 +176,10 @@ void FTL_new_query(unsigned int flags, char *name, struct all_addr *addr, char * queries[queryID].AD = false; // Increase DNS queries counter - counters.queries++; + counters->queries++; // Count this query as unknown as long as no reply has // been found and analyzed - counters.unknown++; + counters->unknown++; // Update overTime data validate_access("overTime", timeidx, true, __LINE__, __FUNCTION__, __FILE__); @@ -237,11 +237,11 @@ static int findQueryID(int id) // MAX(0, a) is used to return 0 in case a is negative (negative array indices are harmful) // Validate access only once for the maximum index (all lower will work) - validate_access("queries", counters.queries-1, false, __LINE__, __FUNCTION__, __FILE__); - int until = MAX(0, counters.queries-MAXITER); + validate_access("queries", counters->queries-1, false, __LINE__, __FUNCTION__, __FILE__); + int until = MAX(0, counters->queries-MAXITER); int i; // Check UUIDs of queries - for(i = counters.queries-1; i >= until; i--) + for(i = counters->queries-1; i >= until; i--) if(queries[i].id == id) return i; @@ -323,7 +323,7 @@ void FTL_forwarded(unsigned int flags, char *name, struct all_addr *addr, int id // This code section acknowledges this by removing one entry from // the cached counters as we will re-brand this query as having been // forwarded in the following. - counters.cached--; + counters->cached--; // Also correct overTime data overTime[j].cached--; @@ -338,7 +338,7 @@ void FTL_forwarded(unsigned int flags, char *name, struct all_addr *addr, int id { // Normal cache reply // Query is no longer unknown - counters.unknown--; + counters->unknown--; // Hereby, this query is now fully determined queries[i].complete = true; } @@ -346,7 +346,7 @@ void FTL_forwarded(unsigned int flags, char *name, struct all_addr *addr, int id overTime[j].forwarded++; // Update couter for forwarded queries - counters.forwardedqueries++; + counters->forwardedqueries++; } // Release allocated memory @@ -362,7 +362,7 @@ void FTL_dnsmasq_reload(void) // Called when dnsmasq re-reads its config and hosts files // Reset number of blocked domains - counters.gravity = 0; + counters->gravity = 0; // Inspect 01-pihole.conf to see if Pi-hole blocking is enabled, // i.e. if /etc/pihole/gravity.list is sourced as addn-hosts file @@ -436,9 +436,9 @@ void FTL_reply(unsigned short flags, char *name, struct all_addr *addr, int id) { // Answered from local configuration, might be a wildcard or user-provided // This query is no longer unknown - counters.unknown--; + counters->unknown--; // Answered from a custom (user provided) cache file - counters.cached++; + counters->cached++; // Detect user-defined blocking rules if(strcmp(answer, "(NXDOMAIN)") == 0 || @@ -524,13 +524,13 @@ void FTL_reply(unsigned short flags, char *name, struct all_addr *addr, int id) static void query_externally_blocked(int i) { // Correct counters as we won't count this as forwarded ... - counters.forwardedqueries--; + counters->forwardedqueries--; overTime[queries[i].timeidx].forwarded--; validate_access("forwarded", queries[i].forwardID, true, __LINE__, __FUNCTION__, __FILE__); forwarded[queries[i].forwardID].count--; // ... but as blocked - counters.blocked++; + counters->blocked++; overTime[queries[i].timeidx].blocked++; validate_access("domains", queries[i].domainID, true, __LINE__, __FUNCTION__, __FILE__); domains[queries[i].domainID].blockedcount++; @@ -627,7 +627,7 @@ void FTL_cache(unsigned int flags, char *name, struct all_addr *addr, char *arg, if(!queries[i].complete) { // This query is no longer unknown - counters.unknown--; + counters->unknown--; // Get time index int querytimestamp, overTimetimestamp; @@ -653,13 +653,13 @@ void FTL_cache(unsigned int flags, char *name, struct all_addr *addr, char *arg, case QUERY_GRAVITY: // gravity.list case QUERY_BLACKLIST: // black.list case QUERY_WILDCARD: // regex blocked - counters.blocked++; + counters->blocked++; overTime[timeidx].blocked++; domains[domainID].blockedcount++; clients[clientID].blockedcount++; break; case QUERY_CACHE: // cached from one of the lists - counters.cached++; + counters->cached++; overTime[timeidx].cached++; break; } @@ -768,26 +768,26 @@ void save_reply_type(unsigned int flags, int queryID, struct timeval response) { // NXDOMAIN queries[queryID].reply = REPLY_NXDOMAIN; - counters.reply_NXDOMAIN++; + counters->reply_NXDOMAIN++; } else { // NODATA(-IPv6) queries[queryID].reply = REPLY_NODATA; - counters.reply_NODATA++; + counters->reply_NODATA++; } } else if(flags & F_CNAME) { // queries[queryID].reply = REPLY_CNAME; - counters.reply_CNAME++; + counters->reply_CNAME++; } else if(flags & F_REVERSE) { // reserve lookup queries[queryID].reply = REPLY_DOMAIN; - counters.reply_domain++; + counters->reply_domain++; } else if(flags & F_RRNAME) { @@ -798,7 +798,7 @@ void save_reply_type(unsigned int flags, int queryID, struct timeval response) { // Valid IP queries[queryID].reply = REPLY_IP; - counters.reply_IP++; + counters->reply_IP++; } // Save response time (relative time) @@ -1111,6 +1111,6 @@ int FTL_listsfile(char* filename, unsigned int index, FILE *f, int cache_size, s } logg("%s: parsed %i domains (took %.1f ms)", filename, added, timer_elapsed_msec(LISTS_TIMER)); - counters.gravity += added; + counters->gravity += added; return name_count; } diff --git a/gc.c b/gc.c index 658d7096d..98374489a 100644 --- a/gc.c +++ b/gc.c @@ -42,7 +42,7 @@ void *GC_thread(void *val) if(debug) logg("GC starting, mintime: %u %s", mintime, ctime(&mintime)); // Process all queries - for(i=0; i < counters.queries; i++) + for(i=0; i < counters->queries; i++) { validate_access("queries", i, true, __LINE__, __FUNCTION__, __FILE__); // Test if this query is too new @@ -51,7 +51,7 @@ void *GC_thread(void *val) // Adjust total counters and total over time data - // We cannot edit counters.queries directly as it is used + // We cannot edit counters->queries directly as it is used // as max ID for the queries[] struct int timeidx = queries[i].timeidx; validate_access("overTime", timeidx, true, __LINE__, __FUNCTION__, __FILE__); @@ -76,31 +76,31 @@ void *GC_thread(void *val) { case QUERY_UNKNOWN: // Unknown (?) - counters.unknown--; + counters->unknown--; break; case QUERY_GRAVITY: // Blocked by Pi-hole's blocking lists - counters.blocked--; + counters->blocked--; overTime[timeidx].blocked--; domains[domainID].blockedcount--; clients[clientID].blockedcount--; break; case QUERY_FORWARDED: // Forwarded to an upstream DNS server - counters.forwardedqueries--; + counters->forwardedqueries--; overTime[timeidx].forwarded--; validate_access("forwarded", queries[i].forwardID, true, __LINE__, __FUNCTION__, __FILE__); forwarded[queries[i].forwardID].count--; break; case QUERY_CACHE: // Answered from local cache _or_ local config - counters.cached--; + counters->cached--; overTime[timeidx].cached--; break; case QUERY_BLACKLIST: // exact blocked case QUERY_WILDCARD: // regex blocked (fall through) case QUERY_EXTERNAL_BLOCKED: // blocked by upstream provider (fall through) - counters.blocked--; + counters->blocked--; overTime[timeidx].blocked--; domains[domainID].blockedcount--; clients[clientID].blockedcount--; @@ -114,23 +114,23 @@ void *GC_thread(void *val) switch(queries[i].reply) { case REPLY_NODATA: // NODATA(-IPv6) - counters.reply_NODATA--; + counters->reply_NODATA--; break; case REPLY_NXDOMAIN: // NXDOMAIN - counters.reply_NXDOMAIN--; + counters->reply_NXDOMAIN--; break; case REPLY_CNAME: // - counters.reply_CNAME--; + counters->reply_CNAME--; break; case REPLY_IP: // valid IP - counters.reply_IP--; + counters->reply_IP--; break; case REPLY_DOMAIN: // reverse lookup - counters.reply_domain--; + counters->reply_domain--; break; default: // Incomplete query or TXT, do nothing @@ -140,7 +140,7 @@ void *GC_thread(void *val) // Update type counters if(queries[i].type >= TYPE_A && queries[i].type < TYPE_MAX) { - counters.querytype[queries[i].type-1]--; + counters->querytype[queries[i].type-1]--; validate_access("overTime", queries[i].timeidx, true, __LINE__, __FUNCTION__, __FILE__); overTime[queries[i].timeidx].querytypedata[queries[i].type-1]--; } @@ -155,13 +155,13 @@ void *GC_thread(void *val) // Example: (I = now invalid, X = still valid queries, F = free space) // Before: IIIIIIXXXXFF // After: XXXXFFFFFFFF - memmove(&queries[0], &queries[removed], (counters.queries - removed)*sizeof(*queries)); + memmove(&queries[0], &queries[removed], (counters->queries - removed)*sizeof(*queries)); // Update queries counter - counters.queries -= removed; + counters->queries -= removed; // Zero out remaining memory (marked as "F" in the above example) - memset(&queries[counters.queries], 0, (counters.queries_MAX - counters.queries)*sizeof(*queries)); + memset(&queries[counters->queries], 0, (counters->queries_MAX - counters->queries)*sizeof(*queries)); if(debug) logg("Notice: GC removed %i queries (took %.2f ms)", removed, timer_elapsed_msec(GC_TIMER)); diff --git a/log.c b/log.c index 63c4b8278..348239cea 100644 --- a/log.c +++ b/log.c @@ -121,14 +121,14 @@ void logg_struct_resize(const char* str, int to, int step) void log_counter_info(void) { - logg(" -> Total DNS queries: %i", counters.queries); - logg(" -> Cached DNS queries: %i", counters.cached); - logg(" -> Forwarded DNS queries: %i", counters.forwardedqueries); - logg(" -> Exactly blocked DNS queries: %i", counters.blocked); - logg(" -> Unknown DNS queries: %i", counters.unknown); - logg(" -> Unique domains: %i", counters.domains); - logg(" -> Unique clients: %i", counters.clients); - logg(" -> Known forward destinations: %i", counters.forwarded); + logg(" -> Total DNS queries: %i", counters->queries); + logg(" -> Cached DNS queries: %i", counters->cached); + logg(" -> Forwarded DNS queries: %i", counters->forwardedqueries); + logg(" -> Exactly blocked DNS queries: %i", counters->blocked); + logg(" -> Unknown DNS queries: %i", counters->unknown); + logg(" -> Unique domains: %i", counters->domains); + logg(" -> Unique clients: %i", counters->clients); + logg(" -> Known forward destinations: %i", counters->forwarded); } void log_FTL_version(void) diff --git a/memory.c b/memory.c index 81265f49f..52b3b6693 100644 --- a/memory.c +++ b/memory.c @@ -33,7 +33,7 @@ logFileNamesStruct files = { }; // Fixed size structs -countersStruct counters = { 0 }; +countersStruct *counters = NULL; ConfigStruct config; // Variable size array structs @@ -48,7 +48,7 @@ void memory_check(int which) switch(which) { case QUERIES: - if(counters.queries >= counters.queries_MAX-1) + if(counters->queries >= counters->queries_MAX-1) { // Have to reallocate shared memory queries = enlarge_shmem_struct('q'); @@ -60,7 +60,7 @@ void memory_check(int which) } break; case FORWARDED: - if(counters.forwarded >= counters.forwarded_MAX-1) + if(counters->forwarded >= counters->forwarded_MAX-1) { // Have to reallocate shared memory forwarded = enlarge_shmem_struct('f'); @@ -72,7 +72,7 @@ void memory_check(int which) } break; case CLIENTS: - if(counters.clients >= counters.clients_MAX-1) + if(counters->clients >= counters->clients_MAX-1) { // Have to reallocate shared memory clients = enlarge_shmem_struct('c'); @@ -84,7 +84,7 @@ void memory_check(int which) } break; case DOMAINS: - if(counters.domains >= counters.domains_MAX-1) + if(counters->domains >= counters->domains_MAX-1) { // Have to reallocate shared memory domains = enlarge_shmem_struct('d'); @@ -96,12 +96,12 @@ void memory_check(int which) } break; case OVERTIME: - if(counters.overTime >= counters.overTime_MAX-1) + if(counters->overTime >= counters->overTime_MAX-1) { // Have to reallocate memory - counters.overTime_MAX += OVERTIMEALLOCSTEP; - logg_struct_resize("overTime",counters.overTime_MAX,OVERTIMEALLOCSTEP); - overTime = realloc(overTime, counters.overTime_MAX*sizeof(overTimeDataStruct)); + counters->overTime_MAX += OVERTIMEALLOCSTEP; + logg_struct_resize("overTime",counters->overTime_MAX,OVERTIMEALLOCSTEP); + overTime = realloc(overTime, counters->overTime_MAX*sizeof(overTimeDataStruct)); if(overTime == NULL) { logg("FATAL: Memory allocation failed! Exiting"); @@ -120,11 +120,11 @@ void memory_check(int which) void validate_access(const char * name, int pos, bool testmagic, int line, const char * function, const char * file) { int limit = 0; - if(name[0] == 'c') limit = counters.clients_MAX; - else if(name[0] == 'd') limit = counters.domains_MAX; - else if(name[0] == 'q') limit = counters.queries_MAX; - else if(name[0] == 'o') limit = counters.overTime_MAX; - else if(name[0] == 'f') limit = counters.forwarded_MAX; + if(name[0] == 'c') limit = counters->clients_MAX; + else if(name[0] == 'd') limit = counters->domains_MAX; + else if(name[0] == 'q') limit = counters->queries_MAX; + else if(name[0] == 'o') limit = counters->overTime_MAX; + else if(name[0] == 'f') limit = counters->forwarded_MAX; else { logg("Validator error (range)"); killed = 1; } if(pos >= limit || pos < 0) diff --git a/regex.c b/regex.c index c29855830..6a0a1ef23 100644 --- a/regex.c +++ b/regex.c @@ -149,9 +149,9 @@ void free_regex(void) // Must reevaluate regex filters after having reread the regex filter // We reset all regex status to unknown to have them being reevaluated - if(counters.domains > 0) - validate_access("domains", counters.domains-1, false, __LINE__, __FUNCTION__, __FILE__); - for(int i=0; i < counters.domains; i++) + if(counters->domains > 0) + validate_access("domains", counters->domains-1, false, __LINE__, __FUNCTION__, __FILE__); + for(int i=0; i < counters->domains; i++) { domains[i].regexmatch = REGEX_UNKNOWN; } diff --git a/resolve.c b/resolve.c index 01b9670b6..175c890fc 100644 --- a/resolve.c +++ b/resolve.c @@ -67,7 +67,7 @@ char *resolveHostname(const char *addr) void reresolveHostnames(void) { int clientID; - for(clientID = 0; clientID < counters.clients; clientID++) + for(clientID = 0; clientID < counters->clients; clientID++) { // Memory validation validate_access("clients", clientID, true, __LINE__, __FUNCTION__, __FILE__); @@ -93,7 +93,7 @@ void reresolveHostnames(void) void resolveNewClients(void) { int i; - for(i = 0; i < counters.clients; i++) + for(i = 0; i < counters->clients; i++) { // Memory validation validate_access("clients", i, true, __LINE__, __FUNCTION__, __FILE__); @@ -107,7 +107,7 @@ void resolveNewClients(void) clients[i].new = false; } } - for(i = 0; i < counters.forwarded; i++) + for(i = 0; i < counters->forwarded; i++) { // Memory validation validate_access("forwarded", i, true, __LINE__, __FUNCTION__, __FILE__); diff --git a/shmem.c b/shmem.c index a71b20109..b28a6834f 100644 --- a/shmem.c +++ b/shmem.c @@ -13,6 +13,7 @@ /// The name of the shared memory. Use this when connecting to the shared memory. #define SHARED_STRINGS_NAME "/FTL-strings" +#define SHARED_COUNTERS_NAME "/FTL-counters" #define SHARED_DOMAINS_NAME "/FTL-domains" #define SHARED_CLIENTS_NAME "/FTL-clients" #define SHARED_QUERIES_NAME "/FTL-queries" @@ -20,6 +21,7 @@ /// The pointer in shared memory to the shared string buffer static SharedMemory shm_strings = { 0 }; +static SharedMemory shm_counters = { 0 }; static SharedMemory shm_domains = { 0 }; static SharedMemory shm_clients = { 0 }; static SharedMemory shm_queries = { 0 }; @@ -83,6 +85,14 @@ bool init_shmem(void) ((char*)shm_strings.ptr)[0] = '\0'; next_pos = 1; + /****************************** shared counters struct ******************************/ + shm_unlink(SHARED_COUNTERS_NAME); + // Try to create shared memory object + shm_counters = create_shm(SHARED_COUNTERS_NAME, sizeof(countersStruct)); + if(shm_counters.ptr == NULL) + return false; + counters = (countersStruct*)shm_counters.ptr; + /****************************** shared domains struct ******************************/ shm_unlink(SHARED_DOMAINS_NAME); // Try to create shared memory object @@ -90,7 +100,7 @@ bool init_shmem(void) if(shm_domains.ptr == NULL) return false; domains = (domainsDataStruct*)shm_domains.ptr; - counters.domains_MAX = pagesize; + counters->domains_MAX = pagesize; /****************************** shared clients struct ******************************/ shm_unlink(SHARED_CLIENTS_NAME); @@ -99,7 +109,7 @@ bool init_shmem(void) if(shm_clients.ptr == NULL) return false; clients = (clientsDataStruct*)shm_clients.ptr; - counters.clients_MAX = pagesize; + counters->clients_MAX = pagesize; /****************************** shared forwarded struct ******************************/ shm_unlink(SHARED_FORWARDED_NAME); @@ -108,7 +118,7 @@ bool init_shmem(void) if(shm_forwarded.ptr == NULL) return false; forwarded = (forwardedDataStruct*)shm_forwarded.ptr; - counters.forwarded_MAX = pagesize; + counters->forwarded_MAX = pagesize; /****************************** shared queries struct ******************************/ shm_unlink(SHARED_QUERIES_NAME); @@ -117,7 +127,7 @@ bool init_shmem(void) if(shm_queries.ptr == NULL) return false; queries = (queriesDataStruct*)shm_queries.ptr; - counters.queries_MAX = pagesize; + counters->queries_MAX = pagesize; return true; } @@ -125,6 +135,7 @@ bool init_shmem(void) void destroy_shmem(void) { delete_shm(&shm_strings); + delete_shm(&shm_counters); delete_shm(&shm_domains); delete_shm(&shm_clients); delete_shm(&shm_queries); @@ -194,22 +205,22 @@ void *enlarge_shmem_struct(char type) case 'q': sharedMemory = &shm_queries; sizeofobj = sizeof(queriesDataStruct); - counter = &counters.queries_MAX; + counter = &counters->queries_MAX; break; case 'c': sharedMemory = &shm_clients; sizeofobj = sizeof(clientsDataStruct); - counter = &counters.clients_MAX; + counter = &counters->clients_MAX; break; case 'd': sharedMemory = &shm_domains; sizeofobj = sizeof(domainsDataStruct); - counter = &counters.domains_MAX; + counter = &counters->domains_MAX; break; case 'f': sharedMemory = &shm_forwarded; sizeofobj = sizeof(forwardedDataStruct); - counter = &counters.forwarded_MAX; + counter = &counters->forwarded_MAX; break; default: logg("Invalid argument in enlarge_shmem_struct(): %c (%i)", type, type); From e1c8ccc97523f9e38ddb57beef00153d2a59958f Mon Sep 17 00:00:00 2001 From: Mcat12 Date: Tue, 18 Sep 2018 11:40:44 -0400 Subject: [PATCH 29/48] Use negative query IDs for queries loaded from the database This is necessary for all queries to have unique IDs. Previously, all queries loaded from the database had an ID of zero. Signed-off-by: Mcat12 --- database.c | 44 ++++++++++++++++++++++++-------------------- 1 file changed, 24 insertions(+), 20 deletions(-) diff --git a/database.c b/database.c index 73b292691..3ad3f68a1 100644 --- a/database.c +++ b/database.c @@ -617,12 +617,6 @@ void read_data_from_DB(void) // Loop through returned database rows while((rc = sqlite3_step(stmt)) == SQLITE_ROW) { - // Ensure we have enough space in the queries struct - memory_check(QUERIES); - - // Set ID for this query - int queryID = counters->queries; - int queryTimeStamp = sqlite3_column_int(stmt, 1); // 1483228800 = 01/01/2017 @ 12:00am (UTC) if(queryTimeStamp < 1483228800) @@ -699,21 +693,31 @@ void read_data_from_DB(void) int timeidx = findOverTimeID(overTimeTimeStamp); validate_access("overTime", timeidx, true, __LINE__, __FUNCTION__, __FILE__); + // Ensure we have enough space in the queries struct + memory_check(QUERIES); + + // Set index for this query + int queryIndex = counters->queries; + + // Set the ID for this query. Queries loaded from the database use negative IDs. + // 1 is added to the counter before flipping the sign so the IDs start at -1 instead of 0. + int queryID = -1 * (counters->queries + 1); + // Store this query in memory - validate_access("queries", queryID, false, __LINE__, __FUNCTION__, __FILE__); - queries[queryID].magic = MAGICBYTE; - queries[queryID].timestamp = queryTimeStamp; - queries[queryID].type = type; - queries[queryID].status = status; - queries[queryID].domainID = domainID; - queries[queryID].clientID = clientID; - queries[queryID].forwardID = forwardID; - queries[queryID].timeidx = timeidx; - queries[queryID].db = true; // Mark this as already present in the database - queries[queryID].id = 0; // This is dnsmasq's internal ID. We don't store it in the database - queries[queryID].complete = true; // Mark as all information is avaiable - queries[queryID].response = 0; - queries[queryID].AD = false; + validate_access("queries", queryIndex, false, __LINE__, __FUNCTION__, __FILE__); + queries[queryIndex].magic = MAGICBYTE; + queries[queryIndex].timestamp = queryTimeStamp; + queries[queryIndex].type = type; + queries[queryIndex].status = status; + queries[queryIndex].domainID = domainID; + queries[queryIndex].clientID = clientID; + queries[queryIndex].forwardID = forwardID; + queries[queryIndex].timeidx = timeidx; + queries[queryIndex].db = true; // Mark this as already present in the database + queries[queryIndex].id = queryID; + queries[queryIndex].complete = true; // Mark as all information is avaiable + queries[queryIndex].response = 0; + queries[queryIndex].AD = false; lastDBimportedtimestamp = queryTimeStamp; // Handle type counters From cd41aaeab49b598adc8bd508f76374e06e768901 Mon Sep 17 00:00:00 2001 From: Mcat12 Date: Wed, 19 Sep 2018 13:00:50 -0400 Subject: [PATCH 30/48] Put overTime data into shared memory The array of overTime structs is separate from the client data, because the client data is dynamically sized. Each client has a separate shared memory block named `/FTL-client-ID` where `ID` is the numerical ID of the client. The client data is an array of ints, which is indexed using the overTime time indexes. Before, the code referenced the client data via `overTime[timeIndex].clientdata[clientID]` Now, the code references the data via `overTimeClientData[clientID][timeIndex]` The client data is updated whenever a new client or overTime slot is added, so referencing the client data should always be valid. Signed-off-by: Mcat12 --- FTL.h | 8 +++-- api.c | 9 +---- database.c | 12 +++---- datastructure.c | 10 ++++-- dnsmasq_interface.c | 3 +- gc.c | 3 +- memory.c | 47 ++++--------------------- routines.h | 13 ++++++- shmem.c | 84 ++++++++++++++++++++++++++++++++++++++++++--- 9 files changed, 118 insertions(+), 71 deletions(-) diff --git a/FTL.h b/FTL.h index 7c7c096f8..f6766bf36 100644 --- a/FTL.h +++ b/FTL.h @@ -203,9 +203,7 @@ typedef struct { int blocked; int cached; int forwarded; - int clientnum; - int *clientdata; - int querytypedata[7]; + int querytypedata[TYPE_MAX-1]; } overTimeDataStruct; typedef struct { @@ -230,6 +228,10 @@ extern clientsDataStruct *clients; extern domainsDataStruct *domains; extern overTimeDataStruct *overTime; +/// Indexed by client ID, then time index (like `overTime`). +/// This gets automatically updated whenever a new client or overTime slot is added. +extern int **overTimeClientData; + extern FILE *logfile; extern volatile sig_atomic_t killed; diff --git a/api.c b/api.c index 377193c35..e19292c60 100644 --- a/api.c +++ b/api.c @@ -1063,17 +1063,10 @@ void getClientsOverTime(int *sock) int j; for(j = 0; j < counters->clients; j++) { - int thisclient = 0; - if(skipclient[j]) continue; - if(j < overTime[i].clientnum) - { - // This client entry does already exist at this timestamp - // -> use counter of requests sent to this destination - thisclient = overTime[i].clientdata[j]; - } + int thisclient = overTimeClientData[j][i]; if(istelnet[*sock]) ssend(*sock, " %i", thisclient); diff --git a/database.c b/database.c index 3ad3f68a1..66b24aea4 100644 --- a/database.c +++ b/database.c @@ -671,10 +671,6 @@ void read_data_from_DB(void) continue; } - // Obtain IDs only after filtering which queries we want to keep - int domainID = findDomainID(domain); - int clientID = findClientID(client); - const char *forwarddest = (const char *)sqlite3_column_text(stmt, 6); int forwardID = 0; // Determine forwardID only when status == 2 (forwarded) as the @@ -689,9 +685,11 @@ void read_data_from_DB(void) forwardID = findForwardID(forwarddest, true); } + // Obtain IDs only after filtering which queries we want to keep int overTimeTimeStamp = queryTimeStamp - (queryTimeStamp % 600) + 300; int timeidx = findOverTimeID(overTimeTimeStamp); - validate_access("overTime", timeidx, true, __LINE__, __FUNCTION__, __FILE__); + int domainID = findDomainID(domain); + int clientID = findClientID(client); // Ensure we have enough space in the queries struct memory_check(QUERIES); @@ -704,6 +702,7 @@ void read_data_from_DB(void) int queryID = -1 * (counters->queries + 1); // Store this query in memory + validate_access("overTime", timeidx, true, __LINE__, __FUNCTION__, __FILE__); validate_access("queries", queryIndex, false, __LINE__, __FUNCTION__, __FILE__); queries[queryIndex].magic = MAGICBYTE; queries[queryIndex].timestamp = queryTimeStamp; @@ -731,8 +730,7 @@ void read_data_from_DB(void) overTime[timeidx].total++; // Update overTime data structure with the new client - validate_access_oTcl(timeidx, clientID, __LINE__, __FUNCTION__, __FILE__); - overTime[timeidx].clientdata[clientID]++; + overTimeClientData[clientID][timeidx]++; // Increase DNS queries counter counters->queries++; diff --git a/datastructure.c b/datastructure.c index 1528f8545..41fad9d17 100644 --- a/datastructure.c +++ b/datastructure.c @@ -66,11 +66,12 @@ int findOverTimeID(int overTimetimestamp) overTime[timeidx].blocked = 0; overTime[timeidx].cached = 0; // overTime[timeidx].querytypedata is static - overTime[timeidx].clientnum = 0; - overTime[timeidx].clientdata = NULL; counters->overTime++; - // Update time stamp for next loop interation + // Create new overTime slot in client shared memory + addOverTimeClientSlot(); + + // Update time stamp for next loop interaction if(counters->overTime != 0) { validate_access("overTime", counters->overTime-1, false, __LINE__, __FUNCTION__, __FILE__); @@ -214,6 +215,9 @@ int findClientID(const char *client) // Increase counter by one counters->clients++; + // Create new overTime client data + newOverTimeClient(); + return clientID; } diff --git a/dnsmasq_interface.c b/dnsmasq_interface.c index a1ca0d3d5..80d40ef07 100644 --- a/dnsmasq_interface.c +++ b/dnsmasq_interface.c @@ -187,8 +187,7 @@ void FTL_new_query(unsigned int flags, char *name, struct all_addr *addr, char * overTime[timeidx].total++; // Update overTime data structure with the new client - validate_access_oTcl(timeidx, clientID, __LINE__, __FUNCTION__, __FILE__); - overTime[timeidx].clientdata[clientID]++; + overTimeClientData[clientID][timeidx]++; // Try blocking regex if configured validate_access("domains", domainID, false, __LINE__, __FUNCTION__, __FILE__); diff --git a/gc.c b/gc.c index 98374489a..432f4ac4b 100644 --- a/gc.c +++ b/gc.c @@ -63,8 +63,7 @@ void *GC_thread(void *val) clients[clientID].count--; // Adjust corresponding overTime counters - validate_access_oTcl(timeidx, clientID, __LINE__, __FUNCTION__, __FILE__); - overTime[timeidx].clientdata[clientID]--; + overTimeClientData[timeidx][clientID]--; // Adjust domain counter (no overTime information) int domainID = queries[i].domainID; diff --git a/memory.c b/memory.c index 52b3b6693..6c020c928 100644 --- a/memory.c +++ b/memory.c @@ -42,6 +42,7 @@ forwardedDataStruct *forwarded = NULL; clientsDataStruct *clients = NULL; domainsDataStruct *domains = NULL; overTimeDataStruct *overTime = NULL; +int **overTimeClientData = NULL; void memory_check(int which) { @@ -51,7 +52,7 @@ void memory_check(int which) if(counters->queries >= counters->queries_MAX-1) { // Have to reallocate shared memory - queries = enlarge_shmem_struct('q'); + queries = enlarge_shmem_struct(QUERIES); if(queries == NULL) { logg("FATAL: Memory allocation failed! Exiting"); @@ -63,7 +64,7 @@ void memory_check(int which) if(counters->forwarded >= counters->forwarded_MAX-1) { // Have to reallocate shared memory - forwarded = enlarge_shmem_struct('f'); + forwarded = enlarge_shmem_struct(FORWARDED); if(forwarded == NULL) { logg("FATAL: Memory allocation failed! Exiting"); @@ -75,7 +76,7 @@ void memory_check(int which) if(counters->clients >= counters->clients_MAX-1) { // Have to reallocate shared memory - clients = enlarge_shmem_struct('c'); + clients = enlarge_shmem_struct(CLIENTS); if(clients == NULL) { logg("FATAL: Memory allocation failed! Exiting"); @@ -87,7 +88,7 @@ void memory_check(int which) if(counters->domains >= counters->domains_MAX-1) { // Have to reallocate shared memory - domains = enlarge_shmem_struct('d'); + domains = enlarge_shmem_struct(DOMAINS); if(domains == NULL) { logg("FATAL: Memory allocation failed! Exiting"); @@ -98,10 +99,8 @@ void memory_check(int which) case OVERTIME: if(counters->overTime >= counters->overTime_MAX-1) { - // Have to reallocate memory - counters->overTime_MAX += OVERTIMEALLOCSTEP; - logg_struct_resize("overTime",counters->overTime_MAX,OVERTIMEALLOCSTEP); - overTime = realloc(overTime, counters->overTime_MAX*sizeof(overTimeDataStruct)); + // Have to reallocate shared memory + overTime = enlarge_shmem_struct(OVERTIME); if(overTime == NULL) { logg("FATAL: Memory allocation failed! Exiting"); @@ -150,38 +149,6 @@ void validate_access(const char * name, int pos, bool testmagic, int line, const } } -void validate_access_oTcl(int timeidx, int clientID, int line, const char * function, const char * file) -{ - if(clientID < 0) - { - logg("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"); - logg("FATAL ERROR: Trying to access overTime.clientdata[%i]", clientID); - logg(" found in %s() (%s:%i)", function, file, line); - } - // Determine if there is enough space for saving the current - // clientID in the overTime data structure, allocate space otherwise - if(overTime[timeidx].clientnum <= clientID) - { - // Reallocate more space for clientdata - overTime[timeidx].clientdata = realloc(overTime[timeidx].clientdata, (clientID+1)*sizeof(*overTime[timeidx].clientdata)); - // Initialize new data fields with zeroes - int i; - for(i = overTime[timeidx].clientnum; i <= clientID; i++) - { - overTime[timeidx].clientdata[i] = 0; - } - // Update counter - overTime[timeidx].clientnum = clientID + 1; - } - int limit = overTime[timeidx].clientnum; - if(clientID >= limit) - { - logg("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"); - logg("FATAL ERROR: Trying to access overTime.clientdata[%i], but maximum is %i", clientID, limit); - logg(" found in %s() (%s:%i)", function, file, line); - } -} - // The special memory handling routines have to be the last ones in this source file // as we restore the original definition of the strdup, free, calloc, and realloc // functions in here, i.e. if anything extra would come below these lines, it would diff --git a/routines.h b/routines.h index e36cf0d2b..be7bc4d38 100644 --- a/routines.h +++ b/routines.h @@ -92,7 +92,6 @@ void *FTLcalloc(size_t nmemb, size_t size, const char *file, const char *functio void *FTLrealloc(void *ptr_in, size_t size, const char *file, const char *function, int line); void FTLfree(void *ptr, const char* file, const char *function, int line); void validate_access(const char * name, int pos, bool testmagic, int line, const char * function, const char * file); -void validate_access_oTcl(int timeidx, int clientID, int line, const char * function, const char * file); int main_dnsmasq(int argc, char **argv); @@ -115,3 +114,15 @@ void destroy_shmem(void); unsigned long long addstr(const char *str); char *getstr(unsigned long long pos); void *enlarge_shmem_struct(char type); + +/** + * Create a new overTime client shared memory block. + * This also updates `overTimeClientData`. + */ +void newOverTimeClient(); + +/** + * Add a new overTime slot to each overTime client shared memory block. + * This also updates `overTimeClientData`. + */ +void addOverTimeClientSlot(); diff --git a/shmem.c b/shmem.c index b28a6834f..d4e8fc199 100644 --- a/shmem.c +++ b/shmem.c @@ -18,6 +18,8 @@ #define SHARED_CLIENTS_NAME "/FTL-clients" #define SHARED_QUERIES_NAME "/FTL-queries" #define SHARED_FORWARDED_NAME "/FTL-forwarded" +#define SHARED_OVERTIME_NAME "/FTL-overTime" +#define SHARED_OVERTIMECLIENT_PREFIX "/FTL-client-" /// The pointer in shared memory to the shared string buffer static SharedMemory shm_strings = { 0 }; @@ -26,6 +28,10 @@ static SharedMemory shm_domains = { 0 }; static SharedMemory shm_clients = { 0 }; static SharedMemory shm_queries = { 0 }; static SharedMemory shm_forwarded = { 0 }; +static SharedMemory shm_overTime = { 0 }; + +static SharedMemory *shm_overTimeClients = NULL; +static int overTimeClientCount = 0; static int pagesize; static unsigned int next_pos = 0; @@ -66,6 +72,54 @@ char *getstr(unsigned long long pos) return &((char*)shm_strings.ptr)[pos]; } +static char *clientShmName(int id) { + int name_len = 1 + snprintf(NULL, 0, "%s%d", SHARED_OVERTIMECLIENT_PREFIX, id); + char *name = malloc(sizeof(char) * name_len); + snprintf(name, (size_t) name_len, "%s%d", SHARED_OVERTIMECLIENT_PREFIX, id); + + return name; +} + +void newOverTimeClient() { + // Get the name of the new shared memory. + // This will be used in the struct, so it should not be immediately freed. + char *name = clientShmName(overTimeClientCount); + + // Create the shared memory with enough space for the current overTime slots + shm_unlink(name); + SharedMemory shm = create_shm(name, sizeof(int) * counters->overTime); + if(shm.ptr == NULL) { + free(shm.name); + logg("Failed to initialize new overTime client %d", overTimeClientCount); + return; + } + + // Make space for the new shared memory + shm_overTimeClients = realloc(shm_overTimeClients, sizeof(SharedMemory) * (overTimeClientCount + 1)); + overTimeClientCount++; + shm_overTimeClients[overTimeClientCount-1] = shm; + + // Add to overTimeClientData + overTimeClientData = realloc(overTimeClientData, sizeof(int*) * (overTimeClientCount)); + overTimeClientData[overTimeClientCount-1] = shm.ptr; +} + +void addOverTimeClientSlot() { + // For each client slot, add an overTime slot + for(int i = 0; i < overTimeClientCount; i++) { + // Get the name of the shared memory + char *name = clientShmName(i); + + // Reallocate with one more slot + realloc_shm(&shm_overTimeClients[i], shm_overTimeClients[i].size + sizeof(int)); + + // Update overTimeClientData + overTimeClientData[i] = shm_overTimeClients[i].ptr; + + free(name); + } +} + bool init_shmem(void) { // Get kernel's page size @@ -129,6 +183,15 @@ bool init_shmem(void) queries = (queriesDataStruct*)shm_queries.ptr; counters->queries_MAX = pagesize; + /****************************** shared overTime struct ******************************/ + shm_unlink(SHARED_OVERTIME_NAME); + // Try to create shared memory object + shm_overTime = create_shm(SHARED_OVERTIME_NAME, pagesize*sizeof(overTimeDataStruct)); + if(shm_overTime.ptr == NULL) + return false; + overTime = (overTimeDataStruct*)shm_overTime.ptr; + counters->overTime_MAX = pagesize; + return true; } @@ -140,6 +203,12 @@ void destroy_shmem(void) delete_shm(&shm_clients); delete_shm(&shm_queries); delete_shm(&shm_forwarded); + delete_shm(&shm_overTime); + + for(int i = 0; i < overTimeClientCount; i++) { + free(shm_overTimeClients[i].name); + delete_shm(&shm_overTimeClients[i]); + } } SharedMemory create_shm(char *name, size_t size) @@ -202,28 +271,33 @@ void *enlarge_shmem_struct(char type) // Select type of struct that should be enlarged switch(type) { - case 'q': + case QUERIES: sharedMemory = &shm_queries; sizeofobj = sizeof(queriesDataStruct); counter = &counters->queries_MAX; break; - case 'c': + case CLIENTS: sharedMemory = &shm_clients; sizeofobj = sizeof(clientsDataStruct); counter = &counters->clients_MAX; break; - case 'd': + case DOMAINS: sharedMemory = &shm_domains; sizeofobj = sizeof(domainsDataStruct); counter = &counters->domains_MAX; break; - case 'f': + case FORWARDED: sharedMemory = &shm_forwarded; sizeofobj = sizeof(forwardedDataStruct); counter = &counters->forwarded_MAX; break; + case OVERTIME: + sharedMemory = &shm_overTime; + sizeofobj = sizeof(overTimeDataStruct); + counter = &counters->overTime_MAX; + break; default: - logg("Invalid argument in enlarge_shmem_struct(): %c (%i)", type, type); + logg("Invalid argument in enlarge_shmem_struct(): %i", type); return 0; } From b936e2b3a97f42ad5fe0a05c875d2391489bcbed Mon Sep 17 00:00:00 2001 From: Mcat12 Date: Mon, 24 Sep 2018 22:46:52 -0400 Subject: [PATCH 31/48] Fix error freeing client shared memory The client name was freed before the shared memory was freed, so it was used after free in delete_shm. Signed-off-by: Mcat12 --- shmem.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shmem.c b/shmem.c index d4e8fc199..19c3c2e0d 100644 --- a/shmem.c +++ b/shmem.c @@ -206,8 +206,8 @@ void destroy_shmem(void) delete_shm(&shm_overTime); for(int i = 0; i < overTimeClientCount; i++) { - free(shm_overTimeClients[i].name); delete_shm(&shm_overTimeClients[i]); + free(shm_overTimeClients[i].name); } } From 325fc6495f407cd7079fa2589f8eee7ef24ede99 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 25 Sep 2018 14:16:15 +0200 Subject: [PATCH 32/48] Fix interchanged indizes in GC routine Signed-off-by: DL6ER --- gc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gc.c b/gc.c index 432f4ac4b..4ecc1af26 100644 --- a/gc.c +++ b/gc.c @@ -63,7 +63,7 @@ void *GC_thread(void *val) clients[clientID].count--; // Adjust corresponding overTime counters - overTimeClientData[timeidx][clientID]--; + overTimeClientData[clientID][timeidx]--; // Adjust domain counter (no overTime information) int domainID = queries[i].domainID; From 776f91c3016ed70061bf5aed315587a49d327f16 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 26 Sep 2018 23:37:44 +0200 Subject: [PATCH 33/48] Allocate individual client overTime objects guaranteed pagesize-aligned Signed-off-by: DL6ER --- shmem.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/shmem.c b/shmem.c index 19c3c2e0d..82a72ac1b 100644 --- a/shmem.c +++ b/shmem.c @@ -87,7 +87,7 @@ void newOverTimeClient() { // Create the shared memory with enough space for the current overTime slots shm_unlink(name); - SharedMemory shm = create_shm(name, sizeof(int) * counters->overTime); + SharedMemory shm = create_shm(name, (counters->overTime/pagesize + 1)*pagesize*sizeof(int)); if(shm.ptr == NULL) { free(shm.name); logg("Failed to initialize new overTime client %d", overTimeClientCount); @@ -105,18 +105,18 @@ void newOverTimeClient() { } void addOverTimeClientSlot() { - // For each client slot, add an overTime slot - for(int i = 0; i < overTimeClientCount; i++) { - // Get the name of the shared memory - char *name = clientShmName(i); + // For each client slot, add pagesize overTime slots + for(int i = 0; i < overTimeClientCount; i++) + { + // Only increase size of the shm object is needed + if(shm_overTimeClients[i].size > (size_t)counters->overTime) + continue; // Reallocate with one more slot - realloc_shm(&shm_overTimeClients[i], shm_overTimeClients[i].size + sizeof(int)); + realloc_shm(&shm_overTimeClients[i], (counters->overTime/pagesize + 1)*pagesize*sizeof(int)); // Update overTimeClientData overTimeClientData[i] = shm_overTimeClients[i].ptr; - - free(name); } } From 8c68aac456d766050d562e52e3ae16df5e2a7036 Mon Sep 17 00:00:00 2001 From: Mcat12 Date: Fri, 5 Oct 2018 01:18:40 -0400 Subject: [PATCH 34/48] Add shared memory lock This replaces the previous mutex managed by threads.c. The lock is stored in shared memory with the name "/FTL-lock", and any clients connecting to the shared memory are expected to use the lock when accessing memory. Signed-off-by: Mcat12 --- database.c | 11 +++++---- dnsmasq_interface.c | 57 +++++++++++++++++++++++---------------------- gc.c | 6 +++-- main.c | 1 - request.c | 3 ++- routines.h | 5 ---- shmem.c | 53 +++++++++++++++++++++++++++++++++++++++++ shmem.h | 9 +++++++ socket.c | 9 +++---- threads.c | 46 ------------------------------------ 10 files changed, 108 insertions(+), 92 deletions(-) delete mode 100644 threads.c diff --git a/database.c b/database.c index 66b24aea4..714d7e1b6 100644 --- a/database.c +++ b/database.c @@ -9,6 +9,7 @@ * Please see LICENSE file for your rights under this license. */ #include "FTL.h" +#include "shmem.h" sqlite3 *db; bool database = false; @@ -552,15 +553,15 @@ void *DB_thread(void *val) // to prevent locking the resolver resolveNewClients(); - // Lock FTL's data structure, since it is - // likely that it will be changed here - enable_thread_lock(); + // Lock FTL's data structures, since it is + // likely that they will be changed here + shm_write_lock(); // Save data to database save_to_DB(); - // Release thread lock - disable_thread_lock(); + // Release data lock + shm_unlock_lock(); // Check if GC should be done on the database if(DBdeleteoldqueries) diff --git a/dnsmasq_interface.c b/dnsmasq_interface.c index 80d40ef07..4118ff08c 100644 --- a/dnsmasq_interface.c +++ b/dnsmasq_interface.c @@ -12,6 +12,7 @@ #undef __USE_XOPEN #include "FTL.h" #include "dnsmasq_interface.h" +#include "shmem.h" void print_flags(unsigned int flags); void save_reply_type(unsigned int flags, int queryID, struct timeval response); @@ -30,7 +31,7 @@ void FTL_new_query(unsigned int flags, char *name, struct all_addr *addr, char * return; // Create new query in data structure - enable_thread_lock(); + shm_write_lock(); // Get timestamp int querytimestamp, overTimetimestamp; @@ -60,7 +61,7 @@ void FTL_new_query(unsigned int flags, char *name, struct all_addr *addr, char * { // Return early to avoid accessing querytypedata out of bounds if(debug) logg("Notice: Skipping unknown query type: %s (%i)", types, id); - disable_thread_lock(); + shm_unlock_lock(); return; } @@ -68,7 +69,7 @@ void FTL_new_query(unsigned int flags, char *name, struct all_addr *addr, char * if(!config.analyze_AAAA && querytype == TYPE_AAAA) { if(debug) logg("Not analyzing AAAA query"); - disable_thread_lock(); + shm_unlock_lock(); return; } @@ -85,7 +86,7 @@ void FTL_new_query(unsigned int flags, char *name, struct all_addr *addr, char * { // free memory already allocated here free(domain); - disable_thread_lock(); + shm_unlock_lock(); return; } @@ -114,7 +115,7 @@ void FTL_new_query(unsigned int flags, char *name, struct all_addr *addr, char * free(domain); free(domainbuffer); free(client); - disable_thread_lock(); + shm_unlock_lock(); return; } @@ -145,7 +146,7 @@ void FTL_new_query(unsigned int flags, char *name, struct all_addr *addr, char * free(domain); free(domainbuffer); free(client); - disable_thread_lock(); + shm_unlock_lock(); return; } @@ -223,7 +224,7 @@ void FTL_new_query(unsigned int flags, char *name, struct all_addr *addr, char * free(domainbuffer); // Release thread lock - disable_thread_lock(); + shm_unlock_lock(); } static int findQueryID(int id) @@ -256,7 +257,7 @@ void FTL_forwarded(unsigned int flags, char *name, struct all_addr *addr, int id return; // Save that this query got forwarded to an upstream server - enable_thread_lock(); + shm_write_lock(); // Get forward destination IP address char dest[ADDRSTRLEN]; @@ -275,7 +276,7 @@ void FTL_forwarded(unsigned int flags, char *name, struct all_addr *addr, int id // This may happen e.g. if the original query was a PTR query or "pi.hole" // as we ignore them altogether free(forward); - disable_thread_lock(); + shm_unlock_lock(); return; } @@ -291,7 +292,7 @@ void FTL_forwarded(unsigned int flags, char *name, struct all_addr *addr, int id if(queries[i].complete && queries[i].status != QUERY_CACHE) { free(forward); - disable_thread_lock(); + shm_unlock_lock(); return; } @@ -351,7 +352,7 @@ void FTL_forwarded(unsigned int flags, char *name, struct all_addr *addr, int id // Release allocated memory free(forward); - disable_thread_lock(); + shm_unlock_lock(); } void FTL_dnsmasq_reload(void) @@ -387,7 +388,7 @@ void FTL_reply(unsigned short flags, char *name, struct all_addr *addr, int id) return; // Interpret hosts files that have been read by dnsmasq - enable_thread_lock(); + shm_write_lock(); // Determine returned result if available char dest[ADDRSTRLEN]; dest[0] = '\0'; if(addr) @@ -421,14 +422,14 @@ void FTL_reply(unsigned short flags, char *name, struct all_addr *addr, int id) { // This may happen e.g. if the original query was "pi.hole" if(debug) logg("FTL_reply(): Query %i has not been found", id); - disable_thread_lock(); + shm_unlock_lock(); return; } if(queries[i].reply != REPLY_UNKNOWN) { // Nothing to be done here - disable_thread_lock(); + shm_unlock_lock(); return; } @@ -493,7 +494,7 @@ void FTL_reply(unsigned short flags, char *name, struct all_addr *addr, int id) print_flags(flags); } - disable_thread_lock(); + shm_unlock_lock(); } static void detect_blocked_IP(unsigned short flags, char* answer, int queryID) @@ -578,7 +579,7 @@ void FTL_cache(unsigned int flags, char *name, struct all_addr *addr, char *arg, return; // Save that this query got answered from cache - enable_thread_lock(); + shm_write_lock(); char dest[ADDRSTRLEN]; dest[0] = '\0'; if(addr) { @@ -594,7 +595,7 @@ void FTL_cache(unsigned int flags, char *name, struct all_addr *addr, char *arg, { // free memory already allocated here free(domain); - disable_thread_lock(); + shm_unlock_lock(); return; } free(domain); @@ -651,7 +652,7 @@ void FTL_cache(unsigned int flags, char *name, struct all_addr *addr, char *arg, { // This may happen e.g. if the original query was a PTR query or "pi.hole" // as we ignore them altogether - disable_thread_lock(); + shm_unlock_lock(); return; } @@ -717,7 +718,7 @@ void FTL_cache(unsigned int flags, char *name, struct all_addr *addr, char *arg, logg("*************************** unknown CACHE reply (2) ***************************"); print_flags(flags); } - disable_thread_lock(); + shm_unlock_lock(); } void FTL_dnssec(int status, int id) @@ -727,13 +728,13 @@ void FTL_dnssec(int status, int id) return; // Process DNSSEC result for a domain - enable_thread_lock(); + shm_write_lock(); // Search for corresponding query identified by ID int i = findQueryID(id); if(i < 0) { // This may happen e.g. if the original query was an unhandled query type - disable_thread_lock(); + shm_unlock_lock(); return; } @@ -753,7 +754,7 @@ void FTL_dnssec(int status, int id) else queries[i].dnssec = DNSSEC_BOGUS; - disable_thread_lock(); + shm_unlock_lock(); } void FTL_header_ADbit(unsigned char header4, int id) @@ -762,12 +763,12 @@ void FTL_header_ADbit(unsigned char header4, int id) if(config.privacylevel >= PRIVACY_NOSTATS) return; - enable_thread_lock(); + shm_write_lock(); // Check if AD bit is set in DNS header if(!(header4 & 0x20)) { // AD bit not set - disable_thread_lock(); + shm_unlock_lock(); return; } @@ -776,14 +777,14 @@ void FTL_header_ADbit(unsigned char header4, int id) if(i < 0) { // This may happen e.g. if the original query was an unhandled query type - disable_thread_lock(); + shm_unlock_lock(); return; } // Store AD bit in query data queries[i].AD = true; - disable_thread_lock(); + shm_unlock_lock(); } void print_flags(unsigned int flags) @@ -940,7 +941,7 @@ void FTL_forwarding_failed(struct server *server) return; // Save that this query got forwarded to an upstream server - enable_thread_lock(); + shm_write_lock(); char dest[ADDRSTRLEN]; if(server->addr.sa.sa_family == AF_INET) inet_ntop(AF_INET, &server->addr.in.sin_addr, dest, ADDRSTRLEN); @@ -957,7 +958,7 @@ void FTL_forwarding_failed(struct server *server) forwarded[forwardID].failed++; free(forward); - disable_thread_lock(); + shm_unlock_lock(); return; } diff --git a/gc.c b/gc.c index 4ecc1af26..f1a2bbc92 100644 --- a/gc.c +++ b/gc.c @@ -9,6 +9,8 @@ * Please see LICENSE file for your rights under this license. */ #include "FTL.h" +#include "shmem.h" + bool doGC = false; int lastGCrun = 0; @@ -30,7 +32,7 @@ void *GC_thread(void *val) // Lock FTL's data structure, since it is likely that it will be changed here // Requests should not be processed/answered when data is about to change - enable_thread_lock(); + shm_write_lock(); // Get minimum time stamp to keep time_t mintime = time(NULL) - config.maxlogage; @@ -165,7 +167,7 @@ void *GC_thread(void *val) if(debug) logg("Notice: GC removed %i queries (took %.2f ms)", removed, timer_elapsed_msec(GC_TIMER)); // Release thread lock - disable_thread_lock(); + shm_unlock_lock(); // Reresolve client hostnames to account for changes // Have to this outside of the thread lock diff --git a/main.c b/main.c index ddf1fd0b9..a785a650f 100644 --- a/main.c +++ b/main.c @@ -35,7 +35,6 @@ int main (int argc, char* argv[]) timer_start(EXIT_TIMER); logg("########## FTL started! ##########"); log_FTL_version(); - init_thread_lock(); // Initialize shared memory if(!init_shmem()) diff --git a/request.c b/request.c index 72bc04e5c..54cbe0a3f 100644 --- a/request.c +++ b/request.c @@ -10,6 +10,7 @@ #include "FTL.h" #include "api.h" +#include "shmem.h" bool command(char *client_message, const char* cmd) { return strstr(client_message, cmd) != NULL; @@ -118,7 +119,7 @@ void process_request(char *client_message, int *sock) logg("Received API request to re-resolve host names"); // Need to release the thread lock already here to allow // the resolver to process the incoming PTR requests - disable_thread_lock(); + shm_unlock_lock(); reresolveHostnames(); logg("Done re-resolving host names"); } diff --git a/routines.h b/routines.h index be7bc4d38..d6c1ebdd1 100644 --- a/routines.h +++ b/routines.h @@ -65,11 +65,6 @@ void parse_args(int argc, char* argv[]); char* find_equals(const char* s); -// threads.c -void enable_thread_lock(void); -void disable_thread_lock(void); -void init_thread_lock(void); - // config.c void read_FTLconf(void); void get_privacy_level(FILE *fp); diff --git a/shmem.c b/shmem.c index 19c3c2e0d..14e49c9da 100644 --- a/shmem.c +++ b/shmem.c @@ -12,6 +12,7 @@ #include "shmem.h" /// The name of the shared memory. Use this when connecting to the shared memory. +#define SHARED_LOCK_NAME "/FTL-lock" #define SHARED_STRINGS_NAME "/FTL-strings" #define SHARED_COUNTERS_NAME "/FTL-counters" #define SHARED_DOMAINS_NAME "/FTL-domains" @@ -22,6 +23,7 @@ #define SHARED_OVERTIMECLIENT_PREFIX "/FTL-client-" /// The pointer in shared memory to the shared string buffer +static SharedMemory shm_lock = { 0 }; static SharedMemory shm_strings = { 0 }; static SharedMemory shm_counters = { 0 }; static SharedMemory shm_domains = { 0 }; @@ -32,6 +34,7 @@ static SharedMemory shm_overTime = { 0 }; static SharedMemory *shm_overTimeClients = NULL; static int overTimeClientCount = 0; +static pthread_rwlock_t *sharedMemoryLock = NULL; static int pagesize; static unsigned int next_pos = 0; @@ -120,11 +123,58 @@ void addOverTimeClientSlot() { } } +/// Create a read/write lock +pthread_rwlock_t create_rwlock() { + pthread_rwlockattr_t lock_attr = { 0 }; + pthread_rwlock_t lock = { 0 }; + + // Initialize the lock attributes + pthread_rwlockattr_init(&lock_attr); + + // Allow the lock to be used by other processes + pthread_rwlockattr_setpshared(&lock_attr, PTHREAD_PROCESS_SHARED); + + // Initialize the lock + pthread_rwlock_init(&lock, &lock_attr); + + return lock; +} + +void shm_read_lock() { + int result = pthread_rwlock_rdlock(sharedMemoryLock); + + if(result != 0) + logg("Failed to obtain SHM read lock: %s", strerror(result)); +} + +void shm_write_lock() { + int result = pthread_rwlock_wrlock(sharedMemoryLock); + + if(result != 0) + logg("Failed to obtain SHM write lock: %s", strerror(result)); +} + +void shm_unlock_lock() { + int result = pthread_rwlock_unlock(sharedMemoryLock); + + if(result != 0) + logg("Failed to unlock SHM lock: %s", strerror(result)); +} + bool init_shmem(void) { // Get kernel's page size pagesize = getpagesize(); + /****************************** shared memory lock ******************************/ + shm_unlink(SHARED_LOCK_NAME); + // Try to create shared memory object + shm_lock = create_shm(SHARED_LOCK_NAME, sizeof(pthread_rwlock_t)); + if(shm_lock.ptr == NULL) + return false; + *((pthread_rwlock_t*)shm_lock.ptr) = create_rwlock(); + sharedMemoryLock = shm_lock.ptr; + /****************************** shared strings buffer ******************************/ // Try unlinking the shared memory object before creating a new one // If the object is still existing, e.g., due to a past unclean exit @@ -197,6 +247,9 @@ bool init_shmem(void) void destroy_shmem(void) { + pthread_rwlock_destroy(sharedMemoryLock); + + delete_shm(&shm_lock); delete_shm(&shm_strings); delete_shm(&shm_counters); delete_shm(&shm_domains); diff --git a/shmem.h b/shmem.h index 908c742f4..4b66d8ad3 100644 --- a/shmem.h +++ b/shmem.h @@ -40,4 +40,13 @@ bool realloc_shm(SharedMemory *sharedMemory, size_t size); /// \param sharedMemory the shared memory struct void delete_shm(SharedMemory *sharedMemory); +/// Block until a read lock can be obtained +void shm_read_lock(); + +/// Block until a write lock can be obtained +void shm_write_lock(); + +/// Unlock the lock. Only call this if there is an active lock. +void shm_unlock_lock(); + #endif //SHARED_MEMORY_SERVER_H diff --git a/socket.c b/socket.c index 619441924..e33ce6f40 100644 --- a/socket.c +++ b/socket.c @@ -10,6 +10,7 @@ #include "FTL.h" #include "api.h" +#include "shmem.h" // The backlog argument defines the maximum length // to which the queue of pending connections for @@ -321,13 +322,13 @@ void *telnet_connection_handler_thread(void *socket_desc) // Lock FTL data structure, since it is likely that it will be changed here // Requests should not be processed/answered when data is about to change - enable_thread_lock(); + shm_write_lock(); process_request(message, &sock); free(message); // Release thread lock - disable_thread_lock(); + shm_unlock_lock(); if(sock == 0) { @@ -378,13 +379,13 @@ void *socket_connection_handler_thread(void *socket_desc) // Lock FTL data structure, since it is likely that it will be changed here // Requests should not be processed/answered when data is about to change - enable_thread_lock(); + shm_write_lock(); process_request(message, &sock); free(message); // Release thread lock - disable_thread_lock(); + shm_unlock_lock(); if(sock == 0) { diff --git a/threads.c b/threads.c deleted file mode 100644 index dd4f1d757..000000000 --- a/threads.c +++ /dev/null @@ -1,46 +0,0 @@ -/* Pi-hole: A black hole for Internet advertisements -* (c) 2017 Pi-hole, LLC (https://pi-hole.net) -* Network-wide ad blocking via your own hardware. -* -* FTL Engine -* Thread routines -* -* This file is copyright under the latest version of the EUPL. -* Please see LICENSE file for your rights under this license. */ - -#include "FTL.h" - -// Logic of the locks: -// Any of the various threads (logparser, GC, client threads) is accessing FTL's data structure. Hence, they should -// never run at the same time since the data can change half-way through, leading to unspecified behavior. -// threadlock: The threadlock ensures that only one thread can be active at any given time -pthread_mutex_t threadlock; - -void enable_thread_lock(void) -{ - // logg("At thread lock: waiting"); - int ret = pthread_mutex_lock(&threadlock); - // logg("At thread lock: passed"); - - if(ret != 0) - logg("Thread lock error: %i",ret); -} - -void disable_thread_lock(void) -{ - int ret = pthread_mutex_unlock(&threadlock); - // logg("At thread lock: unlocked"); - - if(ret != 0) - logg("Thread unlock error: %i",ret); -} - -void init_thread_lock(void) -{ - if (pthread_mutex_init(&threadlock, NULL) != 0) - { - logg("FATAL: Thread mutex init failed\n"); - // Return failure - exit(EXIT_FAILURE); - } -} From 3283d687e1601644c3fa839991e5932d2eafda79 Mon Sep 17 00:00:00 2001 From: Mcat12 Date: Fri, 5 Oct 2018 01:22:02 -0400 Subject: [PATCH 35/48] Remove threads.o from Makefile Signed-off-by: Mcat12 --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index d55f5c94a..8cccffa07 100644 --- a/Makefile +++ b/Makefile @@ -14,7 +14,7 @@ DNSMASQOPTS = -DHAVE_DNSSEC -DHAVE_DNSSEC_STATIC # Flags for compiling with libidn2: -DHAVE_LIBIDN2 -DIDN2_VERSION_NUMBER=0x02000003 FTLDEPS = FTL.h routines.h version.h api.h dnsmasq_interface.h shmem.h -FTLOBJ = main.o memory.o log.o daemon.o datastructure.o signals.o socket.o request.o grep.o setupVars.o args.o threads.o gc.o config.o database.o msgpack.o api.o dnsmasq_interface.o resolve.o regex.o shmem.o +FTLOBJ = main.o memory.o log.o daemon.o datastructure.o signals.o socket.o request.o grep.o setupVars.o args.o gc.o config.o database.o msgpack.o api.o dnsmasq_interface.o resolve.o regex.o shmem.o DNSMASQDEPS = config.h dhcp-protocol.h dns-protocol.h radv-protocol.h dhcp6-protocol.h dnsmasq.h ip6addr.h DNSMASQOBJ = arp.o dbus.o domain.o lease.o outpacket.o rrfilter.o auth.o dhcp6.o edns0.o log.o poll.o slaac.o blockdata.o dhcp.o forward.o loop.o radv.o tables.o bpf.o dhcp-common.o helper.o netlink.o rfc1035.o tftp.o cache.o dnsmasq.o inotify.o network.o rfc2131.o util.o conntrack.o dnssec.o ipset.o option.o rfc3315.o crypto.o From fa9e7b1cdb461ec972880155a2c12b47acd94ad4 Mon Sep 17 00:00:00 2001 From: Mcat12 Date: Fri, 5 Oct 2018 01:31:39 -0400 Subject: [PATCH 36/48] Fix lock initializers for arm build The arm compiler does not like the `{ 0 }` initializers for some reason. Signed-off-by: Mcat12 --- shmem.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/shmem.c b/shmem.c index 14e49c9da..fede34529 100644 --- a/shmem.c +++ b/shmem.c @@ -125,8 +125,8 @@ void addOverTimeClientSlot() { /// Create a read/write lock pthread_rwlock_t create_rwlock() { - pthread_rwlockattr_t lock_attr = { 0 }; - pthread_rwlock_t lock = { 0 }; + pthread_rwlockattr_t lock_attr = {}; + pthread_rwlock_t lock = {}; // Initialize the lock attributes pthread_rwlockattr_init(&lock_attr); From 215fa14923588dd51424262369a5f47606944192 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 7 Oct 2018 17:53:39 +0200 Subject: [PATCH 37/48] Review comments Signed-off-by: DL6ER --- shmem.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/shmem.c b/shmem.c index 82a72ac1b..a00613e7b 100644 --- a/shmem.c +++ b/shmem.c @@ -108,12 +108,16 @@ void addOverTimeClientSlot() { // For each client slot, add pagesize overTime slots for(int i = 0; i < overTimeClientCount; i++) { - // Only increase size of the shm object is needed - if(shm_overTimeClients[i].size > (size_t)counters->overTime) + // Only increase the size of the shm object if needed + // shm_overTimeClients[i].size stores the size of the memory in bytes whereas + // counters->overTime (effectively) stores the number of slots each overTime + // client should have. Hence, counters->overTime needs to be multiplied by + // sizeof(int) to get the actual requested memory size + if(shm_overTimeClients[i].size > (size_t)counters->overTime*sizeof(int)) continue; // Reallocate with one more slot - realloc_shm(&shm_overTimeClients[i], (counters->overTime/pagesize + 1)*pagesize*sizeof(int)); + realloc_shm(&shm_overTimeClients[i], (counters->overTime + pagesize)*sizeof(int)); // Update overTimeClientData overTimeClientData[i] = shm_overTimeClients[i].ptr; From 1279b93294e6b7ea17ae4daef0cc251b2b92e91d Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 7 Oct 2018 20:50:21 +0200 Subject: [PATCH 38/48] Store database ID in queries struct. This information will be availabe through shmem to the API to support transparent overlap with the SQlite3 database Signed-off-by: DL6ER --- FTL.h | 2 +- database.c | 48 +++++++++++++++++++++++++++++++++++++++++++----- 2 files changed, 44 insertions(+), 6 deletions(-) diff --git a/FTL.h b/FTL.h index f6766bf36..181047ff1 100644 --- a/FTL.h +++ b/FTL.h @@ -160,7 +160,7 @@ typedef struct { int domainID; int clientID; int forwardID; - bool db; + sqlite3_int64 db; int id; // the ID is a (signed) int in dnsmasq, so no need for a long int here bool complete; bool private; diff --git a/database.c b/database.c index 66b24aea4..85cbeb218 100644 --- a/database.c +++ b/database.c @@ -317,6 +317,33 @@ int number_of_queries_in_DB(void) return result; } +static sqlite3_int64 last_ID_in_DB(void) +{ + sqlite3_stmt* stmt; + + int rc = sqlite3_prepare_v2(db, "SELECT MAX(ID) FROM queries", -1, &stmt, NULL); + if( rc ){ + logg("last_ID_in_DB() - SQL error prepare (%i): %s", rc, sqlite3_errmsg(db)); + dbclose(); + check_database(rc); + return -1; + } + + rc = sqlite3_step(stmt); + if( rc != SQLITE_ROW ){ + logg("last_ID_in_DB() - SQL error step (%i): %s", rc, sqlite3_errmsg(db)); + dbclose(); + check_database(rc); + return -1; + } + + sqlite3_int64 result = sqlite3_column_int64(stmt, 0); + + sqlite3_finalize(stmt); + + return result; +} + int get_number_of_queries_in_DB(void) { int result = -1; @@ -378,7 +405,7 @@ void save_to_DB(void) for(i = 0; i < counters->queries; i++) { validate_access("queries", i, true, __LINE__, __FUNCTION__, __FILE__); - if(queries[i].db) + if(queries[i].db != 0) { // Skip, already saved in database continue; @@ -451,8 +478,9 @@ void save_to_DB(void) } saved++; - // Mark this query as saved in the database only if successful - queries[i].db = true; + // Mark this query as saved in the database by setting to a non-zero + // value. The correct ID will be inserted later + queries[i].db = 1; // Total counter information (delta computation) total++; @@ -487,12 +515,21 @@ void save_to_DB(void) return; } + // Update individual queryIDs in the queries struct + sqlite3_int64 lastID = last_ID_in_DB(); + for(i=0; i < total; i++) + { + // Subtract i from coutners-queries-1 as the database + // loop goes only until i < counters->queries + queries[(counters->queries-1)-i].db = lastID-i; + } + // Close database dbclose(); if(debug) { - logg("Notice: Queries stored in DB: %u (took %.1f ms)", saved, timer_elapsed_msec(DATABASE_WRITE_TIMER)); + logg("Notice: Queries stored in DB: %u (took %.1f ms, last SQLite ID %llu)", saved, timer_elapsed_msec(DATABASE_WRITE_TIMER), lastID); if(saved_error > 0) logg(" There are queries that have not been saved"); } @@ -617,6 +654,7 @@ void read_data_from_DB(void) // Loop through returned database rows while((rc = sqlite3_step(stmt)) == SQLITE_ROW) { + sqlite3_int64 dbid = sqlite3_column_int64(stmt, 0); int queryTimeStamp = sqlite3_column_int(stmt, 1); // 1483228800 = 01/01/2017 @ 12:00am (UTC) if(queryTimeStamp < 1483228800) @@ -712,7 +750,7 @@ void read_data_from_DB(void) queries[queryIndex].clientID = clientID; queries[queryIndex].forwardID = forwardID; queries[queryIndex].timeidx = timeidx; - queries[queryIndex].db = true; // Mark this as already present in the database + queries[queryIndex].db = dbid; queries[queryIndex].id = queryID; queries[queryIndex].complete = true; // Mark as all information is avaiable queries[queryIndex].response = 0; From 601722d4411edbe36f2e3a6d3f2ad60e2dee13d3 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 11 Oct 2018 06:10:18 +0200 Subject: [PATCH 39/48] Obtain ID before insertion transaction and store IDs only when a query has actually been saved instead of trying to do this in a post-processing step Signed-off-by: DL6ER --- database.c | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/database.c b/database.c index 85cbeb218..ce6fad824 100644 --- a/database.c +++ b/database.c @@ -382,6 +382,9 @@ void save_to_DB(void) long int i; sqlite3_stmt* stmt; + // Get last ID stored in the database + sqlite3_int64 lastID = last_ID_in_DB(); + bool ret = dbquery("BEGIN TRANSACTION"); if(!ret) { @@ -478,9 +481,8 @@ void save_to_DB(void) } saved++; - // Mark this query as saved in the database by setting to a non-zero - // value. The correct ID will be inserted later - queries[i].db = 1; + // Mark this query as saved in the database by setting the corresponding ID + queries[i].db = ++lastID; // Total counter information (delta computation) total++; @@ -515,15 +517,6 @@ void save_to_DB(void) return; } - // Update individual queryIDs in the queries struct - sqlite3_int64 lastID = last_ID_in_DB(); - for(i=0; i < total; i++) - { - // Subtract i from coutners-queries-1 as the database - // loop goes only until i < counters->queries - queries[(counters->queries-1)-i].db = lastID-i; - } - // Close database dbclose(); From ca2e38ba60d1ed08ce05597338d86d279beae3f3 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 11 Oct 2018 19:23:53 +0200 Subject: [PATCH 40/48] Initialize db rowID with zero Signed-off-by: DL6ER --- dnsmasq_interface.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dnsmasq_interface.c b/dnsmasq_interface.c index 80d40ef07..84f7e5107 100644 --- a/dnsmasq_interface.c +++ b/dnsmasq_interface.c @@ -164,7 +164,8 @@ void FTL_new_query(unsigned int flags, char *name, struct all_addr *addr, char * queries[queryID].domainID = domainID; queries[queryID].clientID = clientID; queries[queryID].timeidx = timeidx; - queries[queryID].db = false; + // Initialize database rowID with zero, will be set when the query is stored in the long-term DB + queries[queryID].db = 0; queries[queryID].id = id; queries[queryID].complete = false; queries[queryID].private = (config.privacylevel == PRIVACY_MAXIMUM); From d7b4ce8a5f1a94b814eb8534716445508e954514 Mon Sep 17 00:00:00 2001 From: Mcat12 Date: Thu, 11 Oct 2018 15:59:52 -0400 Subject: [PATCH 41/48] Remove read lock and simplify lock names Signed-off-by: Mcat12 --- database.c | 4 ++-- dnsmasq_interface.c | 56 ++++++++++++++++++++++----------------------- gc.c | 4 ++-- request.c | 2 +- shmem.c | 11 ++------- shmem.h | 7 ++---- socket.c | 8 +++---- 7 files changed, 41 insertions(+), 51 deletions(-) diff --git a/database.c b/database.c index 714d7e1b6..618e0970d 100644 --- a/database.c +++ b/database.c @@ -555,13 +555,13 @@ void *DB_thread(void *val) // Lock FTL's data structures, since it is // likely that they will be changed here - shm_write_lock(); + lock_shm(); // Save data to database save_to_DB(); // Release data lock - shm_unlock_lock(); + unlock_shm(); // Check if GC should be done on the database if(DBdeleteoldqueries) diff --git a/dnsmasq_interface.c b/dnsmasq_interface.c index 4118ff08c..df8856566 100644 --- a/dnsmasq_interface.c +++ b/dnsmasq_interface.c @@ -31,7 +31,7 @@ void FTL_new_query(unsigned int flags, char *name, struct all_addr *addr, char * return; // Create new query in data structure - shm_write_lock(); + lock_shm(); // Get timestamp int querytimestamp, overTimetimestamp; @@ -61,7 +61,7 @@ void FTL_new_query(unsigned int flags, char *name, struct all_addr *addr, char * { // Return early to avoid accessing querytypedata out of bounds if(debug) logg("Notice: Skipping unknown query type: %s (%i)", types, id); - shm_unlock_lock(); + unlock_shm(); return; } @@ -69,7 +69,7 @@ void FTL_new_query(unsigned int flags, char *name, struct all_addr *addr, char * if(!config.analyze_AAAA && querytype == TYPE_AAAA) { if(debug) logg("Not analyzing AAAA query"); - shm_unlock_lock(); + unlock_shm(); return; } @@ -86,7 +86,7 @@ void FTL_new_query(unsigned int flags, char *name, struct all_addr *addr, char * { // free memory already allocated here free(domain); - shm_unlock_lock(); + unlock_shm(); return; } @@ -115,7 +115,7 @@ void FTL_new_query(unsigned int flags, char *name, struct all_addr *addr, char * free(domain); free(domainbuffer); free(client); - shm_unlock_lock(); + unlock_shm(); return; } @@ -146,7 +146,7 @@ void FTL_new_query(unsigned int flags, char *name, struct all_addr *addr, char * free(domain); free(domainbuffer); free(client); - shm_unlock_lock(); + unlock_shm(); return; } @@ -224,7 +224,7 @@ void FTL_new_query(unsigned int flags, char *name, struct all_addr *addr, char * free(domainbuffer); // Release thread lock - shm_unlock_lock(); + unlock_shm(); } static int findQueryID(int id) @@ -257,7 +257,7 @@ void FTL_forwarded(unsigned int flags, char *name, struct all_addr *addr, int id return; // Save that this query got forwarded to an upstream server - shm_write_lock(); + lock_shm(); // Get forward destination IP address char dest[ADDRSTRLEN]; @@ -276,7 +276,7 @@ void FTL_forwarded(unsigned int flags, char *name, struct all_addr *addr, int id // This may happen e.g. if the original query was a PTR query or "pi.hole" // as we ignore them altogether free(forward); - shm_unlock_lock(); + unlock_shm(); return; } @@ -292,7 +292,7 @@ void FTL_forwarded(unsigned int flags, char *name, struct all_addr *addr, int id if(queries[i].complete && queries[i].status != QUERY_CACHE) { free(forward); - shm_unlock_lock(); + unlock_shm(); return; } @@ -352,7 +352,7 @@ void FTL_forwarded(unsigned int flags, char *name, struct all_addr *addr, int id // Release allocated memory free(forward); - shm_unlock_lock(); + unlock_shm(); } void FTL_dnsmasq_reload(void) @@ -388,7 +388,7 @@ void FTL_reply(unsigned short flags, char *name, struct all_addr *addr, int id) return; // Interpret hosts files that have been read by dnsmasq - shm_write_lock(); + lock_shm(); // Determine returned result if available char dest[ADDRSTRLEN]; dest[0] = '\0'; if(addr) @@ -422,14 +422,14 @@ void FTL_reply(unsigned short flags, char *name, struct all_addr *addr, int id) { // This may happen e.g. if the original query was "pi.hole" if(debug) logg("FTL_reply(): Query %i has not been found", id); - shm_unlock_lock(); + unlock_shm(); return; } if(queries[i].reply != REPLY_UNKNOWN) { // Nothing to be done here - shm_unlock_lock(); + unlock_shm(); return; } @@ -494,7 +494,7 @@ void FTL_reply(unsigned short flags, char *name, struct all_addr *addr, int id) print_flags(flags); } - shm_unlock_lock(); + unlock_shm(); } static void detect_blocked_IP(unsigned short flags, char* answer, int queryID) @@ -579,7 +579,7 @@ void FTL_cache(unsigned int flags, char *name, struct all_addr *addr, char *arg, return; // Save that this query got answered from cache - shm_write_lock(); + lock_shm(); char dest[ADDRSTRLEN]; dest[0] = '\0'; if(addr) { @@ -595,7 +595,7 @@ void FTL_cache(unsigned int flags, char *name, struct all_addr *addr, char *arg, { // free memory already allocated here free(domain); - shm_unlock_lock(); + unlock_shm(); return; } free(domain); @@ -652,7 +652,7 @@ void FTL_cache(unsigned int flags, char *name, struct all_addr *addr, char *arg, { // This may happen e.g. if the original query was a PTR query or "pi.hole" // as we ignore them altogether - shm_unlock_lock(); + unlock_shm(); return; } @@ -718,7 +718,7 @@ void FTL_cache(unsigned int flags, char *name, struct all_addr *addr, char *arg, logg("*************************** unknown CACHE reply (2) ***************************"); print_flags(flags); } - shm_unlock_lock(); + unlock_shm(); } void FTL_dnssec(int status, int id) @@ -728,13 +728,13 @@ void FTL_dnssec(int status, int id) return; // Process DNSSEC result for a domain - shm_write_lock(); + lock_shm(); // Search for corresponding query identified by ID int i = findQueryID(id); if(i < 0) { // This may happen e.g. if the original query was an unhandled query type - shm_unlock_lock(); + unlock_shm(); return; } @@ -754,7 +754,7 @@ void FTL_dnssec(int status, int id) else queries[i].dnssec = DNSSEC_BOGUS; - shm_unlock_lock(); + unlock_shm(); } void FTL_header_ADbit(unsigned char header4, int id) @@ -763,12 +763,12 @@ void FTL_header_ADbit(unsigned char header4, int id) if(config.privacylevel >= PRIVACY_NOSTATS) return; - shm_write_lock(); + lock_shm(); // Check if AD bit is set in DNS header if(!(header4 & 0x20)) { // AD bit not set - shm_unlock_lock(); + unlock_shm(); return; } @@ -777,14 +777,14 @@ void FTL_header_ADbit(unsigned char header4, int id) if(i < 0) { // This may happen e.g. if the original query was an unhandled query type - shm_unlock_lock(); + unlock_shm(); return; } // Store AD bit in query data queries[i].AD = true; - shm_unlock_lock(); + unlock_shm(); } void print_flags(unsigned int flags) @@ -941,7 +941,7 @@ void FTL_forwarding_failed(struct server *server) return; // Save that this query got forwarded to an upstream server - shm_write_lock(); + lock_shm(); char dest[ADDRSTRLEN]; if(server->addr.sa.sa_family == AF_INET) inet_ntop(AF_INET, &server->addr.in.sin_addr, dest, ADDRSTRLEN); @@ -958,7 +958,7 @@ void FTL_forwarding_failed(struct server *server) forwarded[forwardID].failed++; free(forward); - shm_unlock_lock(); + unlock_shm(); return; } diff --git a/gc.c b/gc.c index f1a2bbc92..75ddd1676 100644 --- a/gc.c +++ b/gc.c @@ -32,7 +32,7 @@ void *GC_thread(void *val) // Lock FTL's data structure, since it is likely that it will be changed here // Requests should not be processed/answered when data is about to change - shm_write_lock(); + lock_shm(); // Get minimum time stamp to keep time_t mintime = time(NULL) - config.maxlogage; @@ -167,7 +167,7 @@ void *GC_thread(void *val) if(debug) logg("Notice: GC removed %i queries (took %.2f ms)", removed, timer_elapsed_msec(GC_TIMER)); // Release thread lock - shm_unlock_lock(); + unlock_shm(); // Reresolve client hostnames to account for changes // Have to this outside of the thread lock diff --git a/request.c b/request.c index 54cbe0a3f..7874779eb 100644 --- a/request.c +++ b/request.c @@ -119,7 +119,7 @@ void process_request(char *client_message, int *sock) logg("Received API request to re-resolve host names"); // Need to release the thread lock already here to allow // the resolver to process the incoming PTR requests - shm_unlock_lock(); + unlock_shm(); reresolveHostnames(); logg("Done re-resolving host names"); } diff --git a/shmem.c b/shmem.c index fede34529..ca76197a0 100644 --- a/shmem.c +++ b/shmem.c @@ -140,21 +140,14 @@ pthread_rwlock_t create_rwlock() { return lock; } -void shm_read_lock() { - int result = pthread_rwlock_rdlock(sharedMemoryLock); - - if(result != 0) - logg("Failed to obtain SHM read lock: %s", strerror(result)); -} - -void shm_write_lock() { +void lock_shm() { int result = pthread_rwlock_wrlock(sharedMemoryLock); if(result != 0) logg("Failed to obtain SHM write lock: %s", strerror(result)); } -void shm_unlock_lock() { +void unlock_shm() { int result = pthread_rwlock_unlock(sharedMemoryLock); if(result != 0) diff --git a/shmem.h b/shmem.h index 4b66d8ad3..0a7d1701d 100644 --- a/shmem.h +++ b/shmem.h @@ -40,13 +40,10 @@ bool realloc_shm(SharedMemory *sharedMemory, size_t size); /// \param sharedMemory the shared memory struct void delete_shm(SharedMemory *sharedMemory); -/// Block until a read lock can be obtained -void shm_read_lock(); - /// Block until a write lock can be obtained -void shm_write_lock(); +void lock_shm(); /// Unlock the lock. Only call this if there is an active lock. -void shm_unlock_lock(); +void unlock_shm(); #endif //SHARED_MEMORY_SERVER_H diff --git a/socket.c b/socket.c index e33ce6f40..5214c5bdd 100644 --- a/socket.c +++ b/socket.c @@ -322,13 +322,13 @@ void *telnet_connection_handler_thread(void *socket_desc) // Lock FTL data structure, since it is likely that it will be changed here // Requests should not be processed/answered when data is about to change - shm_write_lock(); + lock_shm(); process_request(message, &sock); free(message); // Release thread lock - shm_unlock_lock(); + unlock_shm(); if(sock == 0) { @@ -379,13 +379,13 @@ void *socket_connection_handler_thread(void *socket_desc) // Lock FTL data structure, since it is likely that it will be changed here // Requests should not be processed/answered when data is about to change - shm_write_lock(); + lock_shm(); process_request(message, &sock); free(message); // Release thread lock - shm_unlock_lock(); + unlock_shm(); if(sock == 0) { From a3e9677288381a5074da491de26058fa1d8e98f4 Mon Sep 17 00:00:00 2001 From: Mcat12 Date: Thu, 11 Oct 2018 16:27:00 -0400 Subject: [PATCH 42/48] Change shm lock to a robust mutex Signed-off-by: Mcat12 --- shmem.c | 40 ++++++++++++++++++++++++++-------------- shmem.h | 2 +- 2 files changed, 27 insertions(+), 15 deletions(-) diff --git a/shmem.c b/shmem.c index ca76197a0..333501dfb 100644 --- a/shmem.c +++ b/shmem.c @@ -34,7 +34,7 @@ static SharedMemory shm_overTime = { 0 }; static SharedMemory *shm_overTimeClients = NULL; static int overTimeClientCount = 0; -static pthread_rwlock_t *sharedMemoryLock = NULL; +static pthread_mutex_t *sharedMemoryLock = NULL; static int pagesize; static unsigned int next_pos = 0; @@ -123,32 +123,44 @@ void addOverTimeClientSlot() { } } -/// Create a read/write lock -pthread_rwlock_t create_rwlock() { - pthread_rwlockattr_t lock_attr = {}; - pthread_rwlock_t lock = {}; +/// Create a mutex for shared memory +pthread_mutex_t create_rwlock() { + pthread_mutexattr_t lock_attr = {}; + pthread_mutex_t lock = {}; // Initialize the lock attributes - pthread_rwlockattr_init(&lock_attr); + pthread_mutexattr_init(&lock_attr); // Allow the lock to be used by other processes - pthread_rwlockattr_setpshared(&lock_attr, PTHREAD_PROCESS_SHARED); + pthread_mutexattr_setpshared(&lock_attr, PTHREAD_PROCESS_SHARED); + + // Make the lock robust against process death + pthread_mutexattr_setrobust(&lock_attr, PTHREAD_MUTEX_ROBUST); // Initialize the lock - pthread_rwlock_init(&lock, &lock_attr); + pthread_mutex_init(&lock, &lock_attr); + + // Destroy the lock attribute since we're done with it + pthread_mutexattr_destroy(&lock_attr); return lock; } void lock_shm() { - int result = pthread_rwlock_wrlock(sharedMemoryLock); + int result = pthread_mutex_lock(sharedMemoryLock); + + if(result == EOWNERDEAD) { + // Try to make the lock consistent if the other process died while + // holding the lock + result = pthread_mutex_consistent(sharedMemoryLock); + } if(result != 0) - logg("Failed to obtain SHM write lock: %s", strerror(result)); + logg("Failed to obtain SHM lock: %s", strerror(result)); } void unlock_shm() { - int result = pthread_rwlock_unlock(sharedMemoryLock); + int result = pthread_mutex_unlock(sharedMemoryLock); if(result != 0) logg("Failed to unlock SHM lock: %s", strerror(result)); @@ -162,10 +174,10 @@ bool init_shmem(void) /****************************** shared memory lock ******************************/ shm_unlink(SHARED_LOCK_NAME); // Try to create shared memory object - shm_lock = create_shm(SHARED_LOCK_NAME, sizeof(pthread_rwlock_t)); + shm_lock = create_shm(SHARED_LOCK_NAME, sizeof(pthread_mutex_t)); if(shm_lock.ptr == NULL) return false; - *((pthread_rwlock_t*)shm_lock.ptr) = create_rwlock(); + *((pthread_mutex_t*)shm_lock.ptr) = create_rwlock(); sharedMemoryLock = shm_lock.ptr; /****************************** shared strings buffer ******************************/ @@ -240,7 +252,7 @@ bool init_shmem(void) void destroy_shmem(void) { - pthread_rwlock_destroy(sharedMemoryLock); + pthread_mutex_destroy(sharedMemoryLock); delete_shm(&shm_lock); delete_shm(&shm_strings); diff --git a/shmem.h b/shmem.h index 0a7d1701d..2bca284aa 100644 --- a/shmem.h +++ b/shmem.h @@ -40,7 +40,7 @@ bool realloc_shm(SharedMemory *sharedMemory, size_t size); /// \param sharedMemory the shared memory struct void delete_shm(SharedMemory *sharedMemory); -/// Block until a write lock can be obtained +/// Block until a lock can be obtained void lock_shm(); /// Unlock the lock. Only call this if there is an active lock. From b0c8a54ff486847ec2cb6d951ce3b1f182840199 Mon Sep 17 00:00:00 2001 From: Mcat12 Date: Fri, 12 Oct 2018 21:58:25 -0400 Subject: [PATCH 43/48] Use a condition variable and boolean to notify FTL is waiting for a lock This is checked by accessors of shared memory to defer to FTL so they do not starve FTL of access to shared memory (starving DNS resolution). Signed-off-by: Mcat12 --- shmem.c | 58 +++++++++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 48 insertions(+), 10 deletions(-) diff --git a/shmem.c b/shmem.c index 333501dfb..956e0153e 100644 --- a/shmem.c +++ b/shmem.c @@ -34,7 +34,13 @@ static SharedMemory shm_overTime = { 0 }; static SharedMemory *shm_overTimeClients = NULL; static int overTimeClientCount = 0; -static pthread_mutex_t *sharedMemoryLock = NULL; + +typedef struct { + pthread_mutex_t lock; + pthread_cond_t condVar; + bool waitingForLock; +} ShmLock; +static ShmLock *shmLockStruct = NULL; static int pagesize; static unsigned int next_pos = 0; @@ -124,7 +130,7 @@ void addOverTimeClientSlot() { } /// Create a mutex for shared memory -pthread_mutex_t create_rwlock() { +pthread_mutex_t create_mutex() { pthread_mutexattr_t lock_attr = {}; pthread_mutex_t lock = {}; @@ -140,19 +146,47 @@ pthread_mutex_t create_rwlock() { // Initialize the lock pthread_mutex_init(&lock, &lock_attr); - // Destroy the lock attribute since we're done with it + // Destroy the lock attributes since we're done with it pthread_mutexattr_destroy(&lock_attr); return lock; } +/// Create a conditional variable for shared memory +pthread_cond_t create_cond_var() { + pthread_condattr_t cond_attr; + pthread_cond_t cond_var; + + // Initialize the condition variable attributes + pthread_condattr_init(&cond_attr); + + // Allow the condition variable to be used by other processes + pthread_condattr_setpshared(&cond_attr, PTHREAD_PROCESS_SHARED); + + // Initialize the condition variable + pthread_cond_init(&cond_var, &cond_attr); + + // Destroy the condition variable attributes since we're done with it + pthread_condattr_destroy(&cond_attr); + + return cond_var; +} + void lock_shm() { - int result = pthread_mutex_lock(sharedMemoryLock); + // Signal that FTL is waiting for a lock + shmLockStruct->waitingForLock = true; + + int result = pthread_mutex_lock(&shmLockStruct->lock); + + // Turn off waiting for the lock signal and notify everyone who was + // deferring to FTL that they can jump in the lock queue. + shmLockStruct->waitingForLock = false; + pthread_cond_broadcast(&shmLockStruct->condVar); if(result == EOWNERDEAD) { // Try to make the lock consistent if the other process died while // holding the lock - result = pthread_mutex_consistent(sharedMemoryLock); + result = pthread_mutex_consistent(&shmLockStruct->lock); } if(result != 0) @@ -160,7 +194,7 @@ void lock_shm() { } void unlock_shm() { - int result = pthread_mutex_unlock(sharedMemoryLock); + int result = pthread_mutex_unlock(&shmLockStruct->lock); if(result != 0) logg("Failed to unlock SHM lock: %s", strerror(result)); @@ -174,11 +208,13 @@ bool init_shmem(void) /****************************** shared memory lock ******************************/ shm_unlink(SHARED_LOCK_NAME); // Try to create shared memory object - shm_lock = create_shm(SHARED_LOCK_NAME, sizeof(pthread_mutex_t)); + shm_lock = create_shm(SHARED_LOCK_NAME, sizeof(ShmLock)); if(shm_lock.ptr == NULL) return false; - *((pthread_mutex_t*)shm_lock.ptr) = create_rwlock(); - sharedMemoryLock = shm_lock.ptr; + shmLockStruct = (ShmLock*) shm_lock.ptr; + shmLockStruct->lock = create_mutex(); + shmLockStruct->condVar = create_cond_var(); + shmLockStruct->waitingForLock = false; /****************************** shared strings buffer ******************************/ // Try unlinking the shared memory object before creating a new one @@ -252,7 +288,9 @@ bool init_shmem(void) void destroy_shmem(void) { - pthread_mutex_destroy(sharedMemoryLock); + pthread_mutex_destroy(&shmLockStruct->lock); + pthread_cond_destroy(&shmLockStruct->condVar); + shmLockStruct = NULL; delete_shm(&shm_lock); delete_shm(&shm_strings); From ece89a5dc97f0dccf19ce213c5fee38d98aaddc3 Mon Sep 17 00:00:00 2001 From: Mcat12 Date: Tue, 23 Oct 2018 23:38:39 -0400 Subject: [PATCH 44/48] Remove condition variable from shared memory lock It is no longer needed. Signed-off-by: Mcat12 --- shmem.c | 48 ++++++++++++------------------------------------ 1 file changed, 12 insertions(+), 36 deletions(-) diff --git a/shmem.c b/shmem.c index 956e0153e..27dc89bcf 100644 --- a/shmem.c +++ b/shmem.c @@ -37,10 +37,9 @@ static int overTimeClientCount = 0; typedef struct { pthread_mutex_t lock; - pthread_cond_t condVar; bool waitingForLock; } ShmLock; -static ShmLock *shmLockStruct = NULL; +static ShmLock *shmLock = NULL; static int pagesize; static unsigned int next_pos = 0; @@ -152,41 +151,20 @@ pthread_mutex_t create_mutex() { return lock; } -/// Create a conditional variable for shared memory -pthread_cond_t create_cond_var() { - pthread_condattr_t cond_attr; - pthread_cond_t cond_var; - - // Initialize the condition variable attributes - pthread_condattr_init(&cond_attr); - - // Allow the condition variable to be used by other processes - pthread_condattr_setpshared(&cond_attr, PTHREAD_PROCESS_SHARED); - - // Initialize the condition variable - pthread_cond_init(&cond_var, &cond_attr); - - // Destroy the condition variable attributes since we're done with it - pthread_condattr_destroy(&cond_attr); - - return cond_var; -} - void lock_shm() { // Signal that FTL is waiting for a lock - shmLockStruct->waitingForLock = true; + shmLock->waitingForLock = true; - int result = pthread_mutex_lock(&shmLockStruct->lock); + int result = pthread_mutex_lock(&shmLock->lock); - // Turn off waiting for the lock signal and notify everyone who was + // Turn off the waiting for lock signal to notify everyone who was // deferring to FTL that they can jump in the lock queue. - shmLockStruct->waitingForLock = false; - pthread_cond_broadcast(&shmLockStruct->condVar); + shmLock->waitingForLock = false; if(result == EOWNERDEAD) { // Try to make the lock consistent if the other process died while // holding the lock - result = pthread_mutex_consistent(&shmLockStruct->lock); + result = pthread_mutex_consistent(&shmLock->lock); } if(result != 0) @@ -194,7 +172,7 @@ void lock_shm() { } void unlock_shm() { - int result = pthread_mutex_unlock(&shmLockStruct->lock); + int result = pthread_mutex_unlock(&shmLock->lock); if(result != 0) logg("Failed to unlock SHM lock: %s", strerror(result)); @@ -211,10 +189,9 @@ bool init_shmem(void) shm_lock = create_shm(SHARED_LOCK_NAME, sizeof(ShmLock)); if(shm_lock.ptr == NULL) return false; - shmLockStruct = (ShmLock*) shm_lock.ptr; - shmLockStruct->lock = create_mutex(); - shmLockStruct->condVar = create_cond_var(); - shmLockStruct->waitingForLock = false; + shmLock = (ShmLock*) shm_lock.ptr; + shmLock->lock = create_mutex(); + shmLock->waitingForLock = false; /****************************** shared strings buffer ******************************/ // Try unlinking the shared memory object before creating a new one @@ -288,9 +265,8 @@ bool init_shmem(void) void destroy_shmem(void) { - pthread_mutex_destroy(&shmLockStruct->lock); - pthread_cond_destroy(&shmLockStruct->condVar); - shmLockStruct = NULL; + pthread_mutex_destroy(&shmLock->lock); + shmLock = NULL; delete_shm(&shm_lock); delete_shm(&shm_strings); From 100d92bbd9183a5fbe4e9da934581a8f80bc7e2e Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 25 Oct 2018 21:29:16 +0200 Subject: [PATCH 45/48] Set queryID to zero for queries imported from the database on startup Signed-off-by: DL6ER --- database.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/database.c b/database.c index ce6fad824..1ef902464 100644 --- a/database.c +++ b/database.c @@ -744,7 +744,7 @@ void read_data_from_DB(void) queries[queryIndex].forwardID = forwardID; queries[queryIndex].timeidx = timeidx; queries[queryIndex].db = dbid; - queries[queryIndex].id = queryID; + queries[queryIndex].id = 0; queries[queryIndex].complete = true; // Mark as all information is avaiable queries[queryIndex].response = 0; queries[queryIndex].AD = false; From f8ba1739b3f0ff9906f36ec198f04fc06dd3d24a Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 25 Oct 2018 21:33:55 +0200 Subject: [PATCH 46/48] Remove unused variable queryID Signed-off-by: DL6ER --- database.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/database.c b/database.c index 1ef902464..753cc814c 100644 --- a/database.c +++ b/database.c @@ -728,10 +728,6 @@ void read_data_from_DB(void) // Set index for this query int queryIndex = counters->queries; - // Set the ID for this query. Queries loaded from the database use negative IDs. - // 1 is added to the counter before flipping the sign so the IDs start at -1 instead of 0. - int queryID = -1 * (counters->queries + 1); - // Store this query in memory validate_access("overTime", timeidx, true, __LINE__, __FUNCTION__, __FILE__); validate_access("queries", queryIndex, false, __LINE__, __FUNCTION__, __FILE__); From befc0f81fe950fdd2c3ee2e25d09222bfe8436d7 Mon Sep 17 00:00:00 2001 From: Mcat12 Date: Wed, 21 Nov 2018 15:04:52 -0500 Subject: [PATCH 47/48] Fix reference to old lock functions Signed-off-by: Mcat12 --- resolve.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/resolve.c b/resolve.c index c19e48c6c..df86c690a 100644 --- a/resolve.c +++ b/resolve.c @@ -9,6 +9,7 @@ * Please see LICENSE file for your rights under this license. */ #include "FTL.h" +#include "shmem.h" // Resolve new client and upstream server host names // once every minute @@ -84,12 +85,12 @@ void resolveClients(bool onlynew) if(onlynew && !clients[clientID].new) continue; - enable_thread_lock(); + lock_shm(); clients[clientID].namepos = addstr(resolveHostname(getstr(clients[clientID].ippos))); clients[clientID].new = false; - disable_thread_lock(); + unlock_shm(); } } @@ -107,12 +108,12 @@ void resolveForwardDestinations(bool onlynew) if(onlynew && !forwarded[forwardID].new) continue; - enable_thread_lock(); + lock_shm(); forwarded[forwardID].namepos = addstr(resolveHostname(getstr(forwarded[forwardID].ippos))); forwarded[forwardID].new = false; - disable_thread_lock(); + unlock_shm(); } } From 120aeb44a6610e170ac0e69f6a60ec1e970e26a6 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 9 Dec 2018 20:10:42 +0100 Subject: [PATCH 48/48] Remove obsolete constants Signed-off-by: DL6ER --- FTL.h | 6 ------ 1 file changed, 6 deletions(-) diff --git a/FTL.h b/FTL.h index 38a537a75..1eed9bf00 100644 --- a/FTL.h +++ b/FTL.h @@ -52,12 +52,6 @@ #include "routines.h" -// Next we define the step size in which the struct arrays are reallocated if they -// grow too large. This number should be large enough so that reallocation does not -// have to run very often, but should be as small as possible to avoid wasting memory -#define OVERTIMEALLOCSTEP 100 -#define WILDCARDALLOCSTEP 100 - #define SOCKETBUFFERLEN 1024 // How often do we garbage collect (to ensure we only have data fitting to the MAXLOGAGE defined above)? [seconds]