Merge pull request #208 from pi-hole/new/loadFromDB

Import data from long-term database

FTL.h

@@ -210,6 +210,9 @@ enum { QUERIES, FORWARDED, CLIENTS, DOMAINS, OVERTIME, WILDCARD };
 enum { SOCKET };
 enum { DNSSEC_UNSPECIFIED, DNSSEC_SECURE, DNSSEC_INSECURE, DNSSEC_BOGUS, DNSSEC_ABANDONED, DNSSEC_UNKNOWN };
 
+// Used to check memory integrity in various structs
+#define MAGICBYTE 0x57
+
 logFileNamesStruct files;
 FTLFileNamesStruct FTLfiles;
 countersStruct counters;
@@ -252,3 +255,4 @@ long int lastdbindex;
 bool travis;
 bool DBdeleteoldqueries;
 bool rereadgravity;
+long int lastDBimportedtimestamp;

database.c

@@ -14,6 +14,7 @@ sqlite3 *db;
 bool database = false;
 bool DBdeleteoldqueries = false;
 long int lastdbindex = 0;
+long int lastDBimportedtimestamp = 0;
 
 pthread_mutex_t dblock;
 
@@ -480,3 +481,164 @@ void *DB_thread(void *val)
 
  return NULL;
 }
+
+// Get most recent 24 hours data from long-term database
+void read_data_from_DB(void)
+{
+ // Open database file
+ if(!dbopen())
+ {
+ logg("read_data_from_DB() - Failed to open DB");
+ return;
+ }
+
+ // Prepare request
+ char *rstr = NULL;
+ // Get time stamp 24 hours in the past
+ int differencetofullhour = time(NULL) % GCinterval;
+ long int mintime = ((long)time(NULL) - GCdelay - differencetofullhour) - MAXLOGAGE;
+ int rc = asprintf(&rstr, "SELECT * FROM queries WHERE timestamp >= %li", mintime);
+ if(rc < 1)
+ {
+ logg("read_data_from_DB() - Allocation error (%i): %s", rc, sqlite3_errmsg(db));
+ return;
+ }
+ // Log DB query string in debug mode
+ if(debug) logg(rstr);
+
+ // Prepare SQLite3 statement
+ sqlite3_stmt* stmt;
+ rc = sqlite3_prepare_v2(db, rstr, -1, &stmt, NULL);
+ if( rc ){
+ logg("read_data_from_DB() - SQL error prepare (%i): %s", rc, sqlite3_errmsg(db));
+ dbclose();
+ check_database(rc);
+ return;
+ }
+
+ // Loop through returned database rows
+ while((rc = sqlite3_step(stmt)) == SQLITE_ROW)
+ {
+ // Ensure we have enough space in the queries struct
+ memory_check(QUERIES);
+ memory_check(DOMAINS);
+ memory_check(CLIENTS);
+
+ // Set ID for this query
+ int queryID = counters.queries;
+
+ int queryTimeStamp = sqlite3_column_int(stmt, 1);
+ int type = sqlite3_column_int(stmt, 2);
+ int status = sqlite3_column_int(stmt, 3);
+ int domainID = findDomainID((const char *)sqlite3_column_text(stmt, 4));
+ int clientID = findClientID((const char *)sqlite3_column_text(stmt, 5));
+ int forwardID = findForwardID((const char *)sqlite3_column_text(stmt, 6), true);
+
+ int overTimeTimeStamp = queryTimeStamp - (queryTimeStamp % 600 + 300);
+ int timeidx = findOverTimeID(overTimeTimeStamp);
+ validate_access("overTime", timeidx, true, __LINE__, __FUNCTION__, __FILE__);
+
+ // Store this query in memory
+ validate_access("queries", queryID, false, __LINE__, __FUNCTION__, __FILE__);
+ queries[queryID].magic = MAGICBYTE;
+ queries[queryID].timestamp = queryTimeStamp;
+ queries[queryID].type = type;
+ queries[queryID].status = status;
+ queries[queryID].domainID = domainID;
+ queries[queryID].clientID = clientID;
+ queries[queryID].forwardID = forwardID;
+ queries[queryID].timeidx = timeidx;
+ queries[queryID].valid = true; // Mark this as a valid query (false = it has been garbage collected and should be skipped)
+ queries[queryID].db = true; // Mark this as already present in the database
+ queries[queryID].id = 0; // This is dnsmasq's internal ID. We don't store it in the database
+ queries[queryID].complete = true; // Mark as all information is avaiable
+ queries[queryID].reply = 0; // Reply type is not stored in database
+ queries[queryID].generation = 0; // Log generation is neither available nor important if reading from the database
+ lastDBimportedtimestamp = queryTimeStamp;
+
+ // Handle type counters
+ if(type == 1)
+ {
+ counters.IPv4++;
+ overTime[timeidx].querytypedata[0]++;
+ }
+ else if(type == 2)
+ {
+ counters.IPv6++;
+ overTime[timeidx].querytypedata[1]++;
+ }
+
+ // Update overTime data
+ overTime[timeidx].total++;
+
+ // Update overTime data structure with the new client
+ validate_access_oTcl(timeidx, clientID, __LINE__, __FUNCTION__, __FILE__);
+ overTime[timeidx].clientdata[clientID]++;
+
+ // Increase DNS queries counter
+ counters.queries++;
+
+ // Increment status counters
+ switch(status)
+ {
+ case 0: // Unknown
+ counters.unknown++;
+ break;
+
+ case 1: // Blocked by gravity.list
+ counters.blocked++;
+ overTime[timeidx].blocked++;
+ domains[domainID].blockedcount++;
+ break;
+
+ case 2: // Forwarded
+ counters.forwardedqueries++;
+ // Update overTime data structure
+ validate_access_oTfd(timeidx, forwardID, __LINE__, __FUNCTION__, __FILE__);
+ overTime[timeidx].forwarddata[forwardID]++;
+ break;
+
+ case 3: // Cached or local config
+ counters.cached++;
+ // Update overTime data structure
+ overTime[timeidx].cached++;
+ break;
+
+ case 4: // Wildcard blocked
+ counters.wildcardblocked++;
+
+ // Update overTime data structure
+ overTime[timeidx].blocked++;
+ domains[domainID].blockedcount++;
+ domains[domainID].wildcard = true;
+ break;
+
+ case 5: // black.list
+ counters.blocked++;
+
+ // Update overTime data structure
+ overTime[timeidx].blocked++;
+ domains[domainID].blockedcount++;
+ break;
+
+ default:
+ logg("Error: Found unknown status %i in long term database!", status);
+ logg(" Timestamp: %i", queryTimeStamp);
+ logg(" Continuing anyway...");
+ break;
+ }
+ }
+ logg("Imported %i queries from the long-term database", counters.queries);
+
+ if( rc != SQLITE_DONE ){
+ logg("read_data_from_DB() - SQL error step (%i): %s", rc, sqlite3_errmsg(db));
+ dbclose();
+ check_database(rc);
+ return;
+ }
+
+ // Finalize SQLite3 statement
+ sqlite3_finalize(stmt);
+ dbclose();
+ free(rstr);
+}

main.c

@@ -49,6 +49,10 @@ int main (int argc, char* argv[]) {
  if(config.maxDBdays != 0)
  db_init();
 
+ // Try to import queries from long-term database if available
+ if(database)
+ read_data_from_DB();
+
  logg("Starting initial log file parsing");
  initial_log_parsing();
  logg("Finished initial log file parsing");
@@ -181,7 +185,7 @@ int main (int argc, char* argv[]) {
  save_to_DB();
  logg("Finished final database update");
  }
- 
+
  // Close sockets
  close_telnet_socket();
  close_unix_socket();