Merge pull request #116 from pi-hole/development

FTL v2.10

database.c

@@ -49,7 +49,7 @@ bool dbopen(void)
  pthread_mutex_lock(&dblock);
  int rc = sqlite3_open_v2(FTLfiles.db, &db, SQLITE_OPEN_READWRITE, NULL);
  if( rc ){
- logg("Cannot open database: %s", sqlite3_errmsg(db));
+ logg("dbopen() - SQL error (%i): %s", rc, sqlite3_errmsg(db));
  dbclose();
  return false;
  }
@@ -64,13 +64,12 @@ bool dbquery(const char *format, ...)
  int rc;
 
  va_start(args, format);
-
  char *query = sqlite3_vmprintf(format, args);
+ va_end(args);
 
  if(query == NULL)
  {
  logg("Memory allocation failed in dbquery()");
- va_end(args);
  return false;
  }
 
@@ -79,15 +78,14 @@ bool dbquery(const char *format, ...)
  else
  rc = sqlite3_exec(db, query, NULL, NULL, &zErrMsg);
 
- sqlite3_free(query);
- va_end(args);
-
  if( rc != SQLITE_OK ){
- logg("SQL error (%i): %s", rc, zErrMsg);
+ logg("dbquery(%s) - SQL error (%i): %s", query, rc, zErrMsg);
  sqlite3_free(zErrMsg);
  return false;
  }
 
+ sqlite3_free(query);
+
  return true;
 
 }
@@ -97,7 +95,7 @@ bool db_create(void)
  bool ret;
  int rc = sqlite3_open_v2(FTLfiles.db, &db, SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, NULL);
  if( rc ){
- logg("Can't create database: %s", sqlite3_errmsg(db));
+ logg("db_create() - SQL error (%i): %s", rc, sqlite3_errmsg(db));
  dbclose();
  return false;
  }
@@ -132,7 +130,7 @@ void db_init(void)
 {
  int rc = sqlite3_open_v2(FTLfiles.db, &db, SQLITE_OPEN_READWRITE, NULL);
  if( rc ){
- logg("Cannot open database: %s", sqlite3_errmsg(db));
+ logg("db_init() - Cannot open database (%i): %s", rc, sqlite3_errmsg(db));
  dbclose();
 
  logg("Creating new (empty) database");
@@ -146,7 +144,7 @@ void db_init(void)
 
  if (pthread_mutex_init(&dblock, NULL) != 0)
  {
- printf("FATAL: DB mutex init failed\n");
+ logg("FATAL: DB mutex init failed\n");
  // Return failure
  exit(EXIT_FAILURE);
  }
@@ -172,16 +170,16 @@ int db_get_FTL_property(unsigned int ID)
 
  rc = sqlite3_prepare(db, querystring, -1, &dbstmt, NULL);
  if( rc ){
- printf("Cannot read from database: %s", sqlite3_errmsg(db));
+ logg("db_get_FTL_property() - SQL error prepare (%i): %s", rc, sqlite3_errmsg(db));
  dbclose();
  return -1;
  }
  free(querystring);
 
  // Evaluate SQL statement
- sqlite3_step(dbstmt);
- if( rc ){
- printf("Cannot evaluate in database: %s", sqlite3_errmsg(db));
+ rc = sqlite3_step(dbstmt);
+ if( rc != SQLITE_ROW ){
+ logg("db_get_FTL_property() - SQL error step (%i): %s", rc, sqlite3_errmsg(db));
  dbclose();
  return -1;
  }
@@ -201,15 +199,23 @@ bool db_set_FTL_property(unsigned int ID, int value)
 int number_of_queries_in_DB(void)
 {
  sqlite3_stmt* stmt;
- int result = -1;
 
  // Count number of rows using the index timestamp is faster than select(*)
- sqlite3_prepare_v2(db, "SELECT COUNT(timestamp) FROM queries", -1, &stmt, NULL);
- int rc = sqlite3_step(stmt);
- if (rc == SQLITE_ROW)
- result = sqlite3_column_int(stmt, 0);
- else
- logg("get_number_of_queries_in_DB() - SQL error: %s", sqlite3_errmsg(db));
+ int rc = sqlite3_prepare_v2(db, "SELECT COUNT(timestamp) FROM queries", -1, &stmt, NULL);
+ if( rc ){
+ logg("number_of_queries_in_DB() - SQL error prepare (%i): %s", rc, sqlite3_errmsg(db));
+ dbclose();
+ return -1;
+ }
+
+ rc = sqlite3_step(stmt);
+ if( rc != SQLITE_ROW ){
+ logg("number_of_queries_in_DB() - SQL error step (%i): %s", rc, sqlite3_errmsg(db));
+ dbclose();
+ return -1;
+ }
+
+ int result = sqlite3_column_int(stmt, 0);
 
  sqlite3_finalize(stmt);
 
@@ -239,19 +245,37 @@ void save_to_DB(void)
  // Open database
  if(!dbopen())
  {
- logg("Failed to open DB in save_to_DB()");
+ logg("save_to_DB() - failed to open DB");
  return;
  }
 
  int lasttimestamp = db_get_FTL_property(DB_LASTTIMESTAMP);
+ if(lasttimestamp < 0)
+ {
+ logg("save_to_DB() - error in trying to get last time stamp from database");
+ return;
+ }
  int newlasttimestamp = lasttimestamp;
 
  unsigned int saved = 0, saved_error = 0;
  long int i;
  sqlite3_stmt* stmt;
 
- sqlite3_prepare_v2(db, "INSERT INTO queries VALUES (NULL,?,?,?,?,?,?)", -1, &stmt, NULL);
- dbquery("BEGIN TRANSACTION");
+ bool ret = dbquery("BEGIN TRANSACTION");
+ if(!ret)
+ {
+ logg("save_to_DB() - unable to begin transaction (%i): %s", ret, sqlite3_errmsg(db));
+ dbclose();
+ return;
+ }
+
+ int rc = sqlite3_prepare_v2(db, "INSERT INTO queries VALUES (NULL,?,?,?,?,?,?)", -1, &stmt, NULL);
+ if( rc )
+ {
+ logg("save_to_DB() - error in preparing SQL statement (%i): %s", ret, sqlite3_errmsg(db));
+ dbclose();
+ return;
+ }
 
  for(i = lastdbindex; i < counters.queries; i++)
  {
@@ -298,14 +322,22 @@ void save_to_DB(void)
  }
 
  // Step and check if successful
- int rc = sqlite3_step(stmt);
+ rc = sqlite3_step(stmt);
  sqlite3_clear_bindings(stmt);
  sqlite3_reset(stmt);
 
  if( rc != SQLITE_DONE ){
- logg("save_to_DB() - SQL error: %s", sqlite3_errmsg(db));
+ logg("save_to_DB() - SQL error (%i): %s", rc, sqlite3_errmsg(db));
  saved_error++;
- continue;
+ if(saved_error < 3)
+ {
+ continue;
+ }
+ else
+ {
+ logg("save_to_DB() - exiting due to too many errors");
+ break;
+ }
  }
 
  saved++;
@@ -318,7 +350,8 @@ void save_to_DB(void)
  }
 
  // Finish prepared statement
- dbquery("END TRANSACTION");
+ ret = dbquery("END TRANSACTION");
+ if(!ret){ dbclose(); return; }
  sqlite3_finalize(stmt);
 
  // Store index for next loop interation round and update last time stamp
@@ -334,10 +367,9 @@ void save_to_DB(void)
 
  if(debug)
  {
- if(saved > 0)
- logg("Notice: Queries stored in DB: %u", saved);
+ logg("Notice: Queries stored in DB: %u", saved);
  if(saved_error > 0)
- logg(" Queries NOT stored in DB: %u (due to an error)", saved_error);
+ logg(" There are queries that have not been saved");
  }
 }
 
@@ -360,6 +392,13 @@ void delete_old_queries_in_DB(void)
  return;
  }
 
+ // Get how many rows have been affected (deleted)
+ int affected = sqlite3_changes(db);
+
+ // Print final message only if there is a difference
+ if(debug || affected)
+ logg("Notice: Database size is %.2f MB, deleted %i rows", get_db_filesize(), affected);
+
  // Close database
  dbclose();
 

log.c

@@ -9,6 +9,7 @@
 * Please see LICENSE file for your rights under this license. */
 
 #include "FTL.h"
+#include "version.h"
 
 pthread_mutex_t lock;
 
@@ -139,3 +140,11 @@ void log_counter_info(void)
  logg(" -> Unique domains: %i", counters.domains);
  logg(" -> Unique clients: %i", counters.clients);
 }
+
+void log_FTL_version(void)
+{
+ logg("FTL branch: %s", GIT_BRANCH);
+ logg("FTL hash: %s", GIT_VERSION);
+ logg("FTL date: %s", GIT_DATE);
+ logg("FTL user: %s", username);
+}

main.c

@@ -9,7 +9,6 @@
 * Please see LICENSE file for your rights under this license. */
 
 #include "FTL.h"
-#include "version.h"
 
 char * username;
 bool needGC = false;
@@ -25,10 +24,8 @@ int main (int argc, char* argv[]) {
  open_FTL_log(true);
  open_pihole_log();
  logg("########## FTL started! ##########");
- logg("FTL branch: %s", GIT_BRANCH);
- logg("FTL hash: %s", GIT_VERSION);
- logg("FTL date: %s", GIT_DATE);
- logg("FTL user: %s", username);
+ log_FTL_version();
+ init_thread_lock();
 
  // pihole-FTL should really be run as user "pihole" to not mess up with the file permissions
  // still allow this if "debug" flag is set

parser.c

@@ -53,11 +53,16 @@ void open_pihole_log(void)
 {
  FILE * fp;
  if((fp = fopen(files.log, "r")) == NULL) {
- logg("FATAL: Opening of %s failed!", files.log);
- logg(" Make sure it exists and is readable by user %s", username);
- syslog(LOG_ERR, "Opening of pihole.log failed!");
- // Return failure in exit status
- exit(EXIT_FAILURE);
+ logg("WARN: Opening of %s failed!", files.log);
+ logg(" Make sure it exists and is readable by user %s\n Will try again in 15 seconds.", username);
+
+ sleepms(15000);
+ if((fp = fopen(files.log, "r")) == NULL) {
+ logg("FATAL: Opening of %s failed permanently!", files.log);
+ syslog(LOG_ERR, "Opening of pihole.log failed!");
+ // Return failure in exit status
+ exit(EXIT_FAILURE);
+ }
  }
  fclose(fp);
 }
@@ -198,6 +203,12 @@ void process_pihole_log(int file)
  continue;
  }
 
+ if(strstr(readbuffer, "\"") != NULL)
+ {
+ if(debug) logg("Ignoring \" domain (query)");
+ continue;
+ }
+
  if(!config.analyze_AAAA && strstr(readbuffer,"]: query[AAAA]") != NULL)
  {
  if(debug) logg("Not analyzing AAAA query");
@@ -283,22 +294,35 @@ void process_pihole_log(int file)
  strncat(domain,domainstart+2,domainlen);
  sprintf(domainwithspaces," %s ",domain);
 
+ if(strcmp(domain, "pi.hole") == 0)
+ {
+ // domain is "pi.hole", skip this query
+ // free memory already allocated here
+ free(domain);
+ free(domainwithspaces);
+ continue;
+ }
+
  // Get client
  // domainend+6 = pointer to | in "query[AAAA] host.name from |ww.xx.yy.zz\n"
  const char *clientend = strstr(domainend+6, "\n");
  // Check if buffer pointer is valid
  if(clientend == NULL)
  {
  logg("Notice: Skipping malformated log line (client end missing): %s", readbuffer);
- // Skip this line
+ // Skip this line, free memory already allocated here
+ free(domain);
+ free(domainwithspaces);
  continue;
  }
 
  size_t clientlen = (clientend-domainend)-6;
  if(clientlen < 1)
  {
  logg("Notice: Skipping malformated log line (client length < 1): %s", readbuffer);
- // Skip this line
+ // Skip this line, free memory already allocated here
+ free(domain);
+ free(domainwithspaces);
  continue;
  }