Merge pull request #1542 from pi-hole/development

Release v5.22

.github/.codespellignore

@@ -0,0 +1,4 @@
+ssudo
+tre
+ede
+nd

.github/workflows/build.yml

@@ -24,7 +24,7 @@ jobs:
  steps:
  -
  name: Checkout code
- uses: actions/checkout@v3.3.0
+ uses: actions/checkout@v3.4.0
  -
  name: "Calculate required variables"
  id: variables
@@ -53,7 +53,7 @@ jobs:
 
  needs: smoke-tests
 
- container: ghcr.io/pi-hole/ftl-build:v1.23-${{ matrix.arch }}
+ container: ghcr.io/pi-hole/ftl-build:v1.26-${{ matrix.arch }}
 
  strategy:
  fail-fast: false
@@ -81,14 +81,16 @@ jobs:
  bin_name: pihole-FTL-armv8-linux-gnueabihf
  - arch: aarch64
  bin_name: pihole-FTL-aarch64-linux-gnu
+ - arch: riscv64
+ bin_name: pihole-FTL-riscv64-linux-gnu
 
  env:
  CI_ARCH: ${{ matrix.arch }}${{ matrix.arch_extra }}
 
  steps:
  -
  name: Checkout code
- uses: actions/checkout@v3.3.0
+ uses: actions/checkout@v3.4.0
  -
  name: "Fix ownership of repository"
  run: chown -R root .
@@ -131,7 +133,7 @@ jobs:
  steps:
  -
  name: Checkout code
- uses: actions/checkout@v3.3.0
+ uses: actions/checkout@v3.4.0
  -
  name: Get Binaries built in previous jobs
  uses: actions/download-artifact@v3.0.2

.github/workflows/codespell.yml

@@ -10,10 +10,10 @@ jobs:
  steps:
  -
  name: Checkout repository
- uses: actions/checkout@v3.3.0
+ uses: actions/checkout@v3.4.0
  -
  name: Spell-Checking
  uses: codespell-project/actions-codespell@master
  with:
- ignore_words_file: .codespellignore
- skip: ./src/database/sqlite3.c,./src/database/sqlite3.h,./src/database/shell.c,./src/lua,./src/dnsmasq,./src/tre-regex
+ ignore_words_file: .github/.codespellignore
+ skip: ./src/database/sqlite3.c,./src/database/sqlite3.h,./src/database/shell.c,./src/lua,./src/dnsmasq,./src/tre-regex,./.git,./test/libs

.github/workflows/sync-back-to-dev.yml

@@ -11,7 +11,7 @@ jobs:
  name: Syncing branches
  steps:
  - name: Checkout
- uses: actions/checkout@v3.3.0
+ uses: actions/checkout@v3.4.0
  - name: Opening pull request
  run: gh pr create -B development -H master --title 'Sync master back into development' --body 'Created by Github action' --label 'internal'
  env:

CMakeLists.txt

@@ -11,6 +11,6 @@
 cmake_minimum_required(VERSION 2.8.12)
 project(PIHOLE_FTL C)
 
-set(DNSMASQ_VERSION pi-hole-v2.89)
+set(DNSMASQ_VERSION pi-hole-v2.89-9461807)
 
 add_subdirectory(src)

src/database/gravity-db.c

@@ -47,6 +47,7 @@ static sqlite3 *gravity_db = NULL;
 static sqlite3_stmt* table_stmt = NULL;
 static sqlite3_stmt* auditlist_stmt = NULL;
 bool gravityDB_opened = false;
+static bool gravity_abp_format = false;
 
 // Table names corresponding to the enum defined in gravity-db.h
 static const char* tablename[] = { "vw_gravity", "vw_blacklist", "vw_whitelist", "vw_regex_blacklist", "vw_regex_whitelist" , "" };
@@ -95,6 +96,40 @@ void gravityDB_forked(void)
  gravityDB_open();
 }
 
+static void gravity_check_ABP_format(void)
+{
+ // Check if we have a valid ABP format
+ // We do this by checking the "abp_domains" property in the "info" table
+
+ // Prepare statement
+ sqlite3_stmt *stmt = NULL;
+ int rc = sqlite3_prepare_v2(gravity_db,
+ "SELECT value FROM info WHERE property = 'abp_domains';",
+ -1, &stmt, NULL);
+
+ if( rc != SQLITE_OK )
+ {
+ logg("gravity_check_ABP_format() - SQL error prepare: %s", sqlite3_errstr(rc));
+ return;
+ }
+
+ // Execute statement
+ rc = sqlite3_step(stmt);
+ if( rc != SQLITE_ROW )
+ {
+ // No result
+ gravity_abp_format = false;
+ sqlite3_finalize(stmt);
+ return;
+ }
+
+ // Get result (SQLite3 stores 1 for TRUE, 0 for FALSE)
+ gravity_abp_format = sqlite3_column_int(stmt, 0) != 0;
+
+ // Finalize statement
+ sqlite3_finalize(stmt);
+}
+
 // Open gravity database
 bool gravityDB_open(void)
 {
@@ -194,6 +229,10 @@ bool gravityDB_open(void)
  logg("gravityDB_open() - Cannot set busy handler: %s", sqlite3_errstr(rc));
  }
 
+ // Check (and remember in global variable) if there are any ABP-style
+ // entries in the database
+ gravity_check_ABP_format();
+
  if(config.debug & DEBUG_DATABASE)
  logg("gravityDB_open(): Successfully opened gravity.db");
  return true;
@@ -1290,7 +1329,102 @@ enum db_result in_gravity(const char *domain, clientsData *client)
  if(stmt == NULL)
  stmt = gravity_stmt->get(gravity_stmt, client->id);
 
- return domain_in_list(domain, stmt, "gravity", NULL);
+ // Check if domain is exactly in gravity list
+ const enum db_result exact_match = domain_in_list(domain, stmt, "gravity", NULL);
+ if(config.debug & DEBUG_QUERIES)
+ logg("Checking if \"%s\" is in gravity: %s",
+ domain, exact_match == FOUND ? "yes" : "no");
+ // Return for anything else than "not found" (e.g. "found" or "list not available")
+ if(exact_match != NOT_FOUND)
+ return exact_match;
+
+ // Return early if we are not supposed to check for ABP-style regex
+ // matches. This needs to be enabled in the config file as it is
+ // computationally expensive and not needed in most cases (HOSTS lists).
+ if(!gravity_abp_format)
+ return NOT_FOUND;
+
+ // Make a copy of the domain we will slowly truncate
+ // while extracting the individual components below
+ char *domainBuf = strdup(domain);
+
+ // Buffer to hold the constructed (sub)domain in ABP format
+ char *abpDomain = calloc(strlen(domain) + 4, sizeof(char));
+ // Prime abp matcher with minimal content
+ strcpy(abpDomain, "||^");
+
+ // Get number of domain parts (equals the number of dots + 1)
+ unsigned int N = 1u;
+ for(const char *p = domain; *p != '\0'; p++)
+ if(*p == '.')
+ N++;
+
+ // Loop over domain parts, building matcher from the TLD
+ // going down into domain and subdomains one by one
+ while(N-- > 0)
+ {
+ // Get domain to the *last* occurrence of '.'
+ char *ptr = strrchr(domainBuf, '.');
+
+ // If there are no '.' left in the domain buffer, we use the
+ // remainder which is the left-most domain component
+ if(ptr == NULL)
+ ptr = domainBuf;
+
+ // Get size of this component...
+ const size_t component_size = strlen(ptr);
+ // ... and use it to create a "gap" of the right size in our ABP
+ // format buffer
+ // Insert the domain component into the gap
+ if(ptr[0] == '.')
+ {
+ // If the component starts with a dot, we need
+ // to skip it when copying it into the ABP buffer
+ // Move excluding initial "||" but including final \0 (strlen-2+1 = strlen-1)
+ memmove(abpDomain+2+component_size-1, abpDomain+2, strlen(abpDomain)-1);
+ // Copy component bytes (excl. trailing null-byte)
+ memcpy(abpDomain+2, ptr+1, component_size-1);
+ }
+ else
+ {
+ // Otherwise, we copy the component as-is
+ memmove(abpDomain+2+component_size, abpDomain+2, strlen(abpDomain)-1);
+ // Copy component bytes (excl. trailing null-byte)
+ memcpy(abpDomain+2, ptr, component_size);
+ }
+ // Check if the constructed ABP-style domain is in the gravity list
+ const enum db_result abp_match = domain_in_list(abpDomain, stmt, "gravity", NULL);
+ if(config.debug & DEBUG_QUERIES)
+ logg("Checking if \"%s\" is in gravity: %s",
+ abpDomain, abp_match == FOUND ? "yes" : "no");
+ // Return for anything else than "not found" (e.g. "found" or "list not available")
+ if(abp_match != NOT_FOUND)
+ {
+ free(domainBuf);
+ free(abpDomain);
+ return abp_match;
+ }
+ // Truncate the domain buffer to the left of the
+ // last dot, effectively removing the last component
+ const ssize_t truncate_pos = strlen(domainBuf)-component_size;
+ if(truncate_pos < 1)
+ // This was already the last iteration
+ break;
+
+ // Put a null-byte at the truncation position
+ domainBuf[truncate_pos] = '\0';
+
+ // Move the ABP buffer to the right by one byte ...
+ memmove(abpDomain+3, abpDomain+2, strlen(abpDomain));
+ // ... and insert '.' for the next iteration
+ abpDomain[2] = '.';
+ }
+
+ free(domainBuf);
+ free(abpDomain);
+
+ // Domain not found in gravity list
+ return NOT_FOUND;
 }
 
 enum db_result in_blacklist(const char *domain, DNSCacheData *dns_cache, clientsData *client)