Merge pull request #1212 from pi-hole/tweak/dns_tests

Improve CI tests

src/dnsmasq_interface.c

@@ -1475,9 +1475,17 @@ static bool _FTL_check_blocking(int queryID, int domainID, int clientID, const c
 
 bool _FTL_CNAME(const char *domain, const struct crec *cpp, const int id, const char* file, const int line)
 {
+ if(config.debug & DEBUG_QUERIES)
+ {
+ const char *src = cpp != NULL ? cpp->flags & F_BIGNAME ? cpp->name.bname->name : cpp->name.sname : NULL;
+ logg("FTL_CNAME called with: src = %s, dst = %s, id = %d", src, domain, id);
+ }
+
  // Does the user want to skip deep CNAME inspection?
  if(!config.cname_inspection)
  {
+ if(config.debug & DEBUG_QUERIES)
+ logg("Skipping analysis as cname inspection is disabled");
  return false;
  }
 
@@ -1495,6 +1503,8 @@ bool _FTL_CNAME(const char *domain, const struct crec *cpp, const int id, const
  // This may happen e.g. if the original query was a PTR query
  // or "pi.hole" and we ignored them altogether
  unlock_shm();
+ if(config.debug & DEBUG_QUERIES)
+ logg("Skipping analysis as parent query is not found");
  return false;
  }
 
@@ -1505,6 +1515,8 @@ bool _FTL_CNAME(const char *domain, const struct crec *cpp, const int id, const
  {
  // Nothing to be done here
  unlock_shm();
+ if(config.debug & DEBUG_QUERIES)
+ logg("Skipping analysis as parent query is not valid");
  return false;
  }
 

test/dnsmasq.conf

@@ -6,12 +6,19 @@ log-facility=/var/log/pihole.log
 dnssec
 trust-anchor=.,20326,8,2,E06D44B80B8F1D39A95C0B0D7C65D08458E880409BBC683457104237C7F8EC8D
 
-# We hard-code OpenDNS as resolver for our dig tests as
-# - 8.8.8.8 turned out to not want to resolve HTTPS/SVCB,
-# - 1.1.1.1 doesn't implement ANY, and
-# - 9.9.9.9 doesn't answer reliably (many retries necessary).
-server=/netmeister.org/84.200.69.80
+# Explicity specify the local powerDNS recursor as forward destination for .ftl
+# This ensures FTL knows that ftl. is a local zone for which no DNSSEC
+# validation should be done. Otherwise, we'd get BOGUS for everything as the
+# root servers would tell us that ftl. does not exist
+server=/ftl/127.0.0.1#5555
+
+# Send the HTTPS/SVCB queries to the authorative server without detour over the
+# recursor because the latter returns SEVFAIL (connection between authorative
+# server and recursor is not encrypted)
+server=/https.ftl/127.0.0.1#5554
+server=/svcb.ftl/127.0.0.1#5554
+
+# Use local powerDNS recursor for everything else (DNSSEC enabled)
+server=127.0.0.1#5555
 
-# For everything else, we use Google's DNS server at 8.8.8.8
-server=8.8.8.8
 no-resolv

test/gravity.db.sql

@@ -188,24 +188,37 @@ CREATE TRIGGER tr_client_delete AFTER DELETE ON client
 
 /* ^^^ basic gravity table definition, taken from /advanced/Templates/gravity.db.sql ^^^ */
 /* vvv Test content following vvv */
-INSERT INTO domainlist VALUES(1,0,'whitelisted.test.pi-hole.net',1,1559928803,1559928803,'Migrated from /etc/pihole/whitelist.txt');
-INSERT INTO domainlist VALUES(2,0,'regex1.test.pi-hole.net',1,1559928803,1559928803,'');
+INSERT INTO domainlist VALUES(1,0,'whitelisted.ftl',1,1559928803,1559928803,'Migrated from /etc/pihole/whitelist.txt');
+INSERT INTO domainlist VALUES(2,0,'regex1.ftl',1,1559928803,1559928803,'');
 INSERT INTO domainlist VALUES(3,2,'regex2',1,1559928803,1559928803,'');
-INSERT INTO domainlist VALUES(4,2,'discourse',1,1559928803,1559928803,'');
+INSERT INTO domainlist VALUES(4,2,'^gravity-whitelisted',1,1559928803,1559928803,'');
 
-INSERT INTO domainlist VALUES(5,1,'blacklist-blocked.test.pi-hole.net',1,1559928803,1559928803,'Migrated from /etc/pihole/blacklist.txt');
-INSERT INTO domainlist VALUES(6,3,'regex[0-9].test.pi-hole.net',1,1559928803,1559928803,'Migrated from /etc/pihole/regex.list');
+/* Regular regex */
+INSERT INTO domainlist VALUES(5,1,'blacklisted.ftl',1,1559928803,1559928803,'Migrated from /etc/pihole/blacklist.txt');
+INSERT INTO domainlist VALUES(6,3,'regex[0-9].ftl',1,1559928803,1559928803,'Migrated from /etc/pihole/regex.list');
+
+/* Regex option testing */
+INSERT INTO domainlist VALUES(7,3,'^regex-NXDOMAIN$;reply=NXDOMAIN',1,1559928803,1559928803,'');
+INSERT INTO domainlist VALUES(8,3,'^regex-NODATA$;reply=NODATA',1,1559928803,1559928803,'');
+INSERT INTO domainlist VALUES(9,3,'^regex-REFUSED$;reply=REFUSED',1,1559928803,1559928803,'');
+INSERT INTO domainlist VALUES(10,3,'^regex-REPLYv4$;reply=1.2.3.4',1,1559928803,1559928803,'');
+INSERT INTO domainlist VALUES(11,3,'^regex-REPLYv6$;reply=fe80::1234',1,1559928803,1559928803,'');
+INSERT INTO domainlist VALUES(12,3,'^regex-REPLYv46$;reply=1.2.3.4;reply=fe80::1234',1,1559928803,1559928803,'');
+INSERT INTO domainlist VALUES(13,3,'^regex-A$;querytype=A',1,1559928803,1559928803,'');
+INSERT INTO domainlist VALUES(14,3,'^regex-notA$;querytype=!A',1,1559928803,1559928803,'');
+
+/* Other special domains */
+INSERT INTO domainlist VALUES(15,1,'blacklisted-group-disabled.com',1,1559928803,1559928803,'Entry disabled by a group');
 
 INSERT INTO adlist VALUES(1,'https://hosts-file.net/ad_servers.txt',1,1559928803,1559928803,'Migrated from /etc/pihole/adlists.list');
 
-INSERT INTO gravity VALUES('whitelisted.test.pi-hole.net',1);
-INSERT INTO gravity VALUES('gravity-blocked.test.pi-hole.net',1);
-INSERT INTO gravity VALUES('discourse.pi-hole.net',1);
+INSERT INTO gravity VALUES('whitelisted.ftl',1);
+INSERT INTO gravity VALUES('gravity.ftl',1);
+INSERT INTO gravity VALUES('gravity-whitelisted.ftl',1);
 INSERT INTO info VALUES("gravity_count",3);
 
 INSERT INTO "group" VALUES(1,0,'Test group',1559928803,1559928803,'A disabled test group');
-INSERT INTO domainlist VALUES(7,1,'blacklisted-group-disabled.com',1,1559928803,1559928803,'Entry disabled by a group');
-INSERT INTO domainlist_by_group VALUES(7,1);
+INSERT INTO domainlist_by_group VALUES(15,1);
 
 INSERT INTO domain_audit VALUES(1,'google.com',1559928803);
 

test/pdns/pdns.conf

@@ -0,0 +1,21 @@
+# Pi-hole: A black hole for Internet advertisements
+# (c) 2021 Pi-hole, LLC (https://pi-hole.net)
+# Network-wide ad blocking via your own hardware.
+#
+# Pi-hole testing environment configuration (authorative server)
+#
+# This file is copyright under the latest version of the EUPL.
+# Please see LICENSE file for your rights under this license.
+
+# Local DNS address and port
+local-address=127.0.0.1:5554
+local-ipv6=
+
+# Do not enforce TCP for ANY queries
+any-to-tcp=false
+
+# Launch gsqlite3
+launch=gsqlite3
+
+# Database location
+gsqlite3-database=/var/lib/powerdns/pdns.sqlite3

test/pdns/recursor.conf

@@ -0,0 +1,14 @@
+# Pi-hole: A black hole for Internet advertisements
+# (c) 2021 Pi-hole, LLC (https://pi-hole.net)
+# Network-wide ad blocking via your own hardware.
+#
+# Pi-hole testing environment configuration (recursive resolver)
+#
+# This file is copyright under the latest version of the EUPL.
+# Please see LICENSE file for your rights under this license.
+
+# Local DNS address and port
+local-address=127.0.0.1:5555
+
+# Use authorative server for ftl. and arpa. zones
+forward-zones=ftl=127.0.0.1:5554,168.192.in-addr.arpa=127.0.0.1:5554,ip6.arpa=127.0.0.1:5554

test/pdns/setup.sh

@@ -0,0 +1,134 @@
+#!/bin/bash
+
+echo "************ Installing PowerDNS configuration ************"
+
+# Delete possibly existing zone database
+mkdir -p /var/lib/powerdns/
+rm /var/lib/powerdns/pdns.sqlite3 2> /dev/null
+
+# Install config files
+if [ -d /etc/powerdns ]; then
+ # Debian
+ cp test/pdns/pdns.conf /etc/powerdns/pdns.conf
+ cp test/pdns/recursor.conf /etc/powerdns/recursor.conf
+elif [ -d /etc/pdns ]; then
+ # Alpine
+ cp test/pdns/pdns.conf /etc/pdns/pdns.conf
+ cp test/pdns/recursor.conf /etc/pdns/recursor.conf
+
+ # TODO: Remove this once the containers are updated
+ apk add --no-cache pdns-doc
+else
+ echo "Error: Unable to determine powerDNS config directory"
+ exit 1
+fi
+
+# Create zone database
+if [ -f /usr/share/doc/pdns-backend-sqlite3/schema.sqlite3.sql ]; then
+ # Debian
+ sqlite3 /var/lib/powerdns/pdns.sqlite3 < /usr/share/doc/pdns-backend-sqlite3/schema.sqlite3.sql
+elif [ -f /usr/share/doc/pdns/schema.sqlite3.sql ]; then
+ # Alpine
+ sqlite3 /var/lib/powerdns/pdns.sqlite3 < /usr/share/doc/pdns/schema.sqlite3.sql
+else
+ echo "Error: powerDNS SQL schema not found"
+ exit 1
+fi
+# Create zone ftl
+pdnsutil create-zone ftl ns1.ftl
+pdnsutil add-record ftl. . SOA "ns1.ftl. hostmaster.ftl. 1 10800 3600 604800 3600"
+
+# Create A records
+pdnsutil add-record ftl. a A 192.168.1.1
+pdnsutil add-record ftl. gravity A 192.168.1.2
+pdnsutil add-record ftl. blacklisted A 192.168.1.3
+pdnsutil add-record ftl. whitelisted A 192.168.1.4
+pdnsutil add-record ftl. gravity-whitelisted A 192.168.1.5
+pdnsutil add-record ftl. regex1 A 192.168.2.1
+pdnsutil add-record ftl. regex2 A 192.168.2.2
+pdnsutil add-record ftl. regex5 A 192.168.2.3
+pdnsutil add-record ftl. regexA A 192.168.2.4
+pdnsutil add-record ftl. regex-REPLYv4 A 192.168.2.5
+pdnsutil add-record ftl. regex-REPLYv6 A 192.168.2.6
+pdnsutil add-record ftl. regex-REPLYv46 A 192.168.2.7
+pdnsutil add-record ftl. regex-A A 192.168.2.8
+pdnsutil add-record ftl. regex-notA A 192.168.2.9
+pdnsutil add-record ftl. any A 192.168.3.1
+
+# Create AAAA records
+pdnsutil add-record ftl. aaaa AAAA fe80::1c01
+pdnsutil add-record ftl. regex-REPLYv4 AAAA fe80::2c01
+pdnsutil add-record ftl. regex-REPLYv6 AAAA fe80::2c02
+pdnsutil add-record ftl. regex-REPLYv46 AAAA fe80::2c03
+pdnsutil add-record ftl. any AAAA fe80::3c01
+
+# Create CNAME records
+pdnsutil add-record ftl. cname-1 CNAME gravity.ftl
+pdnsutil add-record ftl. cname-2 CNAME cname-1.ftl
+pdnsutil add-record ftl. cname-3 CNAME cname-2.ftl
+pdnsutil add-record ftl. cname-4 CNAME cname-3.ftl
+pdnsutil add-record ftl. cname-5 CNAME cname-4.ftl
+pdnsutil add-record ftl. cname-6 CNAME cname-5.ftl
+pdnsutil add-record ftl. cname-7 CNAME cname-6.ftl
+pdnsutil add-record ftl. cname-ok CNAME a.ftl
+
+# Create CNAME for SOA test domain
+pdnsutil add-record ftl. soa CNAME ftl
+
+# Create PTR records
+pdnsutil add-record ftl. ptr PTR ptr.ftl.
+
+# Other testing records
+pdnsutil add-record ftl. srv SRV "0 1 80 a.ftl"
+pdnsutil add-record ftl. txt TXT "\"Some example text\""
+# We want this to output $1 without expansion
+# shellcheck disable=SC2016
+pdnsutil add-record ftl. naptr NAPTR '10 10 "u" "smtp+E2U" "!.*([^\.]+[^\.]+)$!mailto:postmaster@$1!i" .'
+pdnsutil add-record ftl. naptr NAPTR '20 10 "s" "http+N2L+N2C+N2R" "" ftl.'
+pdnsutil add-record ftl. mx MX "50 ns1.ftl."
+
+# SVCB + HTTPS
+if ! pdnsutil add-record ftl. svcb SVCB '1 port="80"'; then
+ # see RFC3597: Handling of Unknown DNS Resource Record (RR) Types
+ # and https://ypcs.fi/howto/2020/09/30/announce-https-via-dns/
+ pdnsutil add-record ftl. svcb TYPE64 "\# 13 000109706F72743D2238302200"
+fi
+
+# HTTPS
+if ! pdnsutil add-record ftl. https HTTPS '1 . alpn="h3,h2"'; then
+ # comment above applies
+ pdnsutil add-record ftl. https TYPE65 "\# 15 000100000100080322683303683222"
+fi
+
+# Create reverse lookup zone
+pdnsutil create-zone arpa ns1.ftl
+pdnsutil add-record arpa. 1.1.168.192.in-addr PTR ftl.
+pdnsutil add-record arpa. 2.1.168.192.in-addr PTR a.ftl.
+pdnsutil add-record arpa. 1.0.c.1.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.8.e.f.ip6 PTR ftl.
+pdnsutil add-record arpa. 2.0.c.1.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.8.e.f.ip6 PTR aaaa.ftl.
+
+# Calculates the ‘ordername’ and ‘auth’ fields for all zones so they comply with
+# DNSSEC settings. Can be used to fix up migrated data. Can always safely be
+# run, it does no harm.
+pdnsutil rectify-all-zones
+
+# Do final checking
+pdnsutil check-zone ftl
+pdnsutil check-zone arpa
+
+echo "********* Done installing PowerDNS configuration **********"
+
+# Start services
+if command -v service; then
+ # Debian
+ service pdns restart
+ service pdns-recursor restart
+else
+ # Alpine
+ killall pdns_server
+ pdns_server --daemon
+ # Have to create the socketdir or the recursor will fails to start
+ mkdir -p /var/run/pdns-recursor
+ killall pdns_recursor
+ pdns_recursor --daemon
+fi

test/run.sh

@@ -24,7 +24,9 @@ rm -f /etc/pihole/gravity.db /etc/pihole/pihole-FTL.db /var/log/pihole.log /var/
 
 # Create necessary directories and files
 mkdir -p /home/pihole /etc/pihole /run/pihole /var/log
-touch /var/log/pihole-FTL.log /var/log/pihole.log /run/pihole-FTL.pid /run/pihole-FTL.port
+echo "" > /var/log/pihole-FTL.log
+echo "" > /var/log/pihole.log
+touch /run/pihole-FTL.pid /run/pihole-FTL.port dig.log ptr.log
 chown pihole:pihole /etc/pihole /run/pihole /var/log/pihole.log /var/log/pihole-FTL.log /run/pihole-FTL.pid /run/pihole-FTL.port
 
 # Copy binary into a location the new user pihole can access
@@ -51,6 +53,9 @@ cp test/pihole-FTL.conf /etc/pihole/pihole-FTL.conf
 # Prepare dnsmasq.conf
 cp test/dnsmasq.conf /etc/dnsmasq.conf
 
+# Prepare local powerDNS resolver
+bash test/pdns/setup.sh
+
 # Set restrictive umask
 OLDUMASK=$(umask)
 umask 0022
@@ -105,6 +110,9 @@ if [[ $RET != 0 ]]; then
  echo -n "dig.log: "
  curl_to_tricorder ./dig.log
  echo ""
+ echo -n "ptr.log: "
+ curl_to_tricorder ./ptr.log
+ echo ""
 fi
 
 # Kill pihole-FTL after having completed tests