Merge pull request #455 from pi-hole/new/network-details

Process ARP data for new network table

.gitignore

@@ -19,3 +19,7 @@ version*
 /pihole-FTL.conf
 /pihole-FTL.db
 /pihole-FTL.log
+
+# aux files
+aux/manuf.data
+aux/macvendor.db

FTL.h

@@ -69,7 +69,7 @@
 #define MAXITER 1000
 
 // FTLDNS enums
-enum { DATABASE_WRITE_TIMER, EXIT_TIMER, GC_TIMER, LISTS_TIMER, REGEX_TIMER };
+enum { DATABASE_WRITE_TIMER, EXIT_TIMER, GC_TIMER, LISTS_TIMER, REGEX_TIMER, ARP_TIMER, LAST_TIMER };
 enum { QUERIES, FORWARDED, CLIENTS, DOMAINS, OVERTIME, WILDCARD };
 enum { DNSSEC_UNSPECIFIED, DNSSEC_SECURE, DNSSEC_INSECURE, DNSSEC_BOGUS, DNSSEC_ABANDONED, DNSSEC_UNKNOWN };
 enum { QUERY_UNKNOWN, QUERY_GRAVITY, QUERY_FORWARDED, QUERY_CACHE, QUERY_WILDCARD, QUERY_BLACKLIST, QUERY_EXTERNAL_BLOCKED };
@@ -80,6 +80,11 @@ enum { MODE_IP, MODE_NX, MODE_NULL, MODE_IP_NODATA_AAAA, MODE_NODATA };
 enum { REGEX_UNKNOWN, REGEX_BLOCKED, REGEX_NOTBLOCKED };
 enum { BLOCKING_DISABLED, BLOCKING_ENABLED, BLOCKING_UNKNOWN };
 
+// Database table "ftl"
+enum { DB_VERSION, DB_LASTTIMESTAMP, DB_FIRSTCOUNTERTIMESTAMP };
+// Database table "counters"
+enum { DB_TOTALQUERIES, DB_BLOCKEDQUERIES };
+
 // Privacy mode constants
 #define HIDDEN_DOMAIN "hidden"
 #define HIDDEN_CLIENT "0.0.0.0"
@@ -93,6 +98,7 @@ typedef struct {
  char* port;
  char* db;
  char* socketfile;
+ char* macvendordb;
 } FTLFileNamesStruct;
 
 typedef struct {
@@ -144,6 +150,7 @@ typedef struct {
  bool regex_debugmode;
  bool analyze_only_A_AAAA;
  bool DBimport;
+ bool parse_arp_cache;
 } ConfigStruct;
 
 // Dynamic structs
@@ -182,6 +189,8 @@ typedef struct {
  unsigned long long ippos;
  unsigned long long namepos;
  bool new;
+ time_t lastQuery;
+ unsigned int numQueriesARP;
 } clientsDataStruct;
 
 typedef struct {
@@ -208,11 +217,15 @@ typedef struct {
 } whitelistStruct;
 
 // Prepare timers, used mainly for debugging purposes
-#define NUMTIMERS 5
+#define NUMTIMERS LAST_TIMER
 
 // Used to check memory integrity in various structs
 #define MAGICBYTE 0x57
 
+// Some magic database constants
+#define DB_FAILED -2
+#define DB_NODATA -1
+
 extern logFileNamesStruct files;
 extern FTLFileNamesStruct FTLfiles;
 extern countersStruct *counters;
@@ -250,7 +263,6 @@ extern long int lastdbindex;
 extern bool travis;
 extern bool DBdeleteoldqueries;
 extern bool rereadgravity;
-extern long int lastDBimportedtimestamp;
 extern bool ipv4telnet, ipv6telnet;
 extern bool istelnet[MAXCONNS];
 

Makefile

@@ -14,7 +14,7 @@ DNSMASQOPTS = -DHAVE_DNSSEC -DHAVE_DNSSEC_STATIC
 # Flags for compiling with libidn2: -DHAVE_LIBIDN2 -DIDN2_VERSION_NUMBER=0x02000003
 
 FTLDEPS = FTL.h routines.h version.h api.h dnsmasq_interface.h shmem.h
-FTLOBJ = main.o memory.o log.o daemon.o datastructure.o signals.o socket.o request.o grep.o setupVars.o args.o gc.o config.o database.o msgpack.o api.o dnsmasq_interface.o resolve.o regex.o shmem.o capabilities.o
+FTLOBJ = main.o memory.o log.o daemon.o datastructure.o signals.o socket.o request.o grep.o setupVars.o args.o gc.o config.o database.o msgpack.o api.o dnsmasq_interface.o resolve.o regex.o shmem.o capabilities.o networktable.o
 
 DNSMASQDEPS = config.h dhcp-protocol.h dns-protocol.h radv-protocol.h dhcp6-protocol.h dnsmasq.h ip6addr.h metrics.h ../dnsmasq_interface.h
 DNSMASQOBJ = arp.o dbus.o domain.o lease.o outpacket.o rrfilter.o auth.o dhcp6.o edns0.o log.o poll.o slaac.o blockdata.o dhcp.o forward.o loop.o radv.o tables.o bpf.o dhcp-common.o helper.o netlink.o rfc1035.o tftp.o cache.o dnsmasq.o inotify.o network.o rfc2131.o util.o conntrack.o dnssec.o ipset.o option.o rfc3315.o crypto.o dump.o ubus.o metrics.o

aux/macvendor.py

@@ -0,0 +1,83 @@
+# Pi-hole: A black hole for Internet advertisements
+# (c) 2019 Pi-hole, LLC (https://pi-hole.net)
+# Network-wide ad blocking via your own hardware.
+#
+# FTL Engine - auxiliary files
+# MAC -> Vendor database generator
+#
+# This is a python3 script
+#
+# This file is copyright under the latest version of the EUPL.
+# Please see LICENSE file for your rights under this license.
+
+import os
+import re
+import urllib.request
+import sqlite3
+
+# Download raw data from Wireshark's website
+# We use the official URL recommended in the header of this file
+print("Downloading...")
+urllib.request.urlretrieve("https://code.wireshark.org/review/gitweb?p=wireshark.git;a=blob_plain;f=manuf", "manuf.data")
+print("...done")
+
+# Read file into memory and process lines
+manuf = open("manuf.data", "r")
+data = []
+print("Processing...")
+for line in manuf:
+ line = line.strip()
+
+ # Skip comments and empty lines
+ if line[1] == "#" or line == "":
+ continue
+
+ # Remove quotation marks as these might interfere with later INSERT / UPDATE commands
+ line = re.sub("\'|\"","", line)
+ # \s = Unicode whitespace characters, including [ \t\n\r\f\v]
+ cols = re.split("\s\s+|\t", line)
+ # Use try/except chain to catch empty/incomplete lines without failing hard
+ try:
+ # Strip whitespace and quotation marks (some entries are incomplete and cause errors with the CSV parser otherwise)
+ mac = cols[0].strip().strip("\"")
+ except:
+ continue
+ try:
+ desc_short = cols[1].strip().strip("\"")
+ except:
+ desc_short = ""
+ try:
+ desc_long = cols[2].strip().strip("\"")
+ except:
+ desc_long = ""
+
+ # Only add long description where available
+ # There are a few vendors for which only the
+ # short description field is used
+ if(desc_long):
+ data.append([mac, desc_long])
+ else:
+ data.append([mac, desc_short])
+print("...done")
+manuf.close()
+
+# Create database
+database = "macvendor.db"
+
+# Try to delete old database file, pass if no old file exists
+try:
+ os.remove(database)
+except OSError:
+ pass
+
+print("Generating database...")
+con = sqlite3.connect(database)
+cur = con.cursor()
+cur.execute("CREATE TABLE macvendor (mac TEXT NOT NULL, vendor TEXT NOT NULL, PRIMARY KEY (mac))")
+cur.executemany("INSERT INTO macvendor (mac, vendor) VALUES (?, ?);", data)
+con.commit()
+print("...done.")
+print("Optimizing database...")
+con.execute("VACUUM")
+print("...done")
+print("Lines inserted into database:", cur.rowcount)

config.c

@@ -319,6 +319,22 @@ void read_FTLconf(void)
  // AUDITLISTFILE
  getpath(fp, "AUDITLISTFILE", "/etc/pihole/auditlog.list", &files.auditlist);
 
+ // MACVENDORDB
+ getpath(fp, "MACVENDORDB", "/etc/pihole/macvendor.db", &FTLfiles.macvendordb);
+
+ // PARSE_ARP_CACHE
+ // defaults to: true
+ config.parse_arp_cache = true;
+ buffer = parse_FTLconf(fp, "PARSE_ARP_CACHE");
+
+ if(buffer != NULL && strcasecmp(buffer, "false") == 0)
+ config.parse_arp_cache = false;
+
+ if(config.parse_arp_cache)
+ logg(" PARSE_ARP_CACHE: Active");
+ else
+ logg(" PARSE_ARP_CACHE: Inactive");
+
  logg("Finished config file parsing");
 
  // Release memory