Use shared memory objects for data

FTL.h

@@ -52,16 +52,6 @@
 
 #include "routines.h"
 
-// Next we define the step size in which the struct arrays are reallocated if they
-// grow too large. This number should be large enough so that reallocation does not
-// have to run very often, but should be as small as possible to avoid wasting memory
-#define QUERIESALLOCSTEP 10000
-#define FORWARDEDALLOCSTEP 4
-#define CLIENTSALLOCSTEP 10
-#define DOMAINSALLOCSTEP 1000
-#define OVERTIMEALLOCSTEP 100
-#define WILDCARDALLOCSTEP 100
-
 #define SOCKETBUFFERLEN 1024
 
 // How often do we garbage collect (to ensure we only have data fitting to the MAXLOGAGE defined above)? [seconds]
@@ -166,7 +156,7 @@ typedef struct {
  int domainID;
  int clientID;
  int forwardID;
- bool db;
+ sqlite3_int64 db;
  int id; // the ID is a (signed) int in dnsmasq, so no need for a long int here
  bool complete;
  unsigned char privacylevel;
@@ -180,25 +170,25 @@ typedef struct {
  unsigned char magic;
  int count;
  int failed;
- char *ip;
- char *name;
+ unsigned long long ippos;
+ unsigned long long namepos;
  bool new;
 } forwardedDataStruct;
 
 typedef struct {
  unsigned char magic;
  int count;
  int blockedcount;
- char *ip;
- char *name;
+ unsigned long long ippos;
+ unsigned long long namepos;
  bool new;
 } clientsDataStruct;
 
 typedef struct {
  unsigned char magic;
  int count;
  int blockedcount;
- char *domain;
+ unsigned long long domainpos;
  unsigned char regexmatch;
 } domainsDataStruct;
 
@@ -209,9 +199,7 @@ typedef struct {
  int blocked;
  int cached;
  int forwarded;
- int clientnum;
- int *clientdata;
- int querytypedata[7];
+ int querytypedata[TYPE_MAX-1];
 } overTimeDataStruct;
 
 typedef struct {
@@ -227,7 +215,7 @@ typedef struct {
 
 extern logFileNamesStruct files;
 extern FTLFileNamesStruct FTLfiles;
-extern countersStruct counters;
+extern countersStruct *counters;
 extern ConfigStruct config;
 
 extern queriesDataStruct *queries;
@@ -236,6 +224,10 @@ extern clientsDataStruct *clients;
 extern domainsDataStruct *domains;
 extern overTimeDataStruct *overTime;
 
+/// Indexed by client ID, then time index (like `overTime`).
+/// This gets automatically updated whenever a new client or overTime slot is added.
+extern int **overTimeClientData;
+
 extern FILE *logfile;
 extern volatile sig_atomic_t killed;
 

Makefile

@@ -13,8 +13,8 @@ DNSMASQOPTS = -DHAVE_DNSSEC -DHAVE_DNSSEC_STATIC
 # Flags for compiling with libidn : -DHAVE_IDN
 # Flags for compiling with libidn2: -DHAVE_LIBIDN2 -DIDN2_VERSION_NUMBER=0x02000003
 
-FTLDEPS = FTL.h routines.h version.h api.h dnsmasq_interface.h
-FTLOBJ = main.o memory.o log.o daemon.o datastructure.o signals.o socket.o request.o grep.o setupVars.o args.o threads.o gc.o config.o database.o msgpack.o api.o dnsmasq_interface.o resolve.o regex.o 
+FTLDEPS = FTL.h routines.h version.h api.h dnsmasq_interface.h shmem.h
+FTLOBJ = main.o memory.o log.o daemon.o datastructure.o signals.o socket.o request.o grep.o setupVars.o args.o gc.o config.o database.o msgpack.o api.o dnsmasq_interface.o resolve.o regex.o shmem.o
 
 DNSMASQDEPS = config.h dhcp-protocol.h dns-protocol.h radv-protocol.h dhcp6-protocol.h dnsmasq.h ip6addr.h metrics.h
 DNSMASQOBJ = arp.o dbus.o domain.o lease.o outpacket.o rrfilter.o auth.o dhcp6.o edns0.o log.o poll.o slaac.o blockdata.o dhcp.o forward.o loop.o radv.o tables.o bpf.o dhcp-common.o helper.o netlink.o rfc1035.o tftp.o cache.o dnsmasq.o inotify.o network.o rfc2131.o util.o conntrack.o dnssec.o ipset.o option.o rfc3315.o crypto.o dump.o ubus.o metrics.o
@@ -51,7 +51,7 @@ CCFLAGS=-std=gnu11 -I$(IDIR) -Wall -Wextra -Wno-unused-parameter -D_FILE_OFFSET_
 # for dnsmasq we need the nettle crypto library and the gmp maths library
 # We link the two libraries statically. Althougth this increases the binary file size by about 1 MB, it saves about 5 MB of shared libraries and makes deployment easier
 #LIBS=-pthread -lnettle -lgmp -lhogweed
-LIBS=-pthread -Wl,-Bstatic -L/usr/local/lib -lhogweed -lgmp -lnettle -Wl,-Bdynamic
+LIBS=-pthread -Wl,-Bstatic -L/usr/local/lib -lhogweed -lgmp -lnettle -Wl,-Bdynamic -lrt
 # Flags for compiling with libidn : -lidn
 # Flags for compiling with libidn2: -lidn2