diff --git a/.circleci/config.yml b/.circleci/config.yml index c09390bf0..749309413 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,65 +1,81 @@ version: 2 -.job_template: &job_template - docker: - - image: pihole/ftl-build:$CIRCLE_JOB +.job_steps: &job_steps steps: - checkout - run: name: "Setup" command: | make clean + if [[ $CIRCLE_JOB == *"qemu"* ]] ; then sudo docker run --rm --privileged multiarch/qemu-user-static:register --reset ; fi - run: name: "Build" + no_output_timeout: 30m command: | BRANCH=$([ -z "$CIRCLE_TAG" ] && echo "$CIRCLE_BRANCH" || echo "master") - - make GIT_BRANCH="${BRANCH}" GIT_TAG="${CIRCLE_TAG}" + [[ $CIRCLE_JOB == *"qemu"* ]] && DOCKERIFNEEDED="docker run --rm -v $(pwd):/workspace -w /workspace pihole/ftl-build:arm-qemu " + $DOCKERIFNEEDED make GIT_BRANCH="${BRANCH}" GIT_TAG="${CIRCLE_TAG}" STATIC="${STATIC}" file pihole-FTL - run: name: "Upload" command: | - FOLDER=$([ -z "$CIRCLE_TAG" ] && echo "$CIRCLE_BRANCH" || echo "$CIRCLE_TAG") + [[ "$CIRCLE_PROJECT_REPONAME" == "FTL" && "$CIRCLE_PROJECT_USERNAME" == "pi-hole" ]] || exit 0 + DIR="${CIRCLE_TAG:-${CIRCLE_BRANCH}}" mv pihole-FTL "${BIN_NAME}" sha1sum pihole-FTL-* > ${BIN_NAME}.sha1 - wget https://ftl.pi-hole.net:8080/FTL-client - chmod +x ./FTL-client - [[ "$CIRCLE_PR_NUMBER" == "" ]] && ./FTL-client "${FOLDER}" "${BIN_NAME}" "${FTL_SECRET}" - [[ "$CIRCLE_PR_NUMBER" == "" ]] && ./FTL-client "${FOLDER}" "${BIN_NAME}.sha1" "${FTL_SECRET}" - rm ./FTL-client - ls -lah . + mkdir -p ~/.ssh/ + ssh-keyscan -H $SSH_HOST >> ~/.ssh/known_hosts + sftp -b - $SSH_USER@$SSH_HOST <<< "-mkdir ${DIR} + put ${BIN_NAME}* ${DIR}" + mv "${BIN_NAME}" pihole-FTL + - run: + name: "Test" + command: | + test/run.sh + +.docker_template: &docker_template + docker: + - image: pihole/ftl-build:v1.0-$CIRCLE_JOB + <<: *job_steps jobs: arm: - <<: *job_template + <<: *docker_template environment: BIN_NAME: "pihole-FTL-arm-linux-gnueabi" armhf: - <<: *job_template + <<: *docker_template environment: BIN_NAME: "pihole-FTL-arm-linux-gnueabihf" aarch64: - <<: *job_template + <<: *docker_template environment: BIN_NAME: "pihole-FTL-aarch64-linux-gnu" x86_64: - <<: *job_template + <<: *docker_template environment: BIN_NAME: "pihole-FTL-linux-x86_64" x86_64-musl: - <<: *job_template + <<: *docker_template environment: BIN_NAME: "pihole-FTL-musl-linux-x86_64" x86_32: - <<: *job_template + <<: *docker_template environment: BIN_NAME: "pihole-FTL-linux-x86_32" + arm-qemu: + machine: + enabled: true + environment: + BIN_NAME: "pihole-FTL-armel-native" + <<: *job_steps + workflows: version: 2 build: @@ -68,6 +84,10 @@ workflows: filters: tags: only: /^v.*/ + - arm-qemu: + filters: + tags: + only: /^v.*/ - armhf: filters: tags: @@ -80,7 +100,10 @@ workflows: filters: tags: only: /^v.*/ -# - x86_64-musl + - x86_64-musl: + filters: + tags: + only: /^v.*/ - x86_32: filters: tags: diff --git a/.gitignore b/.gitignore index 005afc238..e878f6888 100644 --- a/.gitignore +++ b/.gitignore @@ -1,10 +1,10 @@ -# Generated binary files -/obj/ -/dnsmasq/obj +# Object files output by the assembler +/build + +# Generated binary pihole-FTL -socket-test -# Versioning files +# Versioning files (generated by Makefile) version* # CMake files @@ -13,13 +13,9 @@ version* # IDE files .idea/ +*.sw* +.vscode/ -# Test-generated files -/pihole.log -/pihole-FTL.conf -/pihole-FTL.db -/pihole-FTL.log - -# aux files -aux/manuf.data -aux/macvendor.db +# MAC->Vendor database files +tools/manuf.data +tools/macvendor.db diff --git a/.gitmodules b/.gitmodules index 5f5a83caf..e69de29bb 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,6 +0,0 @@ -[submodule "test/libs/bats"] - path = test/libs/bats - url = https://github.com/sstephenson/bats -[submodule "test/libs/bats-support"] - path = test/libs/bats-support - url = https://github.com/ztombol/bats-support diff --git a/FTL.h b/FTL.h deleted file mode 100644 index b18408797..000000000 --- a/FTL.h +++ /dev/null @@ -1,318 +0,0 @@ -/* Pi-hole: A black hole for Internet advertisements -* (c) 2017 Pi-hole, LLC (https://pi-hole.net) -* Network-wide ad blocking via your own hardware. -* -* FTL Engine -* Global definitions -* -* This file is copyright under the latest version of the EUPL. -* Please see LICENSE file for your rights under this license. */ - -#define __USE_XOPEN -#define _GNU_SOURCE -#include -// variable argument lists -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -// struct sockaddr_in -#include -// char* inet_ntoa(struct in_addr in) -#include -// getnameinfo(); -#include -#include -#include -#include -//#include -#include -// syslog -#include -// tolower() -#include -// Unix socket -#include -// Interfaces -#include -#include - -// Define MIN and MAX macros, use them only when x and y are of the same type -#define MAX(x,y) (((x) > (y)) ? (x) : (y)) -// MIN(x,y) is already defined in dnsmasq.h - -#include "routines.h" - -#define SOCKETBUFFERLEN 1024 - -// How often do we garbage collect (to ensure we only have data fitting to the MAXLOGAGE defined above)? [seconds] -// Default: 3600 (once per hour) -#define GCinterval 3600 - -// Delay applied to the garbage collecting [seconds] -// Default: -60 (one minute before a full hour) -#define GCdelay (-60) - -// How many client connection do we accept at once? -#define MAXCONNS 255 - -// Over how many queries do we iterate at most when trying to find a match? -#define MAXITER 1000 - -// How many hours do we want to store in FTL's memory? [hours] -#define MAXLOGAGE 24 - -// Interval for overTime data [seconds] -// Default: 600 (10 minute intervals) -#define OVERTIME_INTERVAL 600 - -// How many overTime slots do we need? -// (24+1) hours * number of intervals per hour -// We need to be able to hold 25 hours as we need some reserve -// due to that GC is only running once an hours so the shown data -// can be 24 hours + 59 minutes -#define OVERTIME_SLOTS ((MAXLOGAGE+1)*3600/OVERTIME_INTERVAL) - -// Interval for resolving NEW client and upstream server host names [seconds] -// Default: 60 (once every minute) -#define RESOLVE_INTERVAL 60 - -// Interval for re-resolving ALL known host names [seconds] -// Default: 3600 (once every hour) -#define RERESOLVE_INTERVAL 3600 - -// FTLDNS enums -enum { DATABASE_WRITE_TIMER, EXIT_TIMER, GC_TIMER, LISTS_TIMER, REGEX_TIMER, ARP_TIMER, LAST_TIMER }; -enum { QUERIES, FORWARDED, CLIENTS, DOMAINS, OVERTIME, WILDCARD }; -enum { DNSSEC_UNSPECIFIED, DNSSEC_SECURE, DNSSEC_INSECURE, DNSSEC_BOGUS, DNSSEC_ABANDONED, DNSSEC_UNKNOWN }; -enum { QUERY_UNKNOWN, QUERY_GRAVITY, QUERY_FORWARDED, QUERY_CACHE, QUERY_WILDCARD, QUERY_BLACKLIST, QUERY_EXTERNAL_BLOCKED_IP, QUERY_EXTERNAL_BLOCKED_NULL, QUERY_EXTERNAL_BLOCKED_NXRA }; -enum { TYPE_A = 1, TYPE_AAAA, TYPE_ANY, TYPE_SRV, TYPE_SOA, TYPE_PTR, TYPE_TXT, TYPE_MAX }; -enum { REPLY_UNKNOWN, REPLY_NODATA, REPLY_NXDOMAIN, REPLY_CNAME, REPLY_IP, REPLY_DOMAIN, REPLY_RRNAME, REPLY_SERVFAIL, REPLY_REFUSED, REPLY_NOTIMP, REPLY_OTHER }; -enum { PRIVACY_SHOW_ALL = 0, PRIVACY_HIDE_DOMAINS, PRIVACY_HIDE_DOMAINS_CLIENTS, PRIVACY_MAXIMUM, PRIVACY_NOSTATS }; -enum { MODE_IP, MODE_NX, MODE_NULL, MODE_IP_NODATA_AAAA, MODE_NODATA }; -enum { REGEX_UNKNOWN, REGEX_BLOCKED, REGEX_NOTBLOCKED }; -enum { BLOCKING_DISABLED, BLOCKING_ENABLED, BLOCKING_UNKNOWN }; -enum { - DEBUG_DATABASE = (1 << 0), /* 00000000 00000001 */ - DEBUG_NETWORKING = (1 << 1), /* 00000000 00000010 */ - DEBUG_LOCKS = (1 << 2), /* 00000000 00000100 */ - DEBUG_QUERIES = (1 << 3), /* 00000000 00001000 */ - DEBUG_FLAGS = (1 << 4), /* 00000000 00010000 */ - DEBUG_SHMEM = (1 << 5), /* 00000000 00100000 */ - DEBUG_GC = (1 << 6), /* 00000000 01000000 */ - DEBUG_ARP = (1 << 7), /* 00000000 10000000 */ - DEBUG_REGEX = (1 << 8), /* 00000001 00000000 */ - DEBUG_API = (1 << 9), /* 00000010 00000000 */ - DEBUG_OVERTIME = (1 << 10), /* 00000100 00000000 */ - DEBUG_EXTBLOCKED = (1 << 11), /* 00001000 00000000 */ - DEBUG_CAPS = (1 << 12), /* 00010000 00000000 */ -}; - -// Database table "ftl" -enum { DB_VERSION, DB_LASTTIMESTAMP, DB_FIRSTCOUNTERTIMESTAMP }; -// Database table "counters" -enum { DB_TOTALQUERIES, DB_BLOCKEDQUERIES }; - -// Privacy mode constants -#define HIDDEN_DOMAIN "hidden" -#define HIDDEN_CLIENT "0.0.0.0" - -// Static structs -typedef struct { - const char* conf; - const char* snapConf; - char* log; - char* pid; - char* port; - char* db; - char* socketfile; - char* macvendordb; -} FTLFileNamesStruct; - -typedef struct { - char* whitelist; - char* blacklist; - char* gravity; - char* regexlist; - char* setupVars; - char* auditlist; -} logFileNamesStruct; - -typedef struct { - int queries; - int blocked; - int cached; - int unknown; - int forwarded; - int clients; - int domains; - int queries_MAX; - int forwarded_MAX; - int clients_MAX; - int domains_MAX; - int strings_MAX; - int gravity; - int gravity_conf; - int querytype[TYPE_MAX-1]; - int forwardedqueries; - int reply_NODATA; - int reply_NXDOMAIN; - int reply_CNAME; - int reply_IP; - int reply_domain; -} countersStruct; - -typedef struct { - int maxDBdays; - int DBinterval; - int port; - int maxlogage; - int16_t debug; - unsigned char privacylevel; - unsigned char blockingmode; - bool socket_listenlocal; - bool analyze_AAAA; - bool resolveIPv6; - bool resolveIPv4; - bool ignore_localhost; - bool analyze_only_A_AAAA; - bool DBimport; - bool parse_arp_cache; -} ConfigStruct; - -// Dynamic structs -typedef struct { - unsigned char magic; - unsigned char type; - unsigned char status; - unsigned char privacylevel; - unsigned char reply; - unsigned char dnssec; - time_t timestamp; - int domainID; - int clientID; - int forwardID; - int id; // the ID is a (signed) int in dnsmasq, so no need for a long int here - unsigned long response; // saved in units of 1/10 milliseconds (1 = 0.1ms, 2 = 0.2ms, 2500 = 250.0ms, etc.) - int64_t db; - unsigned int timeidx; - bool complete; -} queriesDataStruct; - -typedef struct { - unsigned char magic; - size_t ippos; - size_t namepos; - int count; - int failed; - bool new; -} forwardedDataStruct; - -typedef struct { - unsigned char magic; - size_t ippos; - size_t namepos; - time_t lastQuery; - int count; - int blockedcount; - int overTime[OVERTIME_SLOTS]; - unsigned int numQueriesARP; - bool new; -} clientsDataStruct; - -typedef struct { - unsigned char magic; - unsigned char regexmatch; - size_t domainpos; - int count; - int blockedcount; -} domainsDataStruct; - -typedef struct { - unsigned char magic; - time_t timestamp; - int total; - int blocked; - int cached; - int forwarded; - int querytypedata[TYPE_MAX-1]; -} overTimeDataStruct; - -typedef struct { - char **domains; - int count; -} whitelistStruct; - -typedef struct { - int version; - unsigned int global_shm_counter; - unsigned int next_str_pos; -} ShmSettings; - -// Prepare timers, used mainly for debugging purposes -#define NUMTIMERS LAST_TIMER - -// Used to check memory integrity in various structs -#define MAGICBYTE 0x57 - -// Some magic database constants -#define DB_FAILED -2 -#define DB_NODATA -1 - -extern logFileNamesStruct files; -extern FTLFileNamesStruct FTLfiles; -extern countersStruct *counters; -extern ConfigStruct config; - -extern queriesDataStruct *queries; -extern forwardedDataStruct *forwarded; -extern clientsDataStruct *clients; -extern domainsDataStruct *domains; -extern overTimeDataStruct *overTime; - -// Used in gc.c, memory.c, resolve.c, signals.c, and socket.c -extern volatile sig_atomic_t killed; -// Used in api.c, grep.c, and dnsmasq_interface.c -extern unsigned char blockingstatus; -// Used in main.c, log.c, and others -extern char * username; -// Used in main.c, args.c, log.c, and others -extern bool daemonmode; -// Used in main.c, database.c, and others -extern bool database; -// Used in database.c and gc.c -extern long int lastdbindex; -// Used in database.c and gc.c -extern bool DBdeleteoldqueries; -// Used in main.c, socket.c, and dnsmasq_interface.c -extern bool ipv4telnet, ipv6telnet; -// Used in api.c, and socket.c -extern bool istelnet[MAXCONNS]; - -// Use out own memory handling functions that will detect possible errors -// and report accordingly in the log. This will make debugging FTL crashs -// caused by insufficient memory or by code bugs (not properly dealing -// with NULL pointers) much easier. -#define free(param) FTLfree(param, __FILE__, __FUNCTION__, __LINE__) -#define lib_strdup() strdup() -#undef strdup -#define strdup(param) FTLstrdup(param, __FILE__, __FUNCTION__, __LINE__) -#define calloc(p1,p2) FTLcalloc(p1,p2, __FILE__, __FUNCTION__, __LINE__) -#define realloc(p1,p2) FTLrealloc(p1,p2, __FILE__, __FUNCTION__, __LINE__) - -extern int argc_dnsmasq; -extern const char ** argv_dnsmasq; - -extern pthread_t telnet_listenthreadv4; -extern pthread_t telnet_listenthreadv6; -extern pthread_t socket_listenthread; -extern pthread_t DBthread; -extern pthread_t GCthread; -extern pthread_t DNSclientthread; diff --git a/Makefile b/Makefile index 009e18f01..aff6722ce 100644 --- a/Makefile +++ b/Makefile @@ -8,16 +8,24 @@ # This file is copyright under the latest version of the EUPL. # Please see LICENSE file for your rights under this license. -DNSMASQVERSION = "pi-hole-2.80" -DNSMASQOPTS = -DHAVE_DNSSEC -DHAVE_DNSSEC_STATIC +IDIR = src +ODIR = build + +DNSMASQ_VERSION = "pi-hole-2.81" +DNSMASQ_OPTS = -DHAVE_DNSSEC -DHAVE_DNSSEC_STATIC # Flags for compiling with libidn : -DHAVE_IDN # Flags for compiling with libidn2: -DHAVE_LIBIDN2 -DIDN2_VERSION_NUMBER=0x02000003 -FTLDEPS = FTL.h routines.h version.h api.h dnsmasq_interface.h shmem.h -FTLOBJ = main.o memory.o log.o daemon.o datastructure.o signals.o socket.o request.o grep.o setupVars.o args.o gc.o config.o database.o msgpack.o api.o dnsmasq_interface.o resolve.o regex.o shmem.o capabilities.o networktable.o overTime.o +FTL_DEPS = *.h database/*.h api/*.h version.h +FTL_DB_OBJ = database/common.o database/query-table.o database/network-table.o database/gravity-db.o database/database-thread.o database/sqlite3-ext.o +FTL_API_OBJ = api/socket.o api/request.o api/msgpack.o api/api.o +FTL_OBJ = $(FTL_DB_OBJ) $(FTL_API_OBJ) main.o memory.o log.o daemon.o datastructure.o signals.o files.o setupVars.o args.o gc.o config.o \ + dnsmasq_interface.o resolve.o regex.o shmem.o capabilities.o overTime.o timers.o vector.o -DNSMASQDEPS = config.h dhcp-protocol.h dns-protocol.h radv-protocol.h dhcp6-protocol.h dnsmasq.h ip6addr.h metrics.h ../dnsmasq_interface.h -DNSMASQOBJ = arp.o dbus.o domain.o lease.o outpacket.o rrfilter.o auth.o dhcp6.o edns0.o log.o poll.o slaac.o blockdata.o dhcp.o forward.o loop.o radv.o tables.o bpf.o dhcp-common.o helper.o netlink.o rfc1035.o tftp.o cache.o dnsmasq.o inotify.o network.o rfc2131.o util.o conntrack.o dnssec.o ipset.o option.o rfc3315.o crypto.o dump.o ubus.o metrics.o +DNSMASQ_DEPS = config.h dhcp-protocol.h dns-protocol.h radv-protocol.h dhcp6-protocol.h dnsmasq.h ip6addr.h metrics.h ../dnsmasq_interface.h +DNSMASQ_OBJ = arp.o dbus.o domain.o lease.o outpacket.o rrfilter.o auth.o dhcp6.o edns0.o log.o poll.o slaac.o blockdata.o dhcp.o forward.o \ + loop.o radv.o tables.o bpf.o dhcp-common.o helper.o netlink.o rfc1035.o tftp.o cache.o dnsmasq.o inotify.o network.o rfc2131.o \ + util.o conntrack.o dnssec.o ipset.o option.o rfc3315.o crypto.o dump.o ubus.o metrics.o # Get git commit version and date GIT_BRANCH := $(shell git branch | sed -n 's/^\* //p') @@ -34,27 +42,29 @@ GCCVERSION8 := $(shell expr `$(CC) -dumpversion | cut -f1 -d.` \>= 8) # -Wp,-D_FORTIFY_SOURCE=2 and -O1 or higher: This causes certain unsafe glibc functions to be replaced with their safer counterparts # -Wl,-z,relro: reduces the possible areas of memory in a program that can be used by an attacker that performs a successful memory corruption exploit # -Wl,-z,now: When combined with RELRO above, this further reduces the regions of memory available to memory corruption attacks -# -pie -fPIE: For ASLR (address space layout randomization) # -g3: More debugging information # -fno-omit-frame-pointer: get nicer stacktraces +# -funwind-tables: Generate static data for unwinding # -fasynchronous-unwind-tables: Increased reliability of backtraces # -fexceptions: Enable table-based thread cancellation # -Wl,-z,defs: Detect and reject underlinking (phenomenon caused by missing shared library arguments when invoking the linked editor to produce another shared library) # -Wl,-z,now: Disable lazy binding # -Wl,-z,relro: Read-only segments after relocation -HARDENING_FLAGS=-fstack-protector-strong -Wp,-D_FORTIFY_SOURCE=2 -O3 -Wl,-z,relro,-z,now -pie -fPIE -fexceptions -fasynchronous-unwind-tables -Wl,-z,defs -Wl,-z,now -Wl,-z,relro +HARDENING_FLAGS=-fstack-protector-strong -Wp,-D_FORTIFY_SOURCE=2 -O3 -Wl,-z,relro,-z,now -fexceptions -funwind-tables -fasynchronous-unwind-tables -Wl,-z,defs -Wl,-z,now -Wl,-z,relro DEBUG_FLAGS=-rdynamic -fno-omit-frame-pointer # -DSQLITE_OMIT_LOAD_EXTENSION: This option omits the entire extension loading mechanism from SQLite, including sqlite3_enable_load_extension() and sqlite3_load_extension() interfaces. (needs -ldl linking option, otherwise) # -DSQLITE_DEFAULT_MEMSTATUS=0: This setting causes the sqlite3_status() interfaces that track memory usage to be disabled. This helps the sqlite3_malloc() routines run much faster, and since SQLite uses sqlite3_malloc() internally, this helps to make the entire library faster. # -DSQLITE_OMIT_DEPRECATED: Omitting deprecated interfaces and features will not help SQLite to run any faster. It will reduce the library footprint, however. And it is the right thing to do. # -DSQLITE_OMIT_PROGRESS_CALLBACK: The progress handler callback counter must be checked in the inner loop of the bytecode engine. By omitting this interface, a single conditional is removed from the inner loop of the bytecode engine, helping SQL statements to run slightly faster. -SQLITEFLAGS=-DSQLITE_OMIT_LOAD_EXTENSION -DSQLITE_DEFAULT_MEMSTATUS=0 -DSQLITE_OMIT_DEPRECATED -DSQLITE_OMIT_PROGRESS_CALLBACK -DSQLITE_OMIT_MEMORYDB +# -DSQLITE_DEFAULT_FOREIGN_KEYS=1: This macro determines whether enforcement of foreign key constraints is enabled or disabled by default for new database connections. +# -DSQLITE_DQS=0: This setting disables the double-quoted string literal misfeature. +SQLITE_FLAGS=-DSQLITE_OMIT_LOAD_EXTENSION -DSQLITE_DEFAULT_MEMSTATUS=0 -DSQLITE_OMIT_DEPRECATED -DSQLITE_OMIT_PROGRESS_CALLBACK -DSQLITE_OMIT_MEMORYDB -DSQLITE_DEFAULT_FOREIGN_KEYS=1 -DSQLITE_DQS=0 # -Wall: This enables all the warnings about constructions that some users consider questionable, and that are easy to avoid (or modify to prevent the warning), even in conjunction with macros. This also enables some language-specific warnings described in C++ Dialect Options and Objective-C and Objective-C++ Dialect Options. # -Wextra: This enables some extra warning flags that are not enabled by -Wall. # -Wno-unused-parameter: Disable warning for unused parameters. For threads that don't need arguments, we still have to provide a void* args which is then unused. -WARNFLAGS=-Wall -Wextra -Wno-unused-parameter +WARN_FLAGS=-Wall -Wextra -Wno-unused-parameter # Extra warning flags we apply only to the FTL part of the code (used not for foreign code such as dnsmasq and SQLite3) # -Werror: Halt on any warnings, useful for enforcing clean code without any warnings (we use it only for our code part) @@ -68,7 +78,6 @@ WARNFLAGS=-Wall -Wextra -Wno-unused-parameter # -Wswitch-enum: Warn whenever a switch statement has an index of enumerated type and lacks a case for one or more of the named codes of that enumeration. # -Wshadow: Warn whenever a local variable or type declaration shadows another variable, parameter, type, class member, or whenever a built-in function is shadowed. # -Wfloat-equal: Warn if floating-point values are used in equality comparisons -# -Wunsafe-loop-optimizations -funsafe-loop-optimizations: Warn if the loop cannot be optimized because the compiler cannot assume anything on the bounds of the loop indices # -Wpointer-arith: Warn about anything that depends on the "size of" a function type or of "void". GNU C assigns these types a size of 1 # -Wundef: Warn if an undefined identifier is evaluated in an "#if" directive # -Wbad-function-cast: Warn when a function call is cast to a non-matching type @@ -84,75 +93,109 @@ ifeq "$(GCCVERSION8)" "1" # -Wduplicated-branches: Warn when an if-else has identical branches # -Wcast-align=strict: Warn whenever a pointer is cast such that the required alignment of the target is increased. For example, warn if a "char *" is cast to an "int *" regardless of the target machine. # -Wlogical-not-parentheses: Warn about logical not used on the left hand side operand of a comparison - EXTRAWARNGCC8=-Wduplicated-cond -Wduplicated-branches -Wcast-align=strict -Wlogical-not-parentheses -Wsuggest-attribute=pure -Wsuggest-attribute=const -Wsuggest-attribute=noreturn -Wsuggest-attribute=malloc -Wsuggest-attribute=format -Wsuggest-attribute=cold + EXTRAWARN_GCC8=-Wduplicated-cond -Wduplicated-branches -Wcast-align=strict -Wlogical-not-parentheses -Wsuggest-attribute=pure -Wsuggest-attribute=const -Wsuggest-attribute=noreturn -Wsuggest-attribute=malloc -Wsuggest-attribute=format -Wsuggest-attribute=cold else - EXTRAWARNGCC8= + EXTRAWARN_GCC8= endif EXTRAWARN=-Werror -Waddress -Wlogical-op -Wmissing-field-initializers -Woverlength-strings -Wformat -Wformat-nonliteral -Wuninitialized -Wswitch-enum -Wshadow \ --Wfloat-equal -Wunsafe-loop-optimizations -funsafe-loop-optimizations -Wbad-function-cast -Wwrite-strings -Wparentheses -Wlogical-op -Wstrict-prototypes -Wmissing-prototypes -Wredundant-decls -Winline $(EXTRAWARNGCC8) +-Wfloat-equal -Wbad-function-cast -Wwrite-strings -Wparentheses -Wlogical-op -Wstrict-prototypes -Wmissing-prototypes -Wredundant-decls -Winline $(EXTRAWARN_GCC8) # -FILE_OFFSET_BITS=64: used by stat(). Avoids problems with files > 2 GB on 32bit machines -CCFLAGS=-std=gnu11 -I$(IDIR) $(WARNFLAGS) -D_FILE_OFFSET_BITS=64 $(HARDENING_FLAGS) $(DEBUG_FLAGS) $(CFLAGS) $(SQLITEFLAGS) +CCFLAGS=-std=gnu11 -pipe -I$(IDIR) $(WARN_FLAGS) -D_FILE_OFFSET_BITS=64 $(HARDENING_FLAGS) $(DEBUG_FLAGS) $(CFLAGS) $(SQLITE_FLAGS) -DHAVE_POLL_H +# We define HAVE_POLL_H as this is needed for the musl builds to succeed # for FTL we need the pthread library # for dnsmasq we need the nettle crypto library and the gmp maths library # We link the two libraries statically. Although this increases the binary file size by about 1 MB, it saves about 5 MB of shared libraries and makes deployment easier -#LIBS=-pthread -lnettle -lgmp -lhogweed -LIBS=-pthread -lrt -Wl,-Bstatic -L/usr/local/lib -lhogweed -lgmp -lnettle -Wl,-Bdynamic +LIBS=-pthread -lrt -Wl,-Bstatic -L/usr/local/lib -lhogweed -lgmp -lnettle # Flags for compiling with libidn : -lidn # Flags for compiling with libidn2: -lidn2 -IDIR = . -ODIR = obj -DNSMASQDIR = dnsmasq -DNSMASQODIR = $(DNSMASQDIR)/obj +# Do we want to compile a statically linked musl executable? +ifeq "$(STATIC)" "true" + CC := $(CC) -Wl,-Bstatic -static-libgcc -static-pie +else + LIBS := $(LIBS) -Wl,-Bdynamic + # -pie -fPIE: (Dynamic) position independent executable + HARDENING_FLAGS := $(HARDENING_FLAGS) -pie -fPIE +endif -_FTLDEPS = $(patsubst %,$(IDIR)/%,$(FTLDEPS)) -_FTLOBJ = $(patsubst %,$(ODIR)/%,$(FTLOBJ)) +DB_OBJ_DIR = $(ODIR)/database +API_OBJ_DIR = $(ODIR)/api +DNSMASQ_OBJ_DIR = $(ODIR)/dnsmasq -_DNSMASQDEPS = $(patsubst %,$(DNSMASQDIR)/%,$(DNSMASQDEPS)) -_DNSMASQOBJ = $(patsubst %,$(DNSMASQODIR)/%,$(DNSMASQOBJ)) +_FTL_DEPS = $(patsubst %,$(IDIR)/%,$(FTL_DEPS)) +_FTL_OBJ = $(patsubst %,$(ODIR)/%,$(FTL_OBJ)) + +_DNSMASQ_DEPS = $(patsubst %,$(IDIR)/dnsmasq/%,$(DNSMASQ_DEPS)) +_DNSMASQ_OBJ = $(patsubst %,$(DNSMASQ_OBJ_DIR)/%,$(DNSMASQ_OBJ)) all: pihole-FTL -$(ODIR)/%.o: %.c $(_FTLDEPS) | $(ODIR) + +# Compile FTL source code files with virtually all possible warnings a modern gcc can generate +$(_FTL_OBJ): $(ODIR)/%.o: $(IDIR)/%.c $(_FTL_DEPS) | $(ODIR) $(DB_OBJ_DIR) $(API_OBJ_DIR) $(CC) -c -o $@ $< -g3 $(CCFLAGS) $(EXTRAWARN) -$(DNSMASQODIR)/%.o: $(DNSMASQDIR)/%.c $(_DNSMASQDEPS) | $(DNSMASQODIR) - $(CC) -c -o $@ $< -g3 $(CCFLAGS) -DVERSION=\"$(DNSMASQVERSION)\" $(DNSMASQOPTS) +# Compile the contained dnsmasq code with much less strict requirements as it would fail to comply +# when enforcing the standards we enforce for the rest of our FTL code base +$(_DNSMASQ_OBJ): $(DNSMASQ_OBJ_DIR)/%.o: $(IDIR)/dnsmasq/%.c $(_DNSMASQ_DEPS) | $(DNSMASQ_OBJ_DIR) + $(CC) -c -o $@ $< -g3 $(CCFLAGS) -DVERSION=\"$(DNSMASQ_VERSION)\" $(DNSMASQ_OPTS) + +$(DB_OBJ_DIR)/sqlite3.o: $(IDIR)/database/sqlite3.c | $(DB_OBJ_DIR) + $(CC) -c -o $@ $< -g3 $(CCFLAGS) $(ODIR): mkdir -p $(ODIR) -$(DNSMASQODIR): - mkdir -p $(DNSMASQODIR) +$(DB_OBJ_DIR): + mkdir -p $(DB_OBJ_DIR) + +$(API_OBJ_DIR): + mkdir -p $(API_OBJ_DIR) -$(ODIR)/sqlite3.o: $(IDIR)/sqlite3.c | $(ODIR) - $(CC) -c -o $@ $< $(CCFLAGS) +$(DNSMASQ_OBJ_DIR): + mkdir -p $(DNSMASQ_OBJ_DIR) -pihole-FTL: $(_FTLOBJ) $(_DNSMASQOBJ) $(ODIR)/sqlite3.o +pihole-FTL: $(_FTL_OBJ) $(_DNSMASQ_OBJ) $(DB_OBJ_DIR)/sqlite3.o $(CC) $(CCFLAGS) -o $@ $^ $(LIBS) .PHONY: clean force install clean: - rm -f $(ODIR)/*.o $(DNSMASQODIR)/*.o pihole-FTL + rm -rf $(ODIR) pihole-FTL + +# If CIRCLE_JOB is unset (local compilation), ask uname -m and add locally compiled comment +ifeq ($(strip $(CIRCLE_JOB)),) +FTL_ARCH := $(shell uname -m) (compiled locally) +else +FTL_ARCH := $(CIRCLE_JOB) (compiled on CI) +endif +# Get compiler version +FTL_CC := $(shell $(CC) --version | head -n 1) # # recreate version.h when GIT_VERSION changes, uses temporary file version~ -version~: force +$(IDIR)/version~: force @echo '$(GIT_BRANCH) $(GIT_VERSION) $(GIT_DATE) $(GIT_TAG)' | cmp -s - $@ || echo '$(GIT_BRANCH) $(GIT_VERSION) $(GIT_DATE) $(GIT_TAG)' > $@ -version.h: version~ - @echo '#define GIT_VERSION "$(GIT_VERSION)"' > "$@" +$(IDIR)/version.h: $(IDIR)/version~ + @echo '#ifndef VERSION_H' > "$@" + @echo '#define VERSION_H' >> "$@" + @echo '#define GIT_VERSION "$(GIT_VERSION)"' >> "$@" @echo '#define GIT_DATE "$(GIT_DATE)"' >> "$@" @echo '#define GIT_BRANCH "$(GIT_BRANCH)"' >> "$@" @echo '#define GIT_TAG "$(GIT_TAG)"' >> "$@" @echo '#define GIT_HASH "$(GIT_HASH)"' >> "$@" - @echo "Making FTL version on branch $(GIT_BRANCH) - $(GIT_VERSION) ($(GIT_DATE))" + @echo '#define FTL_ARCH "$(FTL_ARCH)"' >> "$@" + @echo '#define FTL_CC "$(FTL_CC)"' >> "$@" + @echo '#endif // VERSION_H' >> "$@" + @echo "Making FTL version on branch $(GIT_BRANCH) - $(GIT_VERSION) / $(GIT_TAG) / $(GIT_HASH) ($(GIT_DATE))" -prefix=/usr +PREFIX=/usr +SETCAP = $(shell which setcap) # install target just installs the executable # other requirements (correct ownership of files, etc.) is managed by # the service script on sudo service pihole-FTL (re)start install: pihole-FTL - install -m 0755 pihole-FTL $(prefix)/bin - /sbin/setcap CAP_NET_BIND_SERVICE,CAP_NET_RAW,CAP_NET_ADMIN+eip $(prefix)/bin/pihole-FTL + mkdir -p $(DESTDIR)$(PREFIX)/bin + install -m 0755 pihole-FTL $(DESTDIR)$(PREFIX)/bin + $(SETCAP) CAP_NET_BIND_SERVICE,CAP_NET_RAW,CAP_NET_ADMIN+eip $(DESTDIR)$(PREFIX)/bin/pihole-FTL diff --git a/README.md b/README.md index ced20b9fe..254f15839 100644 --- a/README.md +++ b/README.md @@ -60,7 +60,6 @@ If you'd rather not [donate](https://pi-hole.net/donate/) (_which is okay!_), th - [Patreon](https://patreon.com/pihole) _Become a patron for rewards_ - [Digital Ocean](http://www.digitalocean.com/?refcode=344d234950e1) _affiliate link_ - [Stickermule](https://www.stickermule.com/unlock?ref_id=6055890701&utm_medium=link&utm_source=invite) _earn a $10 credit after your first purchase_ -- [Pi-hole Swag Store](https://pi-hole.net/shop/) _affiliate link_ - [Amazon](http://www.amazon.com/exec/obidos/redirect-home/pihole09-20) _affiliate link_ - [DNS Made Easy](https://cp.dnsmadeeasy.com/u/133706) _affiliate link_ - [Vultr](http://www.vultr.com/?ref=7190426) _affiliate link_ diff --git a/api.h b/api.h deleted file mode 100644 index ed2203d6e..000000000 --- a/api.h +++ /dev/null @@ -1,44 +0,0 @@ -/* Pi-hole: A black hole for Internet advertisements -* (c) 2017 Pi-hole, LLC (https://pi-hole.net) -* Network-wide ad blocking via your own hardware. -* -* FTL Engine -* API commands and MessagePack helpers -* -* This file is copyright under the latest version of the EUPL. -* Please see LICENSE file for your rights under this license. */ - -// Statistic methods -void getStats(int *sock); -void getOverTime(int *sock); -void getTopDomains(const char *client_message, int *sock); -void getTopClients(const char *client_message, int *sock); -void getForwardDestinations(const char *client_message, int *sock); -void getQueryTypes(int *sock); -void getAllQueries(const char *client_message, int *sock); -void getRecentBlocked(const char *client_message, int *sock); -void getQueryTypesOverTime(int *sock); -void getClientsOverTime(int *sock); -void getClientNames(int *sock); -void getDomainDetails(const char *client_message, int *sock); - -// FTL methods -void getClientID(int *sock); -void getVersion(int *sock); -void getDBstats(int *sock); -void getUnknownQueries(int *sock); - -// DNS resolver methods (dnsmasq_interface.c) -void getCacheInformation(int *sock); - -// MessagePack serialization helpers -void pack_eom(int sock); -void pack_bool(int sock, bool value); -void pack_uint8(int sock, uint8_t value); -void pack_uint64(int sock, uint64_t value); -void pack_int32(int sock, int32_t value); -void pack_int64(int sock, int64_t value); -void pack_float(int sock, float value); -bool pack_fixstr(int sock, const char *string); -bool pack_str32(int sock, const char *string); -void pack_map16_start(int sock, uint16_t length); diff --git a/database.c b/database.c deleted file mode 100644 index ee9cfe091..000000000 --- a/database.c +++ /dev/null @@ -1,874 +0,0 @@ -/* Pi-hole: A black hole for Internet advertisements -* (c) 2017 Pi-hole, LLC (https://pi-hole.net) -* Network-wide ad blocking via your own hardware. -* -* FTL Engine -* Database routines -* -* This file is copyright under the latest version of the EUPL. -* Please see LICENSE file for your rights under this license. */ - -#include "FTL.h" -#include "shmem.h" -#include "sqlite3.h" - -static sqlite3 *db; -bool database = false; -bool DBdeleteoldqueries = false; -long int lastdbindex = 0; - -static pthread_mutex_t dblock; - -bool db_set_counter(unsigned int ID, int value); -int db_get_FTL_property(unsigned int ID); - -static void check_database(int rc) -{ - // We will retry if the database is busy at the moment - // However, we won't retry if any other error happened - // and - instead - disable the database functionality - // altogether in FTL (setting database to false) - if(rc != SQLITE_OK && - rc != SQLITE_DONE && - rc != SQLITE_ROW && - rc != SQLITE_BUSY) - { - logg("check_database(%i): Disabling database connection due to error", rc); - database = false; - } -} - -void dbclose(void) -{ - int rc = sqlite3_close(db); - // Report any error - if( rc ) - logg("dbclose() - SQL error (%i): %s", rc, sqlite3_errmsg(db)); - - // Unlock mutex on the database - pthread_mutex_unlock(&dblock); -} - -static double get_db_filesize(void) -{ - struct stat st; - if(stat(FTLfiles.db, &st) != 0) - { - // stat() failed (maybe the DB file does not exist?) - return 0; - } - return 1e-6*st.st_size; -} - -bool dbopen(void) -{ - pthread_mutex_lock(&dblock); - int rc = sqlite3_open_v2(FTLfiles.db, &db, SQLITE_OPEN_READWRITE, NULL); - if( rc ){ - logg("dbopen() - SQL error (%i): %s", rc, sqlite3_errmsg(db)); - dbclose(); - check_database(rc); - return false; - } - - return true; -} - -bool dbquery(const char *format, ...) -{ - char *zErrMsg = NULL; - va_list args; - - va_start(args, format); - char *query = sqlite3_vmprintf(format, args); - va_end(args); - - if(query == NULL) - { - logg("Memory allocation failed in dbquery()"); - return false; - } - - if(config.debug & DEBUG_DATABASE) logg("dbquery: %s", query); - - int rc = sqlite3_exec(db, query, NULL, NULL, &zErrMsg); - - if( rc != SQLITE_OK ){ - logg("dbquery(%s) - SQL error (%i): %s", query, rc, zErrMsg); - sqlite3_free(zErrMsg); - check_database(rc); - return false; - } - - sqlite3_free(query); - - return true; - -} - -static bool create_counter_table(void) -{ - bool ret; - // Create FTL table in the database (holds properties like database version, etc.) - ret = dbquery("CREATE TABLE counters ( id INTEGER PRIMARY KEY NOT NULL, value INTEGER NOT NULL );"); - if(!ret){ dbclose(); return false; } - - // ID 0 = total queries - ret = db_set_counter(DB_TOTALQUERIES, 0); - if(!ret){ dbclose(); return false; } - - // ID 1 = total blocked queries - ret = db_set_counter(DB_BLOCKEDQUERIES, 0); - if(!ret){ dbclose(); return false; } - - // Time stamp of creation of the counters database - ret = db_set_FTL_property(DB_FIRSTCOUNTERTIMESTAMP, time(NULL)); - if(!ret){ dbclose(); return false; } - - // Update database version to 2 - ret = db_set_FTL_property(DB_VERSION, 2); - if(!ret){ dbclose(); return false; } - - return true; -} - -static bool db_create(void) -{ - bool ret; - int rc = sqlite3_open_v2(FTLfiles.db, &db, SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, NULL); - if( rc ){ - logg("db_create() - SQL error (%i): %s", rc, sqlite3_errmsg(db)); - dbclose(); - check_database(rc); - return false; - } - // Create Queries table in the database - ret = dbquery("CREATE TABLE queries ( id INTEGER PRIMARY KEY AUTOINCREMENT, timestamp INTEGER NOT NULL, type INTEGER NOT NULL, status INTEGER NOT NULL, domain TEXT NOT NULL, client TEXT NOT NULL, forward TEXT );"); - if(!ret){ dbclose(); return false; } - // Add an index on the timestamps (not a unique index!) - ret = dbquery("CREATE INDEX idx_queries_timestamps ON queries (timestamp);"); - if(!ret){ dbclose(); return false; } - // Create FTL table in the database (holds properties like database version, etc.) - ret = dbquery("CREATE TABLE ftl ( id INTEGER PRIMARY KEY NOT NULL, value BLOB NOT NULL );"); - if(!ret){ dbclose(); return false; } - - // Set DB version 1 - ret = dbquery("INSERT INTO ftl (ID,VALUE) VALUES(%i,1);", DB_VERSION); - if(!ret){ dbclose(); return false; } - - // Most recent timestamp initialized to 00:00 1 Jan 1970 - ret = dbquery("INSERT INTO ftl (ID,VALUE) VALUES(%i,0);", DB_LASTTIMESTAMP); - if(!ret){ dbclose(); return false; } - - // Create counter table - // Will update DB version to 2 - if(!create_counter_table()) - return false; - - // Create network table - // Will update DB version to 3 - if(!create_network_table()) - return false; - - return true; -} - -void SQLite3LogCallback(void *pArg, int iErrCode, const char *zMsg) -{ - // Note: pArg is NULL and not used - // See https://sqlite.org/rescode.html#extrc for details - // concerning the return codes returned here - logg("SQLite3 message: %s (%d)", zMsg, iErrCode); -} - -void db_init(void) -{ - // First check if the user doesn't want to use the database and set an - // empty string as file name in FTL's config file - if(FTLfiles.db == NULL || strlen(FTLfiles.db) == 0) - { - database = false; - return; - } - - // Initialize SQLite3 logging callback - // This ensures SQLite3 errors and warnings are logged to pihole-FTL.log - // We use this to possibly catch even more errors in places we do not - // explicitly check for failures to have happened - sqlite3_config(SQLITE_CONFIG_LOG, SQLite3LogCallback, NULL); - - int rc = sqlite3_open_v2(FTLfiles.db, &db, SQLITE_OPEN_READWRITE, NULL); - if( rc ){ - logg("db_init() - Cannot open database (%i): %s", rc, sqlite3_errmsg(db)); - dbclose(); - check_database(rc); - - logg("Creating new (empty) database"); - if (!db_create()) - { - logg("Database not available"); - database = false; - return; - } - } - - // Test DB version and see if we need to upgrade the database file - int dbversion = db_get_FTL_property(DB_VERSION); - logg("Database version is %i", dbversion); - if(dbversion < 1) - { - logg("Database version incorrect, database not available"); - database = false; - return; - } - // Update to version 2 if lower - if(dbversion < 2) - { - // Update to version 2: Create counters table - logg("Updating long-term database to version 2"); - if (!create_counter_table()) - { - logg("Counter table not initialized, database not available"); - database = false; - return; - } - // Get updated version - dbversion = db_get_FTL_property(DB_VERSION); - } - // Update to version 3 if lower - if(dbversion < 3) - { - // Update to version 3: Create network table - logg("Updating long-term database to version 3"); - if (!create_network_table()) - { - logg("Network table not initialized, database not available"); - database = false; - return; - } - // Get updated version - dbversion = db_get_FTL_property(DB_VERSION); - } - - // Close database to prevent having it opened all time - // we already closed the database when we returned earlier - sqlite3_close(db); - - if (pthread_mutex_init(&dblock, NULL) != 0) - { - logg("FATAL: DB mutex init failed\n"); - // Return failure - exit(EXIT_FAILURE); - } - - logg("Database successfully initialized"); - database = true; -} - -int db_get_FTL_property(unsigned int ID) -{ - // Prepare SQL statement - char* querystr = NULL; - int ret = asprintf(&querystr, "SELECT VALUE FROM ftl WHERE id = %u;", ID); - - if(querystr == NULL || ret < 0) - { - logg("Memory allocation failed in db_get_FTL_property with ID = %u (%i)", ID, ret); - return DB_FAILED; - } - - int value = db_query_int(querystr); - free(querystr); - - return value; -} - -bool db_set_FTL_property(unsigned int ID, int value) -{ - return dbquery("INSERT OR REPLACE INTO ftl (id, value) VALUES ( %u, %i );", ID, value); -} - -bool db_set_counter(unsigned int ID, int value) -{ - return dbquery("INSERT OR REPLACE INTO counters (id, value) VALUES ( %u, %i );", ID, value); -} - -static bool db_update_counters(int total, int blocked) -{ - if(!dbquery("UPDATE counters SET value = value + %i WHERE id = %i;", total, DB_TOTALQUERIES)) - return false; - if(!dbquery("UPDATE counters SET value = value + %i WHERE id = %i;", blocked, DB_BLOCKEDQUERIES)) - return false; - return true; -} - -int db_query_int(const char* querystr) -{ - sqlite3_stmt* stmt; - int rc = sqlite3_prepare_v2(db, querystr, -1, &stmt, NULL); - if( rc ){ - logg("db_query_int(%s) - SQL error prepare (%i): %s", querystr, rc, sqlite3_errmsg(db)); - dbclose(); - check_database(rc); - return DB_FAILED; - } - - rc = sqlite3_step(stmt); - int result; - - if( rc == SQLITE_ROW ) - { - result = sqlite3_column_int(stmt, 0); - } - else if( rc == SQLITE_DONE ) - { - // No rows available - result = DB_NODATA; - } - else - { - logg("db_query_int(%s) - SQL error step (%i): %s", querystr, rc, sqlite3_errmsg(db)); - dbclose(); - check_database(rc); - return DB_FAILED; - } - - sqlite3_finalize(stmt); - - return result; -} - -static int number_of_queries_in_DB(void) -{ - sqlite3_stmt* stmt; - - // Count number of rows using the index timestamp is faster than select(*) - int rc = sqlite3_prepare_v2(db, "SELECT COUNT(timestamp) FROM queries", -1, &stmt, NULL); - if( rc ){ - logg("number_of_queries_in_DB() - SQL error prepare (%i): %s", rc, sqlite3_errmsg(db)); - dbclose(); - check_database(rc); - return DB_FAILED; - } - - rc = sqlite3_step(stmt); - if( rc != SQLITE_ROW ){ - logg("number_of_queries_in_DB() - SQL error step (%i): %s", rc, sqlite3_errmsg(db)); - dbclose(); - check_database(rc); - return DB_FAILED; - } - - int result = sqlite3_column_int(stmt, 0); - - sqlite3_finalize(stmt); - - return result; -} - -static sqlite3_int64 last_ID_in_DB(void) -{ - sqlite3_stmt* stmt; - - int rc = sqlite3_prepare_v2(db, "SELECT MAX(ID) FROM queries", -1, &stmt, NULL); - if( rc ){ - logg("last_ID_in_DB() - SQL error prepare (%i): %s", rc, sqlite3_errmsg(db)); - dbclose(); - check_database(rc); - return DB_FAILED; - } - - rc = sqlite3_step(stmt); - if( rc != SQLITE_ROW ){ - logg("last_ID_in_DB() - SQL error step (%i): %s", rc, sqlite3_errmsg(db)); - dbclose(); - check_database(rc); - return DB_FAILED; - } - - sqlite3_int64 result = sqlite3_column_int64(stmt, 0); - - sqlite3_finalize(stmt); - - return result; -} - -int get_number_of_queries_in_DB(void) -{ - int result = DB_NODATA; - - if(!dbopen()) - { - logg("Failed to open DB in get_number_of_queries_in_DB()"); - return DB_FAILED; - } - - result = number_of_queries_in_DB(); - - // Close database - dbclose(); - - return result; -} - -void save_to_DB(void) -{ - // Don't save anything to the database if in PRIVACY_NOSTATS mode - if(config.privacylevel >= PRIVACY_NOSTATS) - return; - - // Start database timer - if(config.debug & DEBUG_DATABASE) timer_start(DATABASE_WRITE_TIMER); - - // Open database - if(!dbopen()) - { - logg("save_to_DB() - failed to open DB"); - return; - } - - unsigned int saved = 0, saved_error = 0; - long int i; - sqlite3_stmt* stmt; - - // Get last ID stored in the database - sqlite3_int64 lastID = last_ID_in_DB(); - - bool ret = dbquery("BEGIN TRANSACTION"); - if(!ret) - { - logg("save_to_DB() - unable to begin transaction (%i): %s", ret, sqlite3_errmsg(db)); - dbclose(); - return; - } - - int rc = sqlite3_prepare_v2(db, "INSERT INTO queries VALUES (NULL,?,?,?,?,?,?)", -1, &stmt, NULL); - if( rc ) - { - logg("save_to_DB() - error in preparing SQL statement (%i): %s", ret, sqlite3_errmsg(db)); - dbclose(); - check_database(rc); - return; - } - - int total = 0, blocked = 0; - time_t currenttimestamp = time(NULL); - time_t newlasttimestamp = 0; - for(i = MAX(0, lastdbindex); i < counters->queries; i++) - { - validate_access("queries", i, true, __LINE__, __FUNCTION__, __FILE__); - if(queries[i].db != 0) - { - // Skip, already saved in database - continue; - } - - if(!queries[i].complete && queries[i].timestamp > currenttimestamp-2) - { - // Break if a brand new query (age < 2 seconds) is not yet completed - // giving it a chance to be stored next time - break; - } - - // Memory checks - validate_access("queries", i, true, __LINE__, __FUNCTION__, __FILE__); - - if(queries[i].privacylevel >= PRIVACY_MAXIMUM) - { - // Skip, we never store nor count queries recorded - // while have been in maximum privacy mode in the database - continue; - } - - // TIMESTAMP - sqlite3_bind_int(stmt, 1, queries[i].timestamp); - - // TYPE - sqlite3_bind_int(stmt, 2, queries[i].type); - - // STATUS - sqlite3_bind_int(stmt, 3, queries[i].status); - - // DOMAIN - const char *domain = getDomainString(i); - sqlite3_bind_text(stmt, 4, domain, -1, SQLITE_TRANSIENT); - - // CLIENT - const char *client = getClientIPString(i); - sqlite3_bind_text(stmt, 5, client, -1, SQLITE_TRANSIENT); - - // FORWARD - if(queries[i].status == QUERY_FORWARDED && queries[i].forwardID > -1) - { - validate_access("forwarded", queries[i].forwardID, true, __LINE__, __FUNCTION__, __FILE__); - sqlite3_bind_text(stmt, 6, getstr(forwarded[queries[i].forwardID].ippos), -1, SQLITE_TRANSIENT); - } - else - { - sqlite3_bind_null(stmt, 6); - } - - // Step and check if successful - rc = sqlite3_step(stmt); - sqlite3_clear_bindings(stmt); - sqlite3_reset(stmt); - - if( rc != SQLITE_DONE ){ - logg("save_to_DB() - SQL error (%i): %s", rc, sqlite3_errmsg(db)); - saved_error++; - if(saved_error < 3) - { - continue; - } - else - { - logg("save_to_DB() - exiting due to too many errors"); - break; - } - // Check this error message - check_database(rc); - } - - saved++; - // Mark this query as saved in the database by setting the corresponding ID - queries[i].db = ++lastID; - - // Total counter information (delta computation) - total++; - if(queries[i].status == QUERY_GRAVITY || - queries[i].status == QUERY_BLACKLIST || - queries[i].status == QUERY_WILDCARD || - queries[i].status == QUERY_EXTERNAL_BLOCKED_IP || - queries[i].status == QUERY_EXTERNAL_BLOCKED_NULL || - queries[i].status == QUERY_EXTERNAL_BLOCKED_NXRA) - blocked++; - - // Update lasttimestamp variable with timestamp of the latest stored query - if(queries[i].timestamp > newlasttimestamp) - newlasttimestamp = queries[i].timestamp; - } - - // Finish prepared statement - ret = dbquery("END TRANSACTION"); - int ret2 = sqlite3_finalize(stmt); - if(!ret || ret2 != SQLITE_OK){ dbclose(); return; } - - // Store index for next loop interation round and update last time stamp - // in the database only if all queries have been saved successfully - if(saved > 0 && saved_error == 0) - { - lastdbindex = i; - db_set_FTL_property(DB_LASTTIMESTAMP, newlasttimestamp); - } - - // Update total counters in DB - if(saved > 0 && !db_update_counters(total, blocked)) - { - dbclose(); - return; - } - - // Close database - dbclose(); - - if(config.debug & DEBUG_DATABASE) - { - logg("Notice: Queries stored in DB: %u (took %.1f ms, last SQLite ID %llu)", saved, timer_elapsed_msec(DATABASE_WRITE_TIMER), lastID); - if(saved_error > 0) - logg(" There are queries that have not been saved"); - } -} - -static void delete_old_queries_in_DB(void) -{ - // Open database - if(!dbopen()) - { - logg("Failed to open DB in delete_old_queries_in_DB()"); - return; - } - - int timestamp = time(NULL) - config.maxDBdays * 86400; - - if(!dbquery("DELETE FROM queries WHERE timestamp <= %i", timestamp)) - { - dbclose(); - logg("delete_old_queries_in_DB(): Deleting queries due to age of entries failed!"); - database = true; - return; - } - - // Get how many rows have been affected (deleted) - int affected = sqlite3_changes(db); - - // Print final message only if there is a difference - if((config.debug & DEBUG_DATABASE) || affected) - logg("Notice: Database size is %.2f MB, deleted %i rows", get_db_filesize(), affected); - - // Close database - dbclose(); - - // Re-enable database actions - database = true; -} - -int lastDBsave = 0; -void *DB_thread(void *val) -{ - // Set thread name - prctl(PR_SET_NAME,"database",0,0,0); - - // Save timestamp as we do not want to store immediately - // to the database - lastDBsave = time(NULL) - time(NULL)%config.DBinterval; - - while(!killed && database) - { - if(time(NULL) - lastDBsave >= config.DBinterval) - { - // Update lastDBsave timer - lastDBsave = time(NULL) - time(NULL)%config.DBinterval; - - // Lock FTL's data structures, since it is - // likely that they will be changed here - lock_shm(); - - // Save data to database - save_to_DB(); - - // Release data lock - unlock_shm(); - - // Check if GC should be done on the database - if(DBdeleteoldqueries) - { - // No thread locks needed - delete_old_queries_in_DB(); - DBdeleteoldqueries = false; - } - - // Parse ARP cache (fill network table) if enabled - if (config.parse_arp_cache) - parse_arp_cache(); - } - sleepms(100); - } - - return NULL; -} - -// Get most recent 24 hours data from long-term database -void read_data_from_DB(void) -{ - // Don't try to load anything to the database if in PRIVACY_NOSTATS mode - if(config.privacylevel >= PRIVACY_NOSTATS) - return; - - // Open database file - if(!dbopen()) - { - logg("read_data_from_DB() - Failed to open DB"); - return; - } - - // Prepare request - char *rstr = NULL; - // Get time stamp 24 hours in the past - time_t now = time(NULL); - time_t mintime = now - config.maxlogage; - int rc = asprintf(&rstr, "SELECT * FROM queries WHERE timestamp >= %li", mintime); - if(rc < 1) - { - logg("read_data_from_DB() - Allocation error (%i): %s", rc, sqlite3_errmsg(db)); - return; - } - // Log DB query string in debug mode - if(config.debug & DEBUG_DATABASE) logg("%s", rstr); - - // Prepare SQLite3 statement - sqlite3_stmt* stmt; - rc = sqlite3_prepare_v2(db, rstr, -1, &stmt, NULL); - if( rc ){ - logg("read_data_from_DB() - SQL error prepare (%i): %s", rc, sqlite3_errmsg(db)); - dbclose(); - check_database(rc); - return; - } - - // Loop through returned database rows - while((rc = sqlite3_step(stmt)) == SQLITE_ROW) - { - sqlite3_int64 dbid = sqlite3_column_int64(stmt, 0); - time_t queryTimeStamp = sqlite3_column_int(stmt, 1); - // 1483228800 = 01/01/2017 @ 12:00am (UTC) - if(queryTimeStamp < 1483228800) - { - logg("DB warn: TIMESTAMP should be larger than 01/01/2017 but is %li", queryTimeStamp); - continue; - } - if(queryTimeStamp > now) - { - if(config.debug & DEBUG_DATABASE) logg("DB warn: Skipping query logged in the future (%li)", queryTimeStamp); - continue; - } - - int type = sqlite3_column_int(stmt, 2); - if(type < TYPE_A || type >= TYPE_MAX) - { - logg("DB warn: TYPE should not be %i", type); - continue; - } - // Don't import AAAA queries from database if the user set - // AAAA_QUERY_ANALYSIS=no in pihole-FTL.conf - if(type == TYPE_AAAA && !config.analyze_AAAA) - { - continue; - } - - int status = sqlite3_column_int(stmt, 3); - if(status < QUERY_UNKNOWN || status > QUERY_EXTERNAL_BLOCKED_NXRA) - { - logg("DB warn: STATUS should be within [%i,%i] but is %i", QUERY_UNKNOWN, QUERY_EXTERNAL_BLOCKED_NXRA, status); - continue; - } - - const char * domain = (const char *)sqlite3_column_text(stmt, 4); - if(domain == NULL) - { - logg("DB warn: DOMAIN should never be NULL, %li", queryTimeStamp); - continue; - } - - const char * client = (const char *)sqlite3_column_text(stmt, 5); - if(client == NULL) - { - logg("DB warn: CLIENT should never be NULL, %li", queryTimeStamp); - continue; - } - - // Check if user wants to skip queries coming from localhost - if(config.ignore_localhost && - (strcmp(client, "127.0.0.1") == 0 || strcmp(client, "::1") == 0)) - { - continue; - } - - const char *forwarddest = (const char *)sqlite3_column_text(stmt, 6); - int forwardID = 0; - // Determine forwardID only when status == 2 (forwarded) as the - // field need not to be filled for other query status types - if(status == QUERY_FORWARDED) - { - if(forwarddest == NULL) - { - logg("DB warn: FORWARD should not be NULL with status QUERY_FORWARDED, %li", queryTimeStamp); - continue; - } - forwardID = findForwardID(forwarddest, true); - } - - // Obtain IDs only after filtering which queries we want to keep - int timeidx = getOverTimeID(queryTimeStamp); - int domainID = findDomainID(domain); - int clientID = findClientID(client, true); - - // Ensure we have enough space in the queries struct - memory_check(QUERIES); - - // Set index for this query - int queryIndex = counters->queries; - - // Store this query in memory - validate_access("queries", queryIndex, false, __LINE__, __FUNCTION__, __FILE__); - validate_access("clients", clientID, true, __LINE__, __FUNCTION__, __FILE__); - queries[queryIndex].magic = MAGICBYTE; - queries[queryIndex].timestamp = queryTimeStamp; - queries[queryIndex].type = type; - queries[queryIndex].status = status; - queries[queryIndex].domainID = domainID; - queries[queryIndex].clientID = clientID; - queries[queryIndex].forwardID = forwardID; - queries[queryIndex].timeidx = timeidx; - queries[queryIndex].db = dbid; - queries[queryIndex].id = 0; - queries[queryIndex].complete = true; // Mark as all information is available - queries[queryIndex].response = 0; - queries[queryIndex].dnssec = DNSSEC_UNKNOWN; - queries[queryIndex].reply = REPLY_UNKNOWN; - - // Set lastQuery timer and add one query for network table - clients[clientID].lastQuery = queryTimeStamp; - clients[clientID].numQueriesARP++; - - // Handle type counters - if(type >= TYPE_A && type < TYPE_MAX) - { - counters->querytype[type-1]++; - overTime[timeidx].querytypedata[type-1]++; - } - - // Update overTime data - overTime[timeidx].total++; - // Update overTime data structure with the new client - clients[clientID].overTime[timeidx]++; - - // Increase DNS queries counter - counters->queries++; - - // Increment status counters - switch(status) - { - case QUERY_UNKNOWN: // Unknown - counters->unknown++; - break; - - case QUERY_GRAVITY: // Blocked by gravity.list - case QUERY_WILDCARD: // Blocked by regex filter - case QUERY_BLACKLIST: // Blocked by black.list - case QUERY_EXTERNAL_BLOCKED_IP: // Blocked by external provider - case QUERY_EXTERNAL_BLOCKED_NULL: // Blocked by external provider - case QUERY_EXTERNAL_BLOCKED_NXRA: // Blocked by external provider - counters->blocked++; - domains[domainID].blockedcount++; - clients[clientID].blockedcount++; - // Update overTime data structure - overTime[timeidx].blocked++; - break; - - case QUERY_FORWARDED: // Forwarded - counters->forwardedqueries++; - // Update overTime data structure - overTime[timeidx].forwarded++; - break; - - case QUERY_CACHE: // Cached or local config - counters->cached++; - // Update overTime data structure - overTime[timeidx].cached++; - break; - - default: - logg("Error: Found unknown status %i in long term database!", status); - logg(" Timestamp: %li", queryTimeStamp); - logg(" Continuing anyway..."); - break; - } - } - logg("Imported %i queries from the long-term database", counters->queries); - - // Update lastdbindex so that the next call to save_to_DB() - // skips the queries that we just imported from the database - lastdbindex = counters->queries; - - if( rc != SQLITE_DONE ){ - logg("read_data_from_DB() - SQL error step (%i): %s", rc, sqlite3_errmsg(db)); - dbclose(); - check_database(rc); - return; - } - - // Finalize SQLite3 statement - sqlite3_finalize(stmt); - dbclose(); - free(rstr); -} diff --git a/datastructure.c b/datastructure.c deleted file mode 100644 index 429fc3b3f..000000000 --- a/datastructure.c +++ /dev/null @@ -1,219 +0,0 @@ -/* Pi-hole: A black hole for Internet advertisements -* (c) 2017 Pi-hole, LLC (https://pi-hole.net) -* Network-wide ad blocking via your own hardware. -* -* FTL Engine -* Query processing routines -* -* This file is copyright under the latest version of the EUPL. -* Please see LICENSE file for your rights under this license. */ - -#include "FTL.h" - -// converts upper to lower case, and leaves other characters unchanged -void strtolower(char *str) -{ - int i = 0; - while(str[i]){ str[i] = tolower(str[i]); i++; } -} - -int findForwardID(const char * forward, bool count) -{ - int i, forwardID = -1; - if(counters->forwarded > 0) - validate_access("forwarded", counters->forwarded-1, true, __LINE__, __FUNCTION__, __FILE__); - // Go through already knows forward servers and see if we used one of those - for(i=0; i < counters->forwarded; i++) - { - if(strcmp(getstr(forwarded[i].ippos), forward) == 0) - { - forwardID = i; - if(count) forwarded[forwardID].count++; - return forwardID; - } - } - // This forward server is not known - // Store ID - forwardID = counters->forwarded; - logg("New forward server: %s (%i/%u)", forward, forwardID, counters->forwarded_MAX); - - // Check struct size - memory_check(FORWARDED); - - validate_access("forwarded", forwardID, false, __LINE__, __FUNCTION__, __FILE__); - // Set magic byte - forwarded[forwardID].magic = MAGICBYTE; - // Initialize its counter - if(count) - forwarded[forwardID].count = 1; - else - forwarded[forwardID].count = 0; - // Save forward destination IP address - forwarded[forwardID].ippos = addstr(forward); - forwarded[forwardID].failed = 0; - // Initialize forward hostname - // Due to the nature of us being the resolver, - // the actual resolving of the host name has - // to be done separately to be non-blocking - forwarded[forwardID].new = true; - forwarded[forwardID].namepos = 0; // 0 -> string with length zero - // Increase counter by one - counters->forwarded++; - - return forwardID; -} - -int findDomainID(const char *domain) -{ - int i; - if(counters->domains > 0) - validate_access("domains", counters->domains-1, true, __LINE__, __FUNCTION__, __FILE__); - for(i=0; i < counters->domains; i++) - { - // Quick test: Does the domain start with the same character? - if(getstr(domains[i].domainpos)[0] != domain[0]) - continue; - - // If so, compare the full domain using strcmp - if(strcmp(getstr(domains[i].domainpos), domain) == 0) - { - domains[i].count++; - return i; - } - } - - // If we did not return until here, then this domain is not known - // Store ID - int domainID = counters->domains; - - // Check struct size - memory_check(DOMAINS); - - validate_access("domains", domainID, false, __LINE__, __FUNCTION__, __FILE__); - // Set magic byte - domains[domainID].magic = MAGICBYTE; - // Set its counter to 1 - domains[domainID].count = 1; - // Set blocked counter to zero - domains[domainID].blockedcount = 0; - // Store domain name - no need to check for NULL here as it doesn't harm - domains[domainID].domainpos = addstr(domain); - // RegEx needs to be evaluated for this new domain - domains[domainID].regexmatch = REGEX_UNKNOWN; - // Increase counter by one - counters->domains++; - - return domainID; -} - -int findClientID(const char *client, bool count) -{ - // Compare content of client against known client IP addresses - if(counters->clients > 0) - validate_access("clients", counters->clients-1, true, __LINE__, __FUNCTION__, __FILE__); - for(int i=0; i < counters->clients; i++) - { - // Quick test: Does the clients IP start with the same character? - if(getstr(clients[i].ippos)[0] != client[0]) - continue; - - // If so, compare the full IP using strcmp - if(strcmp(getstr(clients[i].ippos), client) == 0) - { - // Add one if count == true (do not add one, e.g., during ARP table processing) - if(count) clients[i].count++; - return i; - } - } - - // Return -1 (= not found) if count is false ... - if(!count) - return -1; - // ... otherwise proceed with adding a new client entry - - // If we did not return until here, then this client is definitely new - // Store ID - int clientID = counters->clients; - - // Check struct size - memory_check(CLIENTS); - - validate_access("clients", clientID, false, __LINE__, __FUNCTION__, __FILE__); - // Set magic byte - clients[clientID].magic = MAGICBYTE; - // Set its counter to 1 - clients[clientID].count = 1; - // Initialize blocked count to zero - clients[clientID].blockedcount = 0; - // Store client IP - no need to check for NULL here as it doesn't harm - clients[clientID].ippos = addstr(client); - // Initialize client hostname - // Due to the nature of us being the resolver, - // the actual resolving of the host name has - // to be done separately to be non-blocking - clients[clientID].new = true; - clients[clientID].namepos = 0; - // No query seen so far - clients[clientID].lastQuery = 0; - clients[clientID].numQueriesARP = 0; - - // Initialize client-specific overTime data - for(int i = 0; i < OVERTIME_SLOTS; i++) - clients[clientID].overTime[i] = 0; - - // Increase counter by one - counters->clients++; - - return clientID; -} - -bool isValidIPv4(const char *addr) -{ - struct sockaddr_in sa; - return inet_pton(AF_INET, addr, &(sa.sin_addr)) != 0; -} - -bool isValidIPv6(const char *addr) -{ - struct sockaddr_in6 sa; - return inet_pton(AF_INET6, addr, &(sa.sin6_addr)) != 0; -} - -// Privacy-level sensitive subroutine that returns the domain name -// only when appropriate for the requested query -const char *getDomainString(int queryID) -{ - if(queries[queryID].privacylevel < PRIVACY_HIDE_DOMAINS) - { - validate_access("domains", queries[queryID].domainID, true, __LINE__, __FUNCTION__, __FILE__); - return getstr(domains[queries[queryID].domainID].domainpos); - } - else - return HIDDEN_DOMAIN; -} - -// Privacy-level sensitive subroutine that returns the client IP -// only when appropriate for the requested query -const char *getClientIPString(int queryID) -{ - if(queries[queryID].privacylevel < PRIVACY_HIDE_DOMAINS_CLIENTS) - { - validate_access("clients", queries[queryID].clientID, true, __LINE__, __FUNCTION__, __FILE__); - return getstr(clients[queries[queryID].clientID].ippos); - } - else - return HIDDEN_CLIENT; -} - -// Privacy-level sensitive subroutine that returns the client host name -// only when appropriate for the requested query -const char *getClientNameString(int queryID) -{ - if(queries[queryID].privacylevel < PRIVACY_HIDE_DOMAINS_CLIENTS) - { - validate_access("clients", queries[queryID].clientID, true, __LINE__, __FUNCTION__, __FILE__); - return getstr(clients[queries[queryID].clientID].namepos); - } - else - return HIDDEN_CLIENT; -} diff --git a/dnsmasq/ubus.c b/dnsmasq/ubus.c deleted file mode 100644 index aaa326a95..000000000 --- a/dnsmasq/ubus.c +++ /dev/null @@ -1,107 +0,0 @@ -/* dnsmasq is Copyright (c) 2000-2018 Simon Kelley - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; version 2 dated June, 1991, or - (at your option) version 3 dated 29 June, 2007. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -*/ - -#include "dnsmasq.h" - -#ifdef HAVE_UBUS - -#include - -static struct ubus_context *ubus = NULL; -static struct blob_buf b; - -static int ubus_handle_metrics(struct ubus_context *ctx, struct ubus_object *obj, - struct ubus_request_data *req, const char *method, - struct blob_attr *msg); -static struct ubus_method ubus_object_methods[] = { - {.name = "metrics", .handler = ubus_handle_metrics}, -}; - -static struct ubus_object_type ubus_object_type = UBUS_OBJECT_TYPE("dnsmasq", ubus_object_methods); - -static struct ubus_object ubus_object = { - .name = "dnsmasq", - .type = &ubus_object_type, - .methods = ubus_object_methods, - .n_methods = ARRAY_SIZE(ubus_object_methods), -}; - -void set_ubus_listeners() -{ - if (!ubus) - return; - - poll_listen(ubus->sock.fd, POLLIN); - poll_listen(ubus->sock.fd, POLLERR); - poll_listen(ubus->sock.fd, POLLHUP); -} - -void check_ubus_listeners() -{ - if (!ubus) - { - ubus = ubus_connect(NULL); - if (!ubus) - return; - ubus_add_object(ubus, &ubus_object); - } - - if (poll_check(ubus->sock.fd, POLLIN)) - ubus_handle_event(ubus); - - if (poll_check(ubus->sock.fd, POLLHUP)) - { - ubus_free(ubus); - ubus = NULL; - } -} - - -static int ubus_handle_metrics(struct ubus_context *ctx, struct ubus_object *obj, - struct ubus_request_data *req, const char *method, - struct blob_attr *msg) -{ - int i; - blob_buf_init(&b, 0); - - for(i=0; i < __METRIC_MAX; i++) - blobmsg_add_u32(&b, get_metric_name(i), daemon->metrics[i]); - - ubus_send_reply(ctx, req, b.head); - - return 0; -} - -void ubus_event_bcast(const char *type, const char *mac, const char *ip, const char *name, const char *interface) -{ - if (!ubus || !ubus_object.has_subscribers) - return; - - blob_buf_init(&b, 0); - if (mac) - blobmsg_add_string(&b, "mac", mac); - if (ip) - blobmsg_add_string(&b, "ip", ip); - if (name) - blobmsg_add_string(&b, "name", name); - if (interface) - blobmsg_add_string(&b, "interface", interface); - - ubus_notify(ubus, &ubus_object, type, b.head, -1); -} - - -#endif /* HAVE_UBUS */ diff --git a/dnsmasq_interface.c b/dnsmasq_interface.c deleted file mode 100644 index a62945f36..000000000 --- a/dnsmasq_interface.c +++ /dev/null @@ -1,1365 +0,0 @@ -/* Pi-hole: A black hole for Internet advertisements -* (c) 2017 Pi-hole, LLC (https://pi-hole.net) -* Network-wide ad blocking via your own hardware. -* -* FTL Engine -* dnsmasq interfacing routines -* -* This file is copyright under the latest version of the EUPL. -* Please see LICENSE file for your rights under this license. */ - -#define FTLDNS -#include "dnsmasq/dnsmasq.h" -#undef __USE_XOPEN -#include "FTL.h" -#include "dnsmasq_interface.h" -#include "shmem.h" -// Prototype of getCacheInformation() -#include "api.h" - -void print_flags(unsigned int flags); -void save_reply_type(unsigned int flags, int queryID, struct timeval response); -static unsigned long converttimeval(struct timeval time) __attribute__((const)); -static void block_single_domain_regex(char *domain); -static void detect_blocked_IP(unsigned short flags, const char* answer, int queryID); -static void query_externally_blocked(int i, unsigned char status); -static int findQueryID(int id); - -unsigned char* pihole_privacylevel = &config.privacylevel; -char flagnames[28][12] = {"F_IMMORTAL ", "F_NAMEP ", "F_REVERSE ", "F_FORWARD ", "F_DHCP ", "F_NEG ", "F_HOSTS ", "F_IPV4 ", "F_IPV6 ", "F_BIGNAME ", "F_NXDOMAIN ", "F_CNAME ", "F_DNSKEY ", "F_CONFIG ", "F_DS ", "F_DNSSECOK ", "F_UPSTREAM ", "F_RRNAME ", "F_SERVER ", "F_QUERY ", "F_NOERR ", "F_AUTH ", "F_DNSSEC ", "F_KEYTAG ", "F_SECSTAT ", "F_NO_RR ", "F_IPSET ", "F_NOEXTRA "}; - -void _FTL_new_query(unsigned int flags, char *name, struct all_addr *addr, char *types, int id, char type, const char* file, const int line) -{ - // Don't analyze anything if in PRIVACY_NOSTATS mode - if(config.privacylevel >= PRIVACY_NOSTATS) - return; - - // Create new query in data structure - lock_shm(); - - // Get timestamp - time_t querytimestamp = time(NULL); - - // Save request time - struct timeval request; - gettimeofday(&request, 0); - - // Determine query type - unsigned char querytype = 0; - if(strcmp(types,"query[A]") == 0) - querytype = TYPE_A; - else if(strcmp(types,"query[AAAA]") == 0) - querytype = TYPE_AAAA; - else if(strcmp(types,"query[ANY]") == 0) - querytype = TYPE_ANY; - else if(strcmp(types,"query[SRV]") == 0) - querytype = TYPE_SRV; - else if(strcmp(types,"query[SOA]") == 0) - querytype = TYPE_SOA; - else if(strcmp(types,"query[PTR]") == 0) - querytype = TYPE_PTR; - else if(strcmp(types,"query[TXT]") == 0) - querytype = TYPE_TXT; - else - { - // Return early to avoid accessing querytypedata out of bounds - if(config.debug & DEBUG_QUERIES) logg("Notice: Skipping unknown query type: %s (%i)", types, id); - unlock_shm(); - return; - } - - // Skip AAAA queries if user doesn't want to have them analyzed - if(!config.analyze_AAAA && querytype == TYPE_AAAA) - { - if(config.debug & DEBUG_QUERIES) logg("Not analyzing AAAA query"); - unlock_shm(); - return; - } - - // Ensure we have enough space in the queries struct - memory_check(QUERIES); - int queryID = counters->queries; - - // Convert domain to lower case - char *domain = strdup(name); - strtolower(domain); - - // If domain is "pi.hole" we skip this query - if(strcmp(domain, "pi.hole") == 0) - { - // free memory already allocated here - free(domain); - unlock_shm(); - return; - } - - // Store plain text domain in buffer for regex validation - char *domainbuffer = strdup(domain); - - // Get client IP address - char dest[ADDRSTRLEN]; - inet_ntop((flags & F_IPV4) ? AF_INET : AF_INET6, addr, dest, ADDRSTRLEN); - char *client = strdup(dest); - strtolower(client); - - // Check if user wants to skip queries coming from localhost - if(config.ignore_localhost && - (strcmp(client, "127.0.0.1") == 0 || strcmp(client, "::1") == 0)) - { - free(domain); - free(client); - unlock_shm(); - return; - } - - // Log new query if in debug mode - const char *proto = (type == UDP) ? "UDP" : "TCP"; - if(config.debug & DEBUG_QUERIES) - { - logg("**** new %s %s \"%s\" from %s (ID %i, FTL %i, %s:%i)", - proto, types, domain, client, id, queryID, file, line); - } - - // Update counters - counters->querytype[querytype-1]++; - - // Update overTime - unsigned int timeidx = getOverTimeID(querytimestamp); - overTime[timeidx].querytypedata[querytype-1]++; - - // Skip rest of the analysis if this query is not of type A or AAAA - // but user wants to see only A and AAAA queries (pre-v4.1 behavior) - if(config.analyze_only_A_AAAA && querytype != TYPE_A && querytype != TYPE_AAAA) - { - // Don't process this query further here, we already counted it - if(config.debug & DEBUG_QUERIES) logg("Notice: Skipping new query: %s (%i)", types, id); - free(domain); - free(domainbuffer); - free(client); - unlock_shm(); - return; - } - - // Go through already knows domains and see if it is one of them - int domainID = findDomainID(domain); - - // Go through already knows clients and see if it is one of them - int clientID = findClientID(client, true); - - // Save everything - validate_access("queries", queryID, false, __LINE__, __FUNCTION__, __FILE__); - queries[queryID].magic = MAGICBYTE; - queries[queryID].timestamp = querytimestamp; - queries[queryID].type = querytype; - queries[queryID].status = QUERY_UNKNOWN; - queries[queryID].domainID = domainID; - queries[queryID].clientID = clientID; - queries[queryID].timeidx = timeidx; - // Initialize database rowID with zero, will be set when the query is stored in the long-term DB - queries[queryID].db = 0; - queries[queryID].id = id; - queries[queryID].complete = false; - queries[queryID].response = converttimeval(request); - // Initialize reply type - queries[queryID].reply = REPLY_UNKNOWN; - // Store DNSSEC result for this domain - queries[queryID].dnssec = DNSSEC_UNSPECIFIED; - - // Check and apply possible privacy level rules - // The currently set privacy level (at the time the query is - // generated) is stored in the queries structure - get_privacy_level(NULL); - queries[queryID].privacylevel = config.privacylevel; - - // Increase DNS queries counter - counters->queries++; - // Count this query as unknown as long as no reply has - // been found and analyzed - counters->unknown++; - - // Update overTime data - overTime[timeidx].total++; - // Update overTime data structure with the new client - clients[clientID].overTime[timeidx]++; - - // Set lastQuery timer and add one query for network table - clients[clientID].lastQuery = querytimestamp; - clients[clientID].numQueriesARP++; - - // Try blocking regex if configured - validate_access("domains", domainID, false, __LINE__, __FUNCTION__, __FILE__); - if(domains[domainID].regexmatch == REGEX_UNKNOWN && blockingstatus != BLOCKING_DISABLED) - { - // For minimal performance impact, we test the regex only when - // - regex checking is enabled, and - // - this domain has not already been validated against the regex. - // This effectively prevents multiple evaluations of the same domain - // - // If a regex filter matched, we additionally compare the domain - // against all known whitelisted domains to possibly prevent blocking - // of a specific domain. The logic herein is: - // If matched, then compare against whitelist - // If in whitelist, negate matched so this function returns: not-to-be-blocked - if(match_regex(domainbuffer) && !in_whitelist(domainbuffer)) - { - // We have to block this domain - block_single_domain_regex(domainbuffer); - domains[domainID].regexmatch = REGEX_BLOCKED; - } - else - { - // Explicitly mark as not blocked to skip regex test - // next time we see this domain - domains[domainID].regexmatch = REGEX_NOTBLOCKED; - } - } - - // Free allocated memory - free(client); - free(domain); - free(domainbuffer); - - // Release thread lock - unlock_shm(); -} - -static int findQueryID(int id) -{ - // Loop over all queries - we loop in reverse order (start from the most recent query and - // continuously walk older queries while trying to find a match. Ideally, we should always - // find the correct query with zero iterations, but it may happen that queries are processed - // asynchronously, e.g. for slow upstream relies to a huge amount of requests. - // We iterate from the most recent query down to at most MAXITER queries in the past to avoid - // iterating through the entire array of queries - // MAX(0, a) is used to return 0 in case a is negative (negative array indices are harmful) - - // Validate access only once for the maximum index (all lower will work) - int until = MAX(0, counters->queries-MAXITER); - int start = MAX(0, counters->queries-1); - validate_access("queries", until, false, __LINE__, __FUNCTION__, __FILE__); - - // Check UUIDs of queries - for(int i = start; i >= until; i--) - if(queries[i].id == id) - return i; - - // If not found - return -1; -} - -void _FTL_forwarded(unsigned int flags, char *name, struct all_addr *addr, int id, const char* file, const int line) -{ - // Don't analyze anything if in PRIVACY_NOSTATS mode - if(config.privacylevel >= PRIVACY_NOSTATS) - return; - - // Save that this query got forwarded to an upstream server - lock_shm(); - - // Get forward destination IP address - char dest[ADDRSTRLEN]; - // If addr == NULL, we will only duplicate an empty string instead of uninitialized memory - dest[0] = '\0'; - if(addr != NULL) - inet_ntop((flags & F_IPV4) ? AF_INET : AF_INET6, addr, dest, ADDRSTRLEN); - // Convert forward to lower case - char *forward = strdup(dest); - strtolower(forward); - - // Debug logging - if(config.debug & DEBUG_QUERIES) logg("**** forwarded %s to %s (ID %i, %s:%i)", name, forward, id, file, line); - - // Save status and forwardID in corresponding query identified by dnsmasq's ID - int i = findQueryID(id); - if(i < 0) - { - // This may happen e.g. if the original query was a PTR query or "pi.hole" - // as we ignore them altogether - free(forward); - unlock_shm(); - return; - } - - // Proceed only if - // - current query has not been marked as replied to so far - // (it could be that answers from multiple forward - // destinations are coming in for the same query) - // - the query was formally known as cached but had to be forwarded - // (this is a special case further described below) - if(queries[i].complete && queries[i].status != QUERY_CACHE) - { - free(forward); - unlock_shm(); - return; - } - - // Get ID of forward destination, create new forward destination record - // if not found in current data structure - int forwardID = findForwardID(forward, true); - queries[i].forwardID = forwardID; - - unsigned int timeidx = queries[i].timeidx; - - if(queries[i].status == QUERY_CACHE) - { - // Detect if we cached the but need to ask the upstream - // servers for the actual IPs now, we remove this query from the - // counters for cache replied queries as we had to forward a - // request for it. Example: - // Assume a domain a.com is a CNAME which is cached and has a very - // long TTL. It point to another domain server.a.com which has an - // A record but this has a much lower TTL. - // If you now query a.com and then again after some time, you end - // up in a situation where dnsmasq can answer the first level of - // the DNS result (the CNAME) from cache, hence the status of this - // query is marked as "answered from cache" in FTLDNS. However, for - // server.a.com wit the much shorter TTL, we still have to forward - // something and ask the upstream server for the final IP address. - // This code section acknowledges this by removing one entry from - // the cached counters as we will re-brand this query as having been - // forwarded in the following. - counters->cached--; - // Also correct overTime data - overTime[timeidx].cached--; - - // Correct reply timer - struct timeval response; - gettimeofday(&response, 0); - // Reset timer, shift slightly into the past to acknowledge the time - // FTLDNS needed to look up the CNAME in its cache - queries[i].response = converttimeval(response) - queries[i].response; - } - else - { - // Normal forwarded query (status is set below) - // Query is no longer unknown - counters->unknown--; - // Hereby, this query is now fully determined - queries[i].complete = true; - } - - // Set query status to forwarded only after the - // if(queries[i].status == QUERY_CACHE) { ... } - // from above as otherwise this check will always - // be negative - queries[i].status = QUERY_FORWARDED; - - // Update overTime data - overTime[timeidx].forwarded++; - - // Update counter for forwarded queries - counters->forwardedqueries++; - - // Release allocated memory - free(forward); - unlock_shm(); -} - -void FTL_dnsmasq_reload(void) -{ - // This function is called by the dnsmasq code on receive of SIGHUP - // *before* clearing the cache and rereading the lists - // This is the only hook that is not skipped in PRIVACY_NOSTATS mode - - logg("Received SIGHUP, reloading cache"); - - // Called when dnsmasq re-reads its config and hosts files - // Reset number of blocked domains - counters->gravity = 0; - - // Inspect 01-pihole.conf to see if Pi-hole blocking is enabled, - // i.e. if /etc/pihole/gravity.list is sourced as addn-hosts file - check_blocking_status(); - - // Reread pihole-FTL.conf to see which blocking mode the user wants to use - // It is possible to change the blocking mode here as we anyhow clear the - // cache and reread all blocking lists - // Passing NULL to this function means it has to open the config file on - // its own behalf (on initial reading, the config file is already opened) - get_blocking_mode(NULL); - - // Reread regex.list - free_regex(); - read_regex_from_file(); - - // Reread pihole-FTL.conf to see which debugging flags are set - read_debuging_settings(NULL); - - // Print current set of capabilities if requested via debug flag - if(config.debug & DEBUG_CAPS) - check_capabilities(); -} - -void _FTL_reply(unsigned short flags, char *name, struct all_addr *addr, int id, const char* file, const int line) -{ - // Don't analyze anything if in PRIVACY_NOSTATS mode - if(config.privacylevel >= PRIVACY_NOSTATS) - return; - - // Interpret hosts files that have been read by dnsmasq - lock_shm(); - // Determine returned result if available - char dest[ADDRSTRLEN]; dest[0] = '\0'; - if(addr) - { - inet_ntop((flags & F_IPV4) ? AF_INET : AF_INET6, addr, dest, ADDRSTRLEN); - } - - // Extract answer (used e.g. for detecting if a local config is a user-defined - // wildcard blocking entry in form "server=/tobeblocked.com/") - const char *answer = dest; - if(flags & F_CNAME) - answer = "(CNAME)"; - else if((flags & F_NEG) && (flags & F_NXDOMAIN)) - answer = "(NXDOMAIN)"; - else if(flags & F_NEG) - answer = "(NODATA)"; - - if(config.debug & DEBUG_QUERIES) - { - logg("**** got reply %s is %s (ID %i, %s:%i)", name, answer, id, file, line); - print_flags(flags); - } - - // Get response time - struct timeval response; - gettimeofday(&response, 0); - - // Save status in corresponding query identified by dnsmasq's ID - int i = findQueryID(id); - if(i < 0) - { - // This may happen e.g. if the original query was "pi.hole" - if(config.debug & DEBUG_QUERIES) logg("FTL_reply(): Query %i has not been found", id); - unlock_shm(); - return; - } - - if(queries[i].reply != REPLY_UNKNOWN) - { - // Nothing to be done here - unlock_shm(); - return; - } - - // Determine if this reply is an exact match for the queried domain - int domainID = queries[i].domainID; - validate_access("domains", domainID, true, __LINE__, __FUNCTION__, __FILE__); - bool isExactMatch = (name != NULL && strcmp(getstr(domains[domainID].domainpos), name) == 0); - - if((flags & F_CONFIG) && isExactMatch && !queries[i].complete) - { - // Answered from local configuration, might be a wildcard or user-provided - // This query is no longer unknown - counters->unknown--; - - // Get time index - unsigned int timeidx = queries[i].timeidx; - - if(strcmp(answer, "(NXDOMAIN)") == 0 || - strcmp(answer, "0.0.0.0") == 0 || - strcmp(answer, "::") == 0) - { - // Answered from user-defined blocking rules (dnsmasq config files) - counters->blocked++; - overTime[timeidx].blocked++; - - validate_access("domains", queries[i].domainID, true, __LINE__, __FUNCTION__, __FILE__); - domains[queries[i].domainID].blockedcount++; - - validate_access("clients", queries[i].clientID, true, __LINE__, __FUNCTION__, __FILE__); - clients[queries[i].clientID].blockedcount++; - - queries[i].status = QUERY_WILDCARD; - } - else - { - // Answered from a custom (user provided) cache file - counters->cached++; - overTime[timeidx].cached++; - - queries[i].status = QUERY_CACHE; - } - - // Save reply type and update individual reply counters - save_reply_type(flags, i, response); - - // Hereby, this query is now fully determined - queries[i].complete = true; - } - else if((flags & F_FORWARD) && isExactMatch) - { - // Only proceed if query is not already known - // to have been blocked by Quad9 - if(queries[i].reply != QUERY_EXTERNAL_BLOCKED_IP && - queries[i].reply != QUERY_EXTERNAL_BLOCKED_NULL && - queries[i].reply != QUERY_EXTERNAL_BLOCKED_NXRA) - { - // Save reply type and update individual reply counters - save_reply_type(flags, i, response); - - // Detect if returned IP indicates that this query was blocked - detect_blocked_IP(flags, answer, i); - } - } - else if(flags & F_REVERSE) - { - // isExactMatch is not used here as the PTR is special. - // Example: - // Question: PTR 8.8.8.8 - // will lead to: - // domains[domainID].domain = 8.8.8.8.in-addr.arpa - // and will return - // name = google-public-dns-a.google.com - // Hence, isExactMatch is always false - - // Save reply type and update individual reply counters - save_reply_type(flags, i, response); - } - else if(isExactMatch && !queries[i].complete) - { - logg("*************************** unknown REPLY ***************************"); - print_flags(flags); - } - - unlock_shm(); -} - -static void detect_blocked_IP(unsigned short flags, const char* answer, int queryID) -{ - if(flags & F_HOSTS) - { - // Skip replies which originated locally. Otherwise, we would - // count gravity.list blocked queries as externally blocked. - if(config.debug & DEBUG_EXTBLOCKED) - { - logg("Skipping detection of external blocking IP for ID %i as origin is HOSTS", queryID); - } - return; - } - else if(flags & F_REVERSE) - { - // Do not mark responses of PTR requests as externally blocked. - if(config.debug & DEBUG_EXTBLOCKED) - { - logg("Skipping detection of external blocking IP for ID %i as query is PTR", queryID); - } - return; - } - - // If received one of the following IPs as reply, OpenDNS - // (Cisco Umbrella) blocked this query - // See https://support.opendns.com/hc/en-us/articles/227986927-What-are-the-Cisco-Umbrella-Block-Page-IP-Addresses- - // for a full list of these IP addresses - if(flags & F_IPV4 && answer != NULL && - (strcmp("146.112.61.104", answer) == 0 || - strcmp("146.112.61.105", answer) == 0 || - strcmp("146.112.61.106", answer) == 0 || - strcmp("146.112.61.107", answer) == 0 || - strcmp("146.112.61.108", answer) == 0 || - strcmp("146.112.61.109", answer) == 0 || - strcmp("146.112.61.110", answer) == 0 )) - { - if(config.debug & DEBUG_EXTBLOCKED) - { - logg("Upstream responded with known blocking page (IPv4), ID %i:\n\t\"%s\" -> \"%s\"", - queryID, getstr(domains[queryID].domainpos), answer); - } - - // Update status - query_externally_blocked(queryID, QUERY_EXTERNAL_BLOCKED_IP); - } - - else if(flags & F_IPV6 && answer != NULL && - (strcmp("::ffff:146.112.61.104", answer) == 0 || - strcmp("::ffff:146.112.61.105", answer) == 0 || - strcmp("::ffff:146.112.61.106", answer) == 0 || - strcmp("::ffff:146.112.61.107", answer) == 0 || - strcmp("::ffff:146.112.61.108", answer) == 0 || - strcmp("::ffff:146.112.61.109", answer) == 0 || - strcmp("::ffff:146.112.61.110", answer) == 0 )) - { - if(config.debug & DEBUG_EXTBLOCKED) - { - logg("Upstream responded with known blocking page (IPv6), ID %i:\n\t\"%s\" -> \"%s\"", - queryID, getstr(domains[queryID].domainpos), answer); - } - - // Update status - query_externally_blocked(queryID, QUERY_EXTERNAL_BLOCKED_IP); - } - - // If upstream replied with 0.0.0.0 or ::, - // we assume that it filtered the reply as - // nothing is reachable under these addresses - else if(flags & F_IPV4 && answer != NULL && - strcmp("0.0.0.0", answer) == 0) - { - if(config.debug & DEBUG_EXTBLOCKED) - { - logg("Upstream responded with 0.0.0.0, ID %i:\n\t\"%s\" -> \"%s\"", - queryID, getstr(domains[queryID].domainpos), answer); - } - - // Update status - query_externally_blocked(queryID, QUERY_EXTERNAL_BLOCKED_NULL); - } - - else if(flags & F_IPV6 && answer != NULL && - strcmp("::", answer) == 0) - { - if(config.debug & DEBUG_EXTBLOCKED) - { - logg("Upstream responded with ::, ID %i:\n\t\"%s\" -> \"%s\"", - queryID, getstr(domains[queryID].domainpos), answer); - } - - // Update status - query_externally_blocked(queryID, QUERY_EXTERNAL_BLOCKED_NULL); - } -} - -static void query_externally_blocked(int i, unsigned char status) -{ - // If query is already known to be externally blocked, - // then we have nothing to do here - if(queries[i].status == QUERY_EXTERNAL_BLOCKED_IP || - queries[i].status == QUERY_EXTERNAL_BLOCKED_NULL || - queries[i].status == QUERY_EXTERNAL_BLOCKED_NXRA) - return; - - // Get time index of this query - unsigned int timeidx = queries[i].timeidx; - - // Correct counters if necessary ... - if(queries[i].status == QUERY_FORWARDED) - { - counters->forwardedqueries--; - overTime[timeidx].forwarded--; - validate_access("forwarded", queries[i].forwardID, true, __LINE__, __FUNCTION__, __FILE__); - forwarded[queries[i].forwardID].count--; - } - // ... but as blocked - counters->blocked++; - overTime[timeidx].blocked++; - validate_access("domains", queries[i].domainID, true, __LINE__, __FUNCTION__, __FILE__); - domains[queries[i].domainID].blockedcount++; - validate_access("clients", queries[i].clientID, true, __LINE__, __FUNCTION__, __FILE__); - clients[queries[i].clientID].blockedcount++; - - // Update status - queries[i].status = status; -} - -void _FTL_cache(unsigned int flags, char *name, struct all_addr *addr, char *arg, int id, const char* file, const int line) -{ - // Don't analyze anything if in PRIVACY_NOSTATS mode - if(config.privacylevel >= PRIVACY_NOSTATS) - return; - - // Save that this query got answered from cache - lock_shm(); - char dest[ADDRSTRLEN]; dest[0] = '\0'; - if(addr) - { - inet_ntop((flags & F_IPV4) ? AF_INET : AF_INET6, addr, dest, ADDRSTRLEN); - } - - // Convert domain to lower case - char *domain = strdup(name); - strtolower(domain); - - // If domain is "pi.hole", we skip this query - if(strcmp(domain, "pi.hole") == 0) - { - // free memory already allocated here - free(domain); - unlock_shm(); - return; - } - free(domain); - - // Debug logging - if(config.debug & DEBUG_QUERIES) - { - logg("**** got cache answer for %s / %s / %s (ID %i, %s:%i)", name, dest, arg, id, file, line); - print_flags(flags); - } - - // Get response time - struct timeval response; - gettimeofday(&response, 0); - - if(((flags & F_HOSTS) && (flags & F_IMMORTAL)) || - ((flags & F_NAMEP) && (flags & F_DHCP)) || - (flags & F_FORWARD) || - (flags & F_REVERSE) || - (flags & F_RRNAME)) - { - // List data: /etc/pihole/gravity.list, /etc/pihole/black.list, /etc/pihole/local.list, etc. - // or - // DHCP server reply - // or - // regex blocked query - // or - // cached answer to previously forwarded request - - // Determine requesttype - unsigned char requesttype = 0; - if(flags & F_HOSTS) - { - if(arg != NULL && strstr(arg, "/gravity.list") != NULL) - requesttype = QUERY_GRAVITY; - else if(arg != NULL && strstr(arg, "/black.list") != NULL) - requesttype = QUERY_BLACKLIST; - else // local.list, hostname.list, /etc/hosts and others - requesttype = QUERY_CACHE; - } - else if((flags & F_NAMEP) && (flags & F_DHCP)) // DHCP server reply - requesttype = QUERY_CACHE; - else if(flags & F_FORWARD) // cached answer to previously forwarded request - requesttype = QUERY_CACHE; - else if(flags & F_REVERSE) // cached answer to reverse request (PTR) - requesttype = QUERY_CACHE; - else if(flags & F_RRNAME) // cached answer to TXT query - requesttype = QUERY_CACHE; - else - { - logg("*************************** unknown CACHE reply (1) ***************************"); - print_flags(flags); - } - - int i = findQueryID(id); - if(i < 0 || queries[i].complete) - { - // This may happen e.g. if the original query was a PTR query or "pi.hole" - // as we ignore them altogether or if the query is already complete - unlock_shm(); - return; - } - - // This query is no longer unknown - counters->unknown--; - - // Get time index - unsigned int timeidx = queries[i].timeidx; - - int domainID = queries[i].domainID; - validate_access("domains", domainID, true, __LINE__, __FUNCTION__, __FILE__); - - int clientID = queries[i].clientID; - validate_access("clients", clientID, true, __LINE__, __FUNCTION__, __FILE__); - - // Mark this query as blocked if domain was matched by a regex - if(domains[domainID].regexmatch == REGEX_BLOCKED) - requesttype = QUERY_WILDCARD; - - queries[i].status = requesttype; - - // Detect if returned IP indicates that this query was blocked - detect_blocked_IP(flags, dest, i); - - // Re-read requesttype as detect_blocked_IP() might have changed it - requesttype = queries[i].status; - - // Handle counters accordingly - switch(requesttype) - { - case QUERY_GRAVITY: // gravity.list - case QUERY_BLACKLIST: // black.list - case QUERY_WILDCARD: // regex blocked - counters->blocked++; - overTime[timeidx].blocked++; - domains[domainID].blockedcount++; - clients[clientID].blockedcount++; - break; - case QUERY_CACHE: // cached from one of the lists - counters->cached++; - overTime[timeidx].cached++; - break; - case QUERY_EXTERNAL_BLOCKED_IP: - case QUERY_EXTERNAL_BLOCKED_NULL: - case QUERY_EXTERNAL_BLOCKED_NXRA: - // everything has already been done - // in query_externally_blocked() - break; - } - - // Save reply type and update individual reply counters - save_reply_type(flags, i, response); - - // Hereby, this query is now fully determined - queries[i].complete = true; - } - else - { - logg("*************************** unknown CACHE reply (2) ***************************"); - print_flags(flags); - } - unlock_shm(); -} - -void _FTL_dnssec(int status, int id, const char* file, const int line) -{ - // Don't analyze anything if in PRIVACY_NOSTATS mode - if(config.privacylevel >= PRIVACY_NOSTATS) - return; - - // Process DNSSEC result for a domain - lock_shm(); - // Search for corresponding query identified by ID - int i = findQueryID(id); - if(i < 0) - { - // This may happen e.g. if the original query was an unhandled query type - unlock_shm(); - return; - } - - // Debug logging - if(config.debug & DEBUG_QUERIES) - { - int domainID = queries[i].domainID; - validate_access("domains", domainID, true, __LINE__, __FUNCTION__, __FILE__); - logg("**** got DNSSEC details for %s: %i (ID %i, %s:%i)", getstr(domains[domainID].domainpos), status, id, file, line); - } - - // Iterate through possible values - if(status == STAT_SECURE) - queries[i].dnssec = DNSSEC_SECURE; - else if(status == STAT_INSECURE) - queries[i].dnssec = DNSSEC_INSECURE; - else - queries[i].dnssec = DNSSEC_BOGUS; - - unlock_shm(); -} - -void _FTL_upstream_error(unsigned int rcode, int id, const char* file, const int line) -{ - // Process upstream errors - // Queries with error are those where the RCODE - // in the DNS header is neither NOERROR nor NXDOMAIN. - - // Don't analyze anything if in PRIVACY_NOSTATS mode - if(config.privacylevel >= PRIVACY_NOSTATS) - return; - - // Process DNSSEC result for a domain - lock_shm(); - // Search for corresponding query identified by ID - int i = findQueryID(id); - if(i < 0) - { - // This may happen e.g. if the original query was an unhandled query type - unlock_shm(); - return; - } - // Translate dnsmasq's rcode into something we can use - const char *rcodestr = NULL; - switch(rcode) - { - case SERVFAIL: - rcodestr = "SERVFAIL"; - queries[i].reply = REPLY_SERVFAIL; - break; - case REFUSED: - rcodestr = "REFUSED"; - queries[i].reply = REPLY_REFUSED; - break; - case NOTIMP: - rcodestr = "NOT IMPLEMENTED"; - queries[i].reply = REPLY_NOTIMP; - break; - default: - rcodestr = "UNKNOWN"; - queries[i].reply = REPLY_OTHER; - break; - } - - // Debug logging - if(config.debug & DEBUG_QUERIES) - { - int domainID = queries[i].domainID; - validate_access("domains", domainID, true, __LINE__, __FUNCTION__, __FILE__); - logg("**** got error report for %s: %s (ID %i, %s:%i)", getstr(domains[domainID].domainpos), rcodestr, id, file, line); - if(queries[i].reply == REPLY_OTHER) - { - logg("Unknown rcode = %i", rcode); - } - } - - unlock_shm(); -} - -void _FTL_header_analysis(const unsigned char header4, const unsigned int rcode, const int id, const char* file, const int line) -{ - // Don't analyze anything if in PRIVACY_NOSTATS mode - if(config.privacylevel >= PRIVACY_NOSTATS) - return; - - // Check if RA bit is unset in DNS header and rcode is NXDOMAIN - // If the response code (rcode) is NXDOMAIN, we may be seeing a response from - // an externally blocked query. As they are not always accompany a necessary - // SOA record, they are not getting added to our cache and, therefore, - // FTL_reply() is never getting called from within the cache routines. - // Hence, we have to store the necessary information about the NXDOMAIN - // reply already here. - if((header4 & 0x80) || rcode != NXDOMAIN) - { - // RA bit is set or rcode is not NXDOMAIN - return; - } - - lock_shm(); - - // Search for corresponding query identified by ID - int queryID = findQueryID(id); - if(queryID < 0) - { - // This may happen e.g. if the original query was an unhandled query type - unlock_shm(); - return; - } - - if(config.debug & DEBUG_QUERIES) - { - int domainID = queries[queryID].domainID; - validate_access("domains", domainID, true, __LINE__, __FUNCTION__, __FILE__); - logg("**** %s externally blocked (ID %i, FTL %i, %s:%i)", getstr(domains[domainID].domainpos), id, queryID, file, line); - } - - - // Get response time - struct timeval response; - gettimeofday(&response, 0); - - // Store query as externally blocked - query_externally_blocked(queryID, QUERY_EXTERNAL_BLOCKED_NXRA); - - // Store reply type as replied with NXDOMAIN - save_reply_type(F_NEG | F_NXDOMAIN, queryID, response); - - unlock_shm(); -} - -void print_flags(unsigned int flags) -{ - // Debug function, listing resolver flags in clear text - // e.g. "Flags: F_FORWARD F_NEG F_IPV6" - - // Only print flags if corresponding debugging flag is set - if(!(config.debug & DEBUG_FLAGS)) - return; - - unsigned int i; - char *flagstr = calloc(256,sizeof(char)); - for(i = 0; i < sizeof(flags)*8; i++) - if(flags & (1u << i)) - strcat(flagstr, flagnames[i]); - logg(" Flags: %s", flagstr); - free(flagstr); -} - -void save_reply_type(unsigned int flags, int queryID, struct timeval response) -{ - // Iterate through possible values - validate_access("queries", queryID, false, __LINE__, __FUNCTION__, __FILE__); - if(flags & F_NEG) - { - if(flags & F_NXDOMAIN) - { - // NXDOMAIN - queries[queryID].reply = REPLY_NXDOMAIN; - counters->reply_NXDOMAIN++; - } - else - { - // NODATA(-IPv6) - queries[queryID].reply = REPLY_NODATA; - counters->reply_NODATA++; - } - } - else if(flags & F_CNAME) - { - // - queries[queryID].reply = REPLY_CNAME; - counters->reply_CNAME++; - } - else if(flags & F_REVERSE) - { - // reserve lookup - queries[queryID].reply = REPLY_DOMAIN; - counters->reply_domain++; - } - else if(flags & F_RRNAME) - { - // TXT query - queries[queryID].reply = REPLY_RRNAME; - } - else - { - // Valid IP - queries[queryID].reply = REPLY_IP; - counters->reply_IP++; - } - - // Save response time (relative time) - queries[queryID].response = converttimeval(response) - - queries[queryID].response; -} - -pthread_t telnet_listenthreadv4; -pthread_t telnet_listenthreadv6; -pthread_t socket_listenthread; -pthread_t DBthread; -pthread_t GCthread; -pthread_t DNSclientthread; - -void FTL_fork_and_bind_sockets(struct passwd *ent_pw) -{ - if(daemonmode) - go_daemon(); - else - savepid(); - - // We will use the attributes object later to start all threads in detached mode - pthread_attr_t attr; - // Initialize thread attributes object with default attribute values - pthread_attr_init(&attr); - // When a detached thread terminates, its resources are automatically released back to - // the system without the need for another thread to join with the terminated thread - pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); - - // Bind to sockets - bind_sockets(); - - // Start TELNET IPv4 thread - if(ipv4telnet && pthread_create( &telnet_listenthreadv4, &attr, telnet_listening_thread_IPv4, NULL ) != 0) - { - logg("Unable to open IPv4 telnet listening thread. Exiting..."); - exit(EXIT_FAILURE); - } - - // Start TELNET IPv6 thread - if(ipv6telnet && pthread_create( &telnet_listenthreadv6, &attr, telnet_listening_thread_IPv6, NULL ) != 0) - { - logg("Unable to open IPv6 telnet listening thread. Exiting..."); - exit(EXIT_FAILURE); - } - - // Start SOCKET thread - if(pthread_create( &socket_listenthread, &attr, socket_listening_thread, NULL ) != 0) - { - logg("Unable to open Unix socket listening thread. Exiting..."); - exit(EXIT_FAILURE); - } - - // Start database thread if database is used - if(database && pthread_create( &DBthread, &attr, DB_thread, NULL ) != 0) - { - logg("Unable to open database thread. Exiting..."); - exit(EXIT_FAILURE); - } - - // Start thread that will stay in the background until garbage collection needs to be done - if(pthread_create( &GCthread, &attr, GC_thread, NULL ) != 0) - { - logg("Unable to open GC thread. Exiting..."); - exit(EXIT_FAILURE); - } - - // Start thread that will stay in the background until host names needs to be resolved - if(pthread_create( &DNSclientthread, &attr, DNSclient_thread, NULL ) != 0) - { - logg("Unable to open DNS client thread. Exiting..."); - exit(EXIT_FAILURE); - } - - // Chown files if FTL started as user root but a dnsmasq config option - // states to run as a different user/group (e.g. "nobody") - if(ent_pw != NULL && getuid() == 0) - { - if(chown(FTLfiles.log, ent_pw->pw_uid, ent_pw->pw_gid) == -1) - logg("Setting ownership (%i:%i) of %s failed: %s (%i)", ent_pw->pw_uid, ent_pw->pw_gid, FTLfiles.log, strerror(errno), errno); - if(database && chown(FTLfiles.db, ent_pw->pw_uid, ent_pw->pw_gid) == -1) - logg("Setting ownership (%i:%i) of %s failed: %s (%i)", ent_pw->pw_uid, ent_pw->pw_gid, FTLfiles.db, strerror(errno), errno); - } -} - -// int cache_inserted, cache_live_freed are defined in dnsmasq/cache.c -void getCacheInformation(int *sock) -{ - ssend(*sock,"cache-size: %i\ncache-live-freed: %i\ncache-inserted: %i\n", - daemon->cachesize, - daemon->metrics[METRIC_DNS_CACHE_LIVE_FREED], - daemon->metrics[METRIC_DNS_CACHE_INSERTED]); - // cache-size is obvious - // It means the resolver handled names lookups that needed to be sent to - // upstream severes and that was thrown out of the cache - // before reaching the end of its time-to-live, to make room for a newer name. - // For , smaller is better. - // New queries are always cached. If the cache is full with entries - // which haven't reached the end of their time-to-live, then the entry - // which hasn't been looked up for the longest time is evicted. -} - -void _FTL_forwarding_failed(struct server *server, const char* file, const int line) -{ - // Don't analyze anything if in PRIVACY_NOSTATS mode - if(config.privacylevel >= PRIVACY_NOSTATS) - return; - - // Save that this query got forwarded to an upstream server - lock_shm(); - char dest[ADDRSTRLEN]; - if(server->addr.sa.sa_family == AF_INET) - inet_ntop(AF_INET, &server->addr.in.sin_addr, dest, ADDRSTRLEN); - else - inet_ntop(AF_INET6, &server->addr.in6.sin6_addr, dest, ADDRSTRLEN); - - // Convert forward to lower case - char *forward = strdup(dest); - strtolower(forward); - int forwardID = findForwardID(forward, false); - - if(config.debug & DEBUG_QUERIES) logg("**** forwarding to %s (ID %i, %s:%i) failed", dest, forwardID, file, line); - - forwarded[forwardID].failed++; - - free(forward); - unlock_shm(); - return; -} - -static unsigned long __attribute__((const)) converttimeval(struct timeval time) -{ - // Convert time from struct timeval into units - // of 10*milliseconds - return time.tv_sec*10000 + time.tv_usec/100; -} - -// This subroutine prepares IPv4 and IPv6 addresses for blocking queries depending on the configured blocking mode -static void prepare_blocking_mode(struct all_addr *addr4, struct all_addr *addr6, bool *has_IPv4, bool *has_IPv6) -{ - // Read IPv4 address for host entries from setupVars.conf - char* const IPv4addr = read_setupVarsconf("IPV4_ADDRESS"); - if((config.blockingmode == MODE_IP || config.blockingmode == MODE_IP_NODATA_AAAA) && - IPv4addr != NULL && strlen(IPv4addr) > 0) - { - // Strip off everything at the end of the IP (CIDR might be there) - char* a=IPv4addr; for(;*a;a++) if(*a == '/') *a = 0; - // Prepare IPv4 address for records - if(inet_pton(AF_INET, IPv4addr, addr4) > 0) - *has_IPv4 = true; - } - else - { - // Blocking mode will use zero-initialized all_addr struct - *has_IPv4 = true; - } - clearSetupVarsArray(); // will free/invalidate IPv4addr - - // Read IPv6 address for host entries from setupVars.conf - char* const IPv6addr = read_setupVarsconf("IPV6_ADDRESS"); - if(config.blockingmode == MODE_IP && - IPv6addr != NULL && strlen(IPv6addr) > 0) - { - // Strip off everything at the end of the IP (CIDR might be there) - char* a=IPv6addr; for(;*a;a++) if(*a == '/') *a = 0; - // Prepare IPv6 address for records - if(inet_pton(AF_INET6, IPv6addr, addr6) > 0) - *has_IPv6 = true; - } - else if(config.blockingmode == MODE_IP_NODATA_AAAA) - { - // Blocking mode will use zero-initialized all_addr struct - // This is irrelevant, however, as this blocking mode will - // reply with NODATA to AAAA queries. Still, we need to - // generate separate IPv4 (IP) and AAAA (NODATA) records - *has_IPv6 = true; - } - else - { - // Don't create IPv6 cache entries when we don't need them - // Also, don't create them if we are in IP blocking mode and - // strlen(IPv6addr) == 0 - *has_IPv6 = false; - } - clearSetupVarsArray(); // will free/invalidate IPv6addr -} - -// Prototypes from functions in dnsmasq's source -void add_hosts_entry(struct crec *cache, struct all_addr *addr, int addrlen, unsigned int index, struct crec **rhash, int hashsz); -void rehash(int size); - -// This routine adds one domain to the resolver's cache. Depending on the configured blocking mode it may create -// a single entry valid for IPv4 & IPv6 or two entries one for IPv4 and one for IPv6. -// When IPv6 is not available on the machine, we do not add IPv6 cache entries (likewise for IPv4) -static int add_blocked_domain(struct all_addr *addr4, struct all_addr *addr6, bool has_IPv4, bool has_IPv6, - char *domain, int len, struct crec **rhash, int hashsz, unsigned int index) -{ - int name_count = 0; - struct crec *cache4,*cache6; - // Add IPv4 record, allocate enough space for cache entry including arbitrary domain name length - // (the domain name is stored at the end of struct crec) - if(has_IPv4 && - (cache4 = malloc(sizeof(struct crec) + len+1-SMALLDNAME))) - { - strcpy(cache4->name.sname, domain); - cache4->flags = F_HOSTS | F_IMMORTAL | F_FORWARD | F_IPV4; - int memorysize = INADDRSZ; - if(config.blockingmode == MODE_NX) - { - // If we block in NXDOMAIN mode, we add the NXDOMAIN flag and make this host record - // also valid for AAAA requests - cache4->flags |= F_IPV6 | F_NEG | F_NXDOMAIN; - } - else if(config.blockingmode == MODE_NULL) - { - // If we block in NULL mode, we make this host record also valid for AAAA requests - // This is okay as the addr structs have been statically zero-initialized - cache4->flags |= F_IPV6; - memorysize = IN6ADDRSZ; - } - else if(config.blockingmode == MODE_NODATA) - { - // If we block in NODATA mode, we make this host record also valid for AAAA requests - // and apply the NEG response flag (but not the NXDOMAIN flag) - cache4->flags |= F_IPV6 | F_NEG; - } - cache4->ttd = daemon->local_ttl; - add_hosts_entry(cache4, addr4, memorysize, index, rhash, hashsz); - name_count++; - } - // Add IPv6 record only if we respond with a non-NULL IP address to blocked domains - if(has_IPv6 && (config.blockingmode == MODE_IP || config.blockingmode == MODE_IP_NODATA_AAAA) && - (cache6 = malloc(sizeof(struct crec) + len+1-SMALLDNAME))) - { - strcpy(cache6->name.sname, domain); - cache6->flags = F_HOSTS | F_IMMORTAL | F_FORWARD | F_IPV6; - if(config.blockingmode == MODE_IP_NODATA_AAAA) cache6->flags |= F_NEG; - cache6->ttd = daemon->local_ttl; - add_hosts_entry(cache6, addr6, IN6ADDRSZ, index, rhash, hashsz); - name_count++; - } - - // Return 1 if only one cache slot was allocated (IPv4) or 2 if two slots were allocated (IPv4 + IPv6) - return name_count; -} - -// Add a single domain to resolver's cache. This respects the configured blocking mode -// Note: This routine is meant for adding a single domain at a time. It should not be -// invoked for batch processing -static void block_single_domain_regex(char *domain) -{ - struct all_addr addr4 = {{{ 0 }}}, addr6 = {{{ 0 }}}; - bool has_IPv4 = false, has_IPv6 = false; - - // Get IPv4/v6 addresses for blocking depending on user configures blocking mode - prepare_blocking_mode(&addr4, &addr6, &has_IPv4, &has_IPv6); - regexlistname = files.regexlist; - add_blocked_domain(&addr4, &addr6, has_IPv4, has_IPv6, domain, strlen(domain), NULL, 0, SRC_REGEX); - - if(config.debug & DEBUG_QUERIES) logg("Added %s to cache", domain); - - return; -} - -int FTL_listsfile(char* filename, unsigned int index, FILE *f, int cache_size, struct crec **rhash, int hashsz) -{ - int name_count = cache_size; - int added = 0; - size_t size = 0; - char *buffer = NULL; - struct all_addr addr4 = {{{ 0 }}}, addr6 = {{{ 0 }}}; - bool has_IPv4 = false, has_IPv6 = false; - - // Handle only gravity.list and black.list - // Skip all other files (they are interpreted in the usual format) - if(strcmp(filename, files.gravity) != 0 && - strcmp(filename, files.blacklist) != 0) - return cache_size; - - // Start timer for list analysis - timer_start(LISTS_TIMER); - - // Get IPv4/v6 addresses for blocking depending on user configured blocking mode - prepare_blocking_mode(&addr4, &addr6, &has_IPv4, &has_IPv6); - - // If we have neither a valid IPv4 nor a valid IPv6 but the user asked for - // blocking modes MODE_IP or MODE_IP_NODATA_AAAA then we cannot add any entries here - if(!has_IPv4 && !has_IPv6) - { - logg("ERROR: found neither a valid IPV4_ADDRESS nor IPV6_ADDRESS in setupVars.conf"); - return cache_size; - } - - // Walk file line by line - bool firstline = true; - while(getline(&buffer, &size, f) != -1) - { - char *domain = buffer; - // Skip hashed out lines - if(*domain == '#') - continue; - - // Filter leading dots or spaces - while (*domain == '.' || *domain == ' ') domain++; - - // Check for spaces or tabs - // If found, then this list is still in HOSTS format and we - // don't analyze it here. We only check the first line for - // efficiency reasons (strstr() is slow) - if(firstline && - (strstr(domain, " ") != NULL || strstr(domain, "\t") != NULL)) - { - // Reset file pointer back to beginning of the list - rewind(f); - logg("File %s is in HOSTS format, please run pihole -g!", filename); - return name_count; - } - firstline = false; - - // Skip empty lines - int len = strlen(domain); - if(len == 0) - continue; - - // Strip newline character at the end of line we just read - if(domain[len-1] == '\n') - { - domain[len-1] = '\0'; - len -= 1; - } - - // As of here we assume the entry to be valid - // Rehash every 1000 valid names - if(rhash && ((name_count - cache_size) > 1000)) - { - rehash(name_count); - cache_size = name_count; - } - - // Add domain - name_count += add_blocked_domain(&addr4, &addr6, has_IPv4, has_IPv6, domain, len, rhash, hashsz, index); - - // Count added domain - added++; - } - - // Rehash after having read all entries - if(rhash) - rehash(name_count); - - // Free allocated memory - if(buffer != NULL) - { - free(buffer); - buffer = NULL; - } - - logg("%s: parsed %i domains (took %.1f ms)", filename, added, timer_elapsed_msec(LISTS_TIMER)); - counters->gravity += added; - return name_count; -} diff --git a/dnsmasq_interface.h b/dnsmasq_interface.h deleted file mode 100644 index 51b76af9c..000000000 --- a/dnsmasq_interface.h +++ /dev/null @@ -1,40 +0,0 @@ -/* Pi-hole: A black hole for Internet advertisements -* (c) 2017 Pi-hole, LLC (https://pi-hole.net) -* Network-wide ad blocking via your own hardware. -* -* FTL Engine -* dnsmasq server interfacing routines -* -* This file is copyright under the latest version of the EUPL. -* Please see LICENSE file for your rights under this license. */ -extern int socketfd, telnetfd4, telnetfd6; -extern unsigned char* pihole_privacylevel; -enum { TCP, UDP }; - -#define FTL_new_query(flags, name, addr, types, id, type) _FTL_new_query(flags, name, addr, types, id, type, __FILE__, __LINE__) -void _FTL_new_query(unsigned int flags, char *name, struct all_addr *addr, char *types, int id, char type, const char* file, const int line); - -#define FTL_forwarded(flags, name, addr, id) _FTL_forwarded(flags, name, addr, id, __FILE__, __LINE__) -void _FTL_forwarded(unsigned int flags, char *name, struct all_addr *addr, int id, const char* file, const int line); - -#define FTL_reply(flags, name, addr, id) _FTL_reply(flags, name, addr, id, __FILE__, __LINE__) -void _FTL_reply(unsigned short flags, char *name, struct all_addr *addr, int id, const char* file, const int line); - -#define FTL_cache(flags, name, addr, arg, id) _FTL_cache(flags, name, addr, arg, id, __FILE__, __LINE__) -void _FTL_cache(unsigned int flags, char *name, struct all_addr *addr, char * arg, int id, const char* file, const int line); - -#define FTL_dnssec(status, id) _FTL_dnssec(status, id, __FILE__, __LINE__) -void _FTL_dnssec(int status, int id, const char* file, const int line); - -#define FTL_header_analysis(header4, rcode, id) _FTL_header_analysis(header4, rcode, id, __FILE__, __LINE__) -void _FTL_header_analysis(unsigned char header4, unsigned int rcode, int id, const char* file, const int line); - -#define FTL_forwarding_failed(server) _FTL_forwarding_failed(server, __FILE__, __LINE__) -void _FTL_forwarding_failed(struct server *server, const char* file, const int line); - -#define FTL_upstream_error(rcode, id) _FTL_upstream_error(rcode, id, __FILE__, __LINE__) -void _FTL_upstream_error(unsigned int rcode, int id, const char* file, const int line); - -void FTL_dnsmasq_reload(void); -void FTL_fork_and_bind_sockets(struct passwd *ent_pw); -int FTL_listsfile(char* filename, unsigned int index, FILE *f, int cache_size, struct crec **rhash, int hashsz); diff --git a/docker/aarch64/Dockerfile b/docker/aarch64/Dockerfile deleted file mode 100644 index 59f1462b0..000000000 --- a/docker/aarch64/Dockerfile +++ /dev/null @@ -1,13 +0,0 @@ -FROM debian:stretch - -RUN dpkg --add-architecture arm64 && \ - apt-get update && \ - apt-get install -y --no-install-recommends nettle-dev:arm64 gcc-aarch64-linux-gnu libc-dev-arm64-cross \ - make file wget netcat-traditional sqlite3 git ca-certificates ssh libcap-dev:arm64 - -# Install ghr for GitHub Releases: https://github.com/tcnksm/ghr -RUN wget https://github.com/tcnksm/ghr/releases/download/v0.12.0/ghr_v0.12.0_linux_amd64.tar.gz && \ - tar -xzf ghr_*_linux_amd64.tar.gz && \ - mv ghr_*_linux_amd64/ghr /usr/bin/ghr - -ENV CC aarch64-linux-gnu-gcc diff --git a/docker/arm/Dockerfile b/docker/arm/Dockerfile deleted file mode 100644 index 1043a66f2..000000000 --- a/docker/arm/Dockerfile +++ /dev/null @@ -1,32 +0,0 @@ -FROM debian:stretch - -# Packages required to install compiler and libraries -RUN apt-get update && \ - apt-get install -y --no-install-recommends wget git ca-certificates - -# Use Raspbian's GCC -# This command was taken from https://github.com/dockcross/dockcross/blob/master/linux-armv6/Dockerfile -# Slightly modified from the original -RUN mkdir rpi_tools && cd rpi_tools && git init && git remote add origin https://github.com/raspberrypi/tools && \ - git config core.sparseCheckout true && \ - echo "arm-bcm2708/arm-rpi-4.9.3-linux-gnueabihf" >> .git/info/sparse-checkout && \ - git pull --depth=1 origin master && \ - cp -a arm-bcm2708/arm-rpi-4.9.3-linux-gnueabihf/* /usr/ && rm -rf ../rpi_tools - -RUN wget ftl.pi-hole.net/libraries/libgmp.a -O /usr/local/lib/libgmp.a && \ - wget ftl.pi-hole.net/libraries/libnettle.a -O /usr/local/lib/libnettle.a && \ - wget ftl.pi-hole.net/libraries/libhogweed.a -O /usr/local/lib/libhogweed.a && \ - wget ftl.pi-hole.net/libraries/libcap.so.2.25 -O /usr/local/lib/libcap.so - -RUN dpkg --add-architecture armhf && \ - apt-get update && \ - apt-get install -y --no-install-recommends make file netcat-traditional ssh \ - nettle-dev:armhf libcap-dev sqlite3 - -# Install ghr for GitHub Releases: https://github.com/tcnksm/ghr -RUN wget https://github.com/tcnksm/ghr/releases/download/v0.12.0/ghr_v0.12.0_linux_amd64.tar.gz && \ - tar -xzf ghr_*_linux_amd64.tar.gz && \ - mv ghr_*_linux_amd64/ghr /usr/bin/ghr - -# Allow libnettle to be used, because this GCC doesn't have all the right header and library directories -ENV CC "arm-linux-gnueabihf-gcc -I/usr/include -I/usr/include/arm-linux-gnueabihf -L/usr/local/lib" diff --git a/docker/armhf/Dockerfile b/docker/armhf/Dockerfile deleted file mode 100644 index 4262ce335..000000000 --- a/docker/armhf/Dockerfile +++ /dev/null @@ -1,13 +0,0 @@ -FROM debian:stretch - -RUN dpkg --add-architecture armhf && \ - apt-get update && \ - apt-get install -y --no-install-recommends nettle-dev:armhf gcc-arm-linux-gnueabihf libc6-dev-armhf-cross \ - make file wget netcat-traditional sqlite3 git ca-certificates ssh libcap-dev:armhf - -# Install ghr for GitHub Releases: https://github.com/tcnksm/ghr -RUN wget https://github.com/tcnksm/ghr/releases/download/v0.12.0/ghr_v0.12.0_linux_amd64.tar.gz && \ - tar -xzf ghr_*_linux_amd64.tar.gz && \ - mv ghr_*_linux_amd64/ghr /usr/bin/ghr - -ENV CC arm-linux-gnueabihf-gcc diff --git a/docker/x86_32/Dockerfile b/docker/x86_32/Dockerfile deleted file mode 100644 index 4e326d6d8..000000000 --- a/docker/x86_32/Dockerfile +++ /dev/null @@ -1,13 +0,0 @@ -FROM debian:stretch - -RUN dpkg --add-architecture i386 && \ - apt-get update && \ - apt-get install -y --no-install-recommends nettle-dev:i386 gcc gcc-multilib \ - make file wget netcat-traditional sqlite3 git ca-certificates ssh libcap-dev:i386 - -# Install ghr for GitHub Releases: https://github.com/tcnksm/ghr -RUN wget https://github.com/tcnksm/ghr/releases/download/v0.12.0/ghr_v0.12.0_linux_amd64.tar.gz && \ - tar -xzf ghr_*_linux_amd64.tar.gz && \ - mv ghr_*_linux_amd64/ghr /usr/bin/ghr - -ENV CC "gcc -m32" diff --git a/docker/x86_64-musl/Dockerfile b/docker/x86_64-musl/Dockerfile deleted file mode 100644 index 78020f761..000000000 --- a/docker/x86_64-musl/Dockerfile +++ /dev/null @@ -1,12 +0,0 @@ -FROM debian:stretch - -RUN apt-get update && \ - apt-get install -y --no-install-recommends nettle-dev musl-tools \ - make file wget netcat-traditional sqlite3 git ca-certificates ssh - -# Install ghr for GitHub Releases: https://github.com/tcnksm/ghr -RUN wget https://github.com/tcnksm/ghr/releases/download/v0.12.0/ghr_v0.12.0_linux_amd64.tar.gz && \ - tar -xzf ghr_*_linux_amd64.tar.gz && \ - mv ghr_*_linux_amd64/ghr /usr/bin/ghr - -ENV CC musl-gcc diff --git a/docker/x86_64/Dockerfile b/docker/x86_64/Dockerfile deleted file mode 100644 index f1b6944ec..000000000 --- a/docker/x86_64/Dockerfile +++ /dev/null @@ -1,12 +0,0 @@ -FROM debian:stretch - -RUN apt-get update && \ - apt-get install -y --no-install-recommends nettle-dev gcc libc-dev \ - make file wget netcat-traditional sqlite3 git ca-certificates ssh libcap-dev - -# Install ghr for GitHub Releases: https://github.com/tcnksm/ghr -RUN wget https://github.com/tcnksm/ghr/releases/download/v0.12.0/ghr_v0.12.0_linux_amd64.tar.gz && \ - tar -xzf ghr_*_linux_amd64.tar.gz && \ - mv ghr_*_linux_amd64/ghr /usr/bin/ghr - -ENV CC gcc diff --git a/grep.c b/grep.c deleted file mode 100644 index 09cba3c8a..000000000 --- a/grep.c +++ /dev/null @@ -1,127 +0,0 @@ -/* Pi-hole: A black hole for Internet advertisements -* (c) 2017 Pi-hole, LLC (https://pi-hole.net) -* Network-wide ad blocking via your own hardware. -* -* FTL Engine -* grep-like routines -* -* This file is copyright under the latest version of the EUPL. -* Please see LICENSE file for your rights under this license. */ - -#include "FTL.h" - -char ** wildcarddomains = NULL; -unsigned char blockingstatus = 2; - -int countlines(const char* fname) -{ - FILE *fp; - int ch = 0, lines = 0, chars = 0; - - if((fp = fopen(fname, "r")) == NULL) { - return -1; - } - - while ((ch = fgetc(fp)) != EOF) - { - chars++; - if (ch=='\n') - { - // Add one to the lines counter - ++lines; - // Reset chars counter - chars = 0; - } - } - - // Add one more line if there were characters at the - // last line of the file even without a final "\n" - if(chars > 0) - ++lines; - - // Close the file - fclose(fp); - - return lines; -} - -int countlineswith(const char* str, const char* fname) -{ - FILE *fp; - int found = 0; - char *buffer = NULL; - size_t size = 0; - - if((fp = fopen(fname, "r")) == NULL) { - return -1; - } - - // Search through file - // getline reads a string from the specified file up to either a - // newline character or EOF - while(getline(&buffer, &size, fp) != -1) - { - // Strip potential newline character at the end of line we just read - if(buffer[strlen(buffer)-1] == '\n') - buffer[strlen(buffer)-1] = '\0'; - - // Search for exact match - if(strcmp(buffer, str) == 0) - { - found++; - continue; - } - - // If line starts with *, search for partial match of - // needle "buffer+1" in haystack "str" - if(buffer[0] == '*') - { - char * buf = strstr(str, buffer+1); - // The strstr() function finds the first occurrence of - // the substring buffer+1 in the string str. - // These functions return a pointer to the beginning of - // the located substring, or NULL if the substring is not - // found. Hence, we compare the length of the substring to - // the wildcard entry to rule out the possiblity that - // there is anything behind the wildcard. This avoids that given - // "*example.com" "example.com.xxxxx" would also match. - if(buf != NULL && strlen(buf) == strlen(buffer+1)) - found++; - } - } - - // Free allocated memory - if(buffer != NULL) - { - free(buffer); - buffer = NULL; - } - - // Close the file - fclose(fp); - - return found; -} - -void check_blocking_status(void) -{ - char* blocking = read_setupVarsconf("BLOCKING_ENABLED"); - const char* message; - - if(blocking == NULL || getSetupVarsBool(blocking)) - { - // Parameter either not present in setupVars.conf - // or explicitly set to true - blockingstatus = BLOCKING_ENABLED; - message = "enabled"; - clearSetupVarsArray(); - } - else - { - // Disabled - blockingstatus = BLOCKING_DISABLED; - message = "disabled"; - } - - logg("Blocking status is %s", message); -} diff --git a/memory.c b/memory.c deleted file mode 100644 index 503e84d1d..000000000 --- a/memory.c +++ /dev/null @@ -1,217 +0,0 @@ -/* Pi-hole: A black hole for Internet advertisements -* (c) 2017 Pi-hole, LLC (https://pi-hole.net) -* Network-wide ad blocking via your own hardware. -* -* FTL Engine -* Global variable definitions and memory reallocation handling -* -* This file is copyright under the latest version of the EUPL. -* Please see LICENSE file for your rights under this license. */ - -#include "FTL.h" -#include "shmem.h" - -FTLFileNamesStruct FTLfiles = { - // Default path for config file (regular installations) - "/etc/pihole/pihole-FTL.conf", - // Alternative path for config file (snap installations) - "/var/snap/pihole/common/etc/pihole/pihole-FTL.conf", - NULL, - NULL, - NULL, - NULL, - NULL, - NULL -}; - -logFileNamesStruct files = { - NULL, - NULL, - NULL, - NULL, - NULL, - NULL -}; - -// Fixed size structs -countersStruct *counters = NULL; -ConfigStruct config; - -// Variable size array structs -queriesDataStruct *queries = NULL; -forwardedDataStruct *forwarded = NULL; -clientsDataStruct *clients = NULL; -domainsDataStruct *domains = NULL; -overTimeDataStruct *overTime = NULL; - -void memory_check(int which) -{ - switch(which) - { - case QUERIES: - if(counters->queries >= counters->queries_MAX-1) - { - // Have to reallocate shared memory - queries = enlarge_shmem_struct(QUERIES); - if(queries == NULL) - { - logg("FATAL: Memory allocation failed! Exiting"); - exit(EXIT_FAILURE); - } - } - break; - case FORWARDED: - if(counters->forwarded >= counters->forwarded_MAX-1) - { - // Have to reallocate shared memory - forwarded = enlarge_shmem_struct(FORWARDED); - if(forwarded == NULL) - { - logg("FATAL: Memory allocation failed! Exiting"); - exit(EXIT_FAILURE); - } - } - break; - case CLIENTS: - if(counters->clients >= counters->clients_MAX-1) - { - // Have to reallocate shared memory - clients = enlarge_shmem_struct(CLIENTS); - if(clients == NULL) - { - logg("FATAL: Memory allocation failed! Exiting"); - exit(EXIT_FAILURE); - } - } - break; - case DOMAINS: - if(counters->domains >= counters->domains_MAX-1) - { - // Have to reallocate shared memory - domains = enlarge_shmem_struct(DOMAINS); - if(domains == NULL) - { - logg("FATAL: Memory allocation failed! Exiting"); - exit(EXIT_FAILURE); - } - } - break; - default: - /* That cannot happen */ - logg("Fatal error in memory_check(%i)", which); - exit(EXIT_FAILURE); - break; - } -} - -void validate_access(const char * name, int pos, bool testmagic, int line, const char * function, const char * file) -{ - int limit = 0; - if(name[0] == 'c') limit = counters->clients_MAX; - else if(name[0] == 'd') limit = counters->domains_MAX; - else if(name[0] == 'q') limit = counters->queries_MAX; - else if(name[0] == 'f') limit = counters->forwarded_MAX; - else { logg("Validator error (range)"); killed = 1; } - - if(pos >= limit || pos < 0) - { - logg("FATAL ERROR: Trying to access %s[%i], but maximum is %i", name, pos, limit); - logg(" found in %s() (%s:%i)", function, file, line); - } - // Don't test magic byte if detected potential out-of-bounds error - else if(testmagic) - { - unsigned char magic = 0x00; - if(name[0] == 'c') magic = clients[pos].magic; - else if(name[0] == 'd') magic = domains[pos].magic; - else if(name[0] == 'q') magic = queries[pos].magic; - else if(name[0] == 'f') magic = forwarded[pos].magic; - else { logg("Validator error (magic byte)"); killed = 1; } - if(magic != MAGICBYTE) - { - logg("FATAL ERROR: Trying to access %s[%i], but magic byte is %x", name, pos, magic); - logg(" found in %s() (%s:%i)", function, file, line); - } - } -} - -// The special memory handling routines have to be the last ones in this source file -// as we restore the original definition of the strdup, free, calloc, and realloc -// functions in here, i.e. if anything extra would come below these lines, it would -// not be protected by our (error logging) functions! - -#undef strdup -char* __attribute__((malloc)) FTLstrdup(const char *src, const char * file, const char * function, int line) -{ - // The FTLstrdup() function returns a pointer to a new string which is a - // duplicate of the string s. Memory for the new string is obtained with - // calloc(3), and can be freed with free(3). - if(src == NULL) - { - logg("WARN: Trying to copy a NULL string in %s() (%s:%i)", function, file, line); - return NULL; - } - size_t len = strlen(src); - char *dest = calloc(len+1, sizeof(char)); - if(dest == NULL) - { - logg("FATAL: Memory allocation failed in %s() (%s:%i)", function, file, line); - return NULL; - } - // Use memcpy as memory areas cannot overlap - memcpy(dest, src, len); - dest[len] = '\0'; - - return dest; -} - -#undef calloc -void* __attribute__((malloc)) __attribute__((alloc_size(1,2))) FTLcalloc(size_t nmemb, size_t size, const char * file, const char * function, int line) -{ - // The FTLcalloc() function allocates memory for an array of nmemb elements - // of size bytes each and returns a pointer to the allocated memory. The - // memory is set to zero. If nmemb or size is 0, then calloc() returns - // either NULL, or a unique pointer value that can later be successfully - // passed to free(). - void *ptr = calloc(nmemb, size); - if(ptr == NULL) - logg("FATAL: Memory allocation (%zu x %zu) failed in %s() (%s:%i)", - nmemb, size, function, file, line); - - return ptr; -} - -#undef realloc -void __attribute__((alloc_size(2))) *FTLrealloc(void *ptr_in, size_t size, const char * file, const char * function, int line) -{ - // The FTLrealloc() function changes the size of the memory block pointed to - // by ptr to size bytes. The contents will be unchanged in the range from - // the start of the region up to the minimum of the old and new sizes. If - // the new size is larger than the old size, the added memory will not be - // initialized. If ptr is NULL, then the call is equivalent to malloc(size), - // for all values of size; if size is equal to zero, and ptr is - // not NULL, then the call is equivalent to free(ptr). Unless ptr is - // NULL, it must have been returned by an earlier call to malloc(), cal‐ - // loc() or realloc(). If the area pointed to was moved, a free(ptr) is - // done. - void *ptr_out = realloc(ptr_in, size); - if(ptr_out == NULL) - logg("FATAL: Memory reallocation (%p -> %zu) failed in %s() (%s:%i)", - ptr_in, size, function, file, line); - - return ptr_out; -} - -#undef free -void FTLfree(void *ptr, const char * file, const char * function, int line) -{ - // The free() function frees the memory space pointed to by ptr, which - // must have been returned by a previous call to malloc(), calloc(), or - // realloc(). Otherwise, or if free(ptr) has already been called before, - // undefined behavior occurs. If ptr is NULL, no operation is performed. - if(ptr == NULL) - logg("FATAL: Trying to free NULL pointer in %s() (%s:%i)", function, file, line); - - // We intentionally run free() nevertheless to see the crash in the debugger - free(ptr); -} diff --git a/networktable.c b/networktable.c deleted file mode 100644 index d5817f9ca..000000000 --- a/networktable.c +++ /dev/null @@ -1,337 +0,0 @@ -/* Pi-hole: A black hole for Internet advertisements -* (c) 2017 Pi-hole, LLC (https://pi-hole.net) -* Network-wide ad blocking via your own hardware. -* -* FTL Engine -* Network table routines -* -* This file is copyright under the latest version of the EUPL. -* Please see LICENSE file for your rights under this license. */ - -#include "FTL.h" -#include "shmem.h" -#include "sqlite3.h" -#define ARPCACHE "/proc/net/arp" - -// Private prototypes -static char* getMACVendor(const char* hwaddr); - -bool create_network_table(void) -{ - bool ret; - // Create network table in the database - ret = dbquery("CREATE TABLE network ( id INTEGER PRIMARY KEY NOT NULL, " \ - "ip TEXT NOT NULL, " \ - "hwaddr TEXT NOT NULL, " \ - "interface TEXT NOT NULL, " \ - "name TEXT, " \ - "firstSeen INTEGER NOT NULL, " \ - "lastQuery INTEGER NOT NULL, " \ - "numQueries INTEGER NOT NULL," \ - "macVendor TEXT);"); - if(!ret){ dbclose(); return false; } - - // Update database version to 3 - ret = db_set_FTL_property(DB_VERSION, 3); - if(!ret){ dbclose(); return false; } - - return true; -} - -// Read kernel's ARP cache using procfs -void parse_arp_cache(void) -{ - FILE* arpfp = NULL; - // Try to access the kernel's ARP cache - if((arpfp = fopen(ARPCACHE, "r")) == NULL) - { - logg("WARN: Opening of %s failed!", ARPCACHE); - logg(" Message: %s", strerror(errno)); - return; - } - - // Open database file - if(!dbopen()) - { - logg("read_arp_cache() - Failed to open DB"); - fclose(arpfp); - return; - } - - // Start ARP timer - if(config.debug & DEBUG_ARP) timer_start(ARP_TIMER); - - // Prepare buffers - char * linebuffer = NULL; - size_t linebuffersize = 0; - char ip[100], mask[100], hwaddr[100], iface[100]; - int type, flags, entries = 0; - time_t now = time(NULL); - - // Start collecting database commands - dbquery("BEGIN TRANSACTION"); - - // Read ARP cache line by line - while(getline(&linebuffer, &linebuffersize, arpfp) != -1) - { - int num = sscanf(linebuffer, "%99s 0x%x 0x%x %99s %99s %99s\n", - ip, &type, &flags, hwaddr, mask, iface); - - // Skip header and empty lines - if (num < 4) - continue; - - // Skip incomplete entires, i.e., entries without C (complete) flag - if(!(flags & 0x02)) - continue; - - // Get ID of this device in our network database. If it cannot be found, then this is a new device - // We match both IP *and* MAC address - // Same MAC, two IPs: Non-deterministic DHCP server, treat as two entries - // Same IP, two MACs: Either non-deterministic DHCP server or (almost) full DHCP address pool - // We can run this SELECT inside the currently active transaction as only the - // changed to the database are collected for latter commitment. Read-only access - // such as this SELECT command will be executed immediately on the database. - char* querystr = NULL; - int ret = asprintf(&querystr, "SELECT id FROM network WHERE ip = \'%s\' AND hwaddr = \'%s\';", ip, hwaddr); - if(querystr == NULL || ret < 0) - { - logg("Memory allocation failed in parse_arp_cache (%i)", ret); - break; - } - - // Perform SQL query - int dbID = db_query_int(querystr); - free(querystr); - - if(dbID == DB_FAILED) - { - // SQLite error - break; - } - - // If we reach this point, we can check if this client - // is known to pihole-FTL - // false = do not create a new record if the client is - // unknown (only DNS requesting clients do this) - lock_shm(); - int clientID = findClientID(ip, false); - - // This client is known (by its IP address) to pihole-FTL if - // findClientID() returned a non-negative index - bool clientKnown = clientID >= 0; - - // Get hostname of this client if the client is known - const char *hostname = ""; - if(clientKnown) - { - validate_access("clients", clientID, true, __LINE__, __FUNCTION__, __FILE__); - hostname = getstr(clients[clientID].namepos); - } - - // Device not in database, add new entry - if(dbID == DB_NODATA) - { - char* macVendor = getMACVendor(hwaddr); - dbquery("INSERT INTO network "\ - "(ip,hwaddr,interface,firstSeen,lastQuery,numQueries,name,macVendor) "\ - "VALUES (\'%s\',\'%s\',\'%s\',%lu, %ld, %u, \'%s\', \'%s\');",\ - ip, hwaddr, iface, now, - clientKnown ? clients[clientID].lastQuery : 0L, - clientKnown ? clients[clientID].numQueriesARP : 0u, - hostname, - macVendor); - free(macVendor); - } - // Device in database AND client known to Pi-hole - else if(clientKnown) - { - // Update lastQuery. Only use new value if larger - // clients[clientID].lastQuery may be zero if this - // client is only known from a database entry but has - // not been seen since then - dbquery("UPDATE network "\ - "SET lastQuery = MAX(lastQuery, %ld) "\ - "WHERE id = %i;",\ - clients[clientID].lastQuery, dbID); - - // Update numQueries. Add queries seen since last update - // and reset counter afterwards - dbquery("UPDATE network "\ - "SET numQueries = numQueries + %u "\ - "WHERE id = %i;",\ - clients[clientID].numQueriesARP, dbID); - clients[clientID].numQueriesARP = 0; - - // Store hostname if available - if(strlen(hostname) > 0) - { - // Store host name - dbquery("UPDATE network "\ - "SET name = \'%s\' "\ - "WHERE id = %i;",\ - hostname, dbID); - } - } - // else: - // Device in database but not known to Pi-hole: No action required - - unlock_shm(); - - // Count number of processed ARP cache entries - entries++; - } - - // Actually update the database - dbquery("COMMIT"); - - // Debug logging - if(config.debug & DEBUG_ARP) logg("ARP table processing (%i entries) took %.1f ms", entries, timer_elapsed_msec(ARP_TIMER)); - - // Close file handle - fclose(arpfp); - - // Close database connection - dbclose(); -} - -static char* getMACVendor(const char* hwaddr) -{ - struct stat st; - if(stat(FTLfiles.macvendordb, &st) != 0) - { - // File does not exist - if(config.debug & DEBUG_ARP) logg("getMACVenor(%s): %s does not exist", hwaddr, FTLfiles.macvendordb); - return strdup(""); - } - else if(strlen(hwaddr) != 17) - { - // MAC address is incomplete - if(config.debug & DEBUG_ARP) logg("getMACVenor(%s): MAC invalid (length %zu)", hwaddr, strlen(hwaddr)); - return strdup(""); - } - - sqlite3 *macdb; - int rc = sqlite3_open_v2(FTLfiles.macvendordb, &macdb, SQLITE_OPEN_READONLY, NULL); - if( rc ){ - logg("getMACVendor(%s) - SQL error (%i): %s", hwaddr, rc, sqlite3_errmsg(macdb)); - sqlite3_close(macdb); - return strdup(""); - } - - char *querystr = NULL; - // Only keep "XX:YY:ZZ" (8 characters) - char * hwaddrshort = strdup(hwaddr); - hwaddrshort[8] = '\0'; - rc = asprintf(&querystr, "SELECT vendor FROM macvendor WHERE mac LIKE \'%s\';", hwaddrshort); - if(rc < 1) - { - logg("getMACVendor(%s) - Allocation error (%i)", hwaddr, rc); - sqlite3_close(macdb); - return strdup(""); - } - free(hwaddrshort); - - sqlite3_stmt* stmt; - rc = sqlite3_prepare_v2(macdb, querystr, -1, &stmt, NULL); - if( rc ){ - logg("getMACVendor(%s) - SQL error prepare (%s, %i): %s", hwaddr, querystr, rc, sqlite3_errmsg(macdb)); - sqlite3_close(macdb); - return strdup(""); - } - free(querystr); - - char *vendor = NULL; - rc = sqlite3_step(stmt); - if(rc == SQLITE_ROW) - { - vendor = strdup((char*)sqlite3_column_text(stmt, 0)); - } - else - { - // Not found - vendor = strdup(""); - } - - if(rc != SQLITE_DONE && rc != SQLITE_ROW) - { - // Error - logg("getMACVendor(%s) - SQL error step (%i): %s", hwaddr, rc, sqlite3_errmsg(macdb)); - } - - sqlite3_finalize(stmt); - sqlite3_close(macdb); - - return vendor; -} - -void updateMACVendorRecords() -{ - struct stat st; - if(stat(FTLfiles.macvendordb, &st) != 0) - { - // File does not exist - if(config.debug & DEBUG_ARP) logg("updateMACVendorRecords(): %s does not exist", FTLfiles.macvendordb); - return; - } - - sqlite3 *db; - int rc = sqlite3_open_v2(FTLfiles.db, &db, SQLITE_OPEN_READWRITE, NULL); - if( rc ){ - logg("updateMACVendorRecords() - SQL error (%i): %s", rc, sqlite3_errmsg(db)); - sqlite3_close(db); - return; - } - - sqlite3_stmt* stmt; - const char* selectstr = "SELECT id,hwaddr FROM network;"; - rc = sqlite3_prepare_v2(db, selectstr, -1, &stmt, NULL); - if( rc ){ - logg("updateMACVendorRecords() - SQL error prepare (%s, %i): %s", selectstr, rc, sqlite3_errmsg(db)); - sqlite3_close(db); - return; - } - - while((rc = sqlite3_step(stmt)) == SQLITE_ROW) - { - const int id = sqlite3_column_int(stmt, 0); - char* hwaddr = strdup((char*)sqlite3_column_text(stmt, 1)); - - // Get vendor for MAC - char* vendor = getMACVendor(hwaddr); - free(hwaddr); - hwaddr = NULL; - - // Prepare UPDATE statement - char *updatestr = NULL; - if(asprintf(&updatestr, "UPDATE network SET macVendor = \'%s\' WHERE id = %i", vendor, id) < 1) - { - logg("updateMACVendorRecords() - Allocation error 2"); - free(vendor); - break; - } - - // Execute prepared statement - char *zErrMsg = NULL; - rc = sqlite3_exec(db, updatestr, NULL, NULL, &zErrMsg); - if( rc != SQLITE_OK ){ - logg("updateMACVendorRecords() - SQL exec error: %s (%i): %s", updatestr, rc, zErrMsg); - sqlite3_free(zErrMsg); - free(updatestr); - free(vendor); - break; - } - - // Free allocated memory - free(updatestr); - free(vendor); - } - if(rc != SQLITE_DONE) - { - // Error - logg("updateMACVendorRecords() - SQL error step (%i): %s", rc, sqlite3_errmsg(db)); - } - - sqlite3_finalize(stmt); - sqlite3_close(db); -} diff --git a/regex.c b/regex.c deleted file mode 100644 index 68678bf61..000000000 --- a/regex.c +++ /dev/null @@ -1,305 +0,0 @@ -/* Pi-hole: A black hole for Internet advertisements -* (c) 2017 Pi-hole, LLC (https://pi-hole.net) -* Network-wide ad blocking via your own hardware. -* -* FTL Engine -* Regular Expressions -* -* This file is copyright under the latest version of the EUPL. -* Please see LICENSE file for your rights under this license. */ - -#include "FTL.h" -#include - -static int num_regex; -static regex_t *regex = NULL; -static bool *regexconfigured = NULL; -static char **regexbuffer = NULL; -static whitelistStruct whitelist = { NULL, 0 }; - -static void log_regex_error(const char *where, int errcode, int index) -{ - // Regex failed for some reason (probably user syntax error) - // Get error string and log it - size_t length = regerror(errcode, ®ex[index], NULL, 0); - char *buffer = calloc(length,sizeof(char)); - (void) regerror (errcode, ®ex[index], buffer, length); - logg("ERROR %s regex on line %i: %s (%i)", where, index+1, buffer, errcode); - free(buffer); -} - -static bool init_regex(const char *regexin, int index) -{ - // compile regular expressions into data structures that - // can be used with regexec to match against a string - int errcode = regcomp(®ex[index], regexin, REG_EXTENDED); - if(errcode != 0) - { - log_regex_error("compiling", errcode, index); - return false; - } - - // Store compiled regex string in buffer if in regex debug mode - if(config.debug & DEBUG_REGEX) - { - regexbuffer[index] = strdup(regexin); - } - return true; -} - -bool __attribute__((pure)) in_whitelist(char *domain) -{ - bool found = false; - for(int i=0; i < whitelist.count; i++) - { - // strcasecmp() compares two strings ignoring case - if(strcasecmp(whitelist.domains[i], domain) == 0) - { - found = true; - break; - } - } - return found; -} - -static void free_whitelist_domains(void) -{ - for(int i=0; i < whitelist.count; i++) - free(whitelist.domains[i]); - - whitelist.count = 0; - - // Free whitelist domains array only allocated - if(whitelist.domains != NULL) - { - free(whitelist.domains); - whitelist.domains = NULL; - } -} - -bool match_regex(char *input) -{ - int index; - bool matched = false; - - // Start matching timer - timer_start(REGEX_TIMER); - for(index = 0; index < num_regex; index++) - { - // Only check regex which have been successfully compiled - if(!regexconfigured[index]) - continue; - - // Try to match the compiled regular expression against input - int errcode = regexec(®ex[index], input, 0, NULL, 0); - if (errcode == 0) - { - // Match, return true - matched = true; - - // Print match message when in regex debug mode - if(config.debug & DEBUG_REGEX) - logg("Regex in line %i \"%s\" matches \"%s\"", index+1, regexbuffer[index], input); - break; - } - else if (errcode != REG_NOMATCH) - { - // Error, return false afterwards - log_regex_error("matching", errcode, index); - break; - } - } - - double elapsed = timer_elapsed_msec(REGEX_TIMER); - - // Only log evaluation times if they are longer than normal - if(elapsed > 10.0) - logg("WARN: Regex evaluation took %.3f msec", elapsed); - - // No match, no error, return false - return matched; -} - -void free_regex(void) -{ - // Return early if we don't use any regex - if(regex == NULL) - return; - - // Disable blocking regex checking and free regex datastructure - for(int index = 0; index < num_regex; index++) - { - if(regexconfigured[index]) - { - regfree(®ex[index]); - - // Also free buffered regex strings if in regex debug mode - if(config.debug & DEBUG_REGEX) - { - free(regexbuffer[index]); - regexbuffer[index] = NULL; - } - } - } - - // Free array with regex datastructure - free(regex); - regex = NULL; - free(regexconfigured); - regexconfigured = NULL; - - // Reset counter for number of regex - num_regex = 0; - - // Must reevaluate regex filters after having reread the regex filter - // We reset all regex status to unknown to have them being reevaluated - if(counters->domains > 0) - validate_access("domains", counters->domains-1, false, __LINE__, __FUNCTION__, __FILE__); - for(int i=0; i < counters->domains; i++) - { - domains[i].regexmatch = REGEX_UNKNOWN; - } - - // Also free array of whitelisted domains - free_whitelist_domains(); -} - -static void read_whitelist_from_file(void) -{ - FILE *fp; - char *buffer = NULL; - size_t size = 0; - - // Get number of lines in the whitelist file - whitelist.count = countlines(files.whitelist); - - if(whitelist.count < 0) - { - logg("INFO: No whitelist file found"); - return; - } - - if((fp = fopen(files.whitelist, "r")) == NULL) { - logg("WARN: Cannot access whitelist (%s)",files.whitelist); - return; - } - - // Allocate memory for array of whitelisted domains - whitelist.domains = calloc(whitelist.count, sizeof(char*)); - - // Search through file - // getline reads a string from the specified file up to either a - // newline character or EOF - for(int i=0; getline(&buffer, &size, fp) != -1; i++) - { - // Test if file has changed since we counted the lines therein (unlikely - // but not impossible). If so, read only as far as we have reserved memory - if(i >= whitelist.count) - break; - - // Strip potential newline character at the end of line we just read - if(buffer[strlen(buffer)-1] == '\n') - buffer[strlen(buffer)-1] = '\0'; - - // Copy this whitelisted domain into memory - whitelist.domains[i] = strdup(buffer); - } - - // Free allocated memory - if(buffer != NULL) - { - free(buffer); - buffer = NULL; - } - - // Close the file - fclose(fp); -} - -void read_regex_from_file(void) -{ - FILE *fp; - char *buffer = NULL; - size_t size = 0; - int errors = 0, skipped = 0; - - // Start timer for regex compilation analysis - timer_start(REGEX_TIMER); - - // Get number of lines in the regex file - num_regex = countlines(files.regexlist); - - if(num_regex < 0) - { - logg("INFO: No Regex file found"); - return; - } - - if((fp = fopen(files.regexlist, "r")) == NULL) { - logg("WARN: Cannot access Regex file"); - return; - } - - // Allocate memory for regex - regex = calloc(num_regex, sizeof(regex_t)); - regexconfigured = calloc(num_regex, sizeof(bool)); - - // Buffer strings if in regex debug mode - if(config.debug & DEBUG_REGEX) - regexbuffer = calloc(num_regex, sizeof(char*)); - - // Search through file - // getline reads a string from the specified file up to either a - // newline character or EOF - for(int i=0; getline(&buffer, &size, fp) != -1; i++) - { - // Test if file has changed since we counted the lines therein (unlikely - // but not impossible). If so, read only as far as we have reserved memory - if(i >= num_regex) - break; - - // Strip potential newline character at the end of line we just read - if(buffer[strlen(buffer)-1] == '\n') - buffer[strlen(buffer)-1] = '\0'; - - // Skip this entry if empty: an empty regex filter would match - // anything anywhere and hence match (and block) all incoming domains. - // A user can still achieve this with a filter such as ".*", however - // empty lines in regex.list are probably not expected to have such an - // effect and would immediately lead to "blocking the entire Internet" - if(strlen(buffer) < 1) - { - regexconfigured[i] = false; - logg("Skipping empty regex filter on line %i", i+1); - skipped++; - continue; - } - - // Skip this entry if it is commented out - if(buffer[0] == '#') - { - regexconfigured[i] = false; - logg("Skipping commented out regex filter on line %i", i+1); - skipped++; - continue; - } - - // Compile this regex - regexconfigured[i] = init_regex(buffer, i); - } - - // Free allocated memory - if(buffer != NULL) - { - free(buffer); - buffer = NULL; - } - - // Close the file - fclose(fp); - - // Read whitelisted domains from file - read_whitelist_from_file(); - - logg("Compiled %i Regex filters and %i whitelisted domains in %.1f msec (%i errors)", (num_regex-skipped), whitelist.count > 0 ? whitelist.count : 0, timer_elapsed_msec(REGEX_TIMER), errors); -} diff --git a/resolve.c b/resolve.c deleted file mode 100644 index 1f717b704..000000000 --- a/resolve.c +++ /dev/null @@ -1,215 +0,0 @@ -/* Pi-hole: A black hole for Internet advertisements -* (c) 2017 Pi-hole, LLC (https://pi-hole.net) -* Network-wide ad blocking via your own hardware. -* -* FTL Engine -* DNS Client Implementation -* -* This file is copyright under the latest version of the EUPL. -* Please see LICENSE file for your rights under this license. */ - -#include "FTL.h" -#include "shmem.h" - -static char *resolveHostname(const char *addr) -{ - // Get host name - struct hostent *he = NULL; - char *hostname = NULL;; - bool IPv6 = false; - - // Check if this is a hidden client - // if so, return "hidden" as hostname - if(strcmp(addr, "0.0.0.0") == 0) - { - hostname = strdup("hidden"); - //if(hostname == NULL) return NULL; - return hostname; - } - - // Test if we want to resolve an IPv6 address - if(strstr(addr,":") != NULL) - { - IPv6 = true; - } - - if(IPv6 && config.resolveIPv6) // Resolve IPv6 address only if requested - { - struct in6_addr ipaddr; - inet_pton(AF_INET6, addr, &ipaddr); - he = gethostbyaddr(&ipaddr, sizeof ipaddr, AF_INET6); - } - else if(!IPv6 && config.resolveIPv4) // Resolve IPv4 address only if requested - { - struct in_addr ipaddr; - inet_pton(AF_INET, addr, &ipaddr); - he = gethostbyaddr(&ipaddr, sizeof ipaddr, AF_INET); - } - - if(he == NULL) - { - // No hostname found - hostname = strdup(""); - //if(hostname == NULL) return NULL; - } - else - { - // Return hostname copied to new memory location - hostname = strdup(he->h_name); - if(hostname == NULL) return NULL; - // Convert hostname to lower case - strtolower(hostname); - } - return hostname; -} - -// Resolve upstream destination host names -static size_t resolveAndAddHostname(size_t ippos, size_t oldnamepos) -{ - // Get IP and host name strings. They are cloned in case shared memory is - // resized before the next lock - lock_shm(); - char* ipaddr = strdup(getstr(ippos)); - char* oldname = strdup(getstr(oldnamepos)); - unlock_shm(); - - // Important: Don't hold a lock while resolving as the main thread - // (dnsmasq) needs to be operable during the call to resolveHostname() - char* newname = resolveHostname(ipaddr); - - // Only store new newname if it is valid and differs from oldname - // We do not need to check for oldname == NULL as names are - // always initialized with an empty string at position 0 - if(newname != NULL && strcmp(oldname, newname) != 0) - { - lock_shm(); - size_t newnamepos = addstr(newname); - // newname has already been checked against NULL - // so we can safely free it - free(newname); - free(ipaddr); - free(oldname); - unlock_shm(); - return newnamepos; - } - else if(config.debug & DEBUG_SHMEM) - { - // Debugging output - logg("Not adding \"%s\" to buffer (unchanged)", oldname); - } - - free(ipaddr); - free(oldname); - - // Not changed, return old namepos - return oldnamepos; -} - -// Resolve client host names -void resolveClients(bool onlynew) -{ - // Lock counter access here, we use a copy in the following loop - lock_shm(); - int clientscount = counters->clients; - unlock_shm(); - for(int clientID = 0; clientID < clientscount; clientID++) - { - // Memory validation - validate_access("clients", clientID, true, __LINE__, __FUNCTION__, __FILE__); - - // Memory access needs to get locked - lock_shm(); - bool newflag = clients[clientID].new; - size_t ippos = clients[clientID].ippos; - size_t oldnamepos = clients[clientID].namepos; - unlock_shm(); - - // If onlynew flag is set, we will only resolve new clients - // If not, we will try to re-resolve all known clients - if(onlynew && !newflag) - continue; - - // Obtain/update hostname of this client - size_t newnamepos = resolveAndAddHostname(ippos, oldnamepos); - - lock_shm(); - // Store obtained host name (may be unchanged) - clients[clientID].namepos = newnamepos; - - // Mark entry as not new - clients[clientID].new = false; - unlock_shm(); - } -} - -// Resolve upstream destination host names -void resolveForwardDestinations(bool onlynew) -{ - // Lock counter access here, we use a copy in the following loop - lock_shm(); - int forwardedcount = counters->forwarded; - unlock_shm(); - for(int forwardID = 0; forwardID < forwardedcount; forwardID++) - { - // Memory validation - validate_access("forwarded", forwardID, true, __LINE__, __FUNCTION__, __FILE__); - - // Memory access needs to get locked - lock_shm(); - bool newflag = forwarded[forwardID].new; - size_t ippos = forwarded[forwardID].ippos; - size_t oldnamepos = forwarded[forwardID].namepos; - unlock_shm(); - - // If onlynew flag is set, we will only resolve new upstream destinations - // If not, we will try to re-resolve all known upstream destinations - if(onlynew && !newflag) - continue; - - // Obtain/update hostname of this client - size_t newnamepos = resolveAndAddHostname(ippos, oldnamepos); - - lock_shm(); - // Store obtained host name (may be unchanged) - forwarded[forwardID].namepos = newnamepos; - // Mark entry as not new - forwarded[forwardID].new = false; - unlock_shm(); - } -} - -void *DNSclient_thread(void *val) -{ - // Set thread name - prctl(PR_SET_NAME, "DNS client", 0, 0, 0); - - while(!killed) - { - // Run every minute to resolve only new clients and upstream servers - if(time(NULL) % RESOLVE_INTERVAL == 0) - { - // Try to resolve new client host names (onlynew=true) - resolveClients(true); - // Try to resolve new upstream destination host names (onlynew=true) - resolveForwardDestinations(true); - // Prevent immediate re-run of this routine - sleepms(500); - } - - // Run every hour to update possibly changed client host names - if(time(NULL) % RERESOLVE_INTERVAL == 0) - { - // Try to resolve all client host names (onlynew=false) - resolveClients(false); - // Try to resolve all upstream destination host names (onlynew=false) - resolveForwardDestinations(false); - // Prevent immediate re-run of this routine - sleepms(500); - } - - // Idle for 0.5 sec before checking again the time criteria - sleepms(500); - } - - return NULL; -} diff --git a/routines.h b/routines.h deleted file mode 100644 index fe990af79..000000000 --- a/routines.h +++ /dev/null @@ -1,155 +0,0 @@ -/* Pi-hole: A black hole for Internet advertisements -* (c) 2017 Pi-hole, LLC (https://pi-hole.net) -* Network-wide ad blocking via your own hardware. -* -* FTL Engine -* Global prototypes -* -* This file is copyright under the latest version of the EUPL. -* Please see LICENSE file for your rights under this license. */ - -void go_daemon(void); -void timer_start(int i); -double timer_elapsed_msec(int i); -void sleepms(int milliseconds); -void savepid(void); -char * getUserName(void); -void removepid(void); - -void open_FTL_log(bool test); -void logg(const char* format, ...) __attribute__ ((format (gnu_printf, 1, 2))); -void logg_struct_resize(const char* str, int to, int step); -void log_counter_info(void); -void format_memory_size(char *prefix, unsigned long int bytes, double *formated); -void log_FTL_version(bool crashreport); - -// datastructure.c -void strtolower(char *str); -int findForwardID(const char * forward, bool count); -int findDomainID(const char *domain); -int findClientID(const char *client, bool addNew); -bool isValidIPv4(const char *addr); -bool isValidIPv6(const char *addr); -const char *getDomainString(int queryID); -const char *getClientIPString(int queryID); -const char *getClientNameString(int queryID); - -void close_telnet_socket(void); -void close_unix_socket(void); -void seom(int sock); -void ssend(int sock, const char *format, ...) __attribute__ ((format (gnu_printf, 2, 3))); -void swrite(int sock, const void* value, size_t size); -void *telnet_listening_thread_IPv4(void *args); -void *telnet_listening_thread_IPv6(void *args); - -void *socket_listening_thread(void *args); -bool ipv6_available(void); -void bind_sockets(void); - -void process_request(const char *client_message, int *sock); -bool command(const char *client_message, const char* cmd) __attribute__((pure)); -bool matchesEndpoint(char *client_message, const char *cmd); - -// grep.c -int countlines(const char* fname); -int countlineswith(const char* str, const char* fname); -void check_blocking_status(void); - -void check_setupVarsconf(void); -char * read_setupVarsconf(const char * key); -void getSetupVarsArray(const char * input); -void clearSetupVarsArray(void); -bool insetupVarsArray(const char * str); -bool getSetupVarsBool(const char * input) __attribute__((pure)); - -void parse_args(int argc, char* argv[]); - -// setupVars.c -char* find_equals(const char* s) __attribute__((pure)); -void trim_whitespace(char *string); - -// config.c -void getLogFilePath(void); -void read_FTLconf(void); -void get_privacy_level(FILE *fp); -void get_blocking_mode(FILE *fp); -void read_debuging_settings(FILE *fp); - -// gc.c -void *GC_thread(void *val); - -// database.c -void db_init(void); -void *DB_thread(void *val); -int get_number_of_queries_in_DB(void); -void save_to_DB(void); -void read_data_from_DB(void); -bool db_set_FTL_property(unsigned int ID, int value); -bool dbquery(const char *format, ...); -bool dbopen(void); -void dbclose(void); -int db_query_int(const char*); -void SQLite3LogCallback(void *pArg, int iErrCode, const char *zMsg); - -// memory.c -void memory_check(int which); -char *FTLstrdup(const char *src, const char *file, const char *function, int line) __attribute__((malloc)); -void *FTLcalloc(size_t nmemb, size_t size, const char *file, const char *function, int line) __attribute__((malloc)) __attribute__((alloc_size(1,2))); -void *FTLrealloc(void *ptr_in, size_t size, const char *file, const char *function, int line) __attribute__((alloc_size(2))); -void FTLfree(void *ptr, const char* file, const char *function, int line); -void validate_access(const char * name, int pos, bool testmagic, int line, const char * function, const char * file); - -int main_dnsmasq(int argc, const char ** argv); - -// signals.c -void handle_signals(void); - -// resolve.c -void *DNSclient_thread(void *val); -void resolveClients(bool onlynew); -void resolveForwardDestinations(bool onlynew); - -// regex.c -bool match_regex(char *input); -void free_regex(void); -void read_regex_from_file(void); -bool in_whitelist(char *domain) __attribute__((pure)); - -// shmem.c -bool init_shmem(void); -void destroy_shmem(void); -size_t addstr(const char *str); -const char *getstr(size_t pos); -void *enlarge_shmem_struct(char type); - -/** - * Create a new overTime client shared memory block. - * This also updates `overTimeClientData`. - */ -void newOverTimeClient(int clientID); - -/** - * Add a new overTime slot to each overTime client shared memory block. - * This also updates `overTimeClientData`. - */ -void addOverTimeClientSlot(void); - -// overTime.c -void initOverTime(void); -unsigned int getOverTimeID(time_t timestamp); - -/** - * Move the overTime slots so the oldest interval starts with mintime. The time - * given will be aligned to OVERTIME_INTERVAL. - * - * @param mintime The start of the oldest interval - */ -void moveOverTimeMemory(time_t mintime); - -// capabilities.c -bool check_capabilities(void); - -// networktable.c -bool create_network_table(void); -void parse_arp_cache(void); -void updateMACVendorRecords(void); diff --git a/shmem.h b/shmem.h deleted file mode 100644 index 83290262e..000000000 --- a/shmem.h +++ /dev/null @@ -1,53 +0,0 @@ -/* Pi-hole: A black hole for Internet advertisements -* (c) 2018 Pi-hole, LLC (https://pi-hole.net) -* Network-wide ad blocking via your own hardware. -* -* FTL Engine -* Shared memory header -* -* This file is copyright under the latest version of the EUPL. -* Please see LICENSE file for your rights under this license. */ - -#ifndef SHARED_MEMORY_SERVER_H -#define SHARED_MEMORY_SERVER_H -#include /* For shm_* functions */ -#include /* For mode constants */ -#include /* For O_* constants */ -#include - -typedef struct { - const char *name; - size_t size; - void *ptr; -} SharedMemory; - -/// Create shared memory -/// -/// \param name the name of the shared memory -/// \param size the size to allocate -/// \return a structure with a pointer to the mounted shared memory. The pointer -/// will always be valid, because if it failed FTL will have exited. -SharedMemory create_shm(const char *name, size_t size); - -/// Reallocate shared memory -/// -/// \param sharedMemory the shared memory struct -/// \param size the new size -/// \param resize whether the object should be resized or only remapped -/// \return if reallocation was successful -bool realloc_shm(SharedMemory *sharedMemory, size_t size, bool resize); - -/// Disconnect from shared memory. If there are no other connections to shared memory, it will be deleted. -/// -/// \param sharedMemory the shared memory struct -void delete_shm(SharedMemory *sharedMemory); - -/// Block until a lock can be obtained -#define lock_shm() _lock_shm(__FUNCTION__, __LINE__, __FILE__); -void _lock_shm(const char* func, const int line, const char* file); - -/// Unlock the lock. Only call this if there is an active lock. -#define unlock_shm() _unlock_shm(__FUNCTION__, __LINE__, __FILE__); -void _unlock_shm(const char* func, const int line, const char* file); - -#endif //SHARED_MEMORY_SERVER_H diff --git a/src/FTL.h b/src/FTL.h new file mode 100644 index 000000000..38c148590 --- /dev/null +++ b/src/FTL.h @@ -0,0 +1,136 @@ +/* Pi-hole: A black hole for Internet advertisements +* (c) 2017 Pi-hole, LLC (https://pi-hole.net) +* Network-wide ad blocking via your own hardware. +* +* FTL Engine +* Global definitions +* +* This file is copyright under the latest version of the EUPL. +* Please see LICENSE file for your rights under this license. */ +#ifndef FTL_H +#define FTL_H + +#define __USE_XOPEN +#define _GNU_SOURCE +#include +// variable argument lists +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +// struct sockaddr_in +#include +// char* inet_ntoa(struct in_addr in) +#include +// getnameinfo(); +#include +#include +#include +#include +//#include +#include +// syslog +#include +// tolower() +#include +// Unix socket +#include +// Interfaces +#include +#include + +// Define MIN and MAX macros, use them only when x and y are of the same type +#define MAX(x,y) (((x) > (y)) ? (x) : (y)) +// MIN(x,y) is already defined in dnsmasq.h + +#define SOCKETBUFFERLEN 1024 + +// How often do we garbage collect (to ensure we only have data fitting to the MAXLOGAGE defined above)? [seconds] +// Default: 3600 (once per hour) +#define GCinterval 3600 + +// Delay applied to the garbage collecting [seconds] +// Default: -60 (one minute before a full hour) +#define GCdelay (-60) + +// How many client connection do we accept at once? +#define MAXCONNS 255 + +// Over how many queries do we iterate at most when trying to find a match? +#define MAXITER 1000 + +// How many hours do we want to store in FTL's memory? [hours] +#define MAXLOGAGE 24 + +// Interval for overTime data [seconds] +// Default: 600 (10 minute intervals) +#define OVERTIME_INTERVAL 600 + +// How many overTime slots do we need? +// (24+1) hours * number of intervals per hour +// We need to be able to hold 25 hours as we need some reserve +// due to that GC is only running once an hours so the shown data +// can be 24 hours + 59 minutes +#define OVERTIME_SLOTS ((MAXLOGAGE+1)*3600/OVERTIME_INTERVAL) + +// Interval for resolving NEW client and upstream server host names [seconds] +// Default: 60 (once every minute) +#define RESOLVE_INTERVAL 60 + +// Interval for re-resolving ALL known host names [seconds] +// Default: 3600 (once every hour) +#define RERESOLVE_INTERVAL 3600 + +// Privacy mode constants +#define HIDDEN_DOMAIN "hidden" +#define HIDDEN_CLIENT "0.0.0.0" + +// Used to check memory integrity in various structs +#define MAGICBYTE 0x57 + +// Some magic database constants +#define DB_FAILED -2 +#define DB_NODATA -1 + +// Add a timeout for the pihole-FTL.db database connection [milliseconds] +// This prevents immediate failures when the database is busy for a short time. +// Default: 1000 (one second) +#define DATABASE_BUSY_TIMEOUT 1000 + +// FTLDNS enums +enum { QUERIES, UPSTREAMS, CLIENTS, DOMAINS, OVERTIME, WILDCARD, DNS_CACHE }; +enum { DNSSEC_UNSPECIFIED, DNSSEC_SECURE, DNSSEC_INSECURE, DNSSEC_BOGUS, DNSSEC_ABANDONED }; +enum { QUERY_UNKNOWN, QUERY_GRAVITY, QUERY_FORWARDED, QUERY_CACHE, QUERY_REGEX, QUERY_BLACKLIST, \ + QUERY_EXTERNAL_BLOCKED_IP, QUERY_EXTERNAL_BLOCKED_NULL, QUERY_EXTERNAL_BLOCKED_NXRA, \ + QUERY_GRAVITY_CNAME, QUERY_REGEX_CNAME, QUERY_BLACKLIST_CNAME, QUERY_STATUS_MAX }; +enum { TYPE_A = 1, TYPE_AAAA, TYPE_ANY, TYPE_SRV, TYPE_SOA, TYPE_PTR, TYPE_TXT, TYPE_NAPTR, TYPE_MAX }; +enum { REPLY_UNKNOWN, REPLY_NODATA, REPLY_NXDOMAIN, REPLY_CNAME, REPLY_IP, REPLY_DOMAIN, REPLY_RRNAME, REPLY_SERVFAIL, REPLY_REFUSED, REPLY_NOTIMP, REPLY_OTHER }; +enum { PRIVACY_SHOW_ALL = 0, PRIVACY_HIDE_DOMAINS, PRIVACY_HIDE_DOMAINS_CLIENTS, PRIVACY_MAXIMUM, PRIVACY_NOSTATS }; +enum { MODE_IP, MODE_NX, MODE_NULL, MODE_IP_NODATA_AAAA, MODE_NODATA }; +enum { REGEX_BLACKLIST, REGEX_WHITELIST }; + +// Use out own memory handling functions that will detect possible errors +// and report accordingly in the log. This will make debugging FTL crashs +// caused by insufficient memory or by code bugs (not properly dealing +// with NULL pointers) much easier. +#define free(param) FTLfree(param, __FILE__, __FUNCTION__, __LINE__) +#define lib_strdup() strdup() +#undef strdup +#define strdup(param) FTLstrdup(param, __FILE__, __FUNCTION__, __LINE__) +#define calloc(p1,p2) FTLcalloc(p1,p2, __FILE__, __FUNCTION__, __LINE__) +#define realloc(p1,p2) FTLrealloc(p1,p2, __FILE__, __FUNCTION__, __LINE__) + +extern pthread_t telnet_listenthreadv4; +extern pthread_t telnet_listenthreadv6; +extern pthread_t socket_listenthread; +extern pthread_t DBthread; +extern pthread_t GCthread; +extern pthread_t DNSclientthread; + +#endif // FTL_H diff --git a/api.c b/src/api/api.c similarity index 56% rename from api.c rename to src/api/api.c index d74d40c0e..a12931b02 100644 --- a/api.c +++ b/src/api/api.c @@ -9,18 +9,32 @@ * Please see LICENSE file for your rights under this license. */ #include "FTL.h" +#include "memory.h" +#include "shmem.h" +#include "datastructure.h" +#include "setupVars.h" +#include "socket.h" +#include "files.h" +#include "log.h" +#include "request.h" +#include "config.h" +#include "database/common.h" +#include "database/query-table.h" +// in_auditlist() +#include "database/gravity-db.h" +#include "overTime.h" #include "api.h" #include "version.h" -// needed for sqlite3_libversion() -#include "sqlite3.h" +// enum REGEX +#include "regex_r.h" #define min(a,b) ({ __typeof__ (a) _a = (a); __typeof__ (b) _b = (b); _a < _b ? _a : _b; }) /* qsort comparision function (count field), sort ASC */ static int __attribute__((pure)) cmpasc(const void *a, const void *b) { - int *elem1 = (int*)a; - int *elem2 = (int*)b; + const int *elem1 = (int*)a; + const int *elem2 = (int*)b; if (elem1[1] < elem2[1]) return -1; @@ -33,8 +47,8 @@ static int __attribute__((pure)) cmpasc(const void *a, const void *b) // qsort subroutine, sort DESC static int __attribute__((pure)) cmpdesc(const void *a, const void *b) { - int *elem1 = (int*)a; - int *elem2 = (int*)b; + const int *elem1 = (int*)a; + const int *elem2 = (int*)b; if (elem1[1] > elem2[1]) return -1; @@ -44,10 +58,10 @@ static int __attribute__((pure)) cmpdesc(const void *a, const void *b) return 0; } -void getStats(int *sock) +void getStats(const int *sock) { - int blocked = counters->blocked; - int total = counters->queries; + const int blocked = counters->blocked; + const int total = counters->queries; float percentage = 0.0f; // Avoid 1/0 condition @@ -62,11 +76,15 @@ void getStats(int *sock) pack_int32(*sock, counters->gravity); // unique_clients: count only clients that have been active within the most recent 24 hours - int i, activeclients = 0; - for(i=0; i < counters->clients; i++) + int activeclients = 0; + for(int clientID=0; clientID < counters->clients; clientID++) { - validate_access("clients", i, true, __LINE__, __FUNCTION__, __FILE__); - if(clients[i].count > 0) + // Get client pointer + const clientsData* client = getClient(clientID, true); + if(client == NULL) + continue; + + if(client->count > 0) activeclients++; } @@ -74,15 +92,15 @@ void getStats(int *sock) ssend(*sock, "dns_queries_today %i\nads_blocked_today %i\nads_percentage_today %f\n", total, blocked, percentage); ssend(*sock, "unique_domains %i\nqueries_forwarded %i\nqueries_cached %i\n", - counters->domains, counters->forwardedqueries, counters->cached); + counters->domains, counters->forwarded, counters->cached); ssend(*sock, "clients_ever_seen %i\n", counters->clients); ssend(*sock, "unique_clients %i\n", activeclients); // Sum up all query types (A, AAAA, ANY, SRV, SOA, ...) int sumalltypes = 0; - for(i=0; i < TYPE_MAX-1; i++) + for(int queryType=0; queryType < TYPE_MAX-1; queryType++) { - sumalltypes += counters->querytype[i]; + sumalltypes += counters->querytype[queryType]; } ssend(*sock, "dns_queries_all_types %i\n", sumalltypes); @@ -97,7 +115,7 @@ void getStats(int *sock) pack_int32(*sock, blocked); pack_float(*sock, percentage); pack_int32(*sock, counters->domains); - pack_int32(*sock, counters->forwardedqueries); + pack_int32(*sock, counters->forwarded); pack_int32(*sock, counters->cached); pack_int32(*sock, counters->clients); pack_int32(*sock, activeclients); @@ -105,36 +123,36 @@ void getStats(int *sock) // Send status if(istelnet[*sock]) { - ssend(*sock, "status %s\n", counters->gravity > 0 ? "enabled" : "disabled"); + ssend(*sock, "status %s\n", blockingstatus ? "enabled" : "disabled"); } else pack_uint8(*sock, blockingstatus); } -void getOverTime(int *sock) +void getOverTime(const int *sock) { - int i, from = 0, until = OVERTIME_SLOTS; + int from = 0, until = OVERTIME_SLOTS; bool found = false; time_t mintime = overTime[0].timestamp; // Start with the first non-empty overTime slot - for(i=0; i < OVERTIME_SLOTS; i++) + for(int slot = 0; slot < OVERTIME_SLOTS; slot++) { - if((overTime[i].total > 0 || overTime[i].blocked > 0) && - overTime[i].timestamp >= mintime) + if((overTime[slot].total > 0 || overTime[slot].blocked > 0) && + overTime[slot].timestamp >= mintime) { - from = i; + from = slot; found = true; break; } } // End with last non-empty overTime slot - for(i = 0; i < OVERTIME_SLOTS; i++) + for(int slot = 0; slot < OVERTIME_SLOTS; slot++) { - if(overTime[i].timestamp >= time(NULL)) + if(overTime[slot].timestamp >= time(NULL)) { - until = i; + until = slot; break; } } @@ -145,9 +163,12 @@ void getOverTime(int *sock) if(istelnet[*sock]) { - for(i = from; i < until; i++) + for(int slot = from; slot < until; slot++) { - ssend(*sock,"%li %i %i\n",overTime[i].timestamp,overTime[i].total,overTime[i].blocked); + ssend(*sock,"%li %i %i\n", + overTime[slot].timestamp, + overTime[slot].total, + overTime[slot].blocked); } } else @@ -157,26 +178,26 @@ void getOverTime(int *sock) // Send domains over time pack_map16_start(*sock, (uint16_t) (until - from)); - for(i = from; i < until; i++) { - pack_int32(*sock, overTime[i].timestamp); - pack_int32(*sock, overTime[i].total); + for(int slot = from; slot < until; slot++) { + pack_int32(*sock, overTime[slot].timestamp); + pack_int32(*sock, overTime[slot].total); } // Send ads over time pack_map16_start(*sock, (uint16_t) (until - from)); - for(i = from; i < until; i++) { - pack_int32(*sock, overTime[i].timestamp); - pack_int32(*sock, overTime[i].blocked); + for(int slot = from; slot < until; slot++) { + pack_int32(*sock, overTime[slot].timestamp); + pack_int32(*sock, overTime[slot].blocked); } } } -void getTopDomains(const char *client_message, int *sock) +void getTopDomains(const char *client_message, const int *sock) { - int i, temparray[counters->domains][2], count=10, num; - bool blocked, audit = false, asc = false; + int temparray[counters->domains][2], count=10, num; + bool audit = false, asc = false; - blocked = command(client_message, ">top-ads"); + const bool blocked = command(client_message, ">top-ads"); // Exit before processing any data if requested via config setting get_privacy_level(NULL); @@ -205,15 +226,19 @@ void getTopDomains(const char *client_message, int *sock) if(command(client_message, " asc")) asc = true; - for(i=0; i < counters->domains; i++) + for(int domainID=0; domainID < counters->domains; domainID++) { - validate_access("domains", i, true, __LINE__, __FUNCTION__, __FILE__); - temparray[i][0] = i; + // Get domain pointer + const domainsData* domain = getDomain(domainID, true); + if(domain == NULL) + continue; + + temparray[domainID][0] = domainID; if(blocked) - temparray[i][1] = domains[i].blockedcount; + temparray[domainID][1] = domain->blockedcount; else // Count only permitted queries - temparray[i][1] = (domains[i].count - domains[i].blockedcount); + temparray[domainID][1] = (domain->count - domain->blockedcount); } // Sort temporary array @@ -224,7 +249,7 @@ void getTopDomains(const char *client_message, int *sock) // Get filter - char * filter = read_setupVarsconf("API_QUERY_LOG_SHOW"); + const char* filter = read_setupVarsconf("API_QUERY_LOG_SHOW"); bool showpermitted = true, showblocked = true; if(filter != NULL) { @@ -261,65 +286,53 @@ void getTopDomains(const char *client_message, int *sock) } int n = 0; - for(i=0; i < counters->domains; i++) + for(int i=0; i < counters->domains; i++) { - // Get sorted indices - int j = temparray[i][0]; - validate_access("domains", j, true, __LINE__, __FUNCTION__, __FILE__); + // Get sorted index + const int domainID = temparray[i][0]; + // Get domain pointer + const domainsData* domain = getDomain(domainID, true); + if(domain == NULL) + continue; // Skip this domain if there is a filter on it - if(excludedomains != NULL && insetupVarsArray(getstr(domains[j].domainpos))) + if(excludedomains != NULL && insetupVarsArray(getstr(domain->domainpos))) continue; - // Skip this domain if already included in audit - if(audit && countlineswith(getstr(domains[j].domainpos), files.auditlist) > 0) + // Skip this domain if already audited + if(audit && in_auditlist(getstr(domain->domainpos)) > 0) + { + if(config.debug & DEBUG_API) + logg("API: %s has been audited.", getstr(domain->domainpos)); continue; + } // Hidden domain, probably due to privacy level. Skip this in the top lists - if(strcmp(getstr(domains[j].domainpos), HIDDEN_DOMAIN) == 0) + if(strcmp(getstr(domain->domainpos), HIDDEN_DOMAIN) == 0) continue; - if(blocked && showblocked && domains[j].blockedcount > 0) + if(blocked && showblocked && domain->blockedcount > 0) { - if(audit && domains[j].regexmatch == REGEX_BLOCKED) - { - if(istelnet[*sock]) - ssend(*sock, "%i %i %s wildcard\n", n, domains[j].blockedcount, getstr(domains[j].domainpos)); - else { - char *fancyWildcard = calloc(3 + strlen(getstr(domains[j].domainpos)), sizeof(char)); - if(fancyWildcard == NULL) return; - sprintf(fancyWildcard, "*.%s", getstr(domains[j].domainpos)); - - if(!pack_str32(*sock, fancyWildcard)) - return; - - pack_int32(*sock, domains[j].blockedcount); - free(fancyWildcard); - } - } - else - { - if(istelnet[*sock]) - ssend(*sock, "%i %i %s\n", n, domains[j].blockedcount, getstr(domains[j].domainpos)); - else { - if(!pack_str32(*sock, getstr(domains[j].domainpos))) - return; + if(istelnet[*sock]) + ssend(*sock, "%i %i %s\n", n, domain->blockedcount, getstr(domain->domainpos)); + else { + if(!pack_str32(*sock, getstr(domain->domainpos))) + return; - pack_int32(*sock, domains[j].blockedcount); - } + pack_int32(*sock, domain->blockedcount); } n++; } - else if(!blocked && showpermitted && (domains[j].count - domains[j].blockedcount) > 0) + else if(!blocked && showpermitted && (domain->count - domain->blockedcount) > 0) { if(istelnet[*sock]) - ssend(*sock,"%i %i %s\n",n,(domains[j].count - domains[j].blockedcount),getstr(domains[j].domainpos)); + ssend(*sock,"%i %i %s\n",n,(domain->count - domain->blockedcount),getstr(domain->domainpos)); else { - if(!pack_str32(*sock, getstr(domains[j].domainpos))) + if(!pack_str32(*sock, getstr(domain->domainpos))) return; - pack_int32(*sock, domains[j].count - domains[j].blockedcount); + pack_int32(*sock, domain->count - domain->blockedcount); } n++; } @@ -333,9 +346,9 @@ void getTopDomains(const char *client_message, int *sock) clearSetupVarsArray(); } -void getTopClients(const char *client_message, int *sock) +void getTopClients(const char *client_message, const int *sock) { - int i, temparray[counters->clients][2], count=10, num; + int temparray[counters->clients][2], count=10, num; // Exit before processing any data if requested via config setting get_privacy_level(NULL); @@ -368,12 +381,15 @@ void getTopClients(const char *client_message, int *sock) if(command(client_message, " blocked")) blockedonly = true; - for(i=0; i < counters->clients; i++) + for(int clientID = 0; clientID < counters->clients; clientID++) { - validate_access("clients", i, true, __LINE__, __FUNCTION__, __FILE__); - temparray[i][0] = i; + // Get client pointer + const clientsData* client = getClient(clientID, true); + if(client == NULL) + continue; + temparray[clientID][0] = clientID; // Use either blocked or total count based on request string - temparray[i][1] = blockedonly ? clients[i].blockedcount : clients[i].count; + temparray[clientID][1] = blockedonly ? client->blockedcount : client->count; } // Sort in ascending order? @@ -389,7 +405,7 @@ void getTopClients(const char *client_message, int *sock) qsort(temparray, counters->clients, sizeof(int[2]), cmpdesc); // Get clients which the user doesn't want to see - char * excludeclients = read_setupVarsconf("API_EXCLUDE_CLIENTS"); + const char* excludeclients = read_setupVarsconf("API_EXCLUDE_CLIENTS"); if(excludeclients != NULL) { getSetupVarsArray(excludeclients); @@ -402,24 +418,28 @@ void getTopClients(const char *client_message, int *sock) } int n = 0; - for(i=0; i < counters->clients; i++) + for(int i=0; i < counters->clients; i++) { // Get sorted indices and counter values (may be either total or blocked count) - int j = temparray[i][0]; - int ccount = temparray[i][1]; - validate_access("clients", j, true, __LINE__, __FUNCTION__, __FILE__); + const int clientID = temparray[i][0]; + const int ccount = temparray[i][1]; + // Get client pointer + const clientsData* client = getClient(clientID, true); + if(client == NULL) + continue; // Skip this client if there is a filter on it if(excludeclients != NULL && - (insetupVarsArray(getstr(clients[j].ippos)) || insetupVarsArray(getstr(clients[j].namepos)))) + (insetupVarsArray(getstr(client->ippos)) || insetupVarsArray(getstr(client->namepos)))) continue; // Hidden client, probably due to privacy level. Skip this in the top lists - if(strcmp(getstr(clients[j].ippos), HIDDEN_CLIENT) == 0) + if(strcmp(getstr(client->ippos), HIDDEN_CLIENT) == 0) continue; - const char *client_ip = getstr(clients[j].ippos); - const char *client_name = getstr(clients[j].namepos); + // Get client IP and name + const char *client_ip = getstr(client->ippos); + const char *client_name = getstr(client->namepos); // Return this client if either // - "withzero" option is set, and/or @@ -447,34 +467,39 @@ void getTopClients(const char *client_message, int *sock) } -void getForwardDestinations(const char *client_message, int *sock) +void getUpstreamDestinations(const char *client_message, const int *sock) { bool sort = true; - int temparray[counters->forwarded][2], totalqueries = 0; + int temparray[counters->upstreams][2], totalqueries = 0; if(command(client_message, "unsorted")) sort = false; - for(int i = 0; i < counters->forwarded; i++) { - validate_access("forwarded", i, true, __LINE__, __FUNCTION__, __FILE__); + for(int upstreamID = 0; upstreamID < counters->upstreams; upstreamID++) + { // If we want to print a sorted output, we fill the temporary array with // the values we will use for sorting afterwards if(sort) { - temparray[i][0] = i; - temparray[i][1] = forwarded[i].count; + // Get forward pointer + const upstreamsData* forward = getUpstream(upstreamID, true); + if(forward == NULL) + continue; + + temparray[upstreamID][0] = upstreamID; + temparray[upstreamID][1] = forward->count; } } if(sort) { // Sort temporary array in descending order - qsort(temparray, counters->forwarded, sizeof(int[2]), cmpdesc); + qsort(temparray, counters->upstreams, sizeof(int[2]), cmpdesc); } - totalqueries = counters->forwardedqueries + counters->cached + counters->blocked; + totalqueries = counters->forwarded + counters->cached + counters->blocked; // Loop over available forward destinations - for(int i = -2; i < min(counters->forwarded, 8); i++) + for(int i = -2; i < min(counters->upstreams, 8); i++) { float percentage = 0.0f; const char* ip, *name; @@ -503,20 +528,24 @@ void getForwardDestinations(const char *client_message, int *sock) { // Regular forward destionation // Get sorted indices - int j; + int upstreamID; if(sort) - j = temparray[i][0]; + upstreamID = temparray[i][0]; else - j = i; - validate_access("forwarded", j, true, __LINE__, __FUNCTION__, __FILE__); + upstreamID = i; + + // Get forward pointer + const upstreamsData* forward = getUpstream(upstreamID, true); + if(forward == NULL) + continue; // Get IP and host name of forward destination if available - ip = getstr(forwarded[j].ippos); - name = getstr(forwarded[j].namepos); + ip = getstr(forward->ippos); + name = getstr(forward->namepos); // Get percentage if(totalqueries > 0) - percentage = 1e2f * forwarded[j].count / totalqueries; + percentage = 1e2f * forward->count / totalqueries; } // Send data: @@ -538,23 +567,30 @@ void getForwardDestinations(const char *client_message, int *sock) } -void getQueryTypes(int *sock) +void getQueryTypes(const int *sock) { - int i,total = 0; - for(i=0; i < TYPE_MAX-1; i++) + int total = 0; + for(int i=0; i < TYPE_MAX-1; i++) + { total += counters->querytype[i]; + } float percentage[TYPE_MAX-1] = { 0.0 }; // Prevent floating point exceptions by checking if the divisor is != 0 if(total > 0) - for(i=0; i < TYPE_MAX-1; i++) + { + for(int i=0; i < TYPE_MAX-1; i++) + { percentage[i] = 1e2f*counters->querytype[i]/total; + } + } if(istelnet[*sock]) { - ssend(*sock, "A (IPv4): %.2f\nAAAA (IPv6): %.2f\nANY: %.2f\nSRV: %.2f\nSOA: %.2f\nPTR: %.2f\nTXT: %.2f\n", + ssend(*sock, "A (IPv4): %.2f\nAAAA (IPv6): %.2f\nANY: %.2f\nSRV: %.2f\n" + "SOA: %.2f\nPTR: %.2f\nTXT: %.2f\nNAPTR: %.2f\n", percentage[0], percentage[1], percentage[2], percentage[3], - percentage[4], percentage[5], percentage[6]); + percentage[4], percentage[5], percentage[6], percentage[7]); } else { pack_str32(*sock, "A (IPv4)"); @@ -571,12 +607,14 @@ void getQueryTypes(int *sock) pack_float(*sock, percentage[5]); pack_str32(*sock, "TXT"); pack_float(*sock, percentage[6]); + pack_str32(*sock, "NAPTR"); + pack_float(*sock, percentage[7]); } } -const char *querytypes[8] = {"A","AAAA","ANY","SRV","SOA","PTR","TXT","UNKN"}; +const char *querytypes[TYPE_MAX] = {"A","AAAA","ANY","SRV","SOA","PTR","TXT","NAPTR","UNKN"}; -void getAllQueries(const char *client_message, int *sock) +void getAllQueries(const char *client_message, const int *sock) { // Exit before processing any data if requested via config setting get_privacy_level(NULL); @@ -631,16 +669,19 @@ void getAllQueries(const char *client_message, int *sock) else { // Iterate through all known forward destinations - int i; - validate_access("forwards", MAX(0,counters->forwarded-1), true, __LINE__, __FUNCTION__, __FILE__); forwarddestid = -3; - for(i = 0; i < counters->forwarded; i++) + for(int i = 0; i < counters->upstreams; i++) { + // Get forward pointer + const upstreamsData* forward = getUpstream(i, true); + if(forward == NULL) + continue; + // Try to match the requested string against their IP addresses and // (if available) their host names - if(strcmp(getstr(forwarded[i].ippos), forwarddest) == 0 || - (forwarded[i].namepos != 0 && - strcmp(getstr(forwarded[i].namepos), forwarddest) == 0)) + if(strcmp(getstr(forward->ippos), forwarddest) == 0 || + (forward->namepos != 0 && + strcmp(getstr(forward->namepos), forwarddest) == 0)) { forwarddestid = i; break; @@ -664,14 +705,17 @@ void getAllQueries(const char *client_message, int *sock) sscanf(client_message, ">getallqueries-domain %255s", domainname); filterdomainname = true; // Iterate through all known domains - int i; - validate_access("domains", MAX(0,counters->domains-1), true, __LINE__, __FUNCTION__, __FILE__); - for(i = 0; i < counters->domains; i++) + for(int domainID = 0; domainID < counters->domains; domainID++) { + // Get domain pointer + const domainsData* domain = getDomain(domainID, true); + if(domain == NULL) + continue; + // Try to match the requested string - if(strcmp(getstr(domains[i].domainpos), domainname) == 0) + if(strcmp(getstr(domain->domainpos), domainname) == 0) { - domainid = i; + domainid = domainID; break; } } @@ -691,15 +735,19 @@ void getAllQueries(const char *client_message, int *sock) if(clientname == NULL) return; sscanf(client_message, ">getallqueries-client %255s", clientname); filterclientname = true; + // Iterate through all known clients - int i; - validate_access("clients", MAX(0,counters->clients-1), true, __LINE__, __FUNCTION__, __FILE__); - for(i = 0; i < counters->clients; i++) + for(int i = 0; i < counters->clients; i++) { + // Get client pointer + const clientsData* client = getClient(i, true); + if(client == NULL) + continue; + // Try to match the requested string - if(strcmp(getstr(clients[i].ippos), clientname) == 0 || - (clients[i].namepos != 0 && - strcmp(getstr(clients[i].namepos), clientname) == 0)) + if(strcmp(getstr(client->ippos), clientname) == 0 || + (client->namepos != 0 && + strcmp(getstr(client->namepos), clientname) == 0)) { clientid = i; break; @@ -742,95 +790,135 @@ void getAllQueries(const char *client_message, int *sock) } clearSetupVarsArray(); - int i; - for(i=ibeg; i < counters->queries; i++) + for(int queryID = ibeg; queryID < counters->queries; queryID++) { - validate_access("queries", i, true, __LINE__, __FUNCTION__, __FILE__); + const queriesData* query = getQuery(queryID, true); // Check if this query has been create while in maximum privacy mode - if(queries[i].privacylevel >= PRIVACY_MAXIMUM) continue; - - validate_access("domains", queries[i].domainID, true, __LINE__, __FUNCTION__, __FILE__); - validate_access("clients", queries[i].clientID, true, __LINE__, __FUNCTION__, __FILE__); + if(query == NULL || query->privacylevel >= PRIVACY_MAXIMUM) + continue; - const char *qtype = querytypes[queries[i].type - TYPE_A]; + // Verify query type + if(query->type > TYPE_MAX-1) + continue; + // Get query type + const char *qtype = querytypes[query->type - TYPE_A]; // 1 = gravity.list, 4 = wildcard, 5 = black.list - if((queries[i].status == QUERY_GRAVITY || - queries[i].status == QUERY_WILDCARD || - queries[i].status == QUERY_BLACKLIST) && !showblocked) + if((query->status == QUERY_GRAVITY || + query->status == QUERY_REGEX || + query->status == QUERY_BLACKLIST || + query->status == QUERY_GRAVITY_CNAME || + query->status == QUERY_REGEX_CNAME || + query->status == QUERY_BLACKLIST_CNAME) && !showblocked) continue; // 2 = forwarded, 3 = cached - if((queries[i].status == QUERY_FORWARDED || - queries[i].status == QUERY_CACHE) && !showpermitted) + if((query->status == QUERY_FORWARDED || + query->status == QUERY_CACHE) && !showpermitted) continue; // Skip those entries which so not meet the requested timeframe - if((from > queries[i].timestamp && from != 0) || (queries[i].timestamp > until && until != 0)) + if((from > query->timestamp && from != 0) || (query->timestamp > until && until != 0)) continue; // Skip if domain is not identical with what the user wants to see - if(filterdomainname && queries[i].domainID != domainid) + if(filterdomainname && query->domainID != domainid) continue; // Skip if client name and IP are not identical with what the user wants to see - if(filterclientname && queries[i].clientID != clientid) + if(filterclientname && query->clientID != clientid) continue; // Skip if query type is not identical with what the user wants to see - if(querytype != 0 && querytype != queries[i].type) + if(querytype != 0 && querytype != query->type) continue; if(filterforwarddest) { // Does the user want to see queries answered from blocking lists? - if(forwarddestid == -2 && queries[i].status != QUERY_GRAVITY - && queries[i].status != QUERY_WILDCARD - && queries[i].status != QUERY_BLACKLIST) + if(forwarddestid == -2 && query->status != QUERY_GRAVITY + && query->status != QUERY_REGEX + && query->status != QUERY_BLACKLIST + && query->status != QUERY_GRAVITY_CNAME + && query->status != QUERY_REGEX_CNAME + && query->status != QUERY_BLACKLIST_CNAME) continue; // Does the user want to see queries answered from local cache? - else if(forwarddestid == -1 && queries[i].status != QUERY_CACHE) + else if(forwarddestid == -1 && query->status != QUERY_CACHE) continue; // Does the user want to see queries answered by an upstream server? - else if(forwarddestid >= 0 && forwarddestid != queries[i].forwardID) + else if(forwarddestid >= 0 && forwarddestid != query->upstreamID) continue; } // Ask subroutine for domain. It may return "hidden" depending on // the privacy settings at the time the query was made - const char *domain = getDomainString(i); + const char *domain = getDomainString(query); + // Similarly for the client - const char *client; - if(strlen(getstr(clients[queries[i].clientID].namepos)) > 0) - client = getClientNameString(i); + const char *clientIPName = NULL; + // Get client pointer + const clientsData* client = getClient(query->clientID, true); + if(domain == NULL || client == NULL) + continue; + + if(strlen(getstr(client->namepos)) > 0) + clientIPName = getClientNameString(query); else - client = getClientIPString(i); + clientIPName = getClientIPString(query); - unsigned long delay = queries[i].response; + unsigned long delay = query->response; // Check if received (delay should be smaller than 30min) if(delay > 1.8e7) delay = 0; + // Get domain blocked during deep CNAME inspection, if applicable + const char *CNAME_domain = "N/A"; + if(query->CNAME_domainID > -1) + { + CNAME_domain = getCNAMEDomainString(query); + } + + // Get ID of blocking regex, if applicable + int regex_idx = -1; + if (query->status == QUERY_REGEX || query->status == QUERY_REGEX_CNAME) + { + unsigned int cacheID = findCacheID(query->domainID, query->clientID); + DNSCacheData *dns_cache = getDNSCache(cacheID, true); + if(dns_cache != NULL) + regex_idx = dns_cache->black_regex_idx; + } + if(istelnet[*sock]) { - ssend(*sock,"%li %s %s %s %i %i %i %lu",queries[i].timestamp,qtype,domain,client,queries[i].status,queries[i].dnssec,queries[i].reply,delay); + ssend(*sock,"%li %s %s %s %i %i %i %lu %s %i", + query->timestamp, + qtype, + domain, + clientIPName, + query->status, + query->dnssec, + query->reply, + delay, + CNAME_domain, + regex_idx); if(config.debug & DEBUG_API) - ssend(*sock, " %i", i); + ssend(*sock, " %i", queryID); ssend(*sock, "\n"); } else { - pack_int32(*sock, queries[i].timestamp); + pack_int32(*sock, query->timestamp); // Use a fixstr because the length of qtype is always 4 (max is 31 for fixstr) if(!pack_fixstr(*sock, qtype)) return; // Use str32 for domain and client because we have no idea how long they will be (max is 4294967295 for str32) - if(!pack_str32(*sock, domain) || !pack_str32(*sock, client)) + if(!pack_str32(*sock, domain) || !pack_str32(*sock, clientIPName)) return; - pack_uint8(*sock, queries[i].status); - pack_uint8(*sock, queries[i].dnssec); + pack_uint8(*sock, query->status); + pack_uint8(*sock, query->dnssec); } } @@ -845,7 +933,7 @@ void getAllQueries(const char *client_message, int *sock) free(forwarddest); } -void getRecentBlocked(const char *client_message, int *sock) +void getRecentBlocked(const char *client_message, const int *sock) { int num=1; @@ -858,19 +946,26 @@ void getRecentBlocked(const char *client_message, int *sock) // Find most recently blocked query int found = 0; - for(int i = counters->queries - 1; i > 0 ; i--) + for(int queryID = counters->queries - 1; queryID > 0 ; queryID--) { - validate_access("queries", i, true, __LINE__, __FUNCTION__, __FILE__); + const queriesData* query = getQuery(queryID, true); + if(query == NULL) + continue; - if(queries[i].status == QUERY_GRAVITY || - queries[i].status == QUERY_WILDCARD || - queries[i].status == QUERY_BLACKLIST) + if(query->status == QUERY_GRAVITY || + query->status == QUERY_REGEX || + query->status == QUERY_BLACKLIST || + query->status == QUERY_GRAVITY_CNAME || + query->status == QUERY_REGEX_CNAME || + query->status == QUERY_BLACKLIST_CNAME) { found++; // Ask subroutine for domain. It may return "hidden" depending on // the privacy settings at the time the query was made - const char *domain = getDomainString(i); + const char *domain = getDomainString(query); + if(domain == NULL) + continue; if(istelnet[*sock]) ssend(*sock,"%s\n", domain); @@ -883,7 +978,7 @@ void getRecentBlocked(const char *client_message, int *sock) } } -void getClientID(int *sock) +void getClientID(const int *sock) { if(istelnet[*sock]) ssend(*sock,"%i\n", *sock); @@ -891,25 +986,26 @@ void getClientID(int *sock) pack_int32(*sock, *sock); } -void getQueryTypesOverTime(int *sock) +void getQueryTypesOverTime(const int *sock) { - int i, from = -1, until = OVERTIME_SLOTS; - time_t mintime = overTime[0].timestamp; - for(i = 0; i < OVERTIME_SLOTS; i++) + int from = -1, until = OVERTIME_SLOTS; + const time_t mintime = overTime[0].timestamp; + + for(int slot = 0; slot < OVERTIME_SLOTS; slot++) { - if((overTime[i].total > 0 || overTime[i].blocked > 0) && overTime[i].timestamp >= mintime) + if((overTime[slot].total > 0 || overTime[slot].blocked > 0) && overTime[slot].timestamp >= mintime) { - from = i; + from = slot; break; } } // End with last non-empty overTime slot - for(i = 0; i < OVERTIME_SLOTS; i++) + for(int slot = 0; slot < OVERTIME_SLOTS; slot++) { - if(overTime[i].timestamp >= time(NULL)) + if(overTime[slot].timestamp >= time(NULL)) { - until = i; + until = slot; break; } } @@ -918,44 +1014,45 @@ void getQueryTypesOverTime(int *sock) if(from < 0) return; - for(i = from; i < until; i++) + for(int slot = from; slot < until; slot++) { float percentageIPv4 = 0.0, percentageIPv6 = 0.0; - int sum = overTime[i].querytypedata[0] + overTime[i].querytypedata[1]; + int sum = overTime[slot].querytypedata[0] + overTime[slot].querytypedata[1]; if(sum > 0) { - percentageIPv4 = (float) (1e2 * overTime[i].querytypedata[0] / sum); - percentageIPv6 = (float) (1e2 * overTime[i].querytypedata[1] / sum); + percentageIPv4 = (float) (1e2 * overTime[slot].querytypedata[0] / sum); + percentageIPv6 = (float) (1e2 * overTime[slot].querytypedata[1] / sum); } if(istelnet[*sock]) - ssend(*sock, "%li %.2f %.2f\n", overTime[i].timestamp, percentageIPv4, percentageIPv6); + ssend(*sock, "%li %.2f %.2f\n", overTime[slot].timestamp, percentageIPv4, percentageIPv6); else { - pack_int32(*sock, overTime[i].timestamp); + pack_int32(*sock, overTime[slot].timestamp); pack_float(*sock, percentageIPv4); pack_float(*sock, percentageIPv6); } } } -void getVersion(int *sock) +void getVersion(const int *sock) { - const char * commit = GIT_HASH; - const char * tag = GIT_TAG; + const char *commit = GIT_HASH; + const char *tag = GIT_TAG; + const char *version = get_FTL_version(); // Extract first 7 characters of the hash char hash[8]; - strncpy(hash, commit, 7); hash[7] = 0; + memcpy(hash, commit, 7); hash[7] = 0; if(strlen(tag) > 1) { if(istelnet[*sock]) ssend( *sock, "version %s\ntag %s\nbranch %s\nhash %s\ndate %s\n", - GIT_VERSION, tag, GIT_BRANCH, hash, GIT_DATE + version, tag, GIT_BRANCH, hash, GIT_DATE ); else { - if(!pack_str32(*sock, GIT_VERSION) || + if(!pack_str32(*sock, version) || !pack_str32(*sock, (char *) tag) || !pack_str32(*sock, GIT_BRANCH) || !pack_str32(*sock, hash) || @@ -987,16 +1084,10 @@ void getVersion(int *sock) } } -void getDBstats(int *sock) +void getDBstats(const int *sock) { // Get file details - struct stat st; - long int filesize = 0; - if(stat(FTLfiles.db, &st) != 0) - // stat() failed (maybe the file does not exist?) - filesize = -1; - else - filesize = st.st_size; + unsigned long long int filesize = get_FTL_db_filesize(); char *prefix = calloc(2, sizeof(char)); if(prefix == NULL) return; @@ -1004,19 +1095,19 @@ void getDBstats(int *sock) format_memory_size(prefix, filesize, &formated); if(istelnet[*sock]) - ssend(*sock,"queries in database: %i\ndatabase filesize: %.2f %sB\nSQLite version: %s\n", get_number_of_queries_in_DB(), formated, prefix, sqlite3_libversion()); + ssend(*sock,"queries in database: %i\ndatabase filesize: %.2f %sB\nSQLite version: %s\n", get_number_of_queries_in_DB(), formated, prefix, get_sqlite3_version()); else { pack_int32(*sock, get_number_of_queries_in_DB()); pack_int64(*sock, filesize); - if(!pack_str32(*sock, (char *) sqlite3_libversion())) + if(!pack_str32(*sock, (char *) get_sqlite3_version())) return; } } -void getClientsOverTime(int *sock) +void getClientsOverTime(const int *sock) { - int i, sendit = -1, until = OVERTIME_SLOTS; + int sendit = -1, until = OVERTIME_SLOTS; // Exit before processing any data if requested via config setting get_privacy_level(NULL); @@ -1024,12 +1115,12 @@ void getClientsOverTime(int *sock) return; // Find minimum ID to send - for(i = 0; i < OVERTIME_SLOTS; i++) + for(int slot = 0; slot < OVERTIME_SLOTS; slot++) { - if((overTime[i].total > 0 || overTime[i].blocked > 0) && - overTime[i].timestamp >= overTime[0].timestamp) + if((overTime[slot].total > 0 || overTime[slot].blocked > 0) && + overTime[slot].timestamp >= overTime[0].timestamp) { - sendit = i; + sendit = slot; break; } } @@ -1037,11 +1128,11 @@ void getClientsOverTime(int *sock) return; // Find minimum ID to send - for(i = 0; i < OVERTIME_SLOTS; i++) + for(int slot = 0; slot < OVERTIME_SLOTS; slot++) { - if(overTime[i].timestamp >= time(NULL)) + if(overTime[slot].timestamp >= time(NULL)) { - until = i; + until = slot; break; } } @@ -1058,31 +1149,38 @@ void getClientsOverTime(int *sock) { getSetupVarsArray(excludeclients); - for(i=0; i < counters->clients; i++) + for(int clientID=0; clientID < counters->clients; clientID++) { - validate_access("clients", i, true, __LINE__, __FUNCTION__, __FILE__); + // Get client pointer + const clientsData* client = getClient(clientID, true); + if(client == NULL) + continue; // Check if this client should be skipped - if(insetupVarsArray(getstr(clients[i].ippos)) || - insetupVarsArray(getstr(clients[i].namepos))) - skipclient[i] = true; + if(insetupVarsArray(getstr(client->ippos)) || + insetupVarsArray(getstr(client->namepos))) + skipclient[clientID] = true; } } // Main return loop - for(i = sendit; i < until; i++) + for(int slot = sendit; slot < until; slot++) { if(istelnet[*sock]) - ssend(*sock, "%li", overTime[i].timestamp); + ssend(*sock, "%li", overTime[slot].timestamp); else - pack_int32(*sock, overTime[i].timestamp); + pack_int32(*sock, overTime[slot].timestamp); // Loop over forward destinations to generate output to be sent to the client - for(int j = 0; j < counters->clients; j++) + for(int clientID = 0; clientID < counters->clients; clientID++) { - if(skipclient[j]) + if(skipclient[clientID]) continue; - int thisclient = clients[j].overTime[i]; + // Get client pointer + const clientsData* client = getClient(clientID, true); + if(client == NULL) + continue; + const int thisclient = client->overTime[slot]; if(istelnet[*sock]) ssend(*sock, " %i", thisclient); @@ -1100,10 +1198,8 @@ void getClientsOverTime(int *sock) clearSetupVarsArray(); } -void getClientNames(int *sock) +void getClientNames(const int *sock) { - int i; - // Exit before processing any data if requested via config setting get_privacy_level(NULL); if(config.privacylevel >= PRIVACY_HIDE_DOMAINS_CLIENTS) @@ -1121,25 +1217,33 @@ void getClientNames(int *sock) { getSetupVarsArray(excludeclients); - for(i=0; i < counters->clients; i++) + for(int clientID=0; clientID < counters->clients; clientID++) { - validate_access("clients", i, true, __LINE__, __FUNCTION__, __FILE__); + // Get client pointer + const clientsData* client = getClient(clientID, true); + if(client == NULL) + continue; + // Check if this client should be skipped - if(insetupVarsArray(getstr(clients[i].ippos)) || - insetupVarsArray(getstr(clients[i].namepos))) - skipclient[i] = true; + if(insetupVarsArray(getstr(client->ippos)) || + insetupVarsArray(getstr(client->namepos))) + skipclient[clientID] = true; } } // Loop over clients to generate output to be sent to the client - for(i = 0; i < counters->clients; i++) + for(int clientID = 0; clientID < counters->clients; clientID++) { - validate_access("clients", i, true, __LINE__, __FUNCTION__, __FILE__); - if(skipclient[i]) + if(skipclient[clientID]) + continue; + + // Get client pointer + const clientsData* client = getClient(clientID, true); + if(client == NULL) continue; - const char *client_ip = getstr(clients[i].ippos); - const char *client_name = getstr(clients[i].namepos); + const char *client_ip = getstr(client->ippos); + const char *client_name = getstr(client->namepos); if(istelnet[*sock]) ssend(*sock, "%s %s\n", client_name, client_ip); @@ -1153,21 +1257,23 @@ void getClientNames(int *sock) clearSetupVarsArray(); } -void getUnknownQueries(int *sock) +void getUnknownQueries(const int *sock) { // Exit before processing any data if requested via config setting get_privacy_level(NULL); if(config.privacylevel >= PRIVACY_HIDE_DOMAINS) return; - int i; - for(i=0; i < counters->queries; i++) + for(int queryID = 0; queryID < counters->queries; queryID++) { - validate_access("queries", i, true, __LINE__, __FUNCTION__, __FILE__); - if(queries[i].status != QUERY_UNKNOWN && queries[i].complete) continue; + const queriesData* query = getQuery(queryID, true); + + if(query == NULL || + (query->status != QUERY_UNKNOWN && query->complete)) + continue; char type[5]; - if(queries[i].type == TYPE_A) + if(query->type == TYPE_A) { strcpy(type,"IPv4"); } @@ -1176,63 +1282,107 @@ void getUnknownQueries(int *sock) strcpy(type,"IPv6"); } - validate_access("domains", queries[i].domainID, true, __LINE__, __FUNCTION__, __FILE__); - validate_access("clients", queries[i].clientID, true, __LINE__, __FUNCTION__, __FILE__); + // Get domain pointer + const domainsData* domain = getDomain(query->domainID, true); + // Get client pointer + const clientsData* client = getClient(query->clientID, true); + if(domain == NULL || client == NULL) + continue; - const char *client = getstr(clients[queries[i].clientID].ippos); + // Get client IP string + const char *clientIP = getstr(client->ippos); if(istelnet[*sock]) - ssend(*sock, "%li %i %i %s %s %s %i %s\n", queries[i].timestamp, i, queries[i].id, type, getstr(domains[queries[i].domainID].domainpos), client, queries[i].status, queries[i].complete ? "true" : "false"); + ssend(*sock, "%li %i %i %s %s %s %i %s\n", query->timestamp, queryID, query->id, type, getstr(domain->domainpos), clientIP, query->status, query->complete ? "true" : "false"); else { - pack_int32(*sock, queries[i].timestamp); - pack_int32(*sock, queries[i].id); + pack_int32(*sock, query->timestamp); + pack_int32(*sock, query->id); // Use a fixstr because the length of qtype is always 4 (max is 31 for fixstr) if(!pack_fixstr(*sock, type)) return; // Use str32 for domain and client because we have no idea how long they will be (max is 4294967295 for str32) - if(!pack_str32(*sock, getstr(domains[queries[i].domainID].domainpos)) || !pack_str32(*sock, client)) + if(!pack_str32(*sock, getstr(domain->domainpos)) || !pack_str32(*sock, clientIP)) return; - pack_uint8(*sock, queries[i].status); - pack_bool(*sock, queries[i].complete); + pack_uint8(*sock, query->status); + pack_bool(*sock, query->complete); } } } -void getDomainDetails(const char *client_message, int *sock) +void getDomainDetails(const char *client_message, const int *sock) { // Get domain name - char domain[128]; - if(sscanf(client_message, "%*[^ ] %127s", domain) < 1) + bool show_all = false; + char domainString[128]; + if(sscanf(client_message, "%*[^ ] %127s", domainString) < 1) { - ssend(*sock, "Need domain for this request\n"); - return; + ssend(*sock, "No domain specified, listing all known (%d)\n", counters->domains); + show_all = true; } - int i; - for(i = 0; i < counters->domains; i++) + for(int domainID = 0; domainID < counters->domains; domainID++) { - validate_access("domains", i, true, __LINE__, __FUNCTION__, __FILE__); - if(strcmp(getstr(domains[i].domainpos), domain) == 0) + // Get domain pointer + const domainsData* domain = getDomain(domainID, true); + if(domain == NULL) + continue; + + if(show_all || strcmp(getstr(domain->domainpos), domainString) == 0) { - ssend(*sock,"Domain \"%s\", ID: %i\n", domain, i); - ssend(*sock,"Total: %i\n", domains[i].count); - ssend(*sock,"Blocked: %i\n", domains[i].blockedcount); - const char *regexstatus; - if(domains[i].regexmatch == REGEX_BLOCKED) - regexstatus = "blocked"; - if(domains[i].regexmatch == REGEX_NOTBLOCKED) - regexstatus = "not blocked"; - else - regexstatus = "unknown"; - ssend(*sock,"Regex status: %s\n", regexstatus); - return; + ssend(*sock,"Domain \"%s\", ID: %i\n", getstr(domain->domainpos), domainID); + ssend(*sock,"Total: %i\n", domain->count); + ssend(*sock,"Blocked: %i\n", domain->blockedcount); + ssend(*sock,"Client status:\n"); + for(int clientID = 0; clientID < counters->clients; clientID++) + { + clientsData *client = getClient(clientID, true); + if(client == NULL) + { + continue; + } + const char *str = "N/A"; + unsigned int cacheID = findCacheID(domainID, clientID); + DNSCacheData *dns_cache = getDNSCache(cacheID, true); + switch(dns_cache->blocking_status) + { + case UNKNOWN_BLOCKED: + str = "unknown"; + break; + case BLACKLIST_BLOCKED: + str = "blacklisted"; + break; + case GRAVITY_BLOCKED: + str = "gravity"; + break; + case REGEX_BLOCKED: + str = "regex"; + break; + case NOT_BLOCKED: + str = "not blocked"; + break; + default: + str = "this cannot happen"; + break; + } + ssend(*sock, " %s (ID %d): %s\n", getstr(client->ippos), clientID, str); + } + ssend(*sock,"\n"); + + // Return early + if(!show_all) + { + return; + } } } // for loop finished without an exact match - ssend(*sock,"Domain \"%s\" is unknown\n", domain); + if(!show_all) + { + ssend(*sock,"Domain \"%s\" is unknown\n", domainString); + } } diff --git a/src/api/api.h b/src/api/api.h new file mode 100644 index 000000000..6068da054 --- /dev/null +++ b/src/api/api.h @@ -0,0 +1,48 @@ +/* Pi-hole: A black hole for Internet advertisements +* (c) 2017 Pi-hole, LLC (https://pi-hole.net) +* Network-wide ad blocking via your own hardware. +* +* FTL Engine +* API commands and MessagePack helpers +* +* This file is copyright under the latest version of the EUPL. +* Please see LICENSE file for your rights under this license. */ +#ifndef API_H +#define API_H + +// Statistic methods +void getStats(const int *sock); +void getOverTime(const int *sock); +void getTopDomains(const char *client_message, const int *sock); +void getTopClients(const char *client_message, const int *sock); +void getUpstreamDestinations(const char *client_message, const int *sock); +void getQueryTypes(const int *sock); +void getAllQueries(const char *client_message, const int *sock); +void getRecentBlocked(const char *client_message, const int *sock); +void getQueryTypesOverTime(const int *sock); +void getClientsOverTime(const int *sock); +void getClientNames(const int *sock); +void getDomainDetails(const char *client_message, const int *sock); + +// FTL methods +void getClientID(const int *sock); +void getVersion(const int *sock); +void getDBstats(const int *sock); +void getUnknownQueries(const int *sock); + +// DNS resolver methods (dnsmasq_interface.c) +void getCacheInformation(const int *sock); + +// MessagePack serialization helpers +void pack_eom(const int sock); +void pack_bool(const int sock, const bool value); +void pack_uint8(const int sock, const uint8_t value); +void pack_uint64(const int sock, const uint64_t value); +void pack_int32(const int sock, const int32_t value); +void pack_int64(const int sock, const int64_t value); +void pack_float(const int sock, const float value); +bool pack_fixstr(const int sock, const char *string); +bool pack_str32(const int sock, const char *string); +void pack_map16_start(const int sock, const uint16_t length); + +#endif // API_H diff --git a/msgpack.c b/src/api/msgpack.c similarity index 68% rename from msgpack.c rename to src/api/msgpack.c index 58cc7ee96..eae1ff533 100644 --- a/msgpack.c +++ b/src/api/msgpack.c @@ -10,20 +10,22 @@ #include "FTL.h" #include "api.h" +#include "socket.h" +#include "log.h" -void pack_eom(int sock) { +void pack_eom(const int sock) { // This byte is explicitly never used in the MessagePack spec, so it is perfect to use as an EOM for this API. uint8_t eom = 0xc1; swrite(sock, &eom, sizeof(eom)); } -static void pack_basic(int sock, uint8_t format, void *value, size_t size) { +static void pack_basic(const int sock, const uint8_t format, const void *value, const size_t size) { swrite(sock, &format, sizeof(format)); swrite(sock, value, size); } -static uint64_t __attribute__((const)) leToBe64(uint64_t value) { - char *ptr = (char *) &value; +static uint64_t __attribute__((const)) leToBe64(const uint64_t value) { + const char *ptr = (char *) &value; uint32_t part1, part2; // Copy the two halves of the 64 bit input into uint32_t's so we can use htonl @@ -38,26 +40,26 @@ static uint64_t __attribute__((const)) leToBe64(uint64_t value) { return (uint64_t) part1 << 32 | part2; } -void pack_bool(int sock, bool value) { +void pack_bool(const int sock, const bool value) { uint8_t packed = (uint8_t) (value ? 0xc3 : 0xc2); swrite(sock, &packed, sizeof(packed)); } -void pack_uint8(int sock, uint8_t value) { +void pack_uint8(const int sock, const uint8_t value) { pack_basic(sock, 0xcc, &value, sizeof(value)); } -void pack_uint64(int sock, uint64_t value) { - uint64_t bigEValue = leToBe64(value); +void pack_uint64(const int sock, const uint64_t value) { + const uint64_t bigEValue = leToBe64(value); pack_basic(sock, 0xcf, &bigEValue, sizeof(bigEValue)); } -void pack_int32(int sock, int32_t value) { - uint32_t bigEValue = htonl((uint32_t) value); +void pack_int32(const int sock, const int32_t value) { + const uint32_t bigEValue = htonl((uint32_t) value); pack_basic(sock, 0xd2, &bigEValue, sizeof(bigEValue)); } -void pack_int64(int sock, int64_t value) { +void pack_int64(const int sock, const int64_t value) { // Need to use memcpy to do a direct copy without reinterpreting the bytes (making negatives into positives). // It should get optimized away. uint64_t bigEValue; @@ -66,7 +68,7 @@ void pack_int64(int sock, int64_t value) { pack_basic(sock, 0xd3, &bigEValue, sizeof(bigEValue)); } -void pack_float(int sock, float value) { +void pack_float(const int sock, const float value) { // Need to use memcpy to do a direct copy without reinterpreting the bytes. It should get optimized away. uint32_t bigEValue; memcpy(&bigEValue, &value, sizeof(bigEValue)); @@ -75,16 +77,16 @@ void pack_float(int sock, float value) { } // Return true if successful -bool pack_fixstr(int sock, const char *string) { +bool pack_fixstr(const int sock, const char *string) { // Make sure that the length is less than 32 - size_t length = strlen(string); + const size_t length = strlen(string); if(length >= 32) { logg("Tried to send a fixstr longer than 31 bytes!"); return false; } - uint8_t format = (uint8_t) (0xA0 | length); + const uint8_t format = (uint8_t) (0xA0 | length); swrite(sock, &format, sizeof(format)); swrite(sock, string, length); @@ -92,27 +94,27 @@ bool pack_fixstr(int sock, const char *string) { } // Return true if successful -bool pack_str32(int sock, const char *string) { +bool pack_str32(const int sock, const char *string) { // Make sure that the length is less than 4294967296 - size_t length = strlen(string); + const size_t length = strlen(string); if(length >= 2147483648u) { logg("Tried to send a str32 longer than 2147483647 bytes!"); return false; } - uint8_t format = 0xdb; + const uint8_t format = 0xdb; swrite(sock, &format, sizeof(format)); - uint32_t bigELength = htonl((uint32_t) length); + const uint32_t bigELength = htonl((uint32_t) length); swrite(sock, &bigELength, sizeof(bigELength)); swrite(sock, string, length); return true; } -void pack_map16_start(int sock, uint16_t length) { - uint8_t format = 0xde; +void pack_map16_start(const int sock, const uint16_t length) { + const uint8_t format = 0xde; swrite(sock, &format, sizeof(format)); - uint16_t bigELength = htons(length); + const uint16_t bigELength = htons(length); swrite(sock, &bigELength, sizeof(bigELength)); } diff --git a/request.c b/src/api/request.c similarity index 92% rename from request.c rename to src/api/request.c index bb1d36406..607d5b3b7 100644 --- a/request.c +++ b/src/api/request.c @@ -11,6 +11,13 @@ #include "FTL.h" #include "api.h" #include "shmem.h" +#include "timers.h" +#include "request.h" +#include "socket.h" +#include "resolve.h" +#include "regex_r.h" +#include "database/network-table.h" +#include "log.h" bool __attribute__((pure)) command(const char *client_message, const char* cmd) { return strstr(client_message, cmd) != NULL; @@ -55,14 +62,14 @@ void process_request(const char *client_message, int *sock) { processed = true; lock_shm(); - getForwardDestinations(client_message, sock); + getUpstreamDestinations(client_message, sock); unlock_shm(); } else if(command(client_message, ">forward-names")) { processed = true; lock_shm(); - getForwardDestinations(">forward-dest unsorted", sock); + getUpstreamDestinations(">forward-dest unsorted", sock); unlock_shm(); } else if(command(client_message, ">querytypes")) @@ -164,8 +171,9 @@ void process_request(const char *client_message, int *sock) processed = true; logg("Received API request to recompile regex"); lock_shm(); - free_regex(); - read_regex_from_file(); + // Reread regex.list + // Read and compile possible regex filters + read_regex_from_database(); unlock_shm(); } else if(command(client_message, ">update-mac-vendor")) diff --git a/src/api/request.h b/src/api/request.h new file mode 100644 index 000000000..ab2baa0aa --- /dev/null +++ b/src/api/request.h @@ -0,0 +1,16 @@ +/* Pi-hole: A black hole for Internet advertisements +* (c) 2019 Pi-hole, LLC (https://pi-hole.net) +* Network-wide ad blocking via your own hardware. +* +* FTL Engine +* Telnet/Socket request prototypes +* +* This file is copyright under the latest version of the EUPL. +* Please see LICENSE file for your rights under this license. */ +#ifndef REQUEST_H +#define REQUEST_H + +void process_request(const char *client_message, int *sock); +bool command(const char *client_message, const char* cmd) __attribute__((pure)); + +#endif //REQUEST_H diff --git a/socket.c b/src/api/socket.c similarity index 89% rename from socket.c rename to src/api/socket.c index 10adc4117..34c452373 100644 --- a/socket.c +++ b/src/api/socket.c @@ -10,7 +10,13 @@ #include "FTL.h" #include "api.h" -#include "shmem.h" +#include "log.h" +#include "socket.h" +#include "request.h" +#include "config.h" +#include "memory.h" +// global variable killed +#include "signals.h" // The backlog argument defines the maximum length // to which the queue of pending connections for @@ -26,6 +32,7 @@ int socketfd, telnetfd4 = 0, telnetfd6 = 0; bool dualstack = false; bool ipv4telnet = false, ipv6telnet = false; +bool sock_avail = false; bool istelnet[MAXCONNS]; static void saveport(void) @@ -160,8 +167,9 @@ static void bind_to_unix_socket(int *socketdescriptor) if(*socketdescriptor < 0) { - logg("Error opening Unix socket"); - exit(EXIT_FAILURE); + logg("WARNING: Error opening Unix socket."); + logg(" Continuing anyway."); + return; } // Make sure unix socket file handle does not exist, if it exists, remove it @@ -169,24 +177,33 @@ static void bind_to_unix_socket(int *socketdescriptor) struct sockaddr_un address; address.sun_family = AF_LOCAL; - strcpy(address.sun_path, FTLfiles.socketfile); - - // Bild to Unix socket handle + // The sockaddr_un.sum_path may be shorter than the size of the FTLfiles.socketfile + // buffer. Ensure that the string is null-terminated even when the string is too large. + // In case strlen(FTLfiles.socketfile) < sizeof(address.sun_path) [this will virtually + // always be the case], the explicit setting of the last byte to zero is a no-op as + // strncpy() writes additional null bytes to ensure that a total of n bytes are written. + strncpy(address.sun_path, FTLfiles.socketfile, sizeof(address.sun_path)); + address.sun_path[sizeof(address.sun_path)-1] = '\0'; + + // Bind to Unix socket handle errno = 0; if(bind(*socketdescriptor, (struct sockaddr *) &address, sizeof (address)) != 0) { - logg("Error on binding on Unix socket %s: %s (%i)", FTLfiles.socketfile, strerror(errno), errno); - exit(EXIT_FAILURE); + logg("WARNING: Cannot bind on Unix socket %s: %s (%i)", FTLfiles.socketfile, strerror(errno), errno); + logg(" Continuing anyway."); + return; } // The listen system call allows the process to listen on the Unix socket for connections if(listen(*socketdescriptor, BACKLOG) == -1) { - logg("Error listening on Unix socket: %s (%i)", strerror(errno), errno); - exit(EXIT_FAILURE); + logg("WARNING: Cannot listen on Unix socket: %s (%i)", strerror(errno), errno); + logg(" Continuing anyway."); + return; } logg("Listening on Unix socket"); + sock_avail = true; } // Called from main() at graceful shutdown @@ -201,7 +218,7 @@ static void removeport(void) fclose(f); } -void seom(int sock) +void seom(const int sock) { if(istelnet[sock]) ssend(sock, "---EOM---\n\n"); @@ -209,7 +226,7 @@ void seom(int sock) pack_eom(sock); } -void __attribute__ ((format (gnu_printf, 2, 3))) ssend(int sock, const char *format, ...) +void __attribute__ ((format (gnu_printf, 2, 3))) ssend(const int sock, const char *format, ...) { char *buffer; va_list args; @@ -224,12 +241,12 @@ void __attribute__ ((format (gnu_printf, 2, 3))) ssend(int sock, const char *for } } -void swrite(int sock, const void *value, size_t size) { +void swrite(const int sock, const void *value, size_t size) { if(write(sock, value, size) == -1) logg("WARNING: Socket write returned error code %i", errno); } -static inline int checkClientLimit(int socket) { +static inline int checkClientLimit(const int socket) { if(socket < MAXCONNS) { return socket; @@ -243,7 +260,7 @@ static inline int checkClientLimit(int socket) { } } -static int listener(int sockfd, char type) +static int listener(const int sockfd, const char type) { struct sockaddr_un un_addr; struct sockaddr_in in4_addr; @@ -359,7 +376,8 @@ static void *socket_connection_handler_thread(void *socket_desc) char threadname[16]; sprintf(threadname,"socket-%i",sock); prctl(PR_SET_NAME,threadname,0,0,0); - //Receive from client + + // Receive from client ssize_t n; while((n = recv(sock,client_message,SOCKETBUFFERLEN-1, 0))) { @@ -430,7 +448,7 @@ void *telnet_listening_thread_IPv4(void *args) while(!killed) { // Look for new clients that want to connect - int csck = listener(telnetfd4, 4); + const int csck = listener(telnetfd4, 4); if(csck == -1) { logg("IPv4 telnet error: %s (%i)", strerror(errno), errno); @@ -471,7 +489,7 @@ void *telnet_listening_thread_IPv6(void *args) while(!killed) { // Look for new clients that want to connect - int csck = listener(telnetfd6, 6); + const int csck = listener(telnetfd6, 6); if(csck == -1) { logg("IPv6 telnet error: %s (%i)", strerror(errno), errno); @@ -508,11 +526,15 @@ void *socket_listening_thread(void *args) // Set thread name prctl(PR_SET_NAME,"socket listener",0,0,0); + // Return early to avoid CPU spinning if Unix socket is not available + if(!sock_avail) + return NULL; + // Listen as long as FTL is not killed while(!killed) { // Look for new clients that want to connect - int csck = listener(socketfd, 0); + const int csck = listener(socketfd, 0); if(csck < 0) continue; // Allocate memory used to transport client socket ID to client listening thread @@ -544,8 +566,8 @@ bool ipv6_available(void) // Loop over interfaces for (interface = allInterfaces; interface != NULL; interface = interface->ifa_next) { - unsigned int flags = interface->ifa_flags; - struct sockaddr *addr = interface->ifa_addr; + const unsigned int flags = interface->ifa_flags; + const struct sockaddr *addr = interface->ifa_addr; // Check only for up and running IPv4, IPv6 interfaces if ((flags & (IFF_UP|IFF_RUNNING)) && addr != NULL) diff --git a/src/api/socket.h b/src/api/socket.h new file mode 100644 index 000000000..62ce923c4 --- /dev/null +++ b/src/api/socket.h @@ -0,0 +1,27 @@ +/* Pi-hole: A black hole for Internet advertisements +* (c) 2019 Pi-hole, LLC (https://pi-hole.net) +* Network-wide ad blocking via your own hardware. +* +* FTL Engine +* Socket prototypes +* +* This file is copyright under the latest version of the EUPL. +* Please see LICENSE file for your rights under this license. */ +#ifndef SOCKET_H +#define SOCKET_H + +void close_telnet_socket(void); +void close_unix_socket(void); +void seom(const int sock); +void ssend(const int sock, const char *format, ...) __attribute__ ((format (gnu_printf, 2, 3))); +void swrite(const int sock, const void* value, const size_t size); +void *telnet_listening_thread_IPv4(void *args); +void *telnet_listening_thread_IPv6(void *args); +void *socket_listening_thread(void *args); +bool ipv6_available(void); +void bind_sockets(void); + +extern bool ipv4telnet, ipv6telnet; +extern bool istelnet[MAXCONNS]; + +#endif //SOCKET_H diff --git a/args.c b/src/args.c similarity index 64% rename from args.c rename to src/args.c index 8e2bdbf5c..c7bd024af 100644 --- a/args.c +++ b/src/args.c @@ -9,28 +9,101 @@ * Please see LICENSE file for your rights under this license. */ #include "FTL.h" +#include "args.h" #include "version.h" +#include "memory.h" +#include "main.h" +#include "log.h" +// global variable killed +#include "signals.h" static bool debug = false; bool daemonmode = true; -bool travis = false; int argc_dnsmasq = 0; const char** argv_dnsmasq = NULL; +static inline bool strEndsWith(const char *input, const char *end){ + return strcmp(input + strlen(input) - strlen(end), end) == 0; +} + void parse_args(int argc, char* argv[]) { - int i; - // Regardless of any arguments, we always pass "-k" (nofork) to dnsmasq argc_dnsmasq = 2; argv_dnsmasq = calloc(argc_dnsmasq, sizeof(char*)); argv_dnsmasq[0] = ""; argv_dnsmasq[1] = "-k"; - // start from 1, as argv[0] is the executable name "pihole-FTL" - for(i=1; i < argc; i++) + bool consume_for_dnsmasq = false; + // If the binary name is "dnsmasq" (e.g., symlink /usr/bin/dnsmasq -> /usr/bin/pihole-FTL), + // we operate in drop-in mode and consume all arguments for the embedded dnsmasq core + if(strEndsWith(argv[0], "dnsmasq")) + consume_for_dnsmasq = true; + + // start from 1, as argv[0] is the executable name + for(int i = 1; i < argc; i++) { bool ok = false; + + // Implement dnsmasq's test function, no need to prepare the entire FTL + // environment (initialize shared memory, lead queries from long-term + // database, ...) when the task is a simple (dnsmasq) syntax check + if(strcmp(argv[i], "dnsmasq-test") == 0 || + strcmp(argv[i], "--test") == 0) + { + const char *arg[2]; + arg[0] = ""; + arg[1] = "--test"; + main_dnsmasq(2, arg); + ok = true; + } + + // If we find "--" we collect everything behind that for dnsmasq + if(strcmp(argv[i], "--") == 0) + { + // Remember that the rest is for dnsmasq ... + consume_for_dnsmasq = true; + + // ... and skip the current argument ("--") + continue; + } + + // If consume_for_dnsmasq is true, we collect all remaining options for + // dnsmasq + if(consume_for_dnsmasq) + { + argc_dnsmasq = argc - i + 2; + if(argv_dnsmasq != NULL) + free(argv_dnsmasq); + + argv_dnsmasq = calloc(argc_dnsmasq, sizeof(const char*)); + argv_dnsmasq[0] = ""; + + if(debug) + argv_dnsmasq[1] = "-d"; + else + argv_dnsmasq[1] = "-k"; + + if(debug) + { + printf("dnsmasq options: [0]: %s\n", argv_dnsmasq[0]); + printf("dnsmasq options: [1]: %s\n", argv_dnsmasq[1]); + } + + int j = 2; + while(i < argc) + { + argv_dnsmasq[j++] = strdup(argv[i++]); + if(debug) + printf("dnsmasq options: [%i]: %s\n", j-1, argv_dnsmasq[j-1]); + } + + // Return early: We have consumes all available command line arguments + return; + } + + // What follows beyond this point are FTL internal command line arguments + if(strcmp(argv[i], "d") == 0 || strcmp(argv[i], "debug") == 0) { @@ -52,19 +125,7 @@ void parse_args(int argc, char* argv[]) strcmp(argv[i], "version") == 0 || strcmp(argv[i], "--version") == 0) { - const char * commit = GIT_HASH; - const char * tag = GIT_TAG; - if(strlen(tag) > 1) - { - printf("%s\n",GIT_VERSION); - } - else - { - char hash[8]; - // Extract first 7 characters of the hash - strncpy(hash, commit, 7); hash[7] = 0; - printf("vDev-%s\n", hash); - } + printf("%s\n", get_FTL_version()); exit(EXIT_SUCCESS); } @@ -90,42 +151,6 @@ void parse_args(int argc, char* argv[]) ok = true; } - // Use files in local places for Travis-CI tests - if(strcmp(argv[i], "travis-ci") == 0) - { - travis = true; - ok = true; - } - - // Implement dnsmasq's test function - if(strcmp(argv[i], "dnsmasq-test") == 0) - { - const char *arg[2]; - arg[0] = ""; - arg[1] = "--test"; - main_dnsmasq(2, arg); - ok = true; - } - - // If we find "--" we collect everything behind that for dnsmasq - if(strcmp(argv[i], "--") == 0) - { - int j; - argc_dnsmasq = argc - i + 1; - if(argv_dnsmasq != NULL) free(argv_dnsmasq); - argv_dnsmasq = calloc(argc_dnsmasq + 2,sizeof(const char*)); - argv_dnsmasq[0] = ""; - if(debug) argv_dnsmasq[1] = "-d"; - else argv_dnsmasq[1] = "-k"; - - for(j=2; j < argc_dnsmasq; j++) - { - argv_dnsmasq[j] = strdup(argv[i+j-1]); - if(debug) logg("dnsmasq options: [%i]: %s",j,argv_dnsmasq[j]); - } - return; - } - // List of implemented arguments if(strcmp(argv[i], "-h") == 0 || strcmp(argv[i], "help") == 0 || strcmp(argv[i], "--help") == 0) { diff --git a/src/args.h b/src/args.h new file mode 100644 index 000000000..27b4bf304 --- /dev/null +++ b/src/args.h @@ -0,0 +1,19 @@ +/* Pi-hole: A black hole for Internet advertisements +* (c) 2019 Pi-hole, LLC (https://pi-hole.net) +* Network-wide ad blocking via your own hardware. +* +* FTL Engine +* Argument parsing prototypes +* +* This file is copyright under the latest version of the EUPL. +* Please see LICENSE file for your rights under this license. */ +#ifndef ARGS_H +#define ARGS_H + +void parse_args(int argc, char* argv[]); + +extern bool daemonmode; +extern int argc_dnsmasq; +extern const char ** argv_dnsmasq; + +#endif //ARGS_H diff --git a/capabilities.c b/src/capabilities.c similarity index 97% rename from capabilities.c rename to src/capabilities.c index 9290cc2c4..609a1d24f 100644 --- a/capabilities.c +++ b/src/capabilities.c @@ -13,12 +13,16 @@ #include "dnsmasq/dnsmasq.h" #undef __USE_XOPEN #include "FTL.h" +#include "capabilities.h" +#include "memory.h" +#include "config.h" +#include "log.h" static const unsigned int capabilityIDs[] = { CAP_CHOWN , CAP_DAC_OVERRIDE , CAP_DAC_READ_SEARCH , CAP_FOWNER , CAP_FSETID , CAP_KILL , CAP_SETGID , CAP_SETUID , CAP_SETPCAP , CAP_LINUX_IMMUTABLE , CAP_NET_BIND_SERVICE , CAP_NET_BROADCAST , CAP_NET_ADMIN , CAP_NET_RAW , CAP_IPC_LOCK , CAP_IPC_OWNER , CAP_SYS_MODULE , CAP_SYS_RAWIO , CAP_SYS_CHROOT , CAP_SYS_PTRACE , CAP_SYS_PACCT , CAP_SYS_ADMIN , CAP_SYS_BOOT , CAP_SYS_NICE , CAP_SYS_RESOURCE , CAP_SYS_TIME , CAP_SYS_TTY_CONFIG , CAP_MKNOD , CAP_LEASE , CAP_AUDIT_WRITE , CAP_AUDIT_CONTROL , CAP_SETFCAP , CAP_MAC_OVERRIDE , CAP_MAC_ADMIN , CAP_SYSLOG , CAP_WAKE_ALARM , CAP_BLOCK_SUSPEND , CAP_AUDIT_READ }; static const char* capabilityNames[] = {"CAP_CHOWN", "CAP_DAC_OVERRIDE", "CAP_DAC_READ_SEARCH", "CAP_FOWNER", "CAP_FSETID", "CAP_KILL", "CAP_SETGID", "CAP_SETUID", "CAP_SETPCAP", "CAP_LINUX_IMMUTABLE", "CAP_NET_BIND_SERVICE", "CAP_NET_BROADCAST", "CAP_NET_ADMIN", "CAP_NET_RAW", "CAP_IPC_LOCK", "CAP_IPC_OWNER", "CAP_SYS_MODULE", "CAP_SYS_RAWIO", "CAP_SYS_CHROOT", "CAP_SYS_PTRACE", "CAP_SYS_PACCT", "CAP_SYS_ADMIN", "CAP_SYS_BOOT", "CAP_SYS_NICE", "CAP_SYS_RESOURCE", "CAP_SYS_TIME", "CAP_SYS_TTY_CONFIG", "CAP_MKNOD", "CAP_LEASE", "CAP_AUDIT_WRITE", "CAP_AUDIT_CONTROL", "CAP_SETFCAP", "CAP_MAC_OVERRIDE", "CAP_MAC_ADMIN", "CAP_SYSLOG", "CAP_WAKE_ALARM", "CAP_BLOCK_SUSPEND", "CAP_AUDIT_READ"}; static const unsigned int numCaps = sizeof(capabilityIDs) / sizeof(const unsigned int); -bool check_capabilities() +bool check_capabilities(void) { // First assume header version 1 int capsize = 1; // VFS_CAP_U32_1 diff --git a/src/capabilities.h b/src/capabilities.h new file mode 100644 index 000000000..a65113828 --- /dev/null +++ b/src/capabilities.h @@ -0,0 +1,15 @@ +/* Pi-hole: A black hole for Internet advertisements +* (c) 2019 Pi-hole, LLC (https://pi-hole.net) +* Network-wide ad blocking via your own hardware. +* +* FTL Engine +* Linux capabilities prototypes +* +* This file is copyright under the latest version of the EUPL. +* Please see LICENSE file for your rights under this license. */ +#ifndef CAPABILITIES_H +#define CAPABILITIES_H + +bool check_capabilities(void); + +#endif //CAPABILITIES_H diff --git a/config.c b/src/config.c similarity index 73% rename from config.c rename to src/config.c index c867a6a79..9d529c2b9 100644 --- a/config.c +++ b/src/config.c @@ -9,13 +9,36 @@ * Please see LICENSE file for your rights under this license. */ #include "FTL.h" +#include "config.h" +#include "memory.h" +#include "setupVars.h" +#include "log.h" ConfigStruct config; +FTLFileNamesStruct FTLfiles = { + // Default path for config file (regular installations) + "/etc/pihole/pihole-FTL.conf", + // Alternative path for config file (snap installations) + "/var/snap/pihole/common/etc/pihole/pihole-FTL.conf", + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL +}; + +// Private global variables +static char *conflinebuffer = NULL; +static size_t size = 0; + +// Private prototypes static char *parse_FTLconf(FILE *fp, const char * key); static void release_config_memory(void); -void getpath(FILE* fp, const char *option, const char *defaultloc, char **pointer); - -char *conflinebuffer = NULL; +static void getpath(FILE* fp, const char *option, const char *defaultloc, char **pointer); void getLogFilePath(void) { @@ -166,20 +189,20 @@ void read_FTLconf(void) errno = 0; // Use sscanf() to obtain filename from config file parameter only if buffer != NULL - if(!(buffer != NULL && sscanf(buffer, "%127ms", &FTLfiles.db))) + if(!(buffer != NULL && sscanf(buffer, "%127ms", &FTLfiles.FTL_db))) { // Use standard path if no custom path was obtained from the config file - FTLfiles.db = strdup("/etc/pihole/pihole-FTL.db"); + FTLfiles.FTL_db = strdup("/etc/pihole/pihole-FTL.db"); } // Test if memory allocation was successful - if(FTLfiles.db == NULL && errno != 0) + if(FTLfiles.FTL_db == NULL && errno != 0) { - logg("FATAL: Allocating memory for FTLfiles.db failed (%s, %i). Exiting.", strerror(errno), errno); + logg("FATAL: Allocating memory for FTLfiles.FTL_db failed (%s, %i). Exiting.", strerror(errno), errno); exit(EXIT_FAILURE); } - else if(FTLfiles.db != NULL && strlen(FTLfiles.db) > 0) - logg(" DBFILE: Using %s", FTLfiles.db); + else if(FTLfiles.FTL_db != NULL && strlen(FTLfiles.FTL_db) > 0) + logg(" DBFILE: Using %s", FTLfiles.FTL_db); else logg(" DBFILE: Not using database due to empty filename"); @@ -221,7 +244,7 @@ void read_FTLconf(void) logg(" PRIVACYLEVEL: Set to %i", config.privacylevel); // IGNORE_LOCALHOST - // defaults to: No + // defaults to: false config.ignore_localhost = false; buffer = parse_FTLconf(fp, "IGNORE_LOCALHOST"); @@ -256,7 +279,7 @@ void read_FTLconf(void) } // ANALYZE_ONLY_A_AND_AAAA - // defaults to: No + // defaults to: false config.analyze_only_A_AAAA = false; buffer = parse_FTLconf(fp, "ANALYZE_ONLY_A_AND_AAAA"); @@ -288,26 +311,14 @@ void read_FTLconf(void) // SOCKETFILE getpath(fp, "SOCKETFILE", "/var/run/pihole/FTL.sock", &FTLfiles.socketfile); - // WHITELISTFILE - getpath(fp, "WHITELISTFILE", "/etc/pihole/whitelist.txt", &files.whitelist); - - // BLACKLISTFILE - getpath(fp, "BLACKLISTFILE", "/etc/pihole/black.list", &files.blacklist); - - // GRAVITYFILE - getpath(fp, "GRAVITYFILE", "/etc/pihole/gravity.list", &files.gravity); - - // REGEXLISTFILE - getpath(fp, "REGEXLISTFILE", "/etc/pihole/regex.list", &files.regexlist); - // SETUPVARSFILE - getpath(fp, "SETUPVARSFILE", "/etc/pihole/setupVars.conf", &files.setupVars); - - // AUDITLISTFILE - getpath(fp, "AUDITLISTFILE", "/etc/pihole/auditlog.list", &files.auditlist); + getpath(fp, "SETUPVARSFILE", "/etc/pihole/setupVars.conf", &FTLfiles.setupVars); // MACVENDORDB - getpath(fp, "MACVENDORDB", "/etc/pihole/macvendor.db", &FTLfiles.macvendordb); + getpath(fp, "MACVENDORDB", "/etc/pihole/macvendor.db", &FTLfiles.macvendor_db); + + // GRAVITYDB + getpath(fp, "GRAVITYDB", "/etc/pihole/gravity.db", &FTLfiles.gravity_db); // PARSE_ARP_CACHE // defaults to: true @@ -322,6 +333,45 @@ void read_FTLconf(void) else logg(" PARSE_ARP_CACHE: Inactive"); + // CNAME_DEEP_INSPECT + // defaults to: true + config.cname_inspection = true; + buffer = parse_FTLconf(fp, "CNAME_DEEP_INSPECT"); + + if(buffer != NULL && strcasecmp(buffer, "false") == 0) + config.cname_inspection = false; + + if(config.cname_inspection) + logg(" CNAME_DEEP_INSPECT: Active"); + else + logg(" CNAME_DEEP_INSPECT: Inactive"); + + // DELAY_STARTUP + // defaults to: zero (seconds) + buffer = parse_FTLconf(fp, "DELAY_STARTUP"); + + config.delay_startup = 0; + if(buffer != NULL && sscanf(buffer, "%u", &config.delay_startup) && + (config.delay_startup > 0 && config.delay_startup <= 300)) + { + logg(" DELAY_STARTUP: Requested to wait %u seconds during startup.", config.delay_startup); + } + else + logg(" DELAY_STARTUP: No delay requested."); + + // BLOCK_ESNI + // defaults to: true + config.block_esni = true; + buffer = parse_FTLconf(fp, "BLOCK_ESNI"); + + if(buffer != NULL && strcasecmp(buffer, "false") == 0) + config.block_esni = false; + + if(config.block_esni) + logg(" BLOCK_ESNI: Enabled, blocking _esni.{blocked domain}"); + else + logg(" BLOCK_ESNI: Disabled"); + // Read DEBUG_... setting from pihole-FTL.conf read_debuging_settings(fp); @@ -334,7 +384,7 @@ void read_FTLconf(void) fclose(fp); } -void getpath(FILE* fp, const char *option, const char *defaultloc, char **pointer) +static void getpath(FILE* fp, const char *option, const char *defaultloc, char **pointer) { // This subroutine is used to read paths from pihole-FTL.conf // fp: File pointer to opened and readable config file @@ -385,7 +435,6 @@ static char *parse_FTLconf(FILE *fp, const char * key) // Go to beginning of file fseek(fp, 0L, SEEK_SET); - size_t size = 0; errno = 0; while(getline(&conflinebuffer, &size, fp) != -1) { @@ -503,6 +552,22 @@ void get_blocking_mode(FILE *fp) fclose(fp); } +// Routine for setting the debug flags in the config struct +static void setDebugOption(FILE* fp, const char* option, int16_t bitmask) +{ + const char* buffer = parse_FTLconf(fp, option); + + // Return early if the key has not been found in FTL's config file + if(buffer == NULL) + return; + + // Set bit if value equals "true", clear bit otherwise + if(strcasecmp(buffer, "true") == 0) + config.debug |= bitmask; + else + config.debug &= ~bitmask; +} + void read_debuging_settings(FILE *fp) { // Set default (no debug instructions set) @@ -519,111 +584,98 @@ void read_debuging_settings(FILE *fp) opened = true; } + // DEBUG_ALL + // defaults to: false + // ~0 is a shortcut for "all bits set" + setDebugOption(fp, "DEBUG_ALL", ~(int16_t)0); + // DEBUG_DATABASE // defaults to: false - char* buffer = parse_FTLconf(fp, "DEBUG_DATABASE"); - if(buffer != NULL && strcasecmp(buffer, "true") == 0) - config.debug |= DEBUG_DATABASE; + setDebugOption(fp, "DEBUG_DATABASE", DEBUG_DATABASE); // DEBUG_NETWORKING // defaults to: false - buffer = parse_FTLconf(fp, "DEBUG_NETWORKING"); - if(buffer != NULL && strcasecmp(buffer, "true") == 0) - config.debug |= DEBUG_NETWORKING; + setDebugOption(fp, "DEBUG_NETWORKING", DEBUG_NETWORKING); // DEBUG_LOCKS // defaults to: false - buffer = parse_FTLconf(fp, "DEBUG_LOCKS"); - if(buffer != NULL && strcasecmp(buffer, "true") == 0) - config.debug |= DEBUG_LOCKS; + setDebugOption(fp, "DEBUG_LOCKS", DEBUG_LOCKS); // DEBUG_QUERIES // defaults to: false - buffer = parse_FTLconf(fp, "DEBUG_QUERIES"); - if(buffer != NULL && strcasecmp(buffer, "true") == 0) - config.debug |= DEBUG_QUERIES; + setDebugOption(fp, "DEBUG_QUERIES", DEBUG_QUERIES); // DEBUG_FLAGS // defaults to: false - buffer = parse_FTLconf(fp, "DEBUG_FLAGS"); - if(buffer != NULL && strcasecmp(buffer, "true") == 0) - config.debug |= DEBUG_FLAGS; + setDebugOption(fp, "DEBUG_FLAGS", DEBUG_FLAGS); // DEBUG_SHMEM // defaults to: false - buffer = parse_FTLconf(fp, "DEBUG_SHMEM"); - if(buffer != NULL && strcasecmp(buffer, "true") == 0) - config.debug |= DEBUG_SHMEM; + setDebugOption(fp, "DEBUG_SHMEM", DEBUG_SHMEM); // DEBUG_GC // defaults to: false - buffer = parse_FTLconf(fp, "DEBUG_GC"); - if(buffer != NULL && strcasecmp(buffer, "true") == 0) - config.debug |= DEBUG_GC; + setDebugOption(fp, "DEBUG_GC", DEBUG_GC); // DEBUG_ARP // defaults to: false - buffer = parse_FTLconf(fp, "DEBUG_ARP"); - if(buffer != NULL && strcasecmp(buffer, "true") == 0) - config.debug |= DEBUG_ARP; + setDebugOption(fp, "DEBUG_ARP", DEBUG_ARP); // DEBUG_REGEX or REGEX_DEBUGMODE (legacy config option) // defaults to: false - buffer = parse_FTLconf(fp, "DEBUG_REGEX"); - if(buffer != NULL && strcasecmp(buffer, "true") == 0) - config.debug |= DEBUG_REGEX; - buffer = parse_FTLconf(fp, "REGEX_DEBUGMODE"); - if(buffer != NULL && strcasecmp(buffer, "true") == 0) - config.debug |= DEBUG_REGEX; + setDebugOption(fp, "REGEX_DEBUGMODE", DEBUG_REGEX); + setDebugOption(fp, "DEBUG_REGEX", DEBUG_REGEX); // DEBUG_API // defaults to: false - buffer = parse_FTLconf(fp, "DEBUG_API"); - if(buffer != NULL && strcasecmp(buffer, "true") == 0) - config.debug |= DEBUG_API; + setDebugOption(fp, "DEBUG_API", DEBUG_API); // DEBUG_OVERTIME // defaults to: false - buffer = parse_FTLconf(fp, "DEBUG_OVERTIME"); - if(buffer != NULL && strcasecmp(buffer, "true") == 0) - config.debug |= DEBUG_OVERTIME; + setDebugOption(fp, "DEBUG_OVERTIME", DEBUG_OVERTIME); // DEBUG_EXTBLOCKED // defaults to: false - buffer = parse_FTLconf(fp, "DEBUG_EXTBLOCKED"); - if(buffer != NULL && strcasecmp(buffer, "true") == 0) - config.debug |= DEBUG_EXTBLOCKED; + setDebugOption(fp, "DEBUG_EXTBLOCKED", DEBUG_EXTBLOCKED); // DEBUG_CAPS // defaults to: false - buffer = parse_FTLconf(fp, "DEBUG_CAPS"); - if(buffer != NULL && strcasecmp(buffer, "true") == 0) - config.debug |= DEBUG_CAPS; + setDebugOption(fp, "DEBUG_CAPS", DEBUG_CAPS); - // DEBUG_ALL + // DEBUG_DNSMASQ_LINES + setDebugOption(fp, "DEBUG_DNSMASQ_LINES", DEBUG_DNSMASQ_LINES); + extern char debug_dnsmasq_lines; + debug_dnsmasq_lines = config.debug & DEBUG_DNSMASQ_LINES ? 1 : 0; + + // DEBUG_VECTORS // defaults to: false - buffer = parse_FTLconf(fp, "DEBUG_ALL"); - if(buffer != NULL && strcasecmp(buffer, "true") == 0) - config.debug = ~0; + setDebugOption(fp, "DEBUG_VECTORS", DEBUG_VECTORS); + + // DEBUG_RESOLVER + // defaults to: false + setDebugOption(fp, "DEBUG_RESOLVER", DEBUG_RESOLVER); if(config.debug) { - logg("************************"); - logg("* Debugging enabled *"); - logg("* DEBUG_DATABASE %s *", (config.debug & DEBUG_DATABASE)? "YES":"NO "); - logg("* DEBUG_NETWORKING %s *", (config.debug & DEBUG_NETWORKING)? "YES":"NO "); - logg("* DEBUG_LOCKS %s *", (config.debug & DEBUG_LOCKS)? "YES":"NO "); - logg("* DEBUG_QUERIES %s *", (config.debug & DEBUG_QUERIES)? "YES":"NO "); - logg("* DEBUG_FLAGS %s *", (config.debug & DEBUG_FLAGS)? "YES":"NO "); - logg("* DEBUG_SHMEM %s *", (config.debug & DEBUG_SHMEM)? "YES":"NO "); - logg("* DEBUG_GC %s *", (config.debug & DEBUG_GC)? "YES":"NO "); - logg("* DEBUG_ARP %s *", (config.debug & DEBUG_ARP)? "YES":"NO "); - logg("* DEBUG_REGEX %s *", (config.debug & DEBUG_REGEX)? "YES":"NO "); - logg("* DEBUG_API %s *", (config.debug & DEBUG_API)? "YES":"NO "); - logg("* DEBUG_OVERTIME %s *", (config.debug & DEBUG_OVERTIME)? "YES":"NO "); - logg("* DEBUG_EXTBLOCKED %s *", (config.debug & DEBUG_EXTBLOCKED)? "YES":"NO "); - logg("* DEBUG_CAPS %s *", (config.debug & DEBUG_CAPS)? "YES":"NO "); - logg("************************"); + logg("*****************************"); + logg("* Debugging enabled *"); + logg("* DEBUG_DATABASE %s *", (config.debug & DEBUG_DATABASE)? "YES":"NO "); + logg("* DEBUG_NETWORKING %s *", (config.debug & DEBUG_NETWORKING)? "YES":"NO "); + logg("* DEBUG_LOCKS %s *", (config.debug & DEBUG_LOCKS)? "YES":"NO "); + logg("* DEBUG_QUERIES %s *", (config.debug & DEBUG_QUERIES)? "YES":"NO "); + logg("* DEBUG_FLAGS %s *", (config.debug & DEBUG_FLAGS)? "YES":"NO "); + logg("* DEBUG_SHMEM %s *", (config.debug & DEBUG_SHMEM)? "YES":"NO "); + logg("* DEBUG_GC %s *", (config.debug & DEBUG_GC)? "YES":"NO "); + logg("* DEBUG_ARP %s *", (config.debug & DEBUG_ARP)? "YES":"NO "); + logg("* DEBUG_REGEX %s *", (config.debug & DEBUG_REGEX)? "YES":"NO "); + logg("* DEBUG_API %s *", (config.debug & DEBUG_API)? "YES":"NO "); + logg("* DEBUG_OVERTIME %s *", (config.debug & DEBUG_OVERTIME)? "YES":"NO "); + logg("* DEBUG_EXTBLOCKED %s *", (config.debug & DEBUG_EXTBLOCKED)? "YES":"NO "); + logg("* DEBUG_CAPS %s *", (config.debug & DEBUG_CAPS)? "YES":"NO "); + logg("* DEBUG_DNSMASQ_LINES %s *", (config.debug & DEBUG_DNSMASQ_LINES)? "YES":"NO "); + logg("* DEBUG_VECTORS %s *", (config.debug & DEBUG_VECTORS)? "YES":"NO "); + logg("* DEBUG_RESOLVER %s *", (config.debug & DEBUG_RESOLVER)? "YES":"NO "); + logg("*****************************"); } // Have to close the config file if we opened it diff --git a/src/config.h b/src/config.h new file mode 100644 index 000000000..e7ba0f81d --- /dev/null +++ b/src/config.h @@ -0,0 +1,80 @@ +/* Pi-hole: A black hole for Internet advertisements +* (c) 2019 Pi-hole, LLC (https://pi-hole.net) +* Network-wide ad blocking via your own hardware. +* +* FTL Engine +* FTL config file prototypes +* +* This file is copyright under the latest version of the EUPL. +* Please see LICENSE file for your rights under this license. */ +#ifndef CONFIG_H +#define CONFIG_H + +// typedef int16_t +#include + +void getLogFilePath(void); +void read_FTLconf(void); +void get_privacy_level(FILE *fp); +void get_blocking_mode(FILE *fp); +void read_debuging_settings(FILE *fp); + +typedef struct { + int maxDBdays; + int DBinterval; + int port; + int maxlogage; + int dns_port; + unsigned int delay_startup; + int16_t debug; + unsigned char privacylevel; + unsigned char blockingmode; + bool socket_listenlocal; + bool analyze_AAAA; + bool resolveIPv6; + bool resolveIPv4; + bool ignore_localhost; + bool analyze_only_A_AAAA; + bool DBimport; + bool parse_arp_cache; + bool cname_inspection; + bool block_esni; +} ConfigStruct; + +typedef struct { + const char* conf; + const char* snapConf; + char* log; + char* pid; + char* port; + char* socketfile; + char* FTL_db; + char* gravity_db; + char* macvendor_db; + char* setupVars; + char* auditlist; +} FTLFileNamesStruct; + +extern ConfigStruct config; +extern FTLFileNamesStruct FTLfiles; + +enum { + DEBUG_DATABASE = (1 << 0), /* 00000000 00000001 */ + DEBUG_NETWORKING = (1 << 1), /* 00000000 00000010 */ + DEBUG_LOCKS = (1 << 2), /* 00000000 00000100 */ + DEBUG_QUERIES = (1 << 3), /* 00000000 00001000 */ + DEBUG_FLAGS = (1 << 4), /* 00000000 00010000 */ + DEBUG_SHMEM = (1 << 5), /* 00000000 00100000 */ + DEBUG_GC = (1 << 6), /* 00000000 01000000 */ + DEBUG_ARP = (1 << 7), /* 00000000 10000000 */ + DEBUG_REGEX = (1 << 8), /* 00000001 00000000 */ + DEBUG_API = (1 << 9), /* 00000010 00000000 */ + DEBUG_OVERTIME = (1 << 10), /* 00000100 00000000 */ + DEBUG_EXTBLOCKED = (1 << 11), /* 00001000 00000000 */ + DEBUG_CAPS = (1 << 12), /* 00010000 00000000 */ + DEBUG_DNSMASQ_LINES = (1 << 13), /* 00100000 00000000 */ + DEBUG_VECTORS = (1 << 14), /* 01000000 00000000 */ + DEBUG_RESOLVER = (1 << 15), /* 10000000 00000000 */ +}; + +#endif //CONFIG_H diff --git a/daemon.c b/src/daemon.c similarity index 76% rename from daemon.c rename to src/daemon.c index 9236c36a6..a8e1bd21b 100644 --- a/daemon.c +++ b/src/daemon.c @@ -9,16 +9,17 @@ * Please see LICENSE file for your rights under this license. */ #include "FTL.h" - -struct timeval t0[NUMTIMERS]; +#include "daemon.h" +#include "memory.h" +#include "config.h" +#include "log.h" +// sleepms() +#include "timers.h" void go_daemon(void) { - pid_t process_id = 0; - pid_t sid = 0; - // Create child process - process_id = fork(); + pid_t process_id = fork(); // Indication of fork() failure if (process_id < 0) @@ -41,7 +42,7 @@ void go_daemon(void) //set new session // creates a session and sets the process group ID - sid = setsid(); + const pid_t sid = setsid(); if(sid < 0) { // Return failure @@ -80,40 +81,10 @@ void go_daemon(void) // Closing stdin, stdout and stderr is handled by dnsmasq } -void timer_start(int i) -{ - if(i >= NUMTIMERS) - { - logg("Code error: Timer %i not defined in timer_start().", i); - exit(EXIT_FAILURE); - } - gettimeofday(&t0[i], 0); -} - -double timer_elapsed_msec(int i) -{ - if(i >= NUMTIMERS) - { - logg("Code error: Timer %i not defined in timer_elapsed_msec().", i); - exit(EXIT_FAILURE); - } - struct timeval t1; - gettimeofday(&t1, 0); - return (t1.tv_sec - t0[i].tv_sec) * 1000.0f + (t1.tv_usec - t0[i].tv_usec) / 1000.0f; -} - -void sleepms(int milliseconds) -{ - struct timeval tv; - tv.tv_sec = milliseconds / 1000; - tv.tv_usec = (milliseconds % 1000) * 1000; - select(0, NULL, NULL, NULL, &tv); -} - void savepid(void) { FILE *f; - pid_t pid = getpid(); + const pid_t pid = getpid(); if((f = fopen(FTLfiles.pid, "w+")) == NULL) { logg("WARNING: Unable to write PID to file."); @@ -143,8 +114,8 @@ char *getUserName(void) char * name; // the getpwuid() function shall search the user database for an entry with a matching uid // the geteuid() function shall return the effective user ID of the calling process - this is used as the search criteria for the getpwuid() function - uid_t euid = geteuid(); - struct passwd *pw = getpwuid(euid); + const uid_t euid = geteuid(); + const struct passwd *pw = getpwuid(euid); if(pw) { name = strdup(pw->pw_name); @@ -157,3 +128,16 @@ char *getUserName(void) return name; } + +void delay_startup(void) +{ + // Exit early if not sleeping + if(config.delay_startup == 0u) + return; + + // Sleep if requested by DELAY_STARTUP + logg("Sleeping for %d seconds as requested by configuration ...", + config.delay_startup); + sleep(config.delay_startup); + logg("Done sleeping, continuing startup of resolver...\n"); +} diff --git a/src/daemon.h b/src/daemon.h new file mode 100644 index 000000000..d94f4cf53 --- /dev/null +++ b/src/daemon.h @@ -0,0 +1,19 @@ +/* Pi-hole: A black hole for Internet advertisements +* (c) 2019 Pi-hole, LLC (https://pi-hole.net) +* Network-wide ad blocking via your own hardware. +* +* FTL Engine +* Daemon prototypes +* +* This file is copyright under the latest version of the EUPL. +* Please see LICENSE file for your rights under this license. */ +#ifndef DAEMON_H +#define DAEMON_H + +void go_daemon(void); +void savepid(void); +char * getUserName(void); +void removepid(void); +void delay_startup(void); + +#endif //DAEMON_H diff --git a/src/database/common.c b/src/database/common.c new file mode 100644 index 000000000..0cc405b6c --- /dev/null +++ b/src/database/common.c @@ -0,0 +1,540 @@ +/* Pi-hole: A black hole for Internet advertisements +* (c) 2017 Pi-hole, LLC (https://pi-hole.net) +* Network-wide ad blocking via your own hardware. +* +* FTL Engine +* Common database routines for pihole-FTL.db +* +* This file is copyright under the latest version of the EUPL. +* Please see LICENSE file for your rights under this license. */ + +#include "FTL.h" +#include "common.h" +#include "shmem.h" +#include "network-table.h" +#include "memory.h" +#include "config.h" +#include "log.h" +#include "timers.h" +#include "files.h" +#include "database/sqlite3-ext.h" + +sqlite3 *FTL_db = NULL; +bool database = true; +bool DBdeleteoldqueries = false; +long int lastdbindex = 0; + +static pthread_mutex_t dblock; + +void dbclose(void) +{ + // Only try to close an existing database connection + int rc = SQLITE_OK; + if( FTL_db != NULL ) + { + rc = sqlite3_close(FTL_db); + FTL_db = NULL; + } + + // Report any error + if( rc != SQLITE_OK ) + { + logg("Encountered error while trying to close database: %s", sqlite3_errstr(rc)); + database = false; + } + + // Unlock mutex on the database + pthread_mutex_unlock(&dblock); +} + +bool dbopen(void) +{ + // Lock mutex on the database + pthread_mutex_lock(&dblock); + + // Try to open database + int rc = sqlite3_open_v2(FTLfiles.FTL_db, &FTL_db, SQLITE_OPEN_READWRITE, NULL); + if( rc != SQLITE_OK ) + { + logg("Encountered error while trying to open database: %s", sqlite3_errstr(rc)); + database = false; + pthread_mutex_unlock(&dblock); + return false; + } + + // Explicitly set busy handler to value defined in FTL.h + rc = sqlite3_busy_timeout(FTL_db, DATABASE_BUSY_TIMEOUT); + if( rc != SQLITE_OK ) + { + logg("Encountered error while trying to set busy timeout (%d ms) on database: %s", + DATABASE_BUSY_TIMEOUT, sqlite3_errstr(rc)); + database = false; + dbclose(); + return false; + } + + return true; +} + +int dbquery(const char *format, ...) +{ + va_list args; + va_start(args, format); + char *query = sqlite3_vmprintf(format, args); + va_end(args); + + if(query == NULL) + { + logg("Memory allocation failed in dbquery()"); + return SQLITE_ERROR; + } + + // Log generated SQL string when dbquery() is called + // although the database connection is not available + if(database == false || ( FTL_db == NULL && !dbopen() )) + { + logg("dbquery(\"%s\") called but database is not available!", query); + sqlite3_free(query); + return SQLITE_ERROR; + } + + if(config.debug & DEBUG_DATABASE) + { + logg("dbquery: \"%s\"", query); + } + + int rc = sqlite3_exec(FTL_db, query, NULL, NULL, NULL); + if( rc != SQLITE_OK ){ + logg("ERROR: SQL query \"%s\" failed: %s", + query, sqlite3_errstr(rc)); + dbclose(); + return rc; + } + + // Free allocated memory for query string + sqlite3_free(query); + + if(config.debug & DEBUG_DATABASE) + { + logg(" ---> OK"); + } + + // Return success + return SQLITE_OK; +} + +static bool create_counter_table(void) +{ + // Create FTL table in the database (holds properties like database version, etc.) + SQL_bool("CREATE TABLE counters ( id INTEGER PRIMARY KEY NOT NULL, value INTEGER NOT NULL );"); + + // ID 0 = total queries + if(!db_set_counter(DB_TOTALQUERIES, 0)) + { dbclose(); return false; } + + // ID 1 = total blocked queries + if(!db_set_counter(DB_BLOCKEDQUERIES, 0)) + { dbclose(); return false; } + + // Time stamp of creation of the counters database + if(!db_set_FTL_property(DB_FIRSTCOUNTERTIMESTAMP, time(NULL))) + { dbclose(); return false; } + + // Update database version to 2 + if(!db_set_FTL_property(DB_VERSION, 2)) + { dbclose(); return false; } + + return true; +} + +static bool db_create(void) +{ + int rc = sqlite3_open_v2(FTLfiles.FTL_db, &FTL_db, SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, NULL); + if( rc != SQLITE_OK ) + { + logg("Encountered error while trying to create database in rw-mode: %s", sqlite3_errstr(rc)); + return false; + } + // Create Queries table in the database + SQL_bool("CREATE TABLE queries ( id INTEGER PRIMARY KEY AUTOINCREMENT, timestamp INTEGER NOT NULL, type INTEGER NOT NULL, status INTEGER NOT NULL, domain TEXT NOT NULL, client TEXT NOT NULL, forward TEXT );"); + + // Add an index on the timestamps (not a unique index!) + SQL_bool("CREATE INDEX idx_queries_timestamps ON queries (timestamp);"); + + // Create FTL table in the database (holds properties like database version, etc.) + SQL_bool("CREATE TABLE ftl ( id INTEGER PRIMARY KEY NOT NULL, value BLOB NOT NULL );"); + + + // Set FTL_db version 1 + if(dbquery("INSERT INTO ftl (ID,VALUE) VALUES(%i,1);", DB_VERSION) != SQLITE_OK) + return false; + + // Most recent timestamp initialized to 00:00 1 Jan 1970 + if(dbquery("INSERT INTO ftl (ID,VALUE) VALUES(%i,0);", DB_LASTTIMESTAMP) != SQLITE_OK) + return false; + + // Create counter table + // Will update FTL_db version to 2 + if(!create_counter_table()) + return false; + + // Create network table + // Will update FTL_db version to 3 + if(!create_network_table()) + return false; + + // Done initializing the database + // Close database handle + dbclose(); + + // Explicitly set permissions to 0644 + // 644 = u+w u+r g+r o+r + const mode_t mode = S_IWUSR | S_IRUSR | S_IRGRP | S_IROTH; + chmod_file(FTLfiles.FTL_db, mode); + + return true; +} + +void SQLite3LogCallback(void *pArg, int iErrCode, const char *zMsg) +{ + // Note: pArg is NULL and not used + // See https://sqlite.org/rescode.html#extrc for details + // concerning the return codes returned here + logg("SQLite3 message: %s (%d)", zMsg, iErrCode); +} + +void db_init(void) +{ + // Initialize database lock mutex + int rc; + if((rc = pthread_mutex_init(&dblock, NULL)) != 0) + { + logg("FATAL: FTL_db mutex init failed (%s, %i)\n", strerror(rc), rc); + // Return failure + exit(EXIT_FAILURE); + } + + // Lock database thread + pthread_mutex_lock(&dblock); + + // Initialize SQLite3 logging callback + // This ensures SQLite3 errors and warnings are logged to pihole-FTL.log + // We use this to possibly catch even more errors in places we do not + // explicitly check for failures to have happened + sqlite3_config(SQLITE_CONFIG_LOG, SQLite3LogCallback, NULL); + + // Register Pi-hole provided SQLite3 extensions (see sqlite3-ext.c) + sqlite3_auto_extension((void (*)(void))sqlite3_pihole_extensions_init); + + // Only exit early if we already finished the SQLite3 library initialization + if(!use_database()) + { + logg("Not using the long-term database"); + pthread_mutex_unlock(&dblock); + database = false; + return; + } + + // Check if database exists, if not create empty database + if(!file_exists(FTLfiles.FTL_db)) + { + logg("No database file found, creating new (empty) database"); + if (!db_create()) + { + logg("Creation of database failed, database is not available"); + pthread_mutex_unlock(&dblock); + database = false; + return; + } + } + + // Try to open the database connection + rc = sqlite3_open_v2(FTLfiles.FTL_db, &FTL_db, SQLITE_OPEN_READWRITE, NULL); + if( rc != SQLITE_OK ){ + logg("Cannot initialize (open) long-term database: %s", sqlite3_errstr(rc)); + database = false; + return; + } + + // Test FTL_db version and see if we need to upgrade the database file + int dbversion = db_get_FTL_property(DB_VERSION); + if(dbversion < 1) + { + logg("Database not available, please ensure the database is unlocked when starting pihole-FTL !"); + dbclose(); + + database = false; + return; + } + else + { + logg("Database version is %i", dbversion); + } + + + // Update to version 2 if lower + if(dbversion < 2) + { + // Update to version 2: Create counters table + logg("Updating long-term database to version 2"); + if (!create_counter_table()) + { + logg("Counter table not initialized, database not available"); + dbclose(); + + database = false; + return; + } + // Get updated version + dbversion = db_get_FTL_property(DB_VERSION); + } + + // Update to version 3 if lower + if(dbversion < 3) + { + // Update to version 3: Create network table + logg("Updating long-term database to version 3"); + if (!create_network_table()) + { + logg("Network table not initialized, database not available"); + dbclose(); + + database = false; + return; + } + // Get updated version + dbversion = db_get_FTL_property(DB_VERSION); + } + + // Update to version 4 if lower + if(dbversion < 4) + { + // Update to version 4: Unify clients in network table + logg("Updating long-term database to version 4"); + if(!unify_hwaddr()) + { + logg("Unable to unify clients in network table, database not available"); + dbclose(); + + database = false; + return; + } + // Get updated version + dbversion = db_get_FTL_property(DB_VERSION); + } + + // Update to version 5 if lower + if(dbversion < 5) + { + // Update to version 5: Create network-addresses table + logg("Updating long-term database to version 5"); + if(!create_network_addresses_table()) + { + logg("Network-addresses table not initialized, database not available"); + dbclose(); + + database = false; + return; + } + // Get updated version + dbversion = db_get_FTL_property(DB_VERSION); + } + + // Close database to prevent having it opened all time + // We already closed the database when we returned earlier + dbclose(); + + logg("Database successfully initialized"); +} + +int db_get_FTL_property(const unsigned int ID) +{ + if(!database || FTL_db == NULL) + { + logg("db_get_FTL_property(%u) called but database is not available!", ID); + return DB_FAILED; + } + // Prepare SQL statement + char* querystr = NULL; + int ret = asprintf(&querystr, "SELECT VALUE FROM ftl WHERE id = %u;", ID); + + if(querystr == NULL || ret < 0) + { + logg("Memory allocation failed in db_get_FTL_property with ID = %u (%i)", ID, ret); + return DB_FAILED; + } + + int value = db_query_int(querystr); + free(querystr); + + return value; +} + +bool db_set_FTL_property(const unsigned int ID, const int value) +{ + if(!database || FTL_db == NULL) + { + logg("db_set_FTL_property(%u, %i) called but database is not available!", ID, value); + return false; + } + return dbquery("INSERT OR REPLACE INTO ftl (id, value) VALUES ( %u, %i );", ID, value) == SQLITE_OK; +} + +bool db_set_counter(const unsigned int ID, const int value) +{ + if(!database || FTL_db == NULL) + { + logg("db_set_counter(%u, %i) called but database is not available!", ID, value); + return false; + } + return dbquery("INSERT OR REPLACE INTO counters (id, value) VALUES ( %u, %i );", ID, value) == SQLITE_OK; +} + +bool db_update_counters(const int total, const int blocked) +{ + if(!database || FTL_db == NULL) + { + logg("db_update_counters(%i, %i) called but database is not available!", total, blocked); + return false; + } + if(dbquery("UPDATE counters SET value = value + %i WHERE id = %i;", total, DB_TOTALQUERIES) != SQLITE_OK) + return false; + if(dbquery("UPDATE counters SET value = value + %i WHERE id = %i;", blocked, DB_BLOCKEDQUERIES) != SQLITE_OK) + return false; + return true; +} + +int db_query_int(const char* querystr) +{ + if(!database || FTL_db == NULL) + { + logg("db_query_int(\"%s\") called but database is not available!", querystr); + return DB_FAILED; + } + + if(config.debug & DEBUG_DATABASE) + { + logg("dbquery: \"%s\"", querystr); + } + + sqlite3_stmt* stmt; + int rc = sqlite3_prepare_v2(FTL_db, querystr, -1, &stmt, NULL); + if( rc != SQLITE_OK ) + { + if( rc != SQLITE_BUSY ) + { + logg("Encountered prepare error in db_query_int(\"%s\"): %s", querystr, sqlite3_errstr(rc)); + database = false; + } + return DB_FAILED; + } + + rc = sqlite3_step(stmt); + int result; + + if( rc == SQLITE_ROW ) + { + result = sqlite3_column_int(stmt, 0); + + if(config.debug & DEBUG_DATABASE) + { + logg(" ---> Result %i (int)", result); + } + } + else if( rc == SQLITE_DONE ) + { + // No rows available + result = DB_NODATA; + + if(config.debug & DEBUG_DATABASE) + { + logg(" ---> No data"); + } + } + else + { + logg("Encountered step error in db_query_int(\"%s\"): %s", querystr, sqlite3_errstr(rc)); + return DB_FAILED; + } + + sqlite3_finalize(stmt); + return result; +} + +long int get_max_query_ID(void) +{ + if(!database || FTL_db == NULL) + { + logg("get_max_query_ID() called but database is not available!"); + return DB_FAILED; + } + + const char *sql = "SELECT MAX(ID) FROM queries"; + if(config.debug & DEBUG_DATABASE) + { + logg("dbquery: \"%s\"", sql); + } + + sqlite3_stmt* stmt = NULL; + int rc = sqlite3_prepare_v2(FTL_db, sql, -1, &stmt, NULL); + if( rc != SQLITE_OK ) + { + if( rc != SQLITE_BUSY ) + { + logg("Encountered prepare error in get_max_query_ID(): %s", sqlite3_errstr(rc)); + database = false; + } + dbclose(); + return DB_FAILED; + } + + rc = sqlite3_step(stmt); + if( rc != SQLITE_ROW ) + { + logg("Encountered step error in get_max_query_ID(): %s", sqlite3_errstr(rc)); + database = false; + dbclose(); + return DB_FAILED; + } + + sqlite3_int64 result = sqlite3_column_int64(stmt, 0); + if(config.debug & DEBUG_DATABASE) + { + logg(" ---> Result %lli (long long int)", (long long int)result); + } + sqlite3_finalize(stmt); + return result; +} + +// Returns ID of the most recent successful INSERT. +long get_lastID(void) +{ + if(!database || FTL_db == NULL) + { + logg("get_lastID() called but database is not available!"); + return DB_FAILED; + } + return sqlite3_last_insert_rowid(FTL_db); +} + +// Return SQLite3 engine version string +const char *get_sqlite3_version(void) +{ + return sqlite3_libversion(); +} + +// Should the long-term database be used? +__attribute__ ((pure)) bool use_database() +{ + // Check if the user doesn't want to use the database and set an + // empty string as file name in FTL's config file or configured + // a maximum history of zero days. + if(FTLfiles.FTL_db == NULL || + strlen(FTLfiles.FTL_db) == 0 || + config.maxDBdays == 0) + { + return false; + } + + return true; +} diff --git a/src/database/common.h b/src/database/common.h new file mode 100644 index 000000000..73273b335 --- /dev/null +++ b/src/database/common.h @@ -0,0 +1,66 @@ +/* Pi-hole: A black hole for Internet advertisements +* (c) 2019 Pi-hole, LLC (https://pi-hole.net) +* Network-wide ad blocking via your own hardware. +* +* FTL Engine +* Database prototypes +* +* This file is copyright under the latest version of the EUPL. +* Please see LICENSE file for your rights under this license. */ +#ifndef DATABASE_COMMON_H +#define DATABASE_COMMON_H + +#include "sqlite3.h" + +void db_init(void); +int db_get_FTL_property(const unsigned int ID); +bool db_set_FTL_property(const unsigned int ID, const int value); + +/// Execute a formatted SQL query and get the return code +int dbquery(const char *format, ...); + +bool dbopen(void); +void dbclose(void); +int db_query_int(const char*); +long get_lastID(void); +void SQLite3LogCallback(void *pArg, int iErrCode, const char *zMsg); +long int get_max_query_ID(void); +bool db_set_counter(const unsigned int ID, const int value); +bool db_update_counters(const int total, const int blocked); +const char *get_sqlite3_version(void); +bool use_database(void) __attribute__ ((pure)); + +extern sqlite3 *FTL_db; +extern bool database; +extern long int lastdbindex; +extern bool DBdeleteoldqueries; + +// Database macros +#define SQL_bool(sql) {\ + int ret;\ + if((ret = dbquery(sql)) != SQLITE_OK) {\ + if(ret == SQLITE_BUSY)\ + logg("WARNING: Database busy in %s()!", __FUNCTION__);\ + else\ + logg("ERROR: %s() failed!", __FUNCTION__);\ + return false;\ + }\ +} + +#define SQL_void(sql) {\ + int ret;\ + if((ret = dbquery(sql)) != SQLITE_OK) {\ + if(ret == SQLITE_BUSY)\ + logg("WARNING: Database busy in %s()!", __FUNCTION__);\ + else\ + logg("ERROR: %s() failed!", __FUNCTION__);\ + return;\ + }\ +} + +// Database table "ftl" +enum { DB_VERSION, DB_LASTTIMESTAMP, DB_FIRSTCOUNTERTIMESTAMP }; +// Database table "counters" +enum { DB_TOTALQUERIES, DB_BLOCKEDQUERIES }; + +#endif //DATABASE_COMMON_H diff --git a/src/database/database-thread.c b/src/database/database-thread.c new file mode 100644 index 000000000..4382d2eb6 --- /dev/null +++ b/src/database/database-thread.c @@ -0,0 +1,74 @@ +/* Pi-hole: A black hole for Internet advertisements +* (c) 2017 Pi-hole, LLC (https://pi-hole.net) +* Network-wide ad blocking via your own hardware. +* +* FTL Engine +* Database thread +* +* This file is copyright under the latest version of the EUPL. +* Please see LICENSE file for your rights under this license. */ + +#include "FTL.h" +#include "database-thread.h" +#include "common.h" +// [un]lock_shm(); +#include "shmem.h" +// parse_neighbor_cache() +#include "network-table.h" +// DB_save_queries() +#include "query-table.h" +#include "config.h" +#include "log.h" +#include "timers.h" +// global variable killed +#include "signals.h" + +void *DB_thread(void *val) +{ + // Set thread name + prctl(PR_SET_NAME,"database",0,0,0); + + // First check if the user doesn't want to use the database. + // If this is the case, we terminate the database thread as + // there is no point in having it around + if(!use_database()) + return NULL; + + // Save timestamp as we do not want to store immediately + // to the database + time_t lastDBsave = time(NULL) - time(NULL)%config.DBinterval; + + while(!killed && database) + { + if(time(NULL) - lastDBsave >= config.DBinterval) + { + // Update lastDBsave timer + lastDBsave = time(NULL) - time(NULL)%config.DBinterval; + + // Lock FTL's data structures, since it is + // likely that they will be changed here + lock_shm(); + + // Save data to database + DB_save_queries(); + + // Release data lock + unlock_shm(); + + // Check if GC should be done on the database + if(DBdeleteoldqueries) + { + // No thread locks needed + delete_old_queries_in_DB(); + DBdeleteoldqueries = false; + } + + // Parse neighbor cache (fill network table) if enabled + if (config.parse_arp_cache) + parse_neighbor_cache(); + } + sleepms(100); + } + + return NULL; +} diff --git a/src/database/database-thread.h b/src/database/database-thread.h new file mode 100644 index 000000000..dffb44d20 --- /dev/null +++ b/src/database/database-thread.h @@ -0,0 +1,15 @@ +/* Pi-hole: A black hole for Internet advertisements +* (c) 2019 Pi-hole, LLC (https://pi-hole.net) +* Network-wide ad blocking via your own hardware. +* +* FTL Engine +* Database thread prototype +* +* This file is copyright under the latest version of the EUPL. +* Please see LICENSE file for your rights under this license. */ +#ifndef DATABASE_THREAD_H +#define DATABASE_THREAD_H + +void *DB_thread(void *val); + +#endif //DATABASE_THREAD_H diff --git a/src/database/gravity-db.c b/src/database/gravity-db.c new file mode 100644 index 000000000..13c51e963 --- /dev/null +++ b/src/database/gravity-db.c @@ -0,0 +1,861 @@ +/* Pi-hole: A black hole for Internet advertisements +* (c) 2017 Pi-hole, LLC (https://pi-hole.net) +* Network-wide ad blocking via your own hardware. +* +* FTL Engine +* Gravity database routines +* +* This file is copyright under the latest version of the EUPL. +* Please see LICENSE file for your rights under this license. */ + +#include "FTL.h" +#include "sqlite3.h" +#include "datastructure.h" +#include "gravity-db.h" +#include "config.h" +#include "log.h" +// global variable counters +#include "memory.h" +// match_regex() +#include "regex_r.h" +// getstr() +#include "shmem.h" +// SQLite3 prepared statement vectors +#include "../vector.h" + +// Process-private prepared statements are used to support multiple forks (might +// be TCP workers) to use the database simultaneously without corrupting the +// gravity database +sqlite3_stmt_vec *whitelist_stmt = NULL; +sqlite3_stmt_vec *gravity_stmt = NULL; +sqlite3_stmt_vec *blacklist_stmt = NULL; + +// Private variables +static sqlite3 *gravity_db = NULL; +static sqlite3_stmt* table_stmt = NULL; +static sqlite3_stmt* auditlist_stmt = NULL; +bool gravityDB_opened = false; +static pid_t main_process = 0, this_process = 0; + +// Table names corresponding to the enum defined in gravity-db.h +static const char* tablename[] = { "vw_gravity", "vw_blacklist", "vw_whitelist", "vw_regex_blacklist", "vw_regex_whitelist" , ""}; + +// Prototypes from functions in dnsmasq's source +void rehash(int size); + +// Initialize gravity subroutines +static void gravityDB_check_fork(void) +{ + // Memorize main process PID on first call of this funtion (guaranteed to be + // the main dnsmasq thread) + if(main_process == 0) + { + main_process = getpid(); + this_process = main_process; + } + + if(this_process == getpid()) + return; + + // If we reach this point, FTL forked to handle TCP connections with + // dedicated (forked) workers SQLite3's mentions that carrying an open + // database connection across a fork() can lead to all kinds of locking + // problems as SQLite3 was not intended to work under such circumstances. + // Doing so may easily lead to ending up with a corrupted database. + logg("Note: FTL forked to handle TCP requests"); + + // Memorize PID of this thread to avoid re-opening the gravity database + // connection multiple times for the same fork + this_process = getpid(); + + // Pretend that we did not open the database so far so it needs to be + // re-opened, also pretend we have not yet prepared the list statements + gravityDB_opened = false; + gravity_db = NULL; + whitelist_stmt = NULL; + blacklist_stmt = NULL; + gravity_stmt = NULL; + gravityDB_open(); +} + +// Open gravity database +bool gravityDB_open(void) +{ + struct stat st; + if(stat(FTLfiles.gravity_db, &st) != 0) + { + // File does not exist + logg("gravityDB_open(): %s does not exist", FTLfiles.gravity_db); + return false; + } + + if(gravityDB_opened && gravity_db != NULL) + { + if(config.debug & DEBUG_DATABASE) + logg("gravityDB_open(): Database already connected"); + return true; + } + + if(config.debug & DEBUG_DATABASE) + logg("gravityDB_open(): Trying to open %s in read-only mode", FTLfiles.gravity_db); + int rc = sqlite3_open_v2(FTLfiles.gravity_db, &gravity_db, SQLITE_OPEN_READONLY, NULL); + if( rc != SQLITE_OK ) + { + logg("gravityDB_open() - SQL error: %s", sqlite3_errstr(rc)); + gravityDB_close(); + return false; + } + + // Database connection is now open + gravityDB_opened = true; + + // Tell SQLite3 to store temporary tables in memory. This speeds up read operations on + // temporary tables, indices, and views. + if(config.debug & DEBUG_DATABASE) + logg("gravityDB_open(): Setting location for temporary object to MEMORY"); + char *zErrMsg = NULL; + rc = sqlite3_exec(gravity_db, "PRAGMA temp_store = MEMORY", NULL, NULL, &zErrMsg); + if( rc != SQLITE_OK ) + { + logg("gravityDB_open(PRAGMA temp_store) - SQL error (%i): %s", rc, zErrMsg); + sqlite3_free(zErrMsg); + gravityDB_close(); + return false; + } + + // Prepare audit statement + if(config.debug & DEBUG_DATABASE) + logg("gravityDB_open(): Preparing audit query"); + + // We support adding audit domains with a wildcard character (*) + // Example 1: google.de + // matches only google.de + // Example 2: *.google.de + // matches all subdomains of google.de + // BUT NOT google.de itself + // Example 3: *google.de + // matches 'google.de' and all of its subdomains but + // also other domains starting in google.de, like + // abcgoogle.de + rc = sqlite3_prepare_v2(gravity_db, + "SELECT EXISTS(" + "SELECT domain, " + "CASE WHEN substr(domain, 1, 1) = '*' " // Does the database string start in '*' ? + "THEN '*' || substr(:input, - length(domain) + 1) " // If so: Crop the input domain and prepend '*' + "ELSE :input " // If not: Use input domain directly for comparison + "END matcher " + "FROM domain_audit WHERE matcher = domain" // Match where (modified) domain equals the database domain + ");", -1, &auditlist_stmt, NULL); + + if( rc != SQLITE_OK ) + { + logg("gravityDB_open(\"SELECT EXISTS(... domain_audit ...)\") - SQL error prepare: %s", sqlite3_errstr(rc)); + gravityDB_close(); + return false; + } + + // Set SQLite3 busy timeout to a user-defined value (defaults to 1 second) + // to avoid immediate failures when the gravity database is still busy + // writing the changes to disk + if(config.debug & DEBUG_DATABASE) + logg("gravityDB_open(): Setting busy timeout to %d", DATABASE_BUSY_TIMEOUT); + sqlite3_busy_timeout(gravity_db, DATABASE_BUSY_TIMEOUT); + + // Prepare private vector of statements for this process (might be a TCP fork!) + if(whitelist_stmt == NULL) + whitelist_stmt = new_sqlite3_stmt_vec(counters->clients); + if(blacklist_stmt == NULL) + blacklist_stmt = new_sqlite3_stmt_vec(counters->clients); + if(gravity_stmt == NULL) + gravity_stmt = new_sqlite3_stmt_vec(counters->clients); + + // Explicitly set busy handler to zero milliseconds + if(config.debug & DEBUG_DATABASE) + logg("gravityDB_open(): Setting busy timeout to zero"); + rc = sqlite3_busy_timeout(gravity_db, 0); + if(rc != SQLITE_OK) + { + logg("gravityDB_open() - Cannot set busy handler: %s", sqlite3_errstr(rc)); + } + + if(config.debug & DEBUG_DATABASE) + logg("gravityDB_open(): Successfully opened gravity.db"); + return true; +} + +static char* get_client_querystr(const char* table, const char* groups) +{ + // Build query string with group filtering + char *querystr = NULL; + if(asprintf(&querystr, "SELECT EXISTS(SELECT domain from %s WHERE domain = ? AND group_id IN (%s));", table, groups) < 1) + { + logg("get_client_querystr(%s, %s) - asprintf() error", table, groups); + return NULL; + } + + if(config.debug & DEBUG_DATABASE) + logg("get_client_querystr: %s", querystr); + + return querystr; +} + +// Get associated groups for this client (if defined) +static bool get_client_groupids(const clientsData* client, char **groups) +{ + char *querystr = NULL; + const char *ip = getstr(client->ippos); + *groups = NULL; + + // Do not proceed when database is not available + if(!gravityDB_opened && !gravityDB_open()) + { + logg("get_client_groupids(): Gravity database not available"); + return false; + } + + if(config.debug & DEBUG_DATABASE) + logg("Querying gravity database for client %s (counting)", ip); + + // Check if client is configured through the client table + if(asprintf(&querystr, "SELECT COUNT(*) FROM client WHERE subnet_match(ip,'%s') = 1;", ip) < 1) + { + logg("get_client_groupids() - asprintf() error 1"); + return false; + } + + // Prepare query + int rc = sqlite3_prepare_v2(gravity_db, querystr, -1, &table_stmt, NULL); + if(rc != SQLITE_OK){ + logg("get_client_groupids(%s) - SQL error prepare: %s", + querystr, sqlite3_errstr(rc)); + free(querystr); + return false; + } + + // Perform query + rc = sqlite3_step(table_stmt); + if(rc == SQLITE_ROW) + { + // There is a record for this client in the database + const int result = sqlite3_column_int(table_stmt, 0); + + // Found no record for this client in the database + // This makes this client qualify for the special "all" group + if(result == 0) + *groups = strdup("0"); + } + else if(rc == SQLITE_DONE) + { + // Found no record for this client in the database + // This makes this client qualify for the special "all" group + *groups = strdup("0"); + } + else + { + logg("get_client_groupids(%s) - SQL error step: %s", + querystr, sqlite3_errstr(rc)); + gravityDB_finalizeTable(); + free(querystr); + return false; + } + + // Finalize statement nad free allocated memory + gravityDB_finalizeTable(); + free(querystr); + querystr = NULL; + + if(*groups != NULL) + { + // The client is not configured through the client table, return early + return true; + } + + // Build query string to get possible group associations for this particular client + // The SQL GROUP_CONCAT() function returns a string which is the concatenation of all + // non-NULL values of group_id separated by ','. The order of the concatenated elements + // is arbitrary, however, is of no relevance for your use case. + // We check using a possibly defined subnet and use the first result + if(asprintf(&querystr, "SELECT GROUP_CONCAT(group_id) FROM client_by_group WHERE client_id = " + "(SELECT id FROM client WHERE subnet_match(ip,'%s') = 1 LIMIT 1);", ip) < 1) + { + logg("get_client_groupids() - asprintf() error 2"); + return false; + } + + if(config.debug & DEBUG_DATABASE) + logg("Querying gravity database for client %s (getting groups)", ip); + + // Prepare query + rc = sqlite3_prepare_v2(gravity_db, querystr, -1, &table_stmt, NULL); + if(rc != SQLITE_OK){ + logg("get_client_groupids(%s) - SQL error prepare: %s", + querystr, sqlite3_errstr(rc)); + sqlite3_finalize(table_stmt); + free(querystr); + return false; + } + + // Perform query + rc = sqlite3_step(table_stmt); + if(rc == SQLITE_ROW) + { + // There is a record for this client in the database + const char* result = (const char*)sqlite3_column_text(table_stmt, 0); + if(result != NULL) + *groups = strdup(result); + else + *groups = strdup(""); + } + else if(rc == SQLITE_DONE) + { + // Found no record for this client in the database + // -> No associated groups + *groups = strdup(""); + } + else + { + logg("get_client_groupids(%s) - SQL error step: %s", + querystr, sqlite3_errstr(rc)); + gravityDB_finalizeTable(); + free(querystr); + return false; + } + // Finalize statement + gravityDB_finalizeTable(); + + // Free allocated memory and return result + free(querystr); + return true; +} + +// Prepare statements for scanning white- and blacklist as well as gravit for one client +bool gravityDB_prepare_client_statements(const int clientID, clientsData *client) +{ + // Return early if gravity database is not available + if(!gravityDB_opened && !gravityDB_open()) + return false; + + const char *clientip = getstr(client->ippos); + + if(config.debug & DEBUG_DATABASE) + logg("Initializing gravity statements for %s", clientip); + + // Get associated groups for this client (if defined) + char *querystr = NULL; + char *groups = NULL; + if(!get_client_groupids(client, &groups)) + return false; + + // Prepare whitelist statement + // We use SELECT EXISTS() as this is known to efficiently use the index + // We are only interested in whether the domain exists or not in the + // list but don't case about duplicates or similar. SELECT EXISTS(...) + // returns true as soon as it sees the first row from the query inside + // of EXISTS(). + if(config.debug & DEBUG_DATABASE) + logg("gravityDB_open(): Preparing vw_whitelist statement for client %s", clientip); + querystr = get_client_querystr("vw_whitelist", groups); + sqlite3_stmt* stmt = NULL; + int rc = sqlite3_prepare_v2(gravity_db, querystr, -1, &stmt, NULL); + if( rc != SQLITE_OK ) + { + logg("gravityDB_open(\"SELECT EXISTS(... vw_whitelist ...)\") - SQL error prepare: %s", sqlite3_errstr(rc)); + gravityDB_close(); + return false; + } + whitelist_stmt->set(whitelist_stmt, clientID, stmt); + free(querystr); + + // Prepare gravity statement + if(config.debug & DEBUG_DATABASE) + logg("gravityDB_open(): Preparing vw_gravity statement for client %s", clientip); + querystr = get_client_querystr("vw_gravity", groups); + rc = sqlite3_prepare_v2(gravity_db, querystr, -1, &stmt, NULL); + if( rc != SQLITE_OK ) + { + logg("gravityDB_open(\"SELECT EXISTS(... vw_gravity ...)\") - SQL error prepare: %s", sqlite3_errstr(rc)); + gravityDB_close(); + return false; + } + gravity_stmt->set(gravity_stmt, clientID, stmt); + free(querystr); + + // Prepare blacklist statement + if(config.debug & DEBUG_DATABASE) + logg("gravityDB_open(): Preparing vw_blacklist statement for client %s", clientip); + querystr = get_client_querystr("vw_blacklist", groups); + rc = sqlite3_prepare_v2(gravity_db, querystr, -1, &stmt, NULL); + if( rc != SQLITE_OK ) + { + logg("gravityDB_open(\"SELECT EXISTS(... vw_blacklist ...)\") - SQL error prepare: %s", sqlite3_errstr(rc)); + gravityDB_close(); + return false; + } + blacklist_stmt->set(blacklist_stmt, clientID, stmt); + free(querystr); + + // Free groups + free(groups); + + return true; +} + +// Finalize non-NULL prepared statements and set them to NULL for a given client +static inline void gravityDB_finalize_client_statements(const int clientID) +{ + if(whitelist_stmt != NULL && + whitelist_stmt->get(whitelist_stmt, clientID) != NULL) + { + sqlite3_finalize(whitelist_stmt->get(whitelist_stmt, clientID)); + whitelist_stmt->set(whitelist_stmt, clientID, NULL); + } + if(blacklist_stmt != NULL && + blacklist_stmt->get(blacklist_stmt, clientID) != NULL) + { + sqlite3_finalize(blacklist_stmt->get(blacklist_stmt, clientID)); + blacklist_stmt->set(blacklist_stmt, clientID, NULL); + } + if(gravity_stmt != NULL && + gravity_stmt->get(gravity_stmt, clientID) != NULL) + { + sqlite3_finalize(gravity_stmt->get(gravity_stmt, clientID)); + gravity_stmt->set(gravity_stmt, clientID, NULL); + } +} + +// Close gravity database connection +void gravityDB_close(void) +{ + // Return early if gravity database is not available + if(!gravityDB_opened) + return; + + // Finalize prepared list statements for all clients + for(int clientID = 0; clientID < counters->clients; clientID++) + { + gravityDB_finalize_client_statements(clientID); + } + + // Free allocated memory for vectors of prepared client statements + free_sqlite3_stmt_vec(whitelist_stmt); + whitelist_stmt = NULL; + free_sqlite3_stmt_vec(blacklist_stmt); + blacklist_stmt = NULL; + free_sqlite3_stmt_vec(gravity_stmt); + gravity_stmt = NULL; + + // Finalize audit list statement + sqlite3_finalize(auditlist_stmt); + auditlist_stmt = NULL; + + // Close table + sqlite3_close(gravity_db); + gravity_db = NULL; + gravityDB_opened = false; +} + +// Prepare a SQLite3 statement which can be used by gravityDB_getDomain() to get +// blocking domains from a table which is specified when calling this function +bool gravityDB_getTable(const unsigned char list) +{ + // First check if FTL forked to handle TCP connections + gravityDB_check_fork(); + + if(!gravityDB_opened && !gravityDB_open()) + { + logg("gravityDB_getTable(%u): Gravity database not available", list); + return false; + } + + // Checking for smaller than GRAVITY_LIST is omitted due to list being unsigned + if(list >= UNKNOWN_TABLE) + { + logg("gravityDB_getTable(%u): Requested list is not known!", list); + return false; + } + + char *querystr = NULL; + // Build correct query string to be used depending on list to be read + // We GROUP BY id as the view also includes the group_id leading to possible duplicates + // when domains are included in more than one group + if(asprintf(&querystr, "SELECT domain, id FROM %s GROUP BY id", tablename[list]) < 18) + { + logg("readGravity(%u) - asprintf() error", list); + return false; + } + + // Prepare SQLite3 statement + int rc = sqlite3_prepare_v2(gravity_db, querystr, -1, &table_stmt, NULL); + if(rc != SQLITE_OK) + { + logg("readGravity(%s) - SQL error prepare: %s", querystr, sqlite3_errstr(rc)); + gravityDB_close(); + free(querystr); + return false; + } + + // Free allocated memory and return success + free(querystr); + return true; +} + +// Get a single domain from a running SELECT operation +// This function returns a pointer to a string as long +// as there are domains available. Once we reached the +// end of the table, it returns NULL. It also returns +// NULL when it encounters an error (e.g., on reading +// errors). Errors are logged to pihole-FTL.log +// This function is performance critical as it might +// be called millions of times for large blocking lists +inline const char* gravityDB_getDomain(int *rowid) +{ + // Perform step + const int rc = sqlite3_step(table_stmt); + + // Valid row + if(rc == SQLITE_ROW) + { + const char* domain = (char*)sqlite3_column_text(table_stmt, 0); + *rowid = sqlite3_column_int(table_stmt, 1); + return domain; + } + + // Check for error. An error happened when the result is neither + // SQLITE_ROW (we returned earlier in this case), nor + // SQLITE_DONE (we are finished reading the table) + if(rc != SQLITE_DONE) + { + logg("gravityDB_getDomain() - SQL error step: %s", sqlite3_errstr(rc)); + *rowid = -1; + return NULL; + } + + // Finished reading, nothing to get here + *rowid = -1; + return NULL; +} + +// Finalize statement of a gravity database transaction +void gravityDB_finalizeTable(void) +{ + if(!gravityDB_opened) + return; + + // Finalize statement + sqlite3_finalize(table_stmt); + table_stmt = NULL; +} + +// Get number of domains in a specified table of the gravity database +// We return the constant DB_FAILED and log to pihole-FTL.log if we +// encounter any error +int gravityDB_count(const unsigned char list) +{ + if(!gravityDB_opened && !gravityDB_open()) + { + logg("gravityDB_count(%d): Gravity database not available", list); + return DB_FAILED; + } + + // Checking for smaller than GRAVITY_LIST is omitted due to list being unsigned + if(list >= UNKNOWN_TABLE) + { + logg("gravityDB_getTable(%u): Requested list is not known!", list); + return false; + } + + char *querystr = NULL; + // Build correct query string to be used depending on list to be read + if(list != GRAVITY_TABLE && asprintf(&querystr, "SELECT COUNT(DISTINCT domain) FROM %s", tablename[list]) < 18) + { + logg("readGravity(%u) - asprintf() error", list); + return false; + } + // We get the number of unique gravity domains as counted and stored by gravity. Counting the number + // of distinct domains in vw_gravity may take up to several minutes for very large blocking lists on + // very low-end devices such as the Raspierry Pi Zero + else if(list == GRAVITY_TABLE && asprintf(&querystr, "SELECT value FROM info WHERE property = 'gravity_count';") < 18) + { + logg("readGravity(%u) - asprintf() error", list); + return false; + } + + if(config.debug & DEBUG_DATABASE) + logg("Querying count of distinct domains in gravity database table %s", tablename[list]); + + // Prepare query + int rc = sqlite3_prepare_v2(gravity_db, querystr, -1, &table_stmt, NULL); + if(rc != SQLITE_OK){ + logg("gravityDB_count(%s) - SQL error prepare %s", querystr, sqlite3_errstr(rc)); + gravityDB_finalizeTable(); + gravityDB_close(); + free(querystr); + return DB_FAILED; + } + + // Perform query + rc = sqlite3_step(table_stmt); + if(rc != SQLITE_ROW){ + logg("gravityDB_count(%s) - SQL error step %s", querystr, sqlite3_errstr(rc)); + if(list == GRAVITY_TABLE) + { + logg("Count of gravity domains not available. Please run pihole -g"); + } + gravityDB_finalizeTable(); + gravityDB_close(); + free(querystr); + return DB_FAILED; + } + + // Get result when there was no error + const int result = sqlite3_column_int(table_stmt, 0); + + // Finalize statement + gravityDB_finalizeTable(); + + // Free allocated memory and return result + free(querystr); + return result; +} + +static bool domain_in_list(const char *domain, sqlite3_stmt* stmt, const char* listname) +{ + // Do not try to bind text to statement when database is not available + if(!gravityDB_opened && !gravityDB_open()) + { + logg("domain_in_list(\"%s\", %p, %s): Gravity database not available", + domain, stmt, listname); + return false; + } + + int rc; + // Bind domain to prepared statement + // SQLITE_STATIC: Use the string without first duplicating it internally. + // We can do this as domain has dynamic scope that exceeds that of the binding. + // We need to bind the domain onl once even to the prepared audit statement as: + // When the same named SQL parameter is used more than once, second and + // subsequent occurrences have the same index as the first occurrence. + // (https://www.sqlite.org/c3ref/bind_blob.html) + if((rc = sqlite3_bind_text(stmt, 1, domain, -1, SQLITE_STATIC)) != SQLITE_OK) + { + logg("domain_in_list(\"%s\", %p, %s): Failed to bind domain: %s", + domain, stmt, listname, sqlite3_errstr(rc)); + return false; + } + + // Perform step + rc = sqlite3_step(stmt); + if(rc == SQLITE_BUSY) + { + // Database is busy + logg("domain_in_list(\"%s\", %p, %s): Database is busy, assuming domain is NOT on list", + domain, stmt, listname); + sqlite3_reset(stmt); + sqlite3_clear_bindings(stmt); + return false; + } + else if(rc != SQLITE_ROW) + { + // Any return code that is neither SQLITE_BUSY not SQLITE_ROW + // is a real error we should log + logg("domain_in_list(\"%s\", %p, %s): Failed to perform step: %s", + domain, stmt, listname, sqlite3_errstr(rc)); + sqlite3_reset(stmt); + sqlite3_clear_bindings(stmt); + return false; + } + + // Get result of query "SELECT EXISTS(...)" + const int result = sqlite3_column_int(stmt, 0); + + if(config.debug & DEBUG_DATABASE) + logg("domain_in_list(\"%s\", %p, %s): %d", domain, stmt, listname, result); + + // The sqlite3_reset() function is called to reset a prepared statement + // object back to its initial state, ready to be re-executed. Note: Any SQL + // statement variables that had values bound to them using the + // sqlite3_bind_*() API retain their values. + sqlite3_reset(stmt); + + // Contrary to the intuition of many, sqlite3_reset() does not reset the + // bindings on a prepared statement. Use this routine to reset all host + // parameters to NULL. + sqlite3_clear_bindings(stmt); + + // Return if domain was found in current table + // SELECT EXISTS(...) either returns 0 (false) or 1 (true). + return (result == 1); +} + +bool in_whitelist(const char *domain, const int clientID, clientsData* client) +{ + // First check if FTL forked to handle TCP connections + gravityDB_check_fork(); + + // If list statement is not ready and cannot be initialized (e.g. no + // access to the database), we return false to prevent an FTL crash + if(whitelist_stmt == NULL) + return false; + + // Get whitelist statement from vector of prepared statements if available + sqlite3_stmt *stmt = whitelist_stmt->get(whitelist_stmt, clientID); + + // If client statement is not ready and cannot be initialized (e.g. no access to + // the database), we return false (not in whitelist) to prevent an FTL crash + if(stmt == NULL && !gravityDB_prepare_client_statements(clientID, client)) + { + logg("ERROR: Gravity database not available, assuming domain is not whitelisted"); + return false; + } + + // Update statement if has just been initialized + if(stmt == NULL) + { + stmt = whitelist_stmt->get(whitelist_stmt, clientID); + } + + // We have to check both the exact whitelist (using a prepared database statement) + // as well the compiled regex whitelist filters to check if the current domain is + // whitelisted. Due to short-circuit-evaluation in C, the regex evaluations is executed + // only if the exact whitelist lookup does not deliver a positive match. This is an + // optimization as the database lookup will most likely hit (a) more domains and (b) + // will be faster (given a sufficiently large number of regex whitelisting filters). + return domain_in_list(domain, stmt, "whitelist") || + match_regex(domain, clientID, REGEX_WHITELIST) != -1; +} + +bool in_gravity(const char *domain, const int clientID, clientsData* client) +{ + // First check if FTL forked to handle TCP connections + gravityDB_check_fork(); + + // If list statement is not ready and cannot be initialized (e.g. no + // access to the database), we return false to prevent an FTL crash + if(gravity_stmt == NULL) + return false; + + // Get whitelist statement from vector of prepared statements + sqlite3_stmt *stmt = gravity_stmt->get(gravity_stmt, clientID); + + // If client statement is not ready and cannot be initialized (e.g. no access to + // the database), we return false (not in gravity list) to prevent an FTL crash + if(stmt == NULL && !gravityDB_prepare_client_statements(clientID, client)) + { + logg("ERROR: Gravity database not available, assuming domain is not gravity blocked"); + return false; + } + + // Update statement if has just been initialized + if(stmt == NULL) + { + stmt = gravity_stmt->get(gravity_stmt, clientID); + } + + return domain_in_list(domain, stmt, "gravity"); +} + +inline bool in_blacklist(const char *domain, const int clientID, clientsData* client) +{ + // First check if FTL forked to handle TCP connections + gravityDB_check_fork(); + + // If list statement is not ready and cannot be initialized (e.g. no + // access to the database), we return false to prevent an FTL crash + if(blacklist_stmt == NULL) + return false; + + // Get whitelist statement from vector of prepared statements + sqlite3_stmt *stmt = blacklist_stmt->get(blacklist_stmt, clientID); + + // If client statement is not ready and cannot be initialized (e.g. no access to + // the database), we return false (not in blacklist) to prevent an FTL crash + if(stmt == NULL && !gravityDB_prepare_client_statements(clientID, client)) + { + logg("ERROR: Gravity database not available, assuming domain is not blacklisted"); + return false; + } + + // Update statement if has just been initialized + if(stmt == NULL) + { + stmt = blacklist_stmt->get(blacklist_stmt, clientID); + } + + return domain_in_list(domain, stmt, "blacklist"); +} + +bool in_auditlist(const char *domain) +{ + // First check if FTL forked to handle TCP connections + gravityDB_check_fork(); + + // If audit list statement is not ready and cannot be initialized (e.g. no access + // to the database), we return false (not in audit list) to prevent an FTL crash + if(auditlist_stmt == NULL) + return false; + + // We check the domain_audit table for the given domain + return domain_in_list(domain, auditlist_stmt, "auditlist"); +} + +bool gravityDB_get_regex_client_groups(clientsData* client, const int numregex, const int *regexid, + const unsigned char type, const char* table, const int clientID) +{ + // First check if FTL forked to handle TCP connections + gravityDB_check_fork(); + + char *querystr = NULL; + char *groups = NULL; + if(!get_client_groupids(client, &groups)) + return false; + + // Group filtering + if(asprintf(&querystr, "SELECT id from %s WHERE group_id IN (%s);", table, groups) < 1) + { + logg("gravityDB_get_regex_client_groups(%s, %s) - asprintf() error", table, groups); + return false; + } + + // Prepare query + sqlite3_stmt *query_stmt; + int rc = sqlite3_prepare_v2(gravity_db, querystr, -1, &query_stmt, NULL); + if(rc != SQLITE_OK){ + logg("gravityDB_get_regex_client_groups(): %s - SQL error prepare: %s", querystr, sqlite3_errstr(rc)); + gravityDB_close(); + free(querystr); + free(groups); + return false; + } + + // Perform query + if(config.debug & DEBUG_REGEX) + logg("Regex %s: Querying groups for client %s: \"%s\"", regextype[type], getstr(client->ippos), querystr); + while((rc = sqlite3_step(query_stmt)) == SQLITE_ROW) + { + const int result = sqlite3_column_int(query_stmt, 0); + for(int regexID = 0; regexID < numregex; regexID++) + { + if(regexid[regexID] == result) + { + if(type == REGEX_WHITELIST) + regexID += counters->num_regex[REGEX_BLACKLIST]; + + set_per_client_regex(clientID, regexID, true); + + if(config.debug & DEBUG_REGEX) + logg("Regex %s: Enabling regex with DB ID %i for client %s", regextype[type], regexid[regexID], getstr(client->ippos)); + + break; + } + } + } + + // Finalize statement + sqlite3_finalize(query_stmt); + + // Free allocated memory and return result + free(querystr); + free(groups); + + return true; +} diff --git a/src/database/gravity-db.h b/src/database/gravity-db.h new file mode 100644 index 000000000..a675e431b --- /dev/null +++ b/src/database/gravity-db.h @@ -0,0 +1,32 @@ +/* Pi-hole: A black hole for Internet advertisements +* (c) 2019 Pi-hole, LLC (https://pi-hole.net) +* Network-wide ad blocking via your own hardware. +* +* FTL Engine +* gravity database prototypes +* +* This file is copyright under the latest version of the EUPL. +* Please see LICENSE file for your rights under this license. */ +#ifndef GRAVITY_H +#define GRAVITY_H + +// Table indices +enum { GRAVITY_TABLE, EXACT_BLACKLIST_TABLE, EXACT_WHITELIST_TABLE, REGEX_BLACKLIST_TABLE, REGEX_WHITELIST_TABLE, UNKNOWN_TABLE }; + +bool gravityDB_open(void); +bool gravityDB_prepare_client_statements(const int clientID, clientsData* client); +void gravityDB_close(void); +bool gravityDB_getTable(unsigned char list); +const char* gravityDB_getDomain(int *rowid); +void gravityDB_finalizeTable(void); +int gravityDB_count(unsigned char list); +bool in_auditlist(const char *domain); + +bool in_gravity(const char *domain, const int clientID, clientsData* client); +bool in_whitelist(const char *domain, const int clientID, clientsData* client); +bool in_blacklist(const char *domain, const int clientID, clientsData* client); + +bool gravityDB_get_regex_client_groups(clientsData* client, const int numregex, const int *regexid, + const unsigned char type, const char* table, const int clientID); + +#endif //GRAVITY_H diff --git a/src/database/network-table.c b/src/database/network-table.c new file mode 100644 index 000000000..9fca10b57 --- /dev/null +++ b/src/database/network-table.c @@ -0,0 +1,944 @@ +/* Pi-hole: A black hole for Internet advertisements +* (c) 2017 Pi-hole, LLC (https://pi-hole.net) +* Network-wide ad blocking via your own hardware. +* +* FTL Engine +* Network table routines +* +* This file is copyright under the latest version of the EUPL. +* Please see LICENSE file for your rights under this license. */ + +#include "FTL.h" +#include "database/network-table.h" +#include "database/common.h" +#include "shmem.h" +#include "memory.h" +#include "log.h" +#include "timers.h" +#include "config.h" +#include "datastructure.h" + +// Private prototypes +static char* getMACVendor(const char* hwaddr); + +bool create_network_table(void) +{ + // Create network table in the database + SQL_bool("CREATE TABLE network ( id INTEGER PRIMARY KEY NOT NULL, " \ + "ip TEXT NOT NULL, " \ + "hwaddr TEXT NOT NULL, " \ + "interface TEXT NOT NULL, " \ + "name TEXT, " \ + "firstSeen INTEGER NOT NULL, " \ + "lastQuery INTEGER NOT NULL, " \ + "numQueries INTEGER NOT NULL, " \ + "macVendor TEXT);"); + + // Update database version to 3 + if(!db_set_FTL_property(DB_VERSION, 3)) + { + logg("create_network_table(): Failed to update database version!"); + return false; + } + + return true; +} + +bool create_network_addresses_table(void) +{ + // Disable foreign key enforcement for this transaction + // Otherwise, dropping the network table would not be allowed + SQL_bool("PRAGMA foreign_keys=OFF"); + + // Begin new transaction + SQL_bool("BEGIN TRANSACTION"); + + // Create network_addresses table in the database + SQL_bool("CREATE TABLE network_addresses ( network_id INTEGER NOT NULL, "\ + "ip TEXT NOT NULL, "\ + "lastSeen INTEGER NOT NULL DEFAULT (cast(strftime('%%s', 'now') as int)), "\ + "UNIQUE(network_id,ip), "\ + "FOREIGN KEY(network_id) REFERENCES network(id));"); + + // Create a network_addresses row for each entry in the network table + // Ignore possible duplicates as they are harmless and can be skipped + SQL_bool("INSERT OR IGNORE INTO network_addresses (network_id,ip) SELECT id,ip FROM network;"); + + // Remove IP column from network table. + // As ALTER TABLE is severely limited, we have to do the column deletion manually. + // Step 1: We create a new table without the ip column + SQL_bool("CREATE TABLE network_bck ( id INTEGER PRIMARY KEY NOT NULL, " \ + "hwaddr TEXT UNIQUE NOT NULL, " \ + "interface TEXT NOT NULL, " \ + "name TEXT, " \ + "firstSeen INTEGER NOT NULL, " \ + "lastQuery INTEGER NOT NULL, " \ + "numQueries INTEGER NOT NULL, " \ + "macVendor TEXT);"); + + // Step 2: Copy data (except ip column) from network into network_back + // The unique constraint on hwaddr is satisfied by grouping results + // by this field where we chose to take only the most recent entry + SQL_bool("INSERT INTO network_bck "\ + "SELECT id, hwaddr, interface, name, firstSeen, "\ + "lastQuery, numQueries, macVendor "\ + "FROM network GROUP BY hwaddr HAVING max(lastQuery);"); + + // Step 3: Drop the network table, the unique index will be automatically dropped + SQL_bool("DROP TABLE network;"); + + // Step 4: Rename network_bck table to network table as last step + SQL_bool("ALTER TABLE network_bck RENAME TO network;"); + + // Update database version to 5 + if(!db_set_FTL_property(DB_VERSION, 5)) + { + logg("create_network_addresses_table(): Failed to update database version!"); + return false; + } + + // Finish transaction + SQL_bool("COMMIT"); + + // Re-enable foreign key enforcement + SQL_bool("PRAGMA foreign_keys=ON"); + + return true; +} + +// Try to find device by recent usage of this IP address +static int find_device_by_recent_ip(const char *ipaddr) +{ + char *querystr = NULL; + int ret = asprintf(&querystr, + "SELECT network_id FROM network_addresses " + "WHERE ip = \'%s\' AND " + "lastSeen > (cast(strftime('%%s', 'now') as int)-86400) " + "ORDER BY lastSeen DESC LIMIT 1;", + ipaddr); + if(querystr == NULL || ret < 0) + { + logg("Memory allocation failed in find_device_by_recent_ip(\"%s\"): %i", + ipaddr, ret); + return -1; + } + + // Perform SQL query + int network_id = db_query_int(querystr); + free(querystr); + + if(network_id == DB_FAILED) + { + // SQLite error + return -1; + } + else if(network_id == DB_NODATA) + { + // No result found + return -1; + } + + if(config.debug & DEBUG_ARP) + logg("APR: Identified device %s using most recently used IP address", ipaddr); + + // Found network_id + return network_id; +} + +// Try to find device by mock hardware address (generated from IP address) +static int find_device_by_mock_hwaddr(const char *ipaddr) +{ + char *querystr = NULL; + int ret = asprintf(&querystr, "SELECT id FROM network WHERE hwaddr = \'ip-%s\';", ipaddr); + if(querystr == NULL || ret < 0) + { + logg("Memory allocation failed in find_device_by_mock_hwaddr(\"%s\"): %i", + ipaddr, ret); + return -1; + } + + // Perform SQL query + int network_id = db_query_int(querystr); + free(querystr); + + return network_id; +} + +// Try to find device by RECENT mock hardware address (generated from IP address) +static int find_recent_device_by_mock_hwaddr(const char *ipaddr) +{ + char *querystr = NULL; + int ret = asprintf(&querystr, + "SELECT id FROM network WHERE " + "hwaddr = \'ip-%s\' AND " + "firstSeen > (cast(strftime('%%s', 'now') as int)-3600);", + ipaddr); + if(querystr == NULL || ret < 0) + { + logg("Memory allocation failed in find_device_by_recent_mock_hwaddr(\"%s\"): %i", + ipaddr, ret); + return -1; + } + + // Perform SQL query + int network_id = db_query_int(querystr); + free(querystr); + + return network_id; +} + +// Store hostname of device identified by dbID +static int update_netDB_hostname(const int dbID, const char *hostname) +{ + // Skip if hostname is NULL or an empty string (= no result) + if(hostname == NULL || strlen(hostname) < 1) + return SQLITE_OK; + + sqlite3_stmt *query_stmt = NULL; + const char querystr[] = "UPDATE network SET name = ? WHERE id = ?;"; + + int rc = sqlite3_prepare_v2(FTL_db, querystr, -1, &query_stmt, NULL); + if(rc != SQLITE_OK) + { + logg("update_netDB_hostname(%i, \"%s\") - SQL error prepare (%i): %s", + dbID, hostname, rc, sqlite3_errmsg(FTL_db)); + return rc; + } + + if(config.debug & DEBUG_DATABASE) + { + logg("dbquery: \"%s\" with arguments 1 = \"%s\" and 2 = %i", querystr, hostname, dbID); + } + + // Bind hostname to prepared statement (1st argument) + // SQLITE_STATIC: Use the string without first duplicating it internally. + // We can do this as hostname has dynamic scope that exceeds that of the binding. + if((rc = sqlite3_bind_text(query_stmt, 1, hostname, -1, SQLITE_STATIC)) != SQLITE_OK) + { + logg("update_netDB_hostname(%i, \"%s\"): Failed to bind hostname (error %d): %s", + dbID, hostname, rc, sqlite3_errmsg(FTL_db)); + sqlite3_reset(query_stmt); + return rc; + } + + // Bind dbID to prepared statement (2nd argument) + if((rc = sqlite3_bind_int(query_stmt, 2, dbID)) != SQLITE_OK) + { + logg("update_netDB_hostname(%i, \"%s\"): Failed to bind dbID (error %d): %s", + dbID, hostname, rc, sqlite3_errmsg(FTL_db)); + sqlite3_reset(query_stmt); + return rc; + } + + // Perform step + sqlite3_step(query_stmt); + sqlite3_finalize(query_stmt); + + return SQLITE_OK; +} + +// Updates lastQuery. Only use new value if larger than zero. +// client->lastQuery may be zero if this client is only known +// from a database entry but has not been seen since then (skip in this case) +static int update_netDB_lastQuery(const int dbID, const clientsData* client) +{ + // Return early if there is nothing to update + if(client->lastQuery < 1) + return SQLITE_OK; + + return dbquery("UPDATE network "\ + "SET lastQuery = MAX(lastQuery, %ld) "\ + "WHERE id = %i;", + client->lastQuery, dbID); +} + + +// Update numQueries. +// Add queries seen since last update and reset counter afterwards +static int update_netDB_numQueries(const int dbID, clientsData* client) +{ + // Return early if there is nothing to update + if(client->numQueriesARP < 1) + return SQLITE_OK; + + int numQueries = client->numQueriesARP; + client->numQueriesARP = 0; + + return dbquery("UPDATE network "\ + "SET numQueries = numQueries + %u "\ + "WHERE id = %i;", + numQueries, dbID); +} + + +// Add unique pair of ID (corresponds to one particular hardware +// address) and IP address if it does not exist (INSERT). In case +// this pair already exists, the UNIQUE(network_id,ip) trigger +// becomes active and the line is instead REPLACEd, causing the +// lastQuery timestamp to be updated +static int add_netDB_network_address(const int dbID, const char* ipaddr) +{ + return dbquery("INSERT OR REPLACE INTO network_addresses "\ + "(network_id,ip,lastSeen) VALUES (%i,\'%s\',(cast(strftime('%%s', 'now') as int)));", + dbID, ipaddr); +} + +// Parse kernel's neighbor cache +void parse_neighbor_cache(void) +{ + // Open database file + if(!dbopen()) + { + logg("parse_neighbor_cache() - Failed to open DB"); + return; + } + + // Try to access the kernel's neighbor cache + // We are only interested in entries which are in either STALE or REACHABLE state + FILE *arpfp = NULL; + const char neigh_command[] = "ip neigh show"; + if((arpfp = popen(neigh_command, "r")) == NULL) + { + logg("WARN: Command \"%s\" failed!", neigh_command); + logg(" Message: %s", strerror(errno)); + dbclose(); + return; + } + + // Start ARP timer + if(config.debug & DEBUG_ARP) + timer_start(ARP_TIMER); + + // Prepare buffers + char *linebuffer = NULL; + size_t linebuffersize = 0u; + char ip[100], hwaddr[100], iface[100]; + unsigned int entries = 0u, additional_entries = 0u; + time_t now = time(NULL); + + const char sql[] = "BEGIN TRANSACTION IMMEDIATE"; + int rc = dbquery(sql); + if( rc != SQLITE_OK ) + { + const char *text; + if( rc == SQLITE_BUSY ) + { + text = "WARNING"; + } + else + { + text = "ERROR"; + // We shall not use the database any longer + database = false; + } + + // dbquery() above already logs the reson for why the query failed + logg("%s: Storing devices in network table (\"%s\") failed", text, sql); + dbclose(); + return; + } + + // Start collecting database commands + lock_shm(); + + // Initialize array of status for individual clients used to + // remember the status of a client already seen in the neigh cache + enum arp_status { CLIENT_NOT_HANDLED, CLIENT_ARP_COMPLETE, CLIENT_ARP_INCOMPLETE }; + enum arp_status client_status[counters->clients]; + for(int i = 0; i < counters->clients; i++) + { + client_status[i] = CLIENT_NOT_HANDLED; + } + + // Read ARP cache line by line + while(getline(&linebuffer, &linebuffersize, arpfp) != -1) + { + // Skip if line buffer is invalid + if(linebuffer == NULL) + continue; + + int num = sscanf(linebuffer, "%99s dev %99s lladdr %99s", + ip, iface, hwaddr); + + // Ensure strings are null-terminated in case we hit the max. + // length limitation + ip[sizeof(ip)-1] = '\0'; + iface[sizeof(iface)-1] = '\0'; + hwaddr[sizeof(hwaddr)-1] = '\0'; + + // Check if we want to process the line we just read + if(num != 3) + { + if(num == 2) + { + // This line is incomplete, remember this to skip + // mock-device creation after ARP processing + int clientID = findClientID(ip, false); + if(clientID >= 0) + client_status[clientID] = CLIENT_ARP_INCOMPLETE; + } + + // Skip to the next row in the neigh cache rather when + // marking as incomplete client + continue; + } + + // Get ID of this device in our network database. If it cannot be + // found, then this is a new device. We only use the hardware address + // to uniquely identify clients and only use the first returned ID. + // + // Same MAC, two IPs: Non-deterministic (sequential) DHCP server, we + // update the IP address to the last seen one. + // + // We can run this SELECT inside the currently active transaction as + // only the changed to the database are collected for latter + // commitment. Read-only access such as this SELECT command will be + // executed immediately on the database. + char* querystr = NULL; + rc = asprintf(&querystr, "SELECT id FROM network WHERE hwaddr = \'%s\';", hwaddr); + if(querystr == NULL || rc < 0) + { + logg("Memory allocation failed in parse_arp_cache(): %i", rc); + break; + } + + // Perform SQL query + int dbID = db_query_int(querystr); + free(querystr); + + if(dbID == DB_FAILED) + { + // Get SQLite error code and return early from loop + rc = sqlite3_errcode(FTL_db); + break; + } + + // If we reach this point, we can check if this client + // is known to pihole-FTL + // false = do not create a new record if the client is + // unknown (only DNS requesting clients do this) + int clientID = findClientID(ip, false); + + // Get hostname of this client if the client is known + const char *hostname = ""; + // Get client pointer + clientsData* client = NULL; + + // This client is known (by its IP address) to pihole-FTL if + // findClientID() returned a non-negative index + if(clientID >= 0) + { + client_status[clientID] = CLIENT_ARP_COMPLETE; + client = getClient(clientID, true); + hostname = getstr(client->namepos); + } + + // Device not in database, add new entry + if(dbID == DB_NODATA) + { + // Check if we recently added a mock-device with the same IP address + // and the ARP entry just came a bit delayed (reported by at least one user) + dbID = find_recent_device_by_mock_hwaddr(ip); + + char* macVendor = getMACVendor(hwaddr); + if(dbID == DB_NODATA) + { + // Device not known AND no recent mock-device found ---> create new device record + if(config.debug & DEBUG_ARP) + { + logg("Device with IP %s not known and " + "no recent mock-device found ---> creating new record", ip); + } + + // Create new record (INSERT) + dbquery("INSERT INTO network " + "(hwaddr,interface,firstSeen,lastQuery,numQueries,name,macVendor) " + "VALUES (\'%s\',\'%s\',%lu, %ld, %u, \'%s\', \'%s\');", + hwaddr, iface, now, + client != NULL ? client->lastQuery : 0L, + client != NULL ? client->numQueriesARP : 0u, + hostname, + macVendor); + + // Reset client ARP counter (we stored the entry in the database) + if(client != NULL) + { + client->numQueriesARP = 0; + } + + // Obtain ID which was given to this new entry + dbID = get_lastID(); + } + else + { + // Device is ALREADY KNOWN ---> convert mock-device to a "real" one + if(config.debug & DEBUG_ARP) + { + logg("Device with IP %s already known (mock-device) " + "---> converting mock-record to real record", ip); + } + + // Update/replace important device properties + dbquery("UPDATE network SET " + "hwaddr = '%s', " + "interface = '%s', " + "macVendor = '%s' " + "WHERE id = %i;", + hwaddr, iface, macVendor, dbID); + // Host name, count and last query timestamp will be set in the next + // loop interation for the sake of simplicity + } + + // Free allocated mememory + free(macVendor); + } + // Device in database AND client known to Pi-hole + else if(client != NULL) + { + // Update timestamp of last query if applicable + rc = update_netDB_lastQuery(dbID, client); + if(rc != SQLITE_OK) + break; + + // Update number of queries if applicable + rc = update_netDB_numQueries(dbID, client); + if(rc != SQLITE_OK) + break; + + // Update hostname if available + rc = update_netDB_hostname(dbID, hostname); + if(rc != SQLITE_OK) + break; + } + // else: + // Device in database but not known to Pi-hole: No action required + + // Add unique IP address / mock-MAC pair to network_addresses table + rc = add_netDB_network_address(dbID, ip); + if(rc != SQLITE_OK) + break; + + // Count number of processed ARP cache entries + entries++; + } + + // Close pipe handle and free allocated memory + pclose(arpfp); + if(linebuffer != NULL) + free(linebuffer); + + // Finally, loop over all clients known to FTL and ensure we add them + // all to the database + for(int clientID = 0; clientID < counters->clients; clientID++) + { + + // Get client pointer + clientsData* client = getClient(clientID, true); + if(client == NULL) + { + if(config.debug & DEBUG_ARP) + logg("Network table: Client %d returned NULL pointer", clientID); + continue; + } + + // Get hostname and IP address of this client + const char *hostname, *ipaddr; + ipaddr = getstr(client->ippos); + hostname = getstr(client->namepos); + + // Skip if this client was inactive (last query may be older than 24 hours) + // This also reduces database I/O when nothing would change anyways + if(client->count < 1 || client->numQueriesARP < 1) + { + if(config.debug & DEBUG_ARP) + logg("Network table: Client %s has zero new queries (count: %d, ARPcount: %d)", + ipaddr, client->count, client->numQueriesARP); + continue; + } + // Skip if already handled above (first check against clients_array_size as we might have added + // more clients to FTL's memory herein (those known only from the database)) + else if(client_status[clientID] != CLIENT_NOT_HANDLED) + { + if(config.debug & DEBUG_ARP) + logg("Network table: Client %s known through ARP/neigh cache", + ipaddr); + continue; + } + else if(config.debug & DEBUG_ARP) + { + logg("Network table: %s NOT known through ARP/neigh cache", ipaddr); + } + + // + // Variant 1: Try to find a device using the same IP address within the last 24 hours + // + int dbID = find_device_by_recent_ip(ipaddr); + + // + // Variant 2: Try to find a device with mock IP address + // + if(dbID < 0) + dbID = find_device_by_mock_hwaddr(ipaddr); + + if(dbID == DB_FAILED) + { + // SQLite error + break; + } + // Device not in database, add new entry + else if(dbID == DB_NODATA) + { + dbquery("INSERT INTO network "\ + "(hwaddr,interface,firstSeen,lastQuery,numQueries,name,macVendor) "\ + "VALUES (\'ip-%s\',\'N/A\',%lu, %ld, %u, \'%s\', \'\');",\ + ipaddr, now, client->lastQuery, client->numQueriesARP, hostname); + client->numQueriesARP = 0; + + if(rc != SQLITE_OK) + break; + + // Obtain ID which was given to this new entry + dbID = get_lastID(); + } + // Device already in database + else + { + // Update timestamp of last query if applicable + rc = update_netDB_lastQuery(dbID, client); + if(rc != SQLITE_OK) + break; + + // Update number of queries if applicable + rc = update_netDB_numQueries(dbID, client); + if(rc != SQLITE_OK) + break; + + // Update hostname if available + rc = update_netDB_hostname(dbID, hostname); + if(rc != SQLITE_OK) + break; + } + + // Add unique IP address / mock-MAC pair to network_addresses table + rc = add_netDB_network_address(dbID, ipaddr); + if(rc != SQLITE_OK) + break; + + // Add to number of processed ARP cache entries + additional_entries++; + } + + // Check for possible error in loop + if(rc != SQLITE_OK) + { + const char *text; + if( rc == SQLITE_BUSY ) + { + text = "WARNING"; + } + else + { + text = "ERROR"; + // We shall not use the database any longer + database = false; + } + + logg("%s: Storing devices in network table failed: %s", text, sqlite3_errstr(rc)); + unlock_shm(); + dbclose(); + return; + } + + // Actually update the database + if((rc = dbquery("END TRANSACTION")) != SQLITE_OK) { + const char *text; + if( rc == SQLITE_BUSY ) + { + text = "WARNING"; + } + else + { + text = "ERROR"; + // We shall not use the database any longer + database = false; + } + + logg("%s: Storing devices in network table failed: %s", text, sqlite3_errstr(rc)); + unlock_shm(); + dbclose(); + return; + } + + // Close database connection + // We opened the connection in this function + dbclose(); + + unlock_shm(); + + // Debug logging + if(config.debug & DEBUG_ARP) + { + logg("ARP table processing (%i entries from ARP, %i from FTL's cache) took %.1f ms", + entries, additional_entries, timer_elapsed_msec(ARP_TIMER)); + } +} + +// Loop over all entries in network table and unify entries by their hwaddr +// If we find duplicates, we keep the most recent entry, while +// - we replace the first-seen date by the earliest across all rows +// - we sum up the number of queries of all clients with the same hwaddr +bool unify_hwaddr(void) +{ + // We request sets of (id,hwaddr). They are GROUPed BY hwaddr to make + // the set unique in hwaddr. + // The grouping is constrained by the HAVING clause which is + // evaluated once across all rows of a group to ensure the returned + // set represents the most recent entry for a given hwaddr + // Get only duplicated hwaddrs here (HAVING cnt > 1). + const char* querystr = "SELECT id,hwaddr,COUNT(*) AS cnt FROM network GROUP BY hwaddr HAVING MAX(lastQuery) AND cnt > 1;"; + + // Perform SQL query + sqlite3_stmt* stmt; + int rc = sqlite3_prepare_v2(FTL_db, querystr, -1, &stmt, NULL); + if( rc != SQLITE_OK ){ + logg("unify_hwaddr(\"%s\") - SQL error prepare: %s", querystr, sqlite3_errstr(rc)); + return false; + } + + // Loop until no further (id,hwaddr) sets are available + while((rc = sqlite3_step(stmt)) != SQLITE_DONE) + { + // Check if we ran into an error + if(rc != SQLITE_ROW) + { + logg("unify_hwaddr(\"%s\") - SQL error step: %s", querystr, sqlite3_errstr(rc)); + dbclose(); + return false; + } + + // Obtain id and hwaddr of the most recent entry for this particular client + const int id = sqlite3_column_int(stmt, 0); + char *hwaddr = strdup((char*)sqlite3_column_text(stmt, 1)); + + // Reset statement + sqlite3_reset(stmt); + + // Update firstSeen with lowest value across all rows with the same hwaddr + dbquery("UPDATE network "\ + "SET firstSeen = (SELECT MIN(firstSeen) FROM network WHERE hwaddr = \'%s\') "\ + "WHERE id = %i;",\ + hwaddr, id); + + // Update numQueries with sum of all rows with the same hwaddr + dbquery("UPDATE network "\ + "SET numQueries = (SELECT SUM(numQueries) FROM network WHERE hwaddr = \'%s\') "\ + "WHERE id = %i;",\ + hwaddr, id); + + // Remove all other lines with the same hwaddr but a different id + dbquery("DELETE FROM network "\ + "WHERE hwaddr = \'%s\' "\ + "AND id != %i;",\ + hwaddr, id); + + free(hwaddr); + } + + // Finalize statement + sqlite3_finalize(stmt); + + // Update database version to 4 + if(!db_set_FTL_property(DB_VERSION, 4)) + return false; + + return true; +} + +static char* getMACVendor(const char* hwaddr) +{ + struct stat st; + if(stat(FTLfiles.macvendor_db, &st) != 0) + { + // File does not exist + if(config.debug & DEBUG_ARP) + logg("getMACVenor(\"%s\"): %s does not exist", hwaddr, FTLfiles.macvendor_db); + return strdup(""); + } + else if(strlen(hwaddr) != 17 || strstr(hwaddr, "ip-") != NULL) + { + // MAC address is incomplete or mock address (for distant clients) + if(config.debug & DEBUG_ARP) + logg("getMACVenor(\"%s\"): MAC invalid (length %zu)", hwaddr, strlen(hwaddr)); + return strdup(""); + } + + sqlite3 *macvendor_db = NULL; + int rc = sqlite3_open_v2(FTLfiles.macvendor_db, &macvendor_db, SQLITE_OPEN_READONLY, NULL); + if( rc != SQLITE_OK ){ + logg("getMACVendor(\"%s\") - SQL error: %s", hwaddr, sqlite3_errstr(rc)); + sqlite3_close(macvendor_db); + return strdup(""); + } + + char *querystr = NULL; + // Only keep "XX:YY:ZZ" (8 characters) + char * hwaddrshort = strdup(hwaddr); + hwaddrshort[8] = '\0'; + rc = asprintf(&querystr, "SELECT vendor FROM macvendor WHERE mac LIKE \'%s\';", hwaddrshort); + if(rc < 1) + { + logg("getMACVendor(\"%s\") - Allocation error (%i)", hwaddr, rc); + sqlite3_close(macvendor_db); + return strdup(""); + } + free(hwaddrshort); + + sqlite3_stmt* stmt = NULL; + rc = sqlite3_prepare_v2(macvendor_db, querystr, -1, &stmt, NULL); + if( rc != SQLITE_OK ){ + logg("getMACVendor(\"%s\") - SQL error prepare \"%s\": %s", hwaddr, querystr, sqlite3_errstr(rc)); + sqlite3_close(macvendor_db); + return strdup(""); + } + free(querystr); + + char *vendor = NULL; + rc = sqlite3_step(stmt); + if(rc == SQLITE_ROW) + { + vendor = strdup((char*)sqlite3_column_text(stmt, 0)); + } + else + { + // Not found + vendor = strdup(""); + } + + if(rc != SQLITE_DONE && rc != SQLITE_ROW) + { + // Error + logg("getMACVendor(\"%s\") - SQL error step: %s", hwaddr, sqlite3_errstr(rc)); + } + + sqlite3_finalize(stmt); + sqlite3_close(macvendor_db); + + return vendor; +} + +void updateMACVendorRecords(void) +{ + struct stat st; + if(stat(FTLfiles.macvendor_db, &st) != 0) + { + // File does not exist + if(config.debug & DEBUG_ARP) + logg("updateMACVendorRecords(): \"%s\" does not exist", FTLfiles.macvendor_db); + return; + } + + // Open database connection + dbopen(); + + sqlite3_stmt* stmt; + const char* selectstr = "SELECT id,hwaddr FROM network;"; + int rc = sqlite3_prepare_v2(FTL_db, selectstr, -1, &stmt, NULL); + if( rc != SQLITE_OK ){ + logg("updateMACVendorRecords() - SQL error prepare \"%s\": %s", selectstr, sqlite3_errstr(rc)); + dbclose(); + return; + } + + while((rc = sqlite3_step(stmt)) == SQLITE_ROW) + { + const int id = sqlite3_column_int(stmt, 0); + char* hwaddr = strdup((char*)sqlite3_column_text(stmt, 1)); + + // Get vendor for MAC + char* vendor = getMACVendor(hwaddr); + free(hwaddr); + hwaddr = NULL; + + // Prepare UPDATE statement + char *updatestr = NULL; + if(asprintf(&updatestr, "UPDATE network SET macVendor = \'%s\' WHERE id = %i", vendor, id) < 1) + { + logg("updateMACVendorRecords() - Allocation error"); + free(vendor); + break; + } + + // Execute prepared statement + char *zErrMsg = NULL; + rc = sqlite3_exec(FTL_db, updatestr, NULL, NULL, &zErrMsg); + if( rc != SQLITE_OK ){ + logg("updateMACVendorRecords() - SQL exec error: \"%s\": %s", updatestr, zErrMsg); + sqlite3_free(zErrMsg); + free(updatestr); + free(vendor); + break; + } + + // Free allocated memory + free(updatestr); + free(vendor); + } + if(rc != SQLITE_DONE) + { + // Error + logg("updateMACVendorRecords() - SQL error step: %s", sqlite3_errstr(rc)); + } + + sqlite3_finalize(stmt); + dbclose(); +} + +char* __attribute__((malloc)) getDatabaseHostname(const char* ipaddr) +{ + // Open pihole-FTL.db database file + if(!dbopen()) + { + logg("getDatabaseHostname(\"%s\") - Failed to open DB", ipaddr); + return strdup(""); + } + + // Prepare SQLite statement + sqlite3_stmt* stmt = NULL; + const char *querystr = "SELECT name FROM network WHERE id = (SELECT network_id FROM network_addresses WHERE ip = ?);"; + int rc = sqlite3_prepare_v2(FTL_db, querystr, -1, &stmt, NULL); + if( rc != SQLITE_OK ){ + logg("getDatabaseHostname(\"%s\") - SQL error prepare: %s", + ipaddr, sqlite3_errstr(rc)); + return strdup(""); + } + + // Bind ipaddr to prepared statement + if((rc = sqlite3_bind_text(stmt, 1, ipaddr, -1, SQLITE_STATIC)) != SQLITE_OK) + { + logg("getDatabaseHostname(\"%s\"): Failed to bind domain: %s", + ipaddr, sqlite3_errstr(rc)); + sqlite3_reset(stmt); + sqlite3_finalize(stmt); + return strdup(""); + } + + char *hostname = NULL; + rc = sqlite3_step(stmt); + if(rc == SQLITE_ROW) + { + // Database record found (result might be empty) + hostname = strdup((char*)sqlite3_column_text(stmt, 0)); + } + else + { + // Not found or error (will be logged automatically through our SQLite3 hook) + hostname = strdup(""); + } + + // Finalize statement and close database handle + sqlite3_reset(stmt); + sqlite3_finalize(stmt); + dbclose(); + + return hostname; +} diff --git a/src/database/network-table.h b/src/database/network-table.h new file mode 100644 index 000000000..9addd0037 --- /dev/null +++ b/src/database/network-table.h @@ -0,0 +1,20 @@ +/* Pi-hole: A black hole for Internet advertisements +* (c) 2019 Pi-hole, LLC (https://pi-hole.net) +* Network-wide ad blocking via your own hardware. +* +* FTL Engine +* pihole-FTL.db -> network tables prototypes +* +* This file is copyright under the latest version of the EUPL. +* Please see LICENSE file for your rights under this license. */ +#ifndef NETWORKTABLE_H +#define NETWORKTABLE_H + +bool create_network_table(void); +bool create_network_addresses_table(void); +void parse_neighbor_cache(void); +void updateMACVendorRecords(void); +bool unify_hwaddr(void); +char* getDatabaseHostname(const char* ipaddr) __attribute__((malloc)); + +#endif //NETWORKTABLE_H diff --git a/src/database/query-table.c b/src/database/query-table.c new file mode 100644 index 000000000..e27c8e507 --- /dev/null +++ b/src/database/query-table.c @@ -0,0 +1,515 @@ +/* Pi-hole: A black hole for Internet advertisements +* (c) 2019 Pi-hole, LLC (https://pi-hole.net) +* Network-wide ad blocking via your own hardware. +* +* FTL Engine +* Query table database routines +* +* This file is copyright under the latest version of the EUPL. +* Please see LICENSE file for your rights under this license. */ + +#include "FTL.h" +#include "query-table.h" +#include "common.h" +// get[Domain,ClientIP,Forward]String(), etc. +#include "datastructure.h" +// getOverTimeID() +#include "overTime.h" +// get_FTL_db_filesize() +#include "files.h" +#include "memory.h" +#include "timers.h" +#include "log.h" +#include "config.h" +// getstr() +#include "shmem.h" + +static bool saving_failed_before = false; + +int get_number_of_queries_in_DB(void) +{ + // This routine is used by the API routines. + // We need to handle opening/closing of the database herein. + if(!dbopen()) + { + return DB_FAILED; + } + + // Count number of rows using the index timestamp is faster than select(*) + int result = db_query_int("SELECT COUNT(timestamp) FROM queries"); + + // Close pihole-FTL.db database connection + dbclose(); + + return result; +} + +void DB_save_queries(void) +{ + // Don't save anything to the database if in PRIVACY_NOSTATS mode + if(config.privacylevel >= PRIVACY_NOSTATS) + return; + + // Start database timer + if(config.debug & DEBUG_DATABASE) + timer_start(DATABASE_WRITE_TIMER); + + // Open database + if(!dbopen()) + { + logg("Failed to open long-term database when trying to store queries"); + return; + } + + unsigned int saved = 0; + bool error = false; + sqlite3_stmt* stmt = NULL; + + int rc = dbquery("BEGIN TRANSACTION IMMEDIATE"); + if( rc != SQLITE_OK ) + { + const char *text; + if( rc == SQLITE_BUSY ) + { + text = "WARNING"; + } + else + { + text = "ERROR"; + // We shall not use the database any longer + database = false; + } + + logg("%s: Storing queries in long-term database failed: %s", text, sqlite3_errstr(rc)); + dbclose(); + return; + } + + rc = sqlite3_prepare_v2(FTL_db, "INSERT INTO queries VALUES (NULL,?,?,?,?,?,?)", -1, &stmt, NULL); + if( rc != SQLITE_OK ) + { + const char *text, *spaces; + if( rc == SQLITE_BUSY ) + { + text = "WARNING"; + spaces = " "; + } + else + { + text = "ERROR"; + spaces = " "; + // We shall not use the database any longer + database = false; + } + + // dbquery() above already logs the reson for why the query failed + logg("%s: Storing queries in long-term database failed: %s\n", text, sqlite3_errstr(rc)); + logg("%s Keeping queries in memory for later new attempt", spaces); + saving_failed_before = true; + dbclose(); + return; + } + + // Get last ID stored in the database + long int lastID = get_max_query_ID(); + + int total = 0, blocked = 0; + time_t currenttimestamp = time(NULL); + time_t newlasttimestamp = 0; + long int queryID; + for(queryID = MAX(0, lastdbindex); queryID < counters->queries; queryID++) + { + queriesData* query = getQuery(queryID, true); + if(query->db != 0) + { + // Skip, already saved in database + continue; + } + + if(!query->complete && query->timestamp > currenttimestamp-2) + { + // Break if a brand new query (age < 2 seconds) is not yet completed + // giving it a chance to be stored next time + break; + } + + if(query->privacylevel >= PRIVACY_MAXIMUM) + { + // Skip, we never store nor count queries recorded + // while have been in maximum privacy mode in the database + continue; + } + + // TIMESTAMP + sqlite3_bind_int(stmt, 1, query->timestamp); + + // TYPE + sqlite3_bind_int(stmt, 2, query->type); + + // STATUS + sqlite3_bind_int(stmt, 3, query->status); + + // DOMAIN + const char *domain = getDomainString(query); + sqlite3_bind_text(stmt, 4, domain, -1, SQLITE_STATIC); + + // CLIENT + const char *client = getClientIPString(query); + sqlite3_bind_text(stmt, 5, client, -1, SQLITE_STATIC); + + // FORWARD + if(query->status == QUERY_FORWARDED && query->upstreamID > -1) + { + // Get forward pointer + const upstreamsData* upstream = getUpstream(query->upstreamID, true); + sqlite3_bind_text(stmt, 6, getstr(upstream->ippos), -1, SQLITE_STATIC); + } + else + { + sqlite3_bind_null(stmt, 6); + } + + // Step and check if successful + rc = sqlite3_step(stmt); + sqlite3_clear_bindings(stmt); + sqlite3_reset(stmt); + + if( rc != SQLITE_DONE ) + { + logg("Encountered error while trying to store queries in long-term database: %s", sqlite3_errstr(rc)); + error = true; + break; + } + + saved++; + // Mark this query as saved in the database by setting the corresponding ID + query->db = ++lastID; + + // Total counter information (delta computation) + total++; + if(query->status == QUERY_GRAVITY || + query->status == QUERY_BLACKLIST || + query->status == QUERY_REGEX || + query->status == QUERY_EXTERNAL_BLOCKED_IP || + query->status == QUERY_EXTERNAL_BLOCKED_NULL || + query->status == QUERY_EXTERNAL_BLOCKED_NXRA || + query->status == QUERY_GRAVITY_CNAME || + query->status == QUERY_REGEX_CNAME || + query->status == QUERY_BLACKLIST_CNAME) + blocked++; + + // Update lasttimestamp variable with timestamp of the latest stored query + if(query->timestamp > newlasttimestamp) + newlasttimestamp = query->timestamp; + } + + if((rc = sqlite3_finalize(stmt)) != SQLITE_OK) + { + logg("Statement finalization failed when trying to store queries to long-term database: %s", + sqlite3_errstr(rc)); + + if( rc == SQLITE_BUSY ) + { + logg("Keeping queries in memory for later new attempt"); + saving_failed_before = true; + } + else + { + database = false; + } + + dbclose(); + return; + } + + // Finish prepared statement + if((rc = dbquery("END TRANSACTION")) != SQLITE_OK) + { + // No need to log the error string here, dbquery() did that already above + logg("END TRANSACTION failed when trying to store queries to long-term database"); + + if( rc == SQLITE_BUSY ) + { + logg("Keeping queries in memory for later new attempt"); + saving_failed_before = true; + } + else + { + database = false; + } + + dbclose(); + return; + } + + // Store index for next loop interation round and update last time stamp + // in the database only if all queries have been saved successfully + if(saved > 0 && !error) + { + lastdbindex = queryID; + db_set_FTL_property(DB_LASTTIMESTAMP, newlasttimestamp); + db_update_counters(total, blocked); + } + + // Close database + dbclose(); + + if(config.debug & DEBUG_DATABASE || saving_failed_before) + { + logg("Notice: Queries stored in long-term database: %u (took %.1f ms, last SQLite ID %li)", saved, timer_elapsed_msec(DATABASE_WRITE_TIMER), lastID); + if(saving_failed_before) + { + logg(" Queries from earlier attempt(s) stored successfully"); + saving_failed_before = false; + } + } +} + +void delete_old_queries_in_DB(void) +{ + // Open database + if(!dbopen()) + { + logg("Failed to open long-term database when trying to delete old queries"); + return; + } + + int timestamp = time(NULL) - config.maxDBdays * 86400; + + if(dbquery("DELETE FROM queries WHERE timestamp <= %i", timestamp) != SQLITE_OK) + { + logg("delete_old_queries_in_DB(): Deleting queries due to age of entries failed!"); + return; + } + + // Get how many rows have been affected (deleted) + const int affected = sqlite3_changes(FTL_db); + + // Print final message only if there is a difference + if((config.debug & DEBUG_DATABASE) || affected) + logg("Notice: Database size is %.2f MB, deleted %i rows", 1e-6*get_FTL_db_filesize(), affected); + + // Close database + dbclose(); +} + +// Get most recent 24 hours data from long-term database +void DB_read_queries(void) +{ + // Don't try to load anything to the database if in PRIVACY_NOSTATS mode + if(config.privacylevel >= PRIVACY_NOSTATS) + return; + + // Open database file + if(!dbopen()) + { + logg("Failed to open long-term database when trying to read queries"); + return; + } + + // Prepare request + // Get time stamp 24 hours in the past + const time_t now = time(NULL); + const time_t mintime = now - config.maxlogage; + char *querystr = NULL; + int rc = asprintf(&querystr, "SELECT * FROM queries WHERE timestamp >= %li", mintime); + if(rc < 42) + { + logg("DB_read_queries() - Memory allocation error: %s", sqlite3_errstr(rc)); + dbclose(); + return; + } + // Log FTL_db query string in debug mode + if(config.debug & DEBUG_DATABASE) + logg("DB_read_queries(): \"%s\"", querystr); + + // Prepare SQLite3 statement + sqlite3_stmt* stmt = NULL; + rc = sqlite3_prepare_v2(FTL_db, querystr, -1, &stmt, NULL); + if( rc != SQLITE_OK ){ + logg("DB_read_queries() - SQL error prepare: %s", sqlite3_errstr(rc)); + dbclose(); + return; + } + + // Loop through returned database rows + while((rc = sqlite3_step(stmt)) == SQLITE_ROW) + { + const sqlite3_int64 dbid = sqlite3_column_int64(stmt, 0); + const time_t queryTimeStamp = sqlite3_column_int(stmt, 1); + // 1483228800 = 01/01/2017 @ 12:00am (UTC) + if(queryTimeStamp < 1483228800) + { + logg("FTL_db warn: TIMESTAMP should be larger than 01/01/2017 but is %li", queryTimeStamp); + continue; + } + if(queryTimeStamp > now) + { + if(config.debug & DEBUG_DATABASE) logg("FTL_db warn: Skipping query logged in the future (%li)", queryTimeStamp); + continue; + } + + const int type = sqlite3_column_int(stmt, 2); + if(type < TYPE_A || type >= TYPE_MAX) + { + logg("FTL_db warn: TYPE should not be %i", type); + continue; + } + // Don't import AAAA queries from database if the user set + // AAAA_QUERY_ANALYSIS=no in pihole-FTL.conf + if(type == TYPE_AAAA && !config.analyze_AAAA) + { + continue; + } + + const int status = sqlite3_column_int(stmt, 3); + if(status < QUERY_UNKNOWN || status >= QUERY_STATUS_MAX) + { + logg("FTL_db warn: STATUS should be within [%i,%i] but is %i", QUERY_UNKNOWN, QUERY_STATUS_MAX-1, status); + continue; + } + + const char * domainname = (const char *)sqlite3_column_text(stmt, 4); + if(domainname == NULL) + { + logg("FTL_db warn: DOMAIN should never be NULL, %li", queryTimeStamp); + continue; + } + + const char * clientIP = (const char *)sqlite3_column_text(stmt, 5); + if(clientIP == NULL) + { + logg("FTL_db warn: CLIENT should never be NULL, %li", queryTimeStamp); + continue; + } + + // Check if user wants to skip queries coming from localhost + if(config.ignore_localhost && + (strcmp(clientIP, "127.0.0.1") == 0 || strcmp(clientIP, "::1") == 0)) + { + continue; + } + + const char *upstream = (const char *)sqlite3_column_text(stmt, 6); + int upstreamID = 0; + // Determine upstreamID only when status == 2 (forwarded) as the + // field need not to be filled for other query status types + if(status == QUERY_FORWARDED) + { + if(upstream == NULL) + { + logg("WARN (during database import): FORWARD should not be NULL with status QUERY_FORWARDED (timestamp: %li), skipping entry", queryTimeStamp); + continue; + } + upstreamID = findUpstreamID(upstream, true); + } + + // Obtain IDs only after filtering which queries we want to keep + const int timeidx = getOverTimeID(queryTimeStamp); + const int domainID = findDomainID(domainname, true); + const int clientID = findClientID(clientIP, true); + + // Ensure we have enough space in the queries struct + memory_check(QUERIES); + + // Set index for this query + const int queryIndex = counters->queries; + + // Store this query in memory + queriesData* query = getQuery(queryIndex, false); + query->magic = MAGICBYTE; + query->timestamp = queryTimeStamp; + query->type = type; + query->status = status; + query->domainID = domainID; + query->clientID = clientID; + query->upstreamID = upstreamID; + query->timeidx = timeidx; + query->db = dbid; + query->id = 0; + query->complete = true; // Mark as all information is available + query->response = 0; + query->dnssec = DNSSEC_UNSPECIFIED; + query->reply = REPLY_UNKNOWN; + query->CNAME_domainID = -1; + + // Set lastQuery timer for network table + clientsData* client = getClient(clientID, true); + client->lastQuery = queryTimeStamp; + + // Handle type counters + if(type >= TYPE_A && type < TYPE_MAX) + { + counters->querytype[type-1]++; + overTime[timeidx].querytypedata[type-1]++; + } + + // Update overTime data + overTime[timeidx].total++; + // Update overTime data structure with the new client + client->overTime[timeidx]++; + + // Increase DNS queries counter + counters->queries++; + + // Increment status counters + switch(status) + { + case QUERY_UNKNOWN: // Unknown + counters->unknown++; + break; + + case QUERY_GRAVITY: // Blocked by gravity + case QUERY_REGEX: // Blocked by regex blacklist + case QUERY_BLACKLIST: // Blocked by exact blacklist + case QUERY_EXTERNAL_BLOCKED_IP: // Blocked by external provider + case QUERY_EXTERNAL_BLOCKED_NULL: // Blocked by external provider + case QUERY_EXTERNAL_BLOCKED_NXRA: // Blocked by external provider + case QUERY_GRAVITY_CNAME: // Blocked by gravity + case QUERY_REGEX_CNAME: // Blocked by regex blacklist + case QUERY_BLACKLIST_CNAME: // Blocked by exact blacklist + counters->blocked++; + // Get domain pointer + domainsData* domain = getDomain(domainID, true); + domain->blockedcount++; + client->blockedcount++; + // Update overTime data structure + overTime[timeidx].blocked++; + break; + + case QUERY_FORWARDED: // Forwarded + counters->forwarded++; + // Update overTime data structure + overTime[timeidx].forwarded++; + break; + + case QUERY_CACHE: // Cached or local config + counters->cached++; + // Update overTime data structure + overTime[timeidx].cached++; + break; + + default: + logg("Error: Found unknown status %i in long term database!", status); + logg(" Timestamp: %li", queryTimeStamp); + logg(" Continuing anyway..."); + break; + } + } + logg("Imported %i queries from the long-term database", counters->queries); + + // Update lastdbindex so that the next call to DB_save_queries() + // skips the queries that we just imported from the database + lastdbindex = counters->queries; + + if( rc != SQLITE_DONE ){ + logg("DB_read_queries() - SQL error step: %s", sqlite3_errstr(rc)); + dbclose(); + return; + } + + // Finalize SQLite3 statement + sqlite3_finalize(stmt); + dbclose(); + free(querystr); +} diff --git a/src/database/query-table.h b/src/database/query-table.h new file mode 100644 index 000000000..2846d850b --- /dev/null +++ b/src/database/query-table.h @@ -0,0 +1,18 @@ +/* Pi-hole: A black hole for Internet advertisements +* (c) 2019 Pi-hole, LLC (https://pi-hole.net) +* Network-wide ad blocking via your own hardware. +* +* FTL Engine +* Query table database prototypes +* +* This file is copyright under the latest version of the EUPL. +* Please see LICENSE file for your rights under this license. */ +#ifndef DATABASE_QUERY_TABLE_H +#define DATABASE_QUERY_TABLE_H + +int get_number_of_queries_in_DB(void); +void delete_old_queries_in_DB(void); +void DB_save_queries(void); +void DB_read_queries(void); + +#endif //DATABASE_QUERY_TABLE_H diff --git a/src/database/sqlite3-ext.c b/src/database/sqlite3-ext.c new file mode 100644 index 000000000..30e8bd65d --- /dev/null +++ b/src/database/sqlite3-ext.c @@ -0,0 +1,154 @@ +/* Pi-hole: A black hole for Internet advertisements +* (c) 2020 Pi-hole, LLC (https://pi-hole.net) +* Network-wide ad blocking via your own hardware. +* +* FTL Engine +* SQLite3 database engine extensions +* +* This file is copyright under the latest version of the EUPL. +* Please see LICENSE file for your rights under this license. */ + +#include "database/sqlite3.h" +#include "database/sqlite3-ext.h" + +// inet_pton +#include +// sscanf() +#include +// type bool +#include +// strstr() +#include +// free() +#include +// logg() +#include "log.h" +// struct config +#include "config.h" + +static void subnet_match_impl(sqlite3_context *context, int argc, sqlite3_value **argv) +{ + // Exactly two arguments should be submitted to this routine + if(argc != 2) + { + sqlite3_result_error(context, "Passed an invalid number of arguments", -1); + return; + } + + // Return NO MATCH if invoked with non-TEXT arguments + if (sqlite3_value_type(argv[0]) != SQLITE_TEXT || + sqlite3_value_type(argv[1]) != SQLITE_TEXT) + { + logg("Invoked subnet_match() with non-text arguments: %d, %d", + sqlite3_value_type(argv[0]), sqlite3_value_type(argv[1])); + sqlite3_result_int(context, 0); + return; + } + + // Analyze input supplied to our SQLite subroutine + // From the DB side (first argument) ... + const char *addrDBcidr = (const char*)sqlite3_value_text(argv[0]); + bool isIPv6_DB = strchr(addrDBcidr, ':') != NULL; + // ... and from FTL's side (second argument) + const char *addrFTL = (const char*)sqlite3_value_text(argv[1]); + bool isIPv6_FTL = strchr(addrFTL, ':') != NULL; + + // Return early (no match) if IP types are different + // We can skip all computations in this case + if(isIPv6_DB != isIPv6_FTL) + { + sqlite3_result_int(context, 0); + return; + } + + // Extract possible CIDR from database IP string + int cidr = isIPv6_DB ? 128 : 32; + char *addrDB = NULL; + // sscanf() will not overwrite the pre-defined CIDR in cidr if + // no CIDR is specified in the database + sscanf(addrDBcidr, "%m[^/]/%i", &addrDB, &cidr); + + // Convert the Internet host address into binary form in network byte order + // We use in6_addr as variable type here as it is guaranteed to be large enough + // for both, IPv4 and IPv6 addresses (128 bits variable size). + struct in6_addr saddrDB = {{{ 0 }}}, saddrFTL = {{{ 0 }}}; + if (inet_pton(isIPv6_DB ? AF_INET6 : AF_INET, addrDB, &saddrDB) == 0) + { + //sqlite3_result_error(context, "Passed a malformed IP address (database)", -1); + // Return non-fatal "NO MATCH" if address is invalid + logg("Passed a malformed DB IP address: %s/%i (%s)", addrDB, cidr, addrDBcidr); + free(addrDB); + sqlite3_result_int(context, 0); + return; + } + + // Free allocated memory + free(addrDB); + addrDB = NULL; + + // Check and convert client IP address as seen by FTL + if (inet_pton(isIPv6_FTL ? AF_INET6 : AF_INET, addrFTL, &saddrFTL) == 0) + { + //sqlite3_result_error(context, "Passed a malformed IP address (FTL)", -1); + // Return non-fatal "NO MATCH" if address is invalid + logg("Passed a malformed FTL IP address: %s", addrFTL); + sqlite3_result_int(context, 0); + return; + } + + // Construct binary mask from CIDR field + uint8_t bitmask[16] = { 0 }; + for(int i = 0; i < cidr; i++) + { + bitmask[i/8] |= (1 << (7-(i%8))); + } + + // Apply bitmask to both IP addresses + // Note: the upper 12 byte of IPv4 addresses are zero + int match = 1; + for(unsigned int i = 0u; i < 16u; i++) + { + saddrDB.s6_addr[i] &= bitmask[i]; + saddrFTL.s6_addr[i] &= bitmask[i]; + + // Are the addresses different given the applied mask? + if(saddrDB.s6_addr[i] != saddrFTL.s6_addr[i]) + { + match = 0; + break; + } + } + + // Possible debug logging + if(config.debug & DEBUG_DATABASE) + { + char subnet[INET6_ADDRSTRLEN]; + inet_ntop(isIPv6_FTL ? AF_INET6 : AF_INET, &bitmask, subnet, sizeof(subnet)); + logg("SQL: Comparing %s vs. %s (subnet %s) - %s", + addrFTL, addrDBcidr, subnet, + match == 1 ? "!! MATCH !!" : "NO MATCH"); + } + + // Return if we found a match between the two addresses + // given a possibly specified mask + sqlite3_result_int(context, match); +} + +int sqlite3_pihole_extensions_init(sqlite3 *db, const char **pzErrMsg, const struct sqlite3_api_routines *pApi) +{ + (void)pzErrMsg; /* Unused parameter */ + + // Register new sqlite function subnet_match taking 2 arguments in UTF8 format. + // The function is deterministic in the sense of always returning the same output for the same input. + // We define a scalar function here so the last two pointers are NULL. + int rc = sqlite3_create_function(db, "subnet_match", 2, SQLITE_UTF8 | SQLITE_DETERMINISTIC, NULL, + subnet_match_impl, NULL, NULL); + + if(rc != SQLITE_OK) + { + logg("Error while initializing the SQLite3 extension subnet_match: %s", + sqlite3_errstr(rc)); + } + + return rc; +} \ No newline at end of file diff --git a/src/database/sqlite3-ext.h b/src/database/sqlite3-ext.h new file mode 100644 index 000000000..2636eb5d8 --- /dev/null +++ b/src/database/sqlite3-ext.h @@ -0,0 +1,12 @@ +/* Pi-hole: A black hole for Internet advertisements +* (c) 2020 Pi-hole, LLC (https://pi-hole.net) +* Network-wide ad blocking via your own hardware. +* +* FTL Engine +* SQLite3 database engine extension prototypes +* +* This file is copyright under the latest version of the EUPL. +* Please see LICENSE file for your rights under this license. */ + +// Initialization point for SQLite3 extensions +extern int sqlite3_pihole_extensions_init(sqlite3 *db, const char **pzErrMsg, const struct sqlite3_api_routines *pApi); diff --git a/sqlite3.c b/src/database/sqlite3.c similarity index 95% rename from sqlite3.c rename to src/database/sqlite3.c index 70e84b589..8fd740b30 100644 --- a/sqlite3.c +++ b/src/database/sqlite3.c @@ -1,6 +1,6 @@ /****************************************************************************** ** This file is an amalgamation of many separate C source files from SQLite -** version 3.27.1. By combining all the individual C code files into this +** version 3.30.1. By combining all the individual C code files into this ** single large file, the entire code can be compiled as a single translation ** unit. This allows many compilers to do optimizations that would not be ** possible if the files were compiled separately. Performance improvements @@ -39,7 +39,7 @@ ** SQLite was built with. */ -#ifndef SQLITE_OMIT_COMPILEOPTION_DIAGS +#ifndef SQLITE_OMIT_COMPILEOPTION_DIAGS /* IMP: R-16824-07538 */ /* ** Include the configuration header output by 'configure' if we're using the @@ -331,8 +331,6 @@ static const char * const sqlite3azCompileOpt[] = { #endif #if defined(SQLITE_ENABLE_STAT4) "ENABLE_STAT4", -#elif defined(SQLITE_ENABLE_STAT3) - "ENABLE_STAT3", #endif #if SQLITE_ENABLE_STMTVTAB "ENABLE_STMTVTAB", @@ -888,6 +886,11 @@ SQLITE_PRIVATE const char **sqlite3CompileOptions(int *pnOpt){ #pragma warning(disable : 4706) #endif /* defined(_MSC_VER) */ +#if defined(_MSC_VER) && !defined(_WIN64) +#undef SQLITE_4_BYTE_ALIGNED_MALLOC +#define SQLITE_4_BYTE_ALIGNED_MALLOC +#endif /* defined(_MSC_VER) && !defined(_WIN64) */ + #endif /* SQLITE_MSVC_H */ /************** End of msvc.h ************************************************/ @@ -1162,9 +1165,9 @@ extern "C" { ** [sqlite3_libversion_number()], [sqlite3_sourceid()], ** [sqlite_version()] and [sqlite_source_id()]. */ -#define SQLITE_VERSION "3.27.1" -#define SQLITE_VERSION_NUMBER 3027001 -#define SQLITE_SOURCE_ID "2019-02-08 13:17:39 0eca3dd3d38b31c92b49ca2d311128b74584714d9e7de895b1a6286ef959a1dd" +#define SQLITE_VERSION "3.30.1" +#define SQLITE_VERSION_NUMBER 3030001 +#define SQLITE_SOURCE_ID "2019-10-10 20:19:45 18db032d058f1436ce3dea84081f4ee5a0f2259ad97301d43c426bc7f3df1b0b" /* ** CAPI3REF: Run-Time Library Version Numbers @@ -1228,6 +1231,9 @@ SQLITE_API int sqlite3_libversion_number(void); #ifndef SQLITE_OMIT_COMPILEOPTION_DIAGS SQLITE_API int sqlite3_compileoption_used(const char *zOptName); SQLITE_API const char *sqlite3_compileoption_get(int N); +#else +# define sqlite3_compileoption_used(X) 0 +# define sqlite3_compileoption_get(X) ((void*)0) #endif /* @@ -2332,8 +2338,14 @@ typedef struct sqlite3_api_routines sqlite3_api_routines; ** ^The flags argument to xAccess() may be [SQLITE_ACCESS_EXISTS] ** to test for the existence of a file, or [SQLITE_ACCESS_READWRITE] to ** test whether a file is readable and writable, or [SQLITE_ACCESS_READ] -** to test whether a file is at least readable. The file can be a -** directory. +** to test whether a file is at least readable. The SQLITE_ACCESS_READ +** flag is never actually used and is not implemented in the built-in +** VFSes of SQLite. The file is named by the second argument and can be a +** directory. The xAccess method returns [SQLITE_OK] on success or some +** non-zero error code if there is an I/O error or if the name of +** the file given in the second argument is illegal. If SQLITE_OK +** is returned, then non-zero or zero is written into *pResOut to indicate +** whether or not the file is accessible. ** ** ^SQLite will always allocate at least mxPathname+1 bytes for the ** output buffer xFullPathname. The exact size of the output buffer @@ -3123,10 +3135,21 @@ struct sqlite3_mem_methods { ** following this call. The second parameter may be a NULL pointer, in ** which case the trigger setting is not reported back. ** +** [[SQLITE_DBCONFIG_ENABLE_VIEW]] +**
SQLITE_DBCONFIG_ENABLE_VIEW
+**
^This option is used to enable or disable [CREATE VIEW | views]. +** There should be two additional arguments. +** The first argument is an integer which is 0 to disable views, +** positive to enable views or negative to leave the setting unchanged. +** The second parameter is a pointer to an integer into which +** is written 0 or 1 to indicate whether views are disabled or enabled +** following this call. The second parameter may be a NULL pointer, in +** which case the view setting is not reported back.
+** ** [[SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER]] **
SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER
-**
^This option is used to enable or disable the two-argument -** version of the [fts3_tokenizer()] function which is part of the +**
^This option is used to enable or disable the +** [fts3_tokenizer()] function which is part of the ** [FTS3] full-text search engine extension. ** There should be two additional arguments. ** The first argument is an integer which is 0 to disable fts3_tokenizer() or @@ -3234,10 +3257,50 @@ struct sqlite3_mem_methods { ** features include but are not limited to the following: **
    **
  • The [PRAGMA writable_schema=ON] statement. +**
  • The [PRAGMA journal_mode=OFF] statement. **
  • Writes to the [sqlite_dbpage] virtual table. **
  • Direct writes to [shadow tables]. **
**
+** +** [[SQLITE_DBCONFIG_WRITABLE_SCHEMA]]
SQLITE_DBCONFIG_WRITABLE_SCHEMA
+**
The SQLITE_DBCONFIG_WRITABLE_SCHEMA option activates or deactivates the +** "writable_schema" flag. This has the same effect and is logically equivalent +** to setting [PRAGMA writable_schema=ON] or [PRAGMA writable_schema=OFF]. +** The first argument to this setting is an integer which is 0 to disable +** the writable_schema, positive to enable writable_schema, or negative to +** leave the setting unchanged. The second parameter is a pointer to an +** integer into which is written 0 or 1 to indicate whether the writable_schema +** is enabled or disabled following this call. +**
+** +** [[SQLITE_DBCONFIG_LEGACY_ALTER_TABLE]] +**
SQLITE_DBCONFIG_LEGACY_ALTER_TABLE
+**
The SQLITE_DBCONFIG_LEGACY_ALTER_TABLE option activates or deactivates +** the legacy behavior of the [ALTER TABLE RENAME] command such it +** behaves as it did prior to [version 3.24.0] (2018-06-04). See the +** "Compatibility Notice" on the [ALTER TABLE RENAME documentation] for +** additional information. This feature can also be turned on and off +** using the [PRAGMA legacy_alter_table] statement. +**
+** +** [[SQLITE_DBCONFIG_DQS_DML]] +**
SQLITE_DBCONFIG_DQS_DML +**
The SQLITE_DBCONFIG_DQS_DML option activates or deactivates +** the legacy [double-quoted string literal] misfeature for DML statement +** only, that is DELETE, INSERT, SELECT, and UPDATE statements. The +** default value of this setting is determined by the [-DSQLITE_DQS] +** compile-time option. +**
+** +** [[SQLITE_DBCONFIG_DQS_DDL]] +**
SQLITE_DBCONFIG_DQS_DDL +**
The SQLITE_DBCONFIG_DQS option activates or deactivates +** the legacy [double-quoted string literal] misfeature for DDL statements, +** such as CREATE TABLE and CREATE INDEX. The +** default value of this setting is determined by the [-DSQLITE_DQS] +** compile-time option. +**
** */ #define SQLITE_DBCONFIG_MAINDBNAME 1000 /* const char* */ @@ -3251,7 +3314,12 @@ struct sqlite3_mem_methods { #define SQLITE_DBCONFIG_TRIGGER_EQP 1008 /* int int* */ #define SQLITE_DBCONFIG_RESET_DATABASE 1009 /* int int* */ #define SQLITE_DBCONFIG_DEFENSIVE 1010 /* int int* */ -#define SQLITE_DBCONFIG_MAX 1010 /* Largest DBCONFIG */ +#define SQLITE_DBCONFIG_WRITABLE_SCHEMA 1011 /* int int* */ +#define SQLITE_DBCONFIG_LEGACY_ALTER_TABLE 1012 /* int int* */ +#define SQLITE_DBCONFIG_DQS_DML 1013 /* int int* */ +#define SQLITE_DBCONFIG_DQS_DDL 1014 /* int int* */ +#define SQLITE_DBCONFIG_ENABLE_VIEW 1015 /* int int* */ +#define SQLITE_DBCONFIG_MAX 1015 /* Largest DBCONFIG */ /* ** CAPI3REF: Enable Or Disable Extended Result Codes @@ -3408,7 +3476,7 @@ SQLITE_API int sqlite3_changes(sqlite3*); ** not. ^Changes to a view that are intercepted by INSTEAD OF triggers ** are not counted. ** -** This the [sqlite3_total_changes(D)] interface only reports the number +** The [sqlite3_total_changes(D)] interface only reports the number ** of rows that changed due to SQL statement run against database ** connection D. Any changes by other database connections are ignored. ** To detect changes against a database file from other database @@ -4800,7 +4868,7 @@ SQLITE_API int sqlite3_limit(sqlite3*, int id, int newVal); ** ^The specific value of WHERE-clause [parameter] might influence the ** choice of query plan if the parameter is the left-hand side of a [LIKE] ** or [GLOB] operator or if the parameter is compared to an indexed column -** and the [SQLITE_ENABLE_STAT3] compile-time option is enabled. +** and the [SQLITE_ENABLE_STAT4] compile-time option is enabled. ** ** ** @@ -4933,6 +5001,18 @@ SQLITE_API const char *sqlite3_normalized_sql(sqlite3_stmt *pStmt); */ SQLITE_API int sqlite3_stmt_readonly(sqlite3_stmt *pStmt); +/* +** CAPI3REF: Query The EXPLAIN Setting For A Prepared Statement +** METHOD: sqlite3_stmt +** +** ^The sqlite3_stmt_isexplain(S) interface returns 1 if the +** prepared statement S is an EXPLAIN statement, or 2 if the +** statement S is an EXPLAIN QUERY PLAN. +** ^The sqlite3_stmt_isexplain(S) interface returns 0 if S is +** an ordinary statement or a NULL pointer. +*/ +SQLITE_API int sqlite3_stmt_isexplain(sqlite3_stmt *pStmt); + /* ** CAPI3REF: Determine If A Prepared Statement Has Been Reset ** METHOD: sqlite3_stmt @@ -5072,7 +5152,9 @@ typedef struct sqlite3_context sqlite3_context; ** ^The fifth argument to the BLOB and string binding interfaces ** is a destructor used to dispose of the BLOB or ** string after SQLite has finished with it. ^The destructor is called -** to dispose of the BLOB or string even if the call to bind API fails. +** to dispose of the BLOB or string even if the call to the bind API fails, +** except the destructor is not called if the third parameter is a NULL +** pointer or the fourth parameter is negative. ** ^If the fifth argument is ** the special value [SQLITE_STATIC], then SQLite assumes that the ** information is in static, unmanaged space and does not need to be freed. @@ -5821,6 +5903,12 @@ SQLITE_API int sqlite3_reset(sqlite3_stmt *pStmt); ** perform additional optimizations on deterministic functions, so use ** of the [SQLITE_DETERMINISTIC] flag is recommended where possible. ** +** ^The fourth parameter may also optionally include the [SQLITE_DIRECTONLY] +** flag, which if present prevents the function from being invoked from +** within VIEWs or TRIGGERs. For security reasons, the [SQLITE_DIRECTONLY] +** flag is recommended for any application-defined SQL function that has +** side-effects. +** ** ^(The fifth parameter is an arbitrary pointer. The implementation of the ** function can gain access to this pointer using [sqlite3_user_data()].)^ ** @@ -5937,8 +6025,30 @@ SQLITE_API int sqlite3_create_window_function( ** [SQLITE_UTF8 | preferred text encoding] as the fourth argument ** to [sqlite3_create_function()], [sqlite3_create_function16()], or ** [sqlite3_create_function_v2()]. +** +** The SQLITE_DETERMINISTIC flag means that the new function will always +** maps the same inputs into the same output. The abs() function is +** deterministic, for example, but randomblob() is not. +** +** The SQLITE_DIRECTONLY flag means that the function may only be invoked +** from top-level SQL, and cannot be used in VIEWs or TRIGGERs. This is +** a security feature which is recommended for all +** [application-defined SQL functions] that have side-effects. This flag +** prevents an attacker from adding triggers and views to a schema then +** tricking a high-privilege application into causing unintended side-effects +** while performing ordinary queries. +** +** The SQLITE_SUBTYPE flag indicates to SQLite that a function may call +** [sqlite3_value_subtype()] to inspect the sub-types of its arguments. +** Specifying this flag makes no difference for scalar or aggregate user +** functions. However, if it is not specified for a user-defined window +** function, then any sub-types belonging to arguments passed to the window +** function may be discarded before the window function is called (i.e. +** sqlite3_value_subtype() will always return 0). */ -#define SQLITE_DETERMINISTIC 0x800 +#define SQLITE_DETERMINISTIC 0x000000800 +#define SQLITE_DIRECTONLY 0x000080000 +#define SQLITE_SUBTYPE 0x000100000 /* ** CAPI3REF: Deprecated Functions @@ -5989,6 +6099,8 @@ SQLITE_API SQLITE_DEPRECATED int sqlite3_memory_alarm(void(*)(void*,sqlite3_int6 ** sqlite3_value_nochange   ** →  True if the column is unchanged in an UPDATE ** against a virtual table. +** sqlite3_value_frombind   +** →  True if value originated from a [bound parameter] ** ** ** Details: @@ -6050,6 +6162,11 @@ SQLITE_API SQLITE_DEPRECATED int sqlite3_memory_alarm(void(*)(void*,sqlite3_int6 ** than within an [xUpdate] method call for an UPDATE statement, then ** the return value is arbitrary and meaningless. ** +** ^The sqlite3_value_frombind(X) interface returns non-zero if the +** value X originated from one of the [sqlite3_bind_int|sqlite3_bind()] +** interfaces. ^If X comes from an SQL literal value, or a table column, +** and expression, then sqlite3_value_frombind(X) returns zero. +** ** Please pay particular attention to the fact that the pointer returned ** from [sqlite3_value_blob()], [sqlite3_value_text()], or ** [sqlite3_value_text16()] can be invalidated by a subsequent call to @@ -6095,6 +6212,7 @@ SQLITE_API int sqlite3_value_bytes16(sqlite3_value*); SQLITE_API int sqlite3_value_type(sqlite3_value*); SQLITE_API int sqlite3_value_numeric_type(sqlite3_value*); SQLITE_API int sqlite3_value_nochange(sqlite3_value*); +SQLITE_API int sqlite3_value_frombind(sqlite3_value*); /* ** CAPI3REF: Finding The Subtype Of SQL Values @@ -6830,7 +6948,7 @@ SQLITE_API sqlite3 *sqlite3_db_handle(sqlite3_stmt*); ** associated with database N of connection D. ^The main database file ** has the name "main". If there is no attached database N on the database ** connection D, or if database N is a temporary or in-memory database, then -** a NULL pointer is returned. +** this function will return either a NULL pointer or an empty string. ** ** ^The filename returned by this function is the output of the ** xFullPathname method of the [VFS]. ^In other words, the filename @@ -7576,6 +7694,12 @@ struct sqlite3_index_info { ** ^The sqlite3_create_module() ** interface is equivalent to sqlite3_create_module_v2() with a NULL ** destructor. +** +** ^If the third parameter (the pointer to the sqlite3_module object) is +** NULL then no new module is create and any existing modules with the +** same name are dropped. +** +** See also: [sqlite3_drop_modules()] */ SQLITE_API int sqlite3_create_module( sqlite3 *db, /* SQLite connection to register module with */ @@ -7591,6 +7715,23 @@ SQLITE_API int sqlite3_create_module_v2( void(*xDestroy)(void*) /* Module destructor function */ ); +/* +** CAPI3REF: Remove Unnecessary Virtual Table Implementations +** METHOD: sqlite3 +** +** ^The sqlite3_drop_modules(D,L) interface removes all virtual +** table modules from database connection D except those named on list L. +** The L parameter must be either NULL or a pointer to an array of pointers +** to strings where the array is terminated by a single NULL pointer. +** ^If the L parameter is NULL, then all virtual table modules are removed. +** +** See also: [sqlite3_create_module()] +*/ +SQLITE_API int sqlite3_drop_modules( + sqlite3 *db, /* Remove modules from this connection */ + const char **azKeep /* Except, do not remove the ones named here */ +); + /* ** CAPI3REF: Virtual Table Instance Object ** KEYWORDS: sqlite3_vtab @@ -8299,7 +8440,7 @@ SQLITE_API int sqlite3_test_control(int op, ...); #define SQLITE_TESTCTRL_FIRST 5 #define SQLITE_TESTCTRL_PRNG_SAVE 5 #define SQLITE_TESTCTRL_PRNG_RESTORE 6 -#define SQLITE_TESTCTRL_PRNG_RESET 7 +#define SQLITE_TESTCTRL_PRNG_RESET 7 /* NOT USED */ #define SQLITE_TESTCTRL_BITVEC_TEST 8 #define SQLITE_TESTCTRL_FAULT_INSTALL 9 #define SQLITE_TESTCTRL_BENIGN_MALLOC_HOOKS 10 @@ -8321,7 +8462,10 @@ SQLITE_API int sqlite3_test_control(int op, ...); #define SQLITE_TESTCTRL_SORTER_MMAP 24 #define SQLITE_TESTCTRL_IMPOSTER 25 #define SQLITE_TESTCTRL_PARSER_COVERAGE 26 -#define SQLITE_TESTCTRL_LAST 26 /* Largest TESTCTRL */ +#define SQLITE_TESTCTRL_RESULT_INTREAL 27 +#define SQLITE_TESTCTRL_PRNG_SEED 28 +#define SQLITE_TESTCTRL_EXTRA_SCHEMA_CHECKS 29 +#define SQLITE_TESTCTRL_LAST 29 /* Largest TESTCTRL */ /* ** CAPI3REF: SQL Keyword Checking @@ -11931,7 +12075,7 @@ SQLITE_API int sqlite3rebaser_configure( ** in size. This function allocates and populates a buffer with a copy ** of the changeset rebased rebased according to the configuration of the ** rebaser object passed as the first argument. If successful, (*ppOut) -** is set to point to the new buffer containing the rebased changset and +** is set to point to the new buffer containing the rebased changeset and ** (*pnOut) to its size in bytes and SQLITE_OK returned. It is the ** responsibility of the caller to eventually free the new buffer using ** sqlite3_free(). Otherwise, if an error occurs, (*ppOut) and (*pnOut) @@ -12340,7 +12484,7 @@ struct Fts5PhraseIter { ** Save the pointer passed as the second argument as the extension functions ** "auxiliary data". The pointer may then be retrieved by the current or any ** future invocation of the same fts5 extension function made as part of -** of the same MATCH query using the xGetAuxdata() API. +** the same MATCH query using the xGetAuxdata() API. ** ** Each extension function is allocated a single auxiliary data slot for ** each FTS query (MATCH expression). If the extension function is invoked @@ -12355,7 +12499,7 @@ struct Fts5PhraseIter { ** The xDelete callback, if one is specified, is also invoked on the ** auxiliary data pointer after the FTS5 query has finished. ** -** If an error (e.g. an OOM condition) occurs within this function, an +** If an error (e.g. an OOM condition) occurs within this function, ** the auxiliary data is set to NULL and an error code returned. If the ** xDelete parameter was not NULL, it is invoked on the auxiliary data ** pointer before returning. @@ -13018,15 +13162,15 @@ struct fts5_api { ** So we have to define the macros in different ways depending on the ** compiler. */ -#if defined(__PTRDIFF_TYPE__) /* This case should work for GCC */ +#if defined(HAVE_STDINT_H) /* Use this case if we have ANSI headers */ +# define SQLITE_INT_TO_PTR(X) ((void*)(intptr_t)(X)) +# define SQLITE_PTR_TO_INT(X) ((int)(intptr_t)(X)) +#elif defined(__PTRDIFF_TYPE__) /* This case should work for GCC */ # define SQLITE_INT_TO_PTR(X) ((void*)(__PTRDIFF_TYPE__)(X)) # define SQLITE_PTR_TO_INT(X) ((int)(__PTRDIFF_TYPE__)(X)) #elif !defined(__GNUC__) /* Works for compilers other than LLVM */ # define SQLITE_INT_TO_PTR(X) ((void*)&((char*)0)[X]) # define SQLITE_PTR_TO_INT(X) ((int)(((char*)X)-(char*)0)) -#elif defined(HAVE_STDINT_H) /* Use this case if we have ANSI headers */ -# define SQLITE_INT_TO_PTR(X) ((void*)(intptr_t)(X)) -# define SQLITE_PTR_TO_INT(X) ((int)(intptr_t)(X)) #else /* Generates a warning - but it always works */ # define SQLITE_INT_TO_PTR(X) ((void*)(X)) # define SQLITE_PTR_TO_INT(X) ((int)(X)) @@ -13381,7 +13525,7 @@ struct Hash { unsigned int count; /* Number of entries in this table */ HashElem *first; /* The first element of the array */ struct _ht { /* the hash table */ - int count; /* Number of entries with this hash */ + unsigned int count; /* Number of entries with this hash */ HashElem *chain; /* Pointer to first entry with this hash */ } *ht; }; @@ -13516,105 +13660,103 @@ SQLITE_PRIVATE void sqlite3HashClear(Hash*); #define TK_VIEW 79 #define TK_VIRTUAL 80 #define TK_WITH 81 -#define TK_CURRENT 82 -#define TK_FOLLOWING 83 -#define TK_PARTITION 84 -#define TK_PRECEDING 85 -#define TK_RANGE 86 -#define TK_UNBOUNDED 87 -#define TK_REINDEX 88 -#define TK_RENAME 89 -#define TK_CTIME_KW 90 -#define TK_ANY 91 -#define TK_BITAND 92 -#define TK_BITOR 93 -#define TK_LSHIFT 94 -#define TK_RSHIFT 95 -#define TK_PLUS 96 -#define TK_MINUS 97 -#define TK_STAR 98 -#define TK_SLASH 99 -#define TK_REM 100 -#define TK_CONCAT 101 -#define TK_COLLATE 102 -#define TK_BITNOT 103 -#define TK_ON 104 -#define TK_INDEXED 105 -#define TK_STRING 106 -#define TK_JOIN_KW 107 -#define TK_CONSTRAINT 108 -#define TK_DEFAULT 109 -#define TK_NULL 110 -#define TK_PRIMARY 111 -#define TK_UNIQUE 112 -#define TK_CHECK 113 -#define TK_REFERENCES 114 -#define TK_AUTOINCR 115 -#define TK_INSERT 116 -#define TK_DELETE 117 -#define TK_UPDATE 118 -#define TK_SET 119 -#define TK_DEFERRABLE 120 -#define TK_FOREIGN 121 -#define TK_DROP 122 -#define TK_UNION 123 -#define TK_ALL 124 -#define TK_EXCEPT 125 -#define TK_INTERSECT 126 -#define TK_SELECT 127 -#define TK_VALUES 128 -#define TK_DISTINCT 129 -#define TK_DOT 130 -#define TK_FROM 131 -#define TK_JOIN 132 -#define TK_USING 133 -#define TK_ORDER 134 -#define TK_GROUP 135 -#define TK_HAVING 136 -#define TK_LIMIT 137 -#define TK_WHERE 138 -#define TK_INTO 139 -#define TK_NOTHING 140 -#define TK_FLOAT 141 -#define TK_BLOB 142 -#define TK_INTEGER 143 -#define TK_VARIABLE 144 -#define TK_CASE 145 -#define TK_WHEN 146 -#define TK_THEN 147 -#define TK_ELSE 148 -#define TK_INDEX 149 -#define TK_ALTER 150 -#define TK_ADD 151 -#define TK_WINDOW 152 -#define TK_OVER 153 -#define TK_FILTER 154 -#define TK_TRUEFALSE 155 -#define TK_ISNOT 156 -#define TK_FUNCTION 157 -#define TK_COLUMN 158 -#define TK_AGG_FUNCTION 159 -#define TK_AGG_COLUMN 160 -#define TK_UMINUS 161 -#define TK_UPLUS 162 -#define TK_TRUTH 163 -#define TK_REGISTER 164 -#define TK_VECTOR 165 -#define TK_SELECT_COLUMN 166 -#define TK_IF_NULL_ROW 167 -#define TK_ASTERISK 168 -#define TK_SPAN 169 -#define TK_END_OF_FILE 170 -#define TK_UNCLOSED_STRING 171 -#define TK_SPACE 172 -#define TK_ILLEGAL 173 - -/* The token codes above must all fit in 8 bits */ -#define TKFLG_MASK 0xff - -/* Flags that can be added to a token code when it is not -** being stored in a u8: */ -#define TKFLG_DONTFOLD 0x100 /* Omit constant folding optimizations */ +#define TK_NULLS 82 +#define TK_FIRST 83 +#define TK_LAST 84 +#define TK_CURRENT 85 +#define TK_FOLLOWING 86 +#define TK_PARTITION 87 +#define TK_PRECEDING 88 +#define TK_RANGE 89 +#define TK_UNBOUNDED 90 +#define TK_EXCLUDE 91 +#define TK_GROUPS 92 +#define TK_OTHERS 93 +#define TK_TIES 94 +#define TK_REINDEX 95 +#define TK_RENAME 96 +#define TK_CTIME_KW 97 +#define TK_ANY 98 +#define TK_BITAND 99 +#define TK_BITOR 100 +#define TK_LSHIFT 101 +#define TK_RSHIFT 102 +#define TK_PLUS 103 +#define TK_MINUS 104 +#define TK_STAR 105 +#define TK_SLASH 106 +#define TK_REM 107 +#define TK_CONCAT 108 +#define TK_COLLATE 109 +#define TK_BITNOT 110 +#define TK_ON 111 +#define TK_INDEXED 112 +#define TK_STRING 113 +#define TK_JOIN_KW 114 +#define TK_CONSTRAINT 115 +#define TK_DEFAULT 116 +#define TK_NULL 117 +#define TK_PRIMARY 118 +#define TK_UNIQUE 119 +#define TK_CHECK 120 +#define TK_REFERENCES 121 +#define TK_AUTOINCR 122 +#define TK_INSERT 123 +#define TK_DELETE 124 +#define TK_UPDATE 125 +#define TK_SET 126 +#define TK_DEFERRABLE 127 +#define TK_FOREIGN 128 +#define TK_DROP 129 +#define TK_UNION 130 +#define TK_ALL 131 +#define TK_EXCEPT 132 +#define TK_INTERSECT 133 +#define TK_SELECT 134 +#define TK_VALUES 135 +#define TK_DISTINCT 136 +#define TK_DOT 137 +#define TK_FROM 138 +#define TK_JOIN 139 +#define TK_USING 140 +#define TK_ORDER 141 +#define TK_GROUP 142 +#define TK_HAVING 143 +#define TK_LIMIT 144 +#define TK_WHERE 145 +#define TK_INTO 146 +#define TK_NOTHING 147 +#define TK_FLOAT 148 +#define TK_BLOB 149 +#define TK_INTEGER 150 +#define TK_VARIABLE 151 +#define TK_CASE 152 +#define TK_WHEN 153 +#define TK_THEN 154 +#define TK_ELSE 155 +#define TK_INDEX 156 +#define TK_ALTER 157 +#define TK_ADD 158 +#define TK_WINDOW 159 +#define TK_OVER 160 +#define TK_FILTER 161 +#define TK_COLUMN 162 +#define TK_AGG_FUNCTION 163 +#define TK_AGG_COLUMN 164 +#define TK_TRUEFALSE 165 +#define TK_ISNOT 166 +#define TK_FUNCTION 167 +#define TK_UMINUS 168 +#define TK_UPLUS 169 +#define TK_TRUTH 170 +#define TK_REGISTER 171 +#define TK_VECTOR 172 +#define TK_SELECT_COLUMN 173 +#define TK_IF_NULL_ROW 174 +#define TK_ASTERISK 175 +#define TK_SPAN 176 +#define TK_SPACE 177 +#define TK_ILLEGAL 178 /************** End of parse.h ***********************************************/ /************** Continuing where we left off in sqliteInt.h ******************/ @@ -13920,12 +14062,13 @@ typedef INT16_TYPE LogEst; ** at run-time. */ #ifndef SQLITE_BYTEORDER -# if defined(i386) || defined(__i386__) || defined(_M_IX86) || \ - defined(__x86_64) || defined(__x86_64__) || defined(_M_X64) || \ - defined(_M_AMD64) || defined(_M_ARM) || defined(__x86) || \ - defined(__arm__) || defined(_M_ARM64) +# if defined(i386) || defined(__i386__) || defined(_M_IX86) || \ + defined(__x86_64) || defined(__x86_64__) || defined(_M_X64) || \ + defined(_M_AMD64) || defined(_M_ARM) || defined(__x86) || \ + defined(__ARMEL__) || defined(__AARCH64EL__) || defined(_M_ARM64) # define SQLITE_BYTEORDER 1234 -# elif defined(sparc) || defined(__ppc__) +# elif defined(sparc) || defined(__ppc__) || \ + defined(__ARMEB__) || defined(__AARCH64EB__) # define SQLITE_BYTEORDER 4321 # else # define SQLITE_BYTEORDER 0 @@ -14024,20 +14167,6 @@ typedef INT16_TYPE LogEst; # define SQLITE_DEFAULT_MMAP_SIZE SQLITE_MAX_MMAP_SIZE #endif -/* -** Only one of SQLITE_ENABLE_STAT3 or SQLITE_ENABLE_STAT4 can be defined. -** Priority is given to SQLITE_ENABLE_STAT4. If either are defined, also -** define SQLITE_ENABLE_STAT3_OR_STAT4 -*/ -#ifdef SQLITE_ENABLE_STAT4 -# undef SQLITE_ENABLE_STAT3 -# define SQLITE_ENABLE_STAT3_OR_STAT4 1 -#elif SQLITE_ENABLE_STAT3 -# define SQLITE_ENABLE_STAT3_OR_STAT4 1 -#elif SQLITE_ENABLE_STAT3_OR_STAT4 -# undef SQLITE_ENABLE_STAT3_OR_STAT4 -#endif - /* ** SELECTTRACE_ENABLED will be either 1 or 0 depending on whether or not ** the Select query generator tracing logic is turned on. @@ -14546,9 +14675,6 @@ struct BtreePayload { SQLITE_PRIVATE int sqlite3BtreeInsert(BtCursor*, const BtreePayload *pPayload, int flags, int seekResult); SQLITE_PRIVATE int sqlite3BtreeFirst(BtCursor*, int *pRes); -#ifndef SQLITE_OMIT_WINDOWFUNC -SQLITE_PRIVATE void sqlite3BtreeSkipNext(BtCursor*); -#endif SQLITE_PRIVATE int sqlite3BtreeLast(BtCursor*, int *pRes); SQLITE_PRIVATE int sqlite3BtreeNext(BtCursor*, int flags); SQLITE_PRIVATE int sqlite3BtreeEof(BtCursor*); @@ -14906,28 +15032,28 @@ typedef struct VdbeOpList VdbeOpList; #define OP_Offset 89 /* synopsis: r[P3] = sqlite_offset(P1) */ #define OP_Column 90 /* synopsis: r[P3]=PX */ #define OP_Affinity 91 /* synopsis: affinity(r[P1@P2]) */ -#define OP_BitAnd 92 /* same as TK_BITAND, synopsis: r[P3]=r[P1]&r[P2] */ -#define OP_BitOr 93 /* same as TK_BITOR, synopsis: r[P3]=r[P1]|r[P2] */ -#define OP_ShiftLeft 94 /* same as TK_LSHIFT, synopsis: r[P3]=r[P2]<>r[P1] */ -#define OP_Add 96 /* same as TK_PLUS, synopsis: r[P3]=r[P1]+r[P2] */ -#define OP_Subtract 97 /* same as TK_MINUS, synopsis: r[P3]=r[P2]-r[P1] */ -#define OP_Multiply 98 /* same as TK_STAR, synopsis: r[P3]=r[P1]*r[P2] */ -#define OP_Divide 99 /* same as TK_SLASH, synopsis: r[P3]=r[P2]/r[P1] */ -#define OP_Remainder 100 /* same as TK_REM, synopsis: r[P3]=r[P2]%r[P1] */ -#define OP_Concat 101 /* same as TK_CONCAT, synopsis: r[P3]=r[P2]+r[P1] */ -#define OP_MakeRecord 102 /* synopsis: r[P3]=mkrec(r[P1@P2]) */ -#define OP_BitNot 103 /* same as TK_BITNOT, synopsis: r[P2]= ~r[P1] */ -#define OP_Count 104 /* synopsis: r[P2]=count() */ -#define OP_ReadCookie 105 -#define OP_String8 106 /* same as TK_STRING, synopsis: r[P2]='P4' */ -#define OP_SetCookie 107 -#define OP_ReopenIdx 108 /* synopsis: root=P2 iDb=P3 */ -#define OP_OpenRead 109 /* synopsis: root=P2 iDb=P3 */ -#define OP_OpenWrite 110 /* synopsis: root=P2 iDb=P3 */ -#define OP_OpenDup 111 -#define OP_OpenAutoindex 112 /* synopsis: nColumn=P2 */ -#define OP_OpenEphemeral 113 /* synopsis: nColumn=P2 */ +#define OP_MakeRecord 92 /* synopsis: r[P3]=mkrec(r[P1@P2]) */ +#define OP_Count 93 /* synopsis: r[P2]=count() */ +#define OP_ReadCookie 94 +#define OP_SetCookie 95 +#define OP_ReopenIdx 96 /* synopsis: root=P2 iDb=P3 */ +#define OP_OpenRead 97 /* synopsis: root=P2 iDb=P3 */ +#define OP_OpenWrite 98 /* synopsis: root=P2 iDb=P3 */ +#define OP_BitAnd 99 /* same as TK_BITAND, synopsis: r[P3]=r[P1]&r[P2] */ +#define OP_BitOr 100 /* same as TK_BITOR, synopsis: r[P3]=r[P1]|r[P2] */ +#define OP_ShiftLeft 101 /* same as TK_LSHIFT, synopsis: r[P3]=r[P2]<>r[P1] */ +#define OP_Add 103 /* same as TK_PLUS, synopsis: r[P3]=r[P1]+r[P2] */ +#define OP_Subtract 104 /* same as TK_MINUS, synopsis: r[P3]=r[P2]-r[P1] */ +#define OP_Multiply 105 /* same as TK_STAR, synopsis: r[P3]=r[P1]*r[P2] */ +#define OP_Divide 106 /* same as TK_SLASH, synopsis: r[P3]=r[P2]/r[P1] */ +#define OP_Remainder 107 /* same as TK_REM, synopsis: r[P3]=r[P2]%r[P1] */ +#define OP_Concat 108 /* same as TK_CONCAT, synopsis: r[P3]=r[P2]+r[P1] */ +#define OP_OpenDup 109 +#define OP_BitNot 110 /* same as TK_BITNOT, synopsis: r[P2]= ~r[P1] */ +#define OP_OpenAutoindex 111 /* synopsis: nColumn=P2 */ +#define OP_OpenEphemeral 112 /* synopsis: nColumn=P2 */ +#define OP_String8 113 /* same as TK_STRING, synopsis: r[P2]='P4' */ #define OP_SorterOpen 114 #define OP_SequenceTest 115 /* synopsis: if( cursor[P1].ctr++ ) pc = P2 */ #define OP_OpenPseudo 116 /* synopsis: P3 columns in r[P2] */ @@ -14937,57 +15063,56 @@ typedef struct VdbeOpList VdbeOpList; #define OP_Sequence 120 /* synopsis: r[P2]=cursor[P1].ctr++ */ #define OP_NewRowid 121 /* synopsis: r[P2]=rowid */ #define OP_Insert 122 /* synopsis: intkey=r[P3] data=r[P2] */ -#define OP_InsertInt 123 /* synopsis: intkey=P3 data=r[P2] */ -#define OP_Delete 124 -#define OP_ResetCount 125 -#define OP_SorterCompare 126 /* synopsis: if key(P1)!=trim(r[P3],P4) goto P2 */ -#define OP_SorterData 127 /* synopsis: r[P2]=data */ -#define OP_RowData 128 /* synopsis: r[P2]=data */ -#define OP_Rowid 129 /* synopsis: r[P2]=rowid */ -#define OP_NullRow 130 -#define OP_SeekEnd 131 -#define OP_SorterInsert 132 /* synopsis: key=r[P2] */ -#define OP_IdxInsert 133 /* synopsis: key=r[P2] */ -#define OP_IdxDelete 134 /* synopsis: key=r[P2@P3] */ -#define OP_DeferredSeek 135 /* synopsis: Move P3 to P1.rowid if needed */ -#define OP_IdxRowid 136 /* synopsis: r[P2]=rowid */ -#define OP_Destroy 137 -#define OP_Clear 138 -#define OP_ResetSorter 139 -#define OP_CreateBtree 140 /* synopsis: r[P2]=root iDb=P1 flags=P3 */ -#define OP_Real 141 /* same as TK_FLOAT, synopsis: r[P2]=P4 */ -#define OP_SqlExec 142 -#define OP_ParseSchema 143 -#define OP_LoadAnalysis 144 -#define OP_DropTable 145 -#define OP_DropIndex 146 -#define OP_DropTrigger 147 -#define OP_IntegrityCk 148 -#define OP_RowSetAdd 149 /* synopsis: rowset(P1)=r[P2] */ -#define OP_Param 150 -#define OP_FkCounter 151 /* synopsis: fkctr[P1]+=P2 */ -#define OP_MemMax 152 /* synopsis: r[P1]=max(r[P1],r[P2]) */ -#define OP_OffsetLimit 153 /* synopsis: if r[P1]>0 then r[P2]=r[P1]+max(0,r[P3]) else r[P2]=(-1) */ -#define OP_AggInverse 154 /* synopsis: accum=r[P3] inverse(r[P2@P5]) */ -#define OP_AggStep 155 /* synopsis: accum=r[P3] step(r[P2@P5]) */ -#define OP_AggStep1 156 /* synopsis: accum=r[P3] step(r[P2@P5]) */ -#define OP_AggValue 157 /* synopsis: r[P3]=value N=P2 */ -#define OP_AggFinal 158 /* synopsis: accum=r[P1] N=P2 */ -#define OP_Expire 159 -#define OP_TableLock 160 /* synopsis: iDb=P1 root=P2 write=P3 */ -#define OP_VBegin 161 -#define OP_VCreate 162 -#define OP_VDestroy 163 -#define OP_VOpen 164 -#define OP_VColumn 165 /* synopsis: r[P3]=vcolumn(P2) */ -#define OP_VRename 166 -#define OP_Pagecount 167 -#define OP_MaxPgcnt 168 -#define OP_Trace 169 -#define OP_CursorHint 170 -#define OP_Noop 171 -#define OP_Explain 172 -#define OP_Abortable 173 +#define OP_Delete 123 +#define OP_ResetCount 124 +#define OP_SorterCompare 125 /* synopsis: if key(P1)!=trim(r[P3],P4) goto P2 */ +#define OP_SorterData 126 /* synopsis: r[P2]=data */ +#define OP_RowData 127 /* synopsis: r[P2]=data */ +#define OP_Rowid 128 /* synopsis: r[P2]=rowid */ +#define OP_NullRow 129 +#define OP_SeekEnd 130 +#define OP_SorterInsert 131 /* synopsis: key=r[P2] */ +#define OP_IdxInsert 132 /* synopsis: key=r[P2] */ +#define OP_IdxDelete 133 /* synopsis: key=r[P2@P3] */ +#define OP_DeferredSeek 134 /* synopsis: Move P3 to P1.rowid if needed */ +#define OP_IdxRowid 135 /* synopsis: r[P2]=rowid */ +#define OP_Destroy 136 +#define OP_Clear 137 +#define OP_ResetSorter 138 +#define OP_CreateBtree 139 /* synopsis: r[P2]=root iDb=P1 flags=P3 */ +#define OP_SqlExec 140 +#define OP_ParseSchema 141 +#define OP_LoadAnalysis 142 +#define OP_DropTable 143 +#define OP_DropIndex 144 +#define OP_DropTrigger 145 +#define OP_IntegrityCk 146 +#define OP_RowSetAdd 147 /* synopsis: rowset(P1)=r[P2] */ +#define OP_Real 148 /* same as TK_FLOAT, synopsis: r[P2]=P4 */ +#define OP_Param 149 +#define OP_FkCounter 150 /* synopsis: fkctr[P1]+=P2 */ +#define OP_MemMax 151 /* synopsis: r[P1]=max(r[P1],r[P2]) */ +#define OP_OffsetLimit 152 /* synopsis: if r[P1]>0 then r[P2]=r[P1]+max(0,r[P3]) else r[P2]=(-1) */ +#define OP_AggInverse 153 /* synopsis: accum=r[P3] inverse(r[P2@P5]) */ +#define OP_AggStep 154 /* synopsis: accum=r[P3] step(r[P2@P5]) */ +#define OP_AggStep1 155 /* synopsis: accum=r[P3] step(r[P2@P5]) */ +#define OP_AggValue 156 /* synopsis: r[P3]=value N=P2 */ +#define OP_AggFinal 157 /* synopsis: accum=r[P1] N=P2 */ +#define OP_Expire 158 +#define OP_TableLock 159 /* synopsis: iDb=P1 root=P2 write=P3 */ +#define OP_VBegin 160 +#define OP_VCreate 161 +#define OP_VDestroy 162 +#define OP_VOpen 163 +#define OP_VColumn 164 /* synopsis: r[P3]=vcolumn(P2) */ +#define OP_VRename 165 +#define OP_Pagecount 166 +#define OP_MaxPgcnt 167 +#define OP_Trace 168 +#define OP_CursorHint 169 +#define OP_Noop 170 +#define OP_Explain 171 +#define OP_Abortable 172 /* Properties such as "out2" or "jump" that are specified in ** comments following the "case" for each opcode in the vdbe.c @@ -15011,17 +15136,17 @@ typedef struct VdbeOpList VdbeOpList; /* 64 */ 0x00, 0x00, 0x02, 0x02, 0x08, 0x00, 0x10, 0x10,\ /* 72 */ 0x10, 0x10, 0x00, 0x10, 0x10, 0x00, 0x00, 0x10,\ /* 80 */ 0x10, 0x00, 0x00, 0x02, 0x02, 0x02, 0x00, 0x00,\ -/* 88 */ 0x12, 0x20, 0x00, 0x00, 0x26, 0x26, 0x26, 0x26,\ -/* 96 */ 0x26, 0x26, 0x26, 0x26, 0x26, 0x26, 0x00, 0x12,\ -/* 104 */ 0x10, 0x10, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00,\ -/* 112 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,\ +/* 88 */ 0x12, 0x20, 0x00, 0x00, 0x00, 0x10, 0x10, 0x00,\ +/* 96 */ 0x00, 0x00, 0x00, 0x26, 0x26, 0x26, 0x26, 0x26,\ +/* 104 */ 0x26, 0x26, 0x26, 0x26, 0x26, 0x00, 0x12, 0x00,\ +/* 112 */ 0x00, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,\ /* 120 */ 0x10, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,\ -/* 128 */ 0x00, 0x10, 0x00, 0x00, 0x04, 0x04, 0x00, 0x00,\ -/* 136 */ 0x10, 0x10, 0x00, 0x00, 0x10, 0x10, 0x00, 0x00,\ -/* 144 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x06, 0x10, 0x00,\ -/* 152 */ 0x04, 0x1a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,\ -/* 160 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10,\ -/* 168 */ 0x10, 0x00, 0x00, 0x00, 0x00, 0x00,} +/* 128 */ 0x10, 0x00, 0x00, 0x04, 0x04, 0x00, 0x00, 0x10,\ +/* 136 */ 0x10, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x00,\ +/* 144 */ 0x00, 0x00, 0x00, 0x06, 0x10, 0x10, 0x00, 0x04,\ +/* 152 */ 0x1a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,\ +/* 160 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x10,\ +/* 168 */ 0x00, 0x00, 0x00, 0x00, 0x00,} /* The sqlite3P2Values() routine is able to run faster if it knows ** the value of the largest JUMP opcode. The smaller the maximum @@ -15088,10 +15213,10 @@ SQLITE_PRIVATE void sqlite3ExplainBreakpoint(const char*,const char*); # define sqlite3ExplainBreakpoint(A,B) /*no-op*/ #endif SQLITE_PRIVATE void sqlite3VdbeAddParseSchemaOp(Vdbe*,int,char*); -SQLITE_PRIVATE void sqlite3VdbeChangeOpcode(Vdbe*, u32 addr, u8); -SQLITE_PRIVATE void sqlite3VdbeChangeP1(Vdbe*, u32 addr, int P1); -SQLITE_PRIVATE void sqlite3VdbeChangeP2(Vdbe*, u32 addr, int P2); -SQLITE_PRIVATE void sqlite3VdbeChangeP3(Vdbe*, u32 addr, int P3); +SQLITE_PRIVATE void sqlite3VdbeChangeOpcode(Vdbe*, int addr, u8); +SQLITE_PRIVATE void sqlite3VdbeChangeP1(Vdbe*, int addr, int P1); +SQLITE_PRIVATE void sqlite3VdbeChangeP2(Vdbe*, int addr, int P2); +SQLITE_PRIVATE void sqlite3VdbeChangeP3(Vdbe*, int addr, int P3); SQLITE_PRIVATE void sqlite3VdbeChangeP5(Vdbe*, u16 P5); SQLITE_PRIVATE void sqlite3VdbeJumpHere(Vdbe*, int addr); SQLITE_PRIVATE int sqlite3VdbeChangeToNoop(Vdbe*, int addr); @@ -16048,6 +16173,7 @@ SQLITE_PRIVATE void sqlite3OsCloseFree(sqlite3_file *); #define MUTEX_LOGIC(X) #else #define MUTEX_LOGIC(X) X +SQLITE_API int sqlite3_mutex_held(sqlite3_mutex*); #endif /* defined(SQLITE_MUTEX_OMIT) */ /************** End of mutex.h ***********************************************/ @@ -16306,6 +16432,7 @@ struct sqlite3 { unsigned orphanTrigger : 1; /* Last statement is orphaned TEMP trigger */ unsigned imposterTable : 1; /* Building an imposter table */ unsigned reopenMemdb : 1; /* ATTACH is really a reopen using MemDB */ + char **azInit; /* "type", "name", and "tbl_name" columns */ } init; int nVdbeActive; /* Number of VDBEs currently running */ int nVdbeRead; /* Number of active VDBEs that read or write */ @@ -16326,6 +16453,7 @@ struct sqlite3 { void (*xRollbackCallback)(void*); /* Invoked at every commit. */ void *pUpdateArg; void (*xUpdateCallback)(void*,int, const char*,const char*,sqlite_int64); + Parse *pParse; /* Current parse */ #ifdef SQLITE_ENABLE_PREUPDATE_HOOK void *pPreUpdateArg; /* First argument to xPreUpdateCallback */ void (*xPreUpdateCallback)( /* Registered using sqlite3_preupdate_hook() */ @@ -16441,16 +16569,19 @@ struct sqlite3 { #define SQLITE_LegacyAlter 0x04000000 /* Legacy ALTER TABLE behaviour */ #define SQLITE_NoSchemaError 0x08000000 /* Do not report schema parse errors*/ #define SQLITE_Defensive 0x10000000 /* Input SQL is likely hostile */ +#define SQLITE_DqsDDL 0x20000000 /* dbl-quoted strings allowed in DDL*/ +#define SQLITE_DqsDML 0x40000000 /* dbl-quoted strings allowed in DML*/ +#define SQLITE_EnableView 0x80000000 /* Enable the use of views */ /* Flags used only if debugging */ #define HI(X) ((u64)(X)<<32) #ifdef SQLITE_DEBUG -#define SQLITE_SqlTrace HI(0x0001) /* Debug print SQL as it executes */ -#define SQLITE_VdbeListing HI(0x0002) /* Debug listings of VDBE progs */ -#define SQLITE_VdbeTrace HI(0x0004) /* True to trace VDBE execution */ -#define SQLITE_VdbeAddopTrace HI(0x0008) /* Trace sqlite3VdbeAddOp() calls */ -#define SQLITE_VdbeEQP HI(0x0010) /* Debug EXPLAIN QUERY PLAN */ -#define SQLITE_ParserTrace HI(0x0020) /* PRAGMA parser_trace=ON */ +#define SQLITE_SqlTrace HI(0x0100000) /* Debug print SQL as it executes */ +#define SQLITE_VdbeListing HI(0x0200000) /* Debug listings of VDBE progs */ +#define SQLITE_VdbeTrace HI(0x0400000) /* True to trace VDBE execution */ +#define SQLITE_VdbeAddopTrace HI(0x0800000) /* Trace sqlite3VdbeAddOp() calls */ +#define SQLITE_VdbeEQP HI(0x1000000) /* Debug EXPLAIN QUERY PLAN */ +#define SQLITE_ParserTrace HI(0x2000000) /* PRAGMA parser_trace=ON */ #endif /* @@ -16459,7 +16590,8 @@ struct sqlite3 { #define DBFLAG_SchemaChange 0x0001 /* Uncommitted Hash table changes */ #define DBFLAG_PreferBuiltin 0x0002 /* Preference to built-in funcs */ #define DBFLAG_Vacuum 0x0004 /* Currently in a VACUUM */ -#define DBFLAG_SchemaKnownOk 0x0008 /* Schema is known to be valid */ +#define DBFLAG_VacuumInto 0x0008 /* Currently running VACUUM INTO */ +#define DBFLAG_SchemaKnownOk 0x0010 /* Schema is known to be valid */ /* ** Bits of the sqlite3.dbOptFlags field that are used by the @@ -16467,7 +16599,7 @@ struct sqlite3 { ** selectively disable various optimizations. */ #define SQLITE_QueryFlattener 0x0001 /* Query flattening */ - /* 0x0002 available for reuse */ +#define SQLITE_WindowFunc 0x0002 /* Use xInverse for window functions */ #define SQLITE_GroupByOrder 0x0004 /* GROUPBY cover of ORDERBY */ #define SQLITE_FactorOutConst 0x0008 /* Constant factoring */ #define SQLITE_DistinctOpt 0x0010 /* DISTINCT using indexes */ @@ -16477,8 +16609,8 @@ struct sqlite3 { #define SQLITE_OmitNoopJoin 0x0100 /* Omit unused tables in joins */ #define SQLITE_CountOfView 0x0200 /* The count-of-view optimization */ #define SQLITE_CursorHints 0x0400 /* Add OP_CursorHint opcodes */ -#define SQLITE_Stat34 0x0800 /* Use STAT3 or STAT4 data */ - /* TH3 expects the Stat34 ^^^^^^ value to be 0x0800. Don't change it */ +#define SQLITE_Stat4 0x0800 /* Use STAT4 data */ + /* TH3 expects the Stat4 ^^^^^^ value to be 0x0800. Don't change it */ #define SQLITE_PushDown 0x1000 /* The push-down optimization */ #define SQLITE_SimplifyJoin 0x2000 /* Convert LEFT JOIN to JOIN */ #define SQLITE_SkipScan 0x4000 /* Skip-scans */ @@ -16566,6 +16698,7 @@ struct FuncDestructor { ** SQLITE_FUNC_LENGTH == OPFLAG_LENGTHARG ** SQLITE_FUNC_TYPEOF == OPFLAG_TYPEOFARG ** SQLITE_FUNC_CONSTANT == SQLITE_DETERMINISTIC from the API +** SQLITE_FUNC_DIRECT == SQLITE_DIRECTONLY from the API ** SQLITE_FUNC_ENCMASK depends on SQLITE_UTF* macros in the API */ #define SQLITE_FUNC_ENCMASK 0x0003 /* SQLITE_UTF8, SQLITE_UTF16BE or UTF16LE */ @@ -16585,8 +16718,9 @@ struct FuncDestructor { #define SQLITE_FUNC_AFFINITY 0x4000 /* Built-in affinity() function */ #define SQLITE_FUNC_OFFSET 0x8000 /* Built-in sqlite_offset() function */ #define SQLITE_FUNC_WINDOW 0x00010000 /* Built-in window-only function */ -#define SQLITE_FUNC_WINDOW_SIZE 0x20000 /* Requires partition size as arg. */ #define SQLITE_FUNC_INTERNAL 0x00040000 /* For use by NestedParse() only */ +#define SQLITE_FUNC_DIRECT 0x00080000 /* Not for use in TRIGGERs or VIEWs */ +#define SQLITE_FUNC_SUBTYPE 0x00100000 /* Result likely to have sub-type */ /* ** The following three macros, FUNCTION(), LIKEFUNC() and AGGREGATE() are @@ -16700,6 +16834,7 @@ struct Savepoint { struct Module { const sqlite3_module *pModule; /* Callback pointers */ const char *zName; /* Name passed to create_module() */ + int nRefModule; /* Number of pointers to this object */ void *pAux; /* pAux passed to create_module() */ void (*xDestroy)(void *); /* Module destructor function */ Table *pEpoTab; /* Eponymous table for this module */ @@ -16765,11 +16900,12 @@ struct CollSeq { ** Note also that the numeric types are grouped together so that testing ** for a numeric type is a single comparison. And the BLOB type is first. */ -#define SQLITE_AFF_BLOB 'A' -#define SQLITE_AFF_TEXT 'B' -#define SQLITE_AFF_NUMERIC 'C' -#define SQLITE_AFF_INTEGER 'D' -#define SQLITE_AFF_REAL 'E' +#define SQLITE_AFF_NONE 0x40 /* '@' */ +#define SQLITE_AFF_BLOB 0x41 /* 'A' */ +#define SQLITE_AFF_TEXT 0x42 /* 'B' */ +#define SQLITE_AFF_NUMERIC 0x43 /* 'C' */ +#define SQLITE_AFF_INTEGER 0x44 /* 'D' */ +#define SQLITE_AFF_REAL 0x45 /* 'E' */ #define sqlite3IsNumericAffinity(X) ((X)>=SQLITE_AFF_NUMERIC) @@ -17037,10 +17173,16 @@ struct KeyInfo { u16 nKeyField; /* Number of key columns in the index */ u16 nAllField; /* Total columns, including key plus others */ sqlite3 *db; /* The database connection */ - u8 *aSortOrder; /* Sort order for each column. */ + u8 *aSortFlags; /* Sort order for each column. */ CollSeq *aColl[1]; /* Collating sequence for each term of the key */ }; +/* +** Allowed bit values for entries in the KeyInfo.aSortFlags[] array. +*/ +#define KEYINFO_ORDER_DESC 0x01 /* DESC sort order */ +#define KEYINFO_ORDER_BIGNULL 0x02 /* NULL is larger than any other value */ + /* ** This object holds a record which has been parsed out into individual ** fields, for the purposes of doing a comparison. @@ -17147,7 +17289,8 @@ struct Index { unsigned noSkipScan:1; /* Do not try to use skip-scan if true */ unsigned hasStat1:1; /* aiRowLogEst values come from sqlite_stat1 */ unsigned bNoQuery:1; /* Do not use this index to optimize queries */ -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 + unsigned bAscKeyBug:1; /* True if the bba7b69f9849b5bf bug applies */ +#ifdef SQLITE_ENABLE_STAT4 int nSample; /* Number of elements in aSample[] */ int nSampleCol; /* Size of IndexSample.anEq[] and so on */ tRowcnt *aAvgEq; /* Average nEq values for keys not in aSample */ @@ -17179,7 +17322,7 @@ struct Index { #define XN_EXPR (-2) /* Indexed column is an expression */ /* -** Each sample stored in the sqlite_stat3 table is represented in memory +** Each sample stored in the sqlite_stat4 table is represented in memory ** using a structure of this type. See documentation at the top of the ** analyze.c source file for additional information. */ @@ -17337,7 +17480,7 @@ typedef int ynVar; */ struct Expr { u8 op; /* Operation performed by this node */ - char affinity; /* The affinity of the column or 0 if not a column */ + char affExpr; /* affinity, or RAISE type */ u32 flags; /* Various flags. EP_* See below */ union { char *zToken; /* Token value. Zero terminated and dequoted */ @@ -17368,20 +17511,22 @@ struct Expr { ** TK_REGISTER: register number ** TK_TRIGGER: 1 -> new, 0 -> old ** EP_Unlikely: 134217728 times likelihood + ** TK_IN: ephemerial table holding RHS + ** TK_SELECT_COLUMN: Number of columns on the LHS ** TK_SELECT: 1st register of result vector */ ynVar iColumn; /* TK_COLUMN: column index. -1 for rowid. ** TK_VARIABLE: variable number (always >= 1). ** TK_SELECT_COLUMN: column of the result vector */ i16 iAgg; /* Which entry in pAggInfo->aCol[] or ->aFunc[] */ i16 iRightJoinTable; /* If EP_FromJoin, the right table of the join */ - u8 op2; /* TK_REGISTER: original value of Expr.op + u8 op2; /* TK_REGISTER/TK_TRUTH: original value of Expr.op ** TK_COLUMN: the value of p5 for OP_Column ** TK_AGG_FUNCTION: nesting depth */ AggInfo *pAggInfo; /* Used by TK_AGG_COLUMN and TK_AGG_FUNCTION */ union { Table *pTab; /* TK_COLUMN: Table containing column. Can be NULL ** for a column of an index on an expression */ - Window *pWin; /* TK_FUNCTION: Window definition for the func */ + Window *pWin; /* EP_WinFunc: Window/Filter defn for a function */ struct { /* TK_IN, TK_SELECT, and TK_EXISTS */ int iAddr; /* Subroutine entry address */ int regReturn; /* Register used to hold return address */ @@ -17391,34 +17536,42 @@ struct Expr { /* ** The following are the meanings of bits in the Expr.flags field. -*/ -#define EP_FromJoin 0x000001 /* Originates in ON/USING clause of outer join */ -#define EP_Agg 0x000002 /* Contains one or more aggregate functions */ -#define EP_HasFunc 0x000004 /* Contains one or more functions of any kind */ -#define EP_FixedCol 0x000008 /* TK_Column with a known fixed value */ -#define EP_Distinct 0x000010 /* Aggregate function with DISTINCT keyword */ -#define EP_VarSelect 0x000020 /* pSelect is correlated, not constant */ -#define EP_DblQuoted 0x000040 /* token.z was originally in "..." */ -#define EP_InfixFunc 0x000080 /* True for an infix function: LIKE, GLOB, etc */ -#define EP_Collate 0x000100 /* Tree contains a TK_COLLATE operator */ -#define EP_Generic 0x000200 /* Ignore COLLATE or affinity on this tree */ -#define EP_IntValue 0x000400 /* Integer value contained in u.iValue */ -#define EP_xIsSelect 0x000800 /* x.pSelect is valid (otherwise x.pList is) */ -#define EP_Skip 0x001000 /* COLLATE, AS, or UNLIKELY */ -#define EP_Reduced 0x002000 /* Expr struct EXPR_REDUCEDSIZE bytes only */ -#define EP_TokenOnly 0x004000 /* Expr struct EXPR_TOKENONLYSIZE bytes only */ -#define EP_Static 0x008000 /* Held in memory not obtained from malloc() */ -#define EP_MemToken 0x010000 /* Need to sqlite3DbFree() Expr.zToken */ -#define EP_NoReduce 0x020000 /* Cannot EXPRDUP_REDUCE this Expr */ -#define EP_Unlikely 0x040000 /* unlikely() or likelihood() function */ -#define EP_ConstFunc 0x080000 /* A SQLITE_FUNC_CONSTANT or _SLOCHNG function */ -#define EP_CanBeNull 0x100000 /* Can be null despite NOT NULL constraint */ -#define EP_Subquery 0x200000 /* Tree contains a TK_SELECT operator */ -#define EP_Alias 0x400000 /* Is an alias for a result set column */ -#define EP_Leaf 0x800000 /* Expr.pLeft, .pRight, .u.pSelect all NULL */ -#define EP_WinFunc 0x1000000 /* TK_FUNCTION with Expr.y.pWin set */ -#define EP_Subrtn 0x2000000 /* Uses Expr.y.sub. TK_IN, _SELECT, or _EXISTS */ -#define EP_Quoted 0x4000000 /* TK_ID was originally quoted */ +** Value restrictions: +** +** EP_Agg == NC_HasAgg == SF_HasAgg +** EP_Win == NC_HasWin +*/ +#define EP_FromJoin 0x000001 /* Originates in ON/USING clause of outer join */ +#define EP_Distinct 0x000002 /* Aggregate function with DISTINCT keyword */ +#define EP_HasFunc 0x000004 /* Contains one or more functions of any kind */ +#define EP_FixedCol 0x000008 /* TK_Column with a known fixed value */ +#define EP_Agg 0x000010 /* Contains one or more aggregate functions */ +#define EP_VarSelect 0x000020 /* pSelect is correlated, not constant */ +#define EP_DblQuoted 0x000040 /* token.z was originally in "..." */ +#define EP_InfixFunc 0x000080 /* True for an infix function: LIKE, GLOB, etc */ +#define EP_Collate 0x000100 /* Tree contains a TK_COLLATE operator */ + /* 0x000200 Available for reuse */ +#define EP_IntValue 0x000400 /* Integer value contained in u.iValue */ +#define EP_xIsSelect 0x000800 /* x.pSelect is valid (otherwise x.pList is) */ +#define EP_Skip 0x001000 /* Operator does not contribute to affinity */ +#define EP_Reduced 0x002000 /* Expr struct EXPR_REDUCEDSIZE bytes only */ +#define EP_TokenOnly 0x004000 /* Expr struct EXPR_TOKENONLYSIZE bytes only */ +#define EP_Win 0x008000 /* Contains window functions */ +#define EP_MemToken 0x010000 /* Need to sqlite3DbFree() Expr.zToken */ +#define EP_NoReduce 0x020000 /* Cannot EXPRDUP_REDUCE this Expr */ +#define EP_Unlikely 0x040000 /* unlikely() or likelihood() function */ +#define EP_ConstFunc 0x080000 /* A SQLITE_FUNC_CONSTANT or _SLOCHNG function */ +#define EP_CanBeNull 0x100000 /* Can be null despite NOT NULL constraint */ +#define EP_Subquery 0x200000 /* Tree contains a TK_SELECT operator */ +#define EP_Alias 0x400000 /* Is an alias for a result set column */ +#define EP_Leaf 0x800000 /* Expr.pLeft, .pRight, .u.pSelect all NULL */ +#define EP_WinFunc 0x1000000 /* TK_FUNCTION with Expr.y.pWin set */ +#define EP_Subrtn 0x2000000 /* Uses Expr.y.sub. TK_IN, _SELECT, or _EXISTS */ +#define EP_Quoted 0x4000000 /* TK_ID was originally quoted */ +#define EP_Static 0x8000000 /* Held in memory not obtained from malloc() */ +#define EP_IsTrue 0x10000000 /* Always has boolean value of TRUE */ +#define EP_IsFalse 0x20000000 /* Always has boolean value of FALSE */ +#define EP_Indirect 0x40000000 /* Contained within a TRIGGER or a VIEW */ /* ** The EP_Propagate mask is a set of properties that automatically propagate @@ -17434,6 +17587,8 @@ struct Expr { #define ExprHasAllProperty(E,P) (((E)->flags&(P))==(P)) #define ExprSetProperty(E,P) (E)->flags|=(P) #define ExprClearProperty(E,P) (E)->flags&=~(P) +#define ExprAlwaysTrue(E) (((E)->flags&(EP_FromJoin|EP_IsTrue))==EP_IsTrue) +#define ExprAlwaysFalse(E) (((E)->flags&(EP_FromJoin|EP_IsFalse))==EP_IsFalse) /* The ExprSetVVAProperty() macro is used for Verification, Validation, ** and Accreditation only. It works like ExprSetProperty() during VVA @@ -17460,6 +17615,18 @@ struct Expr { */ #define EXPRDUP_REDUCE 0x0001 /* Used reduced-size Expr nodes */ +/* +** True if the expression passed as an argument was a function with +** an OVER() clause (a window function). +*/ +#ifdef SQLITE_OMIT_WINDOWFUNC +# define IsWindowFunc(p) 0 +#else +# define IsWindowFunc(p) ( \ + ExprHasProperty((p), EP_WinFunc) && p->y.pWin->eFrmType!=TK_FILTER \ + ) +#endif + /* ** A list of expressions. Each expression may optionally have a ** name. An expr/name combination can be used in several ways, such @@ -17482,11 +17649,12 @@ struct ExprList { Expr *pExpr; /* The parse tree for this expression */ char *zName; /* Token associated with this expression */ char *zSpan; /* Original text of the expression */ - u8 sortOrder; /* 1 for DESC or 0 for ASC */ + u8 sortFlags; /* Mask of KEYINFO_ORDER_* flags */ unsigned done :1; /* A flag to indicate when processing is finished */ unsigned bSpanIsTab :1; /* zSpan holds DB.TABLE.COLUMN */ unsigned reusable :1; /* Constant expression is reusable */ unsigned bSorterRef :1; /* Defer evaluation until after sorting */ + unsigned bNulls: 1; /* True if explicit "NULLS FIRST/LAST" */ union { struct { u16 iOrderByCol; /* For ORDER BY, column number in result set */ @@ -17650,7 +17818,7 @@ struct NameContext { NameContext *pNext; /* Next outer name context. NULL for outermost */ int nRef; /* Number of names resolved by this context */ int nErr; /* Number of errors encountered while resolving names */ - u16 ncFlags; /* Zero or more NC_* flags defined below */ + int ncFlags; /* Zero or more NC_* flags defined below */ Select *pWinSelect; /* SELECT statement for any window functions */ }; @@ -17658,8 +17826,9 @@ struct NameContext { ** Allowed values for the NameContext, ncFlags field. ** ** Value constraints (all checked via assert()): -** NC_HasAgg == SF_HasAgg +** NC_HasAgg == SF_HasAgg == EP_Agg ** NC_MinMaxAgg == SF_MinMaxAgg == SQLITE_FUNC_MINMAX +** NC_HasWin == EP_Win ** */ #define NC_AllowAgg 0x0001 /* Aggregate functions are allowed here */ @@ -17675,6 +17844,8 @@ struct NameContext { #define NC_MinMaxAgg 0x1000 /* min/max aggregates seen. See note above */ #define NC_Complex 0x2000 /* True if a function or subquery seen */ #define NC_AllowWin 0x4000 /* Window functions are allowed here */ +#define NC_HasWin 0x8000 /* One or more window functions seen */ +#define NC_IsDDL 0x10000 /* Resolving names in a CREATE statement */ /* ** An instance of the following object describes a single ON CONFLICT @@ -17774,6 +17945,7 @@ struct Select { #define SF_Converted 0x10000 /* By convertCompoundSelectToSubquery() */ #define SF_IncludeHidden 0x20000 /* Include hidden columns in output */ #define SF_ComplexResult 0x40000 /* Result contains subquery or function */ +#define SF_WhereBegin 0x80000 /* Really a WhereBegin() call. Debug Only */ /* ** The results of a SELECT can be distributed in several ways, as defined @@ -17989,6 +18161,7 @@ struct Parse { AutoincInfo *pAinc; /* Information about AUTOINCREMENT counters */ Parse *pToplevel; /* Parse structure for main program (or NULL) */ Table *pTriggerTab; /* Table triggers are being coded for */ + Parse *pParentParse; /* Parent parser if this parser is nested */ int addrCrTab; /* Address of OP_CreateBtree opcode on CREATE TABLE */ u32 nQueryLoop; /* Est number of iterations of a query (10*log2(N)) */ u32 oldmask; /* Mask of old.* columns referenced */ @@ -18277,11 +18450,12 @@ typedef struct { */ struct Sqlite3Config { int bMemstat; /* True to enable memory status */ - int bCoreMutex; /* True to enable core mutexing */ - int bFullMutex; /* True to enable full mutexing */ - int bOpenUri; /* True to interpret filenames as URIs */ - int bUseCis; /* Use covering indices for full-scans */ - int bSmallMalloc; /* Avoid large memory allocations if true */ + u8 bCoreMutex; /* True to enable core mutexing */ + u8 bFullMutex; /* True to enable full mutexing */ + u8 bOpenUri; /* True to interpret filenames as URIs */ + u8 bUseCis; /* Use covering indices for full-scans */ + u8 bSmallMalloc; /* Avoid large memory allocations if true */ + u8 bExtraSchemaChecks; /* Verify type,name,tbl_name in schema */ int mxStrlen; /* Maximum string length */ int neverCorrupt; /* Database is always well-formed */ int szLookaside; /* Default lookaside buffer size */ @@ -18333,6 +18507,7 @@ struct Sqlite3Config { int bInternalFunctions; /* Internal SQL functions are visible */ int iOnceResetThreshold; /* When to reset OP_Once counters */ u32 szSorterRef; /* Min size in bytes to use sorter-refs */ + unsigned int iPrngSeed; /* Alternative fixed seed for the PRNG */ }; /* @@ -18429,10 +18604,11 @@ struct TreeView { #endif /* SQLITE_DEBUG */ /* -** This object is used in varioius ways, all related to window functions +** This object is used in various ways, most (but not all) related to window +** functions. ** ** (1) A single instance of this structure is attached to the -** the Expr.pWin field for each window function in an expression tree. +** the Expr.y.pWin field for each window function in an expression tree. ** This object holds the information contained in the OVER clause, ** plus additional fields used during code generation. ** @@ -18443,39 +18619,53 @@ struct TreeView { ** (3) The terms of the WINDOW clause of a SELECT are instances of this ** object on a linked list attached to Select.pWinDefn. ** +** (4) For an aggregate function with a FILTER clause, an instance +** of this object is stored in Expr.y.pWin with eFrmType set to +** TK_FILTER. In this case the only field used is Window.pFilter. +** ** The uses (1) and (2) are really the same Window object that just happens -** to be accessible in two different ways. Use (3) is are separate objects. +** to be accessible in two different ways. Use case (3) are separate objects. */ struct Window { char *zName; /* Name of window (may be NULL) */ + char *zBase; /* Name of base window for chaining (may be NULL) */ ExprList *pPartition; /* PARTITION BY clause */ ExprList *pOrderBy; /* ORDER BY clause */ - u8 eType; /* TK_RANGE or TK_ROWS */ + u8 eFrmType; /* TK_RANGE, TK_GROUPS, TK_ROWS, or 0 */ u8 eStart; /* UNBOUNDED, CURRENT, PRECEDING or FOLLOWING */ u8 eEnd; /* UNBOUNDED, CURRENT, PRECEDING or FOLLOWING */ + u8 bImplicitFrame; /* True if frame was implicitly specified */ + u8 eExclude; /* TK_NO, TK_CURRENT, TK_TIES, TK_GROUP, or 0 */ Expr *pStart; /* Expression for " PRECEDING" */ Expr *pEnd; /* Expression for " FOLLOWING" */ + Window **ppThis; /* Pointer to this object in Select.pWin list */ Window *pNextWin; /* Next window function belonging to this SELECT */ Expr *pFilter; /* The FILTER expression */ FuncDef *pFunc; /* The function */ int iEphCsr; /* Partition buffer or Peer buffer */ - int regAccum; - int regResult; + int regAccum; /* Accumulator */ + int regResult; /* Interim result */ int csrApp; /* Function cursor (used by min/max) */ int regApp; /* Function register (also used by min/max) */ - int regPart; /* First in a set of registers holding PARTITION BY - ** and ORDER BY values for the window */ + int regPart; /* Array of registers for PARTITION BY values */ Expr *pOwner; /* Expression object this window is attached to */ int nBufferCol; /* Number of columns in buffer table */ int iArgCol; /* Offset of first argument for this function */ + int regOne; /* Register containing constant value 1 */ + int regStartRowid; + int regEndRowid; + u8 bExprArgs; /* Defer evaluation of window function arguments + ** due to the SQLITE_SUBTYPE flag */ }; #ifndef SQLITE_OMIT_WINDOWFUNC SQLITE_PRIVATE void sqlite3WindowDelete(sqlite3*, Window*); +SQLITE_PRIVATE void sqlite3WindowUnlinkFromSelect(Window*); SQLITE_PRIVATE void sqlite3WindowListDelete(sqlite3 *db, Window *p); -SQLITE_PRIVATE Window *sqlite3WindowAlloc(Parse*, int, int, Expr*, int , Expr*); +SQLITE_PRIVATE Window *sqlite3WindowAlloc(Parse*, int, int, Expr*, int , Expr*, u8); SQLITE_PRIVATE void sqlite3WindowAttach(Parse*, Expr*, Window*); -SQLITE_PRIVATE int sqlite3WindowCompare(Parse*, Window*, Window*); +SQLITE_PRIVATE void sqlite3WindowLink(Select *pSel, Window *pWin); +SQLITE_PRIVATE int sqlite3WindowCompare(Parse*, Window*, Window*, int); SQLITE_PRIVATE void sqlite3WindowCodeInit(Parse*, Window*); SQLITE_PRIVATE void sqlite3WindowCodeStep(Parse*, Select*, WhereInfo*, int, int); SQLITE_PRIVATE int sqlite3WindowRewrite(Parse*, Select*); @@ -18484,6 +18674,8 @@ SQLITE_PRIVATE void sqlite3WindowUpdate(Parse*, Window*, Window*, FuncDef*); SQLITE_PRIVATE Window *sqlite3WindowDup(sqlite3 *db, Expr *pOwner, Window *p); SQLITE_PRIVATE Window *sqlite3WindowListDup(sqlite3 *db, Window *p); SQLITE_PRIVATE void sqlite3WindowFunctions(void); +SQLITE_PRIVATE void sqlite3WindowChain(Parse*, Window*, Window*); +SQLITE_PRIVATE Window *sqlite3WindowAssemble(Parse*, Window*, ExprList*, ExprList*, Token*); #else # define sqlite3WindowDelete(a,b) # define sqlite3WindowFunctions() @@ -18673,8 +18865,12 @@ SQLITE_PRIVATE void sqlite3MutexWarnOnContention(sqlite3_mutex*); #endif #ifndef SQLITE_OMIT_FLOATING_POINT +# define EXP754 (((u64)0x7ff)<<52) +# define MAN754 ((((u64)1)<<52)-1) +# define IsNaN(X) (((X)&EXP754)==EXP754 && ((X)&MAN754)!=0) SQLITE_PRIVATE int sqlite3IsNaN(double); #else +# define IsNaN(X) 0 # define sqlite3IsNaN(X) 0 #endif @@ -18713,6 +18909,7 @@ SQLITE_PRIVATE void sqlite3TreeViewWinFunc(TreeView*, const Window*, u8); SQLITE_PRIVATE void sqlite3SetString(char **, sqlite3*, const char*); SQLITE_PRIVATE void sqlite3ErrorMsg(Parse*, const char*, ...); +SQLITE_PRIVATE int sqlite3ErrorToParser(sqlite3*,int); SQLITE_PRIVATE void sqlite3Dequote(char*); SQLITE_PRIVATE void sqlite3DequoteExpr(Expr*); SQLITE_PRIVATE void sqlite3TokenInit(Token*,char*); @@ -18732,13 +18929,15 @@ SQLITE_PRIVATE Expr *sqlite3Expr(sqlite3*,int,const char*); SQLITE_PRIVATE void sqlite3ExprAttachSubtrees(sqlite3*,Expr*,Expr*,Expr*); SQLITE_PRIVATE Expr *sqlite3PExpr(Parse*, int, Expr*, Expr*); SQLITE_PRIVATE void sqlite3PExprAddSelect(Parse*, Expr*, Select*); -SQLITE_PRIVATE Expr *sqlite3ExprAnd(sqlite3*,Expr*, Expr*); +SQLITE_PRIVATE Expr *sqlite3ExprAnd(Parse*,Expr*, Expr*); +SQLITE_PRIVATE Expr *sqlite3ExprSimplifiedAndOr(Expr*); SQLITE_PRIVATE Expr *sqlite3ExprFunction(Parse*,ExprList*, Token*, int); SQLITE_PRIVATE void sqlite3ExprAssignVarNumber(Parse*, Expr*, u32); SQLITE_PRIVATE void sqlite3ExprDelete(sqlite3*, Expr*); +SQLITE_PRIVATE void sqlite3ExprUnmapAndDelete(Parse*, Expr*); SQLITE_PRIVATE ExprList *sqlite3ExprListAppend(Parse*,ExprList*,Expr*); SQLITE_PRIVATE ExprList *sqlite3ExprListAppendVector(Parse*,ExprList*,IdList*,Expr*); -SQLITE_PRIVATE void sqlite3ExprListSetSortOrder(ExprList*,int); +SQLITE_PRIVATE void sqlite3ExprListSetSortOrder(ExprList*,int,int); SQLITE_PRIVATE void sqlite3ExprListSetName(Parse*,ExprList*,Token*,int); SQLITE_PRIVATE void sqlite3ExprListSetSpan(Parse*,ExprList*,const char*,const char*); SQLITE_PRIVATE void sqlite3ExprListDelete(sqlite3*, ExprList*); @@ -18757,8 +18956,8 @@ SQLITE_PRIVATE void sqlite3CollapseDatabaseArray(sqlite3*); SQLITE_PRIVATE void sqlite3CommitInternalChanges(sqlite3*); SQLITE_PRIVATE void sqlite3DeleteColumnNames(sqlite3*,Table*); SQLITE_PRIVATE int sqlite3ColumnsFromExprList(Parse*,ExprList*,i16*,Column**); -SQLITE_PRIVATE void sqlite3SelectAddColumnTypeAndCollation(Parse*,Table*,Select*); -SQLITE_PRIVATE Table *sqlite3ResultSetOfSelect(Parse*,Select*); +SQLITE_PRIVATE void sqlite3SelectAddColumnTypeAndCollation(Parse*,Table*,Select*,char); +SQLITE_PRIVATE Table *sqlite3ResultSetOfSelect(Parse*,Select*,char); SQLITE_PRIVATE void sqlite3OpenMasterTable(Parse *, int); SQLITE_PRIVATE Index *sqlite3PrimaryKeyIndex(Table*); SQLITE_PRIVATE i16 sqlite3ColumnOfIndex(Index*, i16); @@ -19044,6 +19243,7 @@ SQLITE_PRIVATE int sqlite3FixSelect(DbFixer*, Select*); SQLITE_PRIVATE int sqlite3FixExpr(DbFixer*, Expr*); SQLITE_PRIVATE int sqlite3FixExprList(DbFixer*, ExprList*); SQLITE_PRIVATE int sqlite3FixTriggerStep(DbFixer*, TriggerStep*); +SQLITE_PRIVATE int sqlite3RealSameAsInt(double,sqlite3_int64); SQLITE_PRIVATE int sqlite3AtoF(const char *z, double*, int, u8); SQLITE_PRIVATE int sqlite3GetInt32(const char *, int*); SQLITE_PRIVATE int sqlite3Atoi(const char*); @@ -19058,7 +19258,7 @@ SQLITE_PRIVATE LogEst sqlite3LogEstAdd(LogEst,LogEst); SQLITE_PRIVATE LogEst sqlite3LogEstFromDouble(double); #endif #if defined(SQLITE_ENABLE_STMT_SCANSTATUS) || \ - defined(SQLITE_ENABLE_STAT3_OR_STAT4) || \ + defined(SQLITE_ENABLE_STAT4) || \ defined(SQLITE_EXPLAIN_ESTIMATED_ROWS) SQLITE_PRIVATE u64 sqlite3LogEstToInt(LogEst); #endif @@ -19124,9 +19324,10 @@ SQLITE_PRIVATE int sqlite3ExprCollSeqMatch(Parse*,Expr*,Expr*); SQLITE_PRIVATE Expr *sqlite3ExprAddCollateToken(Parse *pParse, Expr*, const Token*, int); SQLITE_PRIVATE Expr *sqlite3ExprAddCollateString(Parse*,Expr*,const char*); SQLITE_PRIVATE Expr *sqlite3ExprSkipCollate(Expr*); +SQLITE_PRIVATE Expr *sqlite3ExprSkipCollateAndLikely(Expr*); SQLITE_PRIVATE int sqlite3CheckCollSeq(Parse *, CollSeq *); SQLITE_PRIVATE int sqlite3WritableSchema(sqlite3*); -SQLITE_PRIVATE int sqlite3CheckObjectName(Parse *, const char *); +SQLITE_PRIVATE int sqlite3CheckObjectName(Parse*, const char*,const char*,const char*); SQLITE_PRIVATE void sqlite3VdbeSetChanges(sqlite3 *, int); SQLITE_PRIVATE int sqlite3AddInt64(i64*,i64); SQLITE_PRIVATE int sqlite3SubInt64(i64*,i64); @@ -19145,6 +19346,9 @@ SQLITE_PRIVATE void sqlite3ValueSetStr(sqlite3_value*, int, const void *,u8, void(*)(void*)); SQLITE_PRIVATE void sqlite3ValueSetNull(sqlite3_value*); SQLITE_PRIVATE void sqlite3ValueFree(sqlite3_value*); +#ifndef SQLITE_UNTESTABLE +SQLITE_PRIVATE void sqlite3ResultIntReal(sqlite3_context*); +#endif SQLITE_PRIVATE sqlite3_value *sqlite3ValueNew(sqlite3 *); #ifndef SQLITE_OMIT_UTF16 SQLITE_PRIVATE char *sqlite3Utf16to8(sqlite3 *, const void*, int, u8); @@ -19156,7 +19360,6 @@ SQLITE_PRIVATE const unsigned char sqlite3OpcodeProperty[]; SQLITE_PRIVATE const char sqlite3StrBINARY[]; SQLITE_PRIVATE const unsigned char sqlite3UpperToLower[]; SQLITE_PRIVATE const unsigned char sqlite3CtypeMap[]; -SQLITE_PRIVATE const Token sqlite3IntTokens[]; SQLITE_PRIVATE SQLITE_WSD struct Sqlite3Config sqlite3Config; SQLITE_PRIVATE FuncDefHash sqlite3BuiltinFunctions; #ifndef SQLITE_OMIT_WSD @@ -19174,7 +19377,7 @@ SQLITE_PRIVATE void sqlite3AlterRenameColumn(Parse*, SrcList*, Token*, Token*); SQLITE_PRIVATE int sqlite3GetToken(const unsigned char *, int *); SQLITE_PRIVATE void sqlite3NestedParse(Parse*, const char*, ...); SQLITE_PRIVATE void sqlite3ExpirePreparedStatements(sqlite3*, int); -SQLITE_PRIVATE void sqlite3CodeRhsOfIN(Parse*, Expr*, int, int); +SQLITE_PRIVATE void sqlite3CodeRhsOfIN(Parse*, Expr*, int); SQLITE_PRIVATE int sqlite3CodeSubselect(Parse*, Expr*); SQLITE_PRIVATE void sqlite3SelectPrep(Parse*, Select*, NameContext*); SQLITE_PRIVATE void sqlite3SelectWrongNumTermsError(Parse *pParse, Select *p); @@ -19210,6 +19413,7 @@ SQLITE_PRIVATE void sqlite3KeyInfoUnref(KeyInfo*); SQLITE_PRIVATE KeyInfo *sqlite3KeyInfoRef(KeyInfo*); SQLITE_PRIVATE KeyInfo *sqlite3KeyInfoOfIndex(Parse*, Index*); SQLITE_PRIVATE KeyInfo *sqlite3KeyInfoFromExprList(Parse*, ExprList*, int, int); +SQLITE_PRIVATE int sqlite3HasExplicitNulls(Parse*, ExprList*); #ifdef SQLITE_DEBUG SQLITE_PRIVATE int sqlite3KeyInfoIsWriteable(KeyInfo*); @@ -19242,8 +19446,7 @@ SQLITE_PRIVATE int sqlite3ExprCheckIN(Parse*, Expr*); # define sqlite3ExprCheckIN(x,y) SQLITE_OK #endif -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 -SQLITE_PRIVATE void sqlite3AnalyzeFunctions(void); +#ifdef SQLITE_ENABLE_STAT4 SQLITE_PRIVATE int sqlite3Stat4ProbeSetValue( Parse*,Index*,UnpackedRecord**,Expr*,int,int,int*); SQLITE_PRIVATE int sqlite3Stat4ValueFromExpr(Parse*, Expr*, u8, sqlite3_value**); @@ -19290,6 +19493,7 @@ SQLITE_PRIVATE int sqlite3Utf8To8(unsigned char*); # define sqlite3VtabInSync(db) 0 # define sqlite3VtabLock(X) # define sqlite3VtabUnlock(X) +# define sqlite3VtabModuleUnref(D,X) # define sqlite3VtabUnlockList(X) # define sqlite3VtabSavepoint(X, Y, Z) SQLITE_OK # define sqlite3GetVTable(X,Y) ((VTable*)0) @@ -19301,6 +19505,7 @@ SQLITE_PRIVATE int sqlite3VtabRollback(sqlite3 *db); SQLITE_PRIVATE int sqlite3VtabCommit(sqlite3 *db); SQLITE_PRIVATE void sqlite3VtabLock(VTable *); SQLITE_PRIVATE void sqlite3VtabUnlock(VTable *); +SQLITE_PRIVATE void sqlite3VtabModuleUnref(sqlite3*,Module*); SQLITE_PRIVATE void sqlite3VtabUnlockList(sqlite3*); SQLITE_PRIVATE int sqlite3VtabSavepoint(sqlite3 *, int, int); SQLITE_PRIVATE void sqlite3VtabImportErrmsg(Vdbe*, sqlite3_vtab*); @@ -19707,8 +19912,15 @@ SQLITE_PRIVATE const unsigned char sqlite3CtypeMap[256] = { ** SQLITE_ALLOW_COVERING_INDEX_SCAN compile-time option, or is "on" if ** that compile-time option is omitted. */ -#ifndef SQLITE_ALLOW_COVERING_INDEX_SCAN +#if !defined(SQLITE_ALLOW_COVERING_INDEX_SCAN) # define SQLITE_ALLOW_COVERING_INDEX_SCAN 1 +#else +# if !SQLITE_ALLOW_COVERING_INDEX_SCAN +# error "Compile-time disabling of covering index scan using the\ + -DSQLITE_ALLOW_COVERING_INDEX_SCAN=0 option is deprecated.\ + Contact SQLite developers if this is a problem for you, and\ + delete this #error macro to continue with your build." +# endif #endif /* The minimum PMA size is set to this value multiplied by the database @@ -19761,6 +19973,7 @@ SQLITE_PRIVATE SQLITE_WSD struct Sqlite3Config sqlite3Config = { SQLITE_USE_URI, /* bOpenUri */ SQLITE_ALLOW_COVERING_INDEX_SCAN, /* bUseCis */ 0, /* bSmallMalloc */ + 1, /* bExtraSchemaChecks */ 0x7ffffffe, /* mxStrlen */ 0, /* neverCorrupt */ SQLITE_DEFAULT_LOOKASIDE, /* szLookaside, nLookaside */ @@ -19807,6 +20020,7 @@ SQLITE_PRIVATE SQLITE_WSD struct Sqlite3Config sqlite3Config = { 0, /* bInternalFunctions */ 0x7ffffffe, /* iOnceResetThreshold */ SQLITE_DEFAULT_SORTERREF_SIZE, /* szSorterRef */ + 0, /* iPrngSeed */ }; /* @@ -19816,14 +20030,6 @@ SQLITE_PRIVATE SQLITE_WSD struct Sqlite3Config sqlite3Config = { */ SQLITE_PRIVATE FuncDefHash sqlite3BuiltinFunctions; -/* -** Constant tokens for values 0 and 1. -*/ -SQLITE_PRIVATE const Token sqlite3IntTokens[] = { - { "0", 1 }, - { "1", 1 } -}; - #ifdef VDBE_PROFILE /* ** The following performance counter can be used in place of @@ -20135,12 +20341,12 @@ struct sqlite3_value { #define MEM_Int 0x0004 /* Value is an integer */ #define MEM_Real 0x0008 /* Value is a real number */ #define MEM_Blob 0x0010 /* Value is a BLOB */ -#define MEM_AffMask 0x001f /* Mask of affinity bits */ -/* Available 0x0020 */ -/* Available 0x0040 */ +#define MEM_IntReal 0x0020 /* MEM_Int that stringifies like MEM_Real */ +#define MEM_AffMask 0x003f /* Mask of affinity bits */ +#define MEM_FromBind 0x0040 /* Value originates from sqlite3_bind() */ #define MEM_Undefined 0x0080 /* Value is undefined */ #define MEM_Cleared 0x0100 /* NULL set by OP_Null, not from data */ -#define MEM_TypeMask 0xc1ff /* Mask of type bits */ +#define MEM_TypeMask 0xc1bf /* Mask of type bits */ /* Whenever Mem contains a valid string or blob representation, one of @@ -20172,6 +20378,12 @@ struct sqlite3_value { #define MemSetTypeFlag(p, f) \ ((p)->flags = ((p)->flags&~(MEM_TypeMask|MEM_Zero))|f) +/* +** True if Mem X is a NULL-nochng type. +*/ +#define MemNullNochng(X) \ + ((X)->flags==(MEM_Null|MEM_Zero) && (X)->n==0 && (X)->u.nZero==0) + /* ** Return true if a memory cell is not marked as invalid. This macro ** is for use inside assert() statements only. @@ -20370,7 +20582,6 @@ SQLITE_PRIVATE int sqlite3VdbeCursorMoveto(VdbeCursor**, int*); SQLITE_PRIVATE int sqlite3VdbeCursorRestore(VdbeCursor*); SQLITE_PRIVATE u32 sqlite3VdbeSerialTypeLen(u32); SQLITE_PRIVATE u8 sqlite3VdbeOneByteSerialTypeLen(u8); -SQLITE_PRIVATE u32 sqlite3VdbeSerialType(Mem*, int, u32*); SQLITE_PRIVATE u32 sqlite3VdbeSerialPut(unsigned char*, Mem*, u32); SQLITE_PRIVATE u32 sqlite3VdbeSerialGet(const unsigned char*, u32, Mem*); SQLITE_PRIVATE void sqlite3VdbeDeleteAuxData(sqlite3*, AuxData**, int, int); @@ -21254,7 +21465,7 @@ static int parseDateOrTime( return 0; }else if( sqlite3StrICmp(zDate,"now")==0 && sqlite3NotPureFunc(context) ){ return setDateTimeToCurrent(context, p); - }else if( sqlite3AtoF(zDate, &r, sqlite3Strlen30(zDate), SQLITE_UTF8) ){ + }else if( sqlite3AtoF(zDate, &r, sqlite3Strlen30(zDate), SQLITE_UTF8)>0 ){ setRawDateNumber(p, r); return 0; } @@ -21588,7 +21799,7 @@ static int parseModifier( ** date is already on the appropriate weekday, this is a no-op. */ if( sqlite3_strnicmp(z, "weekday ", 8)==0 - && sqlite3AtoF(&z[8], &r, sqlite3Strlen30(&z[8]), SQLITE_UTF8) + && sqlite3AtoF(&z[8], &r, sqlite3Strlen30(&z[8]), SQLITE_UTF8)>0 && (n=(int)r)==r && n>=0 && r<7 ){ sqlite3_int64 Z; computeYMD_HMS(p); @@ -21647,7 +21858,7 @@ static int parseModifier( double rRounder; int i; for(n=1; z[n] && z[n]!=':' && !sqlite3Isspace(z[n]); n++){} - if( !sqlite3AtoF(z, &r, n, SQLITE_UTF8) ){ + if( sqlite3AtoF(z, &r, n, SQLITE_UTF8)<=0 ){ rc = 1; break; } @@ -22376,7 +22587,15 @@ SQLITE_PRIVATE void sqlite3OsDlClose(sqlite3_vfs *pVfs, void *pHandle){ } #endif /* SQLITE_OMIT_LOAD_EXTENSION */ SQLITE_PRIVATE int sqlite3OsRandomness(sqlite3_vfs *pVfs, int nByte, char *zBufOut){ - return pVfs->xRandomness(pVfs, nByte, zBufOut); + if( sqlite3Config.iPrngSeed ){ + memset(zBufOut, 0, nByte); + if( ALWAYS(nByte>(signed)sizeof(unsigned)) ) nByte = sizeof(unsigned int); + memcpy(zBufOut, &sqlite3Config.iPrngSeed, nByte); + return SQLITE_OK; + }else{ + return pVfs->xRandomness(pVfs, nByte, zBufOut); + } + } SQLITE_PRIVATE int sqlite3OsSleep(sqlite3_vfs *pVfs, int nMicro){ return pVfs->xSleep(pVfs, nMicro); @@ -27121,6 +27340,9 @@ SQLITE_PRIVATE void sqlite3OomFault(sqlite3 *db){ db->u1.isInterrupted = 1; } db->lookaside.bDisable++; + if( db->pParse ){ + db->pParse->rc = SQLITE_NOMEM_BKPT; + } } } @@ -27277,6 +27499,12 @@ static const et_info fmtinfo[] = { { 'r', 10, 1, etORDINAL, 0, 0 }, }; +/* Floating point constants used for rounding */ +static const double arRound[] = { + 5.0e-01, 5.0e-02, 5.0e-03, 5.0e-04, 5.0e-05, + 5.0e-06, 5.0e-07, 5.0e-08, 5.0e-09, 5.0e-10, +}; + /* ** If SQLITE_OMIT_FLOATING_POINT is defined, then none of the floating point ** conversions will work. @@ -27314,7 +27542,8 @@ static char et_getdigit(LONGDOUBLE_TYPE *val, int *cnt){ static void setStrAccumError(StrAccum *p, u8 eError){ assert( eError==SQLITE_NOMEM || eError==SQLITE_TOOBIG ); p->accError = eError; - p->nAlloc = 0; + if( p->mxAlloc ) sqlite3_str_reset(p); + if( eError==SQLITE_TOOBIG ) sqlite3ErrorToParser(p->db, eError); } /* @@ -27344,6 +27573,7 @@ static char *getTextArg(PrintfArguments *p){ */ static char *printfTempBuf(sqlite3_str *pAccum, sqlite3_int64 n){ char *z; + if( pAccum->accError ) return 0; if( n>pAccum->nAlloc && n>pAccum->mxAlloc ){ setStrAccumError(pAccum, SQLITE_TOOBIG); return 0; @@ -27693,8 +27923,18 @@ SQLITE_API void sqlite3_str_vappendf( } if( xtype==etGENERIC && precision>0 ) precision--; testcase( precision>0xfff ); - for(idx=precision&0xfff, rounder=0.5; idx>0; idx--, rounder*=0.1){} - if( xtype==etFLOAT ) realvalue += rounder; + idx = precision & 0xfff; + rounder = arRound[idx%10]; + while( idx>=10 ){ rounder *= 1.0e-10; idx -= 10; } + if( xtype==etFLOAT ){ + double rx = (double)realvalue; + sqlite3_uint64 u; + int ex; + memcpy(&u, &rx, sizeof(u)); + ex = -1023 + (int)((u>>52)&0x7ff); + if( precision+(ex/3) < 15 ) rounder += realvalue*3e-16; + realvalue += rounder; + } /* Normalize realvalue to within 10.0 > realvalue >= 1.0 */ exp = 0; if( sqlite3IsNaN((double)realvalue) ){ @@ -28063,9 +28303,8 @@ static int sqlite3StrAccumEnlarge(StrAccum *p, int N){ return 0; } if( p->mxAlloc==0 ){ - N = p->nAlloc - p->nChar - 1; setStrAccumError(p, SQLITE_TOOBIG); - return N; + return p->nAlloc - p->nChar - 1; }else{ char *zOld = isMalloced(p) ? p->zText : 0; i64 szNew = p->nChar; @@ -28137,7 +28376,7 @@ SQLITE_API void sqlite3_str_append(sqlite3_str *p, const char *z, int N){ assert( z!=0 || N==0 ); assert( p->zText!=0 || p->nChar==0 || p->accError ); assert( N>=0 ); - assert( p->accError==0 || p->nAlloc==0 ); + assert( p->accError==0 || p->nAlloc==0 || p->mxAlloc==0 ); if( p->nChar+N >= p->nAlloc ){ enlargeAndAppend(p,z,N); }else if( N ){ @@ -28637,13 +28876,17 @@ SQLITE_PRIVATE void sqlite3TreeViewSelect(TreeView *pView, const Select *p, u8 m sqlite3TreeViewPush(pView, 1); } do{ - sqlite3TreeViewLine(pView, - "SELECT%s%s (%u/%p) selFlags=0x%x nSelectRow=%d", - ((p->selFlags & SF_Distinct) ? " DISTINCT" : ""), - ((p->selFlags & SF_Aggregate) ? " agg_flag" : ""), - p->selId, p, p->selFlags, - (int)p->nSelectRow - ); + if( p->selFlags & SF_WhereBegin ){ + sqlite3TreeViewLine(pView, "sqlite3WhereBegin()"); + }else{ + sqlite3TreeViewLine(pView, + "SELECT%s%s (%u/%p) selFlags=0x%x nSelectRow=%d", + ((p->selFlags & SF_Distinct) ? " DISTINCT" : ""), + ((p->selFlags & SF_Aggregate) ? " agg_flag" : ""), + p->selId, p, p->selFlags, + (int)p->nSelectRow + ); + } if( cnt++ ) sqlite3TreeViewPop(pView); if( p->pPrior ){ n = 1000; @@ -28660,7 +28903,10 @@ SQLITE_PRIVATE void sqlite3TreeViewSelect(TreeView *pView, const Select *p, u8 m if( p->pWinDefn ) n++; #endif } - sqlite3TreeViewExprList(pView, p->pEList, (n--)>0, "result-set"); + if( p->pEList ){ + sqlite3TreeViewExprList(pView, p->pEList, n>0, "result-set"); + } + n--; #ifndef SQLITE_OMIT_WINDOWFUNC if( p->pWin ){ Window *pX; @@ -28770,24 +29016,62 @@ SQLITE_PRIVATE void sqlite3TreeViewBound( ** Generate a human-readable explanation for a Window object */ SQLITE_PRIVATE void sqlite3TreeViewWindow(TreeView *pView, const Window *pWin, u8 more){ + int nElement = 0; + if( pWin->pFilter ){ + sqlite3TreeViewItem(pView, "FILTER", 1); + sqlite3TreeViewExpr(pView, pWin->pFilter, 0); + sqlite3TreeViewPop(pView); + } pView = sqlite3TreeViewPush(pView, more); if( pWin->zName ){ - sqlite3TreeViewLine(pView, "OVER %s", pWin->zName); + sqlite3TreeViewLine(pView, "OVER %s (%p)", pWin->zName, pWin); }else{ - sqlite3TreeViewLine(pView, "OVER"); + sqlite3TreeViewLine(pView, "OVER (%p)", pWin); + } + if( pWin->zBase ) nElement++; + if( pWin->pOrderBy ) nElement++; + if( pWin->eFrmType ) nElement++; + if( pWin->eExclude ) nElement++; + if( pWin->zBase ){ + sqlite3TreeViewPush(pView, (--nElement)>0); + sqlite3TreeViewLine(pView, "window: %s", pWin->zBase); + sqlite3TreeViewPop(pView); } if( pWin->pPartition ){ - sqlite3TreeViewExprList(pView, pWin->pPartition, 1, "PARTITION-BY"); + sqlite3TreeViewExprList(pView, pWin->pPartition, nElement>0,"PARTITION-BY"); } if( pWin->pOrderBy ){ - sqlite3TreeViewExprList(pView, pWin->pOrderBy, 1, "ORDER-BY"); - } - if( pWin->eType ){ - sqlite3TreeViewItem(pView, pWin->eType==TK_RANGE ? "RANGE" : "ROWS", 0); + sqlite3TreeViewExprList(pView, pWin->pOrderBy, (--nElement)>0, "ORDER-BY"); + } + if( pWin->eFrmType ){ + char zBuf[30]; + const char *zFrmType = "ROWS"; + if( pWin->eFrmType==TK_RANGE ) zFrmType = "RANGE"; + if( pWin->eFrmType==TK_GROUPS ) zFrmType = "GROUPS"; + sqlite3_snprintf(sizeof(zBuf),zBuf,"%s%s",zFrmType, + pWin->bImplicitFrame ? " (implied)" : ""); + sqlite3TreeViewItem(pView, zBuf, (--nElement)>0); sqlite3TreeViewBound(pView, pWin->eStart, pWin->pStart, 1); sqlite3TreeViewBound(pView, pWin->eEnd, pWin->pEnd, 0); sqlite3TreeViewPop(pView); } + if( pWin->eExclude ){ + char zBuf[30]; + const char *zExclude; + switch( pWin->eExclude ){ + case TK_NO: zExclude = "NO OTHERS"; break; + case TK_CURRENT: zExclude = "CURRENT ROW"; break; + case TK_GROUP: zExclude = "GROUP"; break; + case TK_TIES: zExclude = "TIES"; break; + default: + sqlite3_snprintf(sizeof(zBuf),zBuf,"invalid(%d)", pWin->eExclude); + zExclude = zBuf; + break; + } + sqlite3TreeViewPush(pView, 0); + sqlite3TreeViewLine(pView, "EXCLUDE %s", zExclude); + sqlite3TreeViewPop(pView); + } sqlite3TreeViewPop(pView); } #endif /* SQLITE_OMIT_WINDOWFUNC */ @@ -28818,12 +29102,14 @@ SQLITE_PRIVATE void sqlite3TreeViewExpr(TreeView *pView, const Expr *pExpr, u8 m sqlite3TreeViewPop(pView); return; } - if( pExpr->flags ){ + if( pExpr->flags || pExpr->affExpr ){ if( ExprHasProperty(pExpr, EP_FromJoin) ){ - sqlite3_snprintf(sizeof(zFlgs),zFlgs," flags=0x%x iRJT=%d", - pExpr->flags, pExpr->iRightJoinTable); + sqlite3_snprintf(sizeof(zFlgs),zFlgs," fg.af=%x.%c iRJT=%d", + pExpr->flags, pExpr->affExpr ? pExpr->affExpr : 'n', + pExpr->iRightJoinTable); }else{ - sqlite3_snprintf(sizeof(zFlgs),zFlgs," flags=0x%x",pExpr->flags); + sqlite3_snprintf(sizeof(zFlgs),zFlgs," fg.af=%x.%c", + pExpr->flags, pExpr->affExpr ? pExpr->affExpr : 'n'); } }else{ zFlgs[0] = 0; @@ -28937,7 +29223,7 @@ SQLITE_PRIVATE void sqlite3TreeViewExpr(TreeView *pView, const Expr *pExpr, u8 m }; assert( pExpr->op2==TK_IS || pExpr->op2==TK_ISNOT ); assert( pExpr->pRight ); - assert( pExpr->pRight->op==TK_TRUEFALSE ); + assert( sqlite3ExprSkipCollate(pExpr->pRight)->op==TK_TRUEFALSE ); x = (pExpr->op2==TK_ISNOT)*2 + sqlite3ExprTruthValue(pExpr->pRight); zUniOp = azOp[x]; break; @@ -28950,7 +29236,14 @@ SQLITE_PRIVATE void sqlite3TreeViewExpr(TreeView *pView, const Expr *pExpr, u8 m } case TK_COLLATE: { - sqlite3TreeViewLine(pView, "COLLATE %Q", pExpr->u.zToken); + /* COLLATE operators without the EP_Collate flag are intended to + ** emulate collation associated with a table column. These show + ** up in the treeview output as "SOFT-COLLATE". Explicit COLLATE + ** operators that appear in the original SQL always have the + ** EP_Collate bit set and appear in treeview output as just "COLLATE" */ + sqlite3TreeViewLine(pView, "%sCOLLATE %Q%s", + !ExprHasProperty(pExpr, EP_Collate) ? "SOFT-" : "", + pExpr->u.zToken, zFlgs); sqlite3TreeViewExpr(pView, pExpr->pLeft, 0); break; } @@ -28971,10 +29264,10 @@ SQLITE_PRIVATE void sqlite3TreeViewExpr(TreeView *pView, const Expr *pExpr, u8 m #endif } if( pExpr->op==TK_AGG_FUNCTION ){ - sqlite3TreeViewLine(pView, "AGG_FUNCTION%d %Q", - pExpr->op2, pExpr->u.zToken); + sqlite3TreeViewLine(pView, "AGG_FUNCTION%d %Q%s", + pExpr->op2, pExpr->u.zToken, zFlgs); }else{ - sqlite3TreeViewLine(pView, "FUNCTION %Q", pExpr->u.zToken); + sqlite3TreeViewLine(pView, "FUNCTION %Q%s", pExpr->u.zToken, zFlgs); } if( pFarg ){ sqlite3TreeViewExprList(pView, pFarg, pWin!=0, 0); @@ -29051,7 +29344,7 @@ SQLITE_PRIVATE void sqlite3TreeViewExpr(TreeView *pView, const Expr *pExpr, u8 m #ifndef SQLITE_OMIT_TRIGGER case TK_RAISE: { const char *zType = "unk"; - switch( pExpr->affinity ){ + switch( pExpr->affExpr ){ case OE_Rollback: zType = "rollback"; break; case OE_Abort: zType = "abort"; break; case OE_Fail: zType = "fail"; break; @@ -29092,7 +29385,7 @@ SQLITE_PRIVATE void sqlite3TreeViewExpr(TreeView *pView, const Expr *pExpr, u8 m sqlite3TreeViewExpr(pView, pExpr->pRight, 0); }else if( zUniOp ){ sqlite3TreeViewLine(pView, "%s%s", zUniOp, zFlgs); - sqlite3TreeViewExpr(pView, pExpr->pLeft, 0); + sqlite3TreeViewExpr(pView, pExpr->pLeft, 0); } sqlite3TreeViewPop(pView); } @@ -29767,11 +30060,11 @@ SQLITE_PRIVATE u32 sqlite3Utf8Read( ** encoding, or if *pMem does not contain a string value. */ SQLITE_PRIVATE SQLITE_NOINLINE int sqlite3VdbeMemTranslate(Mem *pMem, u8 desiredEnc){ - int len; /* Maximum length of output string in bytes */ - unsigned char *zOut; /* Output buffer */ - unsigned char *zIn; /* Input iterator */ - unsigned char *zTerm; /* End of input */ - unsigned char *z; /* Output iterator */ + sqlite3_int64 len; /* Maximum length of output string in bytes */ + unsigned char *zOut; /* Output buffer */ + unsigned char *zIn; /* Input iterator */ + unsigned char *zTerm; /* End of input */ + unsigned char *z; /* Output iterator */ unsigned int c; assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); @@ -29820,14 +30113,14 @@ SQLITE_PRIVATE SQLITE_NOINLINE int sqlite3VdbeMemTranslate(Mem *pMem, u8 desired ** nul-terminator. */ pMem->n &= ~1; - len = pMem->n * 2 + 1; + len = 2 * (sqlite3_int64)pMem->n + 1; }else{ /* When converting from UTF-8 to UTF-16 the maximum growth is caused ** when a 1-byte UTF-8 character is translated into a 2-byte UTF-16 ** character. Two bytes are required in the output buffer for the ** nul-terminator. */ - len = pMem->n * 2 + 2; + len = 2 * (sqlite3_int64)pMem->n + 2; } /* Set zIn to point at the start of the input buffer and zTerm to point 1 @@ -30119,9 +30412,7 @@ SQLITE_PRIVATE void sqlite3UtfSelfTest(void){ */ /* #include "sqliteInt.h" */ /* #include */ -#if HAVE_ISNAN || SQLITE_HAVE_ISNAN -# include -#endif +#include /* ** Routine needed to support the testcase() macro. @@ -30134,15 +30425,23 @@ SQLITE_PRIVATE void sqlite3Coverage(int x){ #endif /* -** Give a callback to the test harness that can be used to simulate faults -** in places where it is difficult or expensive to do so purely by means -** of inputs. +** Calls to sqlite3FaultSim() are used to simulate a failure during testing, +** or to bypass normal error detection during testing in order to let +** execute proceed futher downstream. +** +** In deployment, sqlite3FaultSim() *always* return SQLITE_OK (0). The +** sqlite3FaultSim() function only returns non-zero during testing. ** -** The intent of the integer argument is to let the fault simulator know -** which of multiple sqlite3FaultSim() calls has been hit. +** During testing, if the test harness has set a fault-sim callback using +** a call to sqlite3_test_control(SQLITE_TESTCTRL_FAULT_INSTALL), then +** each call to sqlite3FaultSim() is relayed to that application-supplied +** callback and the integer return value form the application-supplied +** callback is returned by sqlite3FaultSim(). ** -** Return whatever integer value the test callback returns, or return -** SQLITE_OK if no test callback is installed. +** The integer argument to sqlite3FaultSim() is a code to identify which +** sqlite3FaultSim() instance is being invoked. Each call to sqlite3FaultSim() +** should have a unique code. To prevent legacy testing applications from +** breaking, the codes should not be changed or reused. */ #ifndef SQLITE_UNTESTABLE SQLITE_PRIVATE int sqlite3FaultSim(int iTest){ @@ -30154,47 +30453,11 @@ SQLITE_PRIVATE int sqlite3FaultSim(int iTest){ #ifndef SQLITE_OMIT_FLOATING_POINT /* ** Return true if the floating point value is Not a Number (NaN). -** -** Use the math library isnan() function if compiled with SQLITE_HAVE_ISNAN. -** Otherwise, we have our own implementation that works on most systems. */ SQLITE_PRIVATE int sqlite3IsNaN(double x){ - int rc; /* The value return */ -#if !SQLITE_HAVE_ISNAN && !HAVE_ISNAN - /* - ** Systems that support the isnan() library function should probably - ** make use of it by compiling with -DSQLITE_HAVE_ISNAN. But we have - ** found that many systems do not have a working isnan() function so - ** this implementation is provided as an alternative. - ** - ** This NaN test sometimes fails if compiled on GCC with -ffast-math. - ** On the other hand, the use of -ffast-math comes with the following - ** warning: - ** - ** This option [-ffast-math] should never be turned on by any - ** -O option since it can result in incorrect output for programs - ** which depend on an exact implementation of IEEE or ISO - ** rules/specifications for math functions. - ** - ** Under MSVC, this NaN test may fail if compiled with a floating- - ** point precision mode other than /fp:precise. From the MSDN - ** documentation: - ** - ** The compiler [with /fp:precise] will properly handle comparisons - ** involving NaN. For example, x != x evaluates to true if x is NaN - ** ... - */ -#ifdef __FAST_MATH__ -# error SQLite will not work correctly with the -ffast-math option of GCC. -#endif - volatile double y = x; - volatile double z = y; - rc = (y!=z); -#else /* if HAVE_ISNAN */ - rc = isnan(x); -#endif /* HAVE_ISNAN */ - testcase( rc ); - return rc; + u64 y; + memcpy(&y,&x,sizeof(y)); + return IsNaN(y); } #endif /* SQLITE_OMIT_FLOATING_POINT */ @@ -30327,6 +30590,19 @@ SQLITE_PRIVATE void sqlite3ErrorMsg(Parse *pParse, const char *zFormat, ...){ } } +/* +** If database connection db is currently parsing SQL, then transfer +** error code errCode to that parser if the parser has not already +** encountered some other kind of error. +*/ +SQLITE_PRIVATE int sqlite3ErrorToParser(sqlite3 *db, int errCode){ + Parse *pParse; + if( db==0 || (pParse = db->pParse)==0 ) return errCode; + pParse->rc = errCode; + pParse->nErr++; + return errCode; +} + /* ** Convert an SQL-style quoted string into a normal string by removing ** the quote characters. The conversion is done in-place. If the @@ -30403,12 +30679,18 @@ SQLITE_API int sqlite3_stricmp(const char *zLeft, const char *zRight){ } SQLITE_PRIVATE int sqlite3StrICmp(const char *zLeft, const char *zRight){ unsigned char *a, *b; - int c; + int c, x; a = (unsigned char *)zLeft; b = (unsigned char *)zRight; for(;;){ - c = (int)UpperToLower[*a] - (int)UpperToLower[*b]; - if( c || *a==0 ) break; + c = *a; + x = *b; + if( c==x ){ + if( c==0 ) break; + }else{ + c = (int)UpperToLower[c] - (int)UpperToLower[x]; + if( c ) break; + } a++; b++; } @@ -30436,15 +30718,15 @@ SQLITE_API int sqlite3_strnicmp(const char *zLeft, const char *zRight, int N){ static LONGDOUBLE_TYPE sqlite3Pow10(int E){ #if defined(_MSC_VER) static const LONGDOUBLE_TYPE x[] = { - 1.0e+001, - 1.0e+002, - 1.0e+004, - 1.0e+008, - 1.0e+016, - 1.0e+032, - 1.0e+064, - 1.0e+128, - 1.0e+256 + 1.0e+001L, + 1.0e+002L, + 1.0e+004L, + 1.0e+008L, + 1.0e+016L, + 1.0e+032L, + 1.0e+064L, + 1.0e+128L, + 1.0e+256L }; LONGDOUBLE_TYPE r = 1.0; int i; @@ -30474,8 +30756,15 @@ static LONGDOUBLE_TYPE sqlite3Pow10(int E){ ** uses the encoding enc. The string is not necessarily zero-terminated. ** ** Return TRUE if the result is a valid real number (or integer) and FALSE -** if the string is empty or contains extraneous text. Valid numbers -** are in one of these formats: +** if the string is empty or contains extraneous text. More specifically +** return +** 1 => The input string is a pure integer +** 2 or more => The input has a decimal point or eNNN clause +** 0 or less => The input string is not a valid number +** -1 => Not a valid number, but has a valid prefix which +** includes a decimal point and/or an eNNN clause +** +** Valid numbers are in one of these formats: ** ** [+-]digits[E[+-]digits] ** [+-]digits.[digits][E[+-]digits] @@ -30500,8 +30789,8 @@ SQLITE_PRIVATE int sqlite3AtoF(const char *z, double *pResult, int length, u8 en int e = 0; /* exponent */ int eValid = 1; /* True exponent is either not used or is well-formed */ double result; - int nDigits = 0; - int nonNum = 0; /* True if input contains UTF16 with high byte non-zero */ + int nDigit = 0; /* Number of digits processed */ + int eType = 1; /* 1: pure integer, 2+: fractional -1 or less: bad UTF16 */ assert( enc==SQLITE_UTF8 || enc==SQLITE_UTF16LE || enc==SQLITE_UTF16BE ); *pResult = 0.0; /* Default return value, in case of an error */ @@ -30512,8 +30801,10 @@ SQLITE_PRIVATE int sqlite3AtoF(const char *z, double *pResult, int length, u8 en int i; incr = 2; assert( SQLITE_UTF16LE==2 && SQLITE_UTF16BE==3 ); + testcase( enc==SQLITE_UTF16LE ); + testcase( enc==SQLITE_UTF16BE ); for(i=3-enc; i=((LARGEST_INT64-9)/10) ){ + /* skip non-significant significand digits + ** (increase exponent by d to shift decimal left) */ + while( z=zEnd ) goto do_atof_calc; /* if decimal point is present */ if( *z=='.' ){ z+=incr; + eType++; /* copy digits from after decimal to significand ** (decrease exponent by d to shift decimal right) */ while( z=zEnd ) goto do_atof_calc; @@ -30560,6 +30854,7 @@ SQLITE_PRIVATE int sqlite3AtoF(const char *z, double *pResult, int length, u8 en if( *z=='e' || *z=='E' ){ z+=incr; eValid = 0; + eType++; /* This branch is needed to avoid a (harmless) buffer overread. The ** special comment alerts the mutation tester that the correct answer @@ -30658,7 +30953,13 @@ SQLITE_PRIVATE int sqlite3AtoF(const char *z, double *pResult, int length, u8 en *pResult = result; /* return true if number and no extra non-whitespace chracters after */ - return z==zEnd && nDigits>0 && eValid && nonNum==0; + if( z==zEnd && nDigit>0 && eValid && eType>0 ){ + return eType; + }else if( eType>=2 && (eType==3 || eValid) && nDigit>0 ){ + return -1; + }else{ + return 0; + } #else return !sqlite3Atoi64(z, pResult, length, enc); #endif /* SQLITE_OMIT_FLOATING_POINT */ @@ -30701,6 +31002,7 @@ static int compare2pow63(const char *zNum, int incr){ ** ** Returns: ** +** -1 Not even a prefix of the input text looks like an integer ** 0 Successful transformation. Fits in a 64-bit signed integer. ** 1 Excess non-space text after the integer value ** 2 Integer too large for a 64-bit signed integer or is malformed @@ -30760,9 +31062,9 @@ SQLITE_PRIVATE int sqlite3Atoi64(const char *zNum, i64 *pNum, int length, u8 enc *pNum = (i64)u; } rc = 0; - if( (i==0 && zStart==zNum) /* No digits */ - || nonNum /* UTF16 with high-order bytes non-zero */ - ){ + if( i==0 && zStart==zNum ){ /* No digits */ + rc = -1; + }else if( nonNum ){ /* UTF16 with high-order bytes non-zero */ rc = 1; }else if( &zNum[i]=0 ){ + *v = *p; return 1; } - - p++; - b = *p; - /* b: p1 (unmasked) */ - if (!(b&0x80)) - { - a &= 0x7f; - a = a<<7; - a |= b; - *v = a; + if( ((signed char*)p)[1]>=0 ){ + *v = ((u32)(p[0]&0x7f)<<7) | p[1]; return 2; } @@ -31017,8 +31308,9 @@ SQLITE_PRIVATE u8 sqlite3GetVarint(const unsigned char *p, u64 *v){ assert( SLOT_2_0 == ((0x7f<<14) | (0x7f)) ); assert( SLOT_4_2_0 == ((0xfU<<28) | (0x7f<<14) | (0x7f)) ); - p++; - a = a<<14; + a = ((u32)p[0])<<14; + b = p[1]; + p += 2; a |= *p; /* a: p0<<14 | p2 (unmasked) */ if (!(a&0x80)) @@ -31601,7 +31893,7 @@ SQLITE_PRIVATE LogEst sqlite3LogEstFromDouble(double x){ #endif /* SQLITE_OMIT_VIRTUALTABLE */ #if defined(SQLITE_ENABLE_STMT_SCANSTATUS) || \ - defined(SQLITE_ENABLE_STAT3_OR_STAT4) || \ + defined(SQLITE_ENABLE_STAT4) || \ defined(SQLITE_EXPLAIN_ESTIMATED_ROWS) /* ** Convert a LogEst into an integer. @@ -31619,7 +31911,7 @@ SQLITE_PRIVATE u64 sqlite3LogEstToInt(LogEst x){ defined(SQLITE_EXPLAIN_ESTIMATED_ROWS) if( x>60 ) return (u64)LARGEST_INT64; #else - /* If only SQLITE_ENABLE_STAT3_OR_STAT4 is on, then the largest input + /* If only SQLITE_ENABLE_STAT4 is on, then the largest input ** possible to this routine is 310, resulting in a maximum x of 31 */ assert( x<=60 ); #endif @@ -31678,7 +31970,7 @@ SQLITE_PRIVATE VList *sqlite3VListAdd( assert( pIn==0 || pIn[0]>=3 ); /* Verify ok to add new elements */ if( pIn==0 || pIn[1]+nInt > pIn[0] ){ /* Enlarge the allocation */ - int nAlloc = (pIn ? pIn[0]*2 : 10) + nInt; + sqlite3_int64 nAlloc = (pIn ? 2*(sqlite3_int64)pIn[0] : 10) + nInt; VList *pOut = sqlite3DbRealloc(db, pIn, nAlloc*sizeof(int)); if( pOut==0 ) return pIn; if( pIn==0 ) pOut[1] = 2; @@ -31884,7 +32176,7 @@ static HashElem *findElementWithHash( unsigned int *pHash /* Write the hash value here */ ){ HashElem *elem; /* Used to loop thru the element list */ - int count; /* Number of elements left to test */ + unsigned int count; /* Number of elements left to test */ unsigned int h; /* The computed hash */ static HashElem nullElement = { 0, 0, 0, 0 }; @@ -31932,8 +32224,8 @@ static void removeElementGivenHash( if( pEntry->chain==elem ){ pEntry->chain = elem->next; } + assert( pEntry->count>0 ); pEntry->count--; - assert( pEntry->count>=0 ); } sqlite3_free( elem ); pH->count--; @@ -32108,28 +32400,28 @@ SQLITE_PRIVATE const char *sqlite3OpcodeName(int i){ /* 89 */ "Offset" OpHelp("r[P3] = sqlite_offset(P1)"), /* 90 */ "Column" OpHelp("r[P3]=PX"), /* 91 */ "Affinity" OpHelp("affinity(r[P1@P2])"), - /* 92 */ "BitAnd" OpHelp("r[P3]=r[P1]&r[P2]"), - /* 93 */ "BitOr" OpHelp("r[P3]=r[P1]|r[P2]"), - /* 94 */ "ShiftLeft" OpHelp("r[P3]=r[P2]<>r[P1]"), - /* 96 */ "Add" OpHelp("r[P3]=r[P1]+r[P2]"), - /* 97 */ "Subtract" OpHelp("r[P3]=r[P2]-r[P1]"), - /* 98 */ "Multiply" OpHelp("r[P3]=r[P1]*r[P2]"), - /* 99 */ "Divide" OpHelp("r[P3]=r[P2]/r[P1]"), - /* 100 */ "Remainder" OpHelp("r[P3]=r[P2]%r[P1]"), - /* 101 */ "Concat" OpHelp("r[P3]=r[P2]+r[P1]"), - /* 102 */ "MakeRecord" OpHelp("r[P3]=mkrec(r[P1@P2])"), - /* 103 */ "BitNot" OpHelp("r[P2]= ~r[P1]"), - /* 104 */ "Count" OpHelp("r[P2]=count()"), - /* 105 */ "ReadCookie" OpHelp(""), - /* 106 */ "String8" OpHelp("r[P2]='P4'"), - /* 107 */ "SetCookie" OpHelp(""), - /* 108 */ "ReopenIdx" OpHelp("root=P2 iDb=P3"), - /* 109 */ "OpenRead" OpHelp("root=P2 iDb=P3"), - /* 110 */ "OpenWrite" OpHelp("root=P2 iDb=P3"), - /* 111 */ "OpenDup" OpHelp(""), - /* 112 */ "OpenAutoindex" OpHelp("nColumn=P2"), - /* 113 */ "OpenEphemeral" OpHelp("nColumn=P2"), + /* 92 */ "MakeRecord" OpHelp("r[P3]=mkrec(r[P1@P2])"), + /* 93 */ "Count" OpHelp("r[P2]=count()"), + /* 94 */ "ReadCookie" OpHelp(""), + /* 95 */ "SetCookie" OpHelp(""), + /* 96 */ "ReopenIdx" OpHelp("root=P2 iDb=P3"), + /* 97 */ "OpenRead" OpHelp("root=P2 iDb=P3"), + /* 98 */ "OpenWrite" OpHelp("root=P2 iDb=P3"), + /* 99 */ "BitAnd" OpHelp("r[P3]=r[P1]&r[P2]"), + /* 100 */ "BitOr" OpHelp("r[P3]=r[P1]|r[P2]"), + /* 101 */ "ShiftLeft" OpHelp("r[P3]=r[P2]<>r[P1]"), + /* 103 */ "Add" OpHelp("r[P3]=r[P1]+r[P2]"), + /* 104 */ "Subtract" OpHelp("r[P3]=r[P2]-r[P1]"), + /* 105 */ "Multiply" OpHelp("r[P3]=r[P1]*r[P2]"), + /* 106 */ "Divide" OpHelp("r[P3]=r[P2]/r[P1]"), + /* 107 */ "Remainder" OpHelp("r[P3]=r[P2]%r[P1]"), + /* 108 */ "Concat" OpHelp("r[P3]=r[P2]+r[P1]"), + /* 109 */ "OpenDup" OpHelp(""), + /* 110 */ "BitNot" OpHelp("r[P2]= ~r[P1]"), + /* 111 */ "OpenAutoindex" OpHelp("nColumn=P2"), + /* 112 */ "OpenEphemeral" OpHelp("nColumn=P2"), + /* 113 */ "String8" OpHelp("r[P2]='P4'"), /* 114 */ "SorterOpen" OpHelp(""), /* 115 */ "SequenceTest" OpHelp("if( cursor[P1].ctr++ ) pc = P2"), /* 116 */ "OpenPseudo" OpHelp("P3 columns in r[P2]"), @@ -32139,57 +32431,56 @@ SQLITE_PRIVATE const char *sqlite3OpcodeName(int i){ /* 120 */ "Sequence" OpHelp("r[P2]=cursor[P1].ctr++"), /* 121 */ "NewRowid" OpHelp("r[P2]=rowid"), /* 122 */ "Insert" OpHelp("intkey=r[P3] data=r[P2]"), - /* 123 */ "InsertInt" OpHelp("intkey=P3 data=r[P2]"), - /* 124 */ "Delete" OpHelp(""), - /* 125 */ "ResetCount" OpHelp(""), - /* 126 */ "SorterCompare" OpHelp("if key(P1)!=trim(r[P3],P4) goto P2"), - /* 127 */ "SorterData" OpHelp("r[P2]=data"), - /* 128 */ "RowData" OpHelp("r[P2]=data"), - /* 129 */ "Rowid" OpHelp("r[P2]=rowid"), - /* 130 */ "NullRow" OpHelp(""), - /* 131 */ "SeekEnd" OpHelp(""), - /* 132 */ "SorterInsert" OpHelp("key=r[P2]"), - /* 133 */ "IdxInsert" OpHelp("key=r[P2]"), - /* 134 */ "IdxDelete" OpHelp("key=r[P2@P3]"), - /* 135 */ "DeferredSeek" OpHelp("Move P3 to P1.rowid if needed"), - /* 136 */ "IdxRowid" OpHelp("r[P2]=rowid"), - /* 137 */ "Destroy" OpHelp(""), - /* 138 */ "Clear" OpHelp(""), - /* 139 */ "ResetSorter" OpHelp(""), - /* 140 */ "CreateBtree" OpHelp("r[P2]=root iDb=P1 flags=P3"), - /* 141 */ "Real" OpHelp("r[P2]=P4"), - /* 142 */ "SqlExec" OpHelp(""), - /* 143 */ "ParseSchema" OpHelp(""), - /* 144 */ "LoadAnalysis" OpHelp(""), - /* 145 */ "DropTable" OpHelp(""), - /* 146 */ "DropIndex" OpHelp(""), - /* 147 */ "DropTrigger" OpHelp(""), - /* 148 */ "IntegrityCk" OpHelp(""), - /* 149 */ "RowSetAdd" OpHelp("rowset(P1)=r[P2]"), - /* 150 */ "Param" OpHelp(""), - /* 151 */ "FkCounter" OpHelp("fkctr[P1]+=P2"), - /* 152 */ "MemMax" OpHelp("r[P1]=max(r[P1],r[P2])"), - /* 153 */ "OffsetLimit" OpHelp("if r[P1]>0 then r[P2]=r[P1]+max(0,r[P3]) else r[P2]=(-1)"), - /* 154 */ "AggInverse" OpHelp("accum=r[P3] inverse(r[P2@P5])"), - /* 155 */ "AggStep" OpHelp("accum=r[P3] step(r[P2@P5])"), - /* 156 */ "AggStep1" OpHelp("accum=r[P3] step(r[P2@P5])"), - /* 157 */ "AggValue" OpHelp("r[P3]=value N=P2"), - /* 158 */ "AggFinal" OpHelp("accum=r[P1] N=P2"), - /* 159 */ "Expire" OpHelp(""), - /* 160 */ "TableLock" OpHelp("iDb=P1 root=P2 write=P3"), - /* 161 */ "VBegin" OpHelp(""), - /* 162 */ "VCreate" OpHelp(""), - /* 163 */ "VDestroy" OpHelp(""), - /* 164 */ "VOpen" OpHelp(""), - /* 165 */ "VColumn" OpHelp("r[P3]=vcolumn(P2)"), - /* 166 */ "VRename" OpHelp(""), - /* 167 */ "Pagecount" OpHelp(""), - /* 168 */ "MaxPgcnt" OpHelp(""), - /* 169 */ "Trace" OpHelp(""), - /* 170 */ "CursorHint" OpHelp(""), - /* 171 */ "Noop" OpHelp(""), - /* 172 */ "Explain" OpHelp(""), - /* 173 */ "Abortable" OpHelp(""), + /* 123 */ "Delete" OpHelp(""), + /* 124 */ "ResetCount" OpHelp(""), + /* 125 */ "SorterCompare" OpHelp("if key(P1)!=trim(r[P3],P4) goto P2"), + /* 126 */ "SorterData" OpHelp("r[P2]=data"), + /* 127 */ "RowData" OpHelp("r[P2]=data"), + /* 128 */ "Rowid" OpHelp("r[P2]=rowid"), + /* 129 */ "NullRow" OpHelp(""), + /* 130 */ "SeekEnd" OpHelp(""), + /* 131 */ "SorterInsert" OpHelp("key=r[P2]"), + /* 132 */ "IdxInsert" OpHelp("key=r[P2]"), + /* 133 */ "IdxDelete" OpHelp("key=r[P2@P3]"), + /* 134 */ "DeferredSeek" OpHelp("Move P3 to P1.rowid if needed"), + /* 135 */ "IdxRowid" OpHelp("r[P2]=rowid"), + /* 136 */ "Destroy" OpHelp(""), + /* 137 */ "Clear" OpHelp(""), + /* 138 */ "ResetSorter" OpHelp(""), + /* 139 */ "CreateBtree" OpHelp("r[P2]=root iDb=P1 flags=P3"), + /* 140 */ "SqlExec" OpHelp(""), + /* 141 */ "ParseSchema" OpHelp(""), + /* 142 */ "LoadAnalysis" OpHelp(""), + /* 143 */ "DropTable" OpHelp(""), + /* 144 */ "DropIndex" OpHelp(""), + /* 145 */ "DropTrigger" OpHelp(""), + /* 146 */ "IntegrityCk" OpHelp(""), + /* 147 */ "RowSetAdd" OpHelp("rowset(P1)=r[P2]"), + /* 148 */ "Real" OpHelp("r[P2]=P4"), + /* 149 */ "Param" OpHelp(""), + /* 150 */ "FkCounter" OpHelp("fkctr[P1]+=P2"), + /* 151 */ "MemMax" OpHelp("r[P1]=max(r[P1],r[P2])"), + /* 152 */ "OffsetLimit" OpHelp("if r[P1]>0 then r[P2]=r[P1]+max(0,r[P3]) else r[P2]=(-1)"), + /* 153 */ "AggInverse" OpHelp("accum=r[P3] inverse(r[P2@P5])"), + /* 154 */ "AggStep" OpHelp("accum=r[P3] step(r[P2@P5])"), + /* 155 */ "AggStep1" OpHelp("accum=r[P3] step(r[P2@P5])"), + /* 156 */ "AggValue" OpHelp("r[P3]=value N=P2"), + /* 157 */ "AggFinal" OpHelp("accum=r[P1] N=P2"), + /* 158 */ "Expire" OpHelp(""), + /* 159 */ "TableLock" OpHelp("iDb=P1 root=P2 write=P3"), + /* 160 */ "VBegin" OpHelp(""), + /* 161 */ "VCreate" OpHelp(""), + /* 162 */ "VDestroy" OpHelp(""), + /* 163 */ "VOpen" OpHelp(""), + /* 164 */ "VColumn" OpHelp("r[P3]=vcolumn(P2)"), + /* 165 */ "VRename" OpHelp(""), + /* 166 */ "Pagecount" OpHelp(""), + /* 167 */ "MaxPgcnt" OpHelp(""), + /* 168 */ "Trace" OpHelp(""), + /* 169 */ "CursorHint" OpHelp(""), + /* 170 */ "Noop" OpHelp(""), + /* 171 */ "Explain" OpHelp(""), + /* 172 */ "Abortable" OpHelp(""), }; return azName[i]; } @@ -32304,13 +32595,29 @@ SQLITE_PRIVATE const char *sqlite3OpcodeName(int i){ # include #endif /* SQLITE_ENABLE_LOCKING_STYLE */ -#if defined(__APPLE__) && ((__MAC_OS_X_VERSION_MIN_REQUIRED > 1050) || \ - (__IPHONE_OS_VERSION_MIN_REQUIRED > 2000)) -# if (!defined(TARGET_OS_EMBEDDED) || (TARGET_OS_EMBEDDED==0)) \ - && (!defined(TARGET_IPHONE_SIMULATOR) || (TARGET_IPHONE_SIMULATOR==0)) -# define HAVE_GETHOSTUUID 1 -# else -# warning "gethostuuid() is disabled." +/* +** Try to determine if gethostuuid() is available based on standard +** macros. This might sometimes compute the wrong value for some +** obscure platforms. For those cases, simply compile with one of +** the following: +** +** -DHAVE_GETHOSTUUID=0 +** -DHAVE_GETHOSTUUID=1 +** +** None if this matters except when building on Apple products with +** -DSQLITE_ENABLE_LOCKING_STYLE. +*/ +#ifndef HAVE_GETHOSTUUID +# define HAVE_GETHOSTUUID 0 +# if defined(__APPLE__) && ((__MAC_OS_X_VERSION_MIN_REQUIRED > 1050) || \ + (__IPHONE_OS_VERSION_MIN_REQUIRED > 2000)) +# if (!defined(TARGET_OS_EMBEDDED) || (TARGET_OS_EMBEDDED==0)) \ + && (!defined(TARGET_IPHONE_SIMULATOR) || (TARGET_IPHONE_SIMULATOR==0)) +# undef HAVE_GETHOSTUUID +# define HAVE_GETHOSTUUID 1 +# else +# warning "gethostuuid() is disabled." +# endif # endif #endif @@ -32918,13 +33225,14 @@ static struct unix_syscall { #if defined(__linux__) && defined(SQLITE_ENABLE_BATCH_ATOMIC_WRITE) # ifdef __ANDROID__ { "ioctl", (sqlite3_syscall_ptr)(int(*)(int, int, ...))ioctl, 0 }, +#define osIoctl ((int(*)(int,int,...))aSyscall[28].pCurrent) # else { "ioctl", (sqlite3_syscall_ptr)ioctl, 0 }, +#define osIoctl ((int(*)(int,unsigned long,...))aSyscall[28].pCurrent) # endif #else { "ioctl", (sqlite3_syscall_ptr)0, 0 }, #endif -#define osIoctl ((int(*)(int,int,...))aSyscall[28].pCurrent) }; /* End of the overrideable system calls */ @@ -38166,6 +38474,7 @@ static UnixUnusedFd *findReusableFd(const char *zPath, int flags){ UnixUnusedFd **pp; assert( sqlite3_mutex_notheld(pInode->pLockMutex) ); sqlite3_mutex_enter(pInode->pLockMutex); + flags &= (SQLITE_OPEN_READONLY|SQLITE_OPEN_READWRITE); for(pp=&pInode->pUnused; *pp && (*pp)->flags!=flags; pp=&((*pp)->pNext)); pUnused = *pp; if( pUnused ){ @@ -38219,7 +38528,7 @@ static int getFileMode( ** If the SQLITE_ENABLE_8_3_NAMES option is enabled, then the ** original filename is unavailable. But 8_3_NAMES is only used for ** FAT filesystems and permissions do not matter there, so just use -** the default permissions. +** the default permissions. In 8_3_NAMES mode, leave *pMode set to zero. */ static int findCreateFileMode( const char *zPath, /* Path of file (possibly) being created */ @@ -38454,11 +38763,19 @@ static int unixOpen( goto open_finished; } - /* If this process is running as root and if creating a new rollback - ** journal or WAL file, set the ownership of the journal or WAL to be - ** the same as the original database. + /* The owner of the rollback journal or WAL file should always be the + ** same as the owner of the database file. Try to ensure that this is + ** the case. The chown() system call will be a no-op if the current + ** process lacks root privileges, be we should at least try. Without + ** this step, if a root process opens a database file, it can leave + ** behinds a journal/WAL that is owned by root and hence make the + ** database inaccessible to unprivileged processes. + ** + ** If openMode==0, then that means uid and gid are not set correctly + ** (probably because SQLite is configured to use 8+3 filename mode) and + ** in that case we do not want to attempt the chown(). */ - if( flags & (SQLITE_OPEN_WAL|SQLITE_OPEN_MAIN_JOURNAL) ){ + if( openMode && (flags & (SQLITE_OPEN_WAL|SQLITE_OPEN_MAIN_JOURNAL))!=0 ){ robustFchown(fd, uid, gid); } } @@ -38469,7 +38786,8 @@ static int unixOpen( if( p->pPreallocatedUnused ){ p->pPreallocatedUnused->fd = fd; - p->pPreallocatedUnused->flags = flags; + p->pPreallocatedUnused->flags = + flags & (SQLITE_OPEN_READONLY|SQLITE_OPEN_READWRITE); } if( isDelete ){ @@ -39315,7 +39633,7 @@ SQLITE_API int sqlite3_hostid_num = 0; #define PROXY_HOSTIDLEN 16 /* conch file host id length */ -#ifdef HAVE_GETHOSTUUID +#if HAVE_GETHOSTUUID /* Not always defined in the headers as it ought to be */ extern int gethostuuid(uuid_t id, const struct timespec *wait); #endif @@ -39326,7 +39644,7 @@ extern int gethostuuid(uuid_t id, const struct timespec *wait); static int proxyGetHostID(unsigned char *pHostID, int *pError){ assert(PROXY_HOSTIDLEN == sizeof(uuid_t)); memset(pHostID, 0, PROXY_HOSTIDLEN); -#ifdef HAVE_GETHOSTUUID +#if HAVE_GETHOSTUUID { struct timespec timeout = {1, 0}; /* 1 sec timeout */ if( gethostuuid(pHostID, &timeout) ){ @@ -40000,7 +40318,7 @@ static int proxyFileControl(sqlite3_file *id, int op, void *pArg){ assert( 0 ); /* The call assures that only valid opcodes are sent */ } } - /*NOTREACHED*/ + /*NOTREACHED*/ assert(0); return SQLITE_ERROR; } @@ -44685,6 +45003,7 @@ static int winShmMap( rc = winOpenSharedMemory(pDbFd); if( rc!=SQLITE_OK ) return rc; pShm = pDbFd->pShm; + assert( pShm!=0 ); } pShmNode = pShm->pShmNode; @@ -44987,6 +45306,7 @@ static int winFetch(sqlite3_file *fd, i64 iOff, int nAmt, void **pp){ } } if( pFd->mmapSize >= iOff+nAmt ){ + assert( pFd->pMapRegion!=0 ); *pp = &((u8 *)pFd->pMapRegion)[iOff]; pFd->nFetchOut++; } @@ -47889,9 +48209,10 @@ static int numberOfCachePages(PCache *p){ ** suggested cache size is set to N. */ return p->szCache; }else{ - /* IMPLEMENTATION-OF: R-61436-13639 If the argument N is negative, then - ** the number of cache pages is adjusted to use approximately abs(N*1024) - ** bytes of memory. */ + /* IMPLEMANTATION-OF: R-59858-46238 If the argument N is negative, then the + ** number of cache pages is adjusted to be a number of pages that would + ** use approximately abs(N*1024) bytes of memory based on the current + ** page size. */ return (int)((-1024*(i64)p->szCache)/(p->szPage+p->szExtra)); } } @@ -47907,6 +48228,7 @@ SQLITE_PRIVATE int sqlite3PcacheInitialize(void){ ** built-in default page cache is used instead of the application defined ** page cache. */ sqlite3PCacheSetDefault(); + assert( sqlite3GlobalConfig.pcache2.xInit!=0 ); } return sqlite3GlobalConfig.pcache2.xInit(sqlite3GlobalConfig.pcache2.pArg); } @@ -48953,6 +49275,7 @@ static PgHdr1 *pcache1AllocPage(PCache1 *pCache, int benignMalloc){ assert( sqlite3_mutex_held(pCache->pGroup->mutex) ); if( pCache->pFree || (pCache->nPage==0 && pcache1InitBulk(pCache)) ){ + assert( pCache->pFree!=0 ); p = pCache->pFree; pCache->pFree = p->pNext; p->pNext = 0; @@ -49018,9 +49341,7 @@ static void pcache1FreePage(PgHdr1 *p){ ** exists, this function falls back to sqlite3Malloc(). */ SQLITE_PRIVATE void *sqlite3PageMalloc(int sz){ - /* During rebalance operations on a corrupt database file, it is sometimes - ** (rarely) possible to overread the temporary page buffer by a few bytes. - ** Enlarge the allocation slightly so that this does not cause problems. */ + assert( sz<=65536+8 ); /* These allocations are never very large */ return pcache1Alloc(sz); } @@ -49309,6 +49630,7 @@ static sqlite3_pcache *pcache1Create(int szPage, int szExtra, int bPurgeable){ }else{ pGroup = &pcache1.grp; } + pcache1EnterMutex(pGroup); if( pGroup->lru.isAnchor==0 ){ pGroup->lru.isAnchor = 1; pGroup->lru.pLruPrev = pGroup->lru.pLruNext = &pGroup->lru; @@ -49318,7 +49640,6 @@ static sqlite3_pcache *pcache1Create(int szPage, int szExtra, int bPurgeable){ pCache->szExtra = szExtra; pCache->szAlloc = szPage + szExtra + ROUND8(sizeof(PgHdr1)); pCache->bPurgeable = (bPurgeable ? 1 : 0); - pcache1EnterMutex(pGroup); pcache1ResizeHash(pCache); if( bPurgeable ){ pCache->nMin = 10; @@ -51304,6 +51625,9 @@ static const unsigned char aJournalMagic[] = { SQLITE_PRIVATE int sqlite3PagerDirectReadOk(Pager *pPager, Pgno pgno){ if( pPager->fd->pMethods==0 ) return 0; if( sqlite3PCacheIsDirty(pPager->pPCache) ) return 0; +#ifdef SQLITE_HAS_CODEC + if( pPager->xCodec!=0 ) return 0; +#endif #ifndef SQLITE_OMIT_WAL if( pPager->pWal ){ u32 iRead = 0; @@ -54253,8 +54577,14 @@ SQLITE_PRIVATE int sqlite3PagerSetPagesize(Pager *pPager, u32 *pPageSize, int nR rc = sqlite3OsFileSize(pPager->fd, &nByte); } if( rc==SQLITE_OK ){ - pNew = (char *)sqlite3PageMalloc(pageSize); - if( !pNew ) rc = SQLITE_NOMEM_BKPT; + /* 8 bytes of zeroed overrun space is sufficient so that the b-tree + * cell header parser will never run off the end of the allocation */ + pNew = (char *)sqlite3PageMalloc(pageSize+8); + if( !pNew ){ + rc = SQLITE_NOMEM_BKPT; + }else{ + memset(pNew+pageSize, 0, 8); + } } if( rc==SQLITE_OK ){ @@ -57635,8 +57965,12 @@ SQLITE_PRIVATE int sqlite3PagerMovepage(Pager *pPager, DbPage *pPg, Pgno pgno, i */ pPg->flags &= ~PGHDR_NEED_SYNC; pPgOld = sqlite3PagerLookup(pPager, pgno); - assert( !pPgOld || pPgOld->nRef==1 ); + assert( !pPgOld || pPgOld->nRef==1 || CORRUPT_DB ); if( pPgOld ){ + if( pPgOld->nRef>1 ){ + sqlite3PagerUnrefNotNull(pPgOld); + return SQLITE_CORRUPT_BKPT; + } pPg->flags |= (pPgOld->flags&PGHDR_NEED_SYNC); if( pPager->tempFile ){ /* Do not discard pages from an in-memory database since we might @@ -58164,7 +58498,7 @@ SQLITE_PRIVATE int sqlite3PagerSnapshotCheck(Pager *pPager, sqlite3_snapshot *pS */ SQLITE_PRIVATE void sqlite3PagerSnapshotUnlock(Pager *pPager){ assert( pPager->pWal ); - return sqlite3WalSnapshotUnlock(pPager->pWal); + sqlite3WalSnapshotUnlock(pPager->pWal); } #endif /* SQLITE_ENABLE_SNAPSHOT */ @@ -58765,7 +59099,7 @@ static SQLITE_NOINLINE int walIndexPageRealloc( /* Enlarge the pWal->apWiData[] array if required */ if( pWal->nWiData<=iPage ){ - int nByte = sizeof(u32*)*(iPage+1); + sqlite3_int64 nByte = sizeof(u32*)*(iPage+1); volatile u32 **apNew; apNew = (volatile u32 **)sqlite3_realloc64((void *)pWal->apWiData, nByte); if( !apNew ){ @@ -58869,6 +59203,7 @@ static void walChecksumBytes( assert( nByte>=8 ); assert( (nByte&0x00000007)==0 ); + assert( nByte<=65536 ); if( nativeCksum ){ do { @@ -59176,6 +59511,7 @@ static void walCleanupHash(Wal *pWal){ int iLimit = 0; /* Zero values greater than this */ int nByte; /* Number of bytes to zero in aPgno[] */ int i; /* Used to iterate through aHash[] */ + int rc; /* Return code form walHashGet() */ assert( pWal->writeLock ); testcase( pWal->hdr.mxFrame==HASHTABLE_NPAGE_ONE-1 ); @@ -59186,11 +59522,12 @@ static void walCleanupHash(Wal *pWal){ /* Obtain pointers to the hash-table and page-number array containing ** the entry that corresponds to frame pWal->hdr.mxFrame. It is guaranteed - ** that the page said hash-table and array reside on is already mapped. + ** that the page said hash-table and array reside on is already mapped.(1) */ assert( pWal->nWiData>walFramePage(pWal->hdr.mxFrame) ); assert( pWal->apWiData[walFramePage(pWal->hdr.mxFrame)] ); - walHashGet(pWal, walFramePage(pWal->hdr.mxFrame), &sLoc); + rc = walHashGet(pWal, walFramePage(pWal->hdr.mxFrame), &sLoc); + if( NEVER(rc) ) return; /* Defense-in-depth, in case (1) above is wrong */ /* Zero all hash-table entries that correspond to frame numbers greater ** than pWal->hdr.mxFrame. @@ -59804,7 +60141,7 @@ static int walIteratorInit(Wal *pWal, u32 nBackfill, WalIterator **pp){ WalIterator *p; /* Return value */ int nSegment; /* Number of segments to merge */ u32 iLast; /* Last frame in log */ - int nByte; /* Number of bytes to allocate */ + sqlite3_int64 nByte; /* Number of bytes to allocate */ int i; /* Iterator variable */ ht_slot *aTmp; /* Temp space used by merge-sort */ int rc = SQLITE_OK; /* Return Code */ @@ -61095,9 +61432,9 @@ SQLITE_PRIVATE int sqlite3WalFindFrame( } nCollide = HASHTABLE_NSLOT; for(iKey=walHash(pgno); sLoc.aHash[iKey]; iKey=walNextHash(iKey)){ - u32 iFrame = sLoc.aHash[iKey] + sLoc.iZero; - if( iFrame<=iLast && iFrame>=pWal->minFrame - && sLoc.aPgno[sLoc.aHash[iKey]]==pgno ){ + u32 iH = sLoc.aHash[iKey]; + u32 iFrame = iH + sLoc.iZero; + if( iFrame<=iLast && iFrame>=pWal->minFrame && sLoc.aPgno[iH]==pgno ){ assert( iFrame>iRead || CORRUPT_DB ); iRead = iFrame; } @@ -61665,6 +62002,7 @@ SQLITE_PRIVATE int sqlite3WalFrames( if( rc ) return rc; iOffset += szFrame; nExtra++; + assert( pLast!=0 ); } } if( bSync ){ @@ -61697,6 +62035,7 @@ SQLITE_PRIVATE int sqlite3WalFrames( iFrame++; rc = walIndexAppend(pWal, iFrame, p->pgno); } + assert( pLast!=0 || nExtra==0 ); while( rc==SQLITE_OK && nExtra>0 ){ iFrame++; nExtra--; @@ -62340,7 +62679,7 @@ struct MemPage { u16 maxLocal; /* Copy of BtShared.maxLocal or BtShared.maxLeaf */ u16 minLocal; /* Copy of BtShared.minLocal or BtShared.minLeaf */ u16 cellOffset; /* Index in aData of first cell pointer */ - u16 nFree; /* Number of free bytes on the page */ + int nFree; /* Number of free bytes on the page. -1 for unknown */ u16 nCell; /* Number of cells on this page, local and ovfl */ u16 maskPage; /* Mask for page offset */ u16 aiOvfl[4]; /* Insert the i-th overflow cell before the aiOvfl-th @@ -63894,14 +64233,18 @@ static int btreeMoveto( */ static int btreeRestoreCursorPosition(BtCursor *pCur){ int rc; - int skipNext; + int skipNext = 0; assert( cursorOwnsBtShared(pCur) ); assert( pCur->eState>=CURSOR_REQUIRESEEK ); if( pCur->eState==CURSOR_FAULT ){ return pCur->skipNext; } pCur->eState = CURSOR_INVALID; - rc = btreeMoveto(pCur, pCur->pKey, pCur->nKey, 0, &skipNext); + if( sqlite3FaultSim(410) ){ + rc = SQLITE_IOERR; + }else{ + rc = btreeMoveto(pCur, pCur->pKey, pCur->nKey, 0, &skipNext); + } if( rc==SQLITE_OK ){ sqlite3_free(pCur->pKey); pCur->pKey = 0; @@ -64482,7 +64825,7 @@ static int defragmentPage(MemPage *pPage, int nMaxFrag){ hdr = pPage->hdrOffset; cellOffset = pPage->cellOffset; nCell = pPage->nCell; - assert( nCell==get2byte(&data[hdr+3]) ); + assert( nCell==get2byte(&data[hdr+3]) || CORRUPT_DB ); iCellFirst = cellOffset + 2*nCell; usableSize = pPage->pBt->usableSize; @@ -64493,11 +64836,7 @@ static int defragmentPage(MemPage *pPage, int nMaxFrag){ ** reconstruct the entire page. */ if( (int)data[hdr+7]<=nMaxFrag ){ int iFree = get2byte(&data[hdr+1]); - - /* If the initial freeblock offset were out of bounds, that would - ** have been detected by btreeInitPage() when it was computing the - ** number of free bytes on the page. */ - assert( iFree<=usableSize-4 ); + if( iFree>usableSize-4 ) return SQLITE_CORRUPT_PAGE(pPage); if( iFree ){ int iFree2 = get2byte(&data[iFree]); if( iFree2>usableSize-4 ) return SQLITE_CORRUPT_PAGE(pPage); @@ -64516,7 +64855,10 @@ static int defragmentPage(MemPage *pPage, int nMaxFrag){ if( iFree2+sz2 > usableSize ) return SQLITE_CORRUPT_PAGE(pPage); memmove(&data[iFree+sz+sz2], &data[iFree+sz], iFree2-(iFree+sz)); sz += sz2; + }else if( iFree+sz>usableSize ){ + return SQLITE_CORRUPT_PAGE(pPage); } + cbrk = top+sz; assert( cbrk+(iFree-top) <= usableSize ); memmove(&data[cbrk], &data[top], iFree-top); @@ -64567,6 +64909,7 @@ static int defragmentPage(MemPage *pPage, int nMaxFrag){ data[hdr+7] = 0; defragment_out: + assert( pPage->nFree>=0 ); if( data[hdr+7]+cbrk-iCellFirst!=pPage->nFree ){ return SQLITE_CORRUPT_PAGE(pPage); } @@ -64594,16 +64937,16 @@ static int defragmentPage(MemPage *pPage, int nMaxFrag){ ** causes the fragmentation count to exceed 60. */ static u8 *pageFindSlot(MemPage *pPg, int nByte, int *pRc){ - const int hdr = pPg->hdrOffset; - u8 * const aData = pPg->aData; - int iAddr = hdr + 1; - int pc = get2byte(&aData[iAddr]); - int x; - int usableSize = pPg->pBt->usableSize; - int size; /* Size of the free slot */ + const int hdr = pPg->hdrOffset; /* Offset to page header */ + u8 * const aData = pPg->aData; /* Page data */ + int iAddr = hdr + 1; /* Address of ptr to pc */ + int pc = get2byte(&aData[iAddr]); /* Address of a free slot */ + int x; /* Excess size of the slot */ + int maxPC = pPg->pBt->usableSize - nByte; /* Max address for a usable slot */ + int size; /* Size of the free slot */ assert( pc>0 ); - while( pc<=usableSize-4 ){ + while( pc<=maxPC ){ /* EVIDENCE-OF: R-22710-53328 The third and fourth bytes of each ** freeblock form a big-endian integer which is the size of the freeblock ** in bytes, including the 4-byte header. */ @@ -64611,10 +64954,7 @@ static u8 *pageFindSlot(MemPage *pPg, int nByte, int *pRc){ if( (x = size - nByte)>=0 ){ testcase( x==4 ); testcase( x==3 ); - if( size+pc > usableSize ){ - *pRc = SQLITE_CORRUPT_PAGE(pPg); - return 0; - }else if( x<4 ){ + if( x<4 ){ /* EVIDENCE-OF: R-11498-58022 In a well-formed b-tree page, the total ** number of bytes in fragments may not exceed 60. */ if( aData[hdr+7]>57 ) return 0; @@ -64623,21 +64963,31 @@ static u8 *pageFindSlot(MemPage *pPg, int nByte, int *pRc){ ** fragmented bytes within the page. */ memcpy(&aData[iAddr], &aData[pc], 2); aData[hdr+7] += (u8)x; + }else if( x+pc > maxPC ){ + /* This slot extends off the end of the usable part of the page */ + *pRc = SQLITE_CORRUPT_PAGE(pPg); + return 0; }else{ /* The slot remains on the free-list. Reduce its size to account - ** for the portion used by the new allocation. */ + ** for the portion used by the new allocation. */ put2byte(&aData[pc+2], x); } return &aData[pc + x]; } iAddr = pc; pc = get2byte(&aData[pc]); - if( pcmaxPC+nByte-4 ){ + /* The free slot chain extends off the end of the page */ *pRc = SQLITE_CORRUPT_PAGE(pPg); } - return 0; } @@ -64678,7 +65028,7 @@ static int allocateSpace(MemPage *pPage, int nByte, int *pIdx){ ** However, that integer is too large to be stored in a 2-byte unsigned ** integer, so a value of 0 is used in its place. */ top = get2byte(&data[hdr+5]); - assert( top<=(int)pPage->pBt->usableSize ); /* Prevent by getAndInitPage() */ + assert( top<=(int)pPage->pBt->usableSize ); /* by btreeComputeFreeSpace() */ if( gap>top ){ if( top==0 && pPage->pBt->usableSize==65536 ){ top = 65536; @@ -64687,9 +65037,9 @@ static int allocateSpace(MemPage *pPage, int nByte, int *pIdx){ } } - /* If there is enough space between gap and top for one more cell pointer - ** array entry offset, and if the freelist is not empty, then search the - ** freelist looking for a free slot big enough to satisfy the request. + /* If there is enough space between gap and top for one more cell pointer, + ** and if the freelist is not empty, then search the + ** freelist looking for a slot big enough to satisfy the request. */ testcase( gap+2==top ); testcase( gap+1==top ); @@ -64697,9 +65047,12 @@ static int allocateSpace(MemPage *pPage, int nByte, int *pIdx){ if( (data[hdr+2] || data[hdr+1]) && gap+2<=top ){ u8 *pSpace = pageFindSlot(pPage, nByte, &rc); if( pSpace ){ - assert( pSpace>=data && (pSpace - data)<65536 ); - *pIdx = (int)(pSpace - data); - return SQLITE_OK; + assert( pSpace+nByte<=data+pPage->pBt->usableSize ); + if( (*pIdx = (int)(pSpace-data))<=gap ){ + return SQLITE_CORRUPT_PAGE(pPage); + }else{ + return SQLITE_OK; + } }else if( rc ){ return rc; } @@ -64711,6 +65064,7 @@ static int allocateSpace(MemPage *pPage, int nByte, int *pIdx){ testcase( gap+2+nByte==top ); if( gap+2+nByte>top ){ assert( pPage->nCell>0 || CORRUPT_DB ); + assert( pPage->nFree>=0 ); rc = defragmentPage(pPage, MIN(4, pPage->nFree - (2+nByte))); if( rc ) return rc; top = get2byteNotZero(&data[hdr+5]); @@ -64719,7 +65073,7 @@ static int allocateSpace(MemPage *pPage, int nByte, int *pIdx){ /* Allocate memory from the gap in between the cell pointer array - ** and the cell content area. The btreeInitPage() call has already + ** and the cell content area. The btreeComputeFreeSpace() call has already ** validated the freelist. Given that the freelist is valid, there ** is no way that the allocation can extend off the end of the page. ** The assert() below verifies the previous sentence. @@ -64738,7 +65092,7 @@ static int allocateSpace(MemPage *pPage, int nByte, int *pIdx){ ** ** Adjacent freeblocks are coalesced. ** -** Note that even though the freeblock list was checked by btreeInitPage(), +** Even though the freeblock list was checked by btreeComputeFreeSpace(), ** that routine will not detect overlap between cells or freeblocks. Nor ** does it detect cells or freeblocks that encrouch into the reserved bytes ** at the end of the page. So do additional corruption checks inside this @@ -64900,21 +65254,14 @@ static int decodeFlags(MemPage *pPage, int flagByte){ } /* -** Initialize the auxiliary information for a disk block. -** -** Return SQLITE_OK on success. If we see that the page does -** not contain a well-formed database page, then return -** SQLITE_CORRUPT. Note that a return of SQLITE_OK does not -** guarantee that the page is well-formed. It only shows that -** we failed to detect any corruption. +** Compute the amount of freespace on the page. In other words, fill +** in the pPage->nFree field. */ -static int btreeInitPage(MemPage *pPage){ +static int btreeComputeFreeSpace(MemPage *pPage){ int pc; /* Address of a freeblock within pPage->aData[] */ u8 hdr; /* Offset to beginning of page header */ u8 *data; /* Equal to pPage->aData */ - BtShared *pBt; /* The main btree structure */ int usableSize; /* Amount of usable space on each page */ - u16 cellOffset; /* Offset from start of page to first cell pointer */ int nFree; /* Number of unused bytes on the page */ int top; /* First byte of the cell content area */ int iCellFirst; /* First allowable cell or freeblock offset */ @@ -64926,71 +65273,18 @@ static int btreeInitPage(MemPage *pPage){ assert( pPage->pgno==sqlite3PagerPagenumber(pPage->pDbPage) ); assert( pPage == sqlite3PagerGetExtra(pPage->pDbPage) ); assert( pPage->aData == sqlite3PagerGetData(pPage->pDbPage) ); - assert( pPage->isInit==0 ); + assert( pPage->isInit==1 ); + assert( pPage->nFree<0 ); - pBt = pPage->pBt; + usableSize = pPage->pBt->usableSize; hdr = pPage->hdrOffset; data = pPage->aData; - /* EVIDENCE-OF: R-28594-02890 The one-byte flag at offset 0 indicating - ** the b-tree page type. */ - if( decodeFlags(pPage, data[hdr]) ){ - return SQLITE_CORRUPT_PAGE(pPage); - } - assert( pBt->pageSize>=512 && pBt->pageSize<=65536 ); - pPage->maskPage = (u16)(pBt->pageSize - 1); - pPage->nOverflow = 0; - usableSize = pBt->usableSize; - pPage->cellOffset = cellOffset = hdr + 8 + pPage->childPtrSize; - pPage->aDataEnd = &data[usableSize]; - pPage->aCellIdx = &data[cellOffset]; - pPage->aDataOfst = &data[pPage->childPtrSize]; /* EVIDENCE-OF: R-58015-48175 The two-byte integer at offset 5 designates ** the start of the cell content area. A zero value for this integer is ** interpreted as 65536. */ top = get2byteNotZero(&data[hdr+5]); - /* EVIDENCE-OF: R-37002-32774 The two-byte integer at offset 3 gives the - ** number of cells on the page. */ - pPage->nCell = get2byte(&data[hdr+3]); - if( pPage->nCell>MX_CELL(pBt) ){ - /* To many cells for a single page. The page must be corrupt */ - return SQLITE_CORRUPT_PAGE(pPage); - } - testcase( pPage->nCell==MX_CELL(pBt) ); - /* EVIDENCE-OF: R-24089-57979 If a page contains no cells (which is only - ** possible for a root page of a table that contains no rows) then the - ** offset to the cell content area will equal the page size minus the - ** bytes of reserved space. */ - assert( pPage->nCell>0 || top==usableSize || CORRUPT_DB ); - - /* A malformed database page might cause us to read past the end - ** of page when parsing a cell. - ** - ** The following block of code checks early to see if a cell extends - ** past the end of a page boundary and causes SQLITE_CORRUPT to be - ** returned if it does. - */ - iCellFirst = cellOffset + 2*pPage->nCell; + iCellFirst = hdr + 8 + pPage->childPtrSize + 2*pPage->nCell; iCellLast = usableSize - 4; - if( pBt->db->flags & SQLITE_CellSizeCk ){ - int i; /* Index into the cell pointer array */ - int sz; /* Size of a cell */ - - if( !pPage->leaf ) iCellLast--; - for(i=0; inCell; i++){ - pc = get2byteAligned(&data[cellOffset+i*2]); - testcase( pc==iCellFirst ); - testcase( pc==iCellLast ); - if( pciCellLast ){ - return SQLITE_CORRUPT_PAGE(pPage); - } - sz = pPage->xCellSize(pPage, &data[pc]); - testcase( pc+sz==usableSize ); - if( pc+sz>usableSize ){ - return SQLITE_CORRUPT_PAGE(pPage); - } - } - if( !pPage->leaf ) iCellLast++; - } /* Compute the total free space on the page ** EVIDENCE-OF: R-23588-34450 The two-byte integer at offset 1 gives the @@ -65034,11 +65328,104 @@ static int btreeInitPage(MemPage *pPage){ ** serves to verify that the offset to the start of the cell-content ** area, according to the page header, lies within the page. */ - if( nFree>usableSize ){ + if( nFree>usableSize || nFreenFree = (u16)(nFree - iCellFirst); + return SQLITE_OK; +} + +/* +** Do additional sanity check after btreeInitPage() if +** PRAGMA cell_size_check=ON +*/ +static SQLITE_NOINLINE int btreeCellSizeCheck(MemPage *pPage){ + int iCellFirst; /* First allowable cell or freeblock offset */ + int iCellLast; /* Last possible cell or freeblock offset */ + int i; /* Index into the cell pointer array */ + int sz; /* Size of a cell */ + int pc; /* Address of a freeblock within pPage->aData[] */ + u8 *data; /* Equal to pPage->aData */ + int usableSize; /* Maximum usable space on the page */ + int cellOffset; /* Start of cell content area */ + + iCellFirst = pPage->cellOffset + 2*pPage->nCell; + usableSize = pPage->pBt->usableSize; + iCellLast = usableSize - 4; + data = pPage->aData; + cellOffset = pPage->cellOffset; + if( !pPage->leaf ) iCellLast--; + for(i=0; inCell; i++){ + pc = get2byteAligned(&data[cellOffset+i*2]); + testcase( pc==iCellFirst ); + testcase( pc==iCellLast ); + if( pciCellLast ){ + return SQLITE_CORRUPT_PAGE(pPage); + } + sz = pPage->xCellSize(pPage, &data[pc]); + testcase( pc+sz==usableSize ); + if( pc+sz>usableSize ){ + return SQLITE_CORRUPT_PAGE(pPage); + } + } + return SQLITE_OK; +} + +/* +** Initialize the auxiliary information for a disk block. +** +** Return SQLITE_OK on success. If we see that the page does +** not contain a well-formed database page, then return +** SQLITE_CORRUPT. Note that a return of SQLITE_OK does not +** guarantee that the page is well-formed. It only shows that +** we failed to detect any corruption. +*/ +static int btreeInitPage(MemPage *pPage){ + u8 *data; /* Equal to pPage->aData */ + BtShared *pBt; /* The main btree structure */ + + assert( pPage->pBt!=0 ); + assert( pPage->pBt->db!=0 ); + assert( sqlite3_mutex_held(pPage->pBt->mutex) ); + assert( pPage->pgno==sqlite3PagerPagenumber(pPage->pDbPage) ); + assert( pPage == sqlite3PagerGetExtra(pPage->pDbPage) ); + assert( pPage->aData == sqlite3PagerGetData(pPage->pDbPage) ); + assert( pPage->isInit==0 ); + + pBt = pPage->pBt; + data = pPage->aData + pPage->hdrOffset; + /* EVIDENCE-OF: R-28594-02890 The one-byte flag at offset 0 indicating + ** the b-tree page type. */ + if( decodeFlags(pPage, data[0]) ){ + return SQLITE_CORRUPT_PAGE(pPage); + } + assert( pBt->pageSize>=512 && pBt->pageSize<=65536 ); + pPage->maskPage = (u16)(pBt->pageSize - 1); + pPage->nOverflow = 0; + pPage->cellOffset = pPage->hdrOffset + 8 + pPage->childPtrSize; + pPage->aCellIdx = data + pPage->childPtrSize + 8; + pPage->aDataEnd = pPage->aData + pBt->usableSize; + pPage->aDataOfst = pPage->aData + pPage->childPtrSize; + /* EVIDENCE-OF: R-37002-32774 The two-byte integer at offset 3 gives the + ** number of cells on the page. */ + pPage->nCell = get2byte(&data[3]); + if( pPage->nCell>MX_CELL(pBt) ){ + /* To many cells for a single page. The page must be corrupt */ + return SQLITE_CORRUPT_PAGE(pPage); + } + testcase( pPage->nCell==MX_CELL(pBt) ); + /* EVIDENCE-OF: R-24089-57979 If a page contains no cells (which is only + ** possible for a root page of a table that contains no rows) then the + ** offset to the cell content area will equal the page size minus the + ** bytes of reserved space. */ + assert( pPage->nCell>0 + || get2byteNotZero(&data[5])==(int)pBt->usableSize + || CORRUPT_DB ); + pPage->nFree = -1; /* Indicate that this value is yet uncomputed */ pPage->isInit = 1; + if( pBt->db->flags & SQLITE_CellSizeCk ){ + return btreeCellSizeCheck(pPage); + } return SQLITE_OK; } @@ -65181,19 +65568,18 @@ static int getAndInitPage( if( pgno>btreePagecount(pBt) ){ rc = SQLITE_CORRUPT_BKPT; - goto getAndInitPage_error; + goto getAndInitPage_error1; } rc = sqlite3PagerGet(pBt->pPager, pgno, (DbPage**)&pDbPage, bReadOnly); if( rc ){ - goto getAndInitPage_error; + goto getAndInitPage_error1; } *ppPage = (MemPage*)sqlite3PagerGetExtra(pDbPage); if( (*ppPage)->isInit==0 ){ btreePageFromDbPage(pDbPage, pgno, pBt); rc = btreeInitPage(*ppPage); if( rc!=SQLITE_OK ){ - releasePage(*ppPage); - goto getAndInitPage_error; + goto getAndInitPage_error2; } } assert( (*ppPage)->pgno==pgno ); @@ -65203,12 +65589,13 @@ static int getAndInitPage( ** compatible with the root page. */ if( pCur && ((*ppPage)->nCell<1 || (*ppPage)->intKey!=pCur->curIntKey) ){ rc = SQLITE_CORRUPT_PGNO(pgno); - releasePage(*ppPage); - goto getAndInitPage_error; + goto getAndInitPage_error2; } return SQLITE_OK; -getAndInitPage_error: +getAndInitPage_error2: + releasePage(*ppPage); +getAndInitPage_error1: if( pCur ){ pCur->iPage--; pCur->pPage = pCur->apPage[pCur->iPage]; @@ -67169,6 +67556,18 @@ SQLITE_PRIVATE int sqlite3BtreeTripAllCursors(Btree *pBtree, int errCode, int wr return rc; } +/* +** Set the pBt->nPage field correctly, according to the current +** state of the database. Assume pBt->pPage1 is valid. +*/ +static void btreeSetNPage(BtShared *pBt, MemPage *pPage1){ + int nPage = get4byte(&pPage1->aData[28]); + testcase( nPage==0 ); + if( nPage==0 ) sqlite3PagerPagecount(pBt->pPager, &nPage); + testcase( pBt->nPage!=nPage ); + pBt->nPage = nPage; +} + /* ** Rollback the transaction in progress. ** @@ -67214,11 +67613,7 @@ SQLITE_PRIVATE int sqlite3BtreeRollback(Btree *p, int tripCode, int writeOnly){ ** call btreeGetPage() on page 1 again to make ** sure pPage1->aData is set correctly. */ if( btreeGetPage(pBt, 1, &pPage1, 0)==SQLITE_OK ){ - int nPage = get4byte(28+(u8*)pPage1->aData); - testcase( nPage==0 ); - if( nPage==0 ) sqlite3PagerPagecount(pBt->pPager, &nPage); - testcase( pBt->nPage!=nPage ); - pBt->nPage = nPage; + btreeSetNPage(pBt, pPage1); releasePageOne(pPage1); } assert( countValidCursors(pBt, 1)==0 ); @@ -67298,12 +67693,11 @@ SQLITE_PRIVATE int sqlite3BtreeSavepoint(Btree *p, int op, int iSavepoint){ pBt->nPage = 0; } rc = newDatabase(pBt); - pBt->nPage = get4byte(28 + pBt->pPage1->aData); + btreeSetNPage(pBt, pBt->pPage1); - /* The database size was written into the offset 28 of the header - ** when the transaction started, so we know that the value at offset - ** 28 is nonzero. */ - assert( pBt->nPage>0 ); + /* pBt->nPage might be zero if the database was corrupt when + ** the transaction was started. Otherwise, it must be at least 1. */ + assert( CORRUPT_DB || pBt->nPage>0 ); } sqlite3BtreeLeave(p); } @@ -67885,6 +68279,7 @@ static int accessPayload( assert( aWrite>=pBufStart ); /* due to (6) */ memcpy(aSave, aWrite, 4); rc = sqlite3OsRead(fd, aWrite, a+4, (i64)pBt->pageSize*(nextPage-1)); + if( rc && nextPage>pBt->nPage ) rc = SQLITE_CORRUPT_BKPT; nextPage = get4byte(aWrite); memcpy(aWrite, aSave, 4); }else @@ -68289,23 +68684,6 @@ SQLITE_PRIVATE int sqlite3BtreeFirst(BtCursor *pCur, int *pRes){ return rc; } -/* -** This function is a no-op if cursor pCur does not point to a valid row. -** Otherwise, if pCur is valid, configure it so that the next call to -** sqlite3BtreeNext() is a no-op. -*/ -#ifndef SQLITE_OMIT_WINDOWFUNC -SQLITE_PRIVATE void sqlite3BtreeSkipNext(BtCursor *pCur){ - /* We believe that the cursor must always be in the valid state when - ** this routine is called, but the proof is difficult, so we add an - ** ALWaYS() test just in case we are wrong. */ - if( ALWAYS(pCur->eState==CURSOR_VALID) ){ - pCur->eState = CURSOR_SKIPNEXT; - pCur->skipNext = 1; - } -} -#endif /* SQLITE_OMIT_WINDOWFUNC */ - /* Move the cursor to the last entry in the table. Return SQLITE_OK ** on success. Set *pRes to 0 if the cursor actually points to something ** or set *pRes to 1 if the table is empty. @@ -68328,6 +68706,7 @@ SQLITE_PRIVATE int sqlite3BtreeLast(BtCursor *pCur, int *pRes){ assert( pCur->ix==pCur->pPage->nCell-1 ); assert( pCur->pPage->leaf ); #endif + *pRes = 0; return SQLITE_OK; } @@ -68549,6 +68928,7 @@ SQLITE_PRIVATE int sqlite3BtreeMovetoUnpacked( ** case this happens. */ void *pCellKey; u8 * const pCellBody = pCell - pPage->childPtrSize; + const int nOverrun = 18; /* Size of the overrun padding */ pPage->xParseCell(pPage, pCellBody, &pCur->info); nCell = (int)pCur->info.nKey; testcase( nCell<0 ); /* True if key size is 2^32 or more */ @@ -68559,19 +68939,20 @@ SQLITE_PRIVATE int sqlite3BtreeMovetoUnpacked( rc = SQLITE_CORRUPT_PAGE(pPage); goto moveto_finish; } - pCellKey = sqlite3Malloc( nCell+18 ); + pCellKey = sqlite3Malloc( nCell+nOverrun ); if( pCellKey==0 ){ rc = SQLITE_NOMEM_BKPT; goto moveto_finish; } pCur->ix = (u16)idx; rc = accessPayload(pCur, 0, nCell, (unsigned char*)pCellKey, 0); + memset(((u8*)pCellKey)+nCell,0,nOverrun); /* Fix uninit warnings */ pCur->curFlags &= ~BTCF_ValidOvfl; if( rc ){ sqlite3_free(pCellKey); goto moveto_finish; } - c = xRecordCompare(nCell, pCellKey, pIdxKey); + c = sqlite3VdbeRecordCompare(nCell, pCellKey, pIdxKey); sqlite3_free(pCellKey); } assert( @@ -69203,13 +69584,15 @@ static int freePage2(BtShared *pBt, MemPage *pMemPage, Pgno iPage){ MemPage *pPage1 = pBt->pPage1; /* Local reference to page 1 */ MemPage *pPage; /* Page being freed. May be NULL. */ int rc; /* Return Code */ - int nFree; /* Initial number of pages on free-list */ + u32 nFree; /* Initial number of pages on free-list */ assert( sqlite3_mutex_held(pBt->mutex) ); assert( CORRUPT_DB || iPage>1 ); assert( !pMemPage || pMemPage->pgno==iPage ); - if( iPage<2 ) return SQLITE_CORRUPT_BKPT; + if( iPage<2 || iPage>pBt->nPage ){ + return SQLITE_CORRUPT_BKPT; + } if( pMemPage ){ pPage = pMemPage; sqlite3PagerRef(pPage->pDbPage); @@ -69620,6 +70003,7 @@ static void dropCell(MemPage *pPage, int idx, int sz, int *pRC){ assert( CORRUPT_DB || sz==cellSize(pPage, idx) ); assert( sqlite3PagerIswriteable(pPage->pDbPage) ); assert( sqlite3_mutex_held(pPage->pBt->mutex) ); + assert( pPage->nFree>=0 ); data = pPage->aData; ptr = &pPage->aCellIdx[2*idx]; pc = get2byte(ptr); @@ -69684,12 +70068,8 @@ static void insertCell( assert( pPage->nOverflow<=ArraySize(pPage->apOvfl) ); assert( ArraySize(pPage->apOvfl)==ArraySize(pPage->aiOvfl) ); assert( sqlite3_mutex_held(pPage->pBt->mutex) ); - /* The cell should normally be sized correctly. However, when moving a - ** malformed cell from a leaf page to an interior page, if the cell size - ** wanted to be less than 4 but got rounded up to 4 on the leaf, then size - ** might be less than 8 (leaf-size + pointer) on the interior node. Hence - ** the term after the || in the following assert(). */ - assert( sz==pPage->xCellSize(pPage, pCell) || (sz==8 && iChild>0) ); + assert( sz==pPage->xCellSize(pPage, pCell) || CORRUPT_DB ); + assert( pPage->nFree>=0 ); if( pPage->nOverflow || sz+2>pPage->nFree ){ if( pTemp ){ memcpy(pTemp, pCell, sz); @@ -69747,7 +70127,7 @@ static void insertCell( pPage->nCell++; /* increment the cell count */ if( (++data[pPage->hdrOffset+4])==0 ) data[pPage->hdrOffset+3]++; - assert( get2byte(&data[pPage->hdrOffset+3])==pPage->nCell ); + assert( get2byte(&data[pPage->hdrOffset+3])==pPage->nCell || CORRUPT_DB ); #ifndef SQLITE_OMIT_AUTOVACUUM if( pPage->pBt->autoVacuum ){ /* The cell may contain a pointer to an overflow page. If so, write @@ -69834,8 +70214,13 @@ static void insertCell( ** are used and they point to the leaf pages only, and the ixNx value are: ** ** ixNx[0] = Number of cells in Child-1. -** ixNx[1] = Number of cells in Child-1 and Child-2 + 1 for 1st divider. -** ixNx[2] = Number of cells in Child-1 and Child-2 + both divider cells +** ixNx[1] = Number of cells in Child-1 and Child-2. +** ixNx[2] = Total number of cells. +** +** Sometimes when deleting, a child page can have zero cells. In those +** cases, ixNx[] entries with higher indexes, and the corresponding apEnd[] +** entries, shift down. The end result is that each ixNx[] entry should +** be larger than the previous */ typedef struct CellArray CellArray; struct CellArray { @@ -69915,7 +70300,7 @@ static int rebuildPage( assert( i(u32)usableSize) ){ j = 0; } + if( j>(u32)usableSize ){ j = 0; } memcpy(&pTmp[j], &aData[j], usableSize - j); for(k=0; pCArray->ixNx[k]<=i && ALWAYS(kszCell[i]!=0 ); + sz = pCArray->szCell[i]; if( (aData[1]==0 && aData[2]==0) || (pSlot = pageFindSlot(pPg,sz,&rc))==0 ){ if( (pData - pBegin)aiOvfl[i]) - iNew; if( iCell>=0 && iCellaCellIdx[iCell * 2]; - assert( nCell>=iCell ); - memmove(&pCellptr[2], pCellptr, (nCell - iCell) * 2); + if( nCell>iCell ){ + memmove(&pCellptr[2], pCellptr, (nCell - iCell) * 2); + } nCell++; + cachedCellSize(pCArray, iCell+iNew); if( pageInsertArray( pPg, pBegin, &pData, pCellptr, iCell+iNew, 1, pCArray @@ -70241,8 +70629,10 @@ static int balance_quick(MemPage *pParent, MemPage *pPage, u8 *pSpace){ assert( sqlite3_mutex_held(pPage->pBt->mutex) ); assert( sqlite3PagerIswriteable(pParent->pDbPage) ); assert( pPage->nOverflow==1 ); - + if( pPage->nCell==0 ) return SQLITE_CORRUPT_BKPT; /* dbfuzz001.test */ + assert( pPage->nFree>=0 ); + assert( pParent->nFree>=0 ); /* Allocate a new page. This page will become the right-sibling of ** pPage. Make the parent page writable, so that the new divider cell @@ -70412,6 +70802,7 @@ static void copyNodeContent(MemPage *pFrom, MemPage *pTo, int *pRC){ */ pTo->isInit = 0; rc = btreeInitPage(pTo); + if( rc==SQLITE_OK ) rc = btreeComputeFreeSpace(pTo); if( rc!=SQLITE_OK ){ *pRC = rc; return; @@ -70520,6 +70911,7 @@ static int balance_nonroot( if( !aOvflSpace ){ return SQLITE_NOMEM_BKPT; } + assert( pParent->nFree>=0 ); /* Find the sibling pages to balance. Also locate the cells in pParent ** that divide the siblings. An attempt is made to find NN siblings on @@ -70559,7 +70951,13 @@ static int balance_nonroot( memset(apOld, 0, (i+1)*sizeof(MemPage*)); goto balance_cleanup; } - nMaxCells += 1+apOld[i]->nCell+apOld[i]->nOverflow; + if( apOld[i]->nFree<0 ){ + rc = btreeComputeFreeSpace(apOld[i]); + if( rc ){ + memset(apOld, 0, (i)*sizeof(MemPage*)); + goto balance_cleanup; + } + } if( (i--)==0 ) break; if( pParent->nOverflow && i+nxDiv==pParent->aiOvfl[0] ){ @@ -70603,6 +71001,7 @@ static int balance_nonroot( /* Make nMaxCells a multiple of 4 in order to preserve 8-byte ** alignment */ + nMaxCells = nOld*(MX_CELL(pBt) + ArraySize(pParent->apOvfl)); nMaxCells = (nMaxCells + 3)&~3; /* @@ -70613,7 +71012,7 @@ static int balance_nonroot( + nMaxCells*sizeof(u16) /* b.szCell */ + pBt->pageSize; /* aSpace1 */ - assert( szScratch<=6*(int)pBt->pageSize ); + assert( szScratch<=7*(int)pBt->pageSize ); b.apCell = sqlite3StackAllocRaw(0, szScratch ); if( b.apCell==0 ){ rc = SQLITE_NOMEM_BKPT; @@ -70649,6 +71048,7 @@ static int balance_nonroot( u16 maskPage = pOld->maskPage; u8 *piCell = aData + pOld->cellOffset; u8 *piEnd; + VVA_ONLY( int nCellAtStart = b.nCell; ) /* Verify that all sibling pages are of the same "type" (table-leaf, ** table-interior, index-leaf, or index-interior). @@ -70677,6 +71077,10 @@ static int balance_nonroot( */ memset(&b.szCell[b.nCell], 0, sizeof(b.szCell[0])*(limit+pOld->nOverflow)); if( pOld->nOverflow>0 ){ + if( NEVER(limitaiOvfl[0]) ){ + rc = SQLITE_CORRUPT_BKPT; + goto balance_cleanup; + } limit = pOld->aiOvfl[0]; for(j=0; jnCell+pOld->nOverflow) ); cntOld[i] = b.nCell; if( iaDataEnd; b.ixNx[k] = cntOld[i]; + if( k && b.ixNx[k]==b.ixNx[k-1] ){ + k--; /* Omit b.ixNx[] entry for child pages with no cells */ + } if( !leafData ){ k++; b.apEnd[k] = pParent->aDataEnd; b.ixNx[k] = cntOld[i]+1; } + assert( p->nFree>=0 ); szNew[i] = usableSpace - p->nFree; for(j=0; jnOverflow; j++){ szNew[i] += 2 + p->xCellSize(p, p->apOvfl[j]); @@ -70954,6 +71363,8 @@ static int balance_nonroot( )); assert( sqlite3PagerIswriteable(pParent->pDbPage) ); + assert( nNew>=1 && nNew<=ArraySize(apNew) ); + assert( apNew[nNew-1]!=0 ); put4byte(pRight, apNew[nNew-1]->pgno); /* If the sibling pages are not leaves, ensure that the right-child pointer @@ -70983,18 +71394,18 @@ static int balance_nonroot( if( ISAUTOVACUUM ){ MemPage *pOld; MemPage *pNew = pOld = apNew[0]; - u8 *aOld = pNew->aData; int cntOldNext = pNew->nCell + pNew->nOverflow; - int usableSize = pBt->usableSize; int iNew = 0; int iOld = 0; for(i=0; i=0 && iOldnCell + pOld->nOverflow + !leafData; - aOld = pOld->aData; } if( i==cntNew[iNew] ){ pNew = apNew[++iNew]; @@ -71009,7 +71420,7 @@ static int balance_nonroot( ** overflow cell), we can skip updating the pointer map entries. */ if( iOld>=nNew || pNew->pgno!=aPgno[iOld] - || !SQLITE_WITHIN(pCell,aOld,&aOld[usableSize]) + || !SQLITE_WITHIN(pCell,pOld->aData,pOld->aDataEnd) ){ if( !leafCorrection ){ ptrmapPut(pBt, get4byte(pCell), PTRMAP_BTREE, pNew->pgno, &rc); @@ -71160,7 +71571,8 @@ static int balance_nonroot( rc = defragmentPage(apNew[0], -1); testcase( rc!=SQLITE_OK ); assert( apNew[0]->nFree == - (get2byte(&apNew[0]->aData[5])-apNew[0]->cellOffset-apNew[0]->nCell*2) + (get2byteNotZero(&apNew[0]->aData[5]) - apNew[0]->cellOffset + - apNew[0]->nCell*2) || rc!=SQLITE_OK ); copyNodeContent(apNew[0], pParent, &rc); @@ -71259,7 +71671,7 @@ static int balance_deeper(MemPage *pRoot, MemPage **ppChild){ } assert( sqlite3PagerIswriteable(pChild->pDbPage) ); assert( sqlite3PagerIswriteable(pRoot->pDbPage) ); - assert( pChild->nCell==pRoot->nCell ); + assert( pChild->nCell==pRoot->nCell || CORRUPT_DB ); TRACE(("BALANCE: copy root %d into %d\n", pRoot->pgno, pChild->pgno)); @@ -71298,10 +71710,13 @@ static int balance(BtCursor *pCur){ VVA_ONLY( int balance_deeper_called = 0 ); do { - int iPage = pCur->iPage; + int iPage; MemPage *pPage = pCur->pPage; - if( iPage==0 ){ + if( NEVER(pPage->nFree<0) && btreeComputeFreeSpace(pPage) ) break; + if( pPage->nOverflow==0 && pPage->nFree<=nMin ){ + break; + }else if( (iPage = pCur->iPage)==0 ){ if( pPage->nOverflow ){ /* The root page of the b-tree is overfull. In this case call the ** balance_deeper() function to create a new child for the root-page @@ -71322,13 +71737,14 @@ static int balance(BtCursor *pCur){ }else{ break; } - }else if( pPage->nOverflow==0 && pPage->nFree<=nMin ){ - break; }else{ MemPage * const pParent = pCur->apPage[iPage-1]; int const iIdx = pCur->aiIdx[iPage-1]; rc = sqlite3PagerWrite(pParent->pDbPage); + if( rc==SQLITE_OK && pParent->nFree<0 ){ + rc = btreeComputeFreeSpace(pParent); + } if( rc==SQLITE_OK ){ #ifndef SQLITE_OMIT_QUICKBALANCE if( pPage->intKeyLeaf @@ -71462,7 +71878,9 @@ static int btreeOverwriteCell(BtCursor *pCur, const BtreePayload *pX){ Pgno ovflPgno; /* Next overflow page to write */ u32 ovflPageSize; /* Size to write on overflow page */ - if( pCur->info.pPayload + pCur->info.nLocal > pPage->aDataEnd ){ + if( pCur->info.pPayload + pCur->info.nLocal > pPage->aDataEnd + || pCur->info.pPayload < pPage->aData + pPage->cellOffset + ){ return SQLITE_CORRUPT_BKPT; } /* Overwrite the local portion first */ @@ -71675,6 +72093,10 @@ SQLITE_PRIVATE int sqlite3BtreeInsert( pPage = pCur->pPage; assert( pPage->intKey || pX->nKey>=0 ); assert( pPage->leaf || !pPage->intKey ); + if( pPage->nFree<0 ){ + rc = btreeComputeFreeSpace(pPage); + if( rc ) return rc; + } TRACE(("INSERT: table=%d nkey=%lld ndata=%d page=%d %s\n", pCur->pgnoRoot, pX->nKey, pX->nData, pPage->pgno, @@ -71699,6 +72121,8 @@ SQLITE_PRIVATE int sqlite3BtreeInsert( memcpy(newCell, oldCell, 4); } rc = clearCell(pPage, oldCell, &info); + testcase( pCur->curFlags & BTCF_ValidOvfl ); + invalidateOverflowCache(pCur); if( info.nSize==szNew && info.nLocal==info.nPayload && (!ISAUTOVACUUM || szNewminLocal) ){ @@ -71712,7 +72136,12 @@ SQLITE_PRIVATE int sqlite3BtreeInsert( ** new entry uses overflow pages, as the insertCell() call below is ** necessary to add the PTRMAP_OVERFLOW1 pointer-map entry. */ assert( rc==SQLITE_OK ); /* clearCell never fails when nLocal==nPayload */ - if( oldCell+szNew > pPage->aDataEnd ) return SQLITE_CORRUPT_BKPT; + if( oldCell < pPage->aData+pPage->hdrOffset+10 ){ + return SQLITE_CORRUPT_BKPT; + } + if( oldCell+szNew > pPage->aDataEnd ){ + return SQLITE_CORRUPT_BKPT; + } memcpy(oldCell, newCell, szNew); return SQLITE_OK; } @@ -71817,14 +72246,18 @@ SQLITE_PRIVATE int sqlite3BtreeDelete(BtCursor *pCur, u8 flags){ assert( pCur->curFlags & BTCF_WriteFlag ); assert( hasSharedCacheTableLock(p, pCur->pgnoRoot, pCur->pKeyInfo!=0, 2) ); assert( !hasReadConflicts(p, pCur->pgnoRoot) ); - assert( pCur->ixpPage->nCell ); - assert( pCur->eState==CURSOR_VALID ); assert( (flags & ~(BTREE_SAVEPOSITION | BTREE_AUXDELETE))==0 ); + if( pCur->eState==CURSOR_REQUIRESEEK ){ + rc = btreeRestoreCursorPosition(pCur); + if( rc ) return rc; + } + assert( pCur->eState==CURSOR_VALID ); iCellDepth = pCur->iPage; iCellIdx = pCur->ix; pPage = pCur->pPage; pCell = findCell(pPage, iCellIdx); + if( pPage->nFree<0 && btreeComputeFreeSpace(pPage) ) return SQLITE_CORRUPT; /* If the bPreserve flag is set to true, then the cursor position must ** be preserved following this delete operation. If the current delete @@ -71895,6 +72328,10 @@ SQLITE_PRIVATE int sqlite3BtreeDelete(BtCursor *pCur, u8 flags){ Pgno n; unsigned char *pTmp; + if( pLeaf->nFree<0 ){ + rc = btreeComputeFreeSpace(pLeaf); + if( rc ) return rc; + } if( iCellDepthiPage-1 ){ n = pCur->apPage[iCellDepth+1]->pgno; }else{ @@ -72253,6 +72690,9 @@ static int btreeDropTable(Btree *p, Pgno iTable, int *piMoved){ assert( sqlite3BtreeHoldsMutex(p) ); assert( p->inTrans==TRANS_WRITE ); assert( iTable>=2 ); + if( iTable>btreePagecount(pBt) ){ + return SQLITE_CORRUPT_BKPT; + } rc = btreeGetPage(pBt, (Pgno)iTable, &pPage, 0); if( rc ) return rc; @@ -72601,10 +73041,10 @@ static void checkList( IntegrityCk *pCheck, /* Integrity checking context */ int isFreeList, /* True for a freelist. False for overflow page list */ int iPage, /* Page number for first page in the list */ - int N /* Expected number of pages in the list */ + u32 N /* Expected number of pages in the list */ ){ int i; - int expected = N; + u32 expected = N; int nErrAtStart = pCheck->nErr; while( iPage!=0 && pCheck->mxErr ){ DbPage *pOvflPage; @@ -72786,6 +73226,11 @@ static int checkTreePage( "btreeInitPage() returns error code %d", rc); goto end_of_check; } + if( (rc = btreeComputeFreeSpace(pPage))!=0 ){ + assert( rc==SQLITE_CORRUPT ); + checkAppendMsg(pCheck, "free space corruption", rc); + goto end_of_check; + } data = pPage->aData; hdr = pPage->hdrOffset; @@ -72858,7 +73303,7 @@ static int checkTreePage( /* Check the content overflow list */ if( info.nPayload>info.nLocal ){ - int nPage; /* Number of pages on the overflow chain */ + u32 nPage; /* Number of pages on the overflow chain */ Pgno pgnoOvfl; /* First page of the overflow chain */ assert( pc + info.nSize - 4 <= usableSize ); nPage = (info.nPayload - info.nLocal + usableSize - 5)/(usableSize - 4); @@ -72918,9 +73363,9 @@ static int checkTreePage( i = get2byte(&data[hdr+1]); while( i>0 ){ int size, j; - assert( (u32)i<=usableSize-4 ); /* Enforced by btreeInitPage() */ + assert( (u32)i<=usableSize-4 ); /* Enforced by btreeComputeFreeSpace() */ size = get2byte(&data[i+2]); - assert( (u32)(i+size)<=usableSize ); /* Enforced by btreeInitPage() */ + assert( (u32)(i+size)<=usableSize ); /* due to btreeComputeFreeSpace() */ btreeHeapInsert(heap, (((u32)i)<<16)|(i+size-1)); /* EVIDENCE-OF: R-58208-19414 The first 2 bytes of a freeblock are a ** big-endian integer which is the offset in the b-tree page of the next @@ -72929,8 +73374,8 @@ static int checkTreePage( j = get2byte(&data[i]); /* EVIDENCE-OF: R-06866-39125 Freeblocks are always connected in order of ** increasing offset. */ - assert( j==0 || j>i+size ); /* Enforced by btreeInitPage() */ - assert( (u32)j<=usableSize-4 ); /* Enforced by btreeInitPage() */ + assert( j==0 || j>i+size ); /* Enforced by btreeComputeFreeSpace() */ + assert( (u32)j<=usableSize-4 ); /* Enforced by btreeComputeFreeSpace() */ i = j; } /* Analyze the min-heap looking for overlap between cells and/or @@ -73688,7 +74133,7 @@ static int backupOnePage( if( nSrcReserve!=nDestReserve ){ u32 newPgsz = nSrcPgsz; rc = sqlite3PagerSetPagesize(pDestPager, &newPgsz, nSrcReserve); - if( rc==SQLITE_OK && newPgsz!=nSrcPgsz ) rc = SQLITE_READONLY; + if( rc==SQLITE_OK && newPgsz!=(u32)nSrcPgsz ) rc = SQLITE_READONLY; } #endif @@ -74033,8 +74478,10 @@ SQLITE_API int sqlite3_backup_finish(sqlite3_backup *p){ } if( p->isAttached ){ pp = sqlite3PagerBackupPtr(sqlite3BtreePager(p->pSrc)); + assert( pp!=0 ); while( *pp!=p ){ pp = &(*pp)->pNext; + assert( pp!=0 ); } *pp = p->pNext; } @@ -74235,6 +74682,11 @@ SQLITE_PRIVATE int sqlite3BtreeCopyFile(Btree *pTo, Btree *pFrom){ /* #include "sqliteInt.h" */ /* #include "vdbeInt.h" */ +/* True if X is a power of two. 0 is considered a power of two here. +** In other words, return true if X has at most one bit set. +*/ +#define ISPOWEROF2(X) (((X)&((X)-1))==0) + #ifdef SQLITE_DEBUG /* ** Check invariants on a Mem object. @@ -74254,8 +74706,8 @@ SQLITE_PRIVATE int sqlite3VdbeCheckMemInvariants(Mem *p){ ** That saves a few cycles in inner loops. */ assert( (p->flags & MEM_Dyn)==0 || p->szMalloc==0 ); - /* Cannot be both MEM_Int and MEM_Real at the same time */ - assert( (p->flags & (MEM_Int|MEM_Real))!=(MEM_Int|MEM_Real) ); + /* Cannot have more than one of MEM_Int, MEM_Real, or MEM_IntReal */ + assert( ISPOWEROF2(p->flags & (MEM_Int|MEM_Real|MEM_IntReal)) ); if( p->flags & MEM_Null ){ /* Cannot be both MEM_Null and some other type */ @@ -74274,7 +74726,7 @@ SQLITE_PRIVATE int sqlite3VdbeCheckMemInvariants(Mem *p){ ((p->flags&MEM_Static)!=0 ? 1 : 0) <= 1 ); /* No other bits set */ - assert( (p->flags & ~(MEM_Null|MEM_Term|MEM_Subtype + assert( (p->flags & ~(MEM_Null|MEM_Term|MEM_Subtype|MEM_FromBind |MEM_Dyn|MEM_Ephem|MEM_Static))==0 ); }else{ /* A pure NULL might have other flags, such as MEM_Static, MEM_Dyn, @@ -74309,9 +74761,31 @@ SQLITE_PRIVATE int sqlite3VdbeCheckMemInvariants(Mem *p){ } #endif +/* +** Render a Mem object which is one of MEM_Int, MEM_Real, or MEM_IntReal +** into a buffer. +*/ +static void vdbeMemRenderNum(int sz, char *zBuf, Mem *p){ + StrAccum acc; + assert( p->flags & (MEM_Int|MEM_Real|MEM_IntReal) ); + sqlite3StrAccumInit(&acc, 0, zBuf, sz, 0); + if( p->flags & MEM_Int ){ + sqlite3_str_appendf(&acc, "%lld", p->u.i); + }else if( p->flags & MEM_IntReal ){ + sqlite3_str_appendf(&acc, "%!.15g", (double)p->u.i); + }else{ + sqlite3_str_appendf(&acc, "%!.15g", p->u.r); + } + assert( acc.zText==zBuf && acc.mxAlloc<=0 ); + zBuf[acc.nChar] = 0; /* Fast version of sqlite3StrAccumFinish(&acc) */ +} + #ifdef SQLITE_DEBUG /* -** Check that string value of pMem agrees with its integer or real value. +** Validity checks on pMem. pMem holds a string. +** +** (1) Check that string value of pMem agrees with its integer or real value. +** (2) Check that the string is correctly zero terminated ** ** A single int or real value always converts to the same strings. But ** many different strings can be converted into the same int or real. @@ -74329,17 +74803,24 @@ SQLITE_PRIVATE int sqlite3VdbeCheckMemInvariants(Mem *p){ ** ** This routine is for use inside of assert() statements only. */ -SQLITE_PRIVATE int sqlite3VdbeMemConsistentDualRep(Mem *p){ +SQLITE_PRIVATE int sqlite3VdbeMemValidStrRep(Mem *p){ char zBuf[100]; char *z; int i, j, incr; if( (p->flags & MEM_Str)==0 ) return 1; - if( (p->flags & (MEM_Int|MEM_Real))==0 ) return 1; - if( p->flags & MEM_Int ){ - sqlite3_snprintf(sizeof(zBuf),zBuf,"%lld",p->u.i); - }else{ - sqlite3_snprintf(sizeof(zBuf),zBuf,"%!.15g",p->u.r); - } + if( p->flags & MEM_Term ){ + /* Insure that the string is properly zero-terminated. Pay particular + ** attention to the case where p->n is odd */ + if( p->szMalloc>0 && p->z==p->zMalloc ){ + assert( p->enc==SQLITE_UTF8 || p->szMalloc >= ((p->n+1)&~1)+2 ); + assert( p->enc!=SQLITE_UTF8 || p->szMalloc >= p->n+1 ); + } + assert( p->z[p->n]==0 ); + assert( p->enc==SQLITE_UTF8 || p->z[(p->n+1)&~1]==0 ); + assert( p->enc==SQLITE_UTF8 || p->z[((p->n+1)&~1)+1]==0 ); + } + if( (p->flags & (MEM_Int|MEM_Real|MEM_IntReal))==0 ) return 1; + vdbeMemRenderNum(sizeof(zBuf), zBuf, p); z = p->z; i = j = 0; incr = 1; @@ -74395,8 +74876,7 @@ SQLITE_PRIVATE int sqlite3VdbeChangeEncoding(Mem *pMem, int desiredEnc){ } /* -** Make sure pMem->z points to a writable allocation of at least -** min(n,32) bytes. +** Make sure pMem->z points to a writable allocation of at least n bytes. ** ** If the bPreserve argument is true, then copy of the content of ** pMem->z into the new allocation. pMem must be either a string or @@ -74415,9 +74895,14 @@ SQLITE_PRIVATE SQLITE_NOINLINE int sqlite3VdbeMemGrow(Mem *pMem, int n, int bPre assert( pMem->szMalloc==0 || pMem->szMalloc==sqlite3DbMallocSize(pMem->db, pMem->zMalloc) ); - if( n<32 ) n = 32; if( pMem->szMalloc>0 && bPreserve && pMem->z==pMem->zMalloc ){ - pMem->z = pMem->zMalloc = sqlite3DbReallocOrFree(pMem->db, pMem->z, n); + if( pMem->db ){ + pMem->z = pMem->zMalloc = sqlite3DbReallocOrFree(pMem->db, pMem->z, n); + }else{ + pMem->zMalloc = sqlite3Realloc(pMem->z, n); + if( pMem->zMalloc==0 ) sqlite3_free(pMem->z); + pMem->z = pMem->zMalloc; + } bPreserve = 0; }else{ if( pMem->szMalloc>0 ) sqlite3DbFreeNN(pMem->db, pMem->zMalloc); @@ -74453,8 +74938,8 @@ SQLITE_PRIVATE SQLITE_NOINLINE int sqlite3VdbeMemGrow(Mem *pMem, int n, int bPre ** ** Any prior string or blob content in the pMem object may be discarded. ** The pMem->xDel destructor is called, if it exists. Though MEM_Str -** and MEM_Blob values may be discarded, MEM_Int, MEM_Real, and MEM_Null -** values are preserved. +** and MEM_Blob values may be discarded, MEM_Int, MEM_Real, MEM_IntReal, +** and MEM_Null values are preserved. ** ** Return SQLITE_OK on success or an error code (probably SQLITE_NOMEM) ** if unable to complete the resizing. @@ -74467,20 +74952,26 @@ SQLITE_PRIVATE int sqlite3VdbeMemClearAndResize(Mem *pMem, int szNew){ } assert( (pMem->flags & MEM_Dyn)==0 ); pMem->z = pMem->zMalloc; - pMem->flags &= (MEM_Null|MEM_Int|MEM_Real); + pMem->flags &= (MEM_Null|MEM_Int|MEM_Real|MEM_IntReal); return SQLITE_OK; } /* ** It is already known that pMem contains an unterminated string. ** Add the zero terminator. +** +** Three bytes of zero are added. In this way, there is guaranteed +** to be a double-zero byte at an even byte boundary in order to +** terminate a UTF16 string, even if the initial size of the buffer +** is an odd number of bytes. */ static SQLITE_NOINLINE int vdbeMemAddTerminator(Mem *pMem){ - if( sqlite3VdbeMemGrow(pMem, pMem->n+2, 1) ){ + if( sqlite3VdbeMemGrow(pMem, pMem->n+3, 1) ){ return SQLITE_NOMEM_BKPT; } pMem->z[pMem->n] = 0; pMem->z[pMem->n+1] = 0; + pMem->z[pMem->n+2] = 0; pMem->flags |= MEM_Term; return SQLITE_OK; } @@ -74517,13 +75008,15 @@ SQLITE_PRIVATE int sqlite3VdbeMemMakeWriteable(Mem *pMem){ SQLITE_PRIVATE int sqlite3VdbeMemExpandBlob(Mem *pMem){ int nByte; assert( pMem->flags & MEM_Zero ); - assert( pMem->flags&MEM_Blob ); + assert( (pMem->flags&MEM_Blob)!=0 || MemNullNochng(pMem) ); + testcase( sqlite3_value_nochange(pMem) ); assert( !sqlite3VdbeMemIsRowSet(pMem) ); assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); /* Set nByte to the number of bytes required to store the expanded blob. */ nByte = pMem->n + pMem->u.nZero; if( nByte<=0 ){ + if( (pMem->flags & MEM_Blob)==0 ) return SQLITE_OK; nByte = 1; } if( sqlite3VdbeMemGrow(pMem, nByte, 1) ){ @@ -74552,12 +75045,12 @@ SQLITE_PRIVATE int sqlite3VdbeMemNulTerminate(Mem *pMem){ } /* -** Add MEM_Str to the set of representations for the given Mem. Numbers -** are converted using sqlite3_snprintf(). Converting a BLOB to a string -** is a no-op. +** Add MEM_Str to the set of representations for the given Mem. This +** routine is only called if pMem is a number of some kind, not a NULL +** or a BLOB. ** -** Existing representations MEM_Int and MEM_Real are invalidated if -** bForce is true but are retained if bForce is false. +** Existing representations MEM_Int, MEM_Real, or MEM_IntReal are invalidated +** if bForce is true but are retained if bForce is false. ** ** A MEM_Null value will never be passed to this function. This function is ** used for converting values to text for returning to the user (i.e. via @@ -74566,13 +75059,12 @@ SQLITE_PRIVATE int sqlite3VdbeMemNulTerminate(Mem *pMem){ ** user and the latter is an internal programming error. */ SQLITE_PRIVATE int sqlite3VdbeMemStringify(Mem *pMem, u8 enc, u8 bForce){ - int fg = pMem->flags; const int nByte = 32; assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); - assert( !(fg&MEM_Zero) ); - assert( !(fg&(MEM_Str|MEM_Blob)) ); - assert( fg&(MEM_Int|MEM_Real) ); + assert( !(pMem->flags&MEM_Zero) ); + assert( !(pMem->flags&(MEM_Str|MEM_Blob)) ); + assert( pMem->flags&(MEM_Int|MEM_Real|MEM_IntReal) ); assert( !sqlite3VdbeMemIsRowSet(pMem) ); assert( EIGHT_BYTE_ALIGNMENT(pMem) ); @@ -74582,23 +75074,12 @@ SQLITE_PRIVATE int sqlite3VdbeMemStringify(Mem *pMem, u8 enc, u8 bForce){ return SQLITE_NOMEM_BKPT; } - /* For a Real or Integer, use sqlite3_snprintf() to produce the UTF-8 - ** string representation of the value. Then, if the required encoding - ** is UTF-16le or UTF-16be do a translation. - ** - ** FIX ME: It would be better if sqlite3_snprintf() could do UTF-16. - */ - if( fg & MEM_Int ){ - sqlite3_snprintf(nByte, pMem->z, "%lld", pMem->u.i); - }else{ - assert( fg & MEM_Real ); - sqlite3_snprintf(nByte, pMem->z, "%!.15g", pMem->u.r); - } + vdbeMemRenderNum(nByte, pMem->z, pMem); assert( pMem->z!=0 ); pMem->n = sqlite3Strlen30NN(pMem->z); pMem->enc = SQLITE_UTF8; pMem->flags |= MEM_Str|MEM_Term; - if( bForce ) pMem->flags &= ~(MEM_Int|MEM_Real); + if( bForce ) pMem->flags &= ~(MEM_Int|MEM_Real|MEM_IntReal); sqlite3VdbeChangeEncoding(pMem, enc); return SQLITE_OK; } @@ -74772,7 +75253,8 @@ SQLITE_PRIVATE i64 sqlite3VdbeIntValue(Mem *pMem){ assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); assert( EIGHT_BYTE_ALIGNMENT(pMem) ); flags = pMem->flags; - if( flags & MEM_Int ){ + if( flags & (MEM_Int|MEM_IntReal) ){ + testcase( flags & MEM_IntReal ); return pMem->u.i; }else if( flags & MEM_Real ){ return doubleToInt64(pMem->u.r); @@ -74801,7 +75283,8 @@ SQLITE_PRIVATE double sqlite3VdbeRealValue(Mem *pMem){ assert( EIGHT_BYTE_ALIGNMENT(pMem) ); if( pMem->flags & MEM_Real ){ return pMem->u.r; - }else if( pMem->flags & MEM_Int ){ + }else if( pMem->flags & (MEM_Int|MEM_IntReal) ){ + testcase( pMem->flags & MEM_IntReal ); return (double)pMem->u.i; }else if( pMem->flags & (MEM_Str|MEM_Blob) ){ return memRealValue(pMem); @@ -74816,7 +75299,8 @@ SQLITE_PRIVATE double sqlite3VdbeRealValue(Mem *pMem){ ** Return the value ifNull if pMem is NULL. */ SQLITE_PRIVATE int sqlite3VdbeBooleanValue(Mem *pMem, int ifNull){ - if( pMem->flags & MEM_Int ) return pMem->u.i!=0; + testcase( pMem->flags & MEM_IntReal ); + if( pMem->flags & (MEM_Int|MEM_IntReal) ) return pMem->u.i!=0; if( pMem->flags & MEM_Null ) return ifNull; return sqlite3VdbeRealValue(pMem)!=0.0; } @@ -74879,17 +75363,21 @@ SQLITE_PRIVATE int sqlite3VdbeMemRealify(Mem *pMem){ /* Compare a floating point value to an integer. Return true if the two ** values are the same within the precision of the floating point value. ** +** This function assumes that i was obtained by assignment from r1. +** ** For some versions of GCC on 32-bit machines, if you do the more obvious ** comparison of "r1==(double)i" you sometimes get an answer of false even ** though the r1 and (double)i values are bit-for-bit the same. */ -static int sqlite3RealSameAsInt(double r1, sqlite3_int64 i){ +SQLITE_PRIVATE int sqlite3RealSameAsInt(double r1, sqlite3_int64 i){ double r2 = (double)i; - return memcmp(&r1, &r2, sizeof(r1))==0; + return r1==0.0 + || (memcmp(&r1, &r2, sizeof(r1))==0 + && i >= -2251799813685248LL && i < 2251799813685248LL); } /* -** Convert pMem so that it has types MEM_Real or MEM_Int or both. +** Convert pMem so that it has type MEM_Real or MEM_Int. ** Invalidate any prior representations. ** ** Every effort is made to force the conversion, even if the input @@ -74897,25 +75385,26 @@ static int sqlite3RealSameAsInt(double r1, sqlite3_int64 i){ ** as much of the string as we can and ignore the rest. */ SQLITE_PRIVATE int sqlite3VdbeMemNumerify(Mem *pMem){ - if( (pMem->flags & (MEM_Int|MEM_Real|MEM_Null))==0 ){ + testcase( pMem->flags & MEM_Int ); + testcase( pMem->flags & MEM_Real ); + testcase( pMem->flags & MEM_IntReal ); + testcase( pMem->flags & MEM_Null ); + if( (pMem->flags & (MEM_Int|MEM_Real|MEM_IntReal|MEM_Null))==0 ){ int rc; + sqlite3_int64 ix; assert( (pMem->flags & (MEM_Blob|MEM_Str))!=0 ); assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); - rc = sqlite3Atoi64(pMem->z, &pMem->u.i, pMem->n, pMem->enc); - if( rc==0 ){ + rc = sqlite3AtoF(pMem->z, &pMem->u.r, pMem->n, pMem->enc); + if( ((rc==0 || rc==1) && sqlite3Atoi64(pMem->z, &ix, pMem->n, pMem->enc)<=1) + || sqlite3RealSameAsInt(pMem->u.r, (ix = (i64)pMem->u.r)) + ){ + pMem->u.i = ix; MemSetTypeFlag(pMem, MEM_Int); }else{ - i64 i = pMem->u.i; - sqlite3AtoF(pMem->z, &pMem->u.r, pMem->n, pMem->enc); - if( rc==1 && sqlite3RealSameAsInt(pMem->u.r, i) ){ - pMem->u.i = i; - MemSetTypeFlag(pMem, MEM_Int); - }else{ - MemSetTypeFlag(pMem, MEM_Real); - } + MemSetTypeFlag(pMem, MEM_Real); } } - assert( (pMem->flags & (MEM_Int|MEM_Real|MEM_Null))!=0 ); + assert( (pMem->flags & (MEM_Int|MEM_Real|MEM_IntReal|MEM_Null))!=0 ); pMem->flags &= ~(MEM_Str|MEM_Blob|MEM_Zero); return SQLITE_OK; } @@ -74958,7 +75447,7 @@ SQLITE_PRIVATE void sqlite3VdbeMemCast(Mem *pMem, u8 aff, u8 encoding){ pMem->flags |= (pMem->flags&MEM_Blob)>>3; sqlite3ValueApplyAffinity(pMem, SQLITE_AFF_TEXT, encoding); assert( pMem->flags & MEM_Str || pMem->db->mallocFailed ); - pMem->flags &= ~(MEM_Int|MEM_Real|MEM_Blob|MEM_Zero); + pMem->flags &= ~(MEM_Int|MEM_Real|MEM_IntReal|MEM_Blob|MEM_Zero); break; } } @@ -75142,7 +75631,7 @@ SQLITE_PRIVATE void sqlite3VdbeMemAboutToChange(Vdbe *pVdbe, Mem *pMem){ ** dual type, are allowed, as long as the underlying value is the ** same. */ u16 mFlags = pMem->flags & pX->flags & pX->mScopyFlags; - assert( (mFlags&MEM_Int)==0 || pMem->u.i==pX->u.i ); + assert( (mFlags&(MEM_Int|MEM_IntReal))==0 || pMem->u.i==pX->u.i ); assert( (mFlags&MEM_Real)==0 || pMem->u.r==pX->u.r ); assert( (mFlags&MEM_Str)==0 || (pMem->n==pX->n && pMem->z==pX->z) ); assert( (mFlags&MEM_Blob)==0 || sqlite3BlobCompare(pMem,pX)==0 ); @@ -75264,7 +75753,6 @@ SQLITE_PRIVATE int sqlite3VdbeMemSetStr( assert( enc!=0 ); if( enc==SQLITE_UTF8 ){ nByte = 0x7fffffff & (int)strlen(z); - if( nByte>iLimit ) nByte = iLimit+1; }else{ for(nByte=0; nByte<=iLimit && (z[nByte] | z[nByte+1]); nByte+=2){} } @@ -75276,29 +75764,30 @@ SQLITE_PRIVATE int sqlite3VdbeMemSetStr( ** management (one of MEM_Dyn or MEM_Static). */ if( xDel==SQLITE_TRANSIENT ){ - int nAlloc = nByte; + u32 nAlloc = nByte; if( flags&MEM_Term ){ nAlloc += (enc==SQLITE_UTF8?1:2); } if( nByte>iLimit ){ - return SQLITE_TOOBIG; + return sqlite3ErrorToParser(pMem->db, SQLITE_TOOBIG); } testcase( nAlloc==0 ); testcase( nAlloc==31 ); testcase( nAlloc==32 ); - if( sqlite3VdbeMemClearAndResize(pMem, MAX(nAlloc,32)) ){ + if( sqlite3VdbeMemClearAndResize(pMem, (int)MAX(nAlloc,32)) ){ return SQLITE_NOMEM_BKPT; } memcpy(pMem->z, z, nAlloc); - }else if( xDel==SQLITE_DYNAMIC ){ - sqlite3VdbeMemRelease(pMem); - pMem->zMalloc = pMem->z = (char *)z; - pMem->szMalloc = sqlite3DbMallocSize(pMem->db, pMem->zMalloc); }else{ sqlite3VdbeMemRelease(pMem); pMem->z = (char *)z; - pMem->xDel = xDel; - flags |= ((xDel==SQLITE_STATIC)?MEM_Static:MEM_Dyn); + if( xDel==SQLITE_DYNAMIC ){ + pMem->zMalloc = pMem->z; + pMem->szMalloc = sqlite3DbMallocSize(pMem->db, pMem->zMalloc); + }else{ + pMem->xDel = xDel; + flags |= ((xDel==SQLITE_STATIC)?MEM_Static:MEM_Dyn); + } } pMem->n = nByte; @@ -75417,7 +75906,7 @@ static SQLITE_NOINLINE const void *valueToText(sqlite3_value* pVal, u8 enc){ assert(pVal->enc==(enc & ~SQLITE_UTF16_ALIGNED) || pVal->db==0 || pVal->db->mallocFailed ); if( pVal->enc==(enc & ~SQLITE_UTF16_ALIGNED) ){ - assert( sqlite3VdbeMemConsistentDualRep(pVal) ); + assert( sqlite3VdbeMemValidStrRep(pVal) ); return pVal->z; }else{ return 0; @@ -75440,7 +75929,7 @@ SQLITE_PRIVATE const void *sqlite3ValueText(sqlite3_value* pVal, u8 enc){ assert( (enc&3)==(enc&~SQLITE_UTF16_ALIGNED) ); assert( !sqlite3VdbeMemIsRowSet(pVal) ); if( (pVal->flags&(MEM_Str|MEM_Term))==(MEM_Str|MEM_Term) && pVal->enc==enc ){ - assert( sqlite3VdbeMemConsistentDualRep(pVal) ); + assert( sqlite3VdbeMemValidStrRep(pVal) ); return pVal->z; } if( pVal->flags&MEM_Null ){ @@ -75484,7 +75973,7 @@ struct ValueNewStat4Ctx { ** an sqlite3_value within the UnpackedRecord.a[] array. */ static sqlite3_value *valueNew(sqlite3 *db, struct ValueNewStat4Ctx *p){ -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 +#ifdef SQLITE_ENABLE_STAT4 if( p ){ UnpackedRecord *pRec = p->ppRec[0]; @@ -75520,7 +76009,7 @@ static sqlite3_value *valueNew(sqlite3 *db, struct ValueNewStat4Ctx *p){ } #else UNUSED_PARAMETER(p); -#endif /* defined(SQLITE_ENABLE_STAT3_OR_STAT4) */ +#endif /* defined(SQLITE_ENABLE_STAT4) */ return sqlite3ValueNew(db); } @@ -75544,7 +76033,7 @@ static sqlite3_value *valueNew(sqlite3 *db, struct ValueNewStat4Ctx *p){ ** and sets (*ppVal) to NULL. Or, if an error occurs, (*ppVal) is set to ** NULL and an SQLite error code returned. */ -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 +#ifdef SQLITE_ENABLE_STAT4 static int valueFromFunction( sqlite3 *db, /* The database connection */ Expr *p, /* The expression to evaluate */ @@ -75627,7 +76116,7 @@ static int valueFromFunction( } #else # define valueFromFunction(a,b,c,d,e,f) SQLITE_OK -#endif /* defined(SQLITE_ENABLE_STAT3_OR_STAT4) */ +#endif /* defined(SQLITE_ENABLE_STAT4) */ /* ** Extract a value from the supplied expression in the manner described @@ -75656,7 +76145,7 @@ static int valueFromExpr( assert( pExpr!=0 ); while( (op = pExpr->op)==TK_UPLUS || op==TK_SPAN ) pExpr = pExpr->pLeft; -#if defined(SQLITE_ENABLE_STAT3_OR_STAT4) +#if defined(SQLITE_ENABLE_STAT4) if( op==TK_REGISTER ) op = pExpr->op2; #else if( NEVER(op==TK_REGISTER) ) op = pExpr->op2; @@ -75705,7 +76194,12 @@ static int valueFromExpr( }else{ sqlite3ValueApplyAffinity(pVal, affinity, SQLITE_UTF8); } - if( pVal->flags & (MEM_Int|MEM_Real) ) pVal->flags &= ~MEM_Str; + assert( (pVal->flags & MEM_IntReal)==0 ); + if( pVal->flags & (MEM_Int|MEM_IntReal|MEM_Real) ){ + testcase( pVal->flags & MEM_Int ); + testcase( pVal->flags & MEM_Real ); + pVal->flags &= ~MEM_Str; + } if( enc!=SQLITE_UTF8 ){ rc = sqlite3VdbeChangeEncoding(pVal, enc); } @@ -75728,7 +76222,7 @@ static int valueFromExpr( }else if( op==TK_NULL ){ pVal = valueNew(db, pCtx); if( pVal==0 ) goto no_mem; - sqlite3VdbeMemNumerify(pVal); + sqlite3VdbeMemSetNull(pVal); } #ifndef SQLITE_OMIT_BLOB_LITERAL else if( op==TK_BLOB ){ @@ -75744,7 +76238,7 @@ static int valueFromExpr( 0, SQLITE_DYNAMIC); } #endif -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 +#ifdef SQLITE_ENABLE_STAT4 else if( op==TK_FUNCTION && pCtx!=0 ){ rc = valueFromFunction(db, pExpr, enc, affinity, &pVal, pCtx); } @@ -75761,13 +76255,13 @@ static int valueFromExpr( return rc; no_mem: -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 +#ifdef SQLITE_ENABLE_STAT4 if( pCtx==0 || pCtx->pParse->nErr==0 ) #endif sqlite3OomFault(db); sqlite3DbFree(db, zVal); assert( *ppVal==0 ); -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 +#ifdef SQLITE_ENABLE_STAT4 if( pCtx==0 ) sqlite3ValueFree(pVal); #else assert( pCtx==0 ); sqlite3ValueFree(pVal); @@ -75795,56 +76289,7 @@ SQLITE_PRIVATE int sqlite3ValueFromExpr( return pExpr ? valueFromExpr(db, pExpr, enc, affinity, ppVal, 0) : 0; } -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 -/* -** The implementation of the sqlite_record() function. This function accepts -** a single argument of any type. The return value is a formatted database -** record (a blob) containing the argument value. -** -** This is used to convert the value stored in the 'sample' column of the -** sqlite_stat3 table to the record format SQLite uses internally. -*/ -static void recordFunc( - sqlite3_context *context, - int argc, - sqlite3_value **argv -){ - const int file_format = 1; - u32 iSerial; /* Serial type */ - int nSerial; /* Bytes of space for iSerial as varint */ - u32 nVal; /* Bytes of space required for argv[0] */ - int nRet; - sqlite3 *db; - u8 *aRet; - - UNUSED_PARAMETER( argc ); - iSerial = sqlite3VdbeSerialType(argv[0], file_format, &nVal); - nSerial = sqlite3VarintLen(iSerial); - db = sqlite3_context_db_handle(context); - - nRet = 1 + nSerial + nVal; - aRet = sqlite3DbMallocRawNN(db, nRet); - if( aRet==0 ){ - sqlite3_result_error_nomem(context); - }else{ - aRet[0] = nSerial+1; - putVarint32(&aRet[1], iSerial); - sqlite3VdbeSerialPut(&aRet[1+nSerial], argv[0], iSerial); - sqlite3_result_blob(context, aRet, nRet, SQLITE_TRANSIENT); - sqlite3DbFreeNN(db, aRet); - } -} - -/* -** Register built-in functions used to help read ANALYZE data. -*/ -SQLITE_PRIVATE void sqlite3AnalyzeFunctions(void){ - static FuncDef aAnalyzeTableFuncs[] = { - FUNCTION(sqlite_record, 1, 0, 0, recordFunc), - }; - sqlite3InsertBuiltinFuncs(aAnalyzeTableFuncs, ArraySize(aAnalyzeTableFuncs)); -} - +#ifdef SQLITE_ENABLE_STAT4 /* ** Attempt to extract a value from pExpr and use it to construct *ppVal. ** @@ -76266,9 +76711,11 @@ static int growOpArray(Vdbe *v, int nOp){ ** operation (without SQLITE_TEST_REALLOC_STRESS) is to double the current ** size of the op array or add 1KB of space, whichever is smaller. */ #ifdef SQLITE_TEST_REALLOC_STRESS - int nNew = (v->nOpAlloc>=512 ? v->nOpAlloc*2 : v->nOpAlloc+nOp); + sqlite3_int64 nNew = (v->nOpAlloc>=512 ? 2*(sqlite3_int64)v->nOpAlloc + : (sqlite3_int64)v->nOpAlloc+nOp); #else - int nNew = (v->nOpAlloc ? v->nOpAlloc*2 : (int)(1024/sizeof(Op))); + sqlite3_int64 nNew = (v->nOpAlloc ? 2*(sqlite3_int64)v->nOpAlloc + : (sqlite3_int64)(1024/sizeof(Op))); UNUSED_PARAMETER(nOp); #endif @@ -76738,6 +77185,7 @@ SQLITE_PRIVATE int sqlite3VdbeAssertMayAbort(Vdbe *v, int mayAbort){ int hasAbort = 0; int hasFkCounter = 0; int hasCreateTable = 0; + int hasCreateIndex = 0; int hasInitCoroutine = 0; Op *pOp; VdbeOpIter sIter; @@ -76748,6 +77196,7 @@ SQLITE_PRIVATE int sqlite3VdbeAssertMayAbort(Vdbe *v, int mayAbort){ int opcode = pOp->opcode; if( opcode==OP_Destroy || opcode==OP_VUpdate || opcode==OP_VRename || opcode==OP_VDestroy + || (opcode==OP_ParseSchema && pOp->p4.z==0) || ((opcode==OP_Halt || opcode==OP_HaltIfNull) && ((pOp->p1)!=SQLITE_OK && pOp->p2==OE_Abort)) ){ @@ -76755,6 +77204,14 @@ SQLITE_PRIVATE int sqlite3VdbeAssertMayAbort(Vdbe *v, int mayAbort){ break; } if( opcode==OP_CreateBtree && pOp->p3==BTREE_INTKEY ) hasCreateTable = 1; + if( mayAbort ){ + /* hasCreateIndex may also be set for some DELETE statements that use + ** OP_Clear. So this routine may end up returning true in the case + ** where a "DELETE FROM tbl" has a statement-journal but does not + ** require one. This is not so bad - it is an inefficiency, not a bug. */ + if( opcode==OP_CreateBtree && pOp->p3==BTREE_BLOBKEY ) hasCreateIndex = 1; + if( opcode==OP_Clear ) hasCreateIndex = 1; + } if( opcode==OP_InitCoroutine ) hasInitCoroutine = 1; #ifndef SQLITE_OMIT_FOREIGN_KEY if( opcode==OP_FkCounter && pOp->p1==0 && pOp->p2==1 ){ @@ -76770,7 +77227,8 @@ SQLITE_PRIVATE int sqlite3VdbeAssertMayAbort(Vdbe *v, int mayAbort){ ** true for this case to prevent the assert() in the callers frame ** from failing. */ return ( v->db->mallocFailed || hasAbort==mayAbort || hasFkCounter - || (hasCreateTable && hasInitCoroutine) ); + || (hasCreateTable && hasInitCoroutine) || hasCreateIndex + ); } #endif /* SQLITE_DEBUG - the sqlite3AssertMayAbort() function */ @@ -77055,7 +77513,7 @@ SQLITE_PRIVATE void sqlite3VdbeScanStatus( LogEst nEst, /* Estimated number of output rows */ const char *zName /* Name of table or index being scanned */ ){ - int nByte = (p->nScan+1) * sizeof(ScanStatus); + sqlite3_int64 nByte = (p->nScan+1) * sizeof(ScanStatus); ScanStatus *aNew; aNew = (ScanStatus*)sqlite3DbRealloc(p->db, p->aScan, nByte); if( aNew ){ @@ -77075,16 +77533,16 @@ SQLITE_PRIVATE void sqlite3VdbeScanStatus( ** Change the value of the opcode, or P1, P2, P3, or P5 operands ** for a specific instruction. */ -SQLITE_PRIVATE void sqlite3VdbeChangeOpcode(Vdbe *p, u32 addr, u8 iNewOpcode){ +SQLITE_PRIVATE void sqlite3VdbeChangeOpcode(Vdbe *p, int addr, u8 iNewOpcode){ sqlite3VdbeGetOp(p,addr)->opcode = iNewOpcode; } -SQLITE_PRIVATE void sqlite3VdbeChangeP1(Vdbe *p, u32 addr, int val){ +SQLITE_PRIVATE void sqlite3VdbeChangeP1(Vdbe *p, int addr, int val){ sqlite3VdbeGetOp(p,addr)->p1 = val; } -SQLITE_PRIVATE void sqlite3VdbeChangeP2(Vdbe *p, u32 addr, int val){ +SQLITE_PRIVATE void sqlite3VdbeChangeP2(Vdbe *p, int addr, int val){ sqlite3VdbeGetOp(p,addr)->p2 = val; } -SQLITE_PRIVATE void sqlite3VdbeChangeP3(Vdbe *p, u32 addr, int val){ +SQLITE_PRIVATE void sqlite3VdbeChangeP3(Vdbe *p, int addr, int val){ sqlite3VdbeGetOp(p,addr)->p3 = val; } SQLITE_PRIVATE void sqlite3VdbeChangeP5(Vdbe *p, u16 p5){ @@ -77591,14 +78049,16 @@ static char *displayP4(Op *pOp, char *zTemp, int nTemp){ case P4_KEYINFO: { int j; KeyInfo *pKeyInfo = pOp->p4.pKeyInfo; - assert( pKeyInfo->aSortOrder!=0 ); + assert( pKeyInfo->aSortFlags!=0 ); sqlite3_str_appendf(&x, "k(%d", pKeyInfo->nKeyField); for(j=0; jnKeyField; j++){ CollSeq *pColl = pKeyInfo->aColl[j]; const char *zColl = pColl ? pColl->zName : ""; if( strcmp(zColl, "BINARY")==0 ) zColl = "B"; - sqlite3_str_appendf(&x, ",%s%s", - pKeyInfo->aSortOrder[j] ? "-" : "", zColl); + sqlite3_str_appendf(&x, ",%s%s%s", + (pKeyInfo->aSortFlags[j] & KEYINFO_ORDER_DESC) ? "-" : "", + (pKeyInfo->aSortFlags[j] & KEYINFO_ORDER_BIGNULL)? "N." : "", + zColl); } sqlite3_str_append(&x, ")", 1); break; @@ -77642,7 +78102,7 @@ static char *displayP4(Op *pOp, char *zTemp, int nTemp){ Mem *pMem = pOp->p4.pMem; if( pMem->flags & MEM_Str ){ zP4 = pMem->z; - }else if( pMem->flags & MEM_Int ){ + }else if( pMem->flags & (MEM_Int|MEM_IntReal) ){ sqlite3_str_appendf(&x, "%lld", pMem->u.i); }else if( pMem->flags & MEM_Real ){ sqlite3_str_appendf(&x, "%.16g", pMem->u.r); @@ -78005,8 +78465,11 @@ SQLITE_PRIVATE int sqlite3VdbeList( ** pick up the appropriate opcode. */ int j; i -= p->nOp; + assert( apSub!=0 ); + assert( nSub>0 ); for(j=0; i>=apSub[j]->nOp; j++){ i -= apSub[j]->nOp; + assert( inOp || j+1aOp[i]; } @@ -78176,9 +78639,9 @@ SQLITE_PRIVATE void sqlite3VdbeIOTraceSql(Vdbe *p){ ** of a ReusableSpace object by the allocSpace() routine below. */ struct ReusableSpace { - u8 *pSpace; /* Available memory */ - int nFree; /* Bytes of available memory */ - int nNeeded; /* Total bytes that could not be allocated */ + u8 *pSpace; /* Available memory */ + sqlite3_int64 nFree; /* Bytes of available memory */ + sqlite3_int64 nNeeded; /* Total bytes that could not be allocated */ }; /* Try to allocate nByte bytes of 8-byte aligned bulk memory for pBuf @@ -78198,7 +78661,7 @@ struct ReusableSpace { static void *allocSpace( struct ReusableSpace *p, /* Bulk memory available for allocation */ void *pBuf, /* Pointer to a prior allocation */ - int nByte /* Bytes of memory needed */ + sqlite3_int64 nByte /* Bytes of memory needed */ ){ assert( EIGHT_BYTE_ALIGNMENT(p->pSpace) ); if( pBuf==0 ){ @@ -79004,7 +79467,7 @@ SQLITE_PRIVATE int sqlite3VdbeHalt(Vdbe *p){ } /* Check for immediate foreign key violations. */ - if( p->rc==SQLITE_OK ){ + if( p->rc==SQLITE_OK || (p->errorAction==OE_Fail && !isSpecialError) ){ sqlite3VdbeCheckFk(p, 0); } @@ -79528,8 +79991,17 @@ SQLITE_PRIVATE int sqlite3VdbeCursorMoveto(VdbeCursor **pp, int *piCol){ ** of SQLite will not understand those serial types. */ +#if 0 /* Inlined into the OP_MakeRecord opcode */ /* ** Return the serial-type for the value stored in pMem. +** +** This routine might convert a large MEM_IntReal value into MEM_Real. +** +** 2019-07-11: The primary user of this subroutine was the OP_MakeRecord +** opcode in the byte-code engine. But by moving this routine in-line, we +** can omit some redundant tests and make that opcode a lot faster. So +** this routine is now only used by the STAT3 logic and STAT3 support has +** ended. The code is kept here for historical reference only. */ SQLITE_PRIVATE u32 sqlite3VdbeSerialType(Mem *pMem, int file_format, u32 *pLen){ int flags = pMem->flags; @@ -79540,11 +80012,13 @@ SQLITE_PRIVATE u32 sqlite3VdbeSerialType(Mem *pMem, int file_format, u32 *pLen){ *pLen = 0; return 0; } - if( flags&MEM_Int ){ + if( flags&(MEM_Int|MEM_IntReal) ){ /* Figure out whether to use 1, 2, 4, 6 or 8 bytes. */ # define MAX_6BYTE ((((i64)0x00008000)<<32)-1) i64 i = pMem->u.i; u64 u; + testcase( flags & MEM_Int ); + testcase( flags & MEM_IntReal ); if( i<0 ){ u = ~i; }else{ @@ -79564,6 +80038,15 @@ SQLITE_PRIVATE u32 sqlite3VdbeSerialType(Mem *pMem, int file_format, u32 *pLen){ if( u<=2147483647 ){ *pLen = 4; return 4; } if( u<=MAX_6BYTE ){ *pLen = 6; return 5; } *pLen = 8; + if( flags&MEM_IntReal ){ + /* If the value is IntReal and is going to take up 8 bytes to store + ** as an integer, then we might as well make it an 8-byte floating + ** point value */ + pMem->u.r = (double)pMem->u.i; + pMem->flags &= ~MEM_IntReal; + pMem->flags |= MEM_Real; + return 7; + } return 6; } if( flags&MEM_Real ){ @@ -79579,6 +80062,7 @@ SQLITE_PRIVATE u32 sqlite3VdbeSerialType(Mem *pMem, int file_format, u32 *pLen){ *pLen = n; return ((n*2) + 12 + ((flags&MEM_Str)!=0)); } +#endif /* inlined into OP_MakeRecord */ /* ** The sizes for serial types less than 128 @@ -79737,7 +80221,7 @@ SQLITE_PRIVATE u32 sqlite3VdbeSerialPut(u8 *buf, Mem *pMem, u32 serial_type){ ** routine so that in most cases the overhead of moving the stack pointer ** is avoided. */ -static u32 SQLITE_NOINLINE serialGet( +static u32 serialGet( const unsigned char *buf, /* Buffer to deserialize from */ u32 serial_type, /* Serial type to deserialize */ Mem *pMem /* Memory cell to write value into */ @@ -79769,7 +80253,7 @@ static u32 SQLITE_NOINLINE serialGet( assert( sizeof(x)==8 && sizeof(pMem->u.r)==8 ); swapMixedEndianFloat(x); memcpy(&pMem->u.r, &x, sizeof(x)); - pMem->flags = sqlite3IsNaN(pMem->u.r) ? MEM_Null : MEM_Real; + pMem->flags = IsNaN(x) ? MEM_Null : MEM_Real; } return 8; } @@ -79887,7 +80371,7 @@ SQLITE_PRIVATE UnpackedRecord *sqlite3VdbeAllocUnpackedRecord( p = (UnpackedRecord *)sqlite3DbMallocRaw(pKeyInfo->db, nByte); if( !p ) return 0; p->aMem = (Mem*)&((char*)p)[ROUND8(sizeof(UnpackedRecord))]; - assert( pKeyInfo->aSortOrder!=0 ); + assert( pKeyInfo->aSortFlags!=0 ); p->pKeyInfo = pKeyInfo; p->nField = pKeyInfo->nKeyField + 1; return p; @@ -79986,7 +80470,7 @@ static int vdbeRecordCompareDebug( if( szHdr1>98307 ) return SQLITE_CORRUPT; d1 = szHdr1; assert( pKeyInfo->nAllField>=pPKey2->nField || CORRUPT_DB ); - assert( pKeyInfo->aSortOrder!=0 ); + assert( pKeyInfo->aSortFlags!=0 ); assert( pKeyInfo->nKeyField>0 ); assert( idx1<=szHdr1 || CORRUPT_DB ); do{ @@ -80017,7 +80501,12 @@ static int vdbeRecordCompareDebug( pKeyInfo->nAllField>i ? pKeyInfo->aColl[i] : 0); if( rc!=0 ){ assert( mem1.szMalloc==0 ); /* See comment below */ - if( pKeyInfo->aSortOrder[i] ){ + if( (pKeyInfo->aSortFlags[i] & KEYINFO_ORDER_BIGNULL) + && ((mem1.flags & MEM_Null) || (pPKey2->aMem[i].flags & MEM_Null)) + ){ + rc = -rc; + } + if( pKeyInfo->aSortFlags[i] & KEYINFO_ORDER_DESC ){ rc = -rc; /* Invert the result for DESC sort order. */ } goto debugCompareEnd; @@ -80219,8 +80708,13 @@ SQLITE_PRIVATE int sqlite3MemCompare(const Mem *pMem1, const Mem *pMem2, const C /* At least one of the two values is a number */ - if( combined_flags&(MEM_Int|MEM_Real) ){ - if( (f1 & f2 & MEM_Int)!=0 ){ + if( combined_flags&(MEM_Int|MEM_Real|MEM_IntReal) ){ + testcase( combined_flags & MEM_Int ); + testcase( combined_flags & MEM_Real ); + testcase( combined_flags & MEM_IntReal ); + if( (f1 & f2 & (MEM_Int|MEM_IntReal))!=0 ){ + testcase( f1 & f2 & MEM_Int ); + testcase( f1 & f2 & MEM_IntReal ); if( pMem1->u.i < pMem2->u.i ) return -1; if( pMem1->u.i > pMem2->u.i ) return +1; return 0; @@ -80230,15 +80724,23 @@ SQLITE_PRIVATE int sqlite3MemCompare(const Mem *pMem1, const Mem *pMem2, const C if( pMem1->u.r > pMem2->u.r ) return +1; return 0; } - if( (f1&MEM_Int)!=0 ){ + if( (f1&(MEM_Int|MEM_IntReal))!=0 ){ + testcase( f1 & MEM_Int ); + testcase( f1 & MEM_IntReal ); if( (f2&MEM_Real)!=0 ){ return sqlite3IntFloatCompare(pMem1->u.i, pMem2->u.r); + }else if( (f2&(MEM_Int|MEM_IntReal))!=0 ){ + if( pMem1->u.i < pMem2->u.i ) return -1; + if( pMem1->u.i > pMem2->u.i ) return +1; + return 0; }else{ return -1; } } if( (f1&MEM_Real)!=0 ){ - if( (f2&MEM_Int)!=0 ){ + if( (f2&(MEM_Int|MEM_IntReal))!=0 ){ + testcase( f2 & MEM_Int ); + testcase( f2 & MEM_IntReal ); return -sqlite3IntFloatCompare(pMem2->u.i, pMem1->u.r); }else{ return -1; @@ -80380,14 +80882,16 @@ SQLITE_PRIVATE int sqlite3VdbeRecordCompareWithSkip( VVA_ONLY( mem1.szMalloc = 0; ) /* Only needed by assert() statements */ assert( pPKey2->pKeyInfo->nAllField>=pPKey2->nField || CORRUPT_DB ); - assert( pPKey2->pKeyInfo->aSortOrder!=0 ); + assert( pPKey2->pKeyInfo->aSortFlags!=0 ); assert( pPKey2->pKeyInfo->nKeyField>0 ); assert( idx1<=szHdr1 || CORRUPT_DB ); do{ u32 serial_type; /* RHS is an integer */ - if( pRhs->flags & MEM_Int ){ + if( pRhs->flags & (MEM_Int|MEM_IntReal) ){ + testcase( pRhs->flags & MEM_Int ); + testcase( pRhs->flags & MEM_IntReal ); serial_type = aKey1[idx1]; testcase( serial_type==12 ); if( serial_type>=10 ){ @@ -80501,8 +81005,14 @@ SQLITE_PRIVATE int sqlite3VdbeRecordCompareWithSkip( } if( rc!=0 ){ - if( pPKey2->pKeyInfo->aSortOrder[i] ){ - rc = -rc; + int sortFlags = pPKey2->pKeyInfo->aSortFlags[i]; + if( sortFlags ){ + if( (sortFlags & KEYINFO_ORDER_BIGNULL)==0 + || ((sortFlags & KEYINFO_ORDER_DESC) + !=(serial_type==0 || (pRhs->flags&MEM_Null))) + ){ + rc = -rc; + } } assert( vdbeRecordCompareDebug(nKey1, pKey1, pPKey2, rc) ); assert( mem1.szMalloc==0 ); /* See comment below */ @@ -80670,7 +81180,11 @@ static int vdbeRecordCompareString( nCmp = MIN( pPKey2->aMem[0].n, nStr ); res = memcmp(&aKey1[szHdr], pPKey2->aMem[0].z, nCmp); - if( res==0 ){ + if( res>0 ){ + res = pPKey2->r2; + }else if( res<0 ){ + res = pPKey2->r1; + }else{ res = nStr - pPKey2->aMem[0].n; if( res==0 ){ if( pPKey2->nField>1 ){ @@ -80684,10 +81198,6 @@ static int vdbeRecordCompareString( }else{ res = pPKey2->r1; } - }else if( res>0 ){ - res = pPKey2->r2; - }else{ - res = pPKey2->r1; } } @@ -80719,7 +81229,10 @@ SQLITE_PRIVATE RecordCompare sqlite3VdbeFindCompare(UnpackedRecord *p){ ** header size is (12*5 + 1 + 1) bytes. */ if( p->pKeyInfo->nAllField<=13 ){ int flags = p->aMem[0].flags; - if( p->pKeyInfo->aSortOrder[0] ){ + if( p->pKeyInfo->aSortFlags[0] ){ + if( p->pKeyInfo->aSortFlags[0] & KEYINFO_ORDER_BIGNULL ){ + return sqlite3VdbeRecordCompare; + } p->r1 = 1; p->r2 = -1; }else{ @@ -80732,7 +81245,9 @@ SQLITE_PRIVATE RecordCompare sqlite3VdbeFindCompare(UnpackedRecord *p){ testcase( flags & MEM_Real ); testcase( flags & MEM_Null ); testcase( flags & MEM_Blob ); - if( (flags & (MEM_Real|MEM_Null|MEM_Blob))==0 && p->pKeyInfo->aColl[0]==0 ){ + if( (flags & (MEM_Real|MEM_IntReal|MEM_Null|MEM_Blob))==0 + && p->pKeyInfo->aColl[0]==0 + ){ assert( flags & MEM_Str ); return vdbeRecordCompareString; } @@ -80966,7 +81481,7 @@ SQLITE_PRIVATE void sqlite3VdbeSetVarmask(Vdbe *v, int iVar){ ** features such as 'now'. */ SQLITE_PRIVATE int sqlite3NotPureFunc(sqlite3_context *pCtx){ -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 +#ifdef SQLITE_ENABLE_STAT4 if( pCtx->pVdbe==0 ) return 1; #endif if( pCtx->pVdbe->aOp[pCtx->iOp].opcode==OP_PureFunc ){ @@ -81063,7 +81578,7 @@ SQLITE_PRIVATE void sqlite3VdbePreUpdateHook( preupdate.keyinfo.db = db; preupdate.keyinfo.enc = ENC(db); preupdate.keyinfo.nKeyField = pTab->nCol; - preupdate.keyinfo.aSortOrder = (u8*)&fakeSortOrder; + preupdate.keyinfo.aSortFlags = (u8*)&fakeSortOrder; preupdate.iKey1 = iKey1; preupdate.iKey2 = iKey2; preupdate.pTab = pTab; @@ -81155,7 +81670,7 @@ static SQLITE_NOINLINE void invokeProfileCallback(sqlite3 *db, Vdbe *p){ assert( p->zSql!=0 ); sqlite3OsCurrentTimeInt64(db->pVfs, &iNow); iElapse = (iNow - p->startTime)*1000000; -#ifndef SQLITE_OMIT_DEPRECATED +#ifndef SQLITE_OMIT_DEPRECATED if( db->xProfile ){ db->xProfile(db->pProfileArg, p->zSql, iElapse); } @@ -81322,39 +81837,86 @@ SQLITE_API const void *sqlite3_value_text16le(sqlite3_value *pVal){ */ SQLITE_API int sqlite3_value_type(sqlite3_value* pVal){ static const u8 aType[] = { - SQLITE_BLOB, /* 0x00 */ - SQLITE_NULL, /* 0x01 */ - SQLITE_TEXT, /* 0x02 */ - SQLITE_NULL, /* 0x03 */ - SQLITE_INTEGER, /* 0x04 */ - SQLITE_NULL, /* 0x05 */ - SQLITE_INTEGER, /* 0x06 */ - SQLITE_NULL, /* 0x07 */ - SQLITE_FLOAT, /* 0x08 */ - SQLITE_NULL, /* 0x09 */ - SQLITE_FLOAT, /* 0x0a */ - SQLITE_NULL, /* 0x0b */ - SQLITE_INTEGER, /* 0x0c */ - SQLITE_NULL, /* 0x0d */ - SQLITE_INTEGER, /* 0x0e */ - SQLITE_NULL, /* 0x0f */ - SQLITE_BLOB, /* 0x10 */ - SQLITE_NULL, /* 0x11 */ - SQLITE_TEXT, /* 0x12 */ - SQLITE_NULL, /* 0x13 */ - SQLITE_INTEGER, /* 0x14 */ - SQLITE_NULL, /* 0x15 */ - SQLITE_INTEGER, /* 0x16 */ - SQLITE_NULL, /* 0x17 */ - SQLITE_FLOAT, /* 0x18 */ - SQLITE_NULL, /* 0x19 */ - SQLITE_FLOAT, /* 0x1a */ - SQLITE_NULL, /* 0x1b */ - SQLITE_INTEGER, /* 0x1c */ - SQLITE_NULL, /* 0x1d */ - SQLITE_INTEGER, /* 0x1e */ - SQLITE_NULL, /* 0x1f */ + SQLITE_BLOB, /* 0x00 (not possible) */ + SQLITE_NULL, /* 0x01 NULL */ + SQLITE_TEXT, /* 0x02 TEXT */ + SQLITE_NULL, /* 0x03 (not possible) */ + SQLITE_INTEGER, /* 0x04 INTEGER */ + SQLITE_NULL, /* 0x05 (not possible) */ + SQLITE_INTEGER, /* 0x06 INTEGER + TEXT */ + SQLITE_NULL, /* 0x07 (not possible) */ + SQLITE_FLOAT, /* 0x08 FLOAT */ + SQLITE_NULL, /* 0x09 (not possible) */ + SQLITE_FLOAT, /* 0x0a FLOAT + TEXT */ + SQLITE_NULL, /* 0x0b (not possible) */ + SQLITE_INTEGER, /* 0x0c (not possible) */ + SQLITE_NULL, /* 0x0d (not possible) */ + SQLITE_INTEGER, /* 0x0e (not possible) */ + SQLITE_NULL, /* 0x0f (not possible) */ + SQLITE_BLOB, /* 0x10 BLOB */ + SQLITE_NULL, /* 0x11 (not possible) */ + SQLITE_TEXT, /* 0x12 (not possible) */ + SQLITE_NULL, /* 0x13 (not possible) */ + SQLITE_INTEGER, /* 0x14 INTEGER + BLOB */ + SQLITE_NULL, /* 0x15 (not possible) */ + SQLITE_INTEGER, /* 0x16 (not possible) */ + SQLITE_NULL, /* 0x17 (not possible) */ + SQLITE_FLOAT, /* 0x18 FLOAT + BLOB */ + SQLITE_NULL, /* 0x19 (not possible) */ + SQLITE_FLOAT, /* 0x1a (not possible) */ + SQLITE_NULL, /* 0x1b (not possible) */ + SQLITE_INTEGER, /* 0x1c (not possible) */ + SQLITE_NULL, /* 0x1d (not possible) */ + SQLITE_INTEGER, /* 0x1e (not possible) */ + SQLITE_NULL, /* 0x1f (not possible) */ + SQLITE_FLOAT, /* 0x20 INTREAL */ + SQLITE_NULL, /* 0x21 (not possible) */ + SQLITE_TEXT, /* 0x22 INTREAL + TEXT */ + SQLITE_NULL, /* 0x23 (not possible) */ + SQLITE_FLOAT, /* 0x24 (not possible) */ + SQLITE_NULL, /* 0x25 (not possible) */ + SQLITE_FLOAT, /* 0x26 (not possible) */ + SQLITE_NULL, /* 0x27 (not possible) */ + SQLITE_FLOAT, /* 0x28 (not possible) */ + SQLITE_NULL, /* 0x29 (not possible) */ + SQLITE_FLOAT, /* 0x2a (not possible) */ + SQLITE_NULL, /* 0x2b (not possible) */ + SQLITE_FLOAT, /* 0x2c (not possible) */ + SQLITE_NULL, /* 0x2d (not possible) */ + SQLITE_FLOAT, /* 0x2e (not possible) */ + SQLITE_NULL, /* 0x2f (not possible) */ + SQLITE_BLOB, /* 0x30 (not possible) */ + SQLITE_NULL, /* 0x31 (not possible) */ + SQLITE_TEXT, /* 0x32 (not possible) */ + SQLITE_NULL, /* 0x33 (not possible) */ + SQLITE_FLOAT, /* 0x34 (not possible) */ + SQLITE_NULL, /* 0x35 (not possible) */ + SQLITE_FLOAT, /* 0x36 (not possible) */ + SQLITE_NULL, /* 0x37 (not possible) */ + SQLITE_FLOAT, /* 0x38 (not possible) */ + SQLITE_NULL, /* 0x39 (not possible) */ + SQLITE_FLOAT, /* 0x3a (not possible) */ + SQLITE_NULL, /* 0x3b (not possible) */ + SQLITE_FLOAT, /* 0x3c (not possible) */ + SQLITE_NULL, /* 0x3d (not possible) */ + SQLITE_FLOAT, /* 0x3e (not possible) */ + SQLITE_NULL, /* 0x3f (not possible) */ }; +#ifdef SQLITE_DEBUG + { + int eType = SQLITE_BLOB; + if( pVal->flags & MEM_Null ){ + eType = SQLITE_NULL; + }else if( pVal->flags & (MEM_Real|MEM_IntReal) ){ + eType = SQLITE_FLOAT; + }else if( pVal->flags & MEM_Int ){ + eType = SQLITE_INTEGER; + }else if( pVal->flags & MEM_Str ){ + eType = SQLITE_TEXT; + } + assert( eType == aType[pVal->flags&MEM_AffMask] ); + } +#endif return aType[pVal->flags&MEM_AffMask]; } @@ -81363,6 +81925,11 @@ SQLITE_API int sqlite3_value_nochange(sqlite3_value *pVal){ return (pVal->flags&(MEM_Null|MEM_Zero))==(MEM_Null|MEM_Zero); } +/* Return true if a parameter value originated from an sqlite3_bind() */ +SQLITE_API int sqlite3_value_frombind(sqlite3_value *pVal){ + return (pVal->flags&MEM_FromBind)!=0; +} + /* Make a copy of an sqlite3_value object */ SQLITE_API sqlite3_value *sqlite3_value_dup(const sqlite3_value *pOrig){ @@ -81599,6 +82166,21 @@ SQLITE_API void sqlite3_result_error_nomem(sqlite3_context *pCtx){ sqlite3OomFault(pCtx->pOut->db); } +#ifndef SQLITE_UNTESTABLE +/* Force the INT64 value currently stored as the result to be +** a MEM_IntReal value. See the SQLITE_TESTCTRL_RESULT_INTREAL +** test-control. +*/ +SQLITE_PRIVATE void sqlite3ResultIntReal(sqlite3_context *pCtx){ + assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); + if( pCtx->pOut->flags & MEM_Int ){ + pCtx->pOut->flags &= ~MEM_Int; + pCtx->pOut->flags |= MEM_IntReal; + } +} +#endif + + /* ** This function is called after a transaction has been committed. It ** invokes callbacks registered with sqlite3_wal_hook() as required. @@ -81865,7 +82447,7 @@ SQLITE_API int sqlite3_vtab_nochange(sqlite3_context *p){ */ SQLITE_PRIVATE sqlite3_int64 sqlite3StmtCurrentTime(sqlite3_context *p){ int rc; -#ifndef SQLITE_ENABLE_STAT3_OR_STAT4 +#ifndef SQLITE_ENABLE_STAT4 sqlite3_int64 *piTime = &p->pVdbe->iCurrentTime; assert( p->pVdbe!=0 ); #else @@ -81930,7 +82512,7 @@ SQLITE_API void *sqlite3_get_auxdata(sqlite3_context *pCtx, int iArg){ AuxData *pAuxData; assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); -#if SQLITE_ENABLE_STAT3_OR_STAT4 +#if SQLITE_ENABLE_STAT4 if( pCtx->pVdbe==0 ) return 0; #else assert( pCtx->pVdbe!=0 ); @@ -81964,7 +82546,7 @@ SQLITE_API void sqlite3_set_auxdata( Vdbe *pVdbe = pCtx->pVdbe; assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 +#ifdef SQLITE_ENABLE_STAT4 if( pVdbe==0 ) goto failed; #else assert( pVdbe!=0 ); @@ -82208,10 +82790,10 @@ SQLITE_API int sqlite3_column_type(sqlite3_stmt *pStmt, int i){ ** or a constant) then useTypes 2, 3, and 4 return NULL. */ static const void *columnName( - sqlite3_stmt *pStmt, - int N, - const void *(*xFunc)(Mem*), - int useType + sqlite3_stmt *pStmt, /* The statement */ + int N, /* Which column to get the name for */ + int useUtf16, /* True to return the name as UTF16 */ + int useType /* What type of name */ ){ const void *ret; Vdbe *p; @@ -82232,8 +82814,15 @@ static const void *columnName( N += useType*n; sqlite3_mutex_enter(db->mutex); assert( db->mallocFailed==0 ); - ret = xFunc(&p->aColName[N]); - /* A malloc may have failed inside of the xFunc() call. If this +#ifndef SQLITE_OMIT_UTF16 + if( useUtf16 ){ + ret = sqlite3_value_text16((sqlite3_value*)&p->aColName[N]); + }else +#endif + { + ret = sqlite3_value_text((sqlite3_value*)&p->aColName[N]); + } + /* A malloc may have failed inside of the _text() call. If this ** is the case, clear the mallocFailed flag and return NULL. */ if( db->mallocFailed ){ @@ -82250,13 +82839,11 @@ static const void *columnName( ** statement pStmt. */ SQLITE_API const char *sqlite3_column_name(sqlite3_stmt *pStmt, int N){ - return columnName( - pStmt, N, (const void*(*)(Mem*))sqlite3_value_text, COLNAME_NAME); + return columnName(pStmt, N, 0, COLNAME_NAME); } #ifndef SQLITE_OMIT_UTF16 SQLITE_API const void *sqlite3_column_name16(sqlite3_stmt *pStmt, int N){ - return columnName( - pStmt, N, (const void*(*)(Mem*))sqlite3_value_text16, COLNAME_NAME); + return columnName(pStmt, N, 1, COLNAME_NAME); } #endif @@ -82275,13 +82862,11 @@ SQLITE_API const void *sqlite3_column_name16(sqlite3_stmt *pStmt, int N){ ** of the result set of SQL statement pStmt. */ SQLITE_API const char *sqlite3_column_decltype(sqlite3_stmt *pStmt, int N){ - return columnName( - pStmt, N, (const void*(*)(Mem*))sqlite3_value_text, COLNAME_DECLTYPE); + return columnName(pStmt, N, 0, COLNAME_DECLTYPE); } #ifndef SQLITE_OMIT_UTF16 SQLITE_API const void *sqlite3_column_decltype16(sqlite3_stmt *pStmt, int N){ - return columnName( - pStmt, N, (const void*(*)(Mem*))sqlite3_value_text16, COLNAME_DECLTYPE); + return columnName(pStmt, N, 1, COLNAME_DECLTYPE); } #endif /* SQLITE_OMIT_UTF16 */ #endif /* SQLITE_OMIT_DECLTYPE */ @@ -82293,13 +82878,11 @@ SQLITE_API const void *sqlite3_column_decltype16(sqlite3_stmt *pStmt, int N){ ** anything else which is not an unambiguous reference to a database column. */ SQLITE_API const char *sqlite3_column_database_name(sqlite3_stmt *pStmt, int N){ - return columnName( - pStmt, N, (const void*(*)(Mem*))sqlite3_value_text, COLNAME_DATABASE); + return columnName(pStmt, N, 0, COLNAME_DATABASE); } #ifndef SQLITE_OMIT_UTF16 SQLITE_API const void *sqlite3_column_database_name16(sqlite3_stmt *pStmt, int N){ - return columnName( - pStmt, N, (const void*(*)(Mem*))sqlite3_value_text16, COLNAME_DATABASE); + return columnName(pStmt, N, 1, COLNAME_DATABASE); } #endif /* SQLITE_OMIT_UTF16 */ @@ -82309,13 +82892,11 @@ SQLITE_API const void *sqlite3_column_database_name16(sqlite3_stmt *pStmt, int N ** anything else which is not an unambiguous reference to a database column. */ SQLITE_API const char *sqlite3_column_table_name(sqlite3_stmt *pStmt, int N){ - return columnName( - pStmt, N, (const void*(*)(Mem*))sqlite3_value_text, COLNAME_TABLE); + return columnName(pStmt, N, 0, COLNAME_TABLE); } #ifndef SQLITE_OMIT_UTF16 SQLITE_API const void *sqlite3_column_table_name16(sqlite3_stmt *pStmt, int N){ - return columnName( - pStmt, N, (const void*(*)(Mem*))sqlite3_value_text16, COLNAME_TABLE); + return columnName(pStmt, N, 1, COLNAME_TABLE); } #endif /* SQLITE_OMIT_UTF16 */ @@ -82325,13 +82906,11 @@ SQLITE_API const void *sqlite3_column_table_name16(sqlite3_stmt *pStmt, int N){ ** anything else which is not an unambiguous reference to a database column. */ SQLITE_API const char *sqlite3_column_origin_name(sqlite3_stmt *pStmt, int N){ - return columnName( - pStmt, N, (const void*(*)(Mem*))sqlite3_value_text, COLNAME_COLUMN); + return columnName(pStmt, N, 0, COLNAME_COLUMN); } #ifndef SQLITE_OMIT_UTF16 SQLITE_API const void *sqlite3_column_origin_name16(sqlite3_stmt *pStmt, int N){ - return columnName( - pStmt, N, (const void*(*)(Mem*))sqlite3_value_text16, COLNAME_COLUMN); + return columnName(pStmt, N, 1, COLNAME_COLUMN); } #endif /* SQLITE_OMIT_UTF16 */ #endif /* SQLITE_ENABLE_COLUMN_METADATA */ @@ -82699,6 +83278,14 @@ SQLITE_API int sqlite3_stmt_readonly(sqlite3_stmt *pStmt){ return pStmt ? ((Vdbe*)pStmt)->readOnly : 1; } +/* +** Return 1 if the statement is an EXPLAIN and return 2 if the +** statement is an EXPLAIN QUERY PLAN +*/ +SQLITE_API int sqlite3_stmt_isexplain(sqlite3_stmt *pStmt){ + return pStmt ? ((Vdbe*)pStmt)->explain : 0; +} + /* ** Return true if the prepared statement is in need of being reset. */ @@ -82880,7 +83467,9 @@ SQLITE_API int sqlite3_preupdate_old(sqlite3 *db, int iIdx, sqlite3_value **ppVa }else if( iIdx>=p->pUnpacked->nField ){ *ppValue = (sqlite3_value *)columnNullValue(); }else if( p->pTab->aCol[iIdx].affinity==SQLITE_AFF_REAL ){ - if( pMem->flags & MEM_Int ){ + if( pMem->flags & (MEM_Int|MEM_IntReal) ){ + testcase( pMem->flags & MEM_Int ); + testcase( pMem->flags & MEM_IntReal ); sqlite3VdbeMemRealify(pMem); } } @@ -83199,7 +83788,7 @@ SQLITE_PRIVATE char *sqlite3VdbeExpandSql( pVar = &p->aVar[idx-1]; if( pVar->flags & MEM_Null ){ sqlite3_str_append(&out, "NULL", 4); - }else if( pVar->flags & MEM_Int ){ + }else if( pVar->flags & (MEM_Int|MEM_IntReal) ){ sqlite3_str_appendf(&out, "%lld", pVar->u.i); }else if( pVar->flags & MEM_Real ){ sqlite3_str_appendf(&out, "%!.15g", pVar->u.r); @@ -83388,12 +83977,20 @@ SQLITE_API int sqlite3_found_count = 0; ** feature is used for test suite validation only and does not appear an ** production builds. ** -** M is an integer between 2 and 4. 2 indicates a ordinary two-way -** branch (I=0 means fall through and I=1 means taken). 3 indicates -** a 3-way branch where the third way is when one of the operands is -** NULL. 4 indicates the OP_Jump instruction which has three destinations -** depending on whether the first operand is less than, equal to, or greater -** than the second. +** M is the type of branch. I is the direction taken for this instance of +** the branch. +** +** M: 2 - two-way branch (I=0: fall-thru 1: jump ) +** 3 - two-way + NULL (I=0: fall-thru 1: jump 2: NULL ) +** 4 - OP_Jump (I=0: jump p1 1: jump p2 2: jump p3) +** +** In other words, if M is 2, then I is either 0 (for fall-through) or +** 1 (for when the branch is taken). If M is 3, the I is 0 for an +** ordinary fall-through, I is 1 if the branch was taken, and I is 2 +** if the result of comparison is NULL. For M=3, I=2 the jump may or +** may not be taken, depending on the SQLITE_JUMPIFNULL flags in p5. +** When M is 4, that means that an OP_Jump is being run. I is 0, 1, or 2 +** depending on if the operands are less than, equal, or greater than. ** ** iSrcLine is the source code line (from the __LINE__ macro) that ** generated the VDBE instruction combined with flag bits. The source @@ -83404,9 +84001,9 @@ SQLITE_API int sqlite3_found_count = 0; ** alternate branch are never taken. If a branch is never taken then ** flags should be 0x06 since only the fall-through approach is allowed. ** -** Bit 0x04 of the flags indicates an OP_Jump opcode that is only +** Bit 0x08 of the flags indicates an OP_Jump opcode that is only ** interested in equal or not-equal. In other words, I==0 and I==2 -** should be treated the same. +** should be treated as equivalent ** ** Since only a line number is retained, not the filename, this macro ** only works for amalgamation builds. But that is ok, since these macros @@ -83430,6 +84027,18 @@ SQLITE_API int sqlite3_found_count = 0; mNever = iSrcLine >> 24; assert( (I & mNever)==0 ); if( sqlite3GlobalConfig.xVdbeBranch==0 ) return; /*NO_TEST*/ + /* Invoke the branch coverage callback with three arguments: + ** iSrcLine - the line number of the VdbeCoverage() macro, with + ** flags removed. + ** I - Mask of bits 0x07 indicating which cases are are + ** fulfilled by this instance of the jump. 0x01 means + ** fall-thru, 0x02 means taken, 0x04 means NULL. Any + ** impossible cases (ex: if the comparison is never NULL) + ** are filled in automatically so that the coverage + ** measurement logic does not flag those impossible cases + ** as missed coverage. + ** M - Type of jump. Same as M argument above + */ I |= mNever; if( M==2 ) I |= 0x04; if( M==4 ){ @@ -83441,14 +84050,6 @@ SQLITE_API int sqlite3_found_count = 0; } #endif -/* -** Convert the given register into a string if it isn't one -** already. Return non-zero if a malloc() fails. -*/ -#define Stringify(P, enc) \ - if(((P)->flags&(MEM_Str|MEM_Blob))==0 && sqlite3VdbeMemStringify(P,enc,0)) \ - { goto no_mem; } - /* ** An ephemeral string value (signified by the MEM_Ephem flag) contains ** a pointer to a dynamically allocated string where some other entity @@ -83510,7 +84111,7 @@ static VdbeCursor *allocateCursor( ** is clear. Otherwise, if this is an ephemeral cursor created by ** OP_OpenDup, the cursor will not be closed and will still be part ** of a BtShared.pCursor list. */ - p->apCsr[iCur]->isEphemeral = 0; + if( p->apCsr[iCur]->pBtx==0 ) p->apCsr[iCur]->isEphemeral = 0; sqlite3VdbeFreeCursor(p, p->apCsr[iCur]); p->apCsr[iCur] = 0; } @@ -83530,6 +84131,21 @@ static VdbeCursor *allocateCursor( return pCx; } +/* +** The string in pRec is known to look like an integer and to have a +** floating point value of rValue. Return true and set *piValue to the +** integer value if the string is in range to be an integer. Otherwise, +** return false. +*/ +static int alsoAnInt(Mem *pRec, double rValue, i64 *piValue){ + i64 iValue = (double)rValue; + if( sqlite3RealSameAsInt(rValue,iValue) ){ + *piValue = iValue; + return 1; + } + return 0==sqlite3Atoi64(pRec->z, piValue, pRec->n, pRec->enc); +} + /* ** Try to convert a value into a numeric representation if we can ** do so without loss of information. In other words, if the string @@ -83547,12 +84163,12 @@ static VdbeCursor *allocateCursor( */ static void applyNumericAffinity(Mem *pRec, int bTryForInt){ double rValue; - i64 iValue; u8 enc = pRec->enc; - assert( (pRec->flags & (MEM_Str|MEM_Int|MEM_Real))==MEM_Str ); - if( sqlite3AtoF(pRec->z, &rValue, pRec->n, enc)==0 ) return; - if( 0==sqlite3Atoi64(pRec->z, &iValue, pRec->n, enc) ){ - pRec->u.i = iValue; + int rc; + assert( (pRec->flags & (MEM_Str|MEM_Int|MEM_Real|MEM_IntReal))==MEM_Str ); + rc = sqlite3AtoF(pRec->z, &rValue, pRec->n, enc); + if( rc<=0 ) return; + if( rc==1 && alsoAnInt(pRec, rValue, &pRec->u.i) ){ pRec->flags |= MEM_Int; }else{ pRec->u.r = rValue; @@ -83582,6 +84198,7 @@ static void applyNumericAffinity(Mem *pRec, int bTryForInt){ ** Convert pRec to a text representation. ** ** SQLITE_AFF_BLOB: +** SQLITE_AFF_NONE: ** No-op. pRec is unchanged. */ static void applyAffinity( @@ -83606,11 +84223,14 @@ static void applyAffinity( ** there is already a string rep, but it is pointless to waste those ** CPU cycles. */ if( 0==(pRec->flags&MEM_Str) ){ /*OPTIMIZATION-IF-FALSE*/ - if( (pRec->flags&(MEM_Real|MEM_Int)) ){ + if( (pRec->flags&(MEM_Real|MEM_Int|MEM_IntReal)) ){ + testcase( pRec->flags & MEM_Int ); + testcase( pRec->flags & MEM_Real ); + testcase( pRec->flags & MEM_IntReal ); sqlite3VdbeMemStringify(pRec, enc, 1); } } - pRec->flags &= ~(MEM_Real|MEM_Int); + pRec->flags &= ~(MEM_Real|MEM_Int|MEM_IntReal); } } @@ -83649,13 +84269,21 @@ SQLITE_PRIVATE void sqlite3ValueApplyAffinity( ** accordingly. */ static u16 SQLITE_NOINLINE computeNumericType(Mem *pMem){ - assert( (pMem->flags & (MEM_Int|MEM_Real))==0 ); + int rc; + sqlite3_int64 ix; + assert( (pMem->flags & (MEM_Int|MEM_Real|MEM_IntReal))==0 ); assert( (pMem->flags & (MEM_Str|MEM_Blob))!=0 ); ExpandBlob(pMem); - if( sqlite3AtoF(pMem->z, &pMem->u.r, pMem->n, pMem->enc)==0 ){ - return 0; - } - if( sqlite3Atoi64(pMem->z, &pMem->u.i, pMem->n, pMem->enc)==0 ){ + rc = sqlite3AtoF(pMem->z, &pMem->u.r, pMem->n, pMem->enc); + if( rc<=0 ){ + if( rc==0 && sqlite3Atoi64(pMem->z, &ix, pMem->n, pMem->enc)<=1 ){ + pMem->u.i = ix; + return MEM_Int; + }else{ + return MEM_Real; + } + }else if( rc==1 && sqlite3Atoi64(pMem->z, &ix, pMem->n, pMem->enc)==0 ){ + pMem->u.i = ix; return MEM_Int; } return MEM_Real; @@ -83669,10 +84297,15 @@ static u16 SQLITE_NOINLINE computeNumericType(Mem *pMem){ ** But it does set pMem->u.r and pMem->u.i appropriately. */ static u16 numericType(Mem *pMem){ - if( pMem->flags & (MEM_Int|MEM_Real) ){ - return pMem->flags & (MEM_Int|MEM_Real); + if( pMem->flags & (MEM_Int|MEM_Real|MEM_IntReal) ){ + testcase( pMem->flags & MEM_Int ); + testcase( pMem->flags & MEM_Real ); + testcase( pMem->flags & MEM_IntReal ); + return pMem->flags & (MEM_Int|MEM_Real|MEM_IntReal); } if( pMem->flags & (MEM_Str|MEM_Blob) ){ + testcase( pMem->flags & MEM_Str ); + testcase( pMem->flags & MEM_Blob ); return computeNumericType(pMem); } return 0; @@ -83705,13 +84338,15 @@ SQLITE_PRIVATE void sqlite3VdbeMemPrettyPrint(Mem *pMem, char *zBuf){ c = 's'; } *(zCsr++) = c; + *(zCsr++) = 'x'; sqlite3_snprintf(100, zCsr, "%d[", pMem->n); zCsr += sqlite3Strlen30(zCsr); - for(i=0; i<16 && in; i++){ + for(i=0; i<25 && in; i++){ sqlite3_snprintf(100, zCsr, "%02X", ((int)pMem->z[i] & 0xFF)); zCsr += sqlite3Strlen30(zCsr); } - for(i=0; i<16 && in; i++){ + *zCsr++ = '|'; + for(i=0; i<25 && in; i++){ char z = pMem->z[i]; if( z<32 || z>126 ) *zCsr++ = '.'; else *zCsr++ = z; @@ -83741,7 +84376,7 @@ SQLITE_PRIVATE void sqlite3VdbeMemPrettyPrint(Mem *pMem, char *zBuf){ sqlite3_snprintf(100, &zBuf[k], "%d", pMem->n); k += sqlite3Strlen30(&zBuf[k]); zBuf[k++] = '['; - for(j=0; j<15 && jn; j++){ + for(j=0; j<25 && jn; j++){ u8 c = pMem->z[j]; if( c>=0x20 && c<0x7f ){ zBuf[k++] = c; @@ -83768,11 +84403,13 @@ static void memTracePrint(Mem *p){ printf(p->flags & MEM_Zero ? " NULL-nochng" : " NULL"); }else if( (p->flags & (MEM_Int|MEM_Str))==(MEM_Int|MEM_Str) ){ printf(" si:%lld", p->u.i); + }else if( (p->flags & (MEM_IntReal))!=0 ){ + printf(" ir:%lld", p->u.i); }else if( p->flags & MEM_Int ){ printf(" i:%lld", p->u.i); #ifndef SQLITE_OMIT_FLOATING_POINT }else if( p->flags & MEM_Real ){ - printf(" r:%g", p->u.r); + printf(" r:%.17g", p->u.r); #endif }else if( sqlite3VdbeMemIsRowSet(p) ){ printf(" (rowset)"); @@ -83977,6 +84614,15 @@ SQLITE_PRIVATE int sqlite3VdbeExec( assert( p->magic==VDBE_MAGIC_RUN ); /* sqlite3_step() verifies this */ sqlite3VdbeEnter(p); +#ifndef SQLITE_OMIT_PROGRESS_CALLBACK + if( db->xProgress ){ + u32 iPrior = p->aCounter[SQLITE_STMTSTATUS_VM_STEP]; + assert( 0 < db->nProgressOps ); + nProgressLimit = db->nProgressOps - (iPrior % db->nProgressOps); + }else{ + nProgressLimit = 0xffffffff; + } +#endif if( p->rc==SQLITE_NOMEM ){ /* This happens if a malloc() inside a call to sqlite3_column_text() or ** sqlite3_column_text16() failed. */ @@ -83990,15 +84636,6 @@ SQLITE_PRIVATE int sqlite3VdbeExec( db->busyHandler.nBusy = 0; if( db->u1.isInterrupted ) goto abort_due_to_interrupt; sqlite3VdbeIOTraceSql(p); -#ifndef SQLITE_OMIT_PROGRESS_CALLBACK - if( db->xProgress ){ - u32 iPrior = p->aCounter[SQLITE_STMTSTATUS_VM_STEP]; - assert( 0 < db->nProgressOps ); - nProgressLimit = db->nProgressOps - (iPrior % db->nProgressOps); - }else{ - nProgressLimit = 0xffffffff; - } -#endif #ifdef SQLITE_DEBUG sqlite3BeginBenignMalloc(); if( p->pc==0 @@ -84174,10 +84811,11 @@ case OP_Goto: { /* jump */ ** If the progress callback returns non-zero, exit the virtual machine with ** a return code SQLITE_ABORT. */ - if( nVmStep>=nProgressLimit && db->xProgress!=0 ){ + while( nVmStep>=nProgressLimit && db->xProgress!=0 ){ assert( db->nProgressOps!=0 ); - nProgressLimit = nVmStep + db->nProgressOps - (nVmStep%db->nProgressOps); + nProgressLimit += db->nProgressOps; if( db->xProgress(db->pProgressArg) ){ + nProgressLimit = 0xffffffff; rc = SQLITE_INTERRUPT; goto abort_due_to_error; } @@ -84449,13 +85087,13 @@ case OP_Real: { /* same as TK_FLOAT, out2 */ case OP_String8: { /* same as TK_STRING, out2 */ assert( pOp->p4.z!=0 ); pOut = out2Prerelease(p, pOp); - pOp->opcode = OP_String; pOp->p1 = sqlite3Strlen30(pOp->p4.z); #ifndef SQLITE_OMIT_UTF16 if( encoding!=SQLITE_UTF8 ){ rc = sqlite3VdbeMemSetStr(pOut, pOp->p4.z, -1, SQLITE_UTF8, SQLITE_STATIC); assert( rc==SQLITE_OK || rc==SQLITE_TOOBIG ); + if( rc ) goto too_big; if( SQLITE_OK!=sqlite3VdbeChangeEncoding(pOut, encoding) ) goto no_mem; assert( pOut->szMalloc>0 && pOut->zMalloc==pOut->z ); assert( VdbeMemDynamic(pOut)==0 ); @@ -84468,11 +85106,11 @@ case OP_String8: { /* same as TK_STRING, out2 */ pOp->p4.z = pOut->z; pOp->p1 = pOut->n; } - testcase( rc==SQLITE_TOOBIG ); #endif if( pOp->p1>db->aLimit[SQLITE_LIMIT_LENGTH] ){ goto too_big; } + pOp->opcode = OP_String; assert( rc==SQLITE_OK ); /* Fall through to the next case, OP_String */ } @@ -84590,7 +85228,10 @@ case OP_Variable: { /* out2 */ goto too_big; } pOut = &aMem[pOp->p2]; - sqlite3VdbeMemShallowCopy(pOut, pVar, MEM_Static); + if( VdbeMemDynamic(pOut) ) sqlite3VdbeMemSetNull(pOut); + memcpy(pOut, pVar, MEMCELLSIZE); + pOut->flags &= ~(MEM_Dyn|MEM_Ephem); + pOut->flags |= MEM_Static|MEM_FromBind; UPDATE_MAX_BLOBSIZE(pOut); break; } @@ -84723,18 +85364,6 @@ case OP_ResultRow: { assert( pOp->p1>0 ); assert( pOp->p1+pOp->p2<=(p->nMem+1 - p->nCursor)+1 ); -#ifndef SQLITE_OMIT_PROGRESS_CALLBACK - /* Run the progress counter just before returning. - */ - if( db->xProgress!=0 - && nVmStep>=nProgressLimit - && db->xProgress(db->pProgressArg)!=0 - ){ - rc = SQLITE_INTERRUPT; - goto abort_due_to_error; - } -#endif - /* If this statement has violated immediate foreign key constraints, do ** not return the number of rows modified. And do not RELEASE the statement ** transaction. It needs to be rolled back. */ @@ -84806,33 +85435,57 @@ case OP_ResultRow: { ** to avoid a memcpy(). */ case OP_Concat: { /* same as TK_CONCAT, in1, in2, out3 */ - i64 nByte; + i64 nByte; /* Total size of the output string or blob */ + u16 flags1; /* Initial flags for P1 */ + u16 flags2; /* Initial flags for P2 */ pIn1 = &aMem[pOp->p1]; pIn2 = &aMem[pOp->p2]; pOut = &aMem[pOp->p3]; + testcase( pIn1==pIn2 ); + testcase( pOut==pIn2 ); assert( pIn1!=pOut ); - if( (pIn1->flags | pIn2->flags) & MEM_Null ){ + flags1 = pIn1->flags; + testcase( flags1 & MEM_Null ); + testcase( pIn2->flags & MEM_Null ); + if( (flags1 | pIn2->flags) & MEM_Null ){ sqlite3VdbeMemSetNull(pOut); break; } - if( ExpandBlob(pIn1) || ExpandBlob(pIn2) ) goto no_mem; - Stringify(pIn1, encoding); - Stringify(pIn2, encoding); + if( (flags1 & (MEM_Str|MEM_Blob))==0 ){ + if( sqlite3VdbeMemStringify(pIn1,encoding,0) ) goto no_mem; + flags1 = pIn1->flags & ~MEM_Str; + }else if( (flags1 & MEM_Zero)!=0 ){ + if( sqlite3VdbeMemExpandBlob(pIn1) ) goto no_mem; + flags1 = pIn1->flags & ~MEM_Str; + } + flags2 = pIn2->flags; + if( (flags2 & (MEM_Str|MEM_Blob))==0 ){ + if( sqlite3VdbeMemStringify(pIn2,encoding,0) ) goto no_mem; + flags2 = pIn2->flags & ~MEM_Str; + }else if( (flags2 & MEM_Zero)!=0 ){ + if( sqlite3VdbeMemExpandBlob(pIn2) ) goto no_mem; + flags2 = pIn2->flags & ~MEM_Str; + } nByte = pIn1->n + pIn2->n; if( nByte>db->aLimit[SQLITE_LIMIT_LENGTH] ){ goto too_big; } - if( sqlite3VdbeMemGrow(pOut, (int)nByte+2, pOut==pIn2) ){ + if( sqlite3VdbeMemGrow(pOut, (int)nByte+3, pOut==pIn2) ){ goto no_mem; } MemSetTypeFlag(pOut, MEM_Str); if( pOut!=pIn2 ){ memcpy(pOut->z, pIn2->z, pIn2->n); + assert( (pIn2->flags & MEM_Dyn) == (flags2 & MEM_Dyn) ); + pIn2->flags = flags2; } memcpy(&pOut->z[pIn2->n], pIn1->z, pIn1->n); + assert( (pIn1->flags & MEM_Dyn) == (flags1 & MEM_Dyn) ); + pIn1->flags = flags1; pOut->z[nByte]=0; pOut->z[nByte+1] = 0; + pOut->z[nByte+2] = 0; pOut->flags |= MEM_Term; pOut->n = (int)nByte; pOut->enc = encoding; @@ -84883,7 +85536,6 @@ case OP_Subtract: /* same as TK_MINUS, in1, in2, out3 */ case OP_Multiply: /* same as TK_STAR, in1, in2, out3 */ case OP_Divide: /* same as TK_SLASH, in1, in2, out3 */ case OP_Remainder: { /* same as TK_REM, in1, in2, out3 */ - char bIntint; /* Started out as two integer operands */ u16 flags; /* Combined MEM_* flags from both inputs */ u16 type1; /* Numeric type of left operand */ u16 type2; /* Numeric type of right operand */ @@ -84901,7 +85553,6 @@ case OP_Remainder: { /* same as TK_REM, in1, in2, out3 */ if( (type1 & type2 & MEM_Int)!=0 ){ iA = pIn1->u.i; iB = pIn2->u.i; - bIntint = 1; switch( pOp->opcode ){ case OP_Add: if( sqlite3AddInt64(&iB,iA) ) goto fp_math; break; case OP_Subtract: if( sqlite3SubInt64(&iB,iA) ) goto fp_math; break; @@ -84924,7 +85575,6 @@ case OP_Remainder: { /* same as TK_REM, in1, in2, out3 */ }else if( (flags & MEM_Null)!=0 ){ goto arithmetic_result_is_null; }else{ - bIntint = 0; fp_math: rA = sqlite3VdbeRealValue(pIn1); rB = sqlite3VdbeRealValue(pIn2); @@ -84956,9 +85606,6 @@ case OP_Remainder: { /* same as TK_REM, in1, in2, out3 */ } pOut->u.r = rB; MemSetTypeFlag(pOut, MEM_Real); - if( ((type1|type2)&MEM_Real)==0 && !bIntint ){ - sqlite3VdbeIntegerAffinity(pOut); - } #endif } break; @@ -85100,8 +85747,8 @@ case OP_MustBeInt: { /* jump, in1 */ pIn1 = &aMem[pOp->p1]; if( (pIn1->flags & MEM_Int)==0 ){ applyAffinity(pIn1, SQLITE_AFF_NUMERIC, encoding); - VdbeBranchTaken((pIn1->flags&MEM_Int)==0, 2); if( (pIn1->flags & MEM_Int)==0 ){ + VdbeBranchTaken(1, 2); if( pOp->p2==0 ){ rc = SQLITE_MISMATCH; goto abort_due_to_error; @@ -85110,6 +85757,7 @@ case OP_MustBeInt: { /* jump, in1 */ } } } + VdbeBranchTaken(0, 2); MemSetTypeFlag(pIn1, MEM_Int); break; } @@ -85126,8 +85774,11 @@ case OP_MustBeInt: { /* jump, in1 */ */ case OP_RealAffinity: { /* in1 */ pIn1 = &aMem[pOp->p1]; - if( pIn1->flags & MEM_Int ){ + if( pIn1->flags & (MEM_Int|MEM_IntReal) ){ + testcase( pIn1->flags & MEM_Int ); + testcase( pIn1->flags & MEM_IntReal ); sqlite3VdbeMemRealify(pIn1); + REGISTER_TRACE(pOp->p1, pIn1); } break; } @@ -85284,7 +85935,6 @@ case OP_Ge: { /* same as TK_GE, jump, in1, in3 */ ** OP_Eq or OP_Ne) then take the jump or not depending on whether ** or not both operands are null. */ - assert( pOp->opcode==OP_Eq || pOp->opcode==OP_Ne ); assert( (flags1 & MEM_Cleared)==0 ); assert( (pOp->p5 & SQLITE_JUMPIFNULL)==0 || CORRUPT_DB ); testcase( (pOp->p5 & SQLITE_JUMPIFNULL)!=0 ); @@ -85293,7 +85943,7 @@ case OP_Ge: { /* same as TK_GE, jump, in1, in3 */ ){ res = 0; /* Operands are equal */ }else{ - res = 1; /* Operands are not equal */ + res = ((flags3 & MEM_Null) ? -1 : +1); /* Operands are not equal */ } }else{ /* SQLITE_NULLEQ is clear and at least one operand is NULL, @@ -85319,7 +85969,7 @@ case OP_Ge: { /* same as TK_GE, jump, in1, in3 */ affinity = pOp->p5 & SQLITE_AFF_MASK; if( affinity>=SQLITE_AFF_NUMERIC ){ if( (flags1 | flags3)&MEM_Str ){ - if( (flags1 & (MEM_Int|MEM_Real|MEM_Str))==MEM_Str ){ + if( (flags1 & (MEM_Int|MEM_IntReal|MEM_Real|MEM_Str))==MEM_Str ){ applyNumericAffinity(pIn1,0); assert( flags3==pIn3->flags ); /* testcase( flags3!=pIn3->flags ); @@ -85329,7 +85979,7 @@ case OP_Ge: { /* same as TK_GE, jump, in1, in3 */ ** in case our analysis is incorrect, so it is left in. */ flags3 = pIn3->flags; } - if( (flags3 & (MEM_Int|MEM_Real|MEM_Str))==MEM_Str ){ + if( (flags3 & (MEM_Int|MEM_IntReal|MEM_Real|MEM_Str))==MEM_Str ){ applyNumericAffinity(pIn3,0); } } @@ -85342,17 +85992,19 @@ case OP_Ge: { /* same as TK_GE, jump, in1, in3 */ goto compare_op; } }else if( affinity==SQLITE_AFF_TEXT ){ - if( (flags1 & MEM_Str)==0 && (flags1 & (MEM_Int|MEM_Real))!=0 ){ + if( (flags1 & MEM_Str)==0 && (flags1&(MEM_Int|MEM_Real|MEM_IntReal))!=0 ){ testcase( pIn1->flags & MEM_Int ); testcase( pIn1->flags & MEM_Real ); + testcase( pIn1->flags & MEM_IntReal ); sqlite3VdbeMemStringify(pIn1, encoding, 1); testcase( (flags1&MEM_Dyn) != (pIn1->flags&MEM_Dyn) ); flags1 = (pIn1->flags & ~MEM_TypeMask) | (flags1 & MEM_TypeMask); assert( pIn1!=pIn3 ); } - if( (flags3 & MEM_Str)==0 && (flags3 & (MEM_Int|MEM_Real))!=0 ){ + if( (flags3 & MEM_Str)==0 && (flags3&(MEM_Int|MEM_Real|MEM_IntReal))!=0 ){ testcase( pIn3->flags & MEM_Int ); testcase( pIn3->flags & MEM_Real ); + testcase( pIn3->flags & MEM_IntReal ); sqlite3VdbeMemStringify(pIn3, encoding, 1); testcase( (flags3&MEM_Dyn) != (pIn3->flags&MEM_Dyn) ); flags3 = (pIn3->flags & ~MEM_TypeMask) | (flags3 & MEM_TypeMask); @@ -85411,7 +86063,7 @@ case OP_Ge: { /* same as TK_GE, jump, in1, in3 */ pOut->u.i = res2; REGISTER_TRACE(pOp->p2, pOut); }else{ - VdbeBranchTaken(res!=0, (pOp->p5 & SQLITE_NULLEQ)?2:3); + VdbeBranchTaken(res2!=0, (pOp->p5 & SQLITE_NULLEQ)?2:3); if( res2 ){ goto jump_to_p2; } @@ -85522,9 +86174,14 @@ case OP_Compare: { REGISTER_TRACE(p2+idx, &aMem[p2+idx]); assert( inKeyField ); pColl = pKeyInfo->aColl[i]; - bRev = pKeyInfo->aSortOrder[i]; + bRev = (pKeyInfo->aSortFlags[i] & KEYINFO_ORDER_DESC); iCompare = sqlite3MemCompare(&aMem[p1+idx], &aMem[p2+idx], pColl); if( iCompare ){ + if( (pKeyInfo->aSortFlags[i] & KEYINFO_ORDER_BIGNULL) + && ((aMem[p1+idx].flags & MEM_Null) || (aMem[p2+idx].flags & MEM_Null)) + ){ + iCompare = -iCompare; + } if( bRev ) iCompare = -iCompare; break; } @@ -85815,11 +86472,6 @@ case OP_Offset: { /* out3 */ ** if the P4 argument is a P4_MEM use the value of the P4 argument as ** the result. ** -** If the OPFLAG_CLEARCACHE bit is set on P5 and P1 is a pseudo-table cursor, -** then the cache of the cursor is reset prior to extracting the column. -** The first OP_Column against a pseudo-table after the value of the content -** register has changed should have this bit set. -** ** If the OPFLAG_LENGTHARG and OPFLAG_TYPEOFARG bits are set on P5 then ** the result is guaranteed to only be used as the argument of a length() ** or typeof() function, respectively. The loading of large blobs can be @@ -85961,15 +86613,15 @@ case OP_Column: { zEndHdr = zData + aOffset[0]; testcase( zHdr>=zEndHdr ); do{ - if( (t = zHdr[0])<0x80 ){ + if( (pC->aType[i] = t = zHdr[0])<0x80 ){ zHdr++; offset64 += sqlite3VdbeOneByteSerialTypeLen(t); }else{ zHdr += sqlite3GetVarint32(zHdr, &t); + pC->aType[i] = t; offset64 += sqlite3VdbeSerialTypeLen(t); } - pC->aType[i++] = t; - aOffset[i] = (u32)(offset64 & 0xffffffff); + aOffset[++i] = (u32)(offset64 & 0xffffffff); }while( i<=p2 && zHdrp2>0 ); assert( zAffinity[pOp->p2]==0 ); pIn1 = &aMem[pOp->p1]; - do{ + while( 1 /*exit-by-break*/ ){ assert( pIn1 <= &p->aMem[(p->nMem+1 - p->nCursor)] ); assert( memIsValid(pIn1) ); - applyAffinity(pIn1, *(zAffinity++), encoding); + applyAffinity(pIn1, zAffinity[0], encoding); + if( zAffinity[0]==SQLITE_AFF_REAL && (pIn1->flags & MEM_Int)!=0 ){ + /* When applying REAL affinity, if the result is still an MEM_Int + ** that will fit in 6 bytes, then change the type to MEM_IntReal + ** so that we keep the high-resolution integer value but know that + ** the type really wants to be REAL. */ + testcase( pIn1->u.i==140737488355328LL ); + testcase( pIn1->u.i==140737488355327LL ); + testcase( pIn1->u.i==-140737488355328LL ); + testcase( pIn1->u.i==-140737488355329LL ); + if( pIn1->u.i<=140737488355327LL && pIn1->u.i>=-140737488355328LL ){ + pIn1->flags |= MEM_IntReal; + pIn1->flags &= ~MEM_Int; + }else{ + pIn1->u.r = (double)pIn1->u.i; + pIn1->flags |= MEM_Real; + pIn1->flags &= ~MEM_Int; + } + } + REGISTER_TRACE((int)(pIn1-aMem), pIn1); + zAffinity++; + if( zAffinity[0]==0 ) break; pIn1++; - }while( zAffinity[0] ); + } break; } @@ -86134,7 +86807,6 @@ case OP_Affinity: { ** If P4 is NULL then all index fields have the affinity BLOB. */ case OP_MakeRecord: { - u8 *zNewRecord; /* A buffer to hold the data for the new record */ Mem *pRec; /* The new record */ u64 nData; /* Number of bytes of data space */ int nHdr; /* Number of bytes of header space */ @@ -86147,9 +86819,9 @@ case OP_MakeRecord: { int nField; /* Number of fields in the record */ char *zAffinity; /* The affinity string for the record */ int file_format; /* File format to use for encoding */ - int i; /* Space used in zNewRecord[] header */ - int j; /* Space used in zNewRecord[] content */ u32 len; /* Length of a field */ + u8 *zHdr; /* Where to write next byte of the header */ + u8 *zPayload; /* Where to write next byte of the payload */ /* Assuming the record contains N fields, the record format looks ** like this: @@ -86188,7 +86860,14 @@ case OP_MakeRecord: { if( zAffinity ){ pRec = pData0; do{ - applyAffinity(pRec++, *(zAffinity++), encoding); + applyAffinity(pRec, zAffinity[0], encoding); + if( zAffinity[0]==SQLITE_AFF_REAL && (pRec->flags & MEM_Int) ){ + pRec->flags |= MEM_IntReal; + pRec->flags &= ~(MEM_Int); + } + REGISTER_TRACE((int)(pRec-aMem), pRec); + zAffinity++; + pRec++; assert( zAffinity[0]==0 || pRec<=pLast ); }while( zAffinity[0] ); } @@ -86208,14 +86887,36 @@ case OP_MakeRecord: { #endif /* Loop through the elements that will make up the record to figure - ** out how much space is required for the new record. + ** out how much space is required for the new record. After this loop, + ** the Mem.uTemp field of each term should hold the serial-type that will + ** be used for that term in the generated record: + ** + ** Mem.uTemp value type + ** --------------- --------------- + ** 0 NULL + ** 1 1-byte signed integer + ** 2 2-byte signed integer + ** 3 3-byte signed integer + ** 4 4-byte signed integer + ** 5 6-byte signed integer + ** 6 8-byte signed integer + ** 7 IEEE float + ** 8 Integer constant 0 + ** 9 Integer constant 1 + ** 10,11 reserved for expansion + ** N>=12 and even BLOB + ** N>=13 and odd text + ** + ** The following additional values are computed: + ** nHdr Number of bytes needed for the record header + ** nData Number of bytes of data space needed for the record + ** nZero Zero bytes at the end of the record */ pRec = pLast; do{ assert( memIsValid(pRec) ); - serial_type = sqlite3VdbeSerialType(pRec, file_format, &len); - if( pRec->flags & MEM_Zero ){ - if( serial_type==0 ){ + if( pRec->flags & MEM_Null ){ + if( pRec->flags & MEM_Zero ){ /* Values with MEM_Null and MEM_Zero are created by xColumn virtual ** table methods that never invoke sqlite3_result_xxxxx() while ** computing an unchanging column value in an UPDATE statement. @@ -86223,19 +86924,83 @@ case OP_MakeRecord: { ** so that they can be passed through to xUpdate and have ** a true sqlite3_value_nochange(). */ assert( pOp->p5==OPFLAG_NOCHNG_MAGIC || CORRUPT_DB ); - serial_type = 10; - }else if( nData ){ - if( sqlite3VdbeMemExpandBlob(pRec) ) goto no_mem; + pRec->uTemp = 10; }else{ - nZero += pRec->u.nZero; - len -= pRec->u.nZero; + pRec->uTemp = 0; + } + nHdr++; + }else if( pRec->flags & (MEM_Int|MEM_IntReal) ){ + /* Figure out whether to use 1, 2, 4, 6 or 8 bytes. */ + i64 i = pRec->u.i; + u64 uu; + testcase( pRec->flags & MEM_Int ); + testcase( pRec->flags & MEM_IntReal ); + if( i<0 ){ + uu = ~i; + }else{ + uu = i; + } + nHdr++; + testcase( uu==127 ); testcase( uu==128 ); + testcase( uu==32767 ); testcase( uu==32768 ); + testcase( uu==8388607 ); testcase( uu==8388608 ); + testcase( uu==2147483647 ); testcase( uu==2147483648 ); + testcase( uu==140737488355327LL ); testcase( uu==140737488355328LL ); + if( uu<=127 ){ + if( (i&1)==i && file_format>=4 ){ + pRec->uTemp = 8+(u32)uu; + }else{ + nData++; + pRec->uTemp = 1; + } + }else if( uu<=32767 ){ + nData += 2; + pRec->uTemp = 2; + }else if( uu<=8388607 ){ + nData += 3; + pRec->uTemp = 3; + }else if( uu<=2147483647 ){ + nData += 4; + pRec->uTemp = 4; + }else if( uu<=140737488355327LL ){ + nData += 6; + pRec->uTemp = 5; + }else{ + nData += 8; + if( pRec->flags & MEM_IntReal ){ + /* If the value is IntReal and is going to take up 8 bytes to store + ** as an integer, then we might as well make it an 8-byte floating + ** point value */ + pRec->u.r = (double)pRec->u.i; + pRec->flags &= ~MEM_IntReal; + pRec->flags |= MEM_Real; + pRec->uTemp = 7; + }else{ + pRec->uTemp = 6; + } + } + }else if( pRec->flags & MEM_Real ){ + nHdr++; + nData += 8; + pRec->uTemp = 7; + }else{ + assert( db->mallocFailed || pRec->flags&(MEM_Str|MEM_Blob) ); + assert( pRec->n>=0 ); + len = (u32)pRec->n; + serial_type = (len*2) + 12 + ((pRec->flags & MEM_Str)!=0); + if( pRec->flags & MEM_Zero ){ + serial_type += pRec->u.nZero*2; + if( nData ){ + if( sqlite3VdbeMemExpandBlob(pRec) ) goto no_mem; + len += pRec->u.nZero; + }else{ + nZero += pRec->u.nZero; + } } + nData += len; + nHdr += sqlite3VarintLen(serial_type); + pRec->uTemp = serial_type; } - nData += len; - testcase( serial_type==127 ); - testcase( serial_type==128 ); - nHdr += serial_type<=127 ? 1 : sqlite3VarintLen(serial_type); - pRec->uTemp = serial_type; if( pRec==pData0 ) break; pRec--; }while(1); @@ -86276,34 +87041,34 @@ case OP_MakeRecord: { goto no_mem; } } - zNewRecord = (u8 *)pOut->z; + pOut->n = (int)nByte; + pOut->flags = MEM_Blob; + if( nZero ){ + pOut->u.nZero = nZero; + pOut->flags |= MEM_Zero; + } + UPDATE_MAX_BLOBSIZE(pOut); + zHdr = (u8 *)pOut->z; + zPayload = zHdr + nHdr; /* Write the record */ - i = putVarint32(zNewRecord, nHdr); - j = nHdr; + zHdr += putVarint32(zHdr, nHdr); assert( pData0<=pLast ); pRec = pData0; do{ serial_type = pRec->uTemp; /* EVIDENCE-OF: R-06529-47362 Following the size varint are one or more ** additional varints, one per column. */ - i += putVarint32(&zNewRecord[i], serial_type); /* serial type */ + zHdr += putVarint32(zHdr, serial_type); /* serial type */ /* EVIDENCE-OF: R-64536-51728 The values for each column in the record ** immediately follow the header. */ - j += sqlite3VdbeSerialPut(&zNewRecord[j], pRec, serial_type); /* content */ + zPayload += sqlite3VdbeSerialPut(zPayload, pRec, serial_type); /* content */ }while( (++pRec)<=pLast ); - assert( i==nHdr ); - assert( j==nByte ); + assert( nHdr==(int)(zHdr - (u8*)pOut->z) ); + assert( nByte==(int)(zPayload - (u8*)pOut->z) ); assert( pOp->p3>0 && pOp->p3<=(p->nMem+1 - p->nCursor) ); - pOut->n = (int)nByte; - pOut->flags = MEM_Blob; - if( nZero ){ - pOut->u.nZero = nZero; - pOut->flags |= MEM_Zero; - } REGISTER_TRACE(pOp->p3, pOut); - UPDATE_MAX_BLOBSIZE(pOut); break; } @@ -86333,8 +87098,9 @@ case OP_Count: { /* out2 */ /* Opcode: Savepoint P1 * * P4 * ** ** Open, release or rollback the savepoint named by parameter P4, depending -** on the value of P1. To open a new savepoint, P1==0. To release (commit) an -** existing savepoint, P1==1, or to rollback an existing savepoint P1==2. +** on the value of P1. To open a new savepoint set P1==0 (SAVEPOINT_BEGIN). +** To release (commit) an existing savepoint set P1==1 (SAVEPOINT_RELEASE). +** To rollback an existing savepoint set P1==2 (SAVEPOINT_ROLLBACK). */ case OP_Savepoint: { int p1; /* Value of P1 operand */ @@ -86402,6 +87168,7 @@ case OP_Savepoint: { } } }else{ + assert( p1==SAVEPOINT_RELEASE || p1==SAVEPOINT_ROLLBACK ); iSavepoint = 0; /* Find the named savepoint. If there is no such savepoint, then an @@ -86455,6 +87222,7 @@ case OP_Savepoint: { if( rc!=SQLITE_OK ) goto abort_due_to_error; } }else{ + assert( p1==SAVEPOINT_RELEASE ); isSchemaChange = 0; } for(ii=0; iinDb; ii++){ @@ -86491,6 +87259,7 @@ case OP_Savepoint: { db->nSavepoint--; } }else{ + assert( p1==SAVEPOINT_ROLLBACK ); db->nDeferredCons = pSavepoint->nDeferredCons; db->nDeferredImmCons = pSavepoint->nDeferredImmCons; } @@ -86567,7 +87336,7 @@ case OP_AutoCommit: { rc = SQLITE_ERROR; goto abort_due_to_error; } - break; + /*NOTREACHED*/ assert(0); } /* Opcode: Transaction P1 P2 P3 P4 P5 @@ -86972,6 +87741,7 @@ case OP_OpenDup: { pCx->pKeyInfo = pOrig->pKeyInfo; pCx->isTable = pOrig->isTable; pCx->pgnoRoot = pOrig->pgnoRoot; + pCx->isOrdered = pOrig->isOrdered; rc = sqlite3BtreeCursor(pOrig->pBtx, pCx->pgnoRoot, BTREE_WRCSR, pCx->pKeyInfo, pCx->uc.pCursor); /* The sqlite3BtreeCursor() routine can only fail for the first cursor @@ -87028,11 +87798,15 @@ case OP_OpenEphemeral: { if( pCx ){ /* If the ephermeral table is already open, erase all existing content ** so that the table is empty again, rather than creating a new table. */ - rc = sqlite3BtreeClearTable(pCx->pBtx, pCx->pgnoRoot, 0); + assert( pCx->isEphemeral ); + pCx->seqCount = 0; + pCx->cacheStatus = CACHE_STALE; + if( pCx->pBtx ){ + rc = sqlite3BtreeClearTable(pCx->pBtx, pCx->pgnoRoot, 0); + } }else{ pCx = allocateCursor(p, pOp->p1, pOp->p2, -1, CURTYPE_BTREE); if( pCx==0 ) goto no_mem; - pCx->nullRow = 1; pCx->isEphemeral = 1; rc = sqlite3BtreeOpen(db->pVfs, 0, db, &pCx->pBtx, BTREE_OMIT_JOURNAL | BTREE_SINGLE | pOp->p5, @@ -87068,6 +87842,7 @@ case OP_OpenEphemeral: { pCx->isOrdered = (pOp->p5!=BTREE_UNORDERED); } if( rc ) goto abort_due_to_error; + pCx->nullRow = 1; break; } @@ -87296,7 +88071,10 @@ case OP_SeekGT: { /* jump, in3, group */ pC->seekOp = pOp->opcode; #endif + pC->deferredMoveto = 0; + pC->cacheStatus = CACHE_STALE; if( pC->isTable ){ + u16 flags3, newType; /* The BTREE_SEEK_EQ flag is only set on index cursors */ assert( sqlite3BtreeCursorHasHint(pC->uc.pCursor, BTREE_SEEK_EQ)==0 || CORRUPT_DB ); @@ -87305,20 +88083,27 @@ case OP_SeekGT: { /* jump, in3, group */ ** blob, or NULL. But it needs to be an integer before we can do ** the seek, so convert it. */ pIn3 = &aMem[pOp->p3]; - if( (pIn3->flags & (MEM_Int|MEM_Real|MEM_Str))==MEM_Str ){ + flags3 = pIn3->flags; + if( (flags3 & (MEM_Int|MEM_Real|MEM_IntReal|MEM_Str))==MEM_Str ){ applyNumericAffinity(pIn3, 0); } - iKey = sqlite3VdbeIntValue(pIn3); + iKey = sqlite3VdbeIntValue(pIn3); /* Get the integer key value */ + newType = pIn3->flags; /* Record the type after applying numeric affinity */ + pIn3->flags = flags3; /* But convert the type back to its original */ /* If the P3 value could not be converted into an integer without ** loss of information, then special processing is required... */ - if( (pIn3->flags & MEM_Int)==0 ){ - if( (pIn3->flags & MEM_Real)==0 ){ - /* If the P3 value cannot be converted into any kind of a number, - ** then the seek is not possible, so jump to P2 */ - VdbeBranchTaken(1,2); goto jump_to_p2; - break; - } + if( (newType & (MEM_Int|MEM_IntReal))==0 ){ + if( (newType & MEM_Real)==0 ){ + if( (newType & MEM_Null) || oc>=OP_SeekGE ){ + VdbeBranchTaken(1,2); + goto jump_to_p2; + }else{ + rc = sqlite3BtreeLast(pC->uc.pCursor, &res); + if( rc!=SQLITE_OK ) goto abort_due_to_error; + goto seek_not_found; + } + }else /* If the approximation iKey is larger than the actual real search ** term, substitute >= for > and < for <=. e.g. if the search term @@ -87342,7 +88127,7 @@ case OP_SeekGT: { /* jump, in3, group */ assert( (OP_SeekLT & 0x0001)==(OP_SeekGE & 0x0001) ); if( (oc & 0x0001)==(OP_SeekLT & 0x0001) ) oc++; } - } + } rc = sqlite3BtreeMovetoUnpacked(pC->uc.pCursor, 0, (u64)iKey, 0, &res); pC->movetoTarget = iKey; /* Used by OP_Delete */ if( rc!=SQLITE_OK ){ @@ -87396,8 +88181,6 @@ case OP_SeekGT: { /* jump, in3, group */ goto seek_not_found; } } - pC->deferredMoveto = 0; - pC->cacheStatus = CACHE_STALE; #ifdef SQLITE_TEST sqlite3_search_count++; #endif @@ -87697,23 +88480,29 @@ case OP_SeekRowid: { /* jump, in3 */ u64 iKey; pIn3 = &aMem[pOp->p3]; - if( (pIn3->flags & MEM_Int)==0 ){ - /* Make sure pIn3->u.i contains a valid integer representation of - ** the key value, but do not change the datatype of the register, as - ** other parts of the perpared statement might be depending on the - ** current datatype. */ - u16 origFlags = pIn3->flags; - int isNotInt; - applyAffinity(pIn3, SQLITE_AFF_NUMERIC, encoding); - isNotInt = (pIn3->flags & MEM_Int)==0; - pIn3->flags = origFlags; - if( isNotInt ) goto jump_to_p2; + testcase( pIn3->flags & MEM_Int ); + testcase( pIn3->flags & MEM_IntReal ); + testcase( pIn3->flags & MEM_Real ); + testcase( (pIn3->flags & (MEM_Str|MEM_Int))==MEM_Str ); + if( (pIn3->flags & (MEM_Int|MEM_IntReal))==0 ){ + /* If pIn3->u.i does not contain an integer, compute iKey as the + ** integer value of pIn3. Jump to P2 if pIn3 cannot be converted + ** into an integer without loss of information. Take care to avoid + ** changing the datatype of pIn3, however, as it is used by other + ** parts of the prepared statement. */ + Mem x = pIn3[0]; + applyAffinity(&x, SQLITE_AFF_NUMERIC, encoding); + if( (x.flags & MEM_Int)==0 ) goto jump_to_p2; + iKey = x.u.i; + goto notExistsWithKey; } /* Fall through into OP_NotExists */ case OP_NotExists: /* jump, in3 */ pIn3 = &aMem[pOp->p3]; assert( (pIn3->flags & MEM_Int)!=0 || pOp->opcode==OP_SeekRowid ); assert( pOp->p1>=0 && pOp->p1nCursor ); + iKey = pIn3->u.i; +notExistsWithKey: pC = p->apCsr[pOp->p1]; assert( pC!=0 ); #ifdef SQLITE_DEBUG @@ -87724,7 +88513,6 @@ case OP_NotExists: /* jump, in3 */ pCrsr = pC->uc.pCursor; assert( pCrsr!=0 ); res = 0; - iKey = pIn3->u.i; rc = sqlite3BtreeMovetoUnpacked(pCrsr, 0, iKey, 0, &res); assert( rc==SQLITE_OK || res==0 ); pC->movetoTarget = iKey; /* Used by OP_Delete */ @@ -87935,14 +88723,7 @@ case OP_NewRowid: { /* out2 */ ** This instruction only works on tables. The equivalent instruction ** for indices is OP_IdxInsert. */ -/* Opcode: InsertInt P1 P2 P3 P4 P5 -** Synopsis: intkey=P3 data=r[P2] -** -** This works exactly like OP_Insert except that the key is the -** integer value P3, not the value of the integer stored in register P3. -*/ -case OP_Insert: -case OP_InsertInt: { +case OP_Insert: { Mem *pData; /* MEM cell holding data for the record to be inserted */ Mem *pKey; /* MEM cell holding key for the record */ VdbeCursor *pC; /* Cursor to table into which insert is written */ @@ -87963,16 +88744,11 @@ case OP_InsertInt: { REGISTER_TRACE(pOp->p2, pData); sqlite3VdbeIncrWriteCounter(p, pC); - if( pOp->opcode==OP_Insert ){ - pKey = &aMem[pOp->p3]; - assert( pKey->flags & MEM_Int ); - assert( memIsValid(pKey) ); - REGISTER_TRACE(pOp->p3, pKey); - x.nKey = pKey->u.i; - }else{ - assert( pOp->opcode==OP_InsertInt ); - x.nKey = pOp->p3; - } + pKey = &aMem[pOp->p3]; + assert( pKey->flags & MEM_Int ); + assert( memIsValid(pKey) ); + REGISTER_TRACE(pOp->p3, pKey); + x.nKey = pKey->u.i; if( pOp->p4type==P4_TABLE && HAS_UPDATE_HOOK(db) ){ assert( pC->iDb>=0 ); @@ -88084,7 +88860,7 @@ case OP_Delete: { ** OP_Delete will have also set the pC->movetoTarget field to the rowid of ** the row that is being deleted */ i64 iKey = sqlite3BtreeIntegerKey(pC->uc.pCursor); - assert( pC->movetoTarget==iKey ); + assert( CORRUPT_DB || pC->movetoTarget==iKey ); } #endif @@ -88492,7 +89268,7 @@ case OP_Sort: { /* jump */ p->aCounter[SQLITE_STMTSTATUS_SORT]++; /* Fall through into OP_Rewind */ } -/* Opcode: Rewind P1 P2 * * P5 +/* Opcode: Rewind P1 P2 * * * ** ** The next use of the Rowid or Column or Next instruction for P1 ** will refer to the first entry in the database table or index. @@ -88500,10 +89276,6 @@ case OP_Sort: { /* jump */ ** If the table or index is not empty, fall through to the following ** instruction. ** -** If P5 is non-zero and the table is not empty, then the "skip-next" -** flag is set on the cursor so that the next OP_Next instruction -** executed on it is a no-op. -** ** This opcode leaves the cursor configured to move in forward order, ** from the beginning toward the end. In other words, the cursor is ** configured to use Next, not Prev. @@ -88514,6 +89286,7 @@ case OP_Rewind: { /* jump */ int res; assert( pOp->p1>=0 && pOp->p1nCursor ); + assert( pOp->p5==0 ); pC = p->apCsr[pOp->p1]; assert( pC!=0 ); assert( isSorter(pC)==(pOp->opcode==OP_SorterSort) ); @@ -88528,9 +89301,6 @@ case OP_Rewind: { /* jump */ pCrsr = pC->uc.pCursor; assert( pCrsr ); rc = sqlite3BtreeFirst(pCrsr, &res); -#ifndef SQLITE_OMIT_WINDOWFUNC - if( pOp->p5 ) sqlite3BtreeSkipNext(pCrsr); -#endif pC->deferredMoveto = 0; pC->cacheStatus = CACHE_STALE; } @@ -88624,11 +89394,12 @@ case OP_Next: /* jump */ ** The Prev opcode is only used after SeekLT, SeekLE, and Last. */ assert( pOp->opcode!=OP_Next || pC->seekOp==OP_SeekGT || pC->seekOp==OP_SeekGE - || pC->seekOp==OP_Rewind || pC->seekOp==OP_Found - || pC->seekOp==OP_NullRow|| pC->seekOp==OP_SeekRowid); + || pC->seekOp==OP_Rewind || pC->seekOp==OP_Found + || pC->seekOp==OP_NullRow|| pC->seekOp==OP_SeekRowid + || pC->seekOp==OP_IfNoHope); assert( pOp->opcode!=OP_Prev || pC->seekOp==OP_SeekLT || pC->seekOp==OP_SeekLE - || pC->seekOp==OP_Last + || pC->seekOp==OP_Last || pC->seekOp==OP_IfNoHope || pC->seekOp==OP_NullRow); rc = pOp->p4.xAdvance(pC->uc.pCursor, pOp->p3); @@ -89147,7 +89918,7 @@ case OP_ParseSchema: { initData.pzErrMsg = &p->zErrMsg; initData.mInitFlags = 0; zSql = sqlite3MPrintf(db, - "SELECT name, rootpage, sql FROM '%q'.%s WHERE %s ORDER BY rowid", + "SELECT*FROM\"%w\".%s WHERE %s ORDER BY rowid", db->aDb[iDb].zDbSName, zMaster, pOp->p4.z); if( zSql==0 ){ rc = SQLITE_NOMEM_BKPT; @@ -89540,8 +90311,7 @@ case OP_Program: { /* jump */ } #endif pOp = &aOp[-1]; - - break; + goto check_for_interrupt; } /* Opcode: Param P1 P2 * * * @@ -89913,6 +90683,7 @@ case OP_AggFinal: { assert( (pMem->flags & ~(MEM_Null|MEM_Agg))==0 ); #ifndef SQLITE_OMIT_WINDOWFUNC if( pOp->p3 ){ + memAboutToChange(p, &aMem[pOp->p3]); rc = sqlite3VdbeMemAggValue(pMem, &aMem[pOp->p3], pOp->p4.pFunc); pMem = &aMem[pOp->p3]; }else @@ -90950,7 +91721,16 @@ default: { /* This is really OP_Noop, OP_Explain */ ** release the mutexes on btrees that were acquired at the ** top. */ vdbe_return: - testcase( nVmStep>0 ); +#ifndef SQLITE_OMIT_PROGRESS_CALLBACK + while( nVmStep>=nProgressLimit && db->xProgress!=0 ){ + nProgressLimit += db->nProgressOps; + if( db->xProgress(db->pProgressArg) ){ + nProgressLimit = 0xffffffff; + rc = SQLITE_INTERRUPT; + goto abort_due_to_error; + } + } +#endif p->aCounter[SQLITE_STMTSTATUS_VM_STEP] += (int)nVmStep; sqlite3VdbeLeave(p); assert( rc!=SQLITE_OK || nExtraDelete==0 @@ -91345,11 +92125,12 @@ SQLITE_API int sqlite3_blob_close(sqlite3_blob *pBlob){ sqlite3 *db; if( p ){ + sqlite3_stmt *pStmt = p->pStmt; db = p->db; sqlite3_mutex_enter(db->mutex); - rc = sqlite3_finalize(p->pStmt); sqlite3DbFree(db, p); sqlite3_mutex_leave(db->mutex); + rc = sqlite3_finalize(pStmt); }else{ rc = SQLITE_OK; } @@ -92037,7 +92818,7 @@ static int vdbePmaReadBlob( /* Extend the p->aAlloc[] allocation if required. */ if( p->nAllocnAlloc*2); + sqlite3_int64 nNew = MAX(128, 2*(sqlite3_int64)p->nAlloc); while( nByte>nNew ) nNew = nNew*2; aNew = sqlite3Realloc(p->aAlloc, nNew); if( !aNew ) return SQLITE_NOMEM_BKPT; @@ -92329,7 +93110,8 @@ static int vdbeSorterCompareText( ); } }else{ - if( pTask->pSorter->pKeyInfo->aSortOrder[0] ){ + assert( !(pTask->pSorter->pKeyInfo->aSortFlags[0]&KEYINFO_ORDER_BIGNULL) ); + if( pTask->pSorter->pKeyInfo->aSortFlags[0] ){ res = res * -1; } } @@ -92397,7 +93179,8 @@ static int vdbeSorterCompareInt( pTask, pbKey2Cached, pKey1, nKey1, pKey2, nKey2 ); } - }else if( pTask->pSorter->pKeyInfo->aSortOrder[0] ){ + }else if( pTask->pSorter->pKeyInfo->aSortFlags[0] ){ + assert( !(pTask->pSorter->pKeyInfo->aSortFlags[0]&KEYINFO_ORDER_BIGNULL) ); res = res * -1; } @@ -92512,6 +93295,7 @@ SQLITE_PRIVATE int sqlite3VdbeSorterInit( if( pKeyInfo->nAllField<13 && (pKeyInfo->aColl[0]==0 || pKeyInfo->aColl[0]==db->pDfltColl) + && (pKeyInfo->aSortFlags[0] & KEYINFO_ORDER_BIGNULL)==0 ){ pSorter->typeMask = SORTER_TYPE_INTEGER | SORTER_TYPE_TEXT; } @@ -93228,13 +94012,16 @@ static int vdbeSorterFlushPMA(VdbeSorter *pSorter){ rc = vdbeSorterListToPMA(&pSorter->aTask[nWorker], &pSorter->list); }else{ /* Launch a background thread for this operation */ - u8 *aMem = pTask->list.aMemory; - void *pCtx = (void*)pTask; + u8 *aMem; + void *pCtx; + assert( pTask!=0 ); assert( pTask->pThread==0 && pTask->bDone==0 ); assert( pTask->list.pList==0 ); assert( pTask->list.aMemory==0 || pSorter->list.aMemory!=0 ); + aMem = pTask->list.aMemory; + pCtx = (void*)pTask; pSorter->iPrev = (u8)(pTask - pSorter->aTask); pTask->list = pSorter->list; pSorter->list.pList = 0; @@ -93328,15 +94115,19 @@ SQLITE_PRIVATE int sqlite3VdbeSorterWrite( if( nMin>pSorter->nMemory ){ u8 *aNew; - int iListOff = (u8*)pSorter->list.pList - pSorter->list.aMemory; - int nNew = pSorter->nMemory * 2; + sqlite3_int64 nNew = 2 * (sqlite3_int64)pSorter->nMemory; + int iListOff = -1; + if( pSorter->list.pList ){ + iListOff = (u8*)pSorter->list.pList - pSorter->list.aMemory; + } while( nNew < nMin ) nNew = nNew*2; if( nNew > pSorter->mxPmaSize ) nNew = pSorter->mxPmaSize; if( nNew < nMin ) nNew = nMin; - aNew = sqlite3Realloc(pSorter->list.aMemory, nNew); if( !aNew ) return SQLITE_NOMEM_BKPT; - pSorter->list.pList = (SorterRecord*)&aNew[iListOff]; + if( iListOff>=0 ){ + pSorter->list.pList = (SorterRecord*)&aNew[iListOff]; + } pSorter->list.aMemory = aNew; pSorter->nMemory = nNew; } @@ -94354,14 +95145,9 @@ static int memjrnlRead( int iChunkOffset; FileChunk *pChunk; -#if defined(SQLITE_ENABLE_ATOMIC_WRITE) \ - || defined(SQLITE_ENABLE_BATCH_ATOMIC_WRITE) if( (iAmt+iOfst)>p->endpoint.iOffset ){ return SQLITE_IOERR_SHORT_READ; } -#endif - - assert( (iAmt+iOfst)<=p->endpoint.iOffset ); assert( p->readpoint.iOffset==0 || p->readpoint.pChunk!=0 ); if( p->readpoint.iOffset!=iOfst || iOfst==0 ){ sqlite3_int64 iOff = 0; @@ -94720,9 +95506,22 @@ SQLITE_PRIVATE int sqlite3JournalSize(sqlite3_vfs *pVfs){ static int walkWindowList(Walker *pWalker, Window *pList){ Window *pWin; for(pWin=pList; pWin; pWin=pWin->pNextWin){ - if( sqlite3WalkExprList(pWalker, pWin->pOrderBy) ) return WRC_Abort; - if( sqlite3WalkExprList(pWalker, pWin->pPartition) ) return WRC_Abort; - if( sqlite3WalkExpr(pWalker, pWin->pFilter) ) return WRC_Abort; + int rc; + rc = sqlite3WalkExprList(pWalker, pWin->pOrderBy); + if( rc ) return WRC_Abort; + rc = sqlite3WalkExprList(pWalker, pWin->pPartition); + if( rc ) return WRC_Abort; + rc = sqlite3WalkExpr(pWalker, pWin->pFilter); + if( rc ) return WRC_Abort; + + /* The next two are purely for calls to sqlite3RenameExprUnmap() + ** within sqlite3WindowOffsetExpr(). Because of constraints imposed + ** by sqlite3WindowOffsetExpr(), they can never fail. The results do + ** not matter anyhow. */ + rc = sqlite3WalkExpr(pWalker, pWin->pStart); + if( NEVER(rc) ) return WRC_Abort; + rc = sqlite3WalkExpr(pWalker, pWin->pEnd); + if( NEVER(rc) ) return WRC_Abort; } return WRC_Continue; } @@ -94758,18 +95557,22 @@ static SQLITE_NOINLINE int walkExpr(Walker *pWalker, Expr *pExpr){ if( pExpr->pLeft && walkExpr(pWalker, pExpr->pLeft) ) return WRC_Abort; assert( pExpr->x.pList==0 || pExpr->pRight==0 ); if( pExpr->pRight ){ + assert( !ExprHasProperty(pExpr, EP_WinFunc) ); pExpr = pExpr->pRight; continue; }else if( ExprHasProperty(pExpr, EP_xIsSelect) ){ + assert( !ExprHasProperty(pExpr, EP_WinFunc) ); if( sqlite3WalkSelect(pWalker, pExpr->x.pSelect) ) return WRC_Abort; - }else if( pExpr->x.pList ){ - if( sqlite3WalkExprList(pWalker, pExpr->x.pList) ) return WRC_Abort; - } + }else{ + if( pExpr->x.pList ){ + if( sqlite3WalkExprList(pWalker, pExpr->x.pList) ) return WRC_Abort; + } #ifndef SQLITE_OMIT_WINDOWFUNC - if( ExprHasProperty(pExpr, EP_WinFunc) ){ - if( walkWindowList(pWalker, pExpr->y.pWin) ) return WRC_Abort; - } + if( ExprHasProperty(pExpr, EP_WinFunc) ){ + if( walkWindowList(pWalker, pExpr->y.pWin) ) return WRC_Abort; + } #endif + } } break; } @@ -94811,8 +95614,9 @@ SQLITE_PRIVATE int sqlite3WalkSelectExpr(Walker *pWalker, Select *p){ { Parse *pParse = pWalker->pParse; if( pParse && IN_RENAME_OBJECT ){ + /* The following may return WRC_Abort if there are unresolvable + ** symbols (e.g. a table that does not exist) in a window definition. */ int rc = walkWindowList(pWalker, p->pWinDefn); - assert( rc==WRC_Continue ); return rc; } } @@ -94984,6 +95788,13 @@ static void resolveAlias( pExpr->u.zToken = sqlite3DbStrDup(db, pExpr->u.zToken); pExpr->flags |= EP_MemToken; } + if( ExprHasProperty(pExpr, EP_WinFunc) ){ + if( pExpr->y.pWin!=0 ){ + pExpr->y.pWin->pOwner = pExpr; + }else{ + assert( db->mallocFailed ); + } + } sqlite3DbFree(db, pDup); } ExprSetProperty(pExpr, EP_Alias); @@ -95036,6 +95847,23 @@ SQLITE_PRIVATE int sqlite3MatchSpanName( return 1; } +/* +** Return TRUE if the double-quoted string mis-feature should be supported. +*/ +static int areDoubleQuotedStringsEnabled(sqlite3 *db, NameContext *pTopNC){ + if( db->init.busy ) return 1; /* Always support for legacy schemas */ + if( pTopNC->ncFlags & NC_IsDDL ){ + /* Currently parsing a DDL statement */ + if( sqlite3WritableSchema(db) && (db->flags & SQLITE_DqsDML)!=0 ){ + return 1; + } + return (db->flags & SQLITE_DqsDDL)!=0; + }else{ + /* Currently parsing a DML statement */ + return (db->flags & SQLITE_DqsDML)!=0; + } +} + /* ** Given the name of a column of the form X.Y.Z or Y.Z or just Z, look up ** that name in the set of source tables in pSrcList and make the pExpr @@ -95252,7 +96080,7 @@ static int lookupName( { #ifndef SQLITE_OMIT_TRIGGER if( iCol<0 ){ - pExpr->affinity = SQLITE_AFF_INTEGER; + pExpr->affExpr = SQLITE_AFF_INTEGER; }else if( pExpr->iTable==0 ){ testcase( iCol==31 ); testcase( iCol==32 ); @@ -95284,7 +96112,7 @@ static int lookupName( ){ cnt = 1; pExpr->iColumn = -1; - pExpr->affinity = SQLITE_AFF_INTEGER; + pExpr->affExpr = SQLITE_AFF_INTEGER; } /* @@ -95323,6 +96151,10 @@ static int lookupName( sqlite3ErrorMsg(pParse, "misuse of aliased aggregate %s", zAs); return WRC_Abort; } + if( (pNC->ncFlags&NC_AllowWin)==0 && ExprHasProperty(pOrig, EP_Win) ){ + sqlite3ErrorMsg(pParse, "misuse of aliased window function %s",zAs); + return WRC_Abort; + } if( sqlite3ExprVectorSize(pOrig)!=1 ){ sqlite3ErrorMsg(pParse, "row value misused"); return WRC_Abort; @@ -95360,7 +96192,9 @@ static int lookupName( */ if( cnt==0 && zTab==0 ){ assert( pExpr->op==TK_ID ); - if( ExprHasProperty(pExpr,EP_DblQuoted) ){ + if( ExprHasProperty(pExpr,EP_DblQuoted) + && areDoubleQuotedStringsEnabled(db, pTopNC) + ){ /* If a double-quoted identifier does not match any known column name, ** then treat it as a string. ** @@ -95554,7 +96388,7 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ pExpr->y.pTab = pItem->pTab; pExpr->iTable = pItem->iCursor; pExpr->iColumn = -1; - pExpr->affinity = SQLITE_AFF_INTEGER; + pExpr->affExpr = SQLITE_AFF_INTEGER; break; } #endif /* defined(SQLITE_ENABLE_UPDATE_DELETE_LIMIT) @@ -95613,7 +96447,10 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ const char *zId; /* The function name. */ FuncDef *pDef; /* Information about the function */ u8 enc = ENC(pParse->db); /* The database encoding */ - + int savedAllowFlags = (pNC->ncFlags & (NC_AllowAgg | NC_AllowWin)); +#ifndef SQLITE_OMIT_WINDOWFUNC + Window *pWin = (IsWindowFunc(pExpr) ? pExpr->y.pWin : 0); +#endif assert( !ExprHasProperty(pExpr, EP_xIsSelect) ); zId = pExpr->u.zToken; nId = sqlite3Strlen30(zId); @@ -95628,7 +96465,7 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ }else{ is_agg = pDef->xFinalize!=0; if( pDef->funcFlags & SQLITE_FUNC_UNLIKELY ){ - ExprSetProperty(pExpr, EP_Unlikely|EP_Skip); + ExprSetProperty(pExpr, EP_Unlikely); if( n==2 ){ pExpr->iTable = exprProbability(pList->a[1].pExpr); if( pExpr->iTable<0 ){ @@ -95685,6 +96522,15 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ ** SQL is being compiled using sqlite3NestedParse() */ no_such_func = 1; pDef = 0; + }else + if( (pDef->funcFlags & SQLITE_FUNC_DIRECT)!=0 + && ExprHasProperty(pExpr, EP_Indirect) + && !IN_RENAME_OBJECT + ){ + /* Functions tagged with SQLITE_DIRECTONLY may not be used + ** inside of triggers and views */ + sqlite3ErrorMsg(pParse, "%s() prohibited in triggers and views", + pDef->zName); } } @@ -95694,18 +96540,18 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ || (pDef->xValue==0 && pDef->xInverse==0) || (pDef->xValue && pDef->xInverse && pDef->xSFunc && pDef->xFinalize) ); - if( pDef && pDef->xValue==0 && ExprHasProperty(pExpr, EP_WinFunc) ){ + if( pDef && pDef->xValue==0 && pWin ){ sqlite3ErrorMsg(pParse, "%.*s() may not be used as a window function", nId, zId ); pNC->nErr++; }else if( (is_agg && (pNC->ncFlags & NC_AllowAgg)==0) - || (is_agg && (pDef->funcFlags&SQLITE_FUNC_WINDOW) && !pExpr->y.pWin) - || (is_agg && pExpr->y.pWin && (pNC->ncFlags & NC_AllowWin)==0) + || (is_agg && (pDef->funcFlags&SQLITE_FUNC_WINDOW) && !pWin) + || (is_agg && pWin && (pNC->ncFlags & NC_AllowWin)==0) ){ const char *zType; - if( (pDef->funcFlags & SQLITE_FUNC_WINDOW) || pExpr->y.pWin ){ + if( (pDef->funcFlags & SQLITE_FUNC_WINDOW) || pWin ){ zType = "window"; }else{ zType = "aggregate"; @@ -95733,49 +96579,69 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ nId, zId); pNC->nErr++; } +#ifndef SQLITE_OMIT_WINDOWFUNC + else if( is_agg==0 && ExprHasProperty(pExpr, EP_WinFunc) ){ + sqlite3ErrorMsg(pParse, + "FILTER may not be used with non-aggregate %.*s()", + nId, zId + ); + pNC->nErr++; + } +#endif if( is_agg ){ + /* Window functions may not be arguments of aggregate functions. + ** Or arguments of other window functions. But aggregate functions + ** may be arguments for window functions. */ #ifndef SQLITE_OMIT_WINDOWFUNC - pNC->ncFlags &= ~(pExpr->y.pWin ? NC_AllowWin : NC_AllowAgg); + pNC->ncFlags &= ~(NC_AllowWin | (!pWin ? NC_AllowAgg : 0)); #else pNC->ncFlags &= ~NC_AllowAgg; #endif } } +#ifndef SQLITE_OMIT_WINDOWFUNC + else if( ExprHasProperty(pExpr, EP_WinFunc) ){ + is_agg = 1; + } +#endif sqlite3WalkExprList(pWalker, pList); if( is_agg ){ #ifndef SQLITE_OMIT_WINDOWFUNC - if( pExpr->y.pWin ){ + if( pWin ){ Select *pSel = pNC->pWinSelect; - sqlite3WindowUpdate(pParse, pSel->pWinDefn, pExpr->y.pWin, pDef); - sqlite3WalkExprList(pWalker, pExpr->y.pWin->pPartition); - sqlite3WalkExprList(pWalker, pExpr->y.pWin->pOrderBy); - sqlite3WalkExpr(pWalker, pExpr->y.pWin->pFilter); - if( 0==pSel->pWin - || 0==sqlite3WindowCompare(pParse, pSel->pWin, pExpr->y.pWin) - ){ - pExpr->y.pWin->pNextWin = pSel->pWin; - pSel->pWin = pExpr->y.pWin; + assert( pWin==pExpr->y.pWin ); + if( IN_RENAME_OBJECT==0 ){ + sqlite3WindowUpdate(pParse, pSel->pWinDefn, pWin, pDef); } - pNC->ncFlags |= NC_AllowWin; + sqlite3WalkExprList(pWalker, pWin->pPartition); + sqlite3WalkExprList(pWalker, pWin->pOrderBy); + sqlite3WalkExpr(pWalker, pWin->pFilter); + sqlite3WindowLink(pSel, pWin); + pNC->ncFlags |= NC_HasWin; }else #endif /* SQLITE_OMIT_WINDOWFUNC */ { NameContext *pNC2 = pNC; pExpr->op = TK_AGG_FUNCTION; pExpr->op2 = 0; +#ifndef SQLITE_OMIT_WINDOWFUNC + if( ExprHasProperty(pExpr, EP_WinFunc) ){ + sqlite3WalkExpr(pWalker, pExpr->y.pWin->pFilter); + } +#endif while( pNC2 && !sqlite3FunctionUsesThisSrc(pExpr, pNC2->pSrcList) ){ pExpr->op2++; pNC2 = pNC2->pNext; } - assert( pDef!=0 ); - if( pNC2 ){ + assert( pDef!=0 || IN_RENAME_OBJECT ); + if( pNC2 && pDef ){ assert( SQLITE_FUNC_MINMAX==NC_MinMaxAgg ); testcase( (pDef->funcFlags & SQLITE_FUNC_MINMAX)!=0 ); pNC2->ncFlags |= NC_HasAgg | (pDef->funcFlags & SQLITE_FUNC_MINMAX); } - pNC->ncFlags |= NC_AllowAgg; } + pNC->ncFlags |= savedAllowFlags; } /* FIX ME: Compute pExpr->affinity based on the expected return ** type of the function @@ -95806,11 +96672,11 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ } case TK_IS: case TK_ISNOT: { - Expr *pRight; + Expr *pRight = sqlite3ExprSkipCollateAndLikely(pExpr->pRight); assert( !ExprHasProperty(pExpr, EP_Reduced) ); /* Handle special cases of "x IS TRUE", "x IS FALSE", "x IS NOT TRUE", ** and "x IS NOT FALSE". */ - if( (pRight = pExpr->pRight)->op==TK_ID ){ + if( pRight->op==TK_ID ){ int rc = resolveExprStep(pWalker, pRight); if( rc==WRC_Abort ) return WRC_Abort; if( pRight->op==TK_TRUEFALSE ){ @@ -96017,7 +96883,7 @@ static int resolveCompoundOrderBy( int iCol = -1; Expr *pE, *pDup; if( pItem->done ) continue; - pE = sqlite3ExprSkipCollate(pItem->pExpr); + pE = sqlite3ExprSkipCollateAndLikely(pItem->pExpr); if( sqlite3ExprIsInteger(pE, &iCol) ){ if( iCol<=0 || iCol>pEList->nExpr ){ resolveOutOfRangeError(pParse, "ORDER", i+1, pEList->nExpr); @@ -96111,7 +96977,7 @@ SQLITE_PRIVATE int sqlite3ResolveOrderGroupBy( ExprList *pEList; struct ExprList_item *pItem; - if( pOrderBy==0 || pParse->db->mallocFailed ) return 0; + if( pOrderBy==0 || pParse->db->mallocFailed || IN_RENAME_OBJECT ) return 0; if( pOrderBy->nExpr>db->aLimit[SQLITE_LIMIT_COLUMN] ){ sqlite3ErrorMsg(pParse, "too many terms in %s BY clause", zType); return 1; @@ -96131,6 +96997,36 @@ SQLITE_PRIVATE int sqlite3ResolveOrderGroupBy( return 0; } +#ifndef SQLITE_OMIT_WINDOWFUNC +/* +** Walker callback for windowRemoveExprFromSelect(). +*/ +static int resolveRemoveWindowsCb(Walker *pWalker, Expr *pExpr){ + UNUSED_PARAMETER(pWalker); + if( ExprHasProperty(pExpr, EP_WinFunc) ){ + Window *pWin = pExpr->y.pWin; + sqlite3WindowUnlinkFromSelect(pWin); + } + return WRC_Continue; +} + +/* +** Remove any Window objects owned by the expression pExpr from the +** Select.pWin list of Select object pSelect. +*/ +static void windowRemoveExprFromSelect(Select *pSelect, Expr *pExpr){ + if( pSelect->pWin ){ + Walker sWalker; + memset(&sWalker, 0, sizeof(Walker)); + sWalker.xExprCallback = resolveRemoveWindowsCb; + sWalker.u.pSelect = pSelect; + sqlite3WalkExpr(&sWalker, pExpr); + } +} +#else +# define windowRemoveExprFromSelect(a, b) +#endif /* SQLITE_OMIT_WINDOWFUNC */ + /* ** pOrderBy is an ORDER BY or GROUP BY clause in SELECT statement pSelect. ** The Name context of the SELECT statement is pNC. zType is either @@ -96166,7 +97062,7 @@ static int resolveOrderGroupBy( pParse = pNC->pParse; for(i=0, pItem=pOrderBy->a; inExpr; i++, pItem++){ Expr *pE = pItem->pExpr; - Expr *pE2 = sqlite3ExprSkipCollate(pE); + Expr *pE2 = sqlite3ExprSkipCollateAndLikely(pE); if( zType[0]!='G' ){ iCol = resolveAsName(pParse, pSelect->pEList, pE2); if( iCol>0 ){ @@ -96197,19 +97093,10 @@ static int resolveOrderGroupBy( } for(j=0; jpEList->nExpr; j++){ if( sqlite3ExprCompare(0, pE, pSelect->pEList->a[j].pExpr, -1)==0 ){ -#ifndef SQLITE_OMIT_WINDOWFUNC - if( ExprHasProperty(pE, EP_WinFunc) ){ - /* Since this window function is being changed into a reference - ** to the same window function the result set, remove the instance - ** of this window function from the Select.pWin list. */ - Window **pp; - for(pp=&pSelect->pWin; *pp; pp=&(*pp)->pNextWin){ - if( *pp==pE->y.pWin ){ - *pp = (*pp)->pNextWin; - } - } - } -#endif + /* Since this expresion is being changed into a reference + ** to an identical expression in the result set, remove all Window + ** objects belonging to the expression from the Select.pWin list. */ + windowRemoveExprFromSelect(pSelect, pE); pItem->u.x.iOrderByCol = j+1; } } @@ -96289,7 +97176,7 @@ static int resolveSelectStep(Walker *pWalker, Select *p){ */ for(i=0; ipSrc->nSrc; i++){ struct SrcList_item *pItem = &p->pSrc->a[i]; - if( pItem->pSelect ){ + if( pItem->pSelect && (pItem->pSelect->selFlags & SF_Resolved)==0 ){ NameContext *pNC; /* Used to iterate name contexts */ int nRef = 0; /* Refcount for pOuterNC and outer contexts */ const char *zSavedContext = pParse->zAuthContext; @@ -96421,6 +97308,7 @@ static int resolveSelectStep(Walker *pWalker, Select *p){ } } +#ifndef SQLITE_OMIT_WINDOWFUNC if( IN_RENAME_OBJECT ){ Window *pWin; for(pWin=p->pWinDefn; pWin; pWin=pWin->pNextWin){ @@ -96431,6 +97319,7 @@ static int resolveSelectStep(Walker *pWalker, Select *p){ } } } +#endif /* If this is part of a compound SELECT, check that it has the right ** number of expressions in the select list. */ @@ -96507,12 +97396,12 @@ SQLITE_PRIVATE int sqlite3ResolveExprNames( NameContext *pNC, /* Namespace to resolve expressions in. */ Expr *pExpr /* The expression to be analyzed. */ ){ - u16 savedHasAgg; + int savedHasAgg; Walker w; if( pExpr==0 ) return SQLITE_OK; - savedHasAgg = pNC->ncFlags & (NC_HasAgg|NC_MinMaxAgg); - pNC->ncFlags &= ~(NC_HasAgg|NC_MinMaxAgg); + savedHasAgg = pNC->ncFlags & (NC_HasAgg|NC_MinMaxAgg|NC_HasWin); + pNC->ncFlags &= ~(NC_HasAgg|NC_MinMaxAgg|NC_HasWin); w.pParse = pNC->pParse; w.xExprCallback = resolveExprStep; w.xSelectCallback = resolveSelectStep; @@ -96528,9 +97417,11 @@ SQLITE_PRIVATE int sqlite3ResolveExprNames( #if SQLITE_MAX_EXPR_DEPTH>0 w.pParse->nHeight -= pExpr->nHeight; #endif - if( pNC->ncFlags & NC_HasAgg ){ - ExprSetProperty(pExpr, EP_Agg); - } + assert( EP_Agg==NC_HasAgg ); + assert( EP_Win==NC_HasWin ); + testcase( pNC->ncFlags & NC_HasAgg ); + testcase( pNC->ncFlags & NC_HasWin ); + ExprSetProperty(pExpr, pNC->ncFlags & (NC_HasAgg|NC_HasWin) ); pNC->ncFlags |= savedHasAgg; return pNC->nErr>0 || w.pParse->nErr>0; } @@ -96619,7 +97510,7 @@ SQLITE_PRIVATE int sqlite3ResolveSelfReference( } sNC.pParse = pParse; sNC.pSrcList = &sSrc; - sNC.ncFlags = type; + sNC.ncFlags = type | NC_IsDDL; if( (rc = sqlite3ResolveExprNames(&sNC, pExpr))!=SQLITE_OK ) return rc; if( pList ) rc = sqlite3ResolveExprListNames(&sNC, pList); return rc; @@ -96673,8 +97564,11 @@ SQLITE_PRIVATE char sqlite3TableColumnAffinity(Table *pTab, int iCol){ */ SQLITE_PRIVATE char sqlite3ExprAffinity(Expr *pExpr){ int op; - pExpr = sqlite3ExprSkipCollate(pExpr); - if( pExpr->flags & EP_Generic ) return 0; + while( ExprHasProperty(pExpr, EP_Skip) ){ + assert( pExpr->op==TK_COLLATE ); + pExpr = pExpr->pLeft; + assert( pExpr!=0 ); + } op = pExpr->op; if( op==TK_SELECT ){ assert( pExpr->flags&EP_xIsSelect ); @@ -96696,7 +97590,7 @@ SQLITE_PRIVATE char sqlite3ExprAffinity(Expr *pExpr){ pExpr->pLeft->x.pSelect->pEList->a[pExpr->iColumn].pExpr ); } - return pExpr->affinity; + return pExpr->affExpr; } /* @@ -96731,11 +97625,23 @@ SQLITE_PRIVATE Expr *sqlite3ExprAddCollateString(Parse *pParse, Expr *pExpr, con } /* -** Skip over any TK_COLLATE operators and any unlikely() -** or likelihood() function at the root of an expression. +** Skip over any TK_COLLATE operators. */ SQLITE_PRIVATE Expr *sqlite3ExprSkipCollate(Expr *pExpr){ while( pExpr && ExprHasProperty(pExpr, EP_Skip) ){ + assert( pExpr->op==TK_COLLATE ); + pExpr = pExpr->pLeft; + } + return pExpr; +} + +/* +** Skip over any TK_COLLATE operators and/or any unlikely() +** or likelihood() or likely() functions at the root of an +** expression. +*/ +SQLITE_PRIVATE Expr *sqlite3ExprSkipCollateAndLikely(Expr *pExpr){ + while( pExpr && ExprHasProperty(pExpr, EP_Skip|EP_Unlikely) ){ if( ExprHasProperty(pExpr, EP_Unlikely) ){ assert( !ExprHasProperty(pExpr, EP_xIsSelect) ); assert( pExpr->x.pList->nExpr>0 ); @@ -96769,7 +97675,6 @@ SQLITE_PRIVATE CollSeq *sqlite3ExprCollSeq(Parse *pParse, Expr *pExpr){ Expr *p = pExpr; while( p ){ int op = p->op; - if( p->flags & EP_Generic ) break; if( op==TK_REGISTER ) op = p->op2; if( (op==TK_AGG_COLUMN || op==TK_COLUMN || op==TK_TRIGGER) && p->y.pTab!=0 @@ -96855,7 +97760,7 @@ SQLITE_PRIVATE int sqlite3ExprCollSeqMatch(Parse *pParse, Expr *pE1, Expr *pE2){ */ SQLITE_PRIVATE char sqlite3CompareAffinity(Expr *pExpr, char aff2){ char aff1 = sqlite3ExprAffinity(pExpr); - if( aff1 && aff2 ){ + if( aff1>SQLITE_AFF_NONE && aff2>SQLITE_AFF_NONE ){ /* Both sides of the comparison are columns. If one has numeric ** affinity, use that. Otherwise use no affinity. */ @@ -96864,15 +97769,10 @@ SQLITE_PRIVATE char sqlite3CompareAffinity(Expr *pExpr, char aff2){ }else{ return SQLITE_AFF_BLOB; } - }else if( !aff1 && !aff2 ){ - /* Neither side of the comparison is a column. Compare the - ** results directly. - */ - return SQLITE_AFF_BLOB; }else{ /* One side is a column, the other is not. Use the columns affinity. */ - assert( aff1==0 || aff2==0 ); - return (aff1 + aff2); + assert( aff1<=SQLITE_AFF_NONE || aff2<=SQLITE_AFF_NONE ); + return (aff1<=SQLITE_AFF_NONE ? aff2 : aff1) | SQLITE_AFF_NONE; } } @@ -96905,14 +97805,13 @@ static char comparisonAffinity(Expr *pExpr){ */ SQLITE_PRIVATE int sqlite3IndexAffinityOk(Expr *pExpr, char idx_affinity){ char aff = comparisonAffinity(pExpr); - switch( aff ){ - case SQLITE_AFF_BLOB: - return 1; - case SQLITE_AFF_TEXT: - return idx_affinity==SQLITE_AFF_TEXT; - default: - return sqlite3IsNumericAffinity(idx_affinity); + if( affiAgg = -1; if( pToken ){ if( nExtra==0 ){ - pNew->flags |= EP_IntValue|EP_Leaf; + pNew->flags |= EP_IntValue|EP_Leaf|(iValue?EP_IsTrue:EP_IsFalse); pNew->u.iValue = iValue; }else{ pNew->u.zToken = (char*)&pNew[1]; @@ -97479,20 +98378,16 @@ SQLITE_PRIVATE Expr *sqlite3PExpr( Expr *pRight /* Right operand */ ){ Expr *p; - if( op==TK_AND && pParse->nErr==0 && !IN_RENAME_OBJECT ){ - /* Take advantage of short-circuit false optimization for AND */ - p = sqlite3ExprAnd(pParse->db, pLeft, pRight); - }else{ - p = sqlite3DbMallocRawNN(pParse->db, sizeof(Expr)); - if( p ){ - memset(p, 0, sizeof(Expr)); - p->op = op & TKFLG_MASK; - p->iAgg = -1; - } + p = sqlite3DbMallocRawNN(pParse->db, sizeof(Expr)); + if( p ){ + memset(p, 0, sizeof(Expr)); + p->op = op & 0xff; + p->iAgg = -1; sqlite3ExprAttachSubtrees(pParse->db, p, pLeft, pRight); - } - if( p ) { sqlite3ExprCheckHeight(pParse, p->nHeight); + }else{ + sqlite3ExprDelete(pParse->db, pLeft); + sqlite3ExprDelete(pParse->db, pRight); } return p; } @@ -97513,33 +98408,6 @@ SQLITE_PRIVATE void sqlite3PExprAddSelect(Parse *pParse, Expr *pExpr, Select *pS } -/* -** If the expression is always either TRUE or FALSE (respectively), -** then return 1. If one cannot determine the truth value of the -** expression at compile-time return 0. -** -** This is an optimization. If is OK to return 0 here even if -** the expression really is always false or false (a false negative). -** But it is a bug to return 1 if the expression might have different -** boolean values in different circumstances (a false positive.) -** -** Note that if the expression is part of conditional for a -** LEFT JOIN, then we cannot determine at compile-time whether or not -** is it true or false, so always return 0. -*/ -static int exprAlwaysTrue(Expr *p){ - int v = 0; - if( ExprHasProperty(p, EP_FromJoin) ) return 0; - if( !sqlite3ExprIsInteger(p, &v) ) return 0; - return v!=0; -} -static int exprAlwaysFalse(Expr *p){ - int v = 0; - if( ExprHasProperty(p, EP_FromJoin) ) return 0; - if( !sqlite3ExprIsInteger(p, &v) ) return 0; - return v==0; -} - /* ** Join two expressions using an AND operator. If either expression is ** NULL, then just return the other expression. @@ -97548,19 +98416,18 @@ static int exprAlwaysFalse(Expr *p){ ** of returning an AND expression, just return a constant expression with ** a value of false. */ -SQLITE_PRIVATE Expr *sqlite3ExprAnd(sqlite3 *db, Expr *pLeft, Expr *pRight){ - if( pLeft==0 ){ +SQLITE_PRIVATE Expr *sqlite3ExprAnd(Parse *pParse, Expr *pLeft, Expr *pRight){ + sqlite3 *db = pParse->db; + if( pLeft==0 ){ return pRight; }else if( pRight==0 ){ return pLeft; - }else if( exprAlwaysFalse(pLeft) || exprAlwaysFalse(pRight) ){ - sqlite3ExprDelete(db, pLeft); - sqlite3ExprDelete(db, pRight); - return sqlite3ExprAlloc(db, TK_INTEGER, &sqlite3IntTokens[0], 0); + }else if( ExprAlwaysFalse(pLeft) || ExprAlwaysFalse(pRight) ){ + sqlite3ExprUnmapAndDelete(pParse, pLeft); + sqlite3ExprUnmapAndDelete(pParse, pRight); + return sqlite3Expr(db, TK_INTEGER, "0"); }else{ - Expr *pNew = sqlite3ExprAlloc(db, TK_AND, 0, 0); - sqlite3ExprAttachSubtrees(db, pNew, pLeft, pRight); - return pNew; + return sqlite3PExpr(pParse, TK_AND, pLeft, pRight); } } @@ -97697,15 +98564,18 @@ static SQLITE_NOINLINE void sqlite3ExprDeleteNN(sqlite3 *db, Expr *p){ assert( p->x.pList==0 || p->pRight==0 ); if( p->pLeft && p->op!=TK_SELECT_COLUMN ) sqlite3ExprDeleteNN(db, p->pLeft); if( p->pRight ){ + assert( !ExprHasProperty(p, EP_WinFunc) ); sqlite3ExprDeleteNN(db, p->pRight); }else if( ExprHasProperty(p, EP_xIsSelect) ){ + assert( !ExprHasProperty(p, EP_WinFunc) ); sqlite3SelectDelete(db, p->x.pSelect); }else{ sqlite3ExprListDelete(db, p->x.pList); - } - if( ExprHasProperty(p, EP_WinFunc) ){ - assert( p->op==TK_FUNCTION ); - sqlite3WindowDelete(db, p->y.pWin); +#ifndef SQLITE_OMIT_WINDOWFUNC + if( ExprHasProperty(p, EP_WinFunc) ){ + sqlite3WindowDelete(db, p->y.pWin); + } +#endif } } if( ExprHasProperty(p, EP_MemToken) ) sqlite3DbFree(db, p->u.zToken); @@ -97717,6 +98587,18 @@ SQLITE_PRIVATE void sqlite3ExprDelete(sqlite3 *db, Expr *p){ if( p ) sqlite3ExprDeleteNN(db, p); } +/* Invoke sqlite3RenameExprUnmap() and sqlite3ExprDelete() on the +** expression. +*/ +SQLITE_PRIVATE void sqlite3ExprUnmapAndDelete(Parse *pParse, Expr *p){ + if( p ){ + if( IN_RENAME_OBJECT ){ + sqlite3RenameExprUnmap(pParse, p); + } + sqlite3ExprDeleteNN(pParse->db, p); + } +} + /* ** Return the number of bytes allocated for the expression structure ** passed as the first argument. This is always one of EXPR_FULLSIZE, @@ -97728,16 +98610,6 @@ static int exprStructSize(Expr *p){ return EXPR_FULLSIZE; } -/* -** Copy the complete content of an Expr node, taking care not to read -** past the end of the structure for a reduced-size version of the source -** Expr. -*/ -static void exprNodeCopy(Expr *pDest, Expr *pSrc){ - memset(pDest, 0, sizeof(Expr)); - memcpy(pDest, pSrc, exprStructSize(pSrc)); -} - /* ** The dupedExpr*Size() routines each return the number of bytes required ** to store a copy of an expression or expression tree. They differ in @@ -97951,7 +98823,7 @@ static Expr *exprDup(sqlite3 *db, Expr *p, int dupFlags, u8 **pzBuffer){ static With *withDup(sqlite3 *db, With *p){ With *pRet = 0; if( p ){ - int nByte = sizeof(*p) + sizeof(p->a[0]) * (p->nCte-1); + sqlite3_int64 nByte = sizeof(*p) + sizeof(p->a[0]) * (p->nCte-1); pRet = sqlite3DbMallocZero(db, nByte); if( pRet ){ int i; @@ -97977,10 +98849,13 @@ static With *withDup(sqlite3 *db, With *p){ ** objects found there, assembling them onto the linked list at Select->pWin. */ static int gatherSelectWindowsCallback(Walker *pWalker, Expr *pExpr){ - if( pExpr->op==TK_FUNCTION && pExpr->y.pWin!=0 ){ - assert( ExprHasProperty(pExpr, EP_WinFunc) ); - pExpr->y.pWin->pNextWin = pWalker->u.pSelect->pWin; - pWalker->u.pSelect->pWin = pExpr->y.pWin; + if( pExpr->op==TK_FUNCTION && ExprHasProperty(pExpr, EP_WinFunc) ){ + Select *pSelect = pWalker->u.pSelect; + Window *pWin = pExpr->y.pWin; + assert( pWin ); + assert( IsWindowFunc(pExpr) ); + assert( pWin->ppThis==0 ); + sqlite3WindowLink(pSelect, pWin); } return WRC_Continue; } @@ -98054,8 +98929,9 @@ SQLITE_PRIVATE ExprList *sqlite3ExprListDup(sqlite3 *db, ExprList *p, int flags) } pItem->zName = sqlite3DbStrDup(db, pOldItem->zName); pItem->zSpan = sqlite3DbStrDup(db, pOldItem->zSpan); - pItem->sortOrder = pOldItem->sortOrder; + pItem->sortFlags = pOldItem->sortFlags; pItem->done = 0; + pItem->bNulls = pOldItem->bNulls; pItem->bSpanIsTab = pOldItem->bSpanIsTab; pItem->bSorterRef = pOldItem->bSorterRef; pItem->u = pOldItem->u; @@ -98166,7 +99042,7 @@ SQLITE_PRIVATE Select *sqlite3SelectDup(sqlite3 *db, Select *pDup, int flags){ #ifndef SQLITE_OMIT_WINDOWFUNC pNew->pWin = 0; pNew->pWinDefn = sqlite3WindowListDup(db, p->pWinDefn); - if( p->pWin ) gatherSelectWindows(pNew); + if( p->pWin && db->mallocFailed==0 ) gatherSelectWindows(pNew); #endif pNew->selId = p->selId; *pp = pNew; @@ -98216,7 +99092,7 @@ SQLITE_PRIVATE ExprList *sqlite3ExprListAppend( }else if( (pList->nExpr & (pList->nExpr-1))==0 ){ ExprList *pNew; pNew = sqlite3DbRealloc(db, pList, - sizeof(*pList)+(2*pList->nExpr - 1)*sizeof(pList->a[0])); + sizeof(*pList)+(2*(sqlite3_int64)pList->nExpr-1)*sizeof(pList->a[0])); if( pNew==0 ){ goto no_mem; } @@ -98275,6 +99151,10 @@ SQLITE_PRIVATE ExprList *sqlite3ExprListAppendVector( for(i=0; inId; i++){ Expr *pSubExpr = sqlite3ExprForVectorField(pParse, pExpr, i); + assert( pSubExpr!=0 || db->mallocFailed ); + assert( pSubExpr==0 || pSubExpr->iTable==0 ); + if( pSubExpr==0 ) continue; + pSubExpr->iTable = pColumns->nId; pList = sqlite3ExprListAppend(pParse, pList, pSubExpr); if( pList ){ assert( pList->nExpr==iFirst+i+1 ); @@ -98299,10 +99179,7 @@ SQLITE_PRIVATE ExprList *sqlite3ExprListAppendVector( } vector_append_error: - if( IN_RENAME_OBJECT ){ - sqlite3RenameExprUnmap(pParse, pExpr); - } - sqlite3ExprDelete(db, pExpr); + sqlite3ExprUnmapAndDelete(pParse, pExpr); sqlite3IdListDelete(db, pColumns); return pList; } @@ -98310,15 +99187,34 @@ SQLITE_PRIVATE ExprList *sqlite3ExprListAppendVector( /* ** Set the sort order for the last element on the given ExprList. */ -SQLITE_PRIVATE void sqlite3ExprListSetSortOrder(ExprList *p, int iSortOrder){ +SQLITE_PRIVATE void sqlite3ExprListSetSortOrder(ExprList *p, int iSortOrder, int eNulls){ + struct ExprList_item *pItem; if( p==0 ) return; - assert( SQLITE_SO_UNDEFINED<0 && SQLITE_SO_ASC>=0 && SQLITE_SO_DESC>0 ); assert( p->nExpr>0 ); - if( iSortOrder<0 ){ - assert( p->a[p->nExpr-1].sortOrder==SQLITE_SO_ASC ); - return; + + assert( SQLITE_SO_UNDEFINED<0 && SQLITE_SO_ASC==0 && SQLITE_SO_DESC>0 ); + assert( iSortOrder==SQLITE_SO_UNDEFINED + || iSortOrder==SQLITE_SO_ASC + || iSortOrder==SQLITE_SO_DESC + ); + assert( eNulls==SQLITE_SO_UNDEFINED + || eNulls==SQLITE_SO_ASC + || eNulls==SQLITE_SO_DESC + ); + + pItem = &p->a[p->nExpr-1]; + assert( pItem->bNulls==0 ); + if( iSortOrder==SQLITE_SO_UNDEFINED ){ + iSortOrder = SQLITE_SO_ASC; + } + pItem->sortFlags = (u8)iSortOrder; + + if( eNulls!=SQLITE_SO_UNDEFINED ){ + pItem->bNulls = 1; + if( iSortOrder!=eNulls ){ + pItem->sortFlags |= KEYINFO_ORDER_BIGNULL; + } } - p->a[p->nExpr-1].sortOrder = (u8)iSortOrder; } /* @@ -98450,6 +99346,7 @@ SQLITE_PRIVATE int sqlite3ExprIdToTrueFalse(Expr *pExpr){ || sqlite3StrICmp(pExpr->u.zToken, "false")==0) ){ pExpr->op = TK_TRUEFALSE; + ExprSetProperty(pExpr, pExpr->u.zToken[4]==0 ? EP_IsTrue : EP_IsFalse); return 1; } return 0; @@ -98460,12 +99357,40 @@ SQLITE_PRIVATE int sqlite3ExprIdToTrueFalse(Expr *pExpr){ ** and 0 if it is FALSE. */ SQLITE_PRIVATE int sqlite3ExprTruthValue(const Expr *pExpr){ + pExpr = sqlite3ExprSkipCollate((Expr*)pExpr); assert( pExpr->op==TK_TRUEFALSE ); assert( sqlite3StrICmp(pExpr->u.zToken,"true")==0 || sqlite3StrICmp(pExpr->u.zToken,"false")==0 ); return pExpr->u.zToken[4]==0; } +/* +** If pExpr is an AND or OR expression, try to simplify it by eliminating +** terms that are always true or false. Return the simplified expression. +** Or return the original expression if no simplification is possible. +** +** Examples: +** +** (x<10) AND true => (x<10) +** (x<10) AND false => false +** (x<10) AND (y=22 OR false) => (x<10) AND (y=22) +** (x<10) AND (y=22 OR true) => (x<10) +** (y=22) OR true => true +*/ +SQLITE_PRIVATE Expr *sqlite3ExprSimplifiedAndOr(Expr *pExpr){ + assert( pExpr!=0 ); + if( pExpr->op==TK_AND || pExpr->op==TK_OR ){ + Expr *pRight = sqlite3ExprSimplifiedAndOr(pExpr->pRight); + Expr *pLeft = sqlite3ExprSimplifiedAndOr(pExpr->pLeft); + if( ExprAlwaysTrue(pLeft) || ExprAlwaysFalse(pRight) ){ + pExpr = pExpr->op==TK_AND ? pRight : pLeft; + }else if( ExprAlwaysTrue(pRight) || ExprAlwaysFalse(pLeft) ){ + pExpr = pExpr->op==TK_AND ? pLeft : pRight; + } + } + return pExpr; +} + /* ** These routines are Walker callbacks used to check expressions to @@ -98710,7 +99635,7 @@ SQLITE_PRIVATE int sqlite3ExprContainsSubquery(Expr *p){ */ SQLITE_PRIVATE int sqlite3ExprIsInteger(Expr *p, int *pValue){ int rc = 0; - if( p==0 ) return 0; /* Can only happen following on OOM */ + if( NEVER(p==0) ) return 0; /* Used to only happen following on OOM */ /* If an expression is an integer literal that fits in a signed 32-bit ** integer, then the EP_IntValue flag will have already been set */ @@ -98788,27 +99713,30 @@ SQLITE_PRIVATE int sqlite3ExprCanBeNull(const Expr *p){ */ SQLITE_PRIVATE int sqlite3ExprNeedsNoAffinityChange(const Expr *p, char aff){ u8 op; + int unaryMinus = 0; if( aff==SQLITE_AFF_BLOB ) return 1; - while( p->op==TK_UPLUS || p->op==TK_UMINUS ){ p = p->pLeft; } + while( p->op==TK_UPLUS || p->op==TK_UMINUS ){ + if( p->op==TK_UMINUS ) unaryMinus = 1; + p = p->pLeft; + } op = p->op; if( op==TK_REGISTER ) op = p->op2; switch( op ){ case TK_INTEGER: { - return aff==SQLITE_AFF_INTEGER || aff==SQLITE_AFF_NUMERIC; + return aff>=SQLITE_AFF_NUMERIC; } case TK_FLOAT: { - return aff==SQLITE_AFF_REAL || aff==SQLITE_AFF_NUMERIC; + return aff>=SQLITE_AFF_NUMERIC; } case TK_STRING: { - return aff==SQLITE_AFF_TEXT; + return !unaryMinus && aff==SQLITE_AFF_TEXT; } case TK_BLOB: { - return 1; + return !unaryMinus; } case TK_COLUMN: { assert( p->iTable>=0 ); /* p cannot be part of a CHECK constraint */ - return p->iColumn<0 - && (aff==SQLITE_AFF_INTEGER || aff==SQLITE_AFF_NUMERIC); + return aff>=SQLITE_AFF_NUMERIC && p->iColumn<0; } default: { return 0; @@ -98991,7 +99919,7 @@ static int sqlite3InRhsIsConstant(Expr *pIn){ #ifndef SQLITE_OMIT_SUBQUERY SQLITE_PRIVATE int sqlite3FindInIndex( Parse *pParse, /* Parsing context */ - Expr *pX, /* The right-hand side (RHS) of the IN operator */ + Expr *pX, /* The IN expression */ u32 inFlags, /* IN_INDEX_LOOP, _MEMBERSHIP, and/or _NOOP_OK */ int *prRhsHasNull, /* Register holding NULL status. See notes */ int *aiMap, /* Mapping from Index fields to RHS fields */ @@ -99181,14 +100109,11 @@ SQLITE_PRIVATE int sqlite3FindInIndex( eType = IN_INDEX_EPH; if( inFlags & IN_INDEX_LOOP ){ pParse->nQueryLoop = 0; - if( pX->pLeft->iColumn<0 && !ExprHasProperty(pX, EP_xIsSelect) ){ - eType = IN_INDEX_ROWID; - } }else if( prRhsHasNull ){ *prRhsHasNull = rMayHaveNull = ++pParse->nMem; } assert( pX->op==TK_IN ); - sqlite3CodeRhsOfIN(pParse, pX, iTab, eType==IN_INDEX_ROWID); + sqlite3CodeRhsOfIN(pParse, pX, iTab); if( rMayHaveNull ){ sqlite3SetHasNullFlag(v, iTab, rMayHaveNull); } @@ -99289,12 +100214,6 @@ SQLITE_PRIVATE void sqlite3VectorErrorMsg(Parse *pParse, Expr *pExpr){ ** however the cursor number returned might not be the same, as it might ** have been duplicated using OP_OpenDup. ** -** If parameter isRowid is non-zero, then LHS of the IN operator is guaranteed -** to be a non-null integer. In this case, the ephemeral table can be an -** table B-Tree that keyed by only integers. The more general cases uses -** an index B-Tree which can have arbitrary keys, but is slower to both -** read and write. -** ** If the LHS expression ("x" in the examples) is a column value, or ** the SELECT statement returns a column value, then the affinity of that ** column is used to build the index keys. If both 'x' and the @@ -99306,8 +100225,7 @@ SQLITE_PRIVATE void sqlite3VectorErrorMsg(Parse *pParse, Expr *pExpr){ SQLITE_PRIVATE void sqlite3CodeRhsOfIN( Parse *pParse, /* Parsing context */ Expr *pExpr, /* The IN operator */ - int iTab, /* Use this cursor number */ - int isRowid /* If true, LHS is a rowid */ + int iTab /* Use this cursor number */ ){ int addrOnce = 0; /* Address of the OP_Once instruction at top */ int addr; /* Address of OP_OpenEphemeral instruction */ @@ -99360,14 +100278,12 @@ SQLITE_PRIVATE void sqlite3CodeRhsOfIN( /* Check to see if this is a vector IN operator */ pLeft = pExpr->pLeft; nVal = sqlite3ExprVectorSize(pLeft); - assert( !isRowid || nVal==1 ); /* Construct the ephemeral table that will contain the content of ** RHS of the IN operator. */ pExpr->iTable = iTab; - addr = sqlite3VdbeAddOp2(v, OP_OpenEphemeral, - pExpr->iTable, (isRowid?0:nVal)); + addr = sqlite3VdbeAddOp2(v, OP_OpenEphemeral, pExpr->iTable, nVal); #ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS if( ExprHasProperty(pExpr, EP_xIsSelect) ){ VdbeComment((v, "Result of SELECT %u", pExpr->x.pSelect->selId)); @@ -99375,7 +100291,7 @@ SQLITE_PRIVATE void sqlite3CodeRhsOfIN( VdbeComment((v, "RHS of IN operator")); } #endif - pKeyInfo = isRowid ? 0 : sqlite3KeyInfoAlloc(pParse->db, nVal, 1); + pKeyInfo = sqlite3KeyInfoAlloc(pParse->db, nVal, 1); if( ExprHasProperty(pExpr, EP_xIsSelect) ){ /* Case 1: expr IN (SELECT ...) @@ -99389,7 +100305,6 @@ SQLITE_PRIVATE void sqlite3CodeRhsOfIN( ExplainQueryPlan((pParse, 1, "%sLIST SUBQUERY %d", addrOnce?"":"CORRELATED ", pSelect->selId )); - assert( !isRowid ); /* If the LHS and RHS of the IN operator do not match, that ** error will have been caught long before we reach this point. */ if( ALWAYS(pEList->nExpr==nVal) ){ @@ -99429,9 +100344,9 @@ SQLITE_PRIVATE void sqlite3CodeRhsOfIN( int i; ExprList *pList = pExpr->x.pList; struct ExprList_item *pItem; - int r1, r2, r3; + int r1, r2; affinity = sqlite3ExprAffinity(pLeft); - if( !affinity ){ + if( affinity<=SQLITE_AFF_NONE ){ affinity = SQLITE_AFF_BLOB; } if( pKeyInfo ){ @@ -99442,10 +100357,8 @@ SQLITE_PRIVATE void sqlite3CodeRhsOfIN( /* Loop through each expression in . */ r1 = sqlite3GetTempReg(pParse); r2 = sqlite3GetTempReg(pParse); - if( isRowid ) sqlite3VdbeAddOp4(v, OP_Blob, 0, r2, 0, "", P4_STATIC); for(i=pList->nExpr, pItem=pList->a; i>0; i--, pItem++){ Expr *pE2 = pItem->pExpr; - int iValToIns; /* If the expression is not constant then we will need to ** disable the test that was generated above that makes sure @@ -99454,24 +100367,14 @@ SQLITE_PRIVATE void sqlite3CodeRhsOfIN( */ if( addrOnce && !sqlite3ExprIsConstant(pE2) ){ sqlite3VdbeChangeToNoop(v, addrOnce); + ExprClearProperty(pExpr, EP_Subrtn); addrOnce = 0; } /* Evaluate the expression and insert it into the temp table */ - if( isRowid && sqlite3ExprIsInteger(pE2, &iValToIns) ){ - sqlite3VdbeAddOp3(v, OP_InsertInt, iTab, r2, iValToIns); - }else{ - r3 = sqlite3ExprCodeTarget(pParse, pE2, r1); - if( isRowid ){ - sqlite3VdbeAddOp2(v, OP_MustBeInt, r3, - sqlite3VdbeCurrentAddr(v)+2); - VdbeCoverage(v); - sqlite3VdbeAddOp3(v, OP_Insert, iTab, r2, r3); - }else{ - sqlite3VdbeAddOp4(v, OP_MakeRecord, r3, 1, r2, &affinity, 1); - sqlite3VdbeAddOp4Int(v, OP_IdxInsert, iTab, r2, r3, 1); - } - } + sqlite3ExprCode(pParse, pE2, r1); + sqlite3VdbeAddOp4(v, OP_MakeRecord, r1, 1, r2, &affinity, 1); + sqlite3VdbeAddOp4Int(v, OP_IdxInsert, iTab, r2, r1, 1); } sqlite3ReleaseTempReg(pParse, r1); sqlite3ReleaseTempReg(pParse, r2); @@ -99484,6 +100387,7 @@ SQLITE_PRIVATE void sqlite3CodeRhsOfIN( /* Subroutine return */ sqlite3VdbeAddOp1(v, OP_Return, pExpr->y.sub.regReturn); sqlite3VdbeChangeP1(v, pExpr->y.sub.iAddr-1, sqlite3VdbeCurrentAddr(v)-1); + sqlite3ClearTempRegCache(pParse); } } #endif /* SQLITE_OMIT_SUBQUERY */ @@ -99497,7 +100401,7 @@ SQLITE_PRIVATE void sqlite3CodeRhsOfIN( ** ** The pExpr parameter is the SELECT or EXISTS operator to be coded. ** -** The register that holds the result. For a multi-column SELECT, +** Return the register that holds the result. For a multi-column SELECT, ** the result is stored in a contiguous array of registers and the ** return value is the register of the left-most result column. ** Return 0 if an error occurs. @@ -99575,11 +100479,21 @@ SQLITE_PRIVATE int sqlite3CodeSubselect(Parse *pParse, Expr *pExpr){ sqlite3VdbeAddOp2(v, OP_Integer, 0, dest.iSDParm); VdbeComment((v, "Init EXISTS result")); } - pLimit = sqlite3ExprAlloc(pParse->db, TK_INTEGER,&sqlite3IntTokens[1], 0); if( pSel->pLimit ){ - sqlite3ExprDelete(pParse->db, pSel->pLimit->pLeft); + /* The subquery already has a limit. If the pre-existing limit is X + ** then make the new limit X<>0 so that the new limit is either 1 or 0 */ + sqlite3 *db = pParse->db; + pLimit = sqlite3Expr(db, TK_INTEGER, "0"); + if( pLimit ){ + pLimit->affExpr = SQLITE_AFF_NUMERIC; + pLimit = sqlite3PExpr(pParse, TK_NE, + sqlite3ExprDup(db, pSel->pLimit->pLeft, 0), pLimit); + } + sqlite3ExprDelete(db, pSel->pLimit->pLeft); pSel->pLimit->pLeft = pLimit; }else{ + /* If there is no pre-existing limit add a limit of 1 */ + pLimit = sqlite3Expr(pParse->db, TK_INTEGER, "1"); pSel->pLimit = sqlite3PExpr(pParse, TK_LIMIT, pLimit, 0); } pSel->iLimit = 0; @@ -99594,6 +100508,7 @@ SQLITE_PRIVATE int sqlite3CodeSubselect(Parse *pParse, Expr *pExpr){ /* Subroutine return */ sqlite3VdbeAddOp1(v, OP_Return, pExpr->y.sub.regReturn); sqlite3VdbeChangeP1(v, pExpr->y.sub.iAddr-1, sqlite3VdbeCurrentAddr(v)-1); + sqlite3ClearTempRegCache(pParse); } return rReg; @@ -99741,13 +100656,21 @@ static void sqlite3ExprCodeIN( int r2, regToFree; int regCkNull = 0; int ii; + int bLhsReal; /* True if the LHS of the IN has REAL affinity */ assert( !ExprHasProperty(pExpr, EP_xIsSelect) ); if( destIfNull!=destIfFalse ){ regCkNull = sqlite3GetTempReg(pParse); sqlite3VdbeAddOp3(v, OP_BitAnd, rLhs, rLhs, regCkNull); } + bLhsReal = sqlite3ExprAffinity(pExpr->pLeft)==SQLITE_AFF_REAL; for(ii=0; iinExpr; ii++){ - r2 = sqlite3ExprCodeTemp(pParse, pList->a[ii].pExpr, ®ToFree); + if( bLhsReal ){ + r2 = regToFree = sqlite3GetTempReg(pParse); + sqlite3ExprCode(pParse, pList->a[ii].pExpr, r2); + sqlite3VdbeAddOp4(v, OP_Affinity, r2, 1, 0, "E", P4_STATIC); + }else{ + r2 = sqlite3ExprCodeTemp(pParse, pList->a[ii].pExpr, ®ToFree); + } if( regCkNull && sqlite3ExprCanBeNull(pList->a[ii].pExpr) ){ sqlite3VdbeAddOp3(v, OP_BitAnd, regCkNull, r2, regCkNull); } @@ -100031,7 +100954,8 @@ SQLITE_PRIVATE void sqlite3ExprCodeMove(Parse *pParse, int iFrom, int iTo, int n ** register iReg. The caller must ensure that iReg already contains ** the correct value for the expression. */ -static void exprToRegister(Expr *p, int iReg){ +static void exprToRegister(Expr *pExpr, int iReg){ + Expr *p = sqlite3ExprSkipCollateAndLikely(pExpr); p->op2 = p->op; p->op = TK_REGISTER; p->iTable = iReg; @@ -100132,7 +101056,7 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) */ int iReg = sqlite3ExprCodeTarget(pParse, pExpr->pLeft,target); int aff = sqlite3TableColumnAffinity(pExpr->y.pTab, pExpr->iColumn); - if( aff!=SQLITE_AFF_BLOB ){ + if( aff>SQLITE_AFF_BLOB ){ static const char zAff[] = "B\000C\000D\000E"; assert( SQLITE_AFF_BLOB=='A' ); assert( SQLITE_AFF_TEXT=='B' ); @@ -100148,7 +101072,19 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) if( iTab<0 ){ if( pParse->iSelfTab<0 ){ /* Generating CHECK constraints or inserting into partial index */ - return pExpr->iColumn - pParse->iSelfTab; + assert( pExpr->y.pTab!=0 ); + assert( pExpr->iColumn>=XN_ROWID ); + assert( pExpr->iColumny.pTab->nCol ); + if( pExpr->iColumn>=0 + && pExpr->y.pTab->aCol[pExpr->iColumn].affinity==SQLITE_AFF_REAL + ){ + sqlite3VdbeAddOp2(v, OP_SCopy, pExpr->iColumn - pParse->iSelfTab, + target); + sqlite3VdbeAddOp1(v, OP_RealAffinity, target); + return target; + }else{ + return pExpr->iColumn - pParse->iSelfTab; + } }else{ /* Coding an expression that is part of an index where column names ** in the index refer to the table to which the index belongs */ @@ -100435,7 +101371,7 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) assert( nFarg==1 ); aff = sqlite3ExprAffinity(pFarg->a[0].pExpr); sqlite3VdbeLoadString(v, target, - aff ? azAff[aff-SQLITE_AFF_BLOB] : "none"); + (aff<=SQLITE_AFF_NONE) ? "none" : azAff[aff-SQLITE_AFF_BLOB]); return target; } #endif @@ -100543,8 +101479,8 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) pExpr->pLeft->iTable = sqlite3CodeSubselect(pParse, pExpr->pLeft); } assert( pExpr->iTable==0 || pExpr->pLeft->op==TK_SELECT ); - if( pExpr->iTable - && pExpr->iTable!=(n = sqlite3ExprVectorSize(pExpr->pLeft)) + if( pExpr->iTable!=0 + && pExpr->iTable!=(n = sqlite3ExprVectorSize(pExpr->pLeft)) ){ sqlite3ErrorMsg(pParse, "%d columns assigned %d values", pExpr->iTable, n); @@ -100647,10 +101583,23 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) break; } + /* TK_IF_NULL_ROW Expr nodes are inserted ahead of expressions + ** that derive from the right-hand table of a LEFT JOIN. The + ** Expr.iTable value is the table number for the right-hand table. + ** The expression is only evaluated if that table is not currently + ** on a LEFT JOIN NULL row. + */ case TK_IF_NULL_ROW: { int addrINR; + u8 okConstFactor = pParse->okConstFactor; addrINR = sqlite3VdbeAddOp1(v, OP_IfNullRow, pExpr->iTable); + /* Temporarily disable factoring of constant expressions, since + ** even though expressions may appear to be constant, they are not + ** really constant because they originate from the right-hand side + ** of a LEFT JOIN. */ + pParse->okConstFactor = 0; inReg = sqlite3ExprCodeTarget(pParse, pExpr->pLeft, target); + pParse->okConstFactor = okConstFactor; sqlite3VdbeJumpHere(v, addrINR); sqlite3VdbeChangeP3(v, addrINR, inReg); break; @@ -100687,6 +101636,8 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) Expr opCompare; /* The X==Ei expression */ Expr *pX; /* The X expression */ Expr *pTest = 0; /* X==Ei (form A) or just Ei (form B) */ + Expr *pDel = 0; + sqlite3 *db = pParse->db; assert( !ExprHasProperty(pExpr, EP_xIsSelect) && pExpr->x.pList ); assert(pExpr->x.pList->nExpr > 0); @@ -100695,13 +101646,17 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) nExpr = pEList->nExpr; endLabel = sqlite3VdbeMakeLabel(pParse); if( (pX = pExpr->pLeft)!=0 ){ - exprNodeCopy(&tempX, pX); + pDel = sqlite3ExprDup(db, pX, 0); + if( db->mallocFailed ){ + sqlite3ExprDelete(db, pDel); + break; + } testcase( pX->op==TK_COLUMN ); - exprToRegister(&tempX, exprCodeVector(pParse, &tempX, ®Free1)); + exprToRegister(pDel, exprCodeVector(pParse, pDel, ®Free1)); testcase( regFree1==0 ); memset(&opCompare, 0, sizeof(opCompare)); opCompare.op = TK_EQ; - opCompare.pLeft = &tempX; + opCompare.pLeft = pDel; pTest = &opCompare; /* Ticket b351d95f9cd5ef17e9d9dbae18f5ca8611190001: ** The value in regFree1 might get SCopy-ed into the file result. @@ -100729,32 +101684,33 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) }else{ sqlite3VdbeAddOp2(v, OP_Null, 0, target); } + sqlite3ExprDelete(db, pDel); sqlite3VdbeResolveLabel(v, endLabel); break; } #ifndef SQLITE_OMIT_TRIGGER case TK_RAISE: { - assert( pExpr->affinity==OE_Rollback - || pExpr->affinity==OE_Abort - || pExpr->affinity==OE_Fail - || pExpr->affinity==OE_Ignore + assert( pExpr->affExpr==OE_Rollback + || pExpr->affExpr==OE_Abort + || pExpr->affExpr==OE_Fail + || pExpr->affExpr==OE_Ignore ); if( !pParse->pTriggerTab ){ sqlite3ErrorMsg(pParse, "RAISE() may only be used within a trigger-program"); return 0; } - if( pExpr->affinity==OE_Abort ){ + if( pExpr->affExpr==OE_Abort ){ sqlite3MayAbort(pParse); } assert( !ExprHasProperty(pExpr, EP_IntValue) ); - if( pExpr->affinity==OE_Ignore ){ + if( pExpr->affExpr==OE_Ignore ){ sqlite3VdbeAddOp4( v, OP_Halt, SQLITE_OK, OE_Ignore, 0, pExpr->u.zToken,0); VdbeCoverage(v); }else{ sqlite3HaltConstraint(pParse, SQLITE_CONSTRAINT_TRIGGER, - pExpr->affinity, pExpr->u.zToken, 0, 0); + pExpr->affExpr, pExpr->u.zToken, 0, 0); } break; @@ -100819,7 +101775,7 @@ SQLITE_PRIVATE int sqlite3ExprCodeAtInit( */ SQLITE_PRIVATE int sqlite3ExprCodeTemp(Parse *pParse, Expr *pExpr, int *pReg){ int r2; - pExpr = sqlite3ExprSkipCollate(pExpr); + pExpr = sqlite3ExprSkipCollateAndLikely(pExpr); if( ConstFactorOk(pParse) && pExpr->op!=TK_REGISTER && sqlite3ExprIsConstantNotJoin(pExpr) @@ -101010,40 +101966,44 @@ static void exprCodeBetween( void (*xJump)(Parse*,Expr*,int,int), /* Action to take */ int jumpIfNull /* Take the jump if the BETWEEN is NULL */ ){ - Expr exprAnd; /* The AND operator in x>=y AND x<=z */ + Expr exprAnd; /* The AND operator in x>=y AND x<=z */ Expr compLeft; /* The x>=y term */ Expr compRight; /* The x<=z term */ - Expr exprX; /* The x subexpression */ int regFree1 = 0; /* Temporary use register */ + Expr *pDel = 0; + sqlite3 *db = pParse->db; memset(&compLeft, 0, sizeof(Expr)); memset(&compRight, 0, sizeof(Expr)); memset(&exprAnd, 0, sizeof(Expr)); assert( !ExprHasProperty(pExpr, EP_xIsSelect) ); - exprNodeCopy(&exprX, pExpr->pLeft); - exprAnd.op = TK_AND; - exprAnd.pLeft = &compLeft; - exprAnd.pRight = &compRight; - compLeft.op = TK_GE; - compLeft.pLeft = &exprX; - compLeft.pRight = pExpr->x.pList->a[0].pExpr; - compRight.op = TK_LE; - compRight.pLeft = &exprX; - compRight.pRight = pExpr->x.pList->a[1].pExpr; - exprToRegister(&exprX, exprCodeVector(pParse, &exprX, ®Free1)); - if( xJump ){ - xJump(pParse, &exprAnd, dest, jumpIfNull); - }else{ - /* Mark the expression is being from the ON or USING clause of a join - ** so that the sqlite3ExprCodeTarget() routine will not attempt to move - ** it into the Parse.pConstExpr list. We should use a new bit for this, - ** for clarity, but we are out of bits in the Expr.flags field so we - ** have to reuse the EP_FromJoin bit. Bummer. */ - exprX.flags |= EP_FromJoin; - sqlite3ExprCodeTarget(pParse, &exprAnd, dest); + pDel = sqlite3ExprDup(db, pExpr->pLeft, 0); + if( db->mallocFailed==0 ){ + exprAnd.op = TK_AND; + exprAnd.pLeft = &compLeft; + exprAnd.pRight = &compRight; + compLeft.op = TK_GE; + compLeft.pLeft = pDel; + compLeft.pRight = pExpr->x.pList->a[0].pExpr; + compRight.op = TK_LE; + compRight.pLeft = pDel; + compRight.pRight = pExpr->x.pList->a[1].pExpr; + exprToRegister(pDel, exprCodeVector(pParse, pDel, ®Free1)); + if( xJump ){ + xJump(pParse, &exprAnd, dest, jumpIfNull); + }else{ + /* Mark the expression is being from the ON or USING clause of a join + ** so that the sqlite3ExprCodeTarget() routine will not attempt to move + ** it into the Parse.pConstExpr list. We should use a new bit for this, + ** for clarity, but we are out of bits in the Expr.flags field so we + ** have to reuse the EP_FromJoin bit. Bummer. */ + pDel->flags |= EP_FromJoin; + sqlite3ExprCodeTarget(pParse, &exprAnd, dest); + } + sqlite3ReleaseTempReg(pParse, regFree1); } - sqlite3ReleaseTempReg(pParse, regFree1); + sqlite3ExprDelete(db, pDel); /* Ensure adequate test coverage */ testcase( xJump==sqlite3ExprIfTrue && jumpIfNull==0 && regFree1==0 ); @@ -101083,18 +102043,23 @@ SQLITE_PRIVATE void sqlite3ExprIfTrue(Parse *pParse, Expr *pExpr, int dest, int if( NEVER(pExpr==0) ) return; /* No way this can happen */ op = pExpr->op; switch( op ){ - case TK_AND: { - int d2 = sqlite3VdbeMakeLabel(pParse); - testcase( jumpIfNull==0 ); - sqlite3ExprIfFalse(pParse, pExpr->pLeft, d2,jumpIfNull^SQLITE_JUMPIFNULL); - sqlite3ExprIfTrue(pParse, pExpr->pRight, dest, jumpIfNull); - sqlite3VdbeResolveLabel(v, d2); - break; - } + case TK_AND: case TK_OR: { - testcase( jumpIfNull==0 ); - sqlite3ExprIfTrue(pParse, pExpr->pLeft, dest, jumpIfNull); - sqlite3ExprIfTrue(pParse, pExpr->pRight, dest, jumpIfNull); + Expr *pAlt = sqlite3ExprSimplifiedAndOr(pExpr); + if( pAlt!=pExpr ){ + sqlite3ExprIfTrue(pParse, pAlt, dest, jumpIfNull); + }else if( op==TK_AND ){ + int d2 = sqlite3VdbeMakeLabel(pParse); + testcase( jumpIfNull==0 ); + sqlite3ExprIfFalse(pParse, pExpr->pLeft, d2, + jumpIfNull^SQLITE_JUMPIFNULL); + sqlite3ExprIfTrue(pParse, pExpr->pRight, dest, jumpIfNull); + sqlite3VdbeResolveLabel(v, d2); + }else{ + testcase( jumpIfNull==0 ); + sqlite3ExprIfTrue(pParse, pExpr->pLeft, dest, jumpIfNull); + sqlite3ExprIfTrue(pParse, pExpr->pRight, dest, jumpIfNull); + } break; } case TK_NOT: { @@ -101180,9 +102145,9 @@ SQLITE_PRIVATE void sqlite3ExprIfTrue(Parse *pParse, Expr *pExpr, int dest, int #endif default: { default_expr: - if( exprAlwaysTrue(pExpr) ){ + if( ExprAlwaysTrue(pExpr) ){ sqlite3VdbeGoto(v, dest); - }else if( exprAlwaysFalse(pExpr) ){ + }else if( ExprAlwaysFalse(pExpr) ){ /* No-op */ }else{ r1 = sqlite3ExprCodeTemp(pParse, pExpr, ®Free1); @@ -101250,18 +102215,23 @@ SQLITE_PRIVATE void sqlite3ExprIfFalse(Parse *pParse, Expr *pExpr, int dest, int assert( pExpr->op!=TK_GE || op==OP_Lt ); switch( pExpr->op ){ - case TK_AND: { - testcase( jumpIfNull==0 ); - sqlite3ExprIfFalse(pParse, pExpr->pLeft, dest, jumpIfNull); - sqlite3ExprIfFalse(pParse, pExpr->pRight, dest, jumpIfNull); - break; - } + case TK_AND: case TK_OR: { - int d2 = sqlite3VdbeMakeLabel(pParse); - testcase( jumpIfNull==0 ); - sqlite3ExprIfTrue(pParse, pExpr->pLeft, d2, jumpIfNull^SQLITE_JUMPIFNULL); - sqlite3ExprIfFalse(pParse, pExpr->pRight, dest, jumpIfNull); - sqlite3VdbeResolveLabel(v, d2); + Expr *pAlt = sqlite3ExprSimplifiedAndOr(pExpr); + if( pAlt!=pExpr ){ + sqlite3ExprIfFalse(pParse, pAlt, dest, jumpIfNull); + }else if( pExpr->op==TK_AND ){ + testcase( jumpIfNull==0 ); + sqlite3ExprIfFalse(pParse, pExpr->pLeft, dest, jumpIfNull); + sqlite3ExprIfFalse(pParse, pExpr->pRight, dest, jumpIfNull); + }else{ + int d2 = sqlite3VdbeMakeLabel(pParse); + testcase( jumpIfNull==0 ); + sqlite3ExprIfTrue(pParse, pExpr->pLeft, d2, + jumpIfNull^SQLITE_JUMPIFNULL); + sqlite3ExprIfFalse(pParse, pExpr->pRight, dest, jumpIfNull); + sqlite3VdbeResolveLabel(v, d2); + } break; } case TK_NOT: { @@ -101350,9 +102320,9 @@ SQLITE_PRIVATE void sqlite3ExprIfFalse(Parse *pParse, Expr *pExpr, int dest, int #endif default: { default_expr: - if( exprAlwaysFalse(pExpr) ){ + if( ExprAlwaysFalse(pExpr) ){ sqlite3VdbeGoto(v, dest); - }else if( exprAlwaysTrue(pExpr) ){ + }else if( ExprAlwaysTrue(pExpr) ){ /* no-op */ }else{ r1 = sqlite3ExprCodeTemp(pParse, pExpr, ®Free1); @@ -101472,20 +102442,17 @@ SQLITE_PRIVATE int sqlite3ExprCompare(Parse *pParse, Expr *pA, Expr *pB, int iTa return 2; } if( pA->op!=TK_COLUMN && pA->op!=TK_AGG_COLUMN && pA->u.zToken ){ - if( pA->op==TK_FUNCTION ){ + if( pA->op==TK_FUNCTION || pA->op==TK_AGG_FUNCTION ){ if( sqlite3StrICmp(pA->u.zToken,pB->u.zToken)!=0 ) return 2; #ifndef SQLITE_OMIT_WINDOWFUNC - /* Justification for the assert(): - ** window functions have p->op==TK_FUNCTION but aggregate functions - ** have p->op==TK_AGG_FUNCTION. So any comparison between an aggregate - ** function and a window function should have failed before reaching - ** this point. And, it is not possible to have a window function and - ** a scalar function with the same name and number of arguments. So - ** if we reach this point, either A and B both window functions or - ** neither are a window functions. */ - assert( ExprHasProperty(pA,EP_WinFunc)==ExprHasProperty(pB,EP_WinFunc) ); + assert( pA->op==pB->op ); + if( ExprHasProperty(pA,EP_WinFunc)!=ExprHasProperty(pB,EP_WinFunc) ){ + return 2; + } if( ExprHasProperty(pA,EP_WinFunc) ){ - if( sqlite3WindowCompare(pParse,pA->y.pWin,pB->y.pWin)!=0 ) return 2; + if( sqlite3WindowCompare(pParse, pA->y.pWin, pB->y.pWin, 1)!=0 ){ + return 2; + } } #endif }else if( pA->op==TK_NULL ){ @@ -101508,7 +102475,9 @@ SQLITE_PRIVATE int sqlite3ExprCompare(Parse *pParse, Expr *pA, Expr *pB, int iTa && (combinedFlags & EP_Reduced)==0 ){ if( pA->iColumn!=pB->iColumn ) return 2; - if( pA->iTable!=pB->iTable + if( pA->op2!=pB->op2 ) return 2; + if( pA->op!=TK_IN + && pA->iTable!=pB->iTable && (pA->iTable!=iTab || NEVER(pB->iTable>=0)) ) return 2; } } @@ -101538,7 +102507,7 @@ SQLITE_PRIVATE int sqlite3ExprListCompare(ExprList *pA, ExprList *pB, int iTab){ for(i=0; inExpr; i++){ Expr *pExprA = pA->a[i].pExpr; Expr *pExprB = pB->a[i].pExpr; - if( pA->a[i].sortOrder!=pB->a[i].sortOrder ) return 1; + if( pA->a[i].sortFlags!=pB->a[i].sortFlags ) return 1; if( sqlite3ExprCompare(0, pExprA, pExprB, iTab) ) return 1; } return 0; @@ -101550,11 +102519,88 @@ SQLITE_PRIVATE int sqlite3ExprListCompare(ExprList *pA, ExprList *pB, int iTab){ */ SQLITE_PRIVATE int sqlite3ExprCompareSkip(Expr *pA, Expr *pB, int iTab){ return sqlite3ExprCompare(0, - sqlite3ExprSkipCollate(pA), - sqlite3ExprSkipCollate(pB), + sqlite3ExprSkipCollateAndLikely(pA), + sqlite3ExprSkipCollateAndLikely(pB), iTab); } +/* +** Return non-zero if Expr p can only be true if pNN is not NULL. +** +** Or if seenNot is true, return non-zero if Expr p can only be +** non-NULL if pNN is not NULL +*/ +static int exprImpliesNotNull( + Parse *pParse, /* Parsing context */ + Expr *p, /* The expression to be checked */ + Expr *pNN, /* The expression that is NOT NULL */ + int iTab, /* Table being evaluated */ + int seenNot /* Return true only if p can be any non-NULL value */ +){ + assert( p ); + assert( pNN ); + if( sqlite3ExprCompare(pParse, p, pNN, iTab)==0 ){ + return pNN->op!=TK_NULL; + } + switch( p->op ){ + case TK_IN: { + if( seenNot && ExprHasProperty(p, EP_xIsSelect) ) return 0; + assert( ExprHasProperty(p,EP_xIsSelect) + || (p->x.pList!=0 && p->x.pList->nExpr>0) ); + return exprImpliesNotNull(pParse, p->pLeft, pNN, iTab, 1); + } + case TK_BETWEEN: { + ExprList *pList = p->x.pList; + assert( pList!=0 ); + assert( pList->nExpr==2 ); + if( seenNot ) return 0; + if( exprImpliesNotNull(pParse, pList->a[0].pExpr, pNN, iTab, 1) + || exprImpliesNotNull(pParse, pList->a[1].pExpr, pNN, iTab, 1) + ){ + return 1; + } + return exprImpliesNotNull(pParse, p->pLeft, pNN, iTab, 1); + } + case TK_EQ: + case TK_NE: + case TK_LT: + case TK_LE: + case TK_GT: + case TK_GE: + case TK_PLUS: + case TK_MINUS: + case TK_BITOR: + case TK_LSHIFT: + case TK_RSHIFT: + case TK_CONCAT: + seenNot = 1; + /* Fall thru */ + case TK_STAR: + case TK_REM: + case TK_BITAND: + case TK_SLASH: { + if( exprImpliesNotNull(pParse, p->pRight, pNN, iTab, seenNot) ) return 1; + /* Fall thru into the next case */ + } + case TK_SPAN: + case TK_COLLATE: + case TK_UPLUS: + case TK_UMINUS: { + return exprImpliesNotNull(pParse, p->pLeft, pNN, iTab, seenNot); + } + case TK_TRUTH: { + if( seenNot ) return 0; + if( p->op2!=TK_IS ) return 0; + return exprImpliesNotNull(pParse, p->pLeft, pNN, iTab, 1); + } + case TK_BITNOT: + case TK_NOT: { + return exprImpliesNotNull(pParse, p->pLeft, pNN, iTab, 1); + } + } + return 0; +} + /* ** Return true if we can prove the pE2 will always be true if pE1 is ** true. Return false if we cannot complete the proof or if pE2 might @@ -101590,10 +102636,10 @@ SQLITE_PRIVATE int sqlite3ExprImpliesExpr(Parse *pParse, Expr *pE1, Expr *pE2, i ){ return 1; } - if( pE2->op==TK_NOTNULL && pE1->op!=TK_ISNULL && pE1->op!=TK_IS ){ - Expr *pX = sqlite3ExprSkipCollate(pE1->pLeft); - testcase( pX!=pE1->pLeft ); - if( sqlite3ExprCompare(pParse, pX, pE2->pLeft, iTab)==0 ) return 1; + if( pE2->op==TK_NOTNULL + && exprImpliesNotNull(pParse, pE1, pE2->pLeft, iTab, 0) + ){ + return 1; } return 0; } @@ -101613,7 +102659,6 @@ static int impliesNotNullRow(Walker *pWalker, Expr *pExpr){ if( ExprHasProperty(pExpr, EP_FromJoin) ) return WRC_Prune; switch( pExpr->op ){ case TK_ISNOT: - case TK_NOT: case TK_ISNULL: case TK_NOTNULL: case TK_IS: @@ -101621,8 +102666,8 @@ static int impliesNotNullRow(Walker *pWalker, Expr *pExpr){ case TK_CASE: case TK_IN: case TK_FUNCTION: + case TK_TRUTH: testcase( pExpr->op==TK_ISNOT ); - testcase( pExpr->op==TK_NOT ); testcase( pExpr->op==TK_ISNULL ); testcase( pExpr->op==TK_NOTNULL ); testcase( pExpr->op==TK_IS ); @@ -101630,6 +102675,7 @@ static int impliesNotNullRow(Walker *pWalker, Expr *pExpr){ testcase( pExpr->op==TK_CASE ); testcase( pExpr->op==TK_IN ); testcase( pExpr->op==TK_FUNCTION ); + testcase( pExpr->op==TK_TRUTH ); return WRC_Prune; case TK_COLUMN: if( pWalker->u.iCur==pExpr->iTable ){ @@ -101638,6 +102684,18 @@ static int impliesNotNullRow(Walker *pWalker, Expr *pExpr){ } return WRC_Prune; + case TK_AND: + if( sqlite3ExprImpliesNonNullRow(pExpr->pLeft, pWalker->u.iCur) + && sqlite3ExprImpliesNonNullRow(pExpr->pRight, pWalker->u.iCur) + ){ + pWalker->eCode = 1; + } + return WRC_Prune; + + case TK_BETWEEN: + sqlite3WalkExpr(pWalker, pExpr->pLeft); + return WRC_Prune; + /* Virtual tables are allowed to use constraints like x=NULL. So ** a term of the form x=y does not prove that y is not null if x ** is the column of a virtual table */ @@ -101658,6 +102716,7 @@ static int impliesNotNullRow(Walker *pWalker, Expr *pExpr){ ){ return WRC_Prune; } + default: return WRC_Continue; } @@ -101687,6 +102746,17 @@ static int impliesNotNullRow(Walker *pWalker, Expr *pExpr){ */ SQLITE_PRIVATE int sqlite3ExprImpliesNonNullRow(Expr *p, int iTab){ Walker w; + p = sqlite3ExprSkipCollateAndLikely(p); + while( p ){ + if( p->op==TK_NOTNULL ){ + p = p->pLeft; + }else if( p->op==TK_AND ){ + if( sqlite3ExprImpliesNonNullRow(p->pLeft, iTab) ) return 1; + p = p->pRight; + }else{ + break; + } + } w.xExprCallback = impliesNotNullRow; w.xSelectCallback = 0; w.xSelectCallback2 = 0; @@ -101782,7 +102852,10 @@ static int exprSrcCount(Walker *pWalker, Expr *pExpr){ } if( inThis++; - }else{ + }else if( nSrc==0 || pExpr->iTablea[0].iCursor ){ + /* In a well-formed parse tree (no name resolution errors), + ** TK_COLUMN nodes with smaller Expr.iTable values are in an + ** outer context. Those are the only ones to count as "other" */ p->nOther++; } } @@ -101799,8 +102872,9 @@ SQLITE_PRIVATE int sqlite3FunctionUsesThisSrc(Expr *pExpr, SrcList *pSrcList){ Walker w; struct SrcCount cnt; assert( pExpr->op==TK_AGG_FUNCTION ); + memset(&w, 0, sizeof(w)); w.xExprCallback = exprSrcCount; - w.xSelectCallback = 0; + w.xSelectCallback = sqlite3SelectWalkNoop; w.u.pSrcCount = &cnt; cnt.pSrc = pSrcList; cnt.nThis = 0; @@ -102069,6 +103143,11 @@ SQLITE_PRIVATE void sqlite3ReleaseTempRange(Parse *pParse, int iReg, int nReg){ /* ** Mark all temporary registers as being unavailable for reuse. +** +** Always invoke this procedure after coding a subroutine or co-routine +** that might be invoked from other parts of the code, to ensure that +** the sub/co-routine does not use registers in common with the code that +** invokes the sub/co-routine. */ SQLITE_PRIVATE void sqlite3ClearTempRegCache(Parse *pParse){ pParse->nTempReg = 0; @@ -102156,7 +103235,7 @@ static void renameTestSchema(Parse *pParse, const char *zDb, int bTemp){ sqlite3NestedParse(pParse, "SELECT 1 " "FROM \"%w\".%s " - "WHERE name NOT LIKE 'sqlite_%%'" + "WHERE name NOT LIKE 'sqliteX_%%' ESCAPE 'X'" " AND sql NOT LIKE 'create virtual%%'" " AND sqlite_rename_test(%Q, sql, type, name, %d)=NULL ", zDb, MASTER_NAME, @@ -102167,7 +103246,7 @@ static void renameTestSchema(Parse *pParse, const char *zDb, int bTemp){ sqlite3NestedParse(pParse, "SELECT 1 " "FROM temp.%s " - "WHERE name NOT LIKE 'sqlite_%%'" + "WHERE name NOT LIKE 'sqliteX_%%' ESCAPE 'X'" " AND sql NOT LIKE 'create virtual%%'" " AND sqlite_rename_test(%Q, sql, type, name, 1)=NULL ", MASTER_NAME, zDb @@ -102238,8 +103317,8 @@ SQLITE_PRIVATE void sqlite3AlterRenameTable( if( SQLITE_OK!=isAlterableTable(pParse, pTab) ){ goto exit_rename_table; } - if( SQLITE_OK!=sqlite3CheckObjectName(pParse, zName) ){ goto - exit_rename_table; + if( SQLITE_OK!=sqlite3CheckObjectName(pParse,zName,"table",zName) ){ + goto exit_rename_table; } #ifndef SQLITE_OMIT_VIEW @@ -102268,15 +103347,15 @@ SQLITE_PRIVATE void sqlite3AlterRenameTable( } #endif - /* Begin a transaction for database iDb. - ** Then modify the schema cookie (since the ALTER TABLE modifies the - ** schema). Open a statement transaction if the table is a virtual - ** table. - */ + /* Begin a transaction for database iDb. Then modify the schema cookie + ** (since the ALTER TABLE modifies the schema). Call sqlite3MayAbort(), + ** as the scalar functions (e.g. sqlite_rename_table()) invoked by the + ** nested SQL may raise an exception. */ v = sqlite3GetVdbe(pParse); if( v==0 ){ goto exit_rename_table; } + sqlite3MayAbort(pParse); /* figure out how many UTF-8 characters are in zName */ zTabName = pTab->zName; @@ -102288,7 +103367,7 @@ SQLITE_PRIVATE void sqlite3AlterRenameTable( "UPDATE \"%w\".%s SET " "sql = sqlite_rename_table(%Q, type, name, sql, %Q, %Q, %d) " "WHERE (type!='index' OR tbl_name=%Q COLLATE nocase)" - "AND name NOT LIKE 'sqlite_%%'" + "AND name NOT LIKE 'sqliteX_%%' ESCAPE 'X'" , zDb, MASTER_NAME, zDb, zTabName, zName, (iDb==1), zTabName ); @@ -102299,7 +103378,8 @@ SQLITE_PRIVATE void sqlite3AlterRenameTable( "tbl_name = %Q, " "name = CASE " "WHEN type='table' THEN %Q " - "WHEN name LIKE 'sqlite_autoindex%%' AND type='index' THEN " + "WHEN name LIKE 'sqliteX_autoindex%%' ESCAPE 'X' " + " AND type='index' THEN " "'sqlite_autoindex_' || %Q || substr(name,%d+18) " "ELSE name END " "WHERE tbl_name=%Q COLLATE nocase AND " @@ -102345,7 +103425,6 @@ SQLITE_PRIVATE void sqlite3AlterRenameTable( int i = ++pParse->nMem; sqlite3VdbeLoadString(v, i, zName); sqlite3VdbeAddOp4(v, OP_VRename, i, 0, 0,(const char*)pVTab, P4_VTAB); - sqlite3MayAbort(pParse); } #endif @@ -102537,6 +103616,7 @@ SQLITE_PRIVATE void sqlite3AlterBeginAddColumn(Parse *pParse, SrcList *pSrc){ goto exit_begin_add_column; } + sqlite3MayAbort(pParse); assert( pTab->addColOffset>0 ); iDb = sqlite3SchemaToIndex(db, pTab->pSchema); @@ -102666,6 +103746,7 @@ SQLITE_PRIVATE void sqlite3AlterRenameColumn( ** uses the sqlite_rename_column() SQL function to compute the new ** CREATE statement text for the sqlite_master table. */ + sqlite3MayAbort(pParse); zNew = sqlite3NameFromToken(db, pNew); if( !zNew ) goto exit_rename_column; assert( pNew->n>0 ); @@ -102673,7 +103754,8 @@ SQLITE_PRIVATE void sqlite3AlterRenameColumn( sqlite3NestedParse(pParse, "UPDATE \"%w\".%s SET " "sql = sqlite_rename_column(sql, type, name, %Q, %Q, %d, %Q, %d, %d) " - "WHERE name NOT LIKE 'sqlite_%%' AND (type != 'index' OR tbl_name = %Q)" + "WHERE name NOT LIKE 'sqliteX_%%' ESCAPE 'X' " + " AND (type != 'index' OR tbl_name = %Q)" " AND sql NOT LIKE 'create virtual%%'", zDb, MASTER_NAME, zDb, pTab->zName, iCol, zNew, bQuote, iSchema==1, @@ -102827,6 +103909,29 @@ static int renameUnmapExprCb(Walker *pWalker, Expr *pExpr){ return WRC_Continue; } +/* +** Walker callback used by sqlite3RenameExprUnmap(). +*/ +static int renameUnmapSelectCb(Walker *pWalker, Select *p){ + Parse *pParse = pWalker->pParse; + int i; + if( ALWAYS(p->pEList) ){ + ExprList *pList = p->pEList; + for(i=0; inExpr; i++){ + if( pList->a[i].zName ){ + sqlite3RenameTokenRemap(pParse, 0, (void*)pList->a[i].zName); + } + } + } + if( ALWAYS(p->pSrc) ){ /* Every Select as a SrcList, even if it is empty */ + SrcList *pSrc = p->pSrc; + for(i=0; inSrc; i++){ + sqlite3RenameTokenRemap(pParse, 0, (void*)pSrc->a[i].zName); + } + } + return WRC_Continue; +} + /* ** Remove all nodes that are part of expression pExpr from the rename list. */ @@ -102835,6 +103940,7 @@ SQLITE_PRIVATE void sqlite3RenameExprUnmap(Parse *pParse, Expr *pExpr){ memset(&sWalker, 0, sizeof(Walker)); sWalker.pParse = pParse; sWalker.xExprCallback = renameUnmapExprCb; + sWalker.xSelectCallback = renameUnmapSelectCb; sqlite3WalkExpr(&sWalker, pExpr); } @@ -103774,13 +104880,13 @@ SQLITE_PRIVATE void sqlite3AlterFunctions(void){ ** is between 3.6.18 and 3.7.8, inclusive, and unless SQLite is compiled ** with SQLITE_ENABLE_STAT2. The sqlite_stat2 table is deprecated. ** The sqlite_stat2 table is superseded by sqlite_stat3, which is only -** created and used by SQLite versions 3.7.9 and later and with +** created and used by SQLite versions 3.7.9 through 3.29.0 when ** SQLITE_ENABLE_STAT3 defined. The functionality of sqlite_stat3 -** is a superset of sqlite_stat2. The sqlite_stat4 is an enhanced -** version of sqlite_stat3 and is only available when compiled with -** SQLITE_ENABLE_STAT4 and in SQLite versions 3.8.1 and later. It is -** not possible to enable both STAT3 and STAT4 at the same time. If they -** are both enabled, then STAT4 takes precedence. +** is a superset of sqlite_stat2 and is also now deprecated. The +** sqlite_stat4 is an enhanced version of sqlite_stat3 and is only +** available when compiled with SQLITE_ENABLE_STAT4 and in SQLite +** versions 3.8.1 and later. STAT4 is the only variant that is still +** supported. ** ** For most applications, sqlite_stat1 provides all the statistics required ** for the query planner to make good choices. @@ -103891,17 +104997,11 @@ SQLITE_PRIVATE void sqlite3AlterFunctions(void){ #if defined(SQLITE_ENABLE_STAT4) # define IsStat4 1 -# define IsStat3 0 -#elif defined(SQLITE_ENABLE_STAT3) -# define IsStat4 0 -# define IsStat3 1 #else # define IsStat4 0 -# define IsStat3 0 # undef SQLITE_STAT4_SAMPLES # define SQLITE_STAT4_SAMPLES 1 #endif -#define IsStat34 (IsStat3+IsStat4) /* 1 for STAT3 or STAT4. 0 otherwise */ /* ** This routine generates code that opens the sqlite_statN tables. @@ -103930,14 +105030,10 @@ static void openStatTable( { "sqlite_stat1", "tbl,idx,stat" }, #if defined(SQLITE_ENABLE_STAT4) { "sqlite_stat4", "tbl,idx,neq,nlt,ndlt,sample" }, - { "sqlite_stat3", 0 }, -#elif defined(SQLITE_ENABLE_STAT3) - { "sqlite_stat3", "tbl,idx,neq,nlt,ndlt,sample" }, - { "sqlite_stat4", 0 }, #else - { "sqlite_stat3", 0 }, { "sqlite_stat4", 0 }, #endif + { "sqlite_stat3", 0 }, }; int i; sqlite3 *db = pParse->db; @@ -104018,7 +105114,7 @@ typedef struct Stat4Sample Stat4Sample; struct Stat4Sample { tRowcnt *anEq; /* sqlite_stat4.nEq */ tRowcnt *anDLt; /* sqlite_stat4.nDLt */ -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 +#ifdef SQLITE_ENABLE_STAT4 tRowcnt *anLt; /* sqlite_stat4.nLt */ union { i64 iRowid; /* Rowid in main table of the key */ @@ -104049,7 +105145,7 @@ struct Stat4Accum { /* Reclaim memory used by a Stat4Sample */ -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 +#ifdef SQLITE_ENABLE_STAT4 static void sampleClear(sqlite3 *db, Stat4Sample *p){ assert( db!=0 ); if( p->nRowid ){ @@ -104061,7 +105157,7 @@ static void sampleClear(sqlite3 *db, Stat4Sample *p){ /* Initialize the BLOB value of a ROWID */ -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 +#ifdef SQLITE_ENABLE_STAT4 static void sampleSetRowid(sqlite3 *db, Stat4Sample *p, int n, const u8 *pData){ assert( db!=0 ); if( p->nRowid ) sqlite3DbFree(db, p->u.aRowid); @@ -104077,7 +105173,7 @@ static void sampleSetRowid(sqlite3 *db, Stat4Sample *p, int n, const u8 *pData){ /* Initialize the INTEGER value of a ROWID. */ -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 +#ifdef SQLITE_ENABLE_STAT4 static void sampleSetRowidInt64(sqlite3 *db, Stat4Sample *p, i64 iRowid){ assert( db!=0 ); if( p->nRowid ) sqlite3DbFree(db, p->u.aRowid); @@ -104090,7 +105186,7 @@ static void sampleSetRowidInt64(sqlite3 *db, Stat4Sample *p, i64 iRowid){ /* ** Copy the contents of object (*pFrom) into (*pTo). */ -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 +#ifdef SQLITE_ENABLE_STAT4 static void sampleCopy(Stat4Accum *p, Stat4Sample *pTo, Stat4Sample *pFrom){ pTo->isPSample = pFrom->isPSample; pTo->iCol = pFrom->iCol; @@ -104111,7 +105207,7 @@ static void sampleCopy(Stat4Accum *p, Stat4Sample *pTo, Stat4Sample *pFrom){ */ static void stat4Destructor(void *pOld){ Stat4Accum *p = (Stat4Accum*)pOld; -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 +#ifdef SQLITE_ENABLE_STAT4 int i; for(i=0; inCol; i++) sampleClear(p->db, p->aBest+i); for(i=0; imxSample; i++) sampleClear(p->db, p->a+i); @@ -104131,7 +105227,7 @@ static void stat4Destructor(void *pOld){ ** WITHOUT ROWID table, N is the number of PRIMARY KEY columns, not the ** total number of columns in the table. ** -** Note 2: C is only used for STAT3 and STAT4. +** Note 2: C is only used for STAT4. ** ** For indexes on ordinary rowid tables, N==K+1. But for indexes on ** WITHOUT ROWID tables, N=K+P where P is the number of columns in the @@ -104154,7 +105250,7 @@ static void statInit( int nColUp; /* nCol rounded up for alignment */ int n; /* Bytes of space to allocate */ sqlite3 *db; /* Database connection */ -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 +#ifdef SQLITE_ENABLE_STAT4 int mxSample = SQLITE_STAT4_SAMPLES; #endif @@ -104171,7 +105267,7 @@ static void statInit( n = sizeof(*p) + sizeof(tRowcnt)*nColUp /* Stat4Accum.anEq */ + sizeof(tRowcnt)*nColUp /* Stat4Accum.anDLt */ -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 +#ifdef SQLITE_ENABLE_STAT4 + sizeof(tRowcnt)*nColUp /* Stat4Accum.anLt */ + sizeof(Stat4Sample)*(nCol+mxSample) /* Stat4Accum.aBest[], a[] */ + sizeof(tRowcnt)*3*nColUp*(nCol+mxSample) @@ -104191,7 +105287,7 @@ static void statInit( p->current.anDLt = (tRowcnt*)&p[1]; p->current.anEq = &p->current.anDLt[nColUp]; -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 +#ifdef SQLITE_ENABLE_STAT4 { u8 *pSpace; /* Allocated space not yet assigned */ int i; /* Used to iterate through p->aSample[] */ @@ -104226,7 +105322,7 @@ static void statInit( sqlite3_result_blob(context, p, sizeof(*p), stat4Destructor); } static const FuncDef statInitFuncdef = { - 2+IsStat34, /* nArg */ + 2+IsStat4, /* nArg */ SQLITE_UTF8, /* funcFlags */ 0, /* pUserData */ 0, /* pNext */ @@ -104266,7 +105362,7 @@ static int sampleIsBetterPost( } #endif -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 +#ifdef SQLITE_ENABLE_STAT4 /* ** Return true if pNew is to be preferred over pOld. ** @@ -104285,15 +105381,11 @@ static int sampleIsBetter( assert( IsStat4 || (pNew->iCol==0 && pOld->iCol==0) ); if( (nEqNew>nEqOld) ) return 1; -#ifdef SQLITE_ENABLE_STAT4 if( nEqNew==nEqOld ){ if( pNew->iColiCol ) return 1; return (pNew->iCol==pOld->iCol && sampleIsBetterPost(pAccum, pNew, pOld)); } return 0; -#else - return (nEqNew==nEqOld && pNew->iHash>pOld->iHash); -#endif } /* @@ -104306,7 +105398,6 @@ static void sampleInsert(Stat4Accum *p, Stat4Sample *pNew, int nEqZero){ assert( IsStat4 || nEqZero==0 ); -#ifdef SQLITE_ENABLE_STAT4 /* Stat4Accum.nMaxEqZero is set to the maximum number of leading 0 ** values in the anEq[] array of any sample in Stat4Accum.a[]. In ** other words, if nMaxEqZero is n, then it is guaranteed that there @@ -104340,7 +105431,6 @@ static void sampleInsert(Stat4Accum *p, Stat4Sample *pNew, int nEqZero){ goto find_new_min; } } -#endif /* If necessary, remove sample iMin to make room for the new sample. */ if( p->nSample>=p->mxSample ){ @@ -104361,10 +105451,8 @@ static void sampleInsert(Stat4Accum *p, Stat4Sample *pNew, int nEqZero){ /* The "rows less-than" for the rowid column must be greater than that ** for the last sample in the p->a[] array. Otherwise, the samples would ** be out of order. */ -#ifdef SQLITE_ENABLE_STAT4 assert( p->nSample==0 || pNew->anLt[p->nCol-1] > p->a[p->nSample-1].anLt[p->nCol-1] ); -#endif /* Insert the new sample */ pSample = &p->a[p->nSample]; @@ -104374,9 +105462,7 @@ static void sampleInsert(Stat4Accum *p, Stat4Sample *pNew, int nEqZero){ /* Zero the first nEqZero entries in the anEq[] array. */ memset(pSample->anEq, 0, sizeof(tRowcnt)*nEqZero); -#ifdef SQLITE_ENABLE_STAT4 - find_new_min: -#endif +find_new_min: if( p->nSample>=p->mxSample ){ int iMin = -1; for(i=0; imxSample; i++){ @@ -104389,7 +105475,7 @@ static void sampleInsert(Stat4Accum *p, Stat4Sample *pNew, int nEqZero){ p->iMin = iMin; } } -#endif /* SQLITE_ENABLE_STAT3_OR_STAT4 */ +#endif /* SQLITE_ENABLE_STAT4 */ /* ** Field iChng of the index being scanned has changed. So at this point @@ -104430,28 +105516,7 @@ static void samplePushPrevious(Stat4Accum *p, int iChng){ } #endif -#if defined(SQLITE_ENABLE_STAT3) && !defined(SQLITE_ENABLE_STAT4) - if( iChng==0 ){ - tRowcnt nLt = p->current.anLt[0]; - tRowcnt nEq = p->current.anEq[0]; - - /* Check if this is to be a periodic sample. If so, add it. */ - if( (nLt/p->nPSample)!=(nLt+nEq)/p->nPSample ){ - p->current.isPSample = 1; - sampleInsert(p, &p->current, 0); - p->current.isPSample = 0; - }else - - /* Or if it is a non-periodic sample. Add it in this case too. */ - if( p->nSamplemxSample - || sampleIsBetter(p, &p->current, &p->a[p->iMin]) - ){ - sampleInsert(p, &p->current, 0); - } - } -#endif - -#ifndef SQLITE_ENABLE_STAT3_OR_STAT4 +#ifndef SQLITE_ENABLE_STAT4 UNUSED_PARAMETER( p ); UNUSED_PARAMETER( iChng ); #endif @@ -104471,7 +105536,7 @@ static void samplePushPrevious(Stat4Accum *p, int iChng){ ** index being analyzed. The stat_get() SQL function will later be used to ** extract relevant information for constructing the sqlite_statN tables. ** -** The R parameter is only used for STAT3 and STAT4 +** The R parameter is only used for STAT4 */ static void statPush( sqlite3_context *context, @@ -104503,14 +105568,14 @@ static void statPush( } for(i=iChng; inCol; i++){ p->current.anDLt[i]++; -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 +#ifdef SQLITE_ENABLE_STAT4 p->current.anLt[i] += p->current.anEq[i]; #endif p->current.anEq[i] = 1; } } p->nRow++; -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 +#ifdef SQLITE_ENABLE_STAT4 if( sqlite3_value_type(argv[2])==SQLITE_INTEGER ){ sampleSetRowidInt64(p->db, &p->current, sqlite3_value_int64(argv[2])); }else{ @@ -104543,7 +105608,7 @@ static void statPush( #endif } static const FuncDef statPushFuncdef = { - 2+IsStat34, /* nArg */ + 2+IsStat4, /* nArg */ SQLITE_UTF8, /* funcFlags */ 0, /* pUserData */ 0, /* pNext */ @@ -104574,7 +105639,7 @@ static const FuncDef statPushFuncdef = { ** parameter will always be a poiner to a Stat4Accum object, never a ** NULL. ** -** If neither STAT3 nor STAT4 are enabled, then J is always +** If STAT4 is not enabled, then J is always ** STAT_GET_STAT1 and is hence omitted and this routine becomes ** a one-parameter function, stat_get(P), that always returns the ** stat1 table entry information. @@ -104585,8 +105650,8 @@ static void statGet( sqlite3_value **argv ){ Stat4Accum *p = (Stat4Accum*)sqlite3_value_blob(argv[0]); -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 - /* STAT3 and STAT4 have a parameter on this routine. */ +#ifdef SQLITE_ENABLE_STAT4 + /* STAT4 has a parameter on this routine. */ int eCall = sqlite3_value_int(argv[1]); assert( argc==2 ); assert( eCall==STAT_GET_STAT1 || eCall==STAT_GET_NEQ @@ -104641,7 +105706,7 @@ static void statGet( sqlite3_result_text(context, zRet, -1, sqlite3_free); } -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 +#ifdef SQLITE_ENABLE_STAT4 else if( eCall==STAT_GET_ROWID ){ if( p->iGet<0 ){ samplePushPrevious(p, 0); @@ -104670,9 +105735,7 @@ static void statGet( } } - if( IsStat3 ){ - sqlite3_result_int64(context, (i64)aCnt[0]); - }else{ + { char *zRet = sqlite3MallocZero(p->nCol * 25); if( zRet==0 ){ sqlite3_result_error_nomem(context); @@ -104689,13 +105752,13 @@ static void statGet( } } } -#endif /* SQLITE_ENABLE_STAT3_OR_STAT4 */ +#endif /* SQLITE_ENABLE_STAT4 */ #ifndef SQLITE_DEBUG UNUSED_PARAMETER( argc ); #endif } static const FuncDef statGetFuncdef = { - 1+IsStat34, /* nArg */ + 1+IsStat4, /* nArg */ SQLITE_UTF8, /* funcFlags */ 0, /* pUserData */ 0, /* pNext */ @@ -104708,7 +105771,7 @@ static const FuncDef statGetFuncdef = { static void callStatGet(Vdbe *v, int regStat4, int iParam, int regOut){ assert( regOut!=regStat4 && regOut!=regStat4+1 ); -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 +#ifdef SQLITE_ENABLE_STAT4 sqlite3VdbeAddOp2(v, OP_Integer, iParam, regStat4+1); #elif SQLITE_DEBUG assert( iParam==STAT_GET_STAT1 ); @@ -104717,7 +105780,7 @@ static void callStatGet(Vdbe *v, int regStat4, int iParam, int regOut){ #endif sqlite3VdbeAddOp4(v, OP_Function0, 0, regStat4, regOut, (char*)&statGetFuncdef, P4_FUNCDEF); - sqlite3VdbeChangeP5(v, 1 + IsStat34); + sqlite3VdbeChangeP5(v, 1 + IsStat4); } /* @@ -104744,7 +105807,7 @@ static void analyzeOneTable( int regNewRowid = iMem++; /* Rowid for the inserted record */ int regStat4 = iMem++; /* Register to hold Stat4Accum object */ int regChng = iMem++; /* Index of changed index field */ -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 +#ifdef SQLITE_ENABLE_STAT4 int regRowid = iMem++; /* Rowid argument passed to stat_push() */ #endif int regTemp = iMem++; /* Temporary use register */ @@ -104878,16 +105941,16 @@ static void analyzeOneTable( ** (3) the number of rows in the index, ** ** - ** The third argument is only used for STAT3 and STAT4 + ** The third argument is only used for STAT4 */ -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 +#ifdef SQLITE_ENABLE_STAT4 sqlite3VdbeAddOp2(v, OP_Count, iIdxCur, regStat4+3); #endif sqlite3VdbeAddOp2(v, OP_Integer, nCol, regStat4+1); sqlite3VdbeAddOp2(v, OP_Integer, pIdx->nKeyCol, regStat4+2); sqlite3VdbeAddOp4(v, OP_Function0, 0, regStat4+1, regStat4, (char*)&statInitFuncdef, P4_FUNCDEF); - sqlite3VdbeChangeP5(v, 2+IsStat34); + sqlite3VdbeChangeP5(v, 2+IsStat4); /* Implementation of the following: ** @@ -104958,12 +106021,12 @@ static void analyzeOneTable( /* ** chng_addr_N: - ** regRowid = idx(rowid) // STAT34 only - ** stat_push(P, regChng, regRowid) // 3rd parameter STAT34 only + ** regRowid = idx(rowid) // STAT4 only + ** stat_push(P, regChng, regRowid) // 3rd parameter STAT4 only ** Next csr ** if !eof(csr) goto next_row; */ -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 +#ifdef SQLITE_ENABLE_STAT4 assert( regRowid==(regStat4+2) ); if( HasRowid(pTab) ){ sqlite3VdbeAddOp2(v, OP_IdxRowid, iIdxCur, regRowid); @@ -104984,7 +106047,7 @@ static void analyzeOneTable( assert( regChng==(regStat4+1) ); sqlite3VdbeAddOp4(v, OP_Function0, 1, regStat4, regTemp, (char*)&statPushFuncdef, P4_FUNCDEF); - sqlite3VdbeChangeP5(v, 2+IsStat34); + sqlite3VdbeChangeP5(v, 2+IsStat4); sqlite3VdbeAddOp2(v, OP_Next, iIdxCur, addrNextRow); VdbeCoverage(v); /* Add the entry to the stat1 table. */ @@ -104998,8 +106061,8 @@ static void analyzeOneTable( #endif sqlite3VdbeChangeP5(v, OPFLAG_APPEND); - /* Add the entries to the stat3 or stat4 table. */ -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 + /* Add the entries to the stat4 table. */ +#ifdef SQLITE_ENABLE_STAT4 { int regEq = regStat1; int regLt = regStat1+1; @@ -105022,21 +106085,17 @@ static void analyzeOneTable( callStatGet(v, regStat4, STAT_GET_NDLT, regDLt); sqlite3VdbeAddOp4Int(v, seekOp, iTabCur, addrNext, regSampleRowid, 0); VdbeCoverage(v); -#ifdef SQLITE_ENABLE_STAT3 - sqlite3ExprCodeLoadIndexColumn(pParse, pIdx, iTabCur, 0, regSample); -#else for(i=0; ibUnordered = 1; }else if( sqlite3_strglob("sz=[0-9]*", z)==0 ){ - pIndex->szIdxRow = sqlite3LogEst(sqlite3Atoi(z+3)); + int sz = sqlite3Atoi(z+3); + if( sz<2 ) sz = 2; + pIndex->szIdxRow = sqlite3LogEst(sz); }else if( sqlite3_strglob("noskipscan*", z)==0 ){ pIndex->noSkipScan = 1; } @@ -105298,7 +106359,7 @@ static int analysisLoader(void *pData, int argc, char **argv, char **NotUsed){ if( pIndex ){ tRowcnt *aiRowEst = 0; int nCol = pIndex->nKeyCol+1; -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 +#ifdef SQLITE_ENABLE_STAT4 /* Index.aiRowEst may already be set here if there are duplicate ** sqlite_stat1 entries for this index. In that case just clobber ** the old data with the new instead of allocating a new array. */ @@ -105334,7 +106395,7 @@ static int analysisLoader(void *pData, int argc, char **argv, char **NotUsed){ ** and its contents. */ SQLITE_PRIVATE void sqlite3DeleteIndexSamples(sqlite3 *db, Index *pIdx){ -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 +#ifdef SQLITE_ENABLE_STAT4 if( pIdx->aSample ){ int j; for(j=0; jnSample; j++){ @@ -105350,10 +106411,10 @@ SQLITE_PRIVATE void sqlite3DeleteIndexSamples(sqlite3 *db, Index *pIdx){ #else UNUSED_PARAMETER(db); UNUSED_PARAMETER(pIdx); -#endif /* SQLITE_ENABLE_STAT3_OR_STAT4 */ +#endif /* SQLITE_ENABLE_STAT4 */ } -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 +#ifdef SQLITE_ENABLE_STAT4 /* ** Populate the pIdx->aAvgEq[] array based on the samples currently ** stored in pIdx->aSample[]. @@ -105431,12 +106492,11 @@ static Index *findIndexOrPrimaryKey( } /* -** Load the content from either the sqlite_stat4 or sqlite_stat3 table +** Load the content from either the sqlite_stat4 ** into the relevant Index.aSample[] arrays. ** ** Arguments zSql1 and zSql2 must point to SQL statements that return -** data equivalent to the following (statements are different for stat3, -** see the caller of this function for details): +** data equivalent to the following: ** ** zSql1: SELECT idx,count(*) FROM %Q.sqlite_stat4 GROUP BY idx ** zSql2: SELECT idx,neq,nlt,ndlt,sample FROM %Q.sqlite_stat4 @@ -105445,7 +106505,6 @@ static Index *findIndexOrPrimaryKey( */ static int loadStatTbl( sqlite3 *db, /* Database handle */ - int bStat3, /* Assume single column records only */ const char *zSql1, /* SQL statement 1 (see above) */ const char *zSql2, /* SQL statement 2 (see above) */ const char *zDb /* Database name (e.g. "main") */ @@ -105479,17 +106538,13 @@ static int loadStatTbl( if( zIndex==0 ) continue; nSample = sqlite3_column_int(pStmt, 1); pIdx = findIndexOrPrimaryKey(db, zIndex, zDb); - assert( pIdx==0 || bStat3 || pIdx->nSample==0 ); - /* Index.nSample is non-zero at this point if data has already been - ** loaded from the stat4 table. In this case ignore stat3 data. */ - if( pIdx==0 || pIdx->nSample ) continue; - if( bStat3==0 ){ - assert( !HasRowid(pIdx->pTable) || pIdx->nColumn==pIdx->nKeyCol+1 ); - if( !HasRowid(pIdx->pTable) && IsPrimaryKeyIndex(pIdx) ){ - nIdxCol = pIdx->nKeyCol; - }else{ - nIdxCol = pIdx->nColumn; - } + assert( pIdx==0 || pIdx->nSample==0 ); + if( pIdx==0 ) continue; + assert( !HasRowid(pIdx->pTable) || pIdx->nColumn==pIdx->nKeyCol+1 ); + if( !HasRowid(pIdx->pTable) && IsPrimaryKeyIndex(pIdx) ){ + nIdxCol = pIdx->nKeyCol; + }else{ + nIdxCol = pIdx->nColumn; } pIdx->nSampleCol = nIdxCol; nByte = sizeof(IndexSample) * nSample; @@ -105531,9 +106586,8 @@ static int loadStatTbl( pIdx = findIndexOrPrimaryKey(db, zIndex, zDb); if( pIdx==0 ) continue; /* This next condition is true if data has already been loaded from - ** the sqlite_stat4 table. In this case ignore stat3 data. */ + ** the sqlite_stat4 table. */ nCol = pIdx->nSampleCol; - if( bStat3 && nCol>1 ) continue; if( pIdx!=pPrevIdx ){ initAvgEq(pPrevIdx); pPrevIdx = pIdx; @@ -105566,7 +106620,7 @@ static int loadStatTbl( } /* -** Load content from the sqlite_stat4 and sqlite_stat3 tables into +** Load content from the sqlite_stat4 table into ** the Index.aSample[] arrays of all indices. */ static int loadStat4(sqlite3 *db, const char *zDb){ @@ -105574,37 +106628,28 @@ static int loadStat4(sqlite3 *db, const char *zDb){ assert( db->lookaside.bDisable ); if( sqlite3FindTable(db, "sqlite_stat4", zDb) ){ - rc = loadStatTbl(db, 0, + rc = loadStatTbl(db, "SELECT idx,count(*) FROM %Q.sqlite_stat4 GROUP BY idx", "SELECT idx,neq,nlt,ndlt,sample FROM %Q.sqlite_stat4", zDb ); } - - if( rc==SQLITE_OK && sqlite3FindTable(db, "sqlite_stat3", zDb) ){ - rc = loadStatTbl(db, 1, - "SELECT idx,count(*) FROM %Q.sqlite_stat3 GROUP BY idx", - "SELECT idx,neq,nlt,ndlt,sqlite_record(sample) FROM %Q.sqlite_stat3", - zDb - ); - } - return rc; } -#endif /* SQLITE_ENABLE_STAT3_OR_STAT4 */ +#endif /* SQLITE_ENABLE_STAT4 */ /* -** Load the content of the sqlite_stat1 and sqlite_stat3/4 tables. The +** Load the content of the sqlite_stat1 and sqlite_stat4 tables. The ** contents of sqlite_stat1 are used to populate the Index.aiRowEst[] -** arrays. The contents of sqlite_stat3/4 are used to populate the +** arrays. The contents of sqlite_stat4 are used to populate the ** Index.aSample[] arrays. ** ** If the sqlite_stat1 table is not present in the database, SQLITE_ERROR -** is returned. In this case, even if SQLITE_ENABLE_STAT3/4 was defined -** during compilation and the sqlite_stat3/4 table is present, no data is +** is returned. In this case, even if SQLITE_ENABLE_STAT4 was defined +** during compilation and the sqlite_stat4 table is present, no data is ** read from it. ** -** If SQLITE_ENABLE_STAT3/4 was defined during compilation and the +** If SQLITE_ENABLE_STAT4 was defined during compilation and the ** sqlite_stat4 table is not present in the database, SQLITE_ERROR is ** returned. However, in this case, data is read from the sqlite_stat1 ** table (if it is present) before returning. @@ -105632,7 +106677,7 @@ SQLITE_PRIVATE int sqlite3AnalysisLoad(sqlite3 *db, int iDb){ for(i=sqliteHashFirst(&pSchema->idxHash); i; i=sqliteHashNext(i)){ Index *pIdx = sqliteHashData(i); pIdx->hasStat1 = 0; -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 +#ifdef SQLITE_ENABLE_STAT4 sqlite3DeleteIndexSamples(db, pIdx); pIdx->aSample = 0; #endif @@ -105660,7 +106705,7 @@ SQLITE_PRIVATE int sqlite3AnalysisLoad(sqlite3 *db, int iDb){ } /* Load the statistics from the sqlite_stat4 table. */ -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 +#ifdef SQLITE_ENABLE_STAT4 if( rc==SQLITE_OK ){ db->lookaside.bDisable++; rc = loadStat4(db, sInfo.zDatabase); @@ -105920,12 +106965,14 @@ static void attachFunc( sqlite3BtreeEnterAll(db); db->init.iDb = 0; db->mDbFlags &= ~(DBFLAG_SchemaKnownOk); - rc = sqlite3Init(db, &zErrDyn); + if( !REOPEN_AS_MEMDB(db) ){ + rc = sqlite3Init(db, &zErrDyn); + } sqlite3BtreeLeaveAll(db); assert( zErrDyn==0 || rc!=SQLITE_OK ); } #ifdef SQLITE_USER_AUTHENTICATION - if( rc==SQLITE_OK ){ + if( rc==SQLITE_OK && !REOPEN_AS_MEMDB(db) ){ u8 newAuth = 0; rc = sqlite3UserAuthCheckLogin(db, zName, &newAuth); if( newAuthauth.authLevel ){ @@ -105983,6 +107030,7 @@ static void detachFunc( sqlite3 *db = sqlite3_context_db_handle(context); int i; Db *pDb = 0; + HashElem *pEntry; char zErr[128]; UNUSED_PARAMETER(NotUsed); @@ -106007,6 +107055,18 @@ static void detachFunc( goto detach_error; } + /* If any TEMP triggers reference the schema being detached, move those + ** triggers to reference the TEMP schema itself. */ + assert( db->aDb[1].pSchema ); + pEntry = sqliteHashFirst(&db->aDb[1].pSchema->trigHash); + while( pEntry ){ + Trigger *pTrig = (Trigger*)sqliteHashData(pEntry); + if( pTrig->pTabSchema==pDb->pSchema ){ + pTrig->pTabSchema = pTrig->pSchema; + } + pEntry = sqliteHashNext(pEntry); + } + sqlite3BtreeClose(pDb->pBt); pDb->pBt = 0; pDb->pSchema = 0; @@ -106244,6 +107304,7 @@ SQLITE_PRIVATE int sqlite3FixExpr( Expr *pExpr /* The expression to be fixed to one database */ ){ while( pExpr ){ + ExprSetProperty(pExpr, EP_Indirect); if( pExpr->op==TK_VARIABLE ){ if( pFix->pParse->db->init.busy ){ pExpr->op = TK_NULL; @@ -106396,7 +107457,7 @@ SQLITE_API int sqlite3_set_authorizer( sqlite3_mutex_enter(db->mutex); db->xAuth = (sqlite3_xauth)xAuth; db->pAuthArg = pArg; - sqlite3ExpirePreparedStatements(db, 0); + if( db->xAuth ) sqlite3ExpirePreparedStatements(db, 1); sqlite3_mutex_leave(db->mutex); return SQLITE_OK; } @@ -106854,7 +107915,12 @@ SQLITE_PRIVATE void sqlite3NestedParse(Parse *pParse, const char *zFormat, ...){ zSql = sqlite3VMPrintf(db, zFormat, ap); va_end(ap); if( zSql==0 ){ - return; /* A malloc must have failed */ + /* This can result either from an OOM or because the formatted string + ** exceeds SQLITE_LIMIT_LENGTH. In the latter case, we need to set + ** an error */ + if( !db->mallocFailed ) pParse->rc = SQLITE_TOOBIG; + pParse->nErr++; + return; } pParse->nested++; memcpy(saveBuf, PARSE_TAIL(pParse), PARSE_TAIL_SZ); @@ -107045,7 +108111,7 @@ SQLITE_PRIVATE void sqlite3FreeIndex(sqlite3 *db, Index *p){ sqlite3ExprListDelete(db, p->aColExpr); sqlite3DbFree(db, p->zColAff); if( p->isResized ) sqlite3DbFree(db, (void *)p->azColl); -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 +#ifdef SQLITE_ENABLE_STAT4 sqlite3_free(p->aiRowEst); #endif sqlite3DbFree(db, p); @@ -107207,10 +108273,14 @@ static void SQLITE_NOINLINE deleteTable(sqlite3 *db, Table *pTable){ #ifdef SQLITE_DEBUG /* Record the number of outstanding lookaside allocations in schema Tables - ** prior to doing any free() operations. Since schema Tables do not use - ** lookaside, this number should not change. */ + ** prior to doing any free() operations. Since schema Tables do not use + ** lookaside, this number should not change. + ** + ** If malloc has already failed, it may be that it failed while allocating + ** a Table object that was going to be marked ephemeral. So do not check + ** that no lookaside memory is used in this case either. */ int nLookaside = 0; - if( db && (pTable->tabFlags & TF_Ephemeral)==0 ){ + if( db && !db->mallocFailed && (pTable->tabFlags & TF_Ephemeral)==0 ){ nLookaside = sqlite3LookasideUsed(db, 0); } #endif @@ -107414,13 +108484,40 @@ SQLITE_PRIVATE int sqlite3WritableSchema(sqlite3 *db){ ** trigger). All names are legal except those that begin with the string ** "sqlite_" (in upper, lower or mixed case). This portion of the namespace ** is reserved for internal use. +** +** When parsing the sqlite_master table, this routine also checks to +** make sure the "type", "name", and "tbl_name" columns are consistent +** with the SQL. */ -SQLITE_PRIVATE int sqlite3CheckObjectName(Parse *pParse, const char *zName){ - if( !pParse->db->init.busy && pParse->nested==0 - && sqlite3WritableSchema(pParse->db)==0 - && 0==sqlite3StrNICmp(zName, "sqlite_", 7) ){ - sqlite3ErrorMsg(pParse, "object name reserved for internal use: %s", zName); - return SQLITE_ERROR; +SQLITE_PRIVATE int sqlite3CheckObjectName( + Parse *pParse, /* Parsing context */ + const char *zName, /* Name of the object to check */ + const char *zType, /* Type of this object */ + const char *zTblName /* Parent table name for triggers and indexes */ +){ + sqlite3 *db = pParse->db; + if( sqlite3WritableSchema(db) || db->init.imposterTable ){ + /* Skip these error checks for writable_schema=ON */ + return SQLITE_OK; + } + if( db->init.busy ){ + if( sqlite3_stricmp(zType, db->init.azInit[0]) + || sqlite3_stricmp(zName, db->init.azInit[1]) + || sqlite3_stricmp(zTblName, db->init.azInit[2]) + ){ + if( sqlite3Config.bExtraSchemaChecks ){ + sqlite3ErrorMsg(pParse, ""); /* corruptSchema() will supply the error */ + return SQLITE_ERROR; + } + } + }else{ + if( pParse->nested==0 + && 0==sqlite3StrNICmp(zName, "sqlite_", 7) + ){ + sqlite3ErrorMsg(pParse, "object name reserved for internal use: %s", + zName); + return SQLITE_ERROR; + } } return SQLITE_OK; } @@ -107501,7 +108598,7 @@ SQLITE_PRIVATE void sqlite3StartTable( } pParse->sNameToken = *pName; if( zName==0 ) return; - if( SQLITE_OK!=sqlite3CheckObjectName(pParse, zName) ){ + if( sqlite3CheckObjectName(pParse, zName, isView?"view":"table", zName) ){ goto begin_table_error; } if( db->init.iDb==1 ) isTemp = 1; @@ -107918,7 +109015,7 @@ SQLITE_PRIVATE void sqlite3AddDefaultValue( ** accept it. This routine does the necessary conversion. It converts ** the expression given in its argument from a TK_STRING into a TK_ID ** if the expression is just a TK_STRING with an optional COLLATE clause. -** If the epxression is anything other than TK_STRING, the expression is +** If the expression is anything other than TK_STRING, the expression is ** unchanged. */ static void sqlite3StringToId(Expr *p){ @@ -107994,13 +109091,14 @@ SQLITE_PRIVATE void sqlite3AddPrimaryKey( && sortOrder!=SQLITE_SO_DESC ){ if( IN_RENAME_OBJECT && pList ){ - sqlite3RenameTokenRemap(pParse, &pTab->iPKey, pList->a[0].pExpr); + Expr *pCExpr = sqlite3ExprSkipCollate(pList->a[0].pExpr); + sqlite3RenameTokenRemap(pParse, &pTab->iPKey, pCExpr); } pTab->iPKey = iCol; pTab->keyConf = (u8)onError; assert( autoInc==0 || autoInc==1 ); pTab->tabFlags |= autoInc*TF_Autoincrement; - if( pList ) pParse->iPkSortOrder = pList->a[0].sortOrder; + if( pList ) pParse->iPkSortOrder = pList->a[0].sortFlags; }else if( autoInc ){ #ifndef SQLITE_OMIT_AUTOINCREMENT sqlite3ErrorMsg(pParse, "AUTOINCREMENT is only allowed on an " @@ -108314,10 +109412,51 @@ static void estimateIndexWidth(Index *pIdx){ pIdx->szIdxRow = sqlite3LogEst(wIndex*4); } -/* Return true if value x is found any of the first nCol entries of aiCol[] +/* Return true if column number x is any of the first nCol entries of aiCol[]. +** This is used to determine if the column number x appears in any of the +** first nCol entries of an index. */ static int hasColumn(const i16 *aiCol, int nCol, int x){ - while( nCol-- > 0 ) if( x==*(aiCol++) ) return 1; + while( nCol-- > 0 ){ + assert( aiCol[0]>=0 ); + if( x==*(aiCol++) ){ + return 1; + } + } + return 0; +} + +/* +** Return true if any of the first nKey entries of index pIdx exactly +** match the iCol-th entry of pPk. pPk is always a WITHOUT ROWID +** PRIMARY KEY index. pIdx is an index on the same table. pIdx may +** or may not be the same index as pPk. +** +** The first nKey entries of pIdx are guaranteed to be ordinary columns, +** not a rowid or expression. +** +** This routine differs from hasColumn() in that both the column and the +** collating sequence must match for this routine, but for hasColumn() only +** the column name must match. +*/ +static int isDupColumn(Index *pIdx, int nKey, Index *pPk, int iCol){ + int i, j; + assert( nKey<=pIdx->nColumn ); + assert( iColnColumn,pPk->nKeyCol) ); + assert( pPk->idxType==SQLITE_IDXTYPE_PRIMARYKEY ); + assert( pPk->pTable->tabFlags & TF_WithoutRowid ); + assert( pPk->pTable==pIdx->pTable ); + testcase( pPk==pIdx ); + j = pPk->aiColumn[iCol]; + assert( j!=XN_ROWID && j!=XN_EXPR ); + for(i=0; iaiColumn[i]>=0 || j>=0 ); + if( pIdx->aiColumn[i]==j + && sqlite3StrICmp(pIdx->azColl[i], pPk->azColl[iCol])==0 + ){ + return 1; + } + } return 0; } @@ -108374,6 +109513,7 @@ static void convertToWithoutRowidTable(Parse *pParse, Table *pTab){ Index *pIdx; Index *pPk; int nPk; + int nExtra; int i, j; sqlite3 *db = pParse->db; Vdbe *v = pParse->pVdbe; @@ -108406,15 +109546,20 @@ static void convertToWithoutRowidTable(Parse *pParse, Table *pTab){ pList = sqlite3ExprListAppend(pParse, 0, sqlite3ExprAlloc(db, TK_ID, &ipkToken, 0)); if( pList==0 ) return; - pList->a[0].sortOrder = pParse->iPkSortOrder; + if( IN_RENAME_OBJECT ){ + sqlite3RenameTokenRemap(pParse, pList->a[0].pExpr, &pTab->iPKey); + } + pList->a[0].sortFlags = pParse->iPkSortOrder; assert( pParse->pNewTable==pTab ); + pTab->iPKey = -1; sqlite3CreateIndex(pParse, 0, 0, 0, pList, pTab->keyConf, 0, 0, 0, 0, SQLITE_IDXTYPE_PRIMARYKEY); if( db->mallocFailed || pParse->nErr ) return; pPk = sqlite3PrimaryKeyIndex(pTab); - pTab->iPKey = -1; + assert( pPk->nKeyCol==1 ); }else{ pPk = sqlite3PrimaryKeyIndex(pTab); + assert( pPk!=0 ); /* ** Remove all redundant columns from the PRIMARY KEY. For example, change @@ -108422,9 +109567,12 @@ static void convertToWithoutRowidTable(Parse *pParse, Table *pTab){ ** code assumes the PRIMARY KEY contains no repeated columns. */ for(i=j=1; inKeyCol; i++){ - if( hasColumn(pPk->aiColumn, j, pPk->aiColumn[i]) ){ + if( isDupColumn(pPk, j, pPk, i) ){ pPk->nColumn--; }else{ + testcase( hasColumn(pPk->aiColumn, j, pPk->aiColumn[i]) ); + pPk->azColl[j] = pPk->azColl[i]; + pPk->aSortOrder[j] = pPk->aSortOrder[i]; pPk->aiColumn[j++] = pPk->aiColumn[i]; } } @@ -108433,7 +109581,7 @@ static void convertToWithoutRowidTable(Parse *pParse, Table *pTab){ assert( pPk!=0 ); pPk->isCovering = 1; if( !db->init.imposterTable ) pPk->uniqNotNull = 1; - nPk = pPk->nKeyCol; + nPk = pPk->nColumn = pPk->nKeyCol; /* Bypass the creation of the PRIMARY KEY btree and the sqlite_master ** table entry. This is only required if currently generating VDBE @@ -108454,7 +109602,10 @@ static void convertToWithoutRowidTable(Parse *pParse, Table *pTab){ int n; if( IsPrimaryKeyIndex(pIdx) ) continue; for(i=n=0; iaiColumn, pIdx->nKeyCol, pPk->aiColumn[i]) ) n++; + if( !isDupColumn(pIdx, pIdx->nKeyCol, pPk, i) ){ + testcase( hasColumn(pIdx->aiColumn, pIdx->nKeyCol, pPk->aiColumn[i]) ); + n++; + } } if( n==0 ){ /* This index is a superset of the primary key */ @@ -108463,9 +109614,14 @@ static void convertToWithoutRowidTable(Parse *pParse, Table *pTab){ } if( resizeIndexObject(db, pIdx, pIdx->nKeyCol+n) ) return; for(i=0, j=pIdx->nKeyCol; iaiColumn, pIdx->nKeyCol, pPk->aiColumn[i]) ){ + if( !isDupColumn(pIdx, pIdx->nKeyCol, pPk, i) ){ + testcase( hasColumn(pIdx->aiColumn, pIdx->nKeyCol, pPk->aiColumn[i]) ); pIdx->aiColumn[j] = pPk->aiColumn[i]; pIdx->azColl[j] = pPk->azColl[i]; + if( pPk->aSortOrder[i] ){ + /* See ticket https://www.sqlite.org/src/info/bba7b69f9849b5bf */ + pIdx->bAscKeyBug = 1; + } j++; } } @@ -108475,21 +109631,21 @@ static void convertToWithoutRowidTable(Parse *pParse, Table *pTab){ /* Add all table columns to the PRIMARY KEY index */ - if( nPknCol ){ - if( resizeIndexObject(db, pPk, pTab->nCol) ) return; - for(i=0, j=nPk; inCol; i++){ - if( !hasColumn(pPk->aiColumn, j, i) ){ - assert( jnColumn ); - pPk->aiColumn[j] = i; - pPk->azColl[j] = sqlite3StrBINARY; - j++; - } + nExtra = 0; + for(i=0; inCol; i++){ + if( !hasColumn(pPk->aiColumn, nPk, i) ) nExtra++; + } + if( resizeIndexObject(db, pPk, nPk+nExtra) ) return; + for(i=0, j=nPk; inCol; i++){ + if( !hasColumn(pPk->aiColumn, j, i) ){ + assert( jnColumn ); + pPk->aiColumn[j] = i; + pPk->azColl[j] = sqlite3StrBINARY; + j++; } - assert( pPk->nColumn==j ); - assert( pTab->nCol==j ); - }else{ - pPk->nColumn = pTab->nCol; } + assert( pPk->nColumn==j ); + assert( pTab->nCol<=j ); recomputeColumnsNotIndexed(pPk); } @@ -108584,6 +109740,11 @@ SQLITE_PRIVATE void sqlite3EndTable( if( p->tnum==1 ) p->tabFlags |= TF_Readonly; } + assert( (p->tabFlags & TF_HasPrimaryKey)==0 + || p->iPKey>=0 || sqlite3PrimaryKeyIndex(p)!=0 ); + assert( (p->tabFlags & TF_HasPrimaryKey)!=0 + || (p->iPKey<0 && sqlite3PrimaryKeyIndex(p)==0) ); + /* Special processing for WITHOUT ROWID Tables */ if( tabOpts & TF_WithoutRowid ){ if( (p->tabFlags & TF_Autoincrement) ){ @@ -108681,7 +109842,7 @@ SQLITE_PRIVATE void sqlite3EndTable( addrTop = sqlite3VdbeCurrentAddr(v) + 1; sqlite3VdbeAddOp3(v, OP_InitCoroutine, regYield, 0, addrTop); if( pParse->nErr ) return; - pSelTab = sqlite3ResultSetOfSelect(pParse, pSelect); + pSelTab = sqlite3ResultSetOfSelect(pParse, pSelect, SQLITE_AFF_BLOB); if( pSelTab==0 ) return; assert( p->aCol==0 ); p->nCol = pSelTab->nCol; @@ -108945,10 +110106,10 @@ SQLITE_PRIVATE int sqlite3ViewGetColumnNames(Parse *pParse, Table *pTable){ #ifndef SQLITE_OMIT_AUTHORIZATION xAuth = db->xAuth; db->xAuth = 0; - pSelTab = sqlite3ResultSetOfSelect(pParse, pSel); + pSelTab = sqlite3ResultSetOfSelect(pParse, pSel, SQLITE_AFF_NONE); db->xAuth = xAuth; #else - pSelTab = sqlite3ResultSetOfSelect(pParse, pSel); + pSelTab = sqlite3ResultSetOfSelect(pParse, pSel, SQLITE_AFF_NONE); #endif pParse->nTab = n; if( pTable->pCheck ){ @@ -108964,7 +110125,8 @@ SQLITE_PRIVATE int sqlite3ViewGetColumnNames(Parse *pParse, Table *pTable){ && pParse->nErr==0 && pTable->nCol==pSel->pEList->nExpr ){ - sqlite3SelectAddColumnTypeAndCollation(pParse, pTable, pSel); + sqlite3SelectAddColumnTypeAndCollation(pParse, pTable, pSel, + SQLITE_AFF_NONE); } }else if( pSelTab ){ /* CREATE VIEW name AS... without an argument list. Construct @@ -109309,7 +110471,8 @@ SQLITE_PRIVATE void sqlite3DropTable(Parse *pParse, SrcList *pName, int isView, } #endif if( sqlite3StrNICmp(pTab->zName, "sqlite_", 7)==0 - && sqlite3StrNICmp(pTab->zName, "sqlite_stat", 11)!=0 ){ + && sqlite3StrNICmp(pTab->zName+7, "stat", 4)!=0 + && sqlite3StrNICmp(pTab->zName+7, "parameters", 10)!=0 ){ sqlite3ErrorMsg(pParse, "table %s may not be dropped", pTab->zName); goto exit_drop_table; } @@ -109579,10 +110742,27 @@ static void sqlite3RefillIndex(Parse *pParse, Index *pIndex, int memRootPage){ sqlite3UniqueConstraint(pParse, OE_Abort, pIndex); sqlite3VdbeJumpHere(v, j2); }else{ + /* Most CREATE INDEX and REINDEX statements that are not UNIQUE can not + ** abort. The exception is if one of the indexed expressions contains a + ** user function that throws an exception when it is evaluated. But the + ** overhead of adding a statement journal to a CREATE INDEX statement is + ** very small (since most of the pages written do not contain content that + ** needs to be restored if the statement aborts), so we call + ** sqlite3MayAbort() for all CREATE INDEX statements. */ + sqlite3MayAbort(pParse); addr2 = sqlite3VdbeCurrentAddr(v); } sqlite3VdbeAddOp3(v, OP_SorterData, iSorter, regRecord, iIdx); - sqlite3VdbeAddOp1(v, OP_SeekEnd, iIdx); + if( !pIndex->bAscKeyBug ){ + /* This OP_SeekEnd opcode makes index insert for a REINDEX go much + ** faster by avoiding unnecessary seeks. But the optimization does + ** not work for UNIQUE constraint indexes on WITHOUT ROWID tables + ** with DESC primary keys, since those indexes have there keys in + ** a different order from the main table. + ** See ticket: https://www.sqlite.org/src/info/bba7b69f9849b5bf + */ + sqlite3VdbeAddOp1(v, OP_SeekEnd, iIdx); + } sqlite3VdbeAddOp2(v, OP_IdxInsert, iIdx, regRecord); sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT); sqlite3ReleaseTempReg(pParse, regRecord); @@ -109629,6 +110809,27 @@ SQLITE_PRIVATE Index *sqlite3AllocateIndexObject( return p; } +/* +** If expression list pList contains an expression that was parsed with +** an explicit "NULLS FIRST" or "NULLS LAST" clause, leave an error in +** pParse and return non-zero. Otherwise, return zero. +*/ +SQLITE_PRIVATE int sqlite3HasExplicitNulls(Parse *pParse, ExprList *pList){ + if( pList ){ + int i; + for(i=0; inExpr; i++){ + if( pList->a[i].bNulls ){ + u8 sf = pList->a[i].sortFlags; + sqlite3ErrorMsg(pParse, "unsupported use of NULLS %s", + (sf==0 || sf==3) ? "FIRST" : "LAST" + ); + return 1; + } + } + } + return 0; +} + /* ** Create a new index for an SQL table. pName1.pName2 is the name of the index ** and pTblList is the name of the table that is to be indexed. Both will @@ -109680,6 +110881,9 @@ SQLITE_PRIVATE void sqlite3CreateIndex( if( SQLITE_OK!=sqlite3ReadSchema(pParse) ){ goto exit_create_index; } + if( sqlite3HasExplicitNulls(pParse, pList) ){ + goto exit_create_index; + } /* ** Find the table that is to be indexed. Return early if not found. @@ -109737,13 +110941,13 @@ SQLITE_PRIVATE void sqlite3CreateIndex( assert( pParse->nErr==0 ); if( sqlite3StrNICmp(pTab->zName, "sqlite_", 7)==0 && db->init.busy==0 + && pTblName!=0 #if SQLITE_USER_AUTHENTICATION && sqlite3UserAuthTable(pTab->zName)==0 #endif #ifdef SQLITE_ALLOW_SQLITE_MASTER_INDEX && sqlite3StrICmp(&pTab->zName[7],"master")!=0 #endif - && sqlite3StrNICmp(&pTab->zName[7],"altertab_",9)!=0 ){ sqlite3ErrorMsg(pParse, "table %s may not be indexed", pTab->zName); goto exit_create_index; @@ -109778,7 +110982,7 @@ SQLITE_PRIVATE void sqlite3CreateIndex( zName = sqlite3NameFromToken(db, pName); if( zName==0 ) goto exit_create_index; assert( pName->z!=0 ); - if( SQLITE_OK!=sqlite3CheckObjectName(pParse, zName) ){ + if( SQLITE_OK!=sqlite3CheckObjectName(pParse, zName,"index",pTab->zName) ){ goto exit_create_index; } if( !IN_RENAME_OBJECT ){ @@ -109844,9 +111048,10 @@ SQLITE_PRIVATE void sqlite3CreateIndex( sqlite3ExprAlloc(db, TK_ID, &prevCol, 0)); if( pList==0 ) goto exit_create_index; assert( pList->nExpr==1 ); - sqlite3ExprListSetSortOrder(pList, sortOrder); + sqlite3ExprListSetSortOrder(pList, sortOrder, SQLITE_SO_UNDEFINED); }else{ sqlite3ExprListCheckLength(pParse, pList, "index"); + if( pParse->nErr ) goto exit_create_index; } /* Figure out how many bytes of space are required to store explicitly @@ -109865,6 +111070,7 @@ SQLITE_PRIVATE void sqlite3CreateIndex( */ nName = sqlite3Strlen30(zName); nExtraCol = pPk ? pPk->nKeyCol : 1; + assert( pList->nExpr + nExtraCol <= 32767 /* Fits in i16 */ ); pIndex = sqlite3AllocateIndexObject(db, pList->nExpr + nExtraCol, nName + nExtra + 1, &zExtra); if( db->mallocFailed ){ @@ -109960,7 +111166,7 @@ SQLITE_PRIVATE void sqlite3CreateIndex( goto exit_create_index; } pIndex->azColl[i] = zColl; - requestedSortOrder = pListItem->sortOrder & sortOrderMask; + requestedSortOrder = pListItem->sortFlags & sortOrderMask; pIndex->aSortOrder[i] = (u8)requestedSortOrder; } @@ -109972,9 +111178,10 @@ SQLITE_PRIVATE void sqlite3CreateIndex( for(j=0; jnKeyCol; j++){ int x = pPk->aiColumn[j]; assert( x>=0 ); - if( hasColumn(pIndex->aiColumn, pIndex->nKeyCol, x) ){ + if( isDupColumn(pIndex, pIndex->nKeyCol, pPk, j) ){ pIndex->nColumn--; }else{ + testcase( hasColumn(pIndex->aiColumn,pIndex->nKeyCol,x) ); pIndex->aiColumn[i] = x; pIndex->azColl[i] = pPk->azColl[j]; pIndex->aSortOrder[i] = pPk->aSortOrder[j]; @@ -110134,6 +111341,7 @@ SQLITE_PRIVATE void sqlite3CreateIndex( /* Gather the complete text of the CREATE INDEX statement into ** the zStmt variable */ + assert( pName!=0 || pStart==0 ); if( pStart ){ int n = (int)(pParse->sLastToken.z - pName->z) + pParse->sLastToken.n; if( pName->z[n-1]==';' ) n--; @@ -110348,9 +111556,9 @@ SQLITE_PRIVATE void *sqlite3ArrayAllocate( int *pIdx /* Write the index of a new slot here */ ){ char *z; - int n = *pnEntry; + sqlite3_int64 n = *pIdx = *pnEntry; if( (n & (n-1))==0 ){ - int sz = (n==0) ? 1 : 2*n; + sqlite3_int64 sz = (n==0) ? 1 : 2*n; void *pNew = sqlite3DbRealloc(db, pArray, sz*szEntry); if( pNew==0 ){ *pIdx = -1; @@ -110360,7 +111568,6 @@ SQLITE_PRIVATE void *sqlite3ArrayAllocate( } z = (char*)pArray; memset(&z[n * szEntry], 0, szEntry); - *pIdx = n; ++*pnEntry; return pArray; } @@ -110471,7 +111678,7 @@ SQLITE_PRIVATE SrcList *sqlite3SrcListEnlarge( /* Allocate additional space if needed */ if( (u32)pSrc->nSrc+nExtra>pSrc->nAlloc ){ SrcList *pNew; - int nAlloc = pSrc->nSrc*2+nExtra; + sqlite3_int64 nAlloc = 2*(sqlite3_int64)pSrc->nSrc+nExtra; sqlite3 *db = pParse->db; if( pSrc->nSrc+nExtra>=SQLITE_MAX_SRCLIST ){ @@ -110978,7 +112185,8 @@ SQLITE_PRIVATE void sqlite3UniqueConstraint( StrAccum errMsg; Table *pTab = pIdx->pTable; - sqlite3StrAccumInit(&errMsg, pParse->db, 0, 0, 200); + sqlite3StrAccumInit(&errMsg, pParse->db, 0, 0, + pParse->db->aLimit[SQLITE_LIMIT_LENGTH]); if( pIdx->aColExpr ){ sqlite3_str_appendf(&errMsg, "index '%q'", pIdx->zName); }else{ @@ -111176,7 +112384,8 @@ SQLITE_PRIVATE KeyInfo *sqlite3KeyInfoOfIndex(Parse *pParse, Index *pIdx){ const char *zColl = pIdx->azColl[i]; pKey->aColl[i] = zColl==sqlite3StrBINARY ? 0 : sqlite3LocateCollSeq(pParse, zColl); - pKey->aSortOrder[i] = pIdx->aSortOrder[i]; + pKey->aSortFlags[i] = pIdx->aSortOrder[i]; + assert( 0==(pKey->aSortFlags[i] & KEYINFO_ORDER_BIGNULL) ); } if( pParse->nErr ){ assert( pParse->rc==SQLITE_ERROR_MISSING_COLLSEQ ); @@ -111227,7 +112436,7 @@ SQLITE_PRIVATE With *sqlite3WithAdd( } if( pWith ){ - int nByte = sizeof(*pWith) + (sizeof(pWith->a[1]) * pWith->nCte); + sqlite3_int64 nByte = sizeof(*pWith) + (sizeof(pWith->a[1]) * pWith->nCte); pNew = sqlite3DbRealloc(db, pWith, nByte); }else{ pNew = sqlite3DbMallocZero(db, sizeof(*pWith)); @@ -112745,6 +113954,7 @@ SQLITE_PRIVATE void sqlite3ResolvePartIdxLabel(Parse *pParse, int iLabel){ /* #include "sqliteInt.h" */ /* #include */ /* #include */ +/* #include */ /* #include "vdbeInt.h" */ /* @@ -112931,6 +114141,8 @@ static void instrFunc( int N = 1; int isText; unsigned char firstChar; + sqlite3_value *pC1 = 0; + sqlite3_value *pC2 = 0; UNUSED_PARAMETER(argc); typeHaystack = sqlite3_value_type(argv[0]); @@ -112943,12 +114155,22 @@ static void instrFunc( zHaystack = sqlite3_value_blob(argv[0]); zNeedle = sqlite3_value_blob(argv[1]); isText = 0; - }else{ + }else if( typeHaystack!=SQLITE_BLOB && typeNeedle!=SQLITE_BLOB ){ zHaystack = sqlite3_value_text(argv[0]); zNeedle = sqlite3_value_text(argv[1]); isText = 1; + }else{ + pC1 = sqlite3_value_dup(argv[0]); + zHaystack = sqlite3_value_text(pC1); + if( zHaystack==0 ) goto endInstrOOM; + nHaystack = sqlite3_value_bytes(pC1); + pC2 = sqlite3_value_dup(argv[1]); + zNeedle = sqlite3_value_text(pC2); + if( zNeedle==0 ) goto endInstrOOM; + nNeedle = sqlite3_value_bytes(pC2); + isText = 1; } - if( zNeedle==0 || (nHaystack && zHaystack==0) ) return; + if( zNeedle==0 || (nHaystack && zHaystack==0) ) goto endInstrOOM; firstChar = zNeedle[0]; while( nNeedle<=nHaystack && (zHaystack[0]!=firstChar || memcmp(zHaystack, zNeedle, nNeedle)!=0) @@ -112962,6 +114184,13 @@ static void instrFunc( if( nNeedle>nHaystack ) N = 0; } sqlite3_result_int(context, N); +endInstr: + sqlite3_value_free(pC1); + sqlite3_value_free(pC2); + return; +endInstrOOM: + sqlite3_result_error_nomem(context); + goto endInstr; } /* @@ -113115,10 +114344,10 @@ static void roundFunc(sqlite3_context *context, int argc, sqlite3_value **argv){ ** handle the rounding directly, ** otherwise use printf. */ - if( n==0 && r>=0 && r+4503599627370496.0 ){ + /* The value has no fractional part so there is nothing to round */ + }else if( n==0 ){ + r = (double)((sqlite_int64)(r+(r<0?-0.5:+0.5))); }else{ zBuf = sqlite3_mprintf("%.*f",n,r); if( zBuf==0 ){ @@ -113572,8 +114801,6 @@ static void likeFunc( return; } #endif - zB = sqlite3_value_text(argv[0]); - zA = sqlite3_value_text(argv[1]); /* Limit the length of the LIKE or GLOB pattern to avoid problems ** of deep recursion and N*N behavior in patternCompare(). @@ -113585,8 +114812,6 @@ static void likeFunc( sqlite3_result_error(context, "LIKE or GLOB pattern too complex", -1); return; } - assert( zB==sqlite3_value_text(argv[0]) ); /* Encoding did not change */ - if( argc==3 ){ /* The escape character string must consist of a single UTF-8 character. ** Otherwise, return an error. @@ -113602,6 +114827,8 @@ static void likeFunc( }else{ escape = pInfo->matchSet; } + zB = sqlite3_value_text(argv[0]); + zA = sqlite3_value_text(argv[1]); if( zA && zB ){ #ifdef SQLITE_TEST sqlite3_like_count++; @@ -114527,35 +115754,24 @@ SQLITE_PRIVATE void sqlite3RegisterPerConnectionBuiltinFunctions(sqlite3 *db){ } /* -** Set the LIKEOPT flag on the 2-argument function with the given name. -*/ -static void setLikeOptFlag(sqlite3 *db, const char *zName, u8 flagVal){ - FuncDef *pDef; - pDef = sqlite3FindFunction(db, zName, 2, SQLITE_UTF8, 0); - if( ALWAYS(pDef) ){ - pDef->funcFlags |= flagVal; - } -} - -/* -** Register the built-in LIKE and GLOB functions. The caseSensitive +** Re-register the built-in LIKE functions. The caseSensitive ** parameter determines whether or not the LIKE operator is case -** sensitive. GLOB is always case sensitive. +** sensitive. */ SQLITE_PRIVATE void sqlite3RegisterLikeFunctions(sqlite3 *db, int caseSensitive){ struct compareInfo *pInfo; + int flags; if( caseSensitive ){ pInfo = (struct compareInfo*)&likeInfoAlt; + flags = SQLITE_FUNC_LIKE | SQLITE_FUNC_CASE; }else{ pInfo = (struct compareInfo*)&likeInfoNorm; + flags = SQLITE_FUNC_LIKE; } sqlite3CreateFunc(db, "like", 2, SQLITE_UTF8, pInfo, likeFunc, 0, 0, 0, 0, 0); sqlite3CreateFunc(db, "like", 3, SQLITE_UTF8, pInfo, likeFunc, 0, 0, 0, 0, 0); - sqlite3CreateFunc(db, "glob", 2, SQLITE_UTF8, - (struct compareInfo*)&globInfo, likeFunc, 0, 0, 0, 0, 0); - setLikeOptFlag(db, "glob", SQLITE_FUNC_LIKE | SQLITE_FUNC_CASE); - setLikeOptFlag(db, "like", - caseSensitive ? (SQLITE_FUNC_LIKE | SQLITE_FUNC_CASE) : SQLITE_FUNC_LIKE); + sqlite3FindFunction(db, "like", 2, SQLITE_UTF8, 0)->funcFlags |= flags; + sqlite3FindFunction(db, "like", 3, SQLITE_UTF8, 0)->funcFlags |= flags; } /* @@ -114727,9 +115943,6 @@ SQLITE_PRIVATE void sqlite3RegisterBuiltinFunctions(void){ sqlite3AlterFunctions(); #endif sqlite3WindowFunctions(); -#if defined(SQLITE_ENABLE_STAT3) || defined(SQLITE_ENABLE_STAT4) - sqlite3AnalyzeFunctions(); -#endif sqlite3RegisterDateTimeFunctions(); sqlite3InsertBuiltinFuncs(aBuiltinFunc, ArraySize(aBuiltinFunc)); @@ -115232,13 +116445,13 @@ static Expr *exprTableRegister( if( iCol>=0 && iCol!=pTab->iPKey ){ pCol = &pTab->aCol[iCol]; pExpr->iTable = regBase + iCol + 1; - pExpr->affinity = pCol->affinity; + pExpr->affExpr = pCol->affinity; zColl = pCol->zColl; if( zColl==0 ) zColl = db->pDfltColl->zName; pExpr = sqlite3ExprAddCollateString(pParse, pExpr, zColl); }else{ pExpr->iTable = regBase; - pExpr->affinity = SQLITE_AFF_INTEGER; + pExpr->affExpr = SQLITE_AFF_INTEGER; } } return pExpr; @@ -115345,7 +116558,7 @@ static void fkScanChildren( zCol = pFKey->pFrom->aCol[iCol].zName; pRight = sqlite3Expr(db, TK_ID, zCol); pEq = sqlite3PExpr(pParse, TK_EQ, pLeft, pRight); - pWhere = sqlite3ExprAnd(db, pWhere, pEq); + pWhere = sqlite3ExprAnd(pParse, pWhere, pEq); } /* If the child table is the same as the parent table, then add terms @@ -115379,11 +116592,11 @@ static void fkScanChildren( pLeft = exprTableRegister(pParse, pTab, regData, iCol); pRight = sqlite3Expr(db, TK_ID, pTab->aCol[iCol].zName); pEq = sqlite3PExpr(pParse, TK_IS, pLeft, pRight); - pAll = sqlite3ExprAnd(db, pAll, pEq); + pAll = sqlite3ExprAnd(pParse, pAll, pEq); } pNe = sqlite3PExpr(pParse, TK_NOT, pAll, 0); } - pWhere = sqlite3ExprAnd(db, pWhere, pNe); + pWhere = sqlite3ExprAnd(pParse, pWhere, pNe); } /* Resolve the references in the WHERE clause. */ @@ -115989,7 +117202,7 @@ static Trigger *fkActionTrigger( sqlite3ExprAlloc(db, TK_ID, &tToCol, 0)), sqlite3ExprAlloc(db, TK_ID, &tFromCol, 0) ); - pWhere = sqlite3ExprAnd(db, pWhere, pEq); + pWhere = sqlite3ExprAnd(pParse, pWhere, pEq); /* For ON UPDATE, construct the next term of the WHEN clause. ** The final WHEN clause will be like this: @@ -116005,7 +117218,7 @@ static Trigger *fkActionTrigger( sqlite3ExprAlloc(db, TK_ID, &tNew, 0), sqlite3ExprAlloc(db, TK_ID, &tToCol, 0)) ); - pWhen = sqlite3ExprAnd(db, pWhen, pEq); + pWhen = sqlite3ExprAnd(pParse, pWhen, pEq); } if( action!=OE_Restrict && (action!=OE_Cascade || pChanges) ){ @@ -116041,7 +117254,7 @@ static Trigger *fkActionTrigger( tFrom.n = nFrom; pRaise = sqlite3Expr(db, TK_RAISE, "FOREIGN KEY constraint failed"); if( pRaise ){ - pRaise->affinity = OE_Abort; + pRaise->affExpr = OE_Abort; } pSelect = sqlite3SelectNew(pParse, sqlite3ExprListAppend(pParse, 0, pRaise), @@ -116086,6 +117299,7 @@ static Trigger *fkActionTrigger( return 0; } assert( pStep!=0 ); + assert( pTrigger!=0 ); switch( action ){ case OE_Restrict: @@ -116276,18 +117490,19 @@ SQLITE_PRIVATE const char *sqlite3IndexAffinityStr(sqlite3 *db, Index *pIdx){ } for(n=0; nnColumn; n++){ i16 x = pIdx->aiColumn[n]; + char aff; if( x>=0 ){ - pIdx->zColAff[n] = pTab->aCol[x].affinity; + aff = pTab->aCol[x].affinity; }else if( x==XN_ROWID ){ - pIdx->zColAff[n] = SQLITE_AFF_INTEGER; + aff = SQLITE_AFF_INTEGER; }else{ - char aff; assert( x==XN_EXPR ); assert( pIdx->aColExpr!=0 ); aff = sqlite3ExprAffinity(pIdx->aColExpr->a[n].pExpr); - if( aff==0 ) aff = SQLITE_AFF_BLOB; - pIdx->zColAff[n] = aff; } + if( affSQLITE_AFF_NUMERIC) aff = SQLITE_AFF_NUMERIC; + pIdx->zColAff[n] = aff; } pIdx->zColAff[n] = 0; } @@ -116327,11 +117542,12 @@ SQLITE_PRIVATE void sqlite3TableAffinity(Vdbe *v, Table *pTab, int iReg){ } for(i=0; inCol; i++){ + assert( pTab->aCol[i].affinity!=0 ); zColAff[i] = pTab->aCol[i].affinity; } do{ zColAff[i--] = 0; - }while( i>=0 && zColAff[i]==SQLITE_AFF_BLOB ); + }while( i>=0 && zColAff[i]<=SQLITE_AFF_BLOB ); pTab->zColAff = zColAff; } assert( zColAff!=0 ); @@ -117002,7 +118218,7 @@ SQLITE_PRIVATE void sqlite3Insert( int nIdx; nIdx = sqlite3OpenTableAndIndices(pParse, pTab, OP_OpenWrite, 0, -1, 0, &iDataCur, &iIdxCur); - aRegIdx = sqlite3DbMallocRawNN(db, sizeof(int)*(nIdx+1)); + aRegIdx = sqlite3DbMallocRawNN(db, sizeof(int)*(nIdx+2)); if( aRegIdx==0 ){ goto insert_cleanup; } @@ -117011,6 +118227,7 @@ SQLITE_PRIVATE void sqlite3Insert( aRegIdx[i] = ++pParse->nMem; pParse->nMem += pIdx->nColumn; } + aRegIdx[i] = ++pParse->nMem; /* Register to store the table record */ } #ifndef SQLITE_OMIT_UPSERT if( pUpsert ){ @@ -117019,6 +118236,9 @@ SQLITE_PRIVATE void sqlite3Insert( pTab->zName); goto insert_cleanup; } + if( sqlite3HasExplicitNulls(pParse, pUpsert->pUpsertTarget) ){ + goto insert_cleanup; + } pTabList->a[0].iCursor = iDataCur; pUpsert->pUpsertSrc = pTabList; pUpsert->regData = regData; @@ -117414,6 +118634,14 @@ SQLITE_PRIVATE int sqlite3ExprReferencesUpdatedColumn( ** the same as the order of indices on the linked list of indices ** at pTab->pIndex. ** +** (2019-05-07) The generated code also creates a new record for the +** main table, if pTab is a rowid table, and stores that record in the +** register identified by aRegIdx[nIdx] - in other words in the first +** entry of aRegIdx[] past the last index. It is important that the +** record be generated during constraint checks to avoid affinity changes +** to the register content that occur after constraint checks but before +** the new record is inserted. +** ** The caller must have already opened writeable cursors on the main ** table and all applicable indices (that is to say, all indices for which ** aRegIdx[] is not zero). iDataCur is the cursor for the main table when @@ -117604,7 +118832,7 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( }else{ char *zName = pCheck->a[i].zName; if( zName==0 ) zName = pTab->zName; - if( onError==OE_Replace ) onError = OE_Abort; /* IMP: R-15569-63625 */ + if( onError==OE_Replace ) onError = OE_Abort; /* IMP: R-26383-51744 */ sqlite3HaltConstraint(pParse, SQLITE_CONSTRAINT_CHECK, onError, zName, P4_TRANSIENT, P5_ConstraintCheck); @@ -117857,7 +119085,9 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( sqlite3VdbeAddOp3(v, OP_MakeRecord, regIdx, pIdx->nColumn, aRegIdx[ix]); VdbeComment((v, "for %s", pIdx->zName)); #ifdef SQLITE_ENABLE_NULL_TRIM - if( pIdx->idxType==2 ) sqlite3SetMakeRecordP5(v, pIdx->pTable); + if( pIdx->idxType==SQLITE_IDXTYPE_PRIMARYKEY ){ + sqlite3SetMakeRecordP5(v, pIdx->pTable); + } #endif /* In an UPDATE operation, if this index is the PRIMARY KEY index @@ -118031,6 +119261,16 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( sqlite3VdbeJumpHere(v, ipkBottom); } + /* Generate the table record */ + if( HasRowid(pTab) ){ + int regRec = aRegIdx[ix]; + sqlite3VdbeAddOp3(v, OP_MakeRecord, regNewData+1, pTab->nCol, regRec); + sqlite3SetMakeRecordP5(v, pTab); + if( !bAffinityDone ){ + sqlite3TableAffinity(v, pTab, 0); + } + } + *pbMayReplace = seenReplace; VdbeModuleComment((v, "END: GenCnstCks(%d)", seenReplace)); } @@ -118080,10 +119320,7 @@ SQLITE_PRIVATE void sqlite3CompleteInsertion( Vdbe *v; /* Prepared statements under construction */ Index *pIdx; /* An index being inserted or updated */ u8 pik_flags; /* flag values passed to the btree insert */ - int regData; /* Content registers (after the rowid) */ - int regRec; /* Register holding assembled record for the table */ int i; /* Loop counter */ - u8 bAffinityDone = 0; /* True if OP_Affinity has been run already */ assert( update_flags==0 || update_flags==OPFLAG_ISUPDATE @@ -118095,7 +119332,6 @@ SQLITE_PRIVATE void sqlite3CompleteInsertion( assert( pTab->pSelect==0 ); /* This table is not a VIEW */ for(i=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, i++){ if( aRegIdx[i]==0 ) continue; - bAffinityDone = 1; if( pIdx->pPartIdxWhere ){ sqlite3VdbeAddOp2(v, OP_IsNull, aRegIdx[i], sqlite3VdbeCurrentAddr(v)+2); VdbeCoverage(v); @@ -118107,10 +119343,13 @@ SQLITE_PRIVATE void sqlite3CompleteInsertion( pik_flags |= (update_flags & OPFLAG_SAVEPOSITION); #ifdef SQLITE_ENABLE_PREUPDATE_HOOK if( update_flags==0 ){ - sqlite3VdbeAddOp4(v, OP_InsertInt, - iIdxCur+i, aRegIdx[i], 0, (char*)pTab, P4_TABLE + int r = sqlite3GetTempReg(pParse); + sqlite3VdbeAddOp2(v, OP_Integer, 0, r); + sqlite3VdbeAddOp4(v, OP_Insert, + iIdxCur+i, aRegIdx[i], r, (char*)pTab, P4_TABLE ); sqlite3VdbeChangeP5(v, OPFLAG_ISNOOP); + sqlite3ReleaseTempReg(pParse, r); } #endif } @@ -118120,13 +119359,6 @@ SQLITE_PRIVATE void sqlite3CompleteInsertion( sqlite3VdbeChangeP5(v, pik_flags); } if( !HasRowid(pTab) ) return; - regData = regNewData + 1; - regRec = sqlite3GetTempReg(pParse); - sqlite3VdbeAddOp3(v, OP_MakeRecord, regData, pTab->nCol, regRec); - sqlite3SetMakeRecordP5(v, pTab); - if( !bAffinityDone ){ - sqlite3TableAffinity(v, pTab, 0); - } if( pParse->nested ){ pik_flags = 0; }else{ @@ -118139,7 +119371,7 @@ SQLITE_PRIVATE void sqlite3CompleteInsertion( if( useSeekResult ){ pik_flags |= OPFLAG_USESEEKRESULT; } - sqlite3VdbeAddOp3(v, OP_Insert, iDataCur, regRec, regNewData); + sqlite3VdbeAddOp3(v, OP_Insert, iDataCur, aRegIdx[i], regNewData); if( !pParse->nested ){ sqlite3VdbeAppendP4(v, pTab, P4_TABLE); } @@ -118458,6 +119690,13 @@ static int xferOptimization( if( pSrcIdx==0 ){ return 0; /* pDestIdx has no corresponding index in pSrc */ } + if( pSrcIdx->tnum==pDestIdx->tnum && pSrc->pSchema==pDest->pSchema + && sqlite3FaultSim(411)==SQLITE_OK ){ + /* The sqlite3FaultSim() call allows this corruption test to be + ** bypassed during testing, in order to exercise other corruption tests + ** further downstream. */ + return 0; /* Corrupt schema - two indexes on the same btree */ + } } #ifndef SQLITE_OMIT_CHECK if( pDest->pCheck && sqlite3ExprListCompare(pSrc->pCheck,pDest->pCheck,-1) ){ @@ -118535,7 +119774,7 @@ static int xferOptimization( sqlite3RowidConstraint(pParse, onError, pDest); sqlite3VdbeJumpHere(v, addr2); autoIncStep(pParse, regAutoinc, regRowid); - }else if( pDest->pIndex==0 ){ + }else if( pDest->pIndex==0 && !(db->mDbFlags & DBFLAG_VacuumInto) ){ addr1 = sqlite3VdbeAddOp2(v, OP_NewRowid, iDest, regRowid); }else{ addr1 = sqlite3VdbeAddOp2(v, OP_Rowid, iSrc, regRowid); @@ -118598,7 +119837,7 @@ static int xferOptimization( sqlite3VdbeAddOp1(v, OP_SeekEnd, iDest); } } - if( !HasRowid(pSrc) && pDestIdx->idxType==2 ){ + if( !HasRowid(pSrc) && pDestIdx->idxType==SQLITE_IDXTYPE_PRIMARYKEY ){ idxInsFlags |= OPFLAG_NCHANGE; } sqlite3VdbeAddOp2(v, OP_IdxInsert, iDest, regData); @@ -119110,6 +120349,11 @@ struct sqlite3_api_routines { void(*xDestroy)(void*)); /* Version 3.26.0 and later */ const char *(*normalized_sql)(sqlite3_stmt*); + /* Version 3.28.0 and later */ + int (*stmt_isexplain)(sqlite3_stmt*); + int (*value_frombind)(sqlite3_value*); + /* Version 3.30.0 and later */ + int (*drop_modules)(sqlite3*,const char**); }; /* @@ -119399,6 +120643,11 @@ typedef int (*sqlite3_loadext_entry)( #define sqlite3_create_window_function sqlite3_api->create_window_function /* Version 3.26.0 and later */ #define sqlite3_normalized_sql sqlite3_api->normalized_sql +/* Version 3.28.0 and later */ +#define sqlite3_stmt_isexplain sqlite3_api->isexplain +#define sqlite3_value_frombind sqlite3_api->frombind +/* Version 3.30.0 and later */ +#define sqlite3_drop_modules sqlite3_api->drop_modules #endif /* !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION) */ #if !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION) @@ -119858,9 +121107,18 @@ static const sqlite3_api_routines sqlite3Apis = { sqlite3_create_window_function, /* Version 3.26.0 and later */ #ifdef SQLITE_ENABLE_NORMALIZE - sqlite3_normalized_sql + sqlite3_normalized_sql, #else - 0 + 0, +#endif + /* Version 3.28.0 and later */ + sqlite3_stmt_isexplain, + sqlite3_value_frombind, + /* Version 3.30.0 and later */ +#ifndef SQLITE_OMIT_VIRTUALTABLE + sqlite3_drop_modules, +#else + 0, #endif }; @@ -120309,10 +121567,9 @@ SQLITE_PRIVATE void sqlite3AutoLoadExtensions(sqlite3 *db){ #define PragTyp_WAL_AUTOCHECKPOINT 38 #define PragTyp_WAL_CHECKPOINT 39 #define PragTyp_ACTIVATE_EXTENSIONS 40 -#define PragTyp_HEXKEY 41 -#define PragTyp_KEY 42 -#define PragTyp_LOCK_STATUS 43 -#define PragTyp_STATS 44 +#define PragTyp_KEY 41 +#define PragTyp_LOCK_STATUS 42 +#define PragTyp_STATS 43 /* Property flags associated with various pragma. */ #define PragFlg_NeedSchema 0x01 /* Force schema load before running */ @@ -120441,11 +121698,13 @@ static const PragmaName aPragmaName[] = { /* ColNames: */ 0, 0, /* iArg: */ 0 }, #endif +#if !defined(SQLITE_OMIT_CASE_SENSITIVE_LIKE_PRAGMA) {/* zName: */ "case_sensitive_like", /* ePragTyp: */ PragTyp_CASE_SENSITIVE_LIKE, /* ePragFlg: */ PragFlg_NoColumns, /* ColNames: */ 0, 0, /* iArg: */ 0 }, +#endif {/* zName: */ "cell_size_check", /* ePragTyp: */ PragTyp_FLAG, /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, @@ -120573,7 +121832,7 @@ static const PragmaName aPragmaName[] = { /* iArg: */ SQLITE_FullFSync }, #endif #if !defined(SQLITE_OMIT_SCHEMA_PRAGMAS) -#if defined(SQLITE_INTROSPECTION_PRAGMAS) +#if !defined(SQLITE_OMIT_INTROSPECTION_PRAGMAS) {/* zName: */ "function_list", /* ePragTyp: */ PragTyp_FUNCTION_LIST, /* ePragFlg: */ PragFlg_Result0, @@ -120583,12 +121842,12 @@ static const PragmaName aPragmaName[] = { #endif #if defined(SQLITE_HAS_CODEC) {/* zName: */ "hexkey", - /* ePragTyp: */ PragTyp_HEXKEY, + /* ePragTyp: */ PragTyp_KEY, /* ePragFlg: */ 0, /* ColNames: */ 0, 0, /* iArg: */ 2 }, {/* zName: */ "hexrekey", - /* ePragTyp: */ PragTyp_HEXKEY, + /* ePragTyp: */ PragTyp_KEY, /* ePragFlg: */ 0, /* ColNames: */ 0, 0, /* iArg: */ 3 }, @@ -120697,7 +121956,7 @@ static const PragmaName aPragmaName[] = { #endif #if !defined(SQLITE_OMIT_SCHEMA_PRAGMAS) #if !defined(SQLITE_OMIT_VIRTUALTABLE) -#if defined(SQLITE_INTROSPECTION_PRAGMAS) +#if !defined(SQLITE_OMIT_INTROSPECTION_PRAGMAS) {/* zName: */ "module_list", /* ePragTyp: */ PragTyp_MODULE_LIST, /* ePragFlg: */ PragFlg_Result0, @@ -120732,7 +121991,7 @@ static const PragmaName aPragmaName[] = { /* iArg: */ SQLITE_ParserTrace }, #endif #endif -#if defined(SQLITE_INTROSPECTION_PRAGMAS) +#if !defined(SQLITE_OMIT_INTROSPECTION_PRAGMAS) {/* zName: */ "pragma_list", /* ePragTyp: */ PragTyp_PRAGMA_LIST, /* ePragFlg: */ PragFlg_Result0, @@ -120930,7 +122189,7 @@ static const PragmaName aPragmaName[] = { /* iArg: */ SQLITE_WriteSchema|SQLITE_NoSchemaError }, #endif }; -/* Number of pragmas: 62 on by default, 81 total. */ +/* Number of pragmas: 65 on by default, 81 total. */ /************** End of pragma.h **********************************************/ /************** Continuing where we left off in pragma.c *********************/ @@ -121549,6 +122808,11 @@ SQLITE_PRIVATE void sqlite3Pragma( ** then do a query */ eMode = PAGER_JOURNALMODE_QUERY; } + if( eMode==PAGER_JOURNALMODE_OFF && (db->flags & SQLITE_Defensive)!=0 ){ + /* Do not allow journal-mode "OFF" in defensive since the database + ** can become corrupted using ordinary SQL when the journal is off */ + eMode = PAGER_JOURNALMODE_QUERY; + } } if( eMode==PAGER_JOURNALMODE_QUERY && pId2->n==0 ){ /* Convert "PRAGMA journal_mode" into "PRAGMA main.journal_mode" */ @@ -122057,6 +123321,15 @@ SQLITE_PRIVATE void sqlite3Pragma( Index *pIdx; Table *pTab; pIdx = sqlite3FindIndex(db, zRight, zDb); + if( pIdx==0 ){ + /* If there is no index named zRight, check to see if there is a + ** WITHOUT ROWID table named zRight, and if there is, show the + ** structure of the PRIMARY KEY index for that table. */ + pTab = sqlite3LocateTable(pParse, LOCATE_NOERR, zRight, zDb); + if( pTab && !HasRowid(pTab) ){ + pIdx = sqlite3PrimaryKeyIndex(pTab); + } + } if( pIdx ){ int iIdxDb = sqlite3SchemaToIndex(db, pIdx->pSchema); int i; @@ -122136,7 +123409,7 @@ SQLITE_PRIVATE void sqlite3Pragma( } break; -#ifdef SQLITE_INTROSPECTION_PRAGMAS +#ifndef SQLITE_OMIT_INTROSPECTION_PRAGMAS case PragTyp_FUNCTION_LIST: { int i; HashElem *j; @@ -122326,6 +123599,7 @@ SQLITE_PRIVATE void sqlite3Pragma( #endif /* !defined(SQLITE_OMIT_TRIGGER) */ #endif /* !defined(SQLITE_OMIT_FOREIGN_KEY) */ +#ifndef SQLITE_OMIT_CASE_SENSITIVE_LIKE_PRAGMA /* Reinstall the LIKE and GLOB functions. The variant of LIKE ** used will be case sensitive or not depending on the RHS. */ @@ -122335,6 +123609,7 @@ SQLITE_PRIVATE void sqlite3Pragma( } } break; +#endif /* SQLITE_OMIT_CASE_SENSITIVE_LIKE_PRAGMA */ #ifndef SQLITE_INTEGRITY_CHECK_ERROR_MAX # define SQLITE_INTEGRITY_CHECK_ERROR_MAX 100 @@ -123028,28 +124303,30 @@ SQLITE_PRIVATE void sqlite3Pragma( */ case PragTyp_KEY: { if( zRight ){ - int n = pPragma->iArg<4 ? sqlite3Strlen30(zRight) : -1; - if( (pPragma->iArg & 1)==0 ){ - sqlite3_key_v2(db, zDb, zRight, n); + char zBuf[40]; + const char *zKey = zRight; + int n; + if( pPragma->iArg==2 || pPragma->iArg==3 ){ + u8 iByte; + int i; + for(i=0, iByte=0; iiArg<4 ? sqlite3Strlen30(zRight) : -1; } if( (pPragma->iArg & 1)==0 ){ - sqlite3_key_v2(db, zDb, zKey, i/2); + rc = sqlite3_key_v2(db, zDb, zKey, n); }else{ - sqlite3_rekey_v2(db, zDb, zKey, i/2); + rc = sqlite3_rekey_v2(db, zDb, zKey, n); + } + if( rc==SQLITE_OK && n!=0 ){ + sqlite3VdbeSetNumCols(v, 1); + sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "ok", SQLITE_STATIC); + returnSingleText(v, "ok"); } } break; @@ -123467,9 +124744,11 @@ SQLITE_PRIVATE int sqlite3IndexHasDuplicateRootPage(Index *pIndex){ ** ** Each callback contains the following information: ** -** argv[0] = name of thing being created -** argv[1] = root page number for table or index. 0 for trigger or view. -** argv[2] = SQL text for the CREATE statement. +** argv[0] = type of object: "table", "index", "trigger", or "view". +** argv[1] = name of thing being created +** argv[2] = associated table if an index or trigger +** argv[3] = root page number for table or index. 0 for trigger or view. +** argv[4] = SQL text for the CREATE statement. ** */ SQLITE_PRIVATE int sqlite3InitCallback(void *pInit, int argc, char **argv, char **NotUsed){ @@ -123477,21 +124756,21 @@ SQLITE_PRIVATE int sqlite3InitCallback(void *pInit, int argc, char **argv, char sqlite3 *db = pData->db; int iDb = pData->iDb; - assert( argc==3 ); + assert( argc==5 ); UNUSED_PARAMETER2(NotUsed, argc); assert( sqlite3_mutex_held(db->mutex) ); DbClearProperty(db, iDb, DB_Empty); pData->nInitRow++; if( db->mallocFailed ){ - corruptSchema(pData, argv[0], 0); + corruptSchema(pData, argv[1], 0); return 1; } assert( iDb>=0 && iDbnDb ); if( argv==0 ) return 0; /* Might happen if EMPTY_RESULT_CALLBACKS are on */ - if( argv[1]==0 ){ - corruptSchema(pData, argv[0], 0); - }else if( sqlite3_strnicmp(argv[2],"create ",7)==0 ){ + if( argv[3]==0 ){ + corruptSchema(pData, argv[1], 0); + }else if( sqlite3_strnicmp(argv[4],"create ",7)==0 ){ /* Call the parser to process a CREATE TABLE, INDEX or VIEW. ** But because db->init.busy is set to 1, no VDBE code is generated ** or executed. All the parser does is build the internal data @@ -123504,9 +124783,10 @@ SQLITE_PRIVATE int sqlite3InitCallback(void *pInit, int argc, char **argv, char assert( db->init.busy ); db->init.iDb = iDb; - db->init.newTnum = sqlite3Atoi(argv[1]); + db->init.newTnum = sqlite3Atoi(argv[3]); db->init.orphanTrigger = 0; - TESTONLY(rcp = ) sqlite3_prepare(db, argv[2], -1, &pStmt, 0); + db->init.azInit = argv; + TESTONLY(rcp = ) sqlite3_prepare(db, argv[4], -1, &pStmt, 0); rc = db->errCode; assert( (rc&0xFF)==(rcp&0xFF) ); db->init.iDb = saved_iDb; @@ -123515,17 +124795,17 @@ SQLITE_PRIVATE int sqlite3InitCallback(void *pInit, int argc, char **argv, char if( db->init.orphanTrigger ){ assert( iDb==1 ); }else{ - pData->rc = rc; + if( rc > pData->rc ) pData->rc = rc; if( rc==SQLITE_NOMEM ){ sqlite3OomFault(db); }else if( rc!=SQLITE_INTERRUPT && (rc&0xFF)!=SQLITE_LOCKED ){ - corruptSchema(pData, argv[0], sqlite3_errmsg(db)); + corruptSchema(pData, argv[1], sqlite3_errmsg(db)); } } } sqlite3_finalize(pStmt); - }else if( argv[0]==0 || (argv[2]!=0 && argv[2][0]!=0) ){ - corruptSchema(pData, argv[0], 0); + }else if( argv[1]==0 || (argv[4]!=0 && argv[4][0]!=0) ){ + corruptSchema(pData, argv[1], 0); }else{ /* If the SQL column is blank it means this is an index that ** was created to be the PRIMARY KEY or to fulfill a UNIQUE @@ -123534,13 +124814,13 @@ SQLITE_PRIVATE int sqlite3InitCallback(void *pInit, int argc, char **argv, char ** to do here is record the root page number for that index. */ Index *pIndex; - pIndex = sqlite3FindIndex(db, argv[0], db->aDb[iDb].zDbSName); + pIndex = sqlite3FindIndex(db, argv[1], db->aDb[iDb].zDbSName); if( pIndex==0 - || sqlite3GetInt32(argv[1],&pIndex->tnum)==0 + || sqlite3GetInt32(argv[3],&pIndex->tnum)==0 || pIndex->tnum<2 || sqlite3IndexHasDuplicateRootPage(pIndex) ){ - corruptSchema(pData, argv[0], pIndex?"invalid rootpage":"orphan index"); + corruptSchema(pData, argv[1], pIndex?"invalid rootpage":"orphan index"); } } return 0; @@ -123561,7 +124841,7 @@ SQLITE_PRIVATE int sqlite3InitOne(sqlite3 *db, int iDb, char **pzErrMsg, u32 mFl int size; #endif Db *pDb; - char const *azArg[4]; + char const *azArg[6]; int meta[5]; InitData initData; const char *zMasterName; @@ -123580,18 +124860,20 @@ SQLITE_PRIVATE int sqlite3InitOne(sqlite3 *db, int iDb, char **pzErrMsg, u32 mFl ** table name will be inserted automatically by the parser so we can just ** use the abbreviation "x" here. The parser will also automatically tag ** the schema table as read-only. */ - azArg[0] = zMasterName = SCHEMA_TABLE(iDb); - azArg[1] = "1"; - azArg[2] = "CREATE TABLE x(type text,name text,tbl_name text," + azArg[0] = "table"; + azArg[1] = zMasterName = SCHEMA_TABLE(iDb); + azArg[2] = azArg[1]; + azArg[3] = "1"; + azArg[4] = "CREATE TABLE x(type text,name text,tbl_name text," "rootpage int,sql text)"; - azArg[3] = 0; + azArg[5] = 0; initData.db = db; initData.iDb = iDb; initData.rc = SQLITE_OK; initData.pzErrMsg = pzErrMsg; initData.mInitFlags = mFlags; initData.nInitRow = 0; - sqlite3InitCallback(&initData, 3, (char **)azArg, 0); + sqlite3InitCallback(&initData, 5, (char **)azArg, 0); if( initData.rc ){ rc = initData.rc; goto error_out; @@ -123717,7 +124999,7 @@ SQLITE_PRIVATE int sqlite3InitOne(sqlite3 *db, int iDb, char **pzErrMsg, u32 mFl { char *zSql; zSql = sqlite3MPrintf(db, - "SELECT name, rootpage, sql FROM \"%w\".%s ORDER BY rowid", + "SELECT*FROM\"%w\".%s ORDER BY rowid", db->aDb[iDb].zDbSName, zMasterName); #ifndef SQLITE_OMIT_AUTHORIZATION { @@ -124038,7 +125320,10 @@ static int sqlite3Prepare( rc = sParse.rc; #ifndef SQLITE_OMIT_EXPLAIN - if( rc==SQLITE_OK && sParse.pVdbe && sParse.explain ){ + /* Justification for the ALWAYS(): The only way for rc to be SQLITE_OK and + ** sParse.pVdbe to be NULL is if the input SQL is an empty string, but in + ** that case, sParse.explain will be false. */ + if( sParse.explain && rc==SQLITE_OK && ALWAYS(sParse.pVdbe) ){ static const char * const azColName[] = { "addr", "opcode", "p1", "p2", "p3", "p4", "p5", "comment", "id", "parent", "notused", "detail" @@ -124063,8 +125348,8 @@ static int sqlite3Prepare( if( db->init.busy==0 ){ sqlite3VdbeSetSql(sParse.pVdbe, zSql, (int)(sParse.zTail-zSql), prepFlags); } - if( sParse.pVdbe && (rc!=SQLITE_OK || db->mallocFailed) ){ - sqlite3VdbeFinalize(sParse.pVdbe); + if( rc!=SQLITE_OK || db->mallocFailed ){ + if( sParse.pVdbe ) sqlite3VdbeFinalize(sParse.pVdbe); assert(!(*ppStmt)); }else{ *ppStmt = (sqlite3_stmt*)sParse.pVdbe; @@ -124435,6 +125720,7 @@ static void clearSelect(sqlite3 *db, Select *p, int bFree){ if( OK_IF_ALWAYS_TRUE(p->pWinDefn) ){ sqlite3WindowListDelete(db, p->pWinDefn); } + assert( p->pWin==0 ); #endif if( OK_IF_ALWAYS_TRUE(p->pWith) ) sqlite3WithDelete(db, p->pWith); if( bFree ) sqlite3DbFreeNN(db, p); @@ -124690,7 +125976,7 @@ static void addWhereTerm( ExprSetVVAProperty(pEq, EP_NoReduce); pEq->iRightJoinTable = (i16)pE2->iTable; } - *ppWhere = sqlite3ExprAnd(db, *ppWhere, pEq); + *ppWhere = sqlite3ExprAnd(pParse, *ppWhere, pEq); } /* @@ -124824,7 +126110,7 @@ static int sqliteProcessJoin(Parse *pParse, Select *p){ */ if( pRight->pOn ){ if( isOuter ) setJoinExpr(pRight->pOn, pRight->iCursor); - p->pWhere = sqlite3ExprAnd(pParse->db, p->pWhere, pRight->pOn); + p->pWhere = sqlite3ExprAnd(pParse, p->pWhere, pRight->pOn); pRight->pOn = 0; } @@ -124998,7 +126284,7 @@ static void pushOntoSorter( if( pParse->db->mallocFailed ) return; pOp->p2 = nKey + nData; pKI = pOp->p4.pKeyInfo; - memset(pKI->aSortOrder, 0, pKI->nKeyField); /* Makes OP_Jump testable */ + memset(pKI->aSortFlags, 0, pKI->nKeyField); /* Makes OP_Jump testable */ sqlite3VdbeChangeP4(v, -1, (char*)pKI, P4_KEYINFO); testcase( pKI->nAllField > pKI->nKeyField+2 ); pOp->p4.pKeyInfo = sqlite3KeyInfoFromExprList(pParse,pSort->pOrderBy,nOBSat, @@ -125609,7 +126895,7 @@ SQLITE_PRIVATE KeyInfo *sqlite3KeyInfoAlloc(sqlite3 *db, int N, int X){ int nExtra = (N+X)*(sizeof(CollSeq*)+1) - sizeof(CollSeq*); KeyInfo *p = sqlite3DbMallocRawNN(db, sizeof(KeyInfo) + nExtra); if( p ){ - p->aSortOrder = (u8*)&p->aColl[N+X]; + p->aSortFlags = (u8*)&p->aColl[N+X]; p->nKeyField = (u16)N; p->nAllField = (u16)(N+X); p->enc = ENC(db); @@ -125686,7 +126972,7 @@ SQLITE_PRIVATE KeyInfo *sqlite3KeyInfoFromExprList( assert( sqlite3KeyInfoIsWriteable(pInfo) ); for(i=iStart, pItem=pList->a+iStart; iaColl[i-iStart] = sqlite3ExprNNCollSeq(pParse, pItem->pExpr); - pInfo->aSortOrder[i-iStart] = pItem->sortOrder; + pInfo->aSortFlags[i-iStart] = pItem->sortFlags; } } return pInfo; @@ -125978,8 +127264,6 @@ static const char *columnTypeImpl( assert( pExpr!=0 ); assert( pNC->pSrcList!=0 ); - assert( pExpr->op!=TK_AGG_COLUMN ); /* This routine runes before aggregates - ** are processed */ switch( pExpr->op ){ case TK_COLUMN: { /* The expression is a column. Locate the table the column is being @@ -126296,12 +127580,11 @@ SQLITE_PRIVATE int sqlite3ColumnsFromExprList( if( (zName = pEList->a[i].zName)!=0 ){ /* If the column contains an "AS " phrase, use as the name */ }else{ - Expr *pColExpr = sqlite3ExprSkipCollate(pEList->a[i].pExpr); + Expr *pColExpr = sqlite3ExprSkipCollateAndLikely(pEList->a[i].pExpr); while( pColExpr->op==TK_DOT ){ pColExpr = pColExpr->pRight; assert( pColExpr!=0 ); } - assert( pColExpr->op!=TK_AGG_COLUMN ); if( pColExpr->op==TK_COLUMN ){ /* For columns use the column name name */ int iCol = pColExpr->iColumn; @@ -126369,7 +127652,8 @@ SQLITE_PRIVATE int sqlite3ColumnsFromExprList( SQLITE_PRIVATE void sqlite3SelectAddColumnTypeAndCollation( Parse *pParse, /* Parsing contexts */ Table *pTab, /* Add column type information to this table */ - Select *pSelect /* SELECT used to determine types and collations */ + Select *pSelect, /* SELECT used to determine types and collations */ + char aff /* Default affinity for columns */ ){ sqlite3 *db = pParse->db; NameContext sNC; @@ -126402,7 +127686,7 @@ SQLITE_PRIVATE void sqlite3SelectAddColumnTypeAndCollation( pCol->colFlags |= COLFLAG_HASTYPE; } } - if( pCol->affinity==0 ) pCol->affinity = SQLITE_AFF_BLOB; + if( pCol->affinity<=SQLITE_AFF_NONE ) pCol->affinity = aff; pColl = sqlite3ExprCollSeq(pParse, p); if( pColl && pCol->zColl==0 ){ pCol->zColl = sqlite3DbStrDup(db, pColl->zName); @@ -126415,7 +127699,7 @@ SQLITE_PRIVATE void sqlite3SelectAddColumnTypeAndCollation( ** Given a SELECT statement, generate a Table structure that describes ** the result set of that SELECT. */ -SQLITE_PRIVATE Table *sqlite3ResultSetOfSelect(Parse *pParse, Select *pSelect){ +SQLITE_PRIVATE Table *sqlite3ResultSetOfSelect(Parse *pParse, Select *pSelect, char aff){ Table *pTab; sqlite3 *db = pParse->db; u64 savedFlags; @@ -126431,14 +127715,11 @@ SQLITE_PRIVATE Table *sqlite3ResultSetOfSelect(Parse *pParse, Select *pSelect){ if( pTab==0 ){ return 0; } - /* The sqlite3ResultSetOfSelect() is only used n contexts where lookaside - ** is disabled */ - assert( db->lookaside.bDisable ); pTab->nTabRef = 1; pTab->zName = 0; pTab->nRowLogEst = 200; assert( 200==sqlite3LogEst(1048576) ); sqlite3ColumnsFromExprList(pParse, pSelect->pEList, &pTab->nCol, &pTab->aCol); - sqlite3SelectAddColumnTypeAndCollation(pParse, pTab, pSelect); + sqlite3SelectAddColumnTypeAndCollation(pParse, pTab, pSelect, aff); pTab->iPKey = -1; if( db->mallocFailed ){ sqlite3DeleteTable(db, pTab); @@ -126592,7 +127873,7 @@ static KeyInfo *multiSelectOrderByKeyInfo(Parse *pParse, Select *p, int nExtra){ } assert( sqlite3KeyInfoIsWriteable(pRet) ); pRet->aColl[i] = pColl; - pRet->aSortOrder[i] = pOrderBy->a[i].sortOrder; + pRet->aSortFlags[i] = pOrderBy->a[i].sortFlags; } } @@ -126875,6 +128156,7 @@ static int multiSelect( */ assert( p && p->pPrior ); /* Calling function guarantees this much */ assert( (p->selFlags & SF_Recursive)==0 || p->op==TK_ALL || p->op==TK_UNION ); + assert( p->selFlags & SF_Compound ); db = pParse->db; pPrior = p->pPrior; dest = *pDest; @@ -127302,11 +128584,14 @@ static int generateOutputSubroutine( /* If this is a scalar select that is part of an expression, then ** store the results in the appropriate memory cell and break out - ** of the scan loop. + ** of the scan loop. Note that the select might return multiple columns + ** if it is the RHS of a row-value IN operator. */ case SRT_Mem: { - assert( pIn->nSdst==1 || pParse->nErr>0 ); testcase( pIn->nSdst!=1 ); - sqlite3ExprCodeMove(pParse, pIn->iSdst, pDest->iSDParm, 1); + if( pParse->nErr==0 ){ + testcase( pIn->nSdst>1 ); + sqlite3ExprCodeMove(pParse, pIn->iSdst, pDest->iSDParm, pIn->nSdst); + } /* The LIMIT clause will jump out of the loop for us */ break; } @@ -127563,7 +128848,7 @@ static int multiSelectOrderBy( assert( sqlite3KeyInfoIsWriteable(pKeyDup) ); for(i=0; iaColl[i] = multiSelectCollSeq(pParse, p, i); - pKeyDup->aSortOrder[i] = 0; + pKeyDup->aSortFlags[i] = 0; } } } @@ -127813,6 +129098,18 @@ static Expr *substExpr( } sqlite3ExprDelete(db, pExpr); pExpr = pNew; + + /* Ensure that the expression now has an implicit collation sequence, + ** just as it did when it was a column of a view or sub-query. */ + if( pExpr ){ + if( pExpr->op!=TK_COLUMN && pExpr->op!=TK_COLLATE ){ + CollSeq *pColl = sqlite3ExprCollSeq(pSubst->pParse, pExpr); + pExpr = sqlite3ExprAddCollateString(pSubst->pParse, pExpr, + (pColl ? pColl->zName : "BINARY") + ); + } + ExprClearProperty(pExpr, EP_Collate); + } } } }else{ @@ -127826,6 +129123,14 @@ static Expr *substExpr( }else{ substExprList(pSubst, pExpr->x.pList); } +#ifndef SQLITE_OMIT_WINDOWFUNC + if( ExprHasProperty(pExpr, EP_WinFunc) ){ + Window *pWin = pExpr->y.pWin; + pWin->pFilter = substExpr(pSubst, pWin->pFilter); + substExprList(pSubst, pWin->pPartition); + substExprList(pSubst, pWin->pOrderBy); + } +#endif } return pExpr; } @@ -128286,6 +129591,7 @@ static int flattenSubquery( for(pParent=p; pParent; pParent=pParent->pPrior, pSub=pSub->pPrior){ int nSubSrc; u8 jointype = 0; + assert( pSub!=0 ); pSubSrc = pSub->pSrc; /* FROM clause of subquery */ nSubSrc = pSubSrc->nSrc; /* Number of terms in subquery FROM clause */ pSrc = pParent->pSrc; /* FROM clause of the outer query */ @@ -128369,7 +129675,7 @@ static int flattenSubquery( if( isLeftJoin>0 ){ setJoinExpr(pWhere, iNewParent); } - pParent->pWhere = sqlite3ExprAnd(db, pWhere, pParent->pWhere); + pParent->pWhere = sqlite3ExprAnd(pParse, pWhere, pParent->pWhere); if( db->mallocFailed==0 ){ SubstContext x; x.pParse = pParse; @@ -128380,10 +129686,10 @@ static int flattenSubquery( substSelect(&x, pParent, 0); } - /* The flattened query is distinct if either the inner or the - ** outer query is distinct. - */ - pParent->selFlags |= pSub->selFlags & SF_Distinct; + /* The flattened query is a compound if either the inner or the + ** outer query is a compound. */ + pParent->selFlags |= pSub->selFlags & SF_Compound; + assert( (pSub->selFlags & SF_Distinct)==0 ); /* restriction (17b) */ /* ** SELECT ... FROM (SELECT ... LIMIT a OFFSET b) LIMIT x OFFSET y; @@ -128704,9 +130010,9 @@ static int pushDownWhereTerms( x.pEList = pSubq->pEList; pNew = substExpr(&x, pNew); if( pSubq->selFlags & SF_Aggregate ){ - pSubq->pHaving = sqlite3ExprAnd(pParse->db, pSubq->pHaving, pNew); + pSubq->pHaving = sqlite3ExprAnd(pParse, pSubq->pHaving, pNew); }else{ - pSubq->pWhere = sqlite3ExprAnd(pParse->db, pSubq->pWhere, pNew); + pSubq->pWhere = sqlite3ExprAnd(pParse, pSubq->pWhere, pNew); } pSubq = pSubq->pPrior; } @@ -128736,24 +130042,27 @@ static u8 minMaxQuery(sqlite3 *db, Expr *pFunc, ExprList **ppMinMax){ ExprList *pEList = pFunc->x.pList; /* Arguments to agg function */ const char *zFunc; /* Name of aggregate function pFunc */ ExprList *pOrderBy; - u8 sortOrder; + u8 sortFlags; assert( *ppMinMax==0 ); assert( pFunc->op==TK_AGG_FUNCTION ); - if( pEList==0 || pEList->nExpr!=1 ) return eRet; + assert( !IsWindowFunc(pFunc) ); + if( pEList==0 || pEList->nExpr!=1 || ExprHasProperty(pFunc, EP_WinFunc) ){ + return eRet; + } zFunc = pFunc->u.zToken; if( sqlite3StrICmp(zFunc, "min")==0 ){ eRet = WHERE_ORDERBY_MIN; - sortOrder = SQLITE_SO_ASC; + sortFlags = KEYINFO_ORDER_BIGNULL; }else if( sqlite3StrICmp(zFunc, "max")==0 ){ eRet = WHERE_ORDERBY_MAX; - sortOrder = SQLITE_SO_DESC; + sortFlags = KEYINFO_ORDER_DESC; }else{ return eRet; } *ppMinMax = pOrderBy = sqlite3ExprListDup(db, pEList, 0); assert( pOrderBy!=0 || db->mallocFailed ); - if( pOrderBy ) pOrderBy->a[0].sortOrder = sortOrder; + if( pOrderBy ) pOrderBy->a[0].sortFlags = sortFlags; return eRet; } @@ -128787,7 +130096,7 @@ static Table *isSimpleCount(Select *p, AggInfo *pAggInfo){ if( pExpr->op!=TK_AGG_FUNCTION ) return 0; if( NEVER(pAggInfo->nFunc==0) ) return 0; if( (pAggInfo->aFunc[0].pFunc->funcFlags&SQLITE_FUNC_COUNT)==0 ) return 0; - if( pExpr->flags&EP_Distinct ) return 0; + if( ExprHasProperty(pExpr, EP_Distinct|EP_WinFunc) ) return 0; return pTab; } @@ -129132,7 +130441,7 @@ SQLITE_PRIVATE int sqlite3ExpandSubquery(Parse *pParse, struct SrcList_item *pFr pTab->nRowLogEst = 200; assert( 200==sqlite3LogEst(1048576) ); pTab->tabFlags |= TF_Ephemeral; - return SQLITE_OK; + return pParse->nErr ? SQLITE_ERROR : SQLITE_OK; } /* @@ -129178,6 +130487,10 @@ static int selectExpander(Walker *pWalker, Select *p){ if( (selFlags & SF_Expanded)!=0 ){ return WRC_Prune; } + if( pWalker->eCode ){ + /* Renumber selId because it has been copied from a view */ + p->selId = ++pParse->nSelect; + } pTabList = p->pSrc; pEList = p->pEList; sqlite3WithPush(pParse, p->pWith, 0); @@ -129227,12 +130540,19 @@ static int selectExpander(Walker *pWalker, Select *p){ #if !defined(SQLITE_OMIT_VIEW) || !defined (SQLITE_OMIT_VIRTUALTABLE) if( IsVirtual(pTab) || pTab->pSelect ){ i16 nCol; + u8 eCodeOrig = pWalker->eCode; if( sqlite3ViewGetColumnNames(pParse, pTab) ) return WRC_Abort; assert( pFrom->pSelect==0 ); + if( pTab->pSelect && (db->flags & SQLITE_EnableView)==0 ){ + sqlite3ErrorMsg(pParse, "access to view \"%s\" prohibited", + pTab->zName); + } pFrom->pSelect = sqlite3SelectDup(db, pTab->pSelect, 0); nCol = pTab->nCol; pTab->nCol = -1; + pWalker->eCode = 1; /* Turn on Select.selId renumbering */ sqlite3WalkSelect(pWalker, pFrom->pSelect); + pWalker->eCode = eCodeOrig; pTab->nCol = nCol; } #endif @@ -129482,6 +130802,7 @@ static void sqlite3SelectExpand(Parse *pParse, Select *pSelect){ } w.xSelectCallback = selectExpander; w.xSelectCallback2 = selectPopWith; + w.eCode = 0; sqlite3WalkSelect(&w, pSelect); } @@ -129519,7 +130840,8 @@ static void selectAddSubqueryTypeInfo(Walker *pWalker, Select *p){ Select *pSel = pFrom->pSelect; if( pSel ){ while( pSel->pPrior ) pSel = pSel->pPrior; - sqlite3SelectAddColumnTypeAndCollation(pParse, pTab, pSel); + sqlite3SelectAddColumnTypeAndCollation(pParse, pTab, pSel, + SQLITE_AFF_NONE); } } } @@ -129641,7 +130963,7 @@ static void finalizeAggFunctions(Parse *pParse, AggInfo *pAggInfo){ ** ** If regAcc is non-zero and there are no min() or max() aggregates ** in pAggInfo, then only populate the pAggInfo->nAccumulator accumulator -** registers i register regAcc contains 0. The caller will take care +** registers if register regAcc contains 0. The caller will take care ** of setting and clearing regAcc. */ static void updateAccumulator(Parse *pParse, int regAcc, AggInfo *pAggInfo){ @@ -129659,6 +130981,25 @@ static void updateAccumulator(Parse *pParse, int regAcc, AggInfo *pAggInfo){ int regAgg; ExprList *pList = pF->pExpr->x.pList; assert( !ExprHasProperty(pF->pExpr, EP_xIsSelect) ); + assert( !IsWindowFunc(pF->pExpr) ); + if( ExprHasProperty(pF->pExpr, EP_WinFunc) ){ + Expr *pFilter = pF->pExpr->y.pWin->pFilter; + if( pAggInfo->nAccumulator + && (pF->pFunc->funcFlags & SQLITE_FUNC_NEEDCOLL) + ){ + if( regHit==0 ) regHit = ++pParse->nMem; + /* If this is the first row of the group (regAcc==0), clear the + ** "magnet" register regHit so that the accumulator registers + ** are populated if the FILTER clause jumps over the the + ** invocation of min() or max() altogether. Or, if this is not + ** the first row (regAcc==1), set the magnet register so that the + ** accumulators are not populated unless the min()/max() is invoked and + ** indicates that they should be. */ + sqlite3VdbeAddOp2(v, OP_Copy, regAcc, regHit); + } + addrNext = sqlite3VdbeMakeLabel(pParse); + sqlite3ExprIfFalse(pParse, pFilter, addrNext, SQLITE_JUMPIFNULL); + } if( pList ){ nArg = pList->nExpr; regAgg = sqlite3GetTempRange(pParse, nArg); @@ -129668,7 +131009,9 @@ static void updateAccumulator(Parse *pParse, int regAcc, AggInfo *pAggInfo){ regAgg = 0; } if( pF->iDistinct>=0 ){ - addrNext = sqlite3VdbeMakeLabel(pParse); + if( addrNext==0 ){ + addrNext = sqlite3VdbeMakeLabel(pParse); + } testcase( nArg==0 ); /* Error condition */ testcase( nArg>1 ); /* Also an error */ codeDistinct(pParse, pF->iDistinct, addrNext, 1, regAgg); @@ -129704,6 +131047,7 @@ static void updateAccumulator(Parse *pParse, int regAcc, AggInfo *pAggInfo){ for(i=0, pC=pAggInfo->aCol; inAccumulator; i++, pC++){ sqlite3ExprCode(pParse, pC->pExpr, pC->iMem); } + pAggInfo->directMode = 0; if( addrHitTest ){ sqlite3VdbeJumpHere(v, addrHitTest); @@ -129749,11 +131093,11 @@ static int havingToWhereExprCb(Walker *pWalker, Expr *pExpr){ Select *pS = pWalker->u.pSelect; if( sqlite3ExprIsConstantOrGroupBy(pWalker->pParse, pExpr, pS->pGroupBy) ){ sqlite3 *db = pWalker->pParse->db; - Expr *pNew = sqlite3ExprAlloc(db, TK_INTEGER, &sqlite3IntTokens[1], 0); + Expr *pNew = sqlite3Expr(db, TK_INTEGER, "1"); if( pNew ){ Expr *pWhere = pS->pWhere; SWAP(Expr, *pNew, *pExpr); - pNew = sqlite3ExprAnd(db, pWhere, pNew); + pNew = sqlite3ExprAnd(pWalker->pParse, pWhere, pNew); pS->pWhere = pNew; pWalker->eCode = 1; } @@ -129808,15 +131152,19 @@ static struct SrcList_item *isSelfJoinView( if( pItem->pSelect==0 ) continue; if( pItem->fg.viaCoroutine ) continue; if( pItem->zName==0 ) continue; - if( sqlite3_stricmp(pItem->zDatabase, pThis->zDatabase)!=0 ) continue; + assert( pItem->pTab!=0 ); + assert( pThis->pTab!=0 ); + if( pItem->pTab->pSchema!=pThis->pTab->pSchema ) continue; if( sqlite3_stricmp(pItem->zName, pThis->zName)!=0 ) continue; pS1 = pItem->pSelect; - if( pThis->pSelect->selId!=pS1->selId ){ + if( pItem->pTab->pSchema==0 && pThis->pSelect->selId!=pS1->selId ){ /* The query flattener left two different CTE tables with identical ** names in the same FROM clause. */ continue; } - if( sqlite3ExprCompare(0, pThis->pSelect->pWhere, pS1->pWhere, -1) ){ + if( sqlite3ExprCompare(0, pThis->pSelect->pWhere, pS1->pWhere, -1) + || sqlite3ExprCompare(0, pThis->pSelect->pHaving, pS1->pHaving, -1) + ){ /* The view was modified by some other optimization such as ** pushDownWhereTerms() */ continue; @@ -129841,7 +131189,8 @@ static struct SrcList_item *isSelfJoinView( ** * The subquery is a UNION ALL of two or more terms ** * The subquery does not have a LIMIT clause ** * There is no WHERE or GROUP BY or HAVING clauses on the subqueries -** * The outer query is a simple count(*) +** * The outer query is a simple count(*) with no WHERE clause or other +** extraneous syntax. ** ** Return TRUE if the optimization is undertaken. */ @@ -129852,6 +131201,8 @@ static int countOfViewOptimization(Parse *pParse, Select *p){ sqlite3 *db; if( (p->selFlags & SF_Aggregate)==0 ) return 0; /* This is an aggregate */ if( p->pEList->nExpr!=1 ) return 0; /* Single result column */ + if( p->pWhere ) return 0; + if( p->pGroupBy ) return 0; pExpr = p->pEList->a[0].pExpr; if( pExpr->op!=TK_AGG_FUNCTION ) return 0; /* Result is an aggregate */ if( sqlite3_stricmp(pExpr->u.zToken,"count") ) return 0; /* Is count() */ @@ -130164,7 +131515,7 @@ SQLITE_PRIVATE int sqlite3Select( ** assume the column name is non-NULL and segfault. The use of an empty ** string for the fake column name seems safer. */ - if( pItem->colUsed==0 ){ + if( pItem->colUsed==0 && pItem->zName!=0 ){ sqlite3AuthCheck(pParse, SQLITE_READ, pItem->zName, "", pItem->zDatabase); } @@ -130178,8 +131529,15 @@ SQLITE_PRIVATE int sqlite3Select( ** technically harmless for it to be generated multiple times. The ** following assert() will detect if something changes to cause ** the same subquery to be coded multiple times, as a signal to the - ** developers to try to optimize the situation. */ - assert( pItem->addrFillSub==0 ); + ** developers to try to optimize the situation. + ** + ** Update 2019-07-24: + ** See ticket https://sqlite.org/src/tktview/c52b09c7f38903b1311cec40. + ** The dbsqlfuzz fuzzer found a case where the same subquery gets + ** coded twice. So this assert() now becomes a testcase(). It should + ** be very rare, though. + */ + testcase( pItem->addrFillSub!=0 ); /* Increment Parse.nHeight by the height of the largest expression ** tree referred to by this, the parent select. The child select @@ -130253,7 +131611,7 @@ SQLITE_PRIVATE int sqlite3Select( int retAddr; struct SrcList_item *pPrior; - assert( pItem->addrFillSub==0 ); + testcase( pItem->addrFillSub==0 ); /* Ticket c52b09c7f38903b1311 */ pItem->regReturn = ++pParse->nMem; topAddr = sqlite3VdbeAddOp2(v, OP_Integer, 0, pItem->regReturn); pItem->addrFillSub = topAddr+1; @@ -130493,23 +131851,35 @@ SQLITE_PRIVATE int sqlite3Select( } assert( 66==sqlite3LogEst(100) ); if( p->nSelectRow>66 ) p->nSelectRow = 66; + + /* If there is both a GROUP BY and an ORDER BY clause and they are + ** identical, then it may be possible to disable the ORDER BY clause + ** on the grounds that the GROUP BY will cause elements to come out + ** in the correct order. It also may not - the GROUP BY might use a + ** database index that causes rows to be grouped together as required + ** but not actually sorted. Either way, record the fact that the + ** ORDER BY and GROUP BY clauses are the same by setting the orderByGrp + ** variable. */ + if( sSort.pOrderBy && pGroupBy->nExpr==sSort.pOrderBy->nExpr ){ + int ii; + /* The GROUP BY processing doesn't care whether rows are delivered in + ** ASC or DESC order - only that each group is returned contiguously. + ** So set the ASC/DESC flags in the GROUP BY to match those in the + ** ORDER BY to maximize the chances of rows being delivered in an + ** order that makes the ORDER BY redundant. */ + for(ii=0; iinExpr; ii++){ + u8 sortFlags = sSort.pOrderBy->a[ii].sortFlags & KEYINFO_ORDER_DESC; + pGroupBy->a[ii].sortFlags = sortFlags; + } + if( sqlite3ExprListCompare(pGroupBy, sSort.pOrderBy, -1)==0 ){ + orderByGrp = 1; + } + } }else{ assert( 0==sqlite3LogEst(1) ); p->nSelectRow = 0; } - /* If there is both a GROUP BY and an ORDER BY clause and they are - ** identical, then it may be possible to disable the ORDER BY clause - ** on the grounds that the GROUP BY will cause elements to come out - ** in the correct order. It also may not - the GROUP BY might use a - ** database index that causes rows to be grouped together as required - ** but not actually sorted. Either way, record the fact that the - ** ORDER BY and GROUP BY clauses are the same by setting the orderByGrp - ** variable. */ - if( sqlite3ExprListCompare(pGroupBy, sSort.pOrderBy, -1)==0 ){ - orderByGrp = 1; - } - /* Create a label to jump to when we want to abort the query */ addrEnd = sqlite3VdbeMakeLabel(pParse); @@ -130544,9 +131914,16 @@ SQLITE_PRIVATE int sqlite3Select( minMaxFlag = WHERE_ORDERBY_NORMAL; } for(i=0; ix.pList); + sqlite3ExprAnalyzeAggList(&sNC, pExpr->x.pList); +#ifndef SQLITE_OMIT_WINDOWFUNC + assert( !IsWindowFunc(pExpr) ); + if( ExprHasProperty(pExpr, EP_WinFunc) ){ + sqlite3ExprAnalyzeAggregates(&sNC, pExpr->y.pWin->pFilter); + } +#endif sNC.ncFlags &= ~NC_InAggFunc; } sAggInfo.mxReg = pParse->nMem; @@ -130858,13 +132235,18 @@ SQLITE_PRIVATE int sqlite3Select( { int regAcc = 0; /* "populate accumulators" flag */ - /* If there are accumulator registers but no min() or max() functions, - ** allocate register regAcc. Register regAcc will contain 0 the first - ** time the inner loop runs, and 1 thereafter. The code generated - ** by updateAccumulator() only updates the accumulator registers if - ** regAcc contains 0. */ + /* If there are accumulator registers but no min() or max() functions + ** without FILTER clauses, allocate register regAcc. Register regAcc + ** will contain 0 the first time the inner loop runs, and 1 thereafter. + ** The code generated by updateAccumulator() uses this to ensure + ** that the accumulator registers are (a) updated only once if + ** there are no min() or max functions or (b) always updated for the + ** first row visited by the aggregate, so that they are updated at + ** least once even if the FILTER clause means the min() or max() + ** function visits zero rows. */ if( sAggInfo.nAccumulator ){ for(i=0; ifuncFlags&SQLITE_FUNC_NEEDCOLL ) break; } if( i==sAggInfo.nFunc ){ @@ -131335,7 +132717,11 @@ SQLITE_PRIVATE void sqlite3BeginTrigger( /* Check that the trigger name is not reserved and that no trigger of the ** specified name exists */ zName = sqlite3NameFromToken(db, pName); - if( !zName || SQLITE_OK!=sqlite3CheckObjectName(pParse, zName) ){ + if( zName==0 ){ + assert( db->mallocFailed ); + goto trigger_cleanup; + } + if( sqlite3CheckObjectName(pParse, zName, "trigger", pTab->zName) ){ goto trigger_cleanup; } assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); @@ -131498,6 +132884,7 @@ SQLITE_PRIVATE void sqlite3FinishTrigger( Trigger *pLink = pTrig; Hash *pHash = &db->aDb[iDb].pSchema->trigHash; assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); + assert( pLink!=0 ); pTrig = sqlite3HashInsert(pHash, zName, pTrig); if( pTrig ){ sqlite3OomFault(db); @@ -131616,6 +133003,9 @@ SQLITE_PRIVATE TriggerStep *sqlite3TriggerInsertStep( pTriggerStep->pIdList = pColumn; pTriggerStep->pUpsert = pUpsert; pTriggerStep->orconf = orconf; + if( pUpsert ){ + sqlite3HasExplicitNulls(pParse, pUpsert->pUpsertTarget); + } }else{ testcase( pColumn ); sqlite3IdListDelete(db, pColumn); @@ -131771,10 +133161,9 @@ SQLITE_PRIVATE void sqlite3DropTriggerPtr(Parse *pParse, Trigger *pTrigger){ iDb = sqlite3SchemaToIndex(pParse->db, pTrigger->pSchema); assert( iDb>=0 && iDbnDb ); pTable = tableOfTrigger(pTrigger); - assert( pTable ); - assert( pTable->pSchema==pTrigger->pSchema || iDb==1 ); + assert( (pTable && pTable->pSchema==pTrigger->pSchema) || iDb==1 ); #ifndef SQLITE_OMIT_AUTHORIZATION - { + if( pTable ){ int code = SQLITE_DROP_TRIGGER; const char *zDb = db->aDb[iDb].zDbSName; const char *zTab = SCHEMA_TABLE(iDb); @@ -131788,7 +133177,6 @@ SQLITE_PRIVATE void sqlite3DropTriggerPtr(Parse *pParse, Trigger *pTrigger){ /* Generate code to destroy the database record of the trigger. */ - assert( pTable!=0 ); if( (v = sqlite3GetVdbe(pParse))!=0 ){ sqlite3NestedParse(pParse, "DELETE FROM %Q.%s WHERE name=%Q AND type='trigger'", @@ -131812,9 +133200,11 @@ SQLITE_PRIVATE void sqlite3UnlinkAndDeleteTrigger(sqlite3 *db, int iDb, const ch if( ALWAYS(pTrigger) ){ if( pTrigger->pSchema==pTrigger->pTabSchema ){ Table *pTab = tableOfTrigger(pTrigger); - Trigger **pp; - for(pp=&pTab->pTrigger; *pp!=pTrigger; pp=&((*pp)->pNext)); - *pp = (*pp)->pNext; + if( pTab ){ + Trigger **pp; + for(pp=&pTab->pTrigger; *pp!=pTrigger; pp=&((*pp)->pNext)); + *pp = (*pp)->pNext; + } } sqlite3DeleteTrigger(db, pTrigger); db->mDbFlags |= DBFLAG_SchemaChange; @@ -132502,11 +133892,12 @@ SQLITE_PRIVATE void sqlite3Update( Index *pIdx; /* For looping over indices */ Index *pPk; /* The PRIMARY KEY index for WITHOUT ROWID tables */ int nIdx; /* Number of indices that need updating */ + int nAllIdx; /* Total number of indexes */ int iBaseCur; /* Base cursor number */ int iDataCur; /* Cursor for the canonical data btree */ int iIdxCur; /* Cursor for the first index */ sqlite3 *db; /* The database structure */ - int *aRegIdx = 0; /* First register in array assigned to each index */ + int *aRegIdx = 0; /* Registers for to each index and the main table */ int *aXRef = 0; /* aXRef[i] is the index in pChanges->a[] of the ** an expression for the i-th column of the table. ** aXRef[i]==-1 if the i-th column is not changed. */ @@ -132620,10 +134011,10 @@ SQLITE_PRIVATE void sqlite3Update( /* Allocate space for aXRef[], aRegIdx[], and aToOpen[]. ** Initialize aXRef[] and aToOpen[] to their default values. */ - aXRef = sqlite3DbMallocRawNN(db, sizeof(int) * (pTab->nCol+nIdx) + nIdx+2 ); + aXRef = sqlite3DbMallocRawNN(db, sizeof(int) * (pTab->nCol+nIdx+1) + nIdx+2 ); if( aXRef==0 ) goto update_cleanup; aRegIdx = aXRef+pTab->nCol; - aToOpen = (u8*)(aRegIdx+nIdx); + aToOpen = (u8*)(aRegIdx+nIdx+1); memset(aToOpen, 1, nIdx+1); aToOpen[nIdx+1] = 0; for(i=0; inCol; i++) aXRef[i] = -1; @@ -132702,7 +134093,7 @@ SQLITE_PRIVATE void sqlite3Update( ** the key for accessing each index. */ if( onError==OE_Replace ) bReplace = 1; - for(j=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, j++){ + for(nAllIdx=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, nAllIdx++){ int reg; if( chngKey || hasFK>1 || pIdx==pPk || indexWhereClauseMightChange(pIdx,aXRef,chngRowid) @@ -132722,9 +134113,10 @@ SQLITE_PRIVATE void sqlite3Update( } } } - if( reg==0 ) aToOpen[j+1] = 0; - aRegIdx[j] = reg; + if( reg==0 ) aToOpen[nAllIdx+1] = 0; + aRegIdx[nAllIdx] = reg; } + aRegIdx[nAllIdx] = ++pParse->nMem; /* Register storing the table record */ if( bReplace ){ /* If REPLACE conflict resolution might be invoked, open cursors on all ** indexes in case they are needed to delete records. */ @@ -132739,7 +134131,13 @@ SQLITE_PRIVATE void sqlite3Update( /* Allocate required registers. */ if( !IsVirtual(pTab) ){ - regRowSet = ++pParse->nMem; + /* For now, regRowSet and aRegIdx[nAllIdx] share the same register. + ** If regRowSet turns out to be needed, then aRegIdx[nAllIdx] will be + ** reallocated. aRegIdx[nAllIdx] is the register in which the main + ** table record is written. regRowSet holds the RowSet for the + ** two-pass update algorithm. */ + assert( aRegIdx[nAllIdx]==pParse->nMem ); + regRowSet = aRegIdx[nAllIdx]; regOldRowid = regNewRowid = ++pParse->nMem; if( chngPk || pTrigger || hasFK ){ regOld = pParse->nMem + 1; @@ -132869,6 +134267,8 @@ SQLITE_PRIVATE void sqlite3Update( ** leave it in register regOldRowid. */ sqlite3VdbeAddOp2(v, OP_Rowid, iDataCur, regOldRowid); if( eOnePass==ONEPASS_OFF ){ + /* We need to use regRowSet, so reallocate aRegIdx[nAllIdx] */ + aRegIdx[nAllIdx] = ++pParse->nMem; sqlite3VdbeAddOp2(v, OP_RowSetAdd, regRowSet, regOldRowid); } }else{ @@ -133052,28 +134452,30 @@ SQLITE_PRIVATE void sqlite3Update( } if( !isView ){ - int addr1 = 0; /* Address of jump instruction */ - /* Do constraint checks. */ assert( regOldRowid>0 ); sqlite3GenerateConstraintChecks(pParse, pTab, aRegIdx, iDataCur, iIdxCur, regNewRowid, regOldRowid, chngKey, onError, labelContinue, &bReplace, aXRef, 0); - /* Do FK constraint checks. */ - if( hasFK ){ - sqlite3FkCheck(pParse, pTab, regOldRowid, 0, aXRef, chngKey); - } - - /* Delete the index entries associated with the current record. */ + /* If REPLACE conflict handling may have been used, or if the PK of the + ** row is changing, then the GenerateConstraintChecks() above may have + ** moved cursor iDataCur. Reseek it. */ if( bReplace || chngKey ){ if( pPk ){ - addr1 = sqlite3VdbeAddOp4Int(v, OP_NotFound, iDataCur, 0, regKey, nKey); + sqlite3VdbeAddOp4Int(v, OP_NotFound,iDataCur,labelContinue,regKey,nKey); }else{ - addr1 = sqlite3VdbeAddOp3(v, OP_NotExists, iDataCur, 0, regOldRowid); + sqlite3VdbeAddOp3(v, OP_NotExists, iDataCur, labelContinue,regOldRowid); } VdbeCoverageNeverTaken(v); } + + /* Do FK constraint checks. */ + if( hasFK ){ + sqlite3FkCheck(pParse, pTab, regOldRowid, 0, aXRef, chngKey); + } + + /* Delete the index entries associated with the current record. */ sqlite3GenerateRowIndexDelete(pParse, pTab, iDataCur, iIdxCur, aRegIdx, -1); /* If changing the rowid value, or if there are foreign key constraints @@ -133103,9 +134505,6 @@ SQLITE_PRIVATE void sqlite3Update( sqlite3VdbeAddOp2(v, OP_Delete, iDataCur, 0); } #endif - if( bReplace || chngKey ){ - sqlite3VdbeJumpHere(v, addr1); - } if( hasFK ){ sqlite3FkCheck(pParse, pTab, 0, regNewRowid, aXRef, chngKey); @@ -133544,6 +134943,7 @@ SQLITE_PRIVATE void sqlite3UpsertDoUpdate( sqlite3 *db = pParse->db; SrcList *pSrc; /* FROM clause for the UPDATE */ int iDataCur; + int i; assert( v!=0 ); assert( pUpsert!=0 ); @@ -133560,7 +134960,6 @@ SQLITE_PRIVATE void sqlite3UpsertDoUpdate( Index *pPk = sqlite3PrimaryKeyIndex(pTab); int nPk = pPk->nKeyCol; int iPk = pParse->nMem+1; - int i; pParse->nMem += nPk; for(i=0; ipUpsertSrc, 0); + /* excluded.* columns of type REAL need to be converted to a hard real */ + for(i=0; inCol; i++){ + if( pTab->aCol[i].affinity==SQLITE_AFF_REAL ){ + sqlite3VdbeAddOp1(v, OP_RealAffinity, pUpsert->regData+i); + } + } sqlite3Update(pParse, pSrc, pUpsert->pUpsertSet, pUpsert->pUpsertWhere, OE_Abort, 0, 0, pUpsert); pUpsert->pUpsertSet = 0; /* Will have been deleted by sqlite3Update() */ @@ -133700,6 +135105,7 @@ SQLITE_PRIVATE void sqlite3Vacuum(Parse *pParse, Token *pNm, Expr *pInto){ Vdbe *v = sqlite3GetVdbe(pParse); int iDb = 0; if( v==0 ) goto build_vacuum_end; + if( pParse->nErr ) goto build_vacuum_end; if( pNm ){ #ifndef SQLITE_BUG_COMPATIBLE_20160819 /* Default behavior: Report an error if the argument to VACUUM is @@ -133733,11 +135139,11 @@ SQLITE_PRIVATE void sqlite3Vacuum(Parse *pParse, Token *pNm, Expr *pInto){ /* ** This routine implements the OP_Vacuum opcode of the VDBE. */ -SQLITE_PRIVATE int sqlite3RunVacuum( +SQLITE_PRIVATE SQLITE_NOINLINE int sqlite3RunVacuum( char **pzErrMsg, /* Write error message here */ sqlite3 *db, /* Database connection */ int iDb, /* Which attached DB to vacuum */ - sqlite3_value *pOut /* Write results here, if not NULL */ + sqlite3_value *pOut /* Write results here, if not NULL. VACUUM INTO */ ){ int rc = SQLITE_OK; /* Return code from service routines */ Btree *pMain; /* The database being vacuumed */ @@ -133746,6 +135152,7 @@ SQLITE_PRIVATE int sqlite3RunVacuum( u64 saved_flags; /* Saved value of db->flags */ int saved_nChange; /* Saved value of db->nChange */ int saved_nTotalChange; /* Saved value of db->nTotalChange */ + u32 saved_openFlags; /* Saved value of db->openFlags */ u8 saved_mTrace; /* Saved trace settings */ Db *pDb = 0; /* Database to detach at end of vacuum */ int isMemDb; /* True if vacuuming a :memory: database */ @@ -133756,18 +135163,21 @@ SQLITE_PRIVATE int sqlite3RunVacuum( if( !db->autoCommit ){ sqlite3SetString(pzErrMsg, db, "cannot VACUUM from within a transaction"); - return SQLITE_ERROR; + return SQLITE_ERROR; /* IMP: R-12218-18073 */ } if( db->nVdbeActive>1 ){ sqlite3SetString(pzErrMsg, db,"cannot VACUUM - SQL statements in progress"); - return SQLITE_ERROR; + return SQLITE_ERROR; /* IMP: R-15610-35227 */ } + saved_openFlags = db->openFlags; if( pOut ){ if( sqlite3_value_type(pOut)!=SQLITE_TEXT ){ sqlite3SetString(pzErrMsg, db, "non-text filename"); return SQLITE_ERROR; } zOut = (const char*)sqlite3_value_text(pOut); + db->openFlags &= ~SQLITE_OPEN_READONLY; + db->openFlags |= SQLITE_OPEN_CREATE|SQLITE_OPEN_READWRITE; }else{ zOut = ""; } @@ -133806,6 +135216,7 @@ SQLITE_PRIVATE int sqlite3RunVacuum( */ nDb = db->nDb; rc = execSqlF(db, pzErrMsg, "ATTACH %Q AS vacuum_db", zOut); + db->openFlags = saved_openFlags; if( rc!=SQLITE_OK ) goto end_of_vacuum; assert( (db->nDb-1)==nDb ); pDb = &db->aDb[nDb]; @@ -133819,6 +135230,7 @@ SQLITE_PRIVATE int sqlite3RunVacuum( sqlite3SetString(pzErrMsg, db, "output file already exists"); goto end_of_vacuum; } + db->mDbFlags |= DBFLAG_VacuumInto; } nRes = sqlite3BtreeGetOptimalReserve(pMain); @@ -134037,6 +135449,9 @@ struct VtabCtx { ** Construct and install a Module object for a virtual table. When this ** routine is called, it is guaranteed that all appropriate locks are held ** and the module is not already part of the connection. +** +** If there already exists a module with zName, replace it with the new one. +** If pModule==0, then delete the module zName if it exists. */ SQLITE_PRIVATE Module *sqlite3VtabCreateModule( sqlite3 *db, /* Database in which module is registered */ @@ -134046,25 +135461,36 @@ SQLITE_PRIVATE Module *sqlite3VtabCreateModule( void (*xDestroy)(void *) /* Module destructor function */ ){ Module *pMod; - int nName = sqlite3Strlen30(zName); - pMod = (Module *)sqlite3Malloc(sizeof(Module) + nName + 1); - if( pMod==0 ){ - sqlite3OomFault(db); + Module *pDel; + char *zCopy; + if( pModule==0 ){ + zCopy = (char*)zName; + pMod = 0; }else{ - Module *pDel; - char *zCopy = (char *)(&pMod[1]); + int nName = sqlite3Strlen30(zName); + pMod = (Module *)sqlite3Malloc(sizeof(Module) + nName + 1); + if( pMod==0 ){ + sqlite3OomFault(db); + return 0; + } + zCopy = (char *)(&pMod[1]); memcpy(zCopy, zName, nName+1); pMod->zName = zCopy; pMod->pModule = pModule; pMod->pAux = pAux; pMod->xDestroy = xDestroy; pMod->pEpoTab = 0; - pDel = (Module *)sqlite3HashInsert(&db->aModule,zCopy,(void*)pMod); - assert( pDel==0 || pDel==pMod ); - if( pDel ){ + pMod->nRefModule = 1; + } + pDel = (Module *)sqlite3HashInsert(&db->aModule,zCopy,(void*)pMod); + if( pDel ){ + if( pDel==pMod ){ sqlite3OomFault(db); sqlite3DbFree(db, pDel); pMod = 0; + }else{ + sqlite3VtabEponymousTableClear(db, pDel); + sqlite3VtabModuleUnref(db, pDel); } } return pMod; @@ -134085,11 +135511,7 @@ static int createModule( int rc = SQLITE_OK; sqlite3_mutex_enter(db->mutex); - if( sqlite3HashFind(&db->aModule, zName) ){ - rc = SQLITE_MISUSE_BKPT; - }else{ - (void)sqlite3VtabCreateModule(db, zName, pModule, pAux, xDestroy); - } + (void)sqlite3VtabCreateModule(db, zName, pModule, pAux, xDestroy); rc = sqlite3ApiExit(db, rc); if( rc!=SQLITE_OK && xDestroy ) xDestroy(pAux); sqlite3_mutex_leave(db->mutex); @@ -134128,6 +135550,44 @@ SQLITE_API int sqlite3_create_module_v2( return createModule(db, zName, pModule, pAux, xDestroy); } +/* +** External API to drop all virtual-table modules, except those named +** on the azNames list. +*/ +SQLITE_API int sqlite3_drop_modules(sqlite3 *db, const char** azNames){ + HashElem *pThis, *pNext; +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif + for(pThis=sqliteHashFirst(&db->aModule); pThis; pThis=pNext){ + Module *pMod = (Module*)sqliteHashData(pThis); + pNext = sqliteHashNext(pThis); + if( azNames ){ + int ii; + for(ii=0; azNames[ii]!=0 && strcmp(azNames[ii],pMod->zName)!=0; ii++){} + if( azNames[ii]!=0 ) continue; + } + createModule(db, pMod->zName, 0, 0, 0); + } + return SQLITE_OK; +} + +/* +** Decrement the reference count on a Module object. Destroy the +** module when the reference count reaches zero. +*/ +SQLITE_PRIVATE void sqlite3VtabModuleUnref(sqlite3 *db, Module *pMod){ + assert( pMod->nRefModule>0 ); + pMod->nRefModule--; + if( pMod->nRefModule==0 ){ + if( pMod->xDestroy ){ + pMod->xDestroy(pMod->pAux); + } + assert( pMod->pEpoTab==0 ); + sqlite3DbFree(db, pMod); + } +} + /* ** Lock the virtual table so that it cannot be disconnected. ** Locks nest. Every lock should have a corresponding unlock. @@ -134167,6 +135627,7 @@ SQLITE_PRIVATE void sqlite3VtabUnlock(VTable *pVTab){ pVTab->nRef--; if( pVTab->nRef==0 ){ sqlite3_vtab *p = pVTab->pVtab; + sqlite3VtabModuleUnref(pVTab->db, pVTab->pMod); if( p ){ p->pModule->xDisconnect(p); } @@ -134307,9 +135768,13 @@ SQLITE_PRIVATE void sqlite3VtabClear(sqlite3 *db, Table *p){ ** string will be freed automatically when the table is ** deleted. */ -static void addModuleArgument(sqlite3 *db, Table *pTable, char *zArg){ - int nBytes = sizeof(char *)*(2+pTable->nModuleArg); +static void addModuleArgument(Parse *pParse, Table *pTable, char *zArg){ + sqlite3_int64 nBytes = sizeof(char *)*(2+pTable->nModuleArg); char **azModuleArg; + sqlite3 *db = pParse->db; + if( pTable->nModuleArg+3>=db->aLimit[SQLITE_LIMIT_COLUMN] ){ + sqlite3ErrorMsg(pParse, "too many columns on %s", pTable->zName); + } azModuleArg = sqlite3DbRealloc(db, pTable->azModuleArg, nBytes); if( azModuleArg==0 ){ sqlite3DbFree(db, zArg); @@ -134344,9 +135809,9 @@ SQLITE_PRIVATE void sqlite3VtabBeginParse( db = pParse->db; assert( pTable->nModuleArg==0 ); - addModuleArgument(db, pTable, sqlite3NameFromToken(db, pModuleName)); - addModuleArgument(db, pTable, 0); - addModuleArgument(db, pTable, sqlite3DbStrDup(db, pTable->zName)); + addModuleArgument(pParse, pTable, sqlite3NameFromToken(db, pModuleName)); + addModuleArgument(pParse, pTable, 0); + addModuleArgument(pParse, pTable, sqlite3DbStrDup(db, pTable->zName)); assert( (pParse->sNameToken.z==pName2->z && pName2->z!=0) || (pParse->sNameToken.z==pName1->z && pName2->z==0) ); @@ -134379,7 +135844,7 @@ static void addArgumentToVtab(Parse *pParse){ const char *z = (const char*)pParse->sArg.z; int n = pParse->sArg.n; sqlite3 *db = pParse->db; - addModuleArgument(db, pParse->pNewTable, sqlite3DbStrNDup(db, z, n)); + addModuleArgument(pParse, pParse->pNewTable, sqlite3DbStrNDup(db, z, n)); } } @@ -134567,6 +136032,7 @@ static int vtabCallConstructor( ** the sqlite3_vtab object if successful. */ memset(pVTable->pVtab, 0, sizeof(pVTable->pVtab[0])); pVTable->pVtab->pModule = pMod->pModule; + pMod->nRefModule++; pVTable->nRef = 1; if( sCtx.bDeclared==0 ){ const char *zFormat = "vtable constructor did not declare schema: %s"; @@ -134668,7 +136134,8 @@ static int growVTrans(sqlite3 *db){ /* Grow the sqlite3.aVTrans array if required */ if( (db->nVTrans%ARRAY_INCR)==0 ){ VTable **aVTrans; - int nBytes = sizeof(sqlite3_vtab *) * (db->nVTrans + ARRAY_INCR); + sqlite3_int64 nBytes = sizeof(sqlite3_vtab*)* + ((sqlite3_int64)db->nVTrans + ARRAY_INCR); aVTrans = sqlite3DbRealloc(db, (void *)db->aVTrans, nBytes); if( !aVTrans ){ return SQLITE_NOMEM_BKPT; @@ -134841,6 +136308,7 @@ SQLITE_PRIVATE int sqlite3VtabCallDestroy(sqlite3 *db, int iDb, const char *zTab p = vtabDisconnectAll(db, pTab); xDestroy = p->pMod->pModule->xDestroy; assert( xDestroy!=0 ); /* Checked before the virtual table is created */ + pTab->nTabRef++; rc = xDestroy(p->pVtab); /* Remove the sqlite3_vtab* from the aVTrans[] array, if applicable */ if( rc==SQLITE_OK ){ @@ -134849,6 +136317,7 @@ SQLITE_PRIVATE int sqlite3VtabCallDestroy(sqlite3 *db, int iDb, const char *zTab pTab->pVTable = 0; sqlite3VtabUnlock(p); } + sqlite3DeleteTable(db, pTab); } return rc; @@ -135164,9 +136633,9 @@ SQLITE_PRIVATE int sqlite3VtabEponymousTableInit(Parse *pParse, Module *pMod){ pTab->pSchema = db->aDb[0].pSchema; assert( pTab->nModuleArg==0 ); pTab->iPKey = -1; - addModuleArgument(db, pTab, sqlite3DbStrDup(db, pTab->zName)); - addModuleArgument(db, pTab, 0); - addModuleArgument(db, pTab, sqlite3DbStrDup(db, pTab->zName)); + addModuleArgument(pParse, pTab, sqlite3DbStrDup(db, pTab->zName)); + addModuleArgument(pParse, pTab, 0); + addModuleArgument(pParse, pTab, sqlite3DbStrDup(db, pTab->zName)); rc = vtabCallConstructor(db, pTab, pMod, pModule->xConnect, &zErr); if( rc ){ sqlite3ErrorMsg(pParse, "%s", zErr); @@ -135291,6 +136760,8 @@ SQLITE_API int sqlite3_vtab_config(sqlite3 *db, int op, ...){ ** planner logic in "where.c". These definitions are broken out into ** a separate source file for easier editing. */ +#ifndef SQLITE_WHEREINT_H +#define SQLITE_WHEREINT_H /* ** Trace output macros @@ -135346,13 +136817,15 @@ struct WhereLevel { int addrCont; /* Jump here to continue with the next loop cycle */ int addrFirst; /* First instruction of interior of the loop */ int addrBody; /* Beginning of the body of this loop */ + int regBignull; /* big-null flag reg. True if a NULL-scan is needed */ + int addrBignull; /* Jump here for next part of big-null scan */ #ifndef SQLITE_LIKE_DOESNT_MATCH_BLOBS u32 iLikeRepCntr; /* LIKE range processing counter register (times 2) */ int addrLikeRep; /* LIKE range processing address */ #endif u8 iFrom; /* Which entry in the FROM clause */ u8 op, p3, p5; /* Opcode, P3 & P5 of the opcode that ends the loop */ - int p1, p2; /* Operands of the opcode used to ends the loop */ + int p1, p2; /* Operands of the opcode used to end the loop */ union { /* Information that depends on pWLoop->wsFlags */ struct { int nIn; /* Number of entries in aInLoop[] */ @@ -135403,7 +136876,7 @@ struct WhereLoop { u16 nEq; /* Number of equality constraints */ u16 nBtm; /* Size of BTM vector */ u16 nTop; /* Size of TOP vector */ - u16 nIdxCol; /* Index column used for ORDER BY */ + u16 nDistinctCol; /* Index columns used to sort for DISTINCT */ Index *pIndex; /* Index used, or NULL */ } btree; struct { /* Information for virtual tables */ @@ -135554,16 +137027,17 @@ struct WhereTerm { #define TERM_ORINFO 0x10 /* Need to free the WhereTerm.u.pOrInfo object */ #define TERM_ANDINFO 0x20 /* Need to free the WhereTerm.u.pAndInfo obj */ #define TERM_OR_OK 0x40 /* Used during OR-clause processing */ -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 +#ifdef SQLITE_ENABLE_STAT4 # define TERM_VNULL 0x80 /* Manufactured x>NULL or x<=NULL term */ #else -# define TERM_VNULL 0x00 /* Disabled if not using stat3 */ +# define TERM_VNULL 0x00 /* Disabled if not using stat4 */ #endif #define TERM_LIKEOPT 0x100 /* Virtual terms from the LIKE optimization */ #define TERM_LIKECOND 0x200 /* Conditionally this LIKE operator term */ #define TERM_LIKE 0x400 /* The original LIKE operator */ #define TERM_IS 0x800 /* Term.pExpr is an IS operator */ #define TERM_VARSELECT 0x1000 /* Term.pExpr contains a correlated sub-query */ +#define TERM_NOPARTIDX 0x2000 /* Not for use to enable a partial index */ /* ** An instance of the WhereScan object is used as an iterator for locating @@ -135674,7 +137148,7 @@ struct WhereLoopBuilder { ExprList *pOrderBy; /* ORDER BY clause */ WhereLoop *pNew; /* Template WhereLoop */ WhereOrSet *pOrSet; /* Record best loops here, if not NULL */ -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 +#ifdef SQLITE_ENABLE_STAT4 UnpackedRecord *pRec; /* Probe for stat4 (if required) */ int nRecValid; /* Number of valid fields currently in pRec */ #endif @@ -135861,6 +137335,9 @@ SQLITE_PRIVATE void sqlite3WhereTabFuncArgs(Parse*, struct SrcList_item*, WhereC #define WHERE_UNQ_WANTED 0x00010000 /* WHERE_ONEROW would have been helpful*/ #define WHERE_PARTIALIDX 0x00020000 /* The automatic index is partial */ #define WHERE_IN_EARLYOUT 0x00040000 /* Perhaps quit IN loops early */ +#define WHERE_BIGNULL_SORT 0x00080000 /* Column nEq of index is BIGNULL */ + +#endif /* !defined(SQLITE_WHEREINT_H) */ /************** End of whereInt.h ********************************************/ /************** Continuing where we left off in wherecode.c ******************/ @@ -136163,9 +137640,9 @@ static void disableTerm(WhereLevel *pLevel, WhereTerm *pTerm){ ** Code an OP_Affinity opcode to apply the column affinity string zAff ** to the n registers starting at base. ** -** As an optimization, SQLITE_AFF_BLOB entries (which are no-ops) at the -** beginning and end of zAff are ignored. If all entries in zAff are -** SQLITE_AFF_BLOB, then no code gets generated. +** As an optimization, SQLITE_AFF_BLOB and SQLITE_AFF_NONE entries (which +** are no-ops) at the beginning and end of zAff are ignored. If all entries +** in zAff are SQLITE_AFF_BLOB or SQLITE_AFF_NONE, then no code gets generated. ** ** This routine makes its own copy of zAff so that the caller is free ** to modify zAff after this routine returns. @@ -136178,15 +137655,16 @@ static void codeApplyAffinity(Parse *pParse, int base, int n, char *zAff){ } assert( v!=0 ); - /* Adjust base and n to skip over SQLITE_AFF_BLOB entries at the beginning - ** and end of the affinity string. + /* Adjust base and n to skip over SQLITE_AFF_BLOB and SQLITE_AFF_NONE + ** entries at the beginning and end of the affinity string. */ - while( n>0 && zAff[0]==SQLITE_AFF_BLOB ){ + assert( SQLITE_AFF_NONE0 && zAff[0]<=SQLITE_AFF_BLOB ){ n--; base++; zAff++; } - while( n>1 && zAff[n-1]==SQLITE_AFF_BLOB ){ + while( n>1 && zAff[n-1]<=SQLITE_AFF_BLOB ){ n--; } @@ -136427,7 +137905,6 @@ static int codeEqualityTerm( if( pLoop->aLTerm[i]->pExpr==pX ){ int iOut = iReg + i - iEq; if( eType==IN_INDEX_ROWID ){ - testcase( nEq>1 ); /* Happens with a UNIQUE index on ROWID */ pIn->addrInTop = sqlite3VdbeAddOp2(v, OP_Rowid, iTab, iOut); }else{ int iCol = aiMap ? aiMap[iMap++] : 0; @@ -136845,7 +138322,7 @@ static void codeCursorHint( } /* If we survive all prior tests, that means this term is worth hinting */ - pExpr = sqlite3ExprAnd(db, pExpr, sqlite3ExprDup(db, pTerm->pExpr, 0)); + pExpr = sqlite3ExprAnd(pParse, pExpr, sqlite3ExprDup(db, pTerm->pExpr, 0)); } if( pExpr!=0 ){ sWalker.xExprCallback = codeCursorHintFixExpr; @@ -136962,6 +138439,7 @@ typedef struct IdxExprTrans { static int whereIndexExprTransNode(Walker *p, Expr *pExpr){ IdxExprTrans *pX = p->u.pIdxTrans; if( sqlite3ExprCompare(0, pExpr, pX->pIdxExpr, pX->iTabCur)==0 ){ + pExpr->affExpr = sqlite3ExprAffinity(pExpr); pExpr->op = TK_COLUMN; pExpr->iTable = pX->iIdxCur; pExpr->iColumn = pX->iIdxCol; @@ -137005,6 +138483,34 @@ static void whereIndexExprTrans( } } +/* +** The pTruth expression is always true because it is the WHERE clause +** a partial index that is driving a query loop. Look through all of the +** WHERE clause terms on the query, and if any of those terms must be +** true because pTruth is true, then mark those WHERE clause terms as +** coded. +*/ +static void whereApplyPartialIndexConstraints( + Expr *pTruth, + int iTabCur, + WhereClause *pWC +){ + int i; + WhereTerm *pTerm; + while( pTruth->op==TK_AND ){ + whereApplyPartialIndexConstraints(pTruth->pLeft, iTabCur, pWC); + pTruth = pTruth->pRight; + } + for(i=0, pTerm=pWC->a; inTerm; i++, pTerm++){ + Expr *pExpr; + if( pTerm->wtFlags & TERM_CODED ) continue; + pExpr = pTerm->pExpr; + if( sqlite3ExprCompare(0, pExpr, pTruth, iTabCur)==0 ){ + pTerm->wtFlags |= TERM_CODED; + } + } +} + /* ** Generate code for the start of the iLevel-th loop in the WHERE clause ** implementation described by pWInfo. @@ -137189,6 +138695,9 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( sqlite3VdbeAddOp3(v, OP_SeekRowid, iCur, addrNxt, iRowidReg); VdbeCoverage(v); pLevel->op = OP_Noop; + if( (pTerm->prereqAll & pLevel->notReady)==0 ){ + pTerm->wtFlags |= TERM_CODED; + } }else if( (pLoop->wsFlags & WHERE_IPK)!=0 && (pLoop->wsFlags & WHERE_COLUMN_RANGE)!=0 ){ @@ -137363,32 +138872,12 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( u8 bSeekPastNull = 0; /* True to seek past initial nulls */ u8 bStopAtNull = 0; /* Add condition to terminate at NULLs */ int omitTable; /* True if we use the index only */ - + int regBignull = 0; /* big-null flag register */ pIdx = pLoop->u.btree.pIndex; iIdxCur = pLevel->iIdxCur; assert( nEq>=pLoop->nSkip ); - /* If this loop satisfies a sort order (pOrderBy) request that - ** was passed to this function to implement a "SELECT min(x) ..." - ** query, then the caller will only allow the loop to run for - ** a single iteration. This means that the first row returned - ** should not have a NULL value stored in 'x'. If column 'x' is - ** the first one after the nEq equality constraints in the index, - ** this requires some special handling. - */ - assert( pWInfo->pOrderBy==0 - || pWInfo->pOrderBy->nExpr==1 - || (pWInfo->wctrlFlags&WHERE_ORDERBY_MIN)==0 ); - if( (pWInfo->wctrlFlags&WHERE_ORDERBY_MIN)!=0 - && pWInfo->nOBSat>0 - && (pIdx->nKeyCol>nEq) - ){ - assert( pLoop->nSkip==0 ); - bSeekPastNull = 1; - nExtraReg = 1; - } - /* Find any inequality constraint terms for the start and end ** of the range. */ @@ -137429,6 +138918,25 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( } assert( pRangeEnd==0 || (pRangeEnd->wtFlags & TERM_VNULL)==0 ); + /* If the WHERE_BIGNULL_SORT flag is set, then index column nEq uses + ** a non-default "big-null" sort (either ASC NULLS LAST or DESC NULLS + ** FIRST). In both cases separate ordered scans are made of those + ** index entries for which the column is null and for those for which + ** it is not. For an ASC sort, the non-NULL entries are scanned first. + ** For DESC, NULL entries are scanned first. + */ + if( (pLoop->wsFlags & (WHERE_TOP_LIMIT|WHERE_BTM_LIMIT))==0 + && (pLoop->wsFlags & WHERE_BIGNULL_SORT)!=0 + ){ + assert( bSeekPastNull==0 && nExtraReg==0 && nBtm==0 && nTop==0 ); + assert( pRangeEnd==0 && pRangeStart==0 ); + assert( pLoop->nSkip==0 ); + nExtraReg = 1; + bSeekPastNull = 1; + pLevel->regBignull = regBignull = ++pParse->nMem; + pLevel->addrBignull = sqlite3VdbeMakeLabel(pParse); + } + /* If we are doing a reverse order scan on an ascending index, or ** a forward order scan on a descending index, interchange the ** start and end terms (pRangeStart and pRangeEnd). @@ -137451,7 +138959,7 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( if( zStartAff && nTop ){ zEndAff = sqlite3DbStrDup(db, &zStartAff[nEq]); } - addrNxt = pLevel->addrNxt; + addrNxt = (regBignull ? pLevel->addrBignull : pLevel->addrNxt); testcase( pRangeStart && (pRangeStart->eOperator & WO_LE)!=0 ); testcase( pRangeStart && (pRangeStart->eOperator & WO_GE)!=0 ); @@ -137485,10 +138993,14 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( } bSeekPastNull = 0; }else if( bSeekPastNull ){ + startEq = 0; sqlite3VdbeAddOp2(v, OP_Null, 0, regBase+nEq); + start_constraints = 1; nConstraint++; - startEq = 0; + }else if( regBignull ){ + sqlite3VdbeAddOp2(v, OP_Null, 0, regBase+nEq); start_constraints = 1; + nConstraint++; } codeApplyAffinity(pParse, regBase, nConstraint - bSeekPastNull, zStartAff); if( pLoop->nSkip>0 && nConstraint==pLoop->nSkip ){ @@ -137499,6 +139011,11 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( if( pLoop->wsFlags & WHERE_IN_EARLYOUT ){ sqlite3VdbeAddOp1(v, OP_SeekHit, iIdxCur); } + if( regBignull ){ + sqlite3VdbeAddOp2(v, OP_Integer, 1, regBignull); + VdbeComment((v, "NULL-scan pass ctr")); + } + op = aStartOp[(start_constraints<<2) + (startEq<<1) + bRev]; assert( op!=0 ); sqlite3VdbeAddOp4Int(v, op, iIdxCur, addrNxt, regBase, nConstraint); @@ -137509,6 +139026,23 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( VdbeCoverageIf(v, op==OP_SeekGE); testcase( op==OP_SeekGE ); VdbeCoverageIf(v, op==OP_SeekLE); testcase( op==OP_SeekLE ); VdbeCoverageIf(v, op==OP_SeekLT); testcase( op==OP_SeekLT ); + + assert( bSeekPastNull==0 || bStopAtNull==0 ); + if( regBignull ){ + assert( bSeekPastNull==1 || bStopAtNull==1 ); + assert( bSeekPastNull==!bStopAtNull ); + assert( bStopAtNull==startEq ); + sqlite3VdbeAddOp2(v, OP_Goto, 0, sqlite3VdbeCurrentAddr(v)+2); + op = aStartOp[(nConstraint>1)*4 + 2 + bRev]; + sqlite3VdbeAddOp4Int(v, op, iIdxCur, addrNxt, regBase, + nConstraint-startEq); + VdbeCoverage(v); + VdbeCoverageIf(v, op==OP_Rewind); testcase( op==OP_Rewind ); + VdbeCoverageIf(v, op==OP_Last); testcase( op==OP_Last ); + VdbeCoverageIf(v, op==OP_SeekGE); testcase( op==OP_SeekGE ); + VdbeCoverageIf(v, op==OP_SeekLE); testcase( op==OP_SeekLE ); + assert( op==OP_Rewind || op==OP_Last || op==OP_SeekGE || op==OP_SeekLE); + } } /* Load the value for the inequality constraint at the end of the @@ -137540,8 +139074,10 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( endEq = 1; } }else if( bStopAtNull ){ - sqlite3VdbeAddOp2(v, OP_Null, 0, regBase+nEq); - endEq = 0; + if( regBignull==0 ){ + sqlite3VdbeAddOp2(v, OP_Null, 0, regBase+nEq); + endEq = 0; + } nConstraint++; } sqlite3DbFree(db, zStartAff); @@ -137552,6 +139088,12 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( /* Check if the index cursor is past the end of the range. */ if( nConstraint ){ + if( regBignull ){ + /* Except, skip the end-of-range check while doing the NULL-scan */ + sqlite3VdbeAddOp2(v, OP_IfNot, regBignull, sqlite3VdbeCurrentAddr(v)+3); + VdbeComment((v, "If NULL-scan 2nd pass")); + VdbeCoverage(v); + } op = aEndOp[bRev*2 + endEq]; sqlite3VdbeAddOp4Int(v, op, iIdxCur, addrNxt, regBase, nConstraint); testcase( op==OP_IdxGT ); VdbeCoverageIf(v, op==OP_IdxGT ); @@ -137559,6 +139101,23 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( testcase( op==OP_IdxLT ); VdbeCoverageIf(v, op==OP_IdxLT ); testcase( op==OP_IdxLE ); VdbeCoverageIf(v, op==OP_IdxLE ); } + if( regBignull ){ + /* During a NULL-scan, check to see if we have reached the end of + ** the NULLs */ + assert( bSeekPastNull==!bStopAtNull ); + assert( bSeekPastNull+bStopAtNull==1 ); + assert( nConstraint+bSeekPastNull>0 ); + sqlite3VdbeAddOp2(v, OP_If, regBignull, sqlite3VdbeCurrentAddr(v)+2); + VdbeComment((v, "If NULL-scan 1st pass")); + VdbeCoverage(v); + op = aEndOp[bRev*2 + bSeekPastNull]; + sqlite3VdbeAddOp4Int(v, op, iIdxCur, addrNxt, regBase, + nConstraint+bSeekPastNull); + testcase( op==OP_IdxGT ); VdbeCoverageIf(v, op==OP_IdxGT ); + testcase( op==OP_IdxGE ); VdbeCoverageIf(v, op==OP_IdxGE ); + testcase( op==OP_IdxLT ); VdbeCoverageIf(v, op==OP_IdxLT ); + testcase( op==OP_IdxLE ); VdbeCoverageIf(v, op==OP_IdxLE ); + } if( pLoop->wsFlags & WHERE_IN_EARLYOUT ){ sqlite3VdbeAddOp2(v, OP_SeekHit, iIdxCur, 1); @@ -137611,6 +139170,14 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( whereIndexExprTrans(pIdx, iCur, iIdxCur, pWInfo); } + /* If a partial index is driving the loop, try to eliminate WHERE clause + ** terms from the query that must be true due to the WHERE clause of + ** the partial index + */ + if( pIdx->pPartIdxWhere ){ + whereApplyPartialIndexConstraints(pIdx->pPartIdxWhere, iCur, pWC); + } + /* Record the instruction used to terminate the loop. */ if( pLoop->wsFlags & WHERE_ONEROW ){ pLevel->op = OP_Noop; @@ -137771,10 +139338,15 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( if( (pWC->a[iTerm].eOperator & WO_ALL)==0 ) continue; testcase( pWC->a[iTerm].wtFlags & TERM_ORINFO ); pExpr = sqlite3ExprDup(db, pExpr, 0); - pAndExpr = sqlite3ExprAnd(db, pAndExpr, pExpr); + pAndExpr = sqlite3ExprAnd(pParse, pAndExpr, pExpr); } if( pAndExpr ){ - pAndExpr = sqlite3PExpr(pParse, TK_AND|TKFLG_DONTFOLD, 0, pAndExpr); + /* The extra 0x10000 bit on the opcode is masked off and does not + ** become part of the new Expr.op. However, it does make the + ** op==TK_AND comparison inside of sqlite3PExpr() false, and this + ** prevents sqlite3PExpr() from implementing AND short-circuit + ** optimization, which we do not want here. */ + pAndExpr = sqlite3PExpr(pParse, TK_AND|0x10000, 0, pAndExpr); } } @@ -137917,7 +139489,7 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( sqlite3VdbeGoto(v, pLevel->addrBrk); sqlite3VdbeResolveLabel(v, iLoopBody); - if( pWInfo->nLevel>1 ) sqlite3StackFree(db, pOrTab); + if( pWInfo->nLevel>1 ){ sqlite3StackFree(db, pOrTab); } if( !untestedTerms ) disableTerm(pLevel, pTerm); }else #endif /* SQLITE_OMIT_OR_OPTIMIZATION */ @@ -138004,8 +139576,9 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( u32 x = pLevel->iLikeRepCntr; if( x>0 ){ skipLikeAddr = sqlite3VdbeAddOp1(v, (x&1)?OP_IfNot:OP_If,(int)(x>>1)); + VdbeCoverageIf(v, (x&1)==1); + VdbeCoverageIf(v, (x&1)==0); } - VdbeCoverage(v); #endif } #ifdef WHERETRACE_ENABLED /* 0xffff */ @@ -138171,7 +139744,7 @@ static int whereClauseInsert(WhereClause *pWC, Expr *p, u16 wtFlags){ }else{ pTerm->truthProb = 1; } - pTerm->pExpr = sqlite3ExprSkipCollate(p); + pTerm->pExpr = sqlite3ExprSkipCollateAndLikely(p); pTerm->wtFlags = wtFlags; pTerm->pWC = pWC; pTerm->iParent = -1; @@ -138204,10 +139777,16 @@ static int allowedOp(int op){ ** the left hand side of a comparison overrides any collation sequence ** attached to the right. For the same reason the EP_Collate flag ** is not commuted. +** +** The return value is extra flags that are added to the WhereTerm object +** after it is commuted. The only extra flag ever added is TERM_NOPARTIDX +** which prevents the term from being used to enable a partial index if +** COLLATE changes have been made. */ -static void exprCommute(Parse *pParse, Expr *pExpr){ +static u16 exprCommute(Parse *pParse, Expr *pExpr){ u16 expRight = (pExpr->pRight->flags & EP_Collate); u16 expLeft = (pExpr->pLeft->flags & EP_Collate); + u16 wtFlags = 0; assert( allowedOp(pExpr->op) && pExpr->op!=TK_IN ); if( expRight==expLeft ){ /* Either X and Y both have COLLATE operator or neither do */ @@ -138215,11 +139794,13 @@ static void exprCommute(Parse *pParse, Expr *pExpr){ /* Both X and Y have COLLATE operators. Make sure X is always ** used by clearing the EP_Collate flag from Y. */ pExpr->pRight->flags &= ~EP_Collate; + wtFlags |= TERM_NOPARTIDX; }else if( sqlite3ExprCollSeq(pParse, pExpr->pLeft)!=0 ){ /* Neither X nor Y have COLLATE operators, but X has a non-default ** collating sequence. So add the EP_Collate marker on X to cause ** it to be searched first. */ pExpr->pLeft->flags |= EP_Collate; + wtFlags |= TERM_NOPARTIDX; } } SWAP(Expr*,pExpr->pRight,pExpr->pLeft); @@ -138231,6 +139812,7 @@ static void exprCommute(Parse *pParse, Expr *pExpr){ assert( pExpr->op>=TK_GT && pExpr->op<=TK_GE ); pExpr->op = ((pExpr->op-TK_GT)^2)+TK_GT; } + return wtFlags; } /* @@ -138349,27 +139931,38 @@ static int isLikeOrGlob( zNew[iTo++] = zNew[iFrom]; } zNew[iTo] = 0; + assert( iTo>0 ); - /* If the RHS begins with a digit or a minus sign, then the LHS must be - ** an ordinary column (not a virtual table column) with TEXT affinity. - ** Otherwise the LHS might be numeric and "lhs >= rhs" would be false - ** even though "lhs LIKE rhs" is true. But if the RHS does not start - ** with a digit or '-', then "lhs LIKE rhs" will always be false if - ** the LHS is numeric and so the optimization still works. + /* If the LHS is not an ordinary column with TEXT affinity, then the + ** pattern prefix boundaries (both the start and end boundaries) must + ** not look like a number. Otherwise the pattern might be treated as + ** a number, which will invalidate the LIKE optimization. ** - ** 2018-09-10 ticket c94369cae9b561b1f996d0054bfab11389f9d033 - ** The RHS pattern must not be '/%' because the termination condition - ** will then become "x<'0'" and if the affinity is numeric, will then - ** be converted into "x<0", which is incorrect. + ** Getting this right has been a persistent source of bugs in the + ** LIKE optimization. See, for example: + ** 2018-09-10 https://sqlite.org/src/info/c94369cae9b561b1 + ** 2019-05-02 https://sqlite.org/src/info/b043a54c3de54b28 + ** 2019-06-10 https://sqlite.org/src/info/fd76310a5e843e07 + ** 2019-06-14 https://sqlite.org/src/info/ce8717f0885af975 + ** 2019-09-03 https://sqlite.org/src/info/0f0428096f17252a */ - if( sqlite3Isdigit(zNew[0]) - || zNew[0]=='-' - || (zNew[0]+1=='0' && iTo==1) + if( pLeft->op!=TK_COLUMN + || sqlite3ExprAffinity(pLeft)!=SQLITE_AFF_TEXT + || IsVirtual(pLeft->y.pTab) /* Value might be numeric */ ){ - if( pLeft->op!=TK_COLUMN - || sqlite3ExprAffinity(pLeft)!=SQLITE_AFF_TEXT - || IsVirtual(pLeft->y.pTab) /* Value might be numeric */ - ){ + int isNum; + double rDummy; + isNum = sqlite3AtoF(zNew, &rDummy, iTo, SQLITE_UTF8); + if( isNum<=0 ){ + if( iTo==1 && zNew[0]=='-' ){ + isNum = +1; + }else{ + zNew[iTo-1]++; + isNum = sqlite3AtoF(zNew, &rDummy, iTo, SQLITE_UTF8); + zNew[iTo-1]--; + } + } + if( isNum>0 ){ sqlite3ExprDelete(db, pPrefix); sqlite3ValueFree(pVal); return 0; @@ -139221,7 +140814,7 @@ static void exprAnalyze( pDup = pExpr; pNew = pTerm; } - exprCommute(pParse, pDup); + pNew->wtFlags |= exprCommute(pParse, pDup); pNew->leftCursor = aiCurCol[0]; pNew->u.leftColumn = aiCurCol[1]; testcase( (prereqLeft | extraRight) != prereqLeft ); @@ -139462,8 +141055,8 @@ static void exprAnalyze( } } -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 - /* When sqlite_stat3 histogram data is available an operator of the +#ifdef SQLITE_ENABLE_STAT4 + /* When sqlite_stat4 histogram data is available an operator of the ** form "x IS NOT NULL" can sometimes be evaluated more efficiently ** as "x>NULL" if x is not an INTEGER PRIMARY KEY. So construct a ** virtual term of that form. @@ -139474,7 +141067,7 @@ static void exprAnalyze( && pExpr->pLeft->op==TK_COLUMN && pExpr->pLeft->iColumn>=0 && !ExprHasProperty(pExpr, EP_FromJoin) - && OptimizationEnabled(db, SQLITE_Stat34) + && OptimizationEnabled(db, SQLITE_Stat4) ){ Expr *pNewExpr; Expr *pLeft = pExpr->pLeft; @@ -139499,7 +141092,7 @@ static void exprAnalyze( pNewTerm->prereqAll = pTerm->prereqAll; } } -#endif /* SQLITE_ENABLE_STAT3_OR_STAT4 */ +#endif /* SQLITE_ENABLE_STAT4 */ /* Prevent ON clause terms of a LEFT JOIN from being used to drive ** an index for tables to the left of the join. @@ -139532,7 +141125,7 @@ static void exprAnalyze( ** all terms of the WHERE clause. */ SQLITE_PRIVATE void sqlite3WhereSplit(WhereClause *pWC, Expr *pExpr, u8 op){ - Expr *pE2 = sqlite3ExprSkipCollate(pExpr); + Expr *pE2 = sqlite3ExprSkipCollateAndLikely(pExpr); pWC->op = op; if( pE2==0 ) return; if( pE2->op!=op ){ @@ -139607,6 +141200,12 @@ SQLITE_PRIVATE Bitmask sqlite3WhereExprUsageNN(WhereMaskSet *pMaskSet, Expr *p){ }else if( p->x.pList ){ mask |= sqlite3WhereExprListUsage(pMaskSet, p->x.pList); } +#ifndef SQLITE_OMIT_WINDOWFUNC + if( p->op==TK_FUNCTION && p->y.pWin ){ + mask |= sqlite3WhereExprListUsage(pMaskSet, p->y.pWin->pPartition); + mask |= sqlite3WhereExprListUsage(pMaskSet, p->y.pWin->pOrderBy); + } +#endif return mask; } SQLITE_PRIVATE Bitmask sqlite3WhereExprUsage(WhereMaskSet *pMaskSet, Expr *p){ @@ -139941,7 +141540,8 @@ static WhereTerm *whereScanNext(WhereScan *pScan){ ){ if( (pTerm->eOperator & WO_EQUIV)!=0 && pScan->nEquivaiCur) - && (pX = sqlite3ExprSkipCollate(pTerm->pExpr->pRight))->op==TK_COLUMN + && (pX = sqlite3ExprSkipCollateAndLikely(pTerm->pExpr->pRight))->op + ==TK_COLUMN ){ int j; for(j=0; jnEquiv; j++){ @@ -140137,7 +141737,7 @@ static int findIndexCol( const char *zColl = pIdx->azColl[iCol]; for(i=0; inExpr; i++){ - Expr *p = sqlite3ExprSkipCollate(pList->a[i].pExpr); + Expr *p = sqlite3ExprSkipCollateAndLikely(pList->a[i].pExpr); if( p->op==TK_COLUMN && p->iColumn==pIdx->aiColumn[iCol] && p->iTable==iBase @@ -140201,7 +141801,7 @@ static int isDistinctRedundant( ** current SELECT is a correlated sub-query. */ for(i=0; inExpr; i++){ - Expr *p = sqlite3ExprSkipCollate(pDistinct->a[i].pExpr); + Expr *p = sqlite3ExprSkipCollateAndLikely(pDistinct->a[i].pExpr); if( p->op==TK_COLUMN && p->iTable==iBase && p->iColumn<0 ) return 1; } @@ -140250,17 +141850,17 @@ static LogEst estLog(LogEst N){ ** opcodes into OP_Copy when the table is being accessed via co-routine ** instead of via table lookup. ** -** If the bIncrRowid parameter is 0, then any OP_Rowid instructions on -** cursor iTabCur are transformed into OP_Null. Or, if bIncrRowid is non-zero, -** then each OP_Rowid is transformed into an instruction to increment the -** value stored in its output register. +** If the iAutoidxCur is not zero, then any OP_Rowid instructions on +** cursor iTabCur are transformed into OP_Sequence opcode for the +** iAutoidxCur cursor, in order to generate unique rowids for the +** automatic index being generated. */ static void translateColumnToCopy( Parse *pParse, /* Parsing context */ int iStart, /* Translate from this opcode to the end */ int iTabCur, /* OP_Column/OP_Rowid references to this table */ int iRegister, /* The first column is in this register */ - int bIncrRowid /* If non-zero, transform OP_rowid to OP_AddImm(1) */ + int iAutoidxCur /* If non-zero, cursor of autoindex being generated */ ){ Vdbe *v = pParse->pVdbe; VdbeOp *pOp = sqlite3VdbeGetOp(v, iStart); @@ -140274,11 +141874,9 @@ static void translateColumnToCopy( pOp->p2 = pOp->p3; pOp->p3 = 0; }else if( pOp->opcode==OP_Rowid ){ - if( bIncrRowid ){ - /* Increment the value stored in the P2 operand of the OP_Rowid. */ - pOp->opcode = OP_AddImm; - pOp->p1 = pOp->p2; - pOp->p2 = 1; + if( iAutoidxCur ){ + pOp->opcode = OP_Sequence; + pOp->p1 = iAutoidxCur; }else{ pOp->opcode = OP_Null; pOp->p1 = 0; @@ -140425,7 +142023,7 @@ static void constructAutomaticIndex( && (pTerm->wtFlags & TERM_VIRTUAL)==0 && !ExprHasProperty(pExpr, EP_FromJoin) && sqlite3ExprIsTableConstant(pExpr, pSrc->iCursor) ){ - pPartial = sqlite3ExprAnd(pParse->db, pPartial, + pPartial = sqlite3ExprAnd(pParse, pPartial, sqlite3ExprDup(pParse->db, pExpr, 0)); } if( termCanDriveIndex(pTerm, pSrc, notReady) ){ @@ -140552,8 +142150,9 @@ static void constructAutomaticIndex( if( pTabItem->fg.viaCoroutine ){ sqlite3VdbeChangeP2(v, addrCounter, regBase+n); testcase( pParse->db->mallocFailed ); + assert( pLevel->iIdxCur>0 ); translateColumnToCopy(pParse, addrTop, pLevel->iTabCur, - pTabItem->regResult, 1); + pTabItem->regResult, pLevel->iIdxCur); sqlite3VdbeGoto(v, addrTop); pTabItem->fg.viaCoroutine = 0; }else{ @@ -140622,6 +142221,7 @@ static sqlite3_index_info *allocateIndexInfo( for(i=0; ia[i].pExpr; if( pExpr->op!=TK_COLUMN || pExpr->iTable!=pSrc->iCursor ) break; + if( pOrderBy->a[i].sortFlags & KEYINFO_ORDER_BIGNULL ) break; } if( i==n){ nOrderBy = n; @@ -140720,7 +142320,7 @@ static sqlite3_index_info *allocateIndexInfo( for(i=0; ia[i].pExpr; pIdxOrderBy[i].iColumn = pExpr->iColumn; - pIdxOrderBy[i].desc = pOrderBy->a[i].sortOrder; + pIdxOrderBy[i].desc = pOrderBy->a[i].sortFlags & KEYINFO_ORDER_DESC; } *pmNoOmit = mNoOmit; @@ -140766,7 +142366,7 @@ static int vtabBestIndex(Parse *pParse, Table *pTab, sqlite3_index_info *p){ } #endif /* !defined(SQLITE_OMIT_VIRTUALTABLE) */ -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 +#ifdef SQLITE_ENABLE_STAT4 /* ** Estimate the location of a particular key among all keys in an ** index. Store the results in aStat as follows: @@ -140959,7 +142559,7 @@ static int whereKeyStats( pRec->nField = nField; return i; } -#endif /* SQLITE_ENABLE_STAT3_OR_STAT4 */ +#endif /* SQLITE_ENABLE_STAT4 */ /* ** If it is not NULL, pTerm is a term that provides an upper or lower @@ -140985,7 +142585,7 @@ static LogEst whereRangeAdjust(WhereTerm *pTerm, LogEst nNew){ } -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 +#ifdef SQLITE_ENABLE_STAT4 /* ** Return the affinity for a single column of an index. */ @@ -140994,12 +142594,13 @@ SQLITE_PRIVATE char sqlite3IndexColumnAffinity(sqlite3 *db, Index *pIdx, int iCo if( !pIdx->zColAff ){ if( sqlite3IndexAffinityStr(db, pIdx)==0 ) return SQLITE_AFF_BLOB; } + assert( pIdx->zColAff[iCol]!=0 ); return pIdx->zColAff[iCol]; } #endif -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 +#ifdef SQLITE_ENABLE_STAT4 /* ** This function is called to estimate the number of rows visited by a ** range-scan on a skip-scan index. For example: @@ -141105,7 +142706,7 @@ static int whereRangeSkipScanEst( return rc; } -#endif /* SQLITE_ENABLE_STAT3_OR_STAT4 */ +#endif /* SQLITE_ENABLE_STAT4 */ /* ** This function is used to estimate the number of rows that will be visited @@ -141158,12 +142759,12 @@ static int whereRangeScanEst( int nOut = pLoop->nOut; LogEst nNew; -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 +#ifdef SQLITE_ENABLE_STAT4 Index *p = pLoop->u.btree.pIndex; int nEq = pLoop->u.btree.nEq; - if( p->nSample>0 && nEqnSampleCol - && OptimizationEnabled(pParse->db, SQLITE_Stat34) + if( p->nSample>0 && ALWAYS(nEqnSampleCol) + && OptimizationEnabled(pParse->db, SQLITE_Stat4) ){ if( nEq==pBuilder->nRecValid ){ UnpackedRecord *pRec = pBuilder->pRec; @@ -141261,7 +142862,7 @@ static int whereRangeScanEst( /* TUNING: If both iUpper and iLower are derived from the same ** sample, then assume they are 4x more selective. This brings ** the estimated selectivity more in line with what it would be - ** if estimated without the use of STAT3/4 tables. */ + ** if estimated without the use of STAT4 tables. */ if( iLwrIdx==iUprIdx ) nNew -= 20; assert( 20==sqlite3LogEst(4) ); }else{ nNew = 10; assert( 10==sqlite3LogEst(2) ); @@ -141310,12 +142911,12 @@ static int whereRangeScanEst( return rc; } -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 +#ifdef SQLITE_ENABLE_STAT4 /* ** Estimate the number of rows that will be returned based on ** an equality constraint x=VALUE and where that VALUE occurs in ** the histogram data. This only works when x is the left-most -** column of an index and sqlite_stat3 histogram data is available +** column of an index and sqlite_stat4 histogram data is available ** for that index. When pExpr==NULL that means the constraint is ** "x IS NULL" instead of "x=VALUE". ** @@ -141373,9 +142974,9 @@ static int whereEqualScanEst( return rc; } -#endif /* SQLITE_ENABLE_STAT3_OR_STAT4 */ +#endif /* SQLITE_ENABLE_STAT4 */ -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 +#ifdef SQLITE_ENABLE_STAT4 /* ** Estimate the number of rows that will be returned based on ** an IN constraint where the right-hand side of the IN operator @@ -141422,7 +143023,7 @@ static int whereInScanEst( assert( pBuilder->nRecValid==nRecValid ); return rc; } -#endif /* SQLITE_ENABLE_STAT3_OR_STAT4 */ +#endif /* SQLITE_ENABLE_STAT4 */ #ifdef WHERETRACE_ENABLED @@ -141954,11 +143555,12 @@ static void whereLoopOutputAdjust( ){ WhereTerm *pTerm, *pX; Bitmask notAllowed = ~(pLoop->prereq|pLoop->maskSelf); - int i, j, k; + int i, j; LogEst iReduce = 0; /* pLoop->nOut should not exceed nRow-iReduce */ assert( (pLoop->wsFlags & WHERE_AUTO_INDEX)==0 ); for(i=pWC->nTerm, pTerm=pWC->a; i>0; i--, pTerm++){ + assert( pTerm!=0 ); if( (pTerm->wtFlags & TERM_VIRTUAL)!=0 ) break; if( (pTerm->prereqAll & pLoop->maskSelf)==0 ) continue; if( (pTerm->prereqAll & notAllowed)!=0 ) continue; @@ -141979,6 +143581,7 @@ static void whereLoopOutputAdjust( pLoop->nOut--; if( pTerm->eOperator&(WO_EQ|WO_IS) ){ Expr *pRight = pTerm->pExpr->pRight; + int k = 0; testcase( pTerm->pExpr->op==TK_IS ); if( sqlite3ExprIsInteger(pRight, &k) && k>=(-1) && k<=1 ){ k = 10; @@ -142142,7 +143745,7 @@ static int whereLoopAddBtreeIndex( LogEst rCostIdx; LogEst nOutUnadjusted; /* nOut before IN() and WHERE adjustments */ int nIn = 0; -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 +#ifdef SQLITE_ENABLE_STAT4 int nRecValid = pBuilder->nRecValid; #endif if( (eOp==WO_ISNULL || (pTerm->wtFlags&TERM_VNULL)!=0) @@ -142203,8 +143806,6 @@ static int whereLoopAddBtreeIndex( }else if( ALWAYS(pExpr->x.pList && pExpr->x.pList->nExpr) ){ /* "x IN (value, value, ...)" */ nIn = sqlite3LogEst(pExpr->x.pList->nExpr); - assert( nIn>0 ); /* RHS always has 2 or more terms... The parser - ** changes "x IN (?)" into "x=?". */ } if( pProbe->hasStat1 ){ LogEst M, logK, safetyMargin; @@ -142300,7 +143901,7 @@ static int whereLoopAddBtreeIndex( ** the value of pNew->nOut to account for pTerm (but not nIn/nInMul). */ assert( pNew->nOut==saved_nOut ); if( pNew->wsFlags & WHERE_COLUMN_RANGE ){ - /* Adjust nOut using stat3/stat4 data. Or, if there is no stat3/stat4 + /* Adjust nOut using stat4 data. Or, if there is no stat4 ** data, using some other estimate. */ whereRangeScanEst(pParse, pBuilder, pBtm, pTop, pNew); }else{ @@ -142314,13 +143915,13 @@ static int whereLoopAddBtreeIndex( pNew->nOut += pTerm->truthProb; pNew->nOut -= nIn; }else{ -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 +#ifdef SQLITE_ENABLE_STAT4 tRowcnt nOut = 0; if( nInMul==0 && pProbe->nSample && pNew->u.btree.nEq<=pProbe->nSampleCol && ((eOp & WO_IN)==0 || !ExprHasProperty(pTerm->pExpr, EP_xIsSelect)) - && OptimizationEnabled(db, SQLITE_Stat34) + && OptimizationEnabled(db, SQLITE_Stat4) ){ Expr *pExpr = pTerm->pExpr; if( (eOp & (WO_EQ|WO_ISNULL|WO_IS))!=0 ){ @@ -142357,6 +143958,7 @@ static int whereLoopAddBtreeIndex( ** it to pNew->rRun, which is currently set to the cost of the index ** seek only. Then, if this is a non-covering index, add the cost of ** visiting the rows in the main table. */ + assert( pSrc->pTab->szTabRow>0 ); rCostIdx = pNew->nOut + 1 + (15*pProbe->szIdxRow)/pSrc->pTab->szTabRow; pNew->rRun = sqlite3LogEstAdd(rLogSize, rCostIdx); if( (pNew->wsFlags & (WHERE_IDX_ONLY|WHERE_IPK))==0 ){ @@ -142382,7 +143984,7 @@ static int whereLoopAddBtreeIndex( whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, nInMul+nIn); } pNew->nOut = saved_nOut; -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 +#ifdef SQLITE_ENABLE_STAT4 pBuilder->nRecValid = nRecValid; #endif } @@ -142455,7 +144057,7 @@ static int indexMightHelpWithOrderBy( if( pIndex->bUnordered ) return 0; if( (pOB = pBuilder->pWInfo->pOrderBy)==0 ) return 0; for(ii=0; iinExpr; ii++){ - Expr *pExpr = sqlite3ExprSkipCollate(pOB->a[ii].pExpr); + Expr *pExpr = sqlite3ExprSkipCollateAndLikely(pOB->a[ii].pExpr); if( pExpr->op==TK_COLUMN && pExpr->iTable==iCursor ){ if( pExpr->iColumn<0 ) return 1; for(jj=0; jjnKeyCol; jj++){ @@ -142486,7 +144088,9 @@ static int whereUsablePartialIndex(int iTab, WhereClause *pWC, Expr *pWhere){ } if( pParse->db->flags & SQLITE_EnableQPSG ) pParse = 0; for(i=0, pTerm=pWC->a; inTerm; i++, pTerm++){ - Expr *pExpr = pTerm->pExpr; + Expr *pExpr; + if( pTerm->wtFlags & TERM_NOPARTIDX ) continue; + pExpr = pTerm->pExpr; if( (!ExprHasProperty(pExpr, EP_FromJoin) || pExpr->iRightJoinTable==iTab) && sqlite3ExprImpliesExpr(pParse, pExpr, pWhere, iTab) ){ @@ -142755,7 +144359,7 @@ static int whereLoopAddBtree( ** plan */ pTab->tabFlags |= TF_StatsUsed; } -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 +#ifdef SQLITE_ENABLE_STAT4 sqlite3Stat4ProbeFree(pBuilder->pRec); pBuilder->nRecValid = 0; pBuilder->pRec = 0; @@ -143023,11 +144627,11 @@ static int whereLoopAddVirtual( rc = whereLoopAddVirtualOne(pBuilder, mPrereq, ALLBITS, 0, p, mNoOmit, &bIn); /* If the call to xBestIndex() with all terms enabled produced a plan - ** that does not require any source tables (IOW: a plan with mBest==0), - ** then there is no point in making any further calls to xBestIndex() - ** since they will all return the same result (if the xBestIndex() - ** implementation is sane). */ - if( rc==SQLITE_OK && (mBest = (pNew->prereq & ~mPrereq))!=0 ){ + ** that does not require any source tables (IOW: a plan with mBest==0) + ** and does not use an IN(...) operator, then there is no point in making + ** any further calls to xBestIndex() since they will all return the same + ** result (if the xBestIndex() implementation is sane). */ + if( rc==SQLITE_OK && ((mBest = (pNew->prereq & ~mPrereq))!=0 || bIn) ){ int seenZero = 0; /* True if a plan with no prereqs seen */ int seenZeroNoIN = 0; /* Plan with no prereqs and no IN(...) seen */ Bitmask mPrev = 0; @@ -143383,8 +144987,8 @@ static i8 wherePathSatisfiesOrderBy( if( pLoop->wsFlags & WHERE_VIRTUALTABLE ){ if( pLoop->u.vtab.isOrdered ) obSat = obDone; break; - }else{ - pLoop->u.btree.nIdxCol = 0; + }else if( wctrlFlags & WHERE_DISTINCTBY ){ + pLoop->u.btree.nDistinctCol = 0; } iCur = pWInfo->pTabList->a[pLoop->iTab].iCursor; @@ -143395,7 +144999,7 @@ static i8 wherePathSatisfiesOrderBy( */ for(i=0; ia[i].pExpr); + pOBExpr = sqlite3ExprSkipCollateAndLikely(pOrderBy->a[i].pExpr); if( pOBExpr->op!=TK_COLUMN ) continue; if( pOBExpr->iTable!=iCur ) continue; pTerm = sqlite3WhereFindTerm(&pWInfo->sWC, iCur, pOBExpr->iColumn, @@ -143432,7 +145036,8 @@ static i8 wherePathSatisfiesOrderBy( assert( nColumn==nKeyCol+1 || !HasRowid(pIndex->pTable) ); assert( pIndex->aiColumn[nColumn-1]==XN_ROWID || !HasRowid(pIndex->pTable)); - isOrderDistinct = IsUniqueIndex(pIndex); + isOrderDistinct = IsUniqueIndex(pIndex) + && (pLoop->wsFlags & WHERE_SKIPSCAN)==0; } /* Loop through all columns of the index and deal with the ones @@ -143450,15 +145055,21 @@ static i8 wherePathSatisfiesOrderBy( u16 eOp = pLoop->aLTerm[j]->eOperator; /* Skip over == and IS and ISNULL terms. (Also skip IN terms when - ** doing WHERE_ORDERBY_LIMIT processing). + ** doing WHERE_ORDERBY_LIMIT processing). Except, IS and ISNULL + ** terms imply that the index is not UNIQUE NOT NULL in which case + ** the loop need to be marked as not order-distinct because it can + ** have repeated NULL rows. ** ** If the current term is a column of an ((?,?) IN (SELECT...)) ** expression for which the SELECT returns more than one column, ** check that it is the only column used by this loop. Otherwise, ** if it is one of two or more, none of the columns can be - ** considered to match an ORDER BY term. */ + ** considered to match an ORDER BY term. + */ if( (eOp & eqOpMask)!=0 ){ - if( eOp & WO_ISNULL ){ + if( eOp & (WO_ISNULL|WO_IS) ){ + testcase( eOp & WO_ISNULL ); + testcase( eOp & WO_IS ); testcase( isOrderDistinct ); isOrderDistinct = 0; } @@ -143484,7 +145095,7 @@ static i8 wherePathSatisfiesOrderBy( */ if( pIndex ){ iColumn = pIndex->aiColumn[j]; - revIdx = pIndex->aSortOrder[j]; + revIdx = pIndex->aSortOrder[j] & KEYINFO_ORDER_DESC; if( iColumn==pIndex->pTable->iPKey ) iColumn = XN_ROWID; }else{ iColumn = XN_ROWID; @@ -143508,7 +145119,7 @@ static i8 wherePathSatisfiesOrderBy( isMatch = 0; for(i=0; bOnce && ia[i].pExpr); + pOBExpr = sqlite3ExprSkipCollateAndLikely(pOrderBy->a[i].pExpr); testcase( wctrlFlags & WHERE_GROUPBY ); testcase( wctrlFlags & WHERE_DISTINCTBY ); if( (wctrlFlags & (WHERE_GROUPBY|WHERE_DISTINCTBY))==0 ) bOnce = 0; @@ -143526,7 +145137,9 @@ static i8 wherePathSatisfiesOrderBy( pColl = sqlite3ExprNNCollSeq(pWInfo->pParse, pOrderBy->a[i].pExpr); if( sqlite3StrICmp(pColl->zName, pIndex->azColl[j])!=0 ) continue; } - pLoop->u.btree.nIdxCol = j+1; + if( wctrlFlags & WHERE_DISTINCTBY ){ + pLoop->u.btree.nDistinctCol = j+1; + } isMatch = 1; break; } @@ -143534,13 +145147,22 @@ static i8 wherePathSatisfiesOrderBy( /* Make sure the sort order is compatible in an ORDER BY clause. ** Sort order is irrelevant for a GROUP BY clause. */ if( revSet ){ - if( (rev ^ revIdx)!=pOrderBy->a[i].sortOrder ) isMatch = 0; + if( (rev ^ revIdx)!=(pOrderBy->a[i].sortFlags&KEYINFO_ORDER_DESC) ){ + isMatch = 0; + } }else{ - rev = revIdx ^ pOrderBy->a[i].sortOrder; + rev = revIdx ^ (pOrderBy->a[i].sortFlags & KEYINFO_ORDER_DESC); if( rev ) *pRevMask |= MASKBIT(iLoop); revSet = 1; } } + if( isMatch && (pOrderBy->a[i].sortFlags & KEYINFO_ORDER_BIGNULL) ){ + if( j==pLoop->u.btree.nEq ){ + pLoop->wsFlags |= WHERE_BIGNULL_SORT; + }else{ + isMatch = 0; + } + } if( isMatch ){ if( iColumn==XN_ROWID ){ testcase( distinctColumns==0 ); @@ -144454,6 +146076,16 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( sqlite3DebugPrintf(", limit: %d", iAuxArg); } sqlite3DebugPrintf(")\n"); + if( sqlite3WhereTrace & 0x100 ){ + Select sSelect; + memset(&sSelect, 0, sizeof(sSelect)); + sSelect.selFlags = SF_WhereBegin; + sSelect.pSrc = pTabList; + sSelect.pWhere = pWhere; + sSelect.pOrderBy = pOrderBy; + sSelect.pEList = pResultSet; + sqlite3TreeViewSelect(0, &sSelect, 0); + } } if( sqlite3WhereTrace & 0x100 ){ /* Display all terms of the WHERE clause */ sqlite3WhereClausePrint(sWLB.pWC); @@ -144730,6 +146362,7 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( sqlite3VdbeSetP4KeyInfo(pParse, pIx); if( (pLoop->wsFlags & WHERE_CONSTRAINT)!=0 && (pLoop->wsFlags & (WHERE_COLUMN_RANGE|WHERE_SKIPSCAN))==0 + && (pLoop->wsFlags & WHERE_BIGNULL_SORT)==0 && (pWInfo->wctrlFlags&WHERE_ORDERBY_MIN)==0 && pWInfo->eDistinct!=WHERE_DISTINCT_ORDERED ){ @@ -144847,7 +146480,7 @@ SQLITE_PRIVATE void sqlite3WhereEnd(WhereInfo *pWInfo){ && i==pWInfo->nLevel-1 /* Ticket [ef9318757b152e3] 2017-10-21 */ && (pLoop->wsFlags & WHERE_INDEXED)!=0 && (pIdx = pLoop->u.btree.pIndex)->hasStat1 - && (n = pLoop->u.btree.nIdxCol)>0 + && (n = pLoop->u.btree.nDistinctCol)>0 && pIdx->aiRowLogEst[n]>=36 ){ int r1 = pParse->nMem+1; @@ -144871,6 +146504,11 @@ SQLITE_PRIVATE void sqlite3WhereEnd(WhereInfo *pWInfo){ VdbeCoverageIf(v, pLevel->op==OP_Next); VdbeCoverageIf(v, pLevel->op==OP_Prev); VdbeCoverageIf(v, pLevel->op==OP_VNext); + if( pLevel->regBignull ){ + sqlite3VdbeResolveLabel(v, pLevel->addrBignull); + sqlite3VdbeAddOp2(v, OP_DecrJumpZero, pLevel->regBignull, pLevel->p2-1); + VdbeCoverage(v); + } #ifndef SQLITE_DISABLE_SKIPAHEAD_DISTINCT if( addrSeek ) sqlite3VdbeJumpHere(v, addrSeek); #endif @@ -145260,6 +146898,96 @@ static void dense_rankValueFunc(sqlite3_context *pCtx){ } } +/* +** Implementation of built-in window function nth_value(). This +** implementation is used in "slow mode" only - when the EXCLUDE clause +** is not set to the default value "NO OTHERS". +*/ +struct NthValueCtx { + i64 nStep; + sqlite3_value *pValue; +}; +static void nth_valueStepFunc( + sqlite3_context *pCtx, + int nArg, + sqlite3_value **apArg +){ + struct NthValueCtx *p; + p = (struct NthValueCtx*)sqlite3_aggregate_context(pCtx, sizeof(*p)); + if( p ){ + i64 iVal; + switch( sqlite3_value_numeric_type(apArg[1]) ){ + case SQLITE_INTEGER: + iVal = sqlite3_value_int64(apArg[1]); + break; + case SQLITE_FLOAT: { + double fVal = sqlite3_value_double(apArg[1]); + if( ((i64)fVal)!=fVal ) goto error_out; + iVal = (i64)fVal; + break; + } + default: + goto error_out; + } + if( iVal<=0 ) goto error_out; + + p->nStep++; + if( iVal==p->nStep ){ + p->pValue = sqlite3_value_dup(apArg[0]); + if( !p->pValue ){ + sqlite3_result_error_nomem(pCtx); + } + } + } + UNUSED_PARAMETER(nArg); + UNUSED_PARAMETER(apArg); + return; + + error_out: + sqlite3_result_error( + pCtx, "second argument to nth_value must be a positive integer", -1 + ); +} +static void nth_valueFinalizeFunc(sqlite3_context *pCtx){ + struct NthValueCtx *p; + p = (struct NthValueCtx*)sqlite3_aggregate_context(pCtx, 0); + if( p && p->pValue ){ + sqlite3_result_value(pCtx, p->pValue); + sqlite3_value_free(p->pValue); + p->pValue = 0; + } +} +#define nth_valueInvFunc noopStepFunc +#define nth_valueValueFunc noopValueFunc + +static void first_valueStepFunc( + sqlite3_context *pCtx, + int nArg, + sqlite3_value **apArg +){ + struct NthValueCtx *p; + p = (struct NthValueCtx*)sqlite3_aggregate_context(pCtx, sizeof(*p)); + if( p && p->pValue==0 ){ + p->pValue = sqlite3_value_dup(apArg[0]); + if( !p->pValue ){ + sqlite3_result_error_nomem(pCtx); + } + } + UNUSED_PARAMETER(nArg); + UNUSED_PARAMETER(apArg); +} +static void first_valueFinalizeFunc(sqlite3_context *pCtx){ + struct NthValueCtx *p; + p = (struct NthValueCtx*)sqlite3_aggregate_context(pCtx, sizeof(*p)); + if( p && p->pValue ){ + sqlite3_result_value(pCtx, p->pValue); + sqlite3_value_free(p->pValue); + p->pValue = 0; + } +} +#define first_valueInvFunc noopStepFunc +#define first_valueValueFunc noopValueFunc + /* ** Implementation of built-in window function rank(). Assumes that ** the window frame has been set to: @@ -145295,7 +147023,7 @@ static void rankValueFunc(sqlite3_context *pCtx){ ** Implementation of built-in window function percent_rank(). Assumes that ** the window frame has been set to: ** -** RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW +** GROUPS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING */ static void percent_rankStepFunc( sqlite3_context *pCtx, @@ -145303,38 +147031,44 @@ static void percent_rankStepFunc( sqlite3_value **apArg ){ struct CallCount *p; - UNUSED_PARAMETER(nArg); assert( nArg==1 ); - + UNUSED_PARAMETER(nArg); assert( nArg==0 ); + UNUSED_PARAMETER(apArg); p = (struct CallCount*)sqlite3_aggregate_context(pCtx, sizeof(*p)); if( p ){ - if( p->nTotal==0 ){ - p->nTotal = sqlite3_value_int64(apArg[0]); - } - p->nStep++; - if( p->nValue==0 ){ - p->nValue = p->nStep; - } + p->nTotal++; } } +static void percent_rankInvFunc( + sqlite3_context *pCtx, + int nArg, + sqlite3_value **apArg +){ + struct CallCount *p; + UNUSED_PARAMETER(nArg); assert( nArg==0 ); + UNUSED_PARAMETER(apArg); + p = (struct CallCount*)sqlite3_aggregate_context(pCtx, sizeof(*p)); + p->nStep++; +} static void percent_rankValueFunc(sqlite3_context *pCtx){ struct CallCount *p; p = (struct CallCount*)sqlite3_aggregate_context(pCtx, sizeof(*p)); if( p ){ + p->nValue = p->nStep; if( p->nTotal>1 ){ - double r = (double)(p->nValue-1) / (double)(p->nTotal-1); + double r = (double)p->nValue / (double)(p->nTotal-1); sqlite3_result_double(pCtx, r); }else{ sqlite3_result_double(pCtx, 0.0); } - p->nValue = 0; } } +#define percent_rankFinalizeFunc percent_rankValueFunc /* ** Implementation of built-in window function cume_dist(). Assumes that ** the window frame has been set to: ** -** RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW +** GROUPS BETWEEN 1 FOLLOWING AND UNBOUNDED FOLLOWING */ static void cume_distStepFunc( sqlite3_context *pCtx, @@ -145342,24 +147076,33 @@ static void cume_distStepFunc( sqlite3_value **apArg ){ struct CallCount *p; - assert( nArg==1 ); UNUSED_PARAMETER(nArg); - + UNUSED_PARAMETER(nArg); assert( nArg==0 ); + UNUSED_PARAMETER(apArg); p = (struct CallCount*)sqlite3_aggregate_context(pCtx, sizeof(*p)); if( p ){ - if( p->nTotal==0 ){ - p->nTotal = sqlite3_value_int64(apArg[0]); - } - p->nStep++; + p->nTotal++; } } -static void cume_distValueFunc(sqlite3_context *pCtx){ +static void cume_distInvFunc( + sqlite3_context *pCtx, + int nArg, + sqlite3_value **apArg +){ struct CallCount *p; + UNUSED_PARAMETER(nArg); assert( nArg==0 ); + UNUSED_PARAMETER(apArg); p = (struct CallCount*)sqlite3_aggregate_context(pCtx, sizeof(*p)); - if( p && p->nTotal ){ + p->nStep++; +} +static void cume_distValueFunc(sqlite3_context *pCtx){ + struct CallCount *p; + p = (struct CallCount*)sqlite3_aggregate_context(pCtx, 0); + if( p ){ double r = (double)(p->nStep) / (double)(p->nTotal); sqlite3_result_double(pCtx, r); } } +#define cume_distFinalizeFunc cume_distValueFunc /* ** Context object for ntile() window function. @@ -145374,7 +147117,7 @@ struct NtileCtx { ** Implementation of ntile(). This assumes that the window frame has ** been coerced to: ** -** ROWS UNBOUNDED PRECEDING AND CURRENT ROW +** ROWS CURRENT ROW AND UNBOUNDED FOLLOWING */ static void ntileStepFunc( sqlite3_context *pCtx, @@ -145382,32 +147125,42 @@ static void ntileStepFunc( sqlite3_value **apArg ){ struct NtileCtx *p; - assert( nArg==2 ); UNUSED_PARAMETER(nArg); + assert( nArg==1 ); UNUSED_PARAMETER(nArg); p = (struct NtileCtx*)sqlite3_aggregate_context(pCtx, sizeof(*p)); if( p ){ if( p->nTotal==0 ){ p->nParam = sqlite3_value_int64(apArg[0]); - p->nTotal = sqlite3_value_int64(apArg[1]); if( p->nParam<=0 ){ sqlite3_result_error( pCtx, "argument of ntile must be a positive integer", -1 ); } } - p->iRow++; + p->nTotal++; } } +static void ntileInvFunc( + sqlite3_context *pCtx, + int nArg, + sqlite3_value **apArg +){ + struct NtileCtx *p; + assert( nArg==1 ); UNUSED_PARAMETER(nArg); + UNUSED_PARAMETER(apArg); + p = (struct NtileCtx*)sqlite3_aggregate_context(pCtx, sizeof(*p)); + p->iRow++; +} static void ntileValueFunc(sqlite3_context *pCtx){ struct NtileCtx *p; p = (struct NtileCtx*)sqlite3_aggregate_context(pCtx, sizeof(*p)); if( p && p->nParam>0 ){ int nSize = (p->nTotal / p->nParam); if( nSize==0 ){ - sqlite3_result_int64(pCtx, p->iRow); + sqlite3_result_int64(pCtx, p->iRow+1); }else{ i64 nLarge = p->nTotal - p->nParam*nSize; i64 iSmall = nLarge*(nSize+1); - i64 iRow = p->iRow-1; + i64 iRow = p->iRow; assert( (nLarge*(nSize+1) + (p->nParam-nLarge)*nSize)==p->nTotal ); @@ -145419,6 +147172,7 @@ static void ntileValueFunc(sqlite3_context *pCtx){ } } } +#define ntileFinalizeFunc ntileValueFunc /* ** Context object for last_value() window function. @@ -145468,7 +147222,7 @@ static void last_valueInvFunc( } static void last_valueValueFunc(sqlite3_context *pCtx){ struct LastValueCtx *p; - p = (struct LastValueCtx*)sqlite3_aggregate_context(pCtx, sizeof(*p)); + p = (struct LastValueCtx*)sqlite3_aggregate_context(pCtx, 0); if( p && p->pVal ){ sqlite3_result_value(pCtx, p->pVal); } @@ -145558,12 +147312,12 @@ SQLITE_PRIVATE void sqlite3WindowFunctions(void){ WINDOWFUNCX(row_number, 0, 0), WINDOWFUNCX(dense_rank, 0, 0), WINDOWFUNCX(rank, 0, 0), - WINDOWFUNCX(percent_rank, 0, SQLITE_FUNC_WINDOW_SIZE), - WINDOWFUNCX(cume_dist, 0, SQLITE_FUNC_WINDOW_SIZE), - WINDOWFUNCX(ntile, 1, SQLITE_FUNC_WINDOW_SIZE), + WINDOWFUNCALL(percent_rank, 0, 0), + WINDOWFUNCALL(cume_dist, 0, 0), + WINDOWFUNCALL(ntile, 1, 0), WINDOWFUNCALL(last_value, 1, 0), - WINDOWFUNCNOOP(nth_value, 2, 0), - WINDOWFUNCNOOP(first_value, 1, 0), + WINDOWFUNCALL(nth_value, 2, 0), + WINDOWFUNCALL(first_value, 1, 0), WINDOWFUNCNOOP(lead, 1, 0), WINDOWFUNCNOOP(lead, 2, 0), WINDOWFUNCNOOP(lead, 3, 0), @@ -145574,6 +147328,17 @@ SQLITE_PRIVATE void sqlite3WindowFunctions(void){ sqlite3InsertBuiltinFuncs(aWindowFuncs, ArraySize(aWindowFuncs)); } +static Window *windowFind(Parse *pParse, Window *pList, const char *zName){ + Window *p; + for(p=pList; p; p=p->pNextWin){ + if( sqlite3StrICmp(p->zName, zName)==0 ) break; + } + if( p==0 ){ + sqlite3ErrorMsg(pParse, "no such window: %s", zName); + } + return p; +} + /* ** This function is called immediately after resolving the function name ** for a window function within a SELECT statement. Argument pList is a @@ -145597,48 +147362,66 @@ SQLITE_PRIVATE void sqlite3WindowUpdate( Window *pWin, /* Window frame to update */ FuncDef *pFunc /* Window function definition */ ){ - if( pWin->zName && pWin->eType==0 ){ - Window *p; - for(p=pList; p; p=p->pNextWin){ - if( sqlite3StrICmp(p->zName, pWin->zName)==0 ) break; - } - if( p==0 ){ - sqlite3ErrorMsg(pParse, "no such window: %s", pWin->zName); - return; - } + if( pWin->zName && pWin->eFrmType==0 ){ + Window *p = windowFind(pParse, pList, pWin->zName); + if( p==0 ) return; pWin->pPartition = sqlite3ExprListDup(pParse->db, p->pPartition, 0); pWin->pOrderBy = sqlite3ExprListDup(pParse->db, p->pOrderBy, 0); pWin->pStart = sqlite3ExprDup(pParse->db, p->pStart, 0); pWin->pEnd = sqlite3ExprDup(pParse->db, p->pEnd, 0); pWin->eStart = p->eStart; pWin->eEnd = p->eEnd; - pWin->eType = p->eType; + pWin->eFrmType = p->eFrmType; + pWin->eExclude = p->eExclude; + }else{ + sqlite3WindowChain(pParse, pWin, pList); } + if( (pWin->eFrmType==TK_RANGE) + && (pWin->pStart || pWin->pEnd) + && (pWin->pOrderBy==0 || pWin->pOrderBy->nExpr!=1) + ){ + sqlite3ErrorMsg(pParse, + "RANGE with offset PRECEDING/FOLLOWING requires one ORDER BY expression" + ); + }else if( pFunc->funcFlags & SQLITE_FUNC_WINDOW ){ sqlite3 *db = pParse->db; if( pWin->pFilter ){ sqlite3ErrorMsg(pParse, "FILTER clause may only be used with aggregate window functions" ); - }else - if( pFunc->zName==row_numberName || pFunc->zName==ntileName ){ - sqlite3ExprDelete(db, pWin->pStart); - sqlite3ExprDelete(db, pWin->pEnd); - pWin->pStart = pWin->pEnd = 0; - pWin->eType = TK_ROWS; - pWin->eStart = TK_UNBOUNDED; - pWin->eEnd = TK_CURRENT; - }else - - if( pFunc->zName==dense_rankName || pFunc->zName==rankName - || pFunc->zName==percent_rankName || pFunc->zName==cume_distName - ){ - sqlite3ExprDelete(db, pWin->pStart); - sqlite3ExprDelete(db, pWin->pEnd); - pWin->pStart = pWin->pEnd = 0; - pWin->eType = TK_RANGE; - pWin->eStart = TK_UNBOUNDED; - pWin->eEnd = TK_CURRENT; + }else{ + struct WindowUpdate { + const char *zFunc; + int eFrmType; + int eStart; + int eEnd; + } aUp[] = { + { row_numberName, TK_ROWS, TK_UNBOUNDED, TK_CURRENT }, + { dense_rankName, TK_RANGE, TK_UNBOUNDED, TK_CURRENT }, + { rankName, TK_RANGE, TK_UNBOUNDED, TK_CURRENT }, + { percent_rankName, TK_GROUPS, TK_CURRENT, TK_UNBOUNDED }, + { cume_distName, TK_GROUPS, TK_FOLLOWING, TK_UNBOUNDED }, + { ntileName, TK_ROWS, TK_CURRENT, TK_UNBOUNDED }, + { leadName, TK_ROWS, TK_UNBOUNDED, TK_UNBOUNDED }, + { lagName, TK_ROWS, TK_UNBOUNDED, TK_CURRENT }, + }; + int i; + for(i=0; izName==aUp[i].zFunc ){ + sqlite3ExprDelete(db, pWin->pStart); + sqlite3ExprDelete(db, pWin->pEnd); + pWin->pEnd = pWin->pStart = 0; + pWin->eFrmType = aUp[i].eFrmType; + pWin->eStart = aUp[i].eStart; + pWin->eEnd = aUp[i].eEnd; + pWin->eExclude = 0; + if( pWin->eStart==TK_FOLLOWING ){ + pWin->pStart = sqlite3Expr(db, TK_INTEGER, "1"); + } + break; + } + } } } pWin->pFunc = pFunc; @@ -145653,6 +147436,7 @@ struct WindowRewrite { Window *pWin; SrcList *pSrc; ExprList *pSub; + Table *pTab; Select *pSubSelect; /* Current sub-select, if any */ }; @@ -145664,6 +147448,8 @@ struct WindowRewrite { static int selectWindowRewriteExprCb(Walker *pWalker, Expr *pExpr){ struct WindowRewrite *p = pWalker->u.pRewrite; Parse *pParse = pWalker->pParse; + assert( p!=0 ); + assert( p->pWin!=0 ); /* If this function is being called from within a scalar sub-select ** that used by the SELECT statement being processed, only process @@ -145713,6 +147499,7 @@ static int selectWindowRewriteExprCb(Walker *pWalker, Expr *pExpr){ pExpr->op = TK_COLUMN; pExpr->iColumn = p->pSub->nExpr-1; pExpr->iTable = p->pWin->iEphCsr; + pExpr->y.pTab = p->pTab; } break; @@ -145756,17 +147543,20 @@ static void selectWindowRewriteEList( Window *pWin, SrcList *pSrc, ExprList *pEList, /* Rewrite expressions in this list */ + Table *pTab, ExprList **ppSub /* IN/OUT: Sub-select expression-list */ ){ Walker sWalker; WindowRewrite sRewrite; + assert( pWin!=0 ); memset(&sWalker, 0, sizeof(Walker)); memset(&sRewrite, 0, sizeof(WindowRewrite)); sRewrite.pSub = *ppSub; sRewrite.pWin = pWin; sRewrite.pSrc = pSrc; + sRewrite.pTab = pTab; sWalker.pParse = pParse; sWalker.xExprCallback = selectWindowRewriteExprCb; @@ -145785,15 +147575,20 @@ static void selectWindowRewriteEList( static ExprList *exprListAppendList( Parse *pParse, /* Parsing context */ ExprList *pList, /* List to which to append. Might be NULL */ - ExprList *pAppend /* List of values to append. Might be NULL */ + ExprList *pAppend, /* List of values to append. Might be NULL */ + int bIntToNull ){ if( pAppend ){ int i; int nInit = pList ? pList->nExpr : 0; for(i=0; inExpr; i++){ Expr *pDup = sqlite3ExprDup(pParse->db, pAppend->a[i].pExpr, 0); + if( bIntToNull && pDup && pDup->op==TK_INTEGER ){ + pDup->op = TK_NULL; + pDup->flags &= ~(EP_IntValue|EP_IsTrue|EP_IsFalse); + } pList = sqlite3ExprListAppend(pParse, pList, pDup); - if( pList ) pList->a[nInit+i].sortOrder = pAppend->a[i].sortOrder; + if( pList ) pList->a[nInit+i].sortFlags = pAppend->a[i].sortFlags; } } return pList; @@ -145821,46 +147616,64 @@ SQLITE_PRIVATE int sqlite3WindowRewrite(Parse *pParse, Select *p){ ExprList *pSublist = 0; /* Expression list for sub-query */ Window *pMWin = p->pWin; /* Master window object */ Window *pWin; /* Window object iterator */ + Table *pTab; + + pTab = sqlite3DbMallocZero(db, sizeof(Table)); + if( pTab==0 ){ + return SQLITE_NOMEM; + } p->pSrc = 0; p->pWhere = 0; p->pGroupBy = 0; p->pHaving = 0; + p->selFlags &= ~SF_Aggregate; /* Create the ORDER BY clause for the sub-select. This is the concatenation ** of the window PARTITION and ORDER BY clauses. Then, if this makes it ** redundant, remove the ORDER BY from the parent SELECT. */ pSort = sqlite3ExprListDup(db, pMWin->pPartition, 0); - pSort = exprListAppendList(pParse, pSort, pMWin->pOrderBy); - if( pSort && p->pOrderBy ){ + pSort = exprListAppendList(pParse, pSort, pMWin->pOrderBy, 1); + if( pSort && p->pOrderBy && p->pOrderBy->nExpr<=pSort->nExpr ){ + int nSave = pSort->nExpr; + pSort->nExpr = p->pOrderBy->nExpr; if( sqlite3ExprListCompare(pSort, p->pOrderBy, -1)==0 ){ sqlite3ExprListDelete(db, p->pOrderBy); p->pOrderBy = 0; } + pSort->nExpr = nSave; } /* Assign a cursor number for the ephemeral table used to buffer rows. ** The OpenEphemeral instruction is coded later, after it is known how ** many columns the table will have. */ pMWin->iEphCsr = pParse->nTab++; + pParse->nTab += 3; - selectWindowRewriteEList(pParse, pMWin, pSrc, p->pEList, &pSublist); - selectWindowRewriteEList(pParse, pMWin, pSrc, p->pOrderBy, &pSublist); + selectWindowRewriteEList(pParse, pMWin, pSrc, p->pEList, pTab, &pSublist); + selectWindowRewriteEList(pParse, pMWin, pSrc, p->pOrderBy, pTab, &pSublist); pMWin->nBufferCol = (pSublist ? pSublist->nExpr : 0); /* Append the PARTITION BY and ORDER BY expressions to the to the ** sub-select expression list. They are required to figure out where ** boundaries for partitions and sets of peer rows lie. */ - pSublist = exprListAppendList(pParse, pSublist, pMWin->pPartition); - pSublist = exprListAppendList(pParse, pSublist, pMWin->pOrderBy); + pSublist = exprListAppendList(pParse, pSublist, pMWin->pPartition, 0); + pSublist = exprListAppendList(pParse, pSublist, pMWin->pOrderBy, 0); /* Append the arguments passed to each window function to the ** sub-select expression list. Also allocate two registers for each ** window function - one for the accumulator, another for interim ** results. */ for(pWin=pMWin; pWin; pWin=pWin->pNextWin){ - pWin->iArgCol = (pSublist ? pSublist->nExpr : 0); - pSublist = exprListAppendList(pParse, pSublist, pWin->pOwner->x.pList); + ExprList *pArgs = pWin->pOwner->x.pList; + if( pWin->pFunc->funcFlags & SQLITE_FUNC_SUBTYPE ){ + selectWindowRewriteEList(pParse, pMWin, pSrc, pArgs, pTab, &pSublist); + pWin->iArgCol = (pSublist ? pSublist->nExpr : 0); + pWin->bExprArgs = 1; + }else{ + pWin->iArgCol = (pSublist ? pSublist->nExpr : 0); + pSublist = exprListAppendList(pParse, pSublist, pArgs, 0); + } if( pWin->pFilter ){ Expr *pFilter = sqlite3ExprDup(db, pWin->pFilter, 0); pSublist = sqlite3ExprListAppend(pParse, pSublist, pFilter); @@ -145878,7 +147691,7 @@ SQLITE_PRIVATE int sqlite3WindowRewrite(Parse *pParse, Select *p){ */ if( pSublist==0 ){ pSublist = sqlite3ExprListAppend(pParse, 0, - sqlite3ExprAlloc(db, TK_INTEGER, &sqlite3IntTokens[0], 0) + sqlite3Expr(db, TK_INTEGER, "0") ); } @@ -145887,37 +147700,58 @@ SQLITE_PRIVATE int sqlite3WindowRewrite(Parse *pParse, Select *p){ ); p->pSrc = sqlite3SrcListAppend(pParse, 0, 0, 0); if( p->pSrc ){ + Table *pTab2; p->pSrc->a[0].pSelect = pSub; sqlite3SrcListAssignCursors(pParse, p->pSrc); - if( sqlite3ExpandSubquery(pParse, &p->pSrc->a[0]) ){ + pSub->selFlags |= SF_Expanded; + pTab2 = sqlite3ResultSetOfSelect(pParse, pSub, SQLITE_AFF_NONE); + if( pTab2==0 ){ rc = SQLITE_NOMEM; }else{ - pSub->selFlags |= SF_Expanded; - p->selFlags &= ~SF_Aggregate; - sqlite3SelectPrep(pParse, pSub, 0); + memcpy(pTab, pTab2, sizeof(Table)); + pTab->tabFlags |= TF_Ephemeral; + p->pSrc->a[0].pTab = pTab; + pTab = pTab2; } - sqlite3VdbeAddOp2(v, OP_OpenEphemeral, pMWin->iEphCsr, pSublist->nExpr); + sqlite3VdbeAddOp2(v, OP_OpenDup, pMWin->iEphCsr+1, pMWin->iEphCsr); + sqlite3VdbeAddOp2(v, OP_OpenDup, pMWin->iEphCsr+2, pMWin->iEphCsr); + sqlite3VdbeAddOp2(v, OP_OpenDup, pMWin->iEphCsr+3, pMWin->iEphCsr); }else{ sqlite3SelectDelete(db, pSub); } if( db->mallocFailed ) rc = SQLITE_NOMEM; + sqlite3DbFree(db, pTab); } return rc; } +/* +** Unlink the Window object from the Select to which it is attached, +** if it is attached. +*/ +SQLITE_PRIVATE void sqlite3WindowUnlinkFromSelect(Window *p){ + if( p->ppThis ){ + *p->ppThis = p->pNextWin; + if( p->pNextWin ) p->pNextWin->ppThis = p->ppThis; + p->ppThis = 0; + } +} + /* ** Free the Window object passed as the second argument. */ SQLITE_PRIVATE void sqlite3WindowDelete(sqlite3 *db, Window *p){ if( p ){ + sqlite3WindowUnlinkFromSelect(p); sqlite3ExprDelete(db, p->pFilter); sqlite3ExprListDelete(db, p->pPartition); sqlite3ExprListDelete(db, p->pOrderBy); sqlite3ExprDelete(db, p->pEnd); sqlite3ExprDelete(db, p->pStart); sqlite3DbFree(db, p->zName); + sqlite3DbFree(db, p->zBase); sqlite3DbFree(db, p); } } @@ -145954,16 +147788,18 @@ static Expr *sqlite3WindowOffsetExpr(Parse *pParse, Expr *pExpr){ */ SQLITE_PRIVATE Window *sqlite3WindowAlloc( Parse *pParse, /* Parsing context */ - int eType, /* Frame type. TK_RANGE or TK_ROWS */ + int eType, /* Frame type. TK_RANGE, TK_ROWS, TK_GROUPS, or 0 */ int eStart, /* Start type: CURRENT, PRECEDING, FOLLOWING, UNBOUNDED */ Expr *pStart, /* Start window size if TK_PRECEDING or FOLLOWING */ int eEnd, /* End type: CURRENT, FOLLOWING, TK_UNBOUNDED, PRECEDING */ - Expr *pEnd /* End window size if TK_FOLLOWING or PRECEDING */ + Expr *pEnd, /* End window size if TK_FOLLOWING or PRECEDING */ + u8 eExclude /* EXCLUDE clause */ ){ Window *pWin = 0; + int bImplicitFrame = 0; /* Parser assures the following: */ - assert( eType==TK_RANGE || eType==TK_ROWS ); + assert( eType==0 || eType==TK_RANGE || eType==TK_ROWS || eType==TK_GROUPS ); assert( eStart==TK_CURRENT || eStart==TK_PRECEDING || eStart==TK_UNBOUNDED || eStart==TK_FOLLOWING ); assert( eEnd==TK_CURRENT || eEnd==TK_FOLLOWING @@ -145971,13 +147807,9 @@ SQLITE_PRIVATE Window *sqlite3WindowAlloc( assert( (eStart==TK_PRECEDING || eStart==TK_FOLLOWING)==(pStart!=0) ); assert( (eEnd==TK_FOLLOWING || eEnd==TK_PRECEDING)==(pEnd!=0) ); - - /* If a frame is declared "RANGE" (not "ROWS"), then it may not use - ** either " PRECEDING" or " FOLLOWING". - */ - if( eType==TK_RANGE && (pStart!=0 || pEnd!=0) ){ - sqlite3ErrorMsg(pParse, "RANGE must use only UNBOUNDED or CURRENT ROW"); - goto windowAllocErr; + if( eType==0 ){ + bImplicitFrame = 1; + eType = TK_RANGE; } /* Additionally, the @@ -145997,15 +147829,20 @@ SQLITE_PRIVATE Window *sqlite3WindowAlloc( if( (eStart==TK_CURRENT && eEnd==TK_PRECEDING) || (eStart==TK_FOLLOWING && (eEnd==TK_PRECEDING || eEnd==TK_CURRENT)) ){ - sqlite3ErrorMsg(pParse, "unsupported frame delimiter for ROWS"); + sqlite3ErrorMsg(pParse, "unsupported frame specification"); goto windowAllocErr; } pWin = (Window*)sqlite3DbMallocZero(pParse->db, sizeof(Window)); if( pWin==0 ) goto windowAllocErr; - pWin->eType = eType; + pWin->eFrmType = eType; pWin->eStart = eStart; pWin->eEnd = eEnd; + if( eExclude==0 && OptimizationDisabled(pParse->db, SQLITE_WindowFunc) ){ + eExclude = TK_NO; + } + pWin->eExclude = eExclude; + pWin->bImplicitFrame = bImplicitFrame; pWin->pEnd = sqlite3WindowOffsetExpr(pParse, pEnd); pWin->pStart = sqlite3WindowOffsetExpr(pParse, pStart); return pWin; @@ -146016,41 +147853,124 @@ SQLITE_PRIVATE Window *sqlite3WindowAlloc( return 0; } +/* +** Attach PARTITION and ORDER BY clauses pPartition and pOrderBy to window +** pWin. Also, if parameter pBase is not NULL, set pWin->zBase to the +** equivalent nul-terminated string. +*/ +SQLITE_PRIVATE Window *sqlite3WindowAssemble( + Parse *pParse, + Window *pWin, + ExprList *pPartition, + ExprList *pOrderBy, + Token *pBase +){ + if( pWin ){ + pWin->pPartition = pPartition; + pWin->pOrderBy = pOrderBy; + if( pBase ){ + pWin->zBase = sqlite3DbStrNDup(pParse->db, pBase->z, pBase->n); + } + }else{ + sqlite3ExprListDelete(pParse->db, pPartition); + sqlite3ExprListDelete(pParse->db, pOrderBy); + } + return pWin; +} + +/* +** Window *pWin has just been created from a WINDOW clause. Tokne pBase +** is the base window. Earlier windows from the same WINDOW clause are +** stored in the linked list starting at pWin->pNextWin. This function +** either updates *pWin according to the base specification, or else +** leaves an error in pParse. +*/ +SQLITE_PRIVATE void sqlite3WindowChain(Parse *pParse, Window *pWin, Window *pList){ + if( pWin->zBase ){ + sqlite3 *db = pParse->db; + Window *pExist = windowFind(pParse, pList, pWin->zBase); + if( pExist ){ + const char *zErr = 0; + /* Check for errors */ + if( pWin->pPartition ){ + zErr = "PARTITION clause"; + }else if( pExist->pOrderBy && pWin->pOrderBy ){ + zErr = "ORDER BY clause"; + }else if( pExist->bImplicitFrame==0 ){ + zErr = "frame specification"; + } + if( zErr ){ + sqlite3ErrorMsg(pParse, + "cannot override %s of window: %s", zErr, pWin->zBase + ); + }else{ + pWin->pPartition = sqlite3ExprListDup(db, pExist->pPartition, 0); + if( pExist->pOrderBy ){ + assert( pWin->pOrderBy==0 ); + pWin->pOrderBy = sqlite3ExprListDup(db, pExist->pOrderBy, 0); + } + sqlite3DbFree(db, pWin->zBase); + pWin->zBase = 0; + } + } + } +} + /* ** Attach window object pWin to expression p. */ SQLITE_PRIVATE void sqlite3WindowAttach(Parse *pParse, Expr *p, Window *pWin){ if( p ){ assert( p->op==TK_FUNCTION ); - /* This routine is only called for the parser. If pWin was not - ** allocated due to an OOM, then the parser would fail before ever - ** invoking this routine */ - if( ALWAYS(pWin) ){ - p->y.pWin = pWin; - ExprSetProperty(p, EP_WinFunc); - pWin->pOwner = p; - if( p->flags & EP_Distinct ){ - sqlite3ErrorMsg(pParse, - "DISTINCT is not supported for window functions"); - } + assert( pWin ); + p->y.pWin = pWin; + ExprSetProperty(p, EP_WinFunc); + pWin->pOwner = p; + if( (p->flags & EP_Distinct) && pWin->eFrmType!=TK_FILTER ){ + sqlite3ErrorMsg(pParse, + "DISTINCT is not supported for window functions" + ); } }else{ sqlite3WindowDelete(pParse->db, pWin); } } +/* +** Possibly link window pWin into the list at pSel->pWin (window functions +** to be processed as part of SELECT statement pSel). The window is linked +** in if either (a) there are no other windows already linked to this +** SELECT, or (b) the windows already linked use a compatible window frame. +*/ +SQLITE_PRIVATE void sqlite3WindowLink(Select *pSel, Window *pWin){ + if( 0==pSel->pWin + || 0==sqlite3WindowCompare(0, pSel->pWin, pWin, 0) + ){ + pWin->pNextWin = pSel->pWin; + if( pSel->pWin ){ + pSel->pWin->ppThis = &pWin->pNextWin; + } + pSel->pWin = pWin; + pWin->ppThis = &pSel->pWin; + } +} + /* ** Return 0 if the two window objects are identical, or non-zero otherwise. ** Identical window objects can be processed in a single scan. */ -SQLITE_PRIVATE int sqlite3WindowCompare(Parse *pParse, Window *p1, Window *p2){ - if( p1->eType!=p2->eType ) return 1; +SQLITE_PRIVATE int sqlite3WindowCompare(Parse *pParse, Window *p1, Window *p2, int bFilter){ + if( p1->eFrmType!=p2->eFrmType ) return 1; if( p1->eStart!=p2->eStart ) return 1; if( p1->eEnd!=p2->eEnd ) return 1; + if( p1->eExclude!=p2->eExclude ) return 1; if( sqlite3ExprCompare(pParse, p1->pStart, p2->pStart, -1) ) return 1; if( sqlite3ExprCompare(pParse, p1->pEnd, p2->pEnd, -1) ) return 1; if( sqlite3ExprListCompare(p1->pPartition, p2->pPartition, -1) ) return 1; if( sqlite3ExprListCompare(p1->pOrderBy, p2->pOrderBy, -1) ) return 1; + if( bFilter ){ + if( sqlite3ExprCompare(pParse, p1->pFilter, p2->pFilter, -1) ) return 1; + } return 0; } @@ -146063,12 +147983,27 @@ SQLITE_PRIVATE int sqlite3WindowCompare(Parse *pParse, Window *p1, Window *p2){ SQLITE_PRIVATE void sqlite3WindowCodeInit(Parse *pParse, Window *pMWin){ Window *pWin; Vdbe *v = sqlite3GetVdbe(pParse); - int nPart = (pMWin->pPartition ? pMWin->pPartition->nExpr : 0); - nPart += (pMWin->pOrderBy ? pMWin->pOrderBy->nExpr : 0); - if( nPart ){ + + /* Allocate registers to use for PARTITION BY values, if any. Initialize + ** said registers to NULL. */ + if( pMWin->pPartition ){ + int nExpr = pMWin->pPartition->nExpr; pMWin->regPart = pParse->nMem+1; - pParse->nMem += nPart; - sqlite3VdbeAddOp3(v, OP_Null, 0, pMWin->regPart, pMWin->regPart+nPart-1); + pParse->nMem += nExpr; + sqlite3VdbeAddOp3(v, OP_Null, 0, pMWin->regPart, pMWin->regPart+nExpr-1); + } + + pMWin->regOne = ++pParse->nMem; + sqlite3VdbeAddOp2(v, OP_Integer, 1, pMWin->regOne); + + if( pMWin->eExclude ){ + pMWin->regStartRowid = ++pParse->nMem; + pMWin->regEndRowid = ++pParse->nMem; + pMWin->csrApp = pParse->nTab++; + sqlite3VdbeAddOp2(v, OP_Integer, 1, pMWin->regStartRowid); + sqlite3VdbeAddOp2(v, OP_Integer, 0, pMWin->regEndRowid); + sqlite3VdbeAddOp2(v, OP_OpenDup, pMWin->csrApp, pMWin->iEphCsr); + return; } for(pWin=pMWin; pWin; pWin=pWin->pNextWin){ @@ -146087,8 +148022,8 @@ SQLITE_PRIVATE void sqlite3WindowCodeInit(Parse *pParse, Window *pMWin){ pWin->regApp = pParse->nMem+1; pParse->nMem += 3; if( pKeyInfo && pWin->pFunc->zName[1]=='i' ){ - assert( pKeyInfo->aSortOrder[0]==0 ); - pKeyInfo->aSortOrder[0] = 1; + assert( pKeyInfo->aSortFlags[0]==0 ); + pKeyInfo->aSortFlags[0] = KEYINFO_ORDER_DESC; } sqlite3VdbeAddOp2(v, OP_OpenEphemeral, pWin->csrApp, 2); sqlite3VdbeAppendP4(v, pKeyInfo, P4_KEYINFO); @@ -146097,20 +148032,24 @@ SQLITE_PRIVATE void sqlite3WindowCodeInit(Parse *pParse, Window *pMWin){ else if( p->zName==nth_valueName || p->zName==first_valueName ){ /* Allocate two registers at pWin->regApp. These will be used to ** store the start and end index of the current frame. */ - assert( pMWin->iEphCsr ); pWin->regApp = pParse->nMem+1; pWin->csrApp = pParse->nTab++; pParse->nMem += 2; sqlite3VdbeAddOp2(v, OP_OpenDup, pWin->csrApp, pMWin->iEphCsr); } else if( p->zName==leadName || p->zName==lagName ){ - assert( pMWin->iEphCsr ); pWin->csrApp = pParse->nTab++; sqlite3VdbeAddOp2(v, OP_OpenDup, pWin->csrApp, pMWin->iEphCsr); } } } +#define WINDOW_STARTING_INT 0 +#define WINDOW_ENDING_INT 1 +#define WINDOW_NTH_VALUE_INT 2 +#define WINDOW_STARTING_NUM 3 +#define WINDOW_ENDING_NUM 4 + /* ** A "PRECEDING " (eCond==0) or "FOLLOWING " (eCond==1) or the ** value of the second argument to nth_value() (eCond==2) has just been @@ -146118,25 +148057,42 @@ SQLITE_PRIVATE void sqlite3WindowCodeInit(Parse *pParse, Window *pMWin){ ** code to check that the value is a non-negative integer and throws an ** exception if it is not. */ -static void windowCheckIntValue(Parse *pParse, int reg, int eCond){ +static void windowCheckValue(Parse *pParse, int reg, int eCond){ static const char *azErr[] = { "frame starting offset must be a non-negative integer", "frame ending offset must be a non-negative integer", - "second argument to nth_value must be a positive integer" + "second argument to nth_value must be a positive integer", + "frame starting offset must be a non-negative number", + "frame ending offset must be a non-negative number", }; - static int aOp[] = { OP_Ge, OP_Ge, OP_Gt }; + static int aOp[] = { OP_Ge, OP_Ge, OP_Gt, OP_Ge, OP_Ge }; Vdbe *v = sqlite3GetVdbe(pParse); int regZero = sqlite3GetTempReg(pParse); - assert( eCond==0 || eCond==1 || eCond==2 ); + assert( eCond>=0 && eCond=WINDOW_STARTING_NUM ){ + int regString = sqlite3GetTempReg(pParse); + sqlite3VdbeAddOp4(v, OP_String8, 0, regString, 0, "", P4_STATIC); + sqlite3VdbeAddOp3(v, OP_Ge, regString, sqlite3VdbeCurrentAddr(v)+2, reg); + sqlite3VdbeChangeP5(v, SQLITE_AFF_NUMERIC|SQLITE_JUMPIFNULL); + VdbeCoverage(v); + assert( eCond==3 || eCond==4 ); + VdbeCoverageIf(v, eCond==3); + VdbeCoverageIf(v, eCond==4); + }else{ + sqlite3VdbeAddOp2(v, OP_MustBeInt, reg, sqlite3VdbeCurrentAddr(v)+2); + VdbeCoverage(v); + assert( eCond==0 || eCond==1 || eCond==2 ); + VdbeCoverageIf(v, eCond==0); + VdbeCoverageIf(v, eCond==1); + VdbeCoverageIf(v, eCond==2); + } sqlite3VdbeAddOp3(v, aOp[eCond], regZero, sqlite3VdbeCurrentAddr(v)+2, reg); - VdbeCoverageNeverNullIf(v, eCond==0); - VdbeCoverageNeverNullIf(v, eCond==1); + VdbeCoverageNeverNullIf(v, eCond==0); /* NULL case captured by */ + VdbeCoverageNeverNullIf(v, eCond==1); /* the OP_MustBeInt */ VdbeCoverageNeverNullIf(v, eCond==2); + VdbeCoverageNeverNullIf(v, eCond==3); /* NULL case caught by */ + VdbeCoverageNeverNullIf(v, eCond==4); /* the OP_Ge */ sqlite3MayAbort(pParse); sqlite3VdbeAddOp2(v, OP_Halt, SQLITE_ERROR, OE_Abort); sqlite3VdbeAppendP4(v, (void*)azErr[eCond], P4_STATIC); @@ -146152,6 +148108,108 @@ static int windowArgCount(Window *pWin){ return (pList ? pList->nExpr : 0); } +typedef struct WindowCodeArg WindowCodeArg; +typedef struct WindowCsrAndReg WindowCsrAndReg; + +/* +** See comments above struct WindowCodeArg. +*/ +struct WindowCsrAndReg { + int csr; /* Cursor number */ + int reg; /* First in array of peer values */ +}; + +/* +** A single instance of this structure is allocated on the stack by +** sqlite3WindowCodeStep() and a pointer to it passed to the various helper +** routines. This is to reduce the number of arguments required by each +** helper function. +** +** regArg: +** Each window function requires an accumulator register (just as an +** ordinary aggregate function does). This variable is set to the first +** in an array of accumulator registers - one for each window function +** in the WindowCodeArg.pMWin list. +** +** eDelete: +** The window functions implementation sometimes caches the input rows +** that it processes in a temporary table. If it is not zero, this +** variable indicates when rows may be removed from the temp table (in +** order to reduce memory requirements - it would always be safe just +** to leave them there). Possible values for eDelete are: +** +** WINDOW_RETURN_ROW: +** An input row can be discarded after it is returned to the caller. +** +** WINDOW_AGGINVERSE: +** An input row can be discarded after the window functions xInverse() +** callbacks have been invoked in it. +** +** WINDOW_AGGSTEP: +** An input row can be discarded after the window functions xStep() +** callbacks have been invoked in it. +** +** start,current,end +** Consider a window-frame similar to the following: +** +** (ORDER BY a, b GROUPS BETWEEN 2 PRECEDING AND 2 FOLLOWING) +** +** The windows functions implmentation caches the input rows in a temp +** table, sorted by "a, b" (it actually populates the cache lazily, and +** aggressively removes rows once they are no longer required, but that's +** a mere detail). It keeps three cursors open on the temp table. One +** (current) that points to the next row to return to the query engine +** once its window function values have been calculated. Another (end) +** points to the next row to call the xStep() method of each window function +** on (so that it is 2 groups ahead of current). And a third (start) that +** points to the next row to call the xInverse() method of each window +** function on. +** +** Each cursor (start, current and end) consists of a VDBE cursor +** (WindowCsrAndReg.csr) and an array of registers (starting at +** WindowCodeArg.reg) that always contains a copy of the peer values +** read from the corresponding cursor. +** +** Depending on the window-frame in question, all three cursors may not +** be required. In this case both WindowCodeArg.csr and reg are set to +** 0. +*/ +struct WindowCodeArg { + Parse *pParse; /* Parse context */ + Window *pMWin; /* First in list of functions being processed */ + Vdbe *pVdbe; /* VDBE object */ + int addrGosub; /* OP_Gosub to this address to return one row */ + int regGosub; /* Register used with OP_Gosub(addrGosub) */ + int regArg; /* First in array of accumulator registers */ + int eDelete; /* See above */ + + WindowCsrAndReg start; + WindowCsrAndReg current; + WindowCsrAndReg end; +}; + +/* +** Generate VM code to read the window frames peer values from cursor csr into +** an array of registers starting at reg. +*/ +static void windowReadPeerValues( + WindowCodeArg *p, + int csr, + int reg +){ + Window *pMWin = p->pMWin; + ExprList *pOrderBy = pMWin->pOrderBy; + if( pOrderBy ){ + Vdbe *v = sqlite3GetVdbe(p->pParse); + ExprList *pPart = pMWin->pPartition; + int iColOff = pMWin->nBufferCol + (pPart ? pPart->nExpr : 0); + int i; + for(i=0; inExpr; i++){ + sqlite3VdbeAddOp3(v, OP_Column, csr, iColOff+i, reg+i); + } + } +} + /* ** Generate VM code to invoke either xStep() (if bInverse is 0) or ** xInverse (if bInverse is non-zero) for each window function in the @@ -146172,41 +148230,39 @@ static int windowArgCount(Window *pWin){ ** number of rows in the current partition. */ static void windowAggStep( - Parse *pParse, + WindowCodeArg *p, Window *pMWin, /* Linked list of window functions */ int csr, /* Read arguments from this cursor */ int bInverse, /* True to invoke xInverse instead of xStep */ - int reg, /* Array of registers */ - int regPartSize /* Register containing size of partition */ + int reg /* Array of registers */ ){ + Parse *pParse = p->pParse; Vdbe *v = sqlite3GetVdbe(pParse); Window *pWin; for(pWin=pMWin; pWin; pWin=pWin->pNextWin){ - int flags = pWin->pFunc->funcFlags; + FuncDef *pFunc = pWin->pFunc; int regArg; - int nArg = windowArgCount(pWin); + int nArg = pWin->bExprArgs ? 0 : windowArgCount(pWin); + int i; - if( csr>=0 ){ - int i; - for(i=0; ieStart!=TK_UNBOUNDED ); + + /* All OVER clauses in the same window function aggregate step must + ** be the same. */ + assert( pWin==pMWin || sqlite3WindowCompare(pParse,pWin,pMWin,0)==0 ); + + for(i=0; izName!=nth_valueName ){ sqlite3VdbeAddOp3(v, OP_Column, csr, pWin->iArgCol+i, reg+i); + }else{ + sqlite3VdbeAddOp3(v, OP_Column, pMWin->iEphCsr, pWin->iArgCol+i, reg+i); } - regArg = reg; - if( flags & SQLITE_FUNC_WINDOW_SIZE ){ - if( nArg==0 ){ - regArg = regPartSize; - }else{ - sqlite3VdbeAddOp2(v, OP_SCopy, regPartSize, reg+nArg); - } - nArg++; - } - }else{ - assert( !(flags & SQLITE_FUNC_WINDOW_SIZE) ); - regArg = reg + pWin->iArgCol; } + regArg = reg; - if( (pWin->pFunc->funcFlags & SQLITE_FUNC_MINMAX) - && pWin->eStart!=TK_UNBOUNDED + if( pMWin->regStartRowid==0 + && (pFunc->funcFlags & SQLITE_FUNC_MINMAX) + && (pWin->eStart!=TK_UNBOUNDED) ){ int addrIsNull = sqlite3VdbeAddOp1(v, OP_IsNull, regArg); VdbeCoverage(v); @@ -146223,34 +148279,40 @@ static void windowAggStep( } sqlite3VdbeJumpHere(v, addrIsNull); }else if( pWin->regApp ){ - assert( pWin->pFunc->zName==nth_valueName - || pWin->pFunc->zName==first_valueName + assert( pFunc->zName==nth_valueName + || pFunc->zName==first_valueName ); assert( bInverse==0 || bInverse==1 ); sqlite3VdbeAddOp2(v, OP_AddImm, pWin->regApp+1-bInverse, 1); - }else if( pWin->pFunc->zName==leadName - || pWin->pFunc->zName==lagName - ){ - /* no-op */ - }else{ + }else if( pFunc->xSFunc!=noopStepFunc ){ int addrIf = 0; if( pWin->pFilter ){ int regTmp; - assert( nArg==0 || nArg==pWin->pOwner->x.pList->nExpr ); - assert( nArg || pWin->pOwner->x.pList==0 ); - if( csr>0 ){ - regTmp = sqlite3GetTempReg(pParse); - sqlite3VdbeAddOp3(v, OP_Column, csr, pWin->iArgCol+nArg,regTmp); - }else{ - regTmp = regArg + nArg; - } + assert( pWin->bExprArgs || !nArg ||nArg==pWin->pOwner->x.pList->nExpr ); + assert( pWin->bExprArgs || nArg ||pWin->pOwner->x.pList==0 ); + regTmp = sqlite3GetTempReg(pParse); + sqlite3VdbeAddOp3(v, OP_Column, csr, pWin->iArgCol+nArg,regTmp); addrIf = sqlite3VdbeAddOp3(v, OP_IfNot, regTmp, 0, 1); VdbeCoverage(v); - if( csr>0 ){ - sqlite3ReleaseTempReg(pParse, regTmp); + sqlite3ReleaseTempReg(pParse, regTmp); + } + + if( pWin->bExprArgs ){ + int iStart = sqlite3VdbeCurrentAddr(v); + VdbeOp *pOp, *pEnd; + + nArg = pWin->pOwner->x.pList->nExpr; + regArg = sqlite3GetTempRange(pParse, nArg); + sqlite3ExprCodeExprList(pParse, pWin->pOwner->x.pList, regArg, 0, 0); + + pEnd = sqlite3VdbeGetOp(v, -1); + for(pOp=sqlite3VdbeGetOp(v, iStart); pOp<=pEnd; pOp++){ + if( pOp->opcode==OP_Column && pOp->p1==pWin->iEphCsr ){ + pOp->p1 = csr; + } } } - if( pWin->pFunc->funcFlags & SQLITE_FUNC_NEEDCOLL ){ + if( pFunc->funcFlags & SQLITE_FUNC_NEEDCOLL ){ CollSeq *pColl; assert( nArg>0 ); pColl = sqlite3ExprNNCollSeq(pParse, pWin->pOwner->x.pList->a[0].pExpr); @@ -146258,45 +148320,56 @@ static void windowAggStep( } sqlite3VdbeAddOp3(v, bInverse? OP_AggInverse : OP_AggStep, bInverse, regArg, pWin->regAccum); - sqlite3VdbeAppendP4(v, pWin->pFunc, P4_FUNCDEF); + sqlite3VdbeAppendP4(v, pFunc, P4_FUNCDEF); sqlite3VdbeChangeP5(v, (u8)nArg); + if( pWin->bExprArgs ){ + sqlite3ReleaseTempRange(pParse, regArg, nArg); + } if( addrIf ) sqlite3VdbeJumpHere(v, addrIf); } } } /* -** Generate VM code to invoke either xValue() (bFinal==0) or xFinalize() -** (bFinal==1) for each window function in the linked list starting at +** Values that may be passed as the second argument to windowCodeOp(). +*/ +#define WINDOW_RETURN_ROW 1 +#define WINDOW_AGGINVERSE 2 +#define WINDOW_AGGSTEP 3 + +/* +** Generate VM code to invoke either xValue() (bFin==0) or xFinalize() +** (bFin==1) for each window function in the linked list starting at ** pMWin. Or, for built-in window-functions that do not use the standard ** API, generate the equivalent VM code. */ -static void windowAggFinal(Parse *pParse, Window *pMWin, int bFinal){ +static void windowAggFinal(WindowCodeArg *p, int bFin){ + Parse *pParse = p->pParse; + Window *pMWin = p->pMWin; Vdbe *v = sqlite3GetVdbe(pParse); Window *pWin; for(pWin=pMWin; pWin; pWin=pWin->pNextWin){ - if( (pWin->pFunc->funcFlags & SQLITE_FUNC_MINMAX) - && pWin->eStart!=TK_UNBOUNDED + if( pMWin->regStartRowid==0 + && (pWin->pFunc->funcFlags & SQLITE_FUNC_MINMAX) + && (pWin->eStart!=TK_UNBOUNDED) ){ sqlite3VdbeAddOp2(v, OP_Null, 0, pWin->regResult); sqlite3VdbeAddOp1(v, OP_Last, pWin->csrApp); VdbeCoverage(v); sqlite3VdbeAddOp3(v, OP_Column, pWin->csrApp, 0, pWin->regResult); sqlite3VdbeJumpHere(v, sqlite3VdbeCurrentAddr(v)-2); - if( bFinal ){ - sqlite3VdbeAddOp1(v, OP_ResetSorter, pWin->csrApp); - } }else if( pWin->regApp ){ + assert( pMWin->regStartRowid==0 ); }else{ - if( bFinal ){ - sqlite3VdbeAddOp2(v, OP_AggFinal, pWin->regAccum, windowArgCount(pWin)); + int nArg = windowArgCount(pWin); + if( bFin ){ + sqlite3VdbeAddOp2(v, OP_AggFinal, pWin->regAccum, nArg); sqlite3VdbeAppendP4(v, pWin->pFunc, P4_FUNCDEF); sqlite3VdbeAddOp2(v, OP_Copy, pWin->regAccum, pWin->regResult); sqlite3VdbeAddOp2(v, OP_Null, 0, pWin->regAccum); }else{ - sqlite3VdbeAddOp3(v, OP_AggValue, pWin->regAccum, windowArgCount(pWin), - pWin->regResult); + sqlite3VdbeAddOp3(v, OP_AggValue,pWin->regAccum,nArg,pWin->regResult); sqlite3VdbeAppendP4(v, pWin->pFunc, P4_FUNCDEF); } } @@ -146304,66 +148377,102 @@ static void windowAggFinal(Parse *pParse, Window *pMWin, int bFinal){ } /* -** This function generates VM code to invoke the sub-routine at address -** lblFlushPart once for each partition with the entire partition cached in -** the Window.iEphCsr temp table. +** Generate code to calculate the current values of all window functions in the +** p->pMWin list by doing a full scan of the current window frame. Store the +** results in the Window.regResult registers, ready to return the upper +** layer. */ -static void windowPartitionCache( - Parse *pParse, - Select *p, /* The rewritten SELECT statement */ - WhereInfo *pWInfo, /* WhereInfo to call WhereEnd() on */ - int regFlushPart, /* Register to use with Gosub lblFlushPart */ - int lblFlushPart, /* Subroutine to Gosub to */ - int *pRegSize /* OUT: Register containing partition size */ -){ - Window *pMWin = p->pWin; - Vdbe *v = sqlite3GetVdbe(pParse); - int iSubCsr = p->pSrc->a[0].iCursor; - int nSub = p->pSrc->a[0].pTab->nCol; - int k; +static void windowFullScan(WindowCodeArg *p){ + Window *pWin; + Parse *pParse = p->pParse; + Window *pMWin = p->pMWin; + Vdbe *v = p->pVdbe; - int reg = pParse->nMem+1; - int regRecord = reg+nSub; - int regRowid = regRecord+1; + int regCRowid = 0; /* Current rowid value */ + int regCPeer = 0; /* Current peer values */ + int regRowid = 0; /* AggStep rowid value */ + int regPeer = 0; /* AggStep peer values */ - *pRegSize = regRowid; - pParse->nMem += nSub + 2; + int nPeer; + int lblNext; + int lblBrk; + int addrNext; + int csr; - /* Load the column values for the row returned by the sub-select - ** into an array of registers starting at reg. */ - for(k=0; kcsrApp; + nPeer = (pMWin->pOrderBy ? pMWin->pOrderBy->nExpr : 0); + + lblNext = sqlite3VdbeMakeLabel(pParse); + lblBrk = sqlite3VdbeMakeLabel(pParse); + + regCRowid = sqlite3GetTempReg(pParse); + regRowid = sqlite3GetTempReg(pParse); + if( nPeer ){ + regCPeer = sqlite3GetTempRange(pParse, nPeer); + regPeer = sqlite3GetTempRange(pParse, nPeer); } - sqlite3VdbeAddOp3(v, OP_MakeRecord, reg, nSub, regRecord); - /* Check if this is the start of a new partition. If so, call the - ** flush_partition sub-routine. */ - if( pMWin->pPartition ){ + sqlite3VdbeAddOp2(v, OP_Rowid, pMWin->iEphCsr, regCRowid); + windowReadPeerValues(p, pMWin->iEphCsr, regCPeer); + + for(pWin=pMWin; pWin; pWin=pWin->pNextWin){ + sqlite3VdbeAddOp2(v, OP_Null, 0, pWin->regAccum); + } + + sqlite3VdbeAddOp3(v, OP_SeekGE, csr, lblBrk, pMWin->regStartRowid); + VdbeCoverage(v); + addrNext = sqlite3VdbeCurrentAddr(v); + sqlite3VdbeAddOp2(v, OP_Rowid, csr, regRowid); + sqlite3VdbeAddOp3(v, OP_Gt, pMWin->regEndRowid, lblBrk, regRowid); + VdbeCoverageNeverNull(v); + + if( pMWin->eExclude==TK_CURRENT ){ + sqlite3VdbeAddOp3(v, OP_Eq, regCRowid, lblNext, regRowid); + VdbeCoverageNeverNull(v); + }else if( pMWin->eExclude!=TK_NO ){ int addr; - ExprList *pPart = pMWin->pPartition; - int nPart = pPart->nExpr; - int regNewPart = reg + pMWin->nBufferCol; - KeyInfo *pKeyInfo = sqlite3KeyInfoFromExprList(pParse, pPart, 0, 0); + int addrEq = 0; + KeyInfo *pKeyInfo = 0; - addr = sqlite3VdbeAddOp3(v, OP_Compare, regNewPart, pMWin->regPart,nPart); - sqlite3VdbeAppendP4(v, (void*)pKeyInfo, P4_KEYINFO); - sqlite3VdbeAddOp3(v, OP_Jump, addr+2, addr+4, addr+2); - VdbeCoverageEqNe(v); - sqlite3VdbeAddOp3(v, OP_Copy, regNewPart, pMWin->regPart, nPart-1); - sqlite3VdbeAddOp2(v, OP_Gosub, regFlushPart, lblFlushPart); - VdbeComment((v, "call flush_partition")); + if( pMWin->pOrderBy ){ + pKeyInfo = sqlite3KeyInfoFromExprList(pParse, pMWin->pOrderBy, 0, 0); + } + if( pMWin->eExclude==TK_TIES ){ + addrEq = sqlite3VdbeAddOp3(v, OP_Eq, regCRowid, 0, regRowid); + VdbeCoverageNeverNull(v); + } + if( pKeyInfo ){ + windowReadPeerValues(p, csr, regPeer); + sqlite3VdbeAddOp3(v, OP_Compare, regPeer, regCPeer, nPeer); + sqlite3VdbeAppendP4(v, (void*)pKeyInfo, P4_KEYINFO); + addr = sqlite3VdbeCurrentAddr(v)+1; + sqlite3VdbeAddOp3(v, OP_Jump, addr, lblNext, addr); + VdbeCoverageEqNe(v); + }else{ + sqlite3VdbeAddOp2(v, OP_Goto, 0, lblNext); + } + if( addrEq ) sqlite3VdbeJumpHere(v, addrEq); } - /* Buffer the current row in the ephemeral table. */ - sqlite3VdbeAddOp2(v, OP_NewRowid, pMWin->iEphCsr, regRowid); - sqlite3VdbeAddOp3(v, OP_Insert, pMWin->iEphCsr, regRecord, regRowid); + windowAggStep(p, pMWin, csr, 0, p->regArg); - /* End of the input loop */ - sqlite3WhereEnd(pWInfo); + sqlite3VdbeResolveLabel(v, lblNext); + sqlite3VdbeAddOp2(v, OP_Next, csr, addrNext); + VdbeCoverage(v); + sqlite3VdbeJumpHere(v, addrNext-1); + sqlite3VdbeJumpHere(v, addrNext+1); + sqlite3ReleaseTempReg(pParse, regRowid); + sqlite3ReleaseTempReg(pParse, regCRowid); + if( nPeer ){ + sqlite3ReleaseTempRange(pParse, regPeer, nPeer); + sqlite3ReleaseTempRange(pParse, regCPeer, nPeer); + } - /* Invoke "flush_partition" to deal with the final (or only) partition */ - sqlite3VdbeAddOp2(v, OP_Gosub, regFlushPart, lblFlushPart); - VdbeComment((v, "call flush_partition")); + windowAggFinal(p, 1); + VdbeModuleComment((v, "windowFullScan end")); } /* @@ -146379,110 +148488,74 @@ static void windowPartitionCache( ** lag() ** lead() */ -static void windowReturnOneRow( - Parse *pParse, - Window *pMWin, - int regGosub, - int addrGosub -){ - Vdbe *v = sqlite3GetVdbe(pParse); - Window *pWin; - for(pWin=pMWin; pWin; pWin=pWin->pNextWin){ - FuncDef *pFunc = pWin->pFunc; - if( pFunc->zName==nth_valueName - || pFunc->zName==first_valueName - ){ - int csr = pWin->csrApp; - int lbl = sqlite3VdbeMakeLabel(pParse); - int tmpReg = sqlite3GetTempReg(pParse); - sqlite3VdbeAddOp2(v, OP_Null, 0, pWin->regResult); +static void windowReturnOneRow(WindowCodeArg *p){ + Window *pMWin = p->pMWin; + Vdbe *v = p->pVdbe; - if( pFunc->zName==nth_valueName ){ - sqlite3VdbeAddOp3(v, OP_Column, pMWin->iEphCsr, pWin->iArgCol+1,tmpReg); - windowCheckIntValue(pParse, tmpReg, 2); - }else{ - sqlite3VdbeAddOp2(v, OP_Integer, 1, tmpReg); - } - sqlite3VdbeAddOp3(v, OP_Add, tmpReg, pWin->regApp, tmpReg); - sqlite3VdbeAddOp3(v, OP_Gt, pWin->regApp+1, lbl, tmpReg); - VdbeCoverageNeverNull(v); - sqlite3VdbeAddOp3(v, OP_SeekRowid, csr, 0, tmpReg); - VdbeCoverageNeverTaken(v); - sqlite3VdbeAddOp3(v, OP_Column, csr, pWin->iArgCol, pWin->regResult); - sqlite3VdbeResolveLabel(v, lbl); - sqlite3ReleaseTempReg(pParse, tmpReg); - } - else if( pFunc->zName==leadName || pFunc->zName==lagName ){ - int nArg = pWin->pOwner->x.pList->nExpr; - int iEph = pMWin->iEphCsr; - int csr = pWin->csrApp; - int lbl = sqlite3VdbeMakeLabel(pParse); - int tmpReg = sqlite3GetTempReg(pParse); - - if( nArg<3 ){ + if( pMWin->regStartRowid ){ + windowFullScan(p); + }else{ + Parse *pParse = p->pParse; + Window *pWin; + + for(pWin=pMWin; pWin; pWin=pWin->pNextWin){ + FuncDef *pFunc = pWin->pFunc; + if( pFunc->zName==nth_valueName + || pFunc->zName==first_valueName + ){ + int csr = pWin->csrApp; + int lbl = sqlite3VdbeMakeLabel(pParse); + int tmpReg = sqlite3GetTempReg(pParse); sqlite3VdbeAddOp2(v, OP_Null, 0, pWin->regResult); - }else{ - sqlite3VdbeAddOp3(v, OP_Column, iEph, pWin->iArgCol+2, pWin->regResult); - } - sqlite3VdbeAddOp2(v, OP_Rowid, iEph, tmpReg); - if( nArg<2 ){ - int val = (pFunc->zName==leadName ? 1 : -1); - sqlite3VdbeAddOp2(v, OP_AddImm, tmpReg, val); - }else{ - int op = (pFunc->zName==leadName ? OP_Add : OP_Subtract); - int tmpReg2 = sqlite3GetTempReg(pParse); - sqlite3VdbeAddOp3(v, OP_Column, iEph, pWin->iArgCol+1, tmpReg2); - sqlite3VdbeAddOp3(v, op, tmpReg2, tmpReg, tmpReg); - sqlite3ReleaseTempReg(pParse, tmpReg2); + + if( pFunc->zName==nth_valueName ){ + sqlite3VdbeAddOp3(v, OP_Column,pMWin->iEphCsr,pWin->iArgCol+1,tmpReg); + windowCheckValue(pParse, tmpReg, 2); + }else{ + sqlite3VdbeAddOp2(v, OP_Integer, 1, tmpReg); + } + sqlite3VdbeAddOp3(v, OP_Add, tmpReg, pWin->regApp, tmpReg); + sqlite3VdbeAddOp3(v, OP_Gt, pWin->regApp+1, lbl, tmpReg); + VdbeCoverageNeverNull(v); + sqlite3VdbeAddOp3(v, OP_SeekRowid, csr, 0, tmpReg); + VdbeCoverageNeverTaken(v); + sqlite3VdbeAddOp3(v, OP_Column, csr, pWin->iArgCol, pWin->regResult); + sqlite3VdbeResolveLabel(v, lbl); + sqlite3ReleaseTempReg(pParse, tmpReg); + } + else if( pFunc->zName==leadName || pFunc->zName==lagName ){ + int nArg = pWin->pOwner->x.pList->nExpr; + int csr = pWin->csrApp; + int lbl = sqlite3VdbeMakeLabel(pParse); + int tmpReg = sqlite3GetTempReg(pParse); + int iEph = pMWin->iEphCsr; + + if( nArg<3 ){ + sqlite3VdbeAddOp2(v, OP_Null, 0, pWin->regResult); + }else{ + sqlite3VdbeAddOp3(v, OP_Column, iEph,pWin->iArgCol+2,pWin->regResult); + } + sqlite3VdbeAddOp2(v, OP_Rowid, iEph, tmpReg); + if( nArg<2 ){ + int val = (pFunc->zName==leadName ? 1 : -1); + sqlite3VdbeAddOp2(v, OP_AddImm, tmpReg, val); + }else{ + int op = (pFunc->zName==leadName ? OP_Add : OP_Subtract); + int tmpReg2 = sqlite3GetTempReg(pParse); + sqlite3VdbeAddOp3(v, OP_Column, iEph, pWin->iArgCol+1, tmpReg2); + sqlite3VdbeAddOp3(v, op, tmpReg2, tmpReg, tmpReg); + sqlite3ReleaseTempReg(pParse, tmpReg2); + } + + sqlite3VdbeAddOp3(v, OP_SeekRowid, csr, lbl, tmpReg); + VdbeCoverage(v); + sqlite3VdbeAddOp3(v, OP_Column, csr, pWin->iArgCol, pWin->regResult); + sqlite3VdbeResolveLabel(v, lbl); + sqlite3ReleaseTempReg(pParse, tmpReg); } - - sqlite3VdbeAddOp3(v, OP_SeekRowid, csr, lbl, tmpReg); - VdbeCoverage(v); - sqlite3VdbeAddOp3(v, OP_Column, csr, pWin->iArgCol, pWin->regResult); - sqlite3VdbeResolveLabel(v, lbl); - sqlite3ReleaseTempReg(pParse, tmpReg); } } - sqlite3VdbeAddOp2(v, OP_Gosub, regGosub, addrGosub); -} - -/* -** Invoke the code generated by windowReturnOneRow() and, optionally, the -** xInverse() function for each window function, for one or more rows -** from the Window.iEphCsr temp table. This routine generates VM code -** similar to: -** -** while( regCtr>0 ){ -** regCtr--; -** windowReturnOneRow() -** if( bInverse ){ -** AggInverse -** } -** Next (Window.iEphCsr) -** } -*/ -static void windowReturnRows( - Parse *pParse, - Window *pMWin, /* List of window functions */ - int regCtr, /* Register containing number of rows */ - int regGosub, /* Register for Gosub addrGosub */ - int addrGosub, /* Address of sub-routine for ReturnOneRow */ - int regInvArg, /* Array of registers for xInverse args */ - int regInvSize /* Register containing size of partition */ -){ - int addr; - Vdbe *v = sqlite3GetVdbe(pParse); - windowAggFinal(pParse, pMWin, 0); - addr = sqlite3VdbeAddOp3(v, OP_IfPos, regCtr, sqlite3VdbeCurrentAddr(v)+2 ,1); - VdbeCoverage(v); - sqlite3VdbeAddOp2(v, OP_Goto, 0, 0); - windowReturnOneRow(pParse, pMWin, regGosub, addrGosub); - if( regInvArg ){ - windowAggStep(pParse, pMWin, pMWin->iEphCsr, 1, regInvArg, regInvSize); - } - sqlite3VdbeAddOp2(v, OP_Next, pMWin->iEphCsr, addr); - VdbeCoverage(v); - sqlite3VdbeJumpHere(v, addr+1); /* The OP_Goto */ + sqlite3VdbeAddOp2(v, OP_Gosub, p->regGosub, p->addrGosub); } /* @@ -146500,17 +148573,17 @@ static int windowInitAccum(Parse *pParse, Window *pMWin){ FuncDef *pFunc = pWin->pFunc; sqlite3VdbeAddOp2(v, OP_Null, 0, pWin->regAccum); nArg = MAX(nArg, windowArgCount(pWin)); - if( pFunc->zName==nth_valueName - || pFunc->zName==first_valueName - ){ - sqlite3VdbeAddOp2(v, OP_Integer, 0, pWin->regApp); - sqlite3VdbeAddOp2(v, OP_Integer, 0, pWin->regApp+1); - } + if( pMWin->regStartRowid==0 ){ + if( pFunc->zName==nth_valueName || pFunc->zName==first_valueName ){ + sqlite3VdbeAddOp2(v, OP_Integer, 0, pWin->regApp); + sqlite3VdbeAddOp2(v, OP_Integer, 0, pWin->regApp+1); + } - if( (pFunc->funcFlags & SQLITE_FUNC_MINMAX) && pWin->csrApp ){ - assert( pWin->eStart!=TK_UNBOUNDED ); - sqlite3VdbeAddOp1(v, OP_ResetSorter, pWin->csrApp); - sqlite3VdbeAddOp2(v, OP_Integer, 0, pWin->regApp+1); + if( (pFunc->funcFlags & SQLITE_FUNC_MINMAX) && pWin->csrApp ){ + assert( pWin->eStart!=TK_UNBOUNDED ); + sqlite3VdbeAddOp1(v, OP_ResetSorter, pWin->csrApp); + sqlite3VdbeAddOp2(v, OP_Integer, 0, pWin->regApp+1); + } } } regArg = pParse->nMem+1; @@ -146518,672 +148591,343 @@ static int windowInitAccum(Parse *pParse, Window *pMWin){ return regArg; } +/* +** Return true if the current frame should be cached in the ephemeral table, +** even if there are no xInverse() calls required. +*/ +static int windowCacheFrame(Window *pMWin){ + Window *pWin; + if( pMWin->regStartRowid ) return 1; + for(pWin=pMWin; pWin; pWin=pWin->pNextWin){ + FuncDef *pFunc = pWin->pFunc; + if( (pFunc->zName==nth_valueName) + || (pFunc->zName==first_valueName) + || (pFunc->zName==leadName) + || (pFunc->zName==lagName) + ){ + return 1; + } + } + return 0; +} /* -** This function does the work of sqlite3WindowCodeStep() for all "ROWS" -** window frame types except for "BETWEEN UNBOUNDED PRECEDING AND CURRENT -** ROW". Pseudo-code for each follows. -** -** ROWS BETWEEN PRECEDING AND FOLLOWING -** -** ... -** if( new partition ){ -** Gosub flush_partition -** } -** Insert (record in eph-table) -** sqlite3WhereEnd() -** Gosub flush_partition -** -** flush_partition: -** Once { -** OpenDup (iEphCsr -> csrStart) -** OpenDup (iEphCsr -> csrEnd) -** } -** regStart = // PRECEDING expression -** regEnd = // FOLLOWING expression -** if( regStart<0 || regEnd<0 ){ error! } -** Rewind (csr,csrStart,csrEnd) // if EOF goto flush_partition_done -** Next(csrEnd) // if EOF skip Aggstep -** Aggstep (csrEnd) -** if( (regEnd--)<=0 ){ -** AggFinal (xValue) -** Gosub addrGosub -** Next(csr) // if EOF goto flush_partition_done -** if( (regStart--)<=0 ){ -** AggInverse (csrStart) -** Next(csrStart) -** } -** } -** flush_partition_done: -** ResetSorter (csr) -** Return -** -** ROWS BETWEEN PRECEDING AND CURRENT ROW -** ROWS BETWEEN CURRENT ROW AND FOLLOWING -** ROWS BETWEEN UNBOUNDED PRECEDING AND FOLLOWING -** -** These are similar to the above. For "CURRENT ROW", intialize the -** register to 0. For "UNBOUNDED PRECEDING" to infinity. -** -** ROWS BETWEEN PRECEDING AND UNBOUNDED FOLLOWING -** ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING -** -** Rewind (csr,csrStart,csrEnd) // if EOF goto flush_partition_done -** while( 1 ){ -** Next(csrEnd) // Exit while(1) at EOF -** Aggstep (csrEnd) -** } -** while( 1 ){ -** AggFinal (xValue) -** Gosub addrGosub -** Next(csr) // if EOF goto flush_partition_done -** if( (regStart--)<=0 ){ -** AggInverse (csrStart) -** Next(csrStart) -** } -** } -** -** For the "CURRENT ROW AND UNBOUNDED FOLLOWING" case, the final if() -** condition is always true (as if regStart were initialized to 0). -** -** RANGE BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING -** -** This is the only RANGE case handled by this routine. It modifies the -** second while( 1 ) loop in "ROWS BETWEEN CURRENT ... UNBOUNDED..." to -** be: -** -** while( 1 ){ -** AggFinal (xValue) -** while( 1 ){ -** regPeer++ -** Gosub addrGosub -** Next(csr) // if EOF goto flush_partition_done -** if( new peer ) break; -** } -** while( (regPeer--)>0 ){ -** AggInverse (csrStart) -** Next(csrStart) -** } -** } -** -** ROWS BETWEEN FOLLOWING AND FOLLOWING -** -** regEnd = regEnd - regStart -** Rewind (csr,csrStart,csrEnd) // if EOF goto flush_partition_done -** Aggstep (csrEnd) -** Next(csrEnd) // if EOF fall-through -** if( (regEnd--)<=0 ){ -** if( (regStart--)<=0 ){ -** AggFinal (xValue) -** Gosub addrGosub -** Next(csr) // if EOF goto flush_partition_done -** } -** AggInverse (csrStart) -** Next (csrStart) -** } -** -** ROWS BETWEEN PRECEDING AND PRECEDING -** -** Replace the bit after "Rewind" in the above with: -** -** if( (regEnd--)<=0 ){ -** AggStep (csrEnd) -** Next (csrEnd) -** } -** AggFinal (xValue) -** Gosub addrGosub -** Next(csr) // if EOF goto flush_partition_done -** if( (regStart--)<=0 ){ -** AggInverse (csr2) -** Next (csr2) -** } +** regOld and regNew are each the first register in an array of size +** pOrderBy->nExpr. This function generates code to compare the two +** arrays of registers using the collation sequences and other comparison +** parameters specified by pOrderBy. ** +** If the two arrays are not equal, the contents of regNew is copied to +** regOld and control falls through. Otherwise, if the contents of the arrays +** are equal, an OP_Goto is executed. The address of the OP_Goto is returned. */ -static void windowCodeRowExprStep( - Parse *pParse, - Select *p, - WhereInfo *pWInfo, - int regGosub, - int addrGosub +static void windowIfNewPeer( + Parse *pParse, + ExprList *pOrderBy, + int regNew, /* First in array of new values */ + int regOld, /* First in array of old values */ + int addr /* Jump here */ ){ - Window *pMWin = p->pWin; Vdbe *v = sqlite3GetVdbe(pParse); - int regFlushPart; /* Register for "Gosub flush_partition" */ - int lblFlushPart; /* Label for "Gosub flush_partition" */ - int lblFlushDone; /* Label for "Gosub flush_partition_done" */ - - int regArg; - int addr; - int csrStart = pParse->nTab++; - int csrEnd = pParse->nTab++; - int regStart; /* Value of PRECEDING */ - int regEnd; /* Value of FOLLOWING */ - int addrGoto; - int addrTop; - int addrIfPos1 = 0; - int addrIfPos2 = 0; - int regSize = 0; - - assert( pMWin->eStart==TK_PRECEDING - || pMWin->eStart==TK_CURRENT - || pMWin->eStart==TK_FOLLOWING - || pMWin->eStart==TK_UNBOUNDED - ); - assert( pMWin->eEnd==TK_FOLLOWING - || pMWin->eEnd==TK_CURRENT - || pMWin->eEnd==TK_UNBOUNDED - || pMWin->eEnd==TK_PRECEDING - ); - - /* Allocate register and label for the "flush_partition" sub-routine. */ - regFlushPart = ++pParse->nMem; - lblFlushPart = sqlite3VdbeMakeLabel(pParse); - lblFlushDone = sqlite3VdbeMakeLabel(pParse); - - regStart = ++pParse->nMem; - regEnd = ++pParse->nMem; - - windowPartitionCache(pParse, p, pWInfo, regFlushPart, lblFlushPart, ®Size); - - addrGoto = sqlite3VdbeAddOp0(v, OP_Goto); - - /* Start of "flush_partition" */ - sqlite3VdbeResolveLabel(v, lblFlushPart); - sqlite3VdbeAddOp2(v, OP_Once, 0, sqlite3VdbeCurrentAddr(v)+3); - VdbeCoverage(v); - VdbeComment((v, "Flush_partition subroutine")); - sqlite3VdbeAddOp2(v, OP_OpenDup, csrStart, pMWin->iEphCsr); - sqlite3VdbeAddOp2(v, OP_OpenDup, csrEnd, pMWin->iEphCsr); - - /* If either regStart or regEnd are not non-negative integers, throw - ** an exception. */ - if( pMWin->pStart ){ - sqlite3ExprCode(pParse, pMWin->pStart, regStart); - windowCheckIntValue(pParse, regStart, 0); - } - if( pMWin->pEnd ){ - sqlite3ExprCode(pParse, pMWin->pEnd, regEnd); - windowCheckIntValue(pParse, regEnd, 1); - } - - /* If this is "ROWS FOLLOWING AND ROWS FOLLOWING", do: - ** - ** if( regEndpEnd && pMWin->eStart==TK_FOLLOWING ){ - assert( pMWin->pStart!=0 ); - assert( pMWin->eEnd==TK_FOLLOWING ); - sqlite3VdbeAddOp3(v, OP_Ge, regStart, sqlite3VdbeCurrentAddr(v)+2, regEnd); - VdbeCoverageNeverNull(v); - sqlite3VdbeAddOp2(v, OP_Copy, regSize, regStart); - sqlite3VdbeAddOp3(v, OP_Subtract, regStart, regEnd, regEnd); - } - - if( pMWin->pStart && pMWin->eEnd==TK_PRECEDING ){ - assert( pMWin->pEnd!=0 ); - assert( pMWin->eStart==TK_PRECEDING ); - sqlite3VdbeAddOp3(v, OP_Le, regStart, sqlite3VdbeCurrentAddr(v)+3, regEnd); - VdbeCoverageNeverNull(v); - sqlite3VdbeAddOp2(v, OP_Copy, regSize, regStart); - sqlite3VdbeAddOp2(v, OP_Copy, regSize, regEnd); - } - - /* Initialize the accumulator register for each window function to NULL */ - regArg = windowInitAccum(pParse, pMWin); - - sqlite3VdbeAddOp2(v, OP_Rewind, pMWin->iEphCsr, lblFlushDone); - VdbeCoverage(v); - sqlite3VdbeAddOp2(v, OP_Rewind, csrStart, lblFlushDone); - VdbeCoverageNeverTaken(v); - sqlite3VdbeChangeP5(v, 1); - sqlite3VdbeAddOp2(v, OP_Rewind, csrEnd, lblFlushDone); - VdbeCoverageNeverTaken(v); - sqlite3VdbeChangeP5(v, 1); - - /* Invoke AggStep function for each window function using the row that - ** csrEnd currently points to. Or, if csrEnd is already at EOF, - ** do nothing. */ - addrTop = sqlite3VdbeCurrentAddr(v); - if( pMWin->eEnd==TK_PRECEDING ){ - addrIfPos1 = sqlite3VdbeAddOp3(v, OP_IfPos, regEnd, 0 , 1); - VdbeCoverage(v); - } - sqlite3VdbeAddOp2(v, OP_Next, csrEnd, sqlite3VdbeCurrentAddr(v)+2); - VdbeCoverage(v); - addr = sqlite3VdbeAddOp0(v, OP_Goto); - windowAggStep(pParse, pMWin, csrEnd, 0, regArg, regSize); - if( pMWin->eEnd==TK_UNBOUNDED ){ - sqlite3VdbeAddOp2(v, OP_Goto, 0, addrTop); - sqlite3VdbeJumpHere(v, addr); - addrTop = sqlite3VdbeCurrentAddr(v); + if( pOrderBy ){ + int nVal = pOrderBy->nExpr; + KeyInfo *pKeyInfo = sqlite3KeyInfoFromExprList(pParse, pOrderBy, 0, 0); + sqlite3VdbeAddOp3(v, OP_Compare, regOld, regNew, nVal); + sqlite3VdbeAppendP4(v, (void*)pKeyInfo, P4_KEYINFO); + sqlite3VdbeAddOp3(v, OP_Jump, + sqlite3VdbeCurrentAddr(v)+1, addr, sqlite3VdbeCurrentAddr(v)+1 + ); + VdbeCoverageEqNe(v); + sqlite3VdbeAddOp3(v, OP_Copy, regNew, regOld, nVal-1); }else{ - sqlite3VdbeJumpHere(v, addr); - if( pMWin->eEnd==TK_PRECEDING ){ - sqlite3VdbeJumpHere(v, addrIfPos1); - } - } - - if( pMWin->eEnd==TK_FOLLOWING ){ - addrIfPos1 = sqlite3VdbeAddOp3(v, OP_IfPos, regEnd, 0 , 1); - VdbeCoverage(v); - } - if( pMWin->eStart==TK_FOLLOWING ){ - addrIfPos2 = sqlite3VdbeAddOp3(v, OP_IfPos, regStart, 0 , 1); - VdbeCoverage(v); - } - windowAggFinal(pParse, pMWin, 0); - windowReturnOneRow(pParse, pMWin, regGosub, addrGosub); - sqlite3VdbeAddOp2(v, OP_Next, pMWin->iEphCsr, sqlite3VdbeCurrentAddr(v)+2); - VdbeCoverage(v); - sqlite3VdbeAddOp2(v, OP_Goto, 0, lblFlushDone); - if( pMWin->eStart==TK_FOLLOWING ){ - sqlite3VdbeJumpHere(v, addrIfPos2); + sqlite3VdbeAddOp2(v, OP_Goto, 0, addr); } - - if( pMWin->eStart==TK_CURRENT - || pMWin->eStart==TK_PRECEDING - || pMWin->eStart==TK_FOLLOWING - ){ - int lblSkipInverse = sqlite3VdbeMakeLabel(pParse);; - if( pMWin->eStart==TK_PRECEDING ){ - sqlite3VdbeAddOp3(v, OP_IfPos, regStart, lblSkipInverse, 1); - VdbeCoverage(v); - } - if( pMWin->eStart==TK_FOLLOWING ){ - sqlite3VdbeAddOp2(v, OP_Next, csrStart, sqlite3VdbeCurrentAddr(v)+2); - VdbeCoverage(v); - sqlite3VdbeAddOp2(v, OP_Goto, 0, lblSkipInverse); - }else{ - sqlite3VdbeAddOp2(v, OP_Next, csrStart, sqlite3VdbeCurrentAddr(v)+1); - VdbeCoverageAlwaysTaken(v); - } - windowAggStep(pParse, pMWin, csrStart, 1, regArg, regSize); - sqlite3VdbeResolveLabel(v, lblSkipInverse); - } - if( pMWin->eEnd==TK_FOLLOWING ){ - sqlite3VdbeJumpHere(v, addrIfPos1); - } - sqlite3VdbeAddOp2(v, OP_Goto, 0, addrTop); - - /* flush_partition_done: */ - sqlite3VdbeResolveLabel(v, lblFlushDone); - sqlite3VdbeAddOp1(v, OP_ResetSorter, pMWin->iEphCsr); - sqlite3VdbeAddOp1(v, OP_Return, regFlushPart); - VdbeComment((v, "end flush_partition subroutine")); - - /* Jump to here to skip over flush_partition */ - sqlite3VdbeJumpHere(v, addrGoto); } /* -** This function does the work of sqlite3WindowCodeStep() for cases that -** would normally be handled by windowCodeDefaultStep() when there are -** one or more built-in window-functions that require the entire partition -** to be cached in a temp table before any rows can be returned. Additionally. -** "RANGE BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING" is always handled by -** this function. -** -** Pseudo-code corresponding to the VM code generated by this function -** for each type of window follows. -** -** RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW -** -** flush_partition: -** Once { -** OpenDup (iEphCsr -> csrLead) -** } -** Integer ctr 0 -** foreach row (csrLead){ -** if( new peer ){ -** AggFinal (xValue) -** for(i=0; i= csr2.peerVal ) goto lbl; ** -** ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW +** The value of parameter op may also be OP_Gt or OP_Le. In these cases the +** operator in the above pseudo-code is replaced with ">" or "<=", respectively. ** -** As above, except that the "if( new peer )" branch is always taken. +** If the sort-order for the ORDER BY term in the window is DESC, then the +** comparison is reversed. Instead of adding regVal to csr1.peerVal, it is +** subtracted. And the comparison operator is inverted to - ">=" becomes "<=", +** ">" becomes "<", and so on. So, with DESC sort order, if the argument op +** is OP_Ge, the generated code is equivalent to: ** -** RANGE BETWEEN CURRENT ROW AND CURRENT ROW +** if( csr1.peerVal - regVal <= csr2.peerVal ) goto lbl; ** -** As above, except that each of the for() loops becomes: -** -** for(i=0; i csrLead) -** } -** foreach row (csrLead) { -** AggStep (csrLead) -** } -** foreach row (iEphCsr) { -** Gosub addrGosub -** } -** -** RANGE BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING -** -** flush_partition: -** Once { -** OpenDup (iEphCsr -> csrLead) -** } -** foreach row (csrLead){ -** AggStep (csrLead) -** } -** Rewind (csrLead) -** Integer ctr 0 -** foreach row (csrLead){ -** if( new peer ){ -** AggFinal (xValue) -** for(i=0; ipWin; + Parse *pParse = p->pParse; Vdbe *v = sqlite3GetVdbe(pParse); - int k; - int addr; - ExprList *pPart = pMWin->pPartition; - ExprList *pOrderBy = pMWin->pOrderBy; - int nPeer = pOrderBy ? pOrderBy->nExpr : 0; - int regNewPeer; - - int addrGoto; /* Address of Goto used to jump flush_par.. */ - int addrNext; /* Jump here for next iteration of loop */ - int regFlushPart; - int lblFlushPart; - int csrLead; - int regCtr; - int regArg; /* Register array to martial function args */ - int regSize; - int lblEmpty; - int bReverse = pMWin->pOrderBy && pMWin->eStart==TK_CURRENT - && pMWin->eEnd==TK_UNBOUNDED; - - assert( (pMWin->eStart==TK_UNBOUNDED && pMWin->eEnd==TK_CURRENT) - || (pMWin->eStart==TK_UNBOUNDED && pMWin->eEnd==TK_UNBOUNDED) - || (pMWin->eStart==TK_CURRENT && pMWin->eEnd==TK_CURRENT) - || (pMWin->eStart==TK_CURRENT && pMWin->eEnd==TK_UNBOUNDED) - ); - - lblEmpty = sqlite3VdbeMakeLabel(pParse); - regNewPeer = pParse->nMem+1; - pParse->nMem += nPeer; - - /* Allocate register and label for the "flush_partition" sub-routine. */ - regFlushPart = ++pParse->nMem; - lblFlushPart = sqlite3VdbeMakeLabel(pParse); - - csrLead = pParse->nTab++; - regCtr = ++pParse->nMem; - - windowPartitionCache(pParse, p, pWInfo, regFlushPart, lblFlushPart, ®Size); - addrGoto = sqlite3VdbeAddOp0(v, OP_Goto); + ExprList *pOrderBy = p->pMWin->pOrderBy; /* ORDER BY clause for window */ + int reg1 = sqlite3GetTempReg(pParse); /* Reg. for csr1.peerVal+regVal */ + int reg2 = sqlite3GetTempReg(pParse); /* Reg. for csr2.peerVal */ + int regString = ++pParse->nMem; /* Reg. for constant value '' */ + int arith = OP_Add; /* OP_Add or OP_Subtract */ + int addrGe; /* Jump destination */ + + assert( op==OP_Ge || op==OP_Gt || op==OP_Le ); + assert( pOrderBy && pOrderBy->nExpr==1 ); + if( pOrderBy->a[0].sortFlags & KEYINFO_ORDER_DESC ){ + switch( op ){ + case OP_Ge: op = OP_Le; break; + case OP_Gt: op = OP_Lt; break; + default: assert( op==OP_Le ); op = OP_Ge; break; + } + arith = OP_Subtract; + } - /* Start of "flush_partition" */ - sqlite3VdbeResolveLabel(v, lblFlushPart); - sqlite3VdbeAddOp2(v, OP_Once, 0, sqlite3VdbeCurrentAddr(v)+2); - VdbeCoverage(v); - sqlite3VdbeAddOp2(v, OP_OpenDup, csrLead, pMWin->iEphCsr); + /* Read the peer-value from each cursor into a register */ + windowReadPeerValues(p, csr1, reg1); + windowReadPeerValues(p, csr2, reg2); - /* Initialize the accumulator register for each window function to NULL */ - regArg = windowInitAccum(pParse, pMWin); + VdbeModuleComment((v, "CodeRangeTest: if( R%d %s R%d %s R%d ) goto lbl", + reg1, (arith==OP_Add ? "+" : "-"), regVal, + ((op==OP_Ge) ? ">=" : (op==OP_Le) ? "<=" : (op==OP_Gt) ? ">" : "<"), reg2 + )); - sqlite3VdbeAddOp2(v, OP_Integer, 0, regCtr); - sqlite3VdbeAddOp2(v, OP_Rewind, csrLead, lblEmpty); + /* Register reg1 currently contains csr1.peerVal (the peer-value from csr1). + ** This block adds (or subtracts for DESC) the numeric value in regVal + ** from it. Or, if reg1 is not numeric (it is a NULL, a text value or a blob), + ** then leave reg1 as it is. In pseudo-code, this is implemented as: + ** + ** if( reg1>='' ) goto addrGe; + ** reg1 = reg1 +/- regVal + ** addrGe: + ** + ** Since all strings and blobs are greater-than-or-equal-to an empty string, + ** the add/subtract is skipped for these, as required. If reg1 is a NULL, + ** then the arithmetic is performed, but since adding or subtracting from + ** NULL is always NULL anyway, this case is handled as required too. */ + sqlite3VdbeAddOp4(v, OP_String8, 0, regString, 0, "", P4_STATIC); + addrGe = sqlite3VdbeAddOp3(v, OP_Ge, regString, 0, reg1); VdbeCoverage(v); - sqlite3VdbeAddOp2(v, OP_Rewind, pMWin->iEphCsr, lblEmpty); - VdbeCoverageNeverTaken(v); - - if( bReverse ){ - int addr2 = sqlite3VdbeCurrentAddr(v); - windowAggStep(pParse, pMWin, csrLead, 0, regArg, regSize); - sqlite3VdbeAddOp2(v, OP_Next, csrLead, addr2); - VdbeCoverage(v); - sqlite3VdbeAddOp2(v, OP_Rewind, csrLead, lblEmpty); - VdbeCoverageNeverTaken(v); - } - addrNext = sqlite3VdbeCurrentAddr(v); - - if( pOrderBy && (pMWin->eEnd==TK_CURRENT || pMWin->eStart==TK_CURRENT) ){ - int bCurrent = (pMWin->eStart==TK_CURRENT); - int addrJump = 0; /* Address of OP_Jump below */ - if( pMWin->eType==TK_RANGE ){ - int iOff = pMWin->nBufferCol + (pPart ? pPart->nExpr : 0); - int regPeer = pMWin->regPart + (pPart ? pPart->nExpr : 0); - KeyInfo *pKeyInfo = sqlite3KeyInfoFromExprList(pParse, pOrderBy, 0, 0); - for(k=0; ka[0].sortFlags & KEYINFO_ORDER_BIGNULL ){ + /* This block runs if reg1 contains a NULL. */ + int addr = sqlite3VdbeAddOp1(v, OP_NotNull, reg1); VdbeCoverage(v); + switch( op ){ + case OP_Ge: + sqlite3VdbeAddOp2(v, OP_Goto, 0, lbl); + break; + case OP_Gt: + sqlite3VdbeAddOp2(v, OP_NotNull, reg2, lbl); + VdbeCoverage(v); + break; + case OP_Le: + sqlite3VdbeAddOp2(v, OP_IsNull, reg2, lbl); + VdbeCoverage(v); + break; + default: assert( op==OP_Lt ); /* no-op */ break; } + sqlite3VdbeAddOp2(v, OP_Goto, 0, sqlite3VdbeCurrentAddr(v)+3); - windowReturnRows(pParse, pMWin, regCtr, regGosub, addrGosub, - (bCurrent ? regArg : 0), (bCurrent ? regSize : 0) - ); - if( addrJump ) sqlite3VdbeJumpHere(v, addrJump); + /* This block runs if reg1 is not NULL, but reg2 is. */ + sqlite3VdbeJumpHere(v, addr); + sqlite3VdbeAddOp2(v, OP_IsNull, reg2, lbl); VdbeCoverage(v); + if( op==OP_Gt || op==OP_Ge ){ + sqlite3VdbeChangeP2(v, -1, sqlite3VdbeCurrentAddr(v)+1); + } } - if( bReverse==0 ){ - windowAggStep(pParse, pMWin, csrLead, 0, regArg, regSize); - } - sqlite3VdbeAddOp2(v, OP_AddImm, regCtr, 1); - sqlite3VdbeAddOp2(v, OP_Next, csrLead, addrNext); - VdbeCoverage(v); + /* Compare registers reg2 and reg1, taking the jump if required. Note that + ** control skips over this test if the BIGNULL flag is set and either + ** reg1 or reg2 contain a NULL value. */ + sqlite3VdbeAddOp3(v, op, reg2, lbl, reg1); VdbeCoverage(v); + sqlite3VdbeChangeP5(v, SQLITE_NULLEQ); - windowReturnRows(pParse, pMWin, regCtr, regGosub, addrGosub, 0, 0); + assert( op==OP_Ge || op==OP_Gt || op==OP_Lt || op==OP_Le ); + testcase(op==OP_Ge); VdbeCoverageIf(v, op==OP_Ge); + testcase(op==OP_Lt); VdbeCoverageIf(v, op==OP_Lt); + testcase(op==OP_Le); VdbeCoverageIf(v, op==OP_Le); + testcase(op==OP_Gt); VdbeCoverageIf(v, op==OP_Gt); + sqlite3ReleaseTempReg(pParse, reg1); + sqlite3ReleaseTempReg(pParse, reg2); - sqlite3VdbeResolveLabel(v, lblEmpty); - sqlite3VdbeAddOp1(v, OP_ResetSorter, pMWin->iEphCsr); - sqlite3VdbeAddOp1(v, OP_Return, regFlushPart); - - /* Jump to here to skip over flush_partition */ - sqlite3VdbeJumpHere(v, addrGoto); + VdbeModuleComment((v, "CodeRangeTest: end")); } - /* -** RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW -** -** ... -** if( new partition ){ -** AggFinal (xFinalize) -** Gosub addrGosub -** ResetSorter eph-table -** } -** else if( new peer ){ -** AggFinal (xValue) -** Gosub addrGosub -** ResetSorter eph-table -** } -** AggStep -** Insert (record into eph-table) -** sqlite3WhereEnd() -** AggFinal (xFinalize) -** Gosub addrGosub -** -** RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING -** -** As above, except take no action for a "new peer". Invoke -** the sub-routine once only for each partition. -** -** RANGE BETWEEN CURRENT ROW AND CURRENT ROW -** -** As above, except that the "new peer" condition is handled in the -** same way as "new partition" (so there is no "else if" block). -** -** ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW -** -** As above, except assume every row is a "new peer". +** Helper function for sqlite3WindowCodeStep(). Each call to this function +** generates VM code for a single RETURN_ROW, AGGSTEP or AGGINVERSE +** operation. Refer to the header comment for sqlite3WindowCodeStep() for +** details. */ -static void windowCodeDefaultStep( - Parse *pParse, - Select *p, - WhereInfo *pWInfo, - int regGosub, - int addrGosub +static int windowCodeOp( + WindowCodeArg *p, /* Context object */ + int op, /* WINDOW_RETURN_ROW, AGGSTEP or AGGINVERSE */ + int regCountdown, /* Register for OP_IfPos countdown */ + int jumpOnEof /* Jump here if stepped cursor reaches EOF */ ){ - Window *pMWin = p->pWin; - Vdbe *v = sqlite3GetVdbe(pParse); - int k; - int iSubCsr = p->pSrc->a[0].iCursor; - int nSub = p->pSrc->a[0].pTab->nCol; - int reg = pParse->nMem+1; - int regRecord = reg+nSub; - int regRowid = regRecord+1; - int addr; - ExprList *pPart = pMWin->pPartition; - ExprList *pOrderBy = pMWin->pOrderBy; + int csr, reg; + Parse *pParse = p->pParse; + Window *pMWin = p->pMWin; + int ret = 0; + Vdbe *v = p->pVdbe; + int addrContinue = 0; + int bPeer = (pMWin->eFrmType!=TK_ROWS); - assert( pMWin->eType==TK_RANGE - || (pMWin->eStart==TK_UNBOUNDED && pMWin->eEnd==TK_CURRENT) - ); + int lblDone = sqlite3VdbeMakeLabel(pParse); + int addrNextRange = 0; - assert( (pMWin->eStart==TK_UNBOUNDED && pMWin->eEnd==TK_CURRENT) - || (pMWin->eStart==TK_UNBOUNDED && pMWin->eEnd==TK_UNBOUNDED) - || (pMWin->eStart==TK_CURRENT && pMWin->eEnd==TK_CURRENT) - || (pMWin->eStart==TK_CURRENT && pMWin->eEnd==TK_UNBOUNDED && !pOrderBy) - ); + /* Special case - WINDOW_AGGINVERSE is always a no-op if the frame + ** starts with UNBOUNDED PRECEDING. */ + if( op==WINDOW_AGGINVERSE && pMWin->eStart==TK_UNBOUNDED ){ + assert( regCountdown==0 && jumpOnEof==0 ); + return 0; + } - if( pMWin->eEnd==TK_UNBOUNDED ){ - pOrderBy = 0; + if( regCountdown>0 ){ + if( pMWin->eFrmType==TK_RANGE ){ + addrNextRange = sqlite3VdbeCurrentAddr(v); + assert( op==WINDOW_AGGINVERSE || op==WINDOW_AGGSTEP ); + if( op==WINDOW_AGGINVERSE ){ + if( pMWin->eStart==TK_FOLLOWING ){ + windowCodeRangeTest( + p, OP_Le, p->current.csr, regCountdown, p->start.csr, lblDone + ); + }else{ + windowCodeRangeTest( + p, OP_Ge, p->start.csr, regCountdown, p->current.csr, lblDone + ); + } + }else{ + windowCodeRangeTest( + p, OP_Gt, p->end.csr, regCountdown, p->current.csr, lblDone + ); + } + }else{ + sqlite3VdbeAddOp3(v, OP_IfPos, regCountdown, lblDone, 1); + VdbeCoverage(v); + } } - pParse->nMem += nSub + 2; + if( op==WINDOW_RETURN_ROW && pMWin->regStartRowid==0 ){ + windowAggFinal(p, 0); + } + addrContinue = sqlite3VdbeCurrentAddr(v); - /* Load the individual column values of the row returned by - ** the sub-select into an array of registers. */ - for(k=0; kb). */ + if( pMWin->eStart==pMWin->eEnd && regCountdown + && pMWin->eFrmType==TK_RANGE && op==WINDOW_AGGINVERSE + ){ + int regRowid1 = sqlite3GetTempReg(pParse); + int regRowid2 = sqlite3GetTempReg(pParse); + sqlite3VdbeAddOp2(v, OP_Rowid, p->start.csr, regRowid1); + sqlite3VdbeAddOp2(v, OP_Rowid, p->end.csr, regRowid2); + sqlite3VdbeAddOp3(v, OP_Ge, regRowid2, lblDone, regRowid1); + VdbeCoverage(v); + sqlite3ReleaseTempReg(pParse, regRowid1); + sqlite3ReleaseTempReg(pParse, regRowid2); + assert( pMWin->eStart==TK_PRECEDING || pMWin->eStart==TK_FOLLOWING ); } - /* Check if this is the start of a new partition or peer group. */ - if( pPart || pOrderBy ){ - int nPart = (pPart ? pPart->nExpr : 0); - int addrGoto = 0; - int addrJump = 0; - int nPeer = (pOrderBy ? pOrderBy->nExpr : 0); + switch( op ){ + case WINDOW_RETURN_ROW: + csr = p->current.csr; + reg = p->current.reg; + windowReturnOneRow(p); + break; - if( pPart ){ - int regNewPart = reg + pMWin->nBufferCol; - KeyInfo *pKeyInfo = sqlite3KeyInfoFromExprList(pParse, pPart, 0, 0); - addr = sqlite3VdbeAddOp3(v, OP_Compare, regNewPart, pMWin->regPart,nPart); - sqlite3VdbeAppendP4(v, (void*)pKeyInfo, P4_KEYINFO); - addrJump = sqlite3VdbeAddOp3(v, OP_Jump, addr+2, 0, addr+2); - VdbeCoverageEqNe(v); - windowAggFinal(pParse, pMWin, 1); - if( pOrderBy ){ - addrGoto = sqlite3VdbeAddOp0(v, OP_Goto); + case WINDOW_AGGINVERSE: + csr = p->start.csr; + reg = p->start.reg; + if( pMWin->regStartRowid ){ + assert( pMWin->regEndRowid ); + sqlite3VdbeAddOp2(v, OP_AddImm, pMWin->regStartRowid, 1); + }else{ + windowAggStep(p, pMWin, csr, 1, p->regArg); } - } - - if( pOrderBy ){ - int regNewPeer = reg + pMWin->nBufferCol + nPart; - int regPeer = pMWin->regPart + nPart; + break; - if( addrJump ) sqlite3VdbeJumpHere(v, addrJump); - if( pMWin->eType==TK_RANGE ){ - KeyInfo *pKeyInfo = sqlite3KeyInfoFromExprList(pParse, pOrderBy, 0, 0); - addr = sqlite3VdbeAddOp3(v, OP_Compare, regNewPeer, regPeer, nPeer); - sqlite3VdbeAppendP4(v, (void*)pKeyInfo, P4_KEYINFO); - addrJump = sqlite3VdbeAddOp3(v, OP_Jump, addr+2, 0, addr+2); - VdbeCoverage(v); + default: + assert( op==WINDOW_AGGSTEP ); + csr = p->end.csr; + reg = p->end.reg; + if( pMWin->regStartRowid ){ + assert( pMWin->regEndRowid ); + sqlite3VdbeAddOp2(v, OP_AddImm, pMWin->regEndRowid, 1); }else{ - addrJump = 0; + windowAggStep(p, pMWin, csr, 0, p->regArg); } - windowAggFinal(pParse, pMWin, pMWin->eStart==TK_CURRENT); - if( addrGoto ) sqlite3VdbeJumpHere(v, addrGoto); - } - - sqlite3VdbeAddOp2(v, OP_Rewind, pMWin->iEphCsr,sqlite3VdbeCurrentAddr(v)+3); - VdbeCoverage(v); - sqlite3VdbeAddOp2(v, OP_Gosub, regGosub, addrGosub); - sqlite3VdbeAddOp2(v, OP_Next, pMWin->iEphCsr, sqlite3VdbeCurrentAddr(v)-1); - VdbeCoverage(v); - - sqlite3VdbeAddOp1(v, OP_ResetSorter, pMWin->iEphCsr); - sqlite3VdbeAddOp3( - v, OP_Copy, reg+pMWin->nBufferCol, pMWin->regPart, nPart+nPeer-1 - ); - - if( addrJump ) sqlite3VdbeJumpHere(v, addrJump); + break; } - /* Invoke step function for window functions */ - windowAggStep(pParse, pMWin, -1, 0, reg, 0); + if( op==p->eDelete ){ + sqlite3VdbeAddOp1(v, OP_Delete, csr); + sqlite3VdbeChangeP5(v, OPFLAG_SAVEPOSITION); + } - /* Buffer the current row in the ephemeral table. */ - if( pMWin->nBufferCol>0 ){ - sqlite3VdbeAddOp3(v, OP_MakeRecord, reg, pMWin->nBufferCol, regRecord); + if( jumpOnEof ){ + sqlite3VdbeAddOp2(v, OP_Next, csr, sqlite3VdbeCurrentAddr(v)+2); + VdbeCoverage(v); + ret = sqlite3VdbeAddOp0(v, OP_Goto); }else{ - sqlite3VdbeAddOp2(v, OP_Blob, 0, regRecord); - sqlite3VdbeAppendP4(v, (void*)"", 0); + sqlite3VdbeAddOp2(v, OP_Next, csr, sqlite3VdbeCurrentAddr(v)+1+bPeer); + VdbeCoverage(v); + if( bPeer ){ + sqlite3VdbeAddOp2(v, OP_Goto, 0, lblDone); + } } - sqlite3VdbeAddOp2(v, OP_NewRowid, pMWin->iEphCsr, regRowid); - sqlite3VdbeAddOp3(v, OP_Insert, pMWin->iEphCsr, regRecord, regRowid); - /* End the database scan loop. */ - sqlite3WhereEnd(pWInfo); + if( bPeer ){ + int nReg = (pMWin->pOrderBy ? pMWin->pOrderBy->nExpr : 0); + int regTmp = (nReg ? sqlite3GetTempRange(pParse, nReg) : 0); + windowReadPeerValues(p, csr, regTmp); + windowIfNewPeer(pParse, pMWin->pOrderBy, regTmp, reg, addrContinue); + sqlite3ReleaseTempRange(pParse, regTmp, nReg); + } - windowAggFinal(pParse, pMWin, 1); - sqlite3VdbeAddOp2(v, OP_Rewind, pMWin->iEphCsr,sqlite3VdbeCurrentAddr(v)+3); - VdbeCoverage(v); - sqlite3VdbeAddOp2(v, OP_Gosub, regGosub, addrGosub); - sqlite3VdbeAddOp2(v, OP_Next, pMWin->iEphCsr, sqlite3VdbeCurrentAddr(v)-1); - VdbeCoverage(v); + if( addrNextRange ){ + sqlite3VdbeAddOp2(v, OP_Goto, 0, addrNextRange); + } + sqlite3VdbeResolveLabel(v, lblDone); + return ret; } + /* ** Allocate and return a duplicate of the Window object indicated by the ** third argument. Set the Window.pOwner field of the new object to @@ -147195,16 +148939,20 @@ SQLITE_PRIVATE Window *sqlite3WindowDup(sqlite3 *db, Expr *pOwner, Window *p){ pNew = sqlite3DbMallocZero(db, sizeof(Window)); if( pNew ){ pNew->zName = sqlite3DbStrDup(db, p->zName); + pNew->zBase = sqlite3DbStrDup(db, p->zBase); pNew->pFilter = sqlite3ExprDup(db, p->pFilter, 0); pNew->pFunc = p->pFunc; pNew->pPartition = sqlite3ExprListDup(db, p->pPartition, 0); pNew->pOrderBy = sqlite3ExprListDup(db, p->pOrderBy, 0); - pNew->eType = p->eType; + pNew->eFrmType = p->eFrmType; pNew->eEnd = p->eEnd; pNew->eStart = p->eStart; + pNew->eExclude = p->eExclude; + pNew->regResult = p->regResult; pNew->pStart = sqlite3ExprDup(db, p->pStart, 0); pNew->pEnd = sqlite3ExprDup(db, p->pEnd, 0); pNew->pOwner = pOwner; + pNew->bImplicitFrame = p->bImplicitFrame; } } return pNew; @@ -147228,12 +148976,360 @@ SQLITE_PRIVATE Window *sqlite3WindowListDup(sqlite3 *db, Window *p){ return pRet; } +/* +** Return true if it can be determined at compile time that expression +** pExpr evaluates to a value that, when cast to an integer, is greater +** than zero. False otherwise. +** +** If an OOM error occurs, this function sets the Parse.db.mallocFailed +** flag and returns zero. +*/ +static int windowExprGtZero(Parse *pParse, Expr *pExpr){ + int ret = 0; + sqlite3 *db = pParse->db; + sqlite3_value *pVal = 0; + sqlite3ValueFromExpr(db, pExpr, db->enc, SQLITE_AFF_NUMERIC, &pVal); + if( pVal && sqlite3_value_int(pVal)>0 ){ + ret = 1; + } + sqlite3ValueFree(pVal); + return ret; +} + /* ** sqlite3WhereBegin() has already been called for the SELECT statement ** passed as the second argument when this function is invoked. It generates -** code to populate the Window.regResult register for each window function and -** invoke the sub-routine at instruction addrGosub once for each row. -** This function calls sqlite3WhereEnd() before returning. +** code to populate the Window.regResult register for each window function +** and invoke the sub-routine at instruction addrGosub once for each row. +** sqlite3WhereEnd() is always called before returning. +** +** This function handles several different types of window frames, which +** require slightly different processing. The following pseudo code is +** used to implement window frames of the form: +** +** ROWS BETWEEN PRECEDING AND FOLLOWING +** +** Other window frame types use variants of the following: +** +** ... loop started by sqlite3WhereBegin() ... +** if( new partition ){ +** Gosub flush +** } +** Insert new row into eph table. +** +** if( first row of partition ){ +** // Rewind three cursors, all open on the eph table. +** Rewind(csrEnd); +** Rewind(csrStart); +** Rewind(csrCurrent); +** +** regEnd = // FOLLOWING expression +** regStart = // PRECEDING expression +** }else{ +** // First time this branch is taken, the eph table contains two +** // rows. The first row in the partition, which all three cursors +** // currently point to, and the following row. +** AGGSTEP +** if( (regEnd--)<=0 ){ +** RETURN_ROW +** if( (regStart--)<=0 ){ +** AGGINVERSE +** } +** } +** } +** } +** flush: +** AGGSTEP +** while( 1 ){ +** RETURN ROW +** if( csrCurrent is EOF ) break; +** if( (regStart--)<=0 ){ +** AggInverse(csrStart) +** Next(csrStart) +** } +** } +** +** The pseudo-code above uses the following shorthand: +** +** AGGSTEP: invoke the aggregate xStep() function for each window function +** with arguments read from the current row of cursor csrEnd, then +** step cursor csrEnd forward one row (i.e. sqlite3BtreeNext()). +** +** RETURN_ROW: return a row to the caller based on the contents of the +** current row of csrCurrent and the current state of all +** aggregates. Then step cursor csrCurrent forward one row. +** +** AGGINVERSE: invoke the aggregate xInverse() function for each window +** functions with arguments read from the current row of cursor +** csrStart. Then step csrStart forward one row. +** +** There are two other ROWS window frames that are handled significantly +** differently from the above - "BETWEEN PRECEDING AND PRECEDING" +** and "BETWEEN FOLLOWING AND FOLLOWING". These are special +** cases because they change the order in which the three cursors (csrStart, +** csrCurrent and csrEnd) iterate through the ephemeral table. Cases that +** use UNBOUNDED or CURRENT ROW are much simpler variations on one of these +** three. +** +** ROWS BETWEEN PRECEDING AND PRECEDING +** +** ... loop started by sqlite3WhereBegin() ... +** if( new partition ){ +** Gosub flush +** } +** Insert new row into eph table. +** if( first row of partition ){ +** Rewind(csrEnd) ; Rewind(csrStart) ; Rewind(csrCurrent) +** regEnd = +** regStart = +** }else{ +** if( (regEnd--)<=0 ){ +** AGGSTEP +** } +** RETURN_ROW +** if( (regStart--)<=0 ){ +** AGGINVERSE +** } +** } +** } +** flush: +** if( (regEnd--)<=0 ){ +** AGGSTEP +** } +** RETURN_ROW +** +** +** ROWS BETWEEN FOLLOWING AND FOLLOWING +** +** ... loop started by sqlite3WhereBegin() ... +** if( new partition ){ +** Gosub flush +** } +** Insert new row into eph table. +** if( first row of partition ){ +** Rewind(csrEnd) ; Rewind(csrStart) ; Rewind(csrCurrent) +** regEnd = +** regStart = regEnd - +** }else{ +** AGGSTEP +** if( (regEnd--)<=0 ){ +** RETURN_ROW +** } +** if( (regStart--)<=0 ){ +** AGGINVERSE +** } +** } +** } +** flush: +** AGGSTEP +** while( 1 ){ +** if( (regEnd--)<=0 ){ +** RETURN_ROW +** if( eof ) break; +** } +** if( (regStart--)<=0 ){ +** AGGINVERSE +** if( eof ) break +** } +** } +** while( !eof csrCurrent ){ +** RETURN_ROW +** } +** +** For the most part, the patterns above are adapted to support UNBOUNDED by +** assuming that it is equivalent to "infinity PRECEDING/FOLLOWING" and +** CURRENT ROW by assuming that it is equivilent to "0 PRECEDING/FOLLOWING". +** This is optimized of course - branches that will never be taken and +** conditions that are always true are omitted from the VM code. The only +** exceptional case is: +** +** ROWS BETWEEN FOLLOWING AND UNBOUNDED FOLLOWING +** +** ... loop started by sqlite3WhereBegin() ... +** if( new partition ){ +** Gosub flush +** } +** Insert new row into eph table. +** if( first row of partition ){ +** Rewind(csrEnd) ; Rewind(csrStart) ; Rewind(csrCurrent) +** regStart = +** }else{ +** AGGSTEP +** } +** } +** flush: +** AGGSTEP +** while( 1 ){ +** if( (regStart--)<=0 ){ +** AGGINVERSE +** if( eof ) break +** } +** RETURN_ROW +** } +** while( !eof csrCurrent ){ +** RETURN_ROW +** } +** +** Also requiring special handling are the cases: +** +** ROWS BETWEEN PRECEDING AND PRECEDING +** ROWS BETWEEN FOLLOWING AND FOLLOWING +** +** when (expr1 < expr2). This is detected at runtime, not by this function. +** To handle this case, the pseudo-code programs depicted above are modified +** slightly to be: +** +** ... loop started by sqlite3WhereBegin() ... +** if( new partition ){ +** Gosub flush +** } +** Insert new row into eph table. +** if( first row of partition ){ +** Rewind(csrEnd) ; Rewind(csrStart) ; Rewind(csrCurrent) +** regEnd = +** regStart = +** if( regEnd < regStart ){ +** RETURN_ROW +** delete eph table contents +** continue +** } +** ... +** +** The new "continue" statement in the above jumps to the next iteration +** of the outer loop - the one started by sqlite3WhereBegin(). +** +** The various GROUPS cases are implemented using the same patterns as +** ROWS. The VM code is modified slightly so that: +** +** 1. The else branch in the main loop is only taken if the row just +** added to the ephemeral table is the start of a new group. In +** other words, it becomes: +** +** ... loop started by sqlite3WhereBegin() ... +** if( new partition ){ +** Gosub flush +** } +** Insert new row into eph table. +** if( first row of partition ){ +** Rewind(csrEnd) ; Rewind(csrStart) ; Rewind(csrCurrent) +** regEnd = +** regStart = +** }else if( new group ){ +** ... +** } +** } +** +** 2. Instead of processing a single row, each RETURN_ROW, AGGSTEP or +** AGGINVERSE step processes the current row of the relevant cursor and +** all subsequent rows belonging to the same group. +** +** RANGE window frames are a little different again. As for GROUPS, the +** main loop runs once per group only. And RETURN_ROW, AGGSTEP and AGGINVERSE +** deal in groups instead of rows. As for ROWS and GROUPS, there are three +** basic cases: +** +** RANGE BETWEEN PRECEDING AND FOLLOWING +** +** ... loop started by sqlite3WhereBegin() ... +** if( new partition ){ +** Gosub flush +** } +** Insert new row into eph table. +** if( first row of partition ){ +** Rewind(csrEnd) ; Rewind(csrStart) ; Rewind(csrCurrent) +** regEnd = +** regStart = +** }else{ +** AGGSTEP +** while( (csrCurrent.key + regEnd) < csrEnd.key ){ +** RETURN_ROW +** while( csrStart.key + regStart) < csrCurrent.key ){ +** AGGINVERSE +** } +** } +** } +** } +** flush: +** AGGSTEP +** while( 1 ){ +** RETURN ROW +** if( csrCurrent is EOF ) break; +** while( csrStart.key + regStart) < csrCurrent.key ){ +** AGGINVERSE +** } +** } +** } +** +** In the above notation, "csr.key" means the current value of the ORDER BY +** expression (there is only ever 1 for a RANGE that uses an FOLLOWING +** or PRECEDING AND PRECEDING +** +** ... loop started by sqlite3WhereBegin() ... +** if( new partition ){ +** Gosub flush +** } +** Insert new row into eph table. +** if( first row of partition ){ +** Rewind(csrEnd) ; Rewind(csrStart) ; Rewind(csrCurrent) +** regEnd = +** regStart = +** }else{ +** while( (csrEnd.key + regEnd) <= csrCurrent.key ){ +** AGGSTEP +** } +** while( (csrStart.key + regStart) < csrCurrent.key ){ +** AGGINVERSE +** } +** RETURN_ROW +** } +** } +** flush: +** while( (csrEnd.key + regEnd) <= csrCurrent.key ){ +** AGGSTEP +** } +** while( (csrStart.key + regStart) < csrCurrent.key ){ +** AGGINVERSE +** } +** RETURN_ROW +** +** RANGE BETWEEN FOLLOWING AND FOLLOWING +** +** ... loop started by sqlite3WhereBegin() ... +** if( new partition ){ +** Gosub flush +** } +** Insert new row into eph table. +** if( first row of partition ){ +** Rewind(csrEnd) ; Rewind(csrStart) ; Rewind(csrCurrent) +** regEnd = +** regStart = +** }else{ +** AGGSTEP +** while( (csrCurrent.key + regEnd) < csrEnd.key ){ +** while( (csrCurrent.key + regStart) > csrStart.key ){ +** AGGINVERSE +** } +** RETURN_ROW +** } +** } +** } +** flush: +** AGGSTEP +** while( 1 ){ +** while( (csrCurrent.key + regStart) > csrStart.key ){ +** AGGINVERSE +** if( eof ) break "while( 1 )" loop. +** } +** RETURN_ROW +** } +** while( !eof csrCurrent ){ +** RETURN_ROW +** } +** +** The text above leaves out many details. Refer to the code and comments +** below for a more complete picture. */ SQLITE_PRIVATE void sqlite3WindowCodeStep( Parse *pParse, /* Parse context */ @@ -147243,75 +149339,321 @@ SQLITE_PRIVATE void sqlite3WindowCodeStep( int addrGosub /* OP_Gosub here to return each row */ ){ Window *pMWin = p->pWin; + ExprList *pOrderBy = pMWin->pOrderBy; + Vdbe *v = sqlite3GetVdbe(pParse); + int csrWrite; /* Cursor used to write to eph. table */ + int csrInput = p->pSrc->a[0].iCursor; /* Cursor of sub-select */ + int nInput = p->pSrc->a[0].pTab->nCol; /* Number of cols returned by sub */ + int iInput; /* To iterate through sub cols */ + int addrNe; /* Address of OP_Ne */ + int addrGosubFlush = 0; /* Address of OP_Gosub to flush: */ + int addrInteger = 0; /* Address of OP_Integer */ + int addrEmpty; /* Address of OP_Rewind in flush: */ + int regNew; /* Array of registers holding new input row */ + int regRecord; /* regNew array in record form */ + int regRowid; /* Rowid for regRecord in eph table */ + int regNewPeer = 0; /* Peer values for new row (part of regNew) */ + int regPeer = 0; /* Peer values for current row */ + int regFlushPart = 0; /* Register for "Gosub flush_partition" */ + WindowCodeArg s; /* Context object for sub-routines */ + int lblWhereEnd; /* Label just before sqlite3WhereEnd() code */ + int regStart = 0; /* Value of PRECEDING */ + int regEnd = 0; /* Value of FOLLOWING */ + + assert( pMWin->eStart==TK_PRECEDING || pMWin->eStart==TK_CURRENT + || pMWin->eStart==TK_FOLLOWING || pMWin->eStart==TK_UNBOUNDED + ); + assert( pMWin->eEnd==TK_FOLLOWING || pMWin->eEnd==TK_CURRENT + || pMWin->eEnd==TK_UNBOUNDED || pMWin->eEnd==TK_PRECEDING + ); + assert( pMWin->eExclude==0 || pMWin->eExclude==TK_CURRENT + || pMWin->eExclude==TK_GROUP || pMWin->eExclude==TK_TIES + || pMWin->eExclude==TK_NO + ); - /* There are three different functions that may be used to do the work - ** of this one, depending on the window frame and the specific built-in - ** window functions used (if any). - ** - ** windowCodeRowExprStep() handles all "ROWS" window frames, except for: - ** - ** ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW - ** - ** The exception is because windowCodeRowExprStep() implements all window - ** frame types by caching the entire partition in a temp table, and - ** "ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW" is easy enough to - ** implement without such a cache. - ** - ** windowCodeCacheStep() is used for: - ** - ** RANGE BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING - ** - ** It is also used for anything not handled by windowCodeRowExprStep() - ** that invokes a built-in window function that requires the entire - ** partition to be cached in a temp table before any rows are returned - ** (e.g. nth_value() or percent_rank()). - ** - ** Finally, assuming there is no built-in window function that requires - ** the partition to be cached, windowCodeDefaultStep() is used for: - ** - ** RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW - ** RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING - ** RANGE BETWEEN CURRENT ROW AND CURRENT ROW - ** ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW - ** - ** windowCodeDefaultStep() is the only one of the three functions that - ** does not cache each partition in a temp table before beginning to - ** return rows. - */ - if( pMWin->eType==TK_ROWS - && (pMWin->eStart!=TK_UNBOUNDED||pMWin->eEnd!=TK_CURRENT||!pMWin->pOrderBy) - ){ - VdbeModuleComment((pParse->pVdbe, "Begin RowExprStep()")); - windowCodeRowExprStep(pParse, p, pWInfo, regGosub, addrGosub); - }else{ - Window *pWin; - int bCache = 0; /* True to use CacheStep() */ - - if( pMWin->eStart==TK_CURRENT && pMWin->eEnd==TK_UNBOUNDED ){ - bCache = 1; - }else{ - for(pWin=pMWin; pWin; pWin=pWin->pNextWin){ - FuncDef *pFunc = pWin->pFunc; - if( (pFunc->funcFlags & SQLITE_FUNC_WINDOW_SIZE) - || (pFunc->zName==nth_valueName) - || (pFunc->zName==first_valueName) - || (pFunc->zName==leadName) - || (pFunc->zName==lagName) - ){ - bCache = 1; - break; + lblWhereEnd = sqlite3VdbeMakeLabel(pParse); + + /* Fill in the context object */ + memset(&s, 0, sizeof(WindowCodeArg)); + s.pParse = pParse; + s.pMWin = pMWin; + s.pVdbe = v; + s.regGosub = regGosub; + s.addrGosub = addrGosub; + s.current.csr = pMWin->iEphCsr; + csrWrite = s.current.csr+1; + s.start.csr = s.current.csr+2; + s.end.csr = s.current.csr+3; + + /* Figure out when rows may be deleted from the ephemeral table. There + ** are four options - they may never be deleted (eDelete==0), they may + ** be deleted as soon as they are no longer part of the window frame + ** (eDelete==WINDOW_AGGINVERSE), they may be deleted as after the row + ** has been returned to the caller (WINDOW_RETURN_ROW), or they may + ** be deleted after they enter the frame (WINDOW_AGGSTEP). */ + switch( pMWin->eStart ){ + case TK_FOLLOWING: + if( pMWin->eFrmType!=TK_RANGE + && windowExprGtZero(pParse, pMWin->pStart) + ){ + s.eDelete = WINDOW_RETURN_ROW; + } + break; + case TK_UNBOUNDED: + if( windowCacheFrame(pMWin)==0 ){ + if( pMWin->eEnd==TK_PRECEDING ){ + if( pMWin->eFrmType!=TK_RANGE + && windowExprGtZero(pParse, pMWin->pEnd) + ){ + s.eDelete = WINDOW_AGGSTEP; + } + }else{ + s.eDelete = WINDOW_RETURN_ROW; + } + } + break; + default: + s.eDelete = WINDOW_AGGINVERSE; + break; + } + + /* Allocate registers for the array of values from the sub-query, the + ** samve values in record form, and the rowid used to insert said record + ** into the ephemeral table. */ + regNew = pParse->nMem+1; + pParse->nMem += nInput; + regRecord = ++pParse->nMem; + regRowid = ++pParse->nMem; + + /* If the window frame contains an " PRECEDING" or " FOLLOWING" + ** clause, allocate registers to store the results of evaluating each + ** . */ + if( pMWin->eStart==TK_PRECEDING || pMWin->eStart==TK_FOLLOWING ){ + regStart = ++pParse->nMem; + } + if( pMWin->eEnd==TK_PRECEDING || pMWin->eEnd==TK_FOLLOWING ){ + regEnd = ++pParse->nMem; + } + + /* If this is not a "ROWS BETWEEN ..." frame, then allocate arrays of + ** registers to store copies of the ORDER BY expressions (peer values) + ** for the main loop, and for each cursor (start, current and end). */ + if( pMWin->eFrmType!=TK_ROWS ){ + int nPeer = (pOrderBy ? pOrderBy->nExpr : 0); + regNewPeer = regNew + pMWin->nBufferCol; + if( pMWin->pPartition ) regNewPeer += pMWin->pPartition->nExpr; + regPeer = pParse->nMem+1; pParse->nMem += nPeer; + s.start.reg = pParse->nMem+1; pParse->nMem += nPeer; + s.current.reg = pParse->nMem+1; pParse->nMem += nPeer; + s.end.reg = pParse->nMem+1; pParse->nMem += nPeer; + } + + /* Load the column values for the row returned by the sub-select + ** into an array of registers starting at regNew. Assemble them into + ** a record in register regRecord. */ + for(iInput=0; iInputpPartition ){ + int addr; + ExprList *pPart = pMWin->pPartition; + int nPart = pPart->nExpr; + int regNewPart = regNew + pMWin->nBufferCol; + KeyInfo *pKeyInfo = sqlite3KeyInfoFromExprList(pParse, pPart, 0, 0); + + regFlushPart = ++pParse->nMem; + addr = sqlite3VdbeAddOp3(v, OP_Compare, regNewPart, pMWin->regPart, nPart); + sqlite3VdbeAppendP4(v, (void*)pKeyInfo, P4_KEYINFO); + sqlite3VdbeAddOp3(v, OP_Jump, addr+2, addr+4, addr+2); + VdbeCoverageEqNe(v); + addrGosubFlush = sqlite3VdbeAddOp1(v, OP_Gosub, regFlushPart); + VdbeComment((v, "call flush_partition")); + sqlite3VdbeAddOp3(v, OP_Copy, regNewPart, pMWin->regPart, nPart-1); + } + + /* Insert the new row into the ephemeral table */ + sqlite3VdbeAddOp2(v, OP_NewRowid, csrWrite, regRowid); + sqlite3VdbeAddOp3(v, OP_Insert, csrWrite, regRecord, regRowid); + addrNe = sqlite3VdbeAddOp3(v, OP_Ne, pMWin->regOne, 0, regRowid); + VdbeCoverageNeverNull(v); + + /* This block is run for the first row of each partition */ + s.regArg = windowInitAccum(pParse, pMWin); + + if( regStart ){ + sqlite3ExprCode(pParse, pMWin->pStart, regStart); + windowCheckValue(pParse, regStart, 0 + (pMWin->eFrmType==TK_RANGE?3:0)); + } + if( regEnd ){ + sqlite3ExprCode(pParse, pMWin->pEnd, regEnd); + windowCheckValue(pParse, regEnd, 1 + (pMWin->eFrmType==TK_RANGE?3:0)); + } + + if( pMWin->eFrmType!=TK_RANGE && pMWin->eStart==pMWin->eEnd && regStart ){ + int op = ((pMWin->eStart==TK_FOLLOWING) ? OP_Ge : OP_Le); + int addrGe = sqlite3VdbeAddOp3(v, op, regStart, 0, regEnd); + VdbeCoverageNeverNullIf(v, op==OP_Ge); /* NeverNull because bound */ + VdbeCoverageNeverNullIf(v, op==OP_Le); /* values previously checked */ + windowAggFinal(&s, 0); + sqlite3VdbeAddOp2(v, OP_Rewind, s.current.csr, 1); + VdbeCoverageNeverTaken(v); + windowReturnOneRow(&s); + sqlite3VdbeAddOp1(v, OP_ResetSorter, s.current.csr); + sqlite3VdbeAddOp2(v, OP_Goto, 0, lblWhereEnd); + sqlite3VdbeJumpHere(v, addrGe); + } + if( pMWin->eStart==TK_FOLLOWING && pMWin->eFrmType!=TK_RANGE && regEnd ){ + assert( pMWin->eEnd==TK_FOLLOWING ); + sqlite3VdbeAddOp3(v, OP_Subtract, regStart, regEnd, regStart); + } + + if( pMWin->eStart!=TK_UNBOUNDED ){ + sqlite3VdbeAddOp2(v, OP_Rewind, s.start.csr, 1); + VdbeCoverageNeverTaken(v); + } + sqlite3VdbeAddOp2(v, OP_Rewind, s.current.csr, 1); + VdbeCoverageNeverTaken(v); + sqlite3VdbeAddOp2(v, OP_Rewind, s.end.csr, 1); + VdbeCoverageNeverTaken(v); + if( regPeer && pOrderBy ){ + sqlite3VdbeAddOp3(v, OP_Copy, regNewPeer, regPeer, pOrderBy->nExpr-1); + sqlite3VdbeAddOp3(v, OP_Copy, regPeer, s.start.reg, pOrderBy->nExpr-1); + sqlite3VdbeAddOp3(v, OP_Copy, regPeer, s.current.reg, pOrderBy->nExpr-1); + sqlite3VdbeAddOp3(v, OP_Copy, regPeer, s.end.reg, pOrderBy->nExpr-1); + } + + sqlite3VdbeAddOp2(v, OP_Goto, 0, lblWhereEnd); + + sqlite3VdbeJumpHere(v, addrNe); + + /* Beginning of the block executed for the second and subsequent rows. */ + if( regPeer ){ + windowIfNewPeer(pParse, pOrderBy, regNewPeer, regPeer, lblWhereEnd); + } + if( pMWin->eStart==TK_FOLLOWING ){ + windowCodeOp(&s, WINDOW_AGGSTEP, 0, 0); + if( pMWin->eEnd!=TK_UNBOUNDED ){ + if( pMWin->eFrmType==TK_RANGE ){ + int lbl = sqlite3VdbeMakeLabel(pParse); + int addrNext = sqlite3VdbeCurrentAddr(v); + windowCodeRangeTest(&s, OP_Ge, s.current.csr, regEnd, s.end.csr, lbl); + windowCodeOp(&s, WINDOW_AGGINVERSE, regStart, 0); + windowCodeOp(&s, WINDOW_RETURN_ROW, 0, 0); + sqlite3VdbeAddOp2(v, OP_Goto, 0, addrNext); + sqlite3VdbeResolveLabel(v, lbl); + }else{ + windowCodeOp(&s, WINDOW_RETURN_ROW, regEnd, 0); + windowCodeOp(&s, WINDOW_AGGINVERSE, regStart, 0); + } + } + }else + if( pMWin->eEnd==TK_PRECEDING ){ + int bRPS = (pMWin->eStart==TK_PRECEDING && pMWin->eFrmType==TK_RANGE); + windowCodeOp(&s, WINDOW_AGGSTEP, regEnd, 0); + if( bRPS ) windowCodeOp(&s, WINDOW_AGGINVERSE, regStart, 0); + windowCodeOp(&s, WINDOW_RETURN_ROW, 0, 0); + if( !bRPS ) windowCodeOp(&s, WINDOW_AGGINVERSE, regStart, 0); + }else{ + int addr = 0; + windowCodeOp(&s, WINDOW_AGGSTEP, 0, 0); + if( pMWin->eEnd!=TK_UNBOUNDED ){ + if( pMWin->eFrmType==TK_RANGE ){ + int lbl = 0; + addr = sqlite3VdbeCurrentAddr(v); + if( regEnd ){ + lbl = sqlite3VdbeMakeLabel(pParse); + windowCodeRangeTest(&s, OP_Ge, s.current.csr, regEnd, s.end.csr, lbl); + } + windowCodeOp(&s, WINDOW_RETURN_ROW, 0, 0); + windowCodeOp(&s, WINDOW_AGGINVERSE, regStart, 0); + if( regEnd ){ + sqlite3VdbeAddOp2(v, OP_Goto, 0, addr); + sqlite3VdbeResolveLabel(v, lbl); + } + }else{ + if( regEnd ){ + addr = sqlite3VdbeAddOp3(v, OP_IfPos, regEnd, 0, 1); + VdbeCoverage(v); } + windowCodeOp(&s, WINDOW_RETURN_ROW, 0, 0); + windowCodeOp(&s, WINDOW_AGGINVERSE, regStart, 0); + if( regEnd ) sqlite3VdbeJumpHere(v, addr); } } + } - /* Otherwise, call windowCodeDefaultStep(). */ - if( bCache ){ - VdbeModuleComment((pParse->pVdbe, "Begin CacheStep()")); - windowCodeCacheStep(pParse, p, pWInfo, regGosub, addrGosub); - }else{ - VdbeModuleComment((pParse->pVdbe, "Begin DefaultStep()")); - windowCodeDefaultStep(pParse, p, pWInfo, regGosub, addrGosub); + /* End of the main input loop */ + sqlite3VdbeResolveLabel(v, lblWhereEnd); + sqlite3WhereEnd(pWInfo); + + /* Fall through */ + if( pMWin->pPartition ){ + addrInteger = sqlite3VdbeAddOp2(v, OP_Integer, 0, regFlushPart); + sqlite3VdbeJumpHere(v, addrGosubFlush); + } + + addrEmpty = sqlite3VdbeAddOp1(v, OP_Rewind, csrWrite); + VdbeCoverage(v); + if( pMWin->eEnd==TK_PRECEDING ){ + int bRPS = (pMWin->eStart==TK_PRECEDING && pMWin->eFrmType==TK_RANGE); + windowCodeOp(&s, WINDOW_AGGSTEP, regEnd, 0); + if( bRPS ) windowCodeOp(&s, WINDOW_AGGINVERSE, regStart, 0); + windowCodeOp(&s, WINDOW_RETURN_ROW, 0, 0); + }else if( pMWin->eStart==TK_FOLLOWING ){ + int addrStart; + int addrBreak1; + int addrBreak2; + int addrBreak3; + windowCodeOp(&s, WINDOW_AGGSTEP, 0, 0); + if( pMWin->eFrmType==TK_RANGE ){ + addrStart = sqlite3VdbeCurrentAddr(v); + addrBreak2 = windowCodeOp(&s, WINDOW_AGGINVERSE, regStart, 1); + addrBreak1 = windowCodeOp(&s, WINDOW_RETURN_ROW, 0, 1); + }else + if( pMWin->eEnd==TK_UNBOUNDED ){ + addrStart = sqlite3VdbeCurrentAddr(v); + addrBreak1 = windowCodeOp(&s, WINDOW_RETURN_ROW, regStart, 1); + addrBreak2 = windowCodeOp(&s, WINDOW_AGGINVERSE, 0, 1); + }else{ + assert( pMWin->eEnd==TK_FOLLOWING ); + addrStart = sqlite3VdbeCurrentAddr(v); + addrBreak1 = windowCodeOp(&s, WINDOW_RETURN_ROW, regEnd, 1); + addrBreak2 = windowCodeOp(&s, WINDOW_AGGINVERSE, regStart, 1); + } + sqlite3VdbeAddOp2(v, OP_Goto, 0, addrStart); + sqlite3VdbeJumpHere(v, addrBreak2); + addrStart = sqlite3VdbeCurrentAddr(v); + addrBreak3 = windowCodeOp(&s, WINDOW_RETURN_ROW, 0, 1); + sqlite3VdbeAddOp2(v, OP_Goto, 0, addrStart); + sqlite3VdbeJumpHere(v, addrBreak1); + sqlite3VdbeJumpHere(v, addrBreak3); + }else{ + int addrBreak; + int addrStart; + windowCodeOp(&s, WINDOW_AGGSTEP, 0, 0); + addrStart = sqlite3VdbeCurrentAddr(v); + addrBreak = windowCodeOp(&s, WINDOW_RETURN_ROW, 0, 1); + windowCodeOp(&s, WINDOW_AGGINVERSE, regStart, 0); + sqlite3VdbeAddOp2(v, OP_Goto, 0, addrStart); + sqlite3VdbeJumpHere(v, addrBreak); + } + sqlite3VdbeJumpHere(v, addrEmpty); + + sqlite3VdbeAddOp1(v, OP_ResetSorter, s.current.csr); + if( pMWin->pPartition ){ + if( pMWin->regStartRowid ){ + sqlite3VdbeAddOp2(v, OP_Integer, 1, pMWin->regStartRowid); + sqlite3VdbeAddOp2(v, OP_Integer, 0, pMWin->regEndRowid); } + sqlite3VdbeChangeP1(v, addrInteger, sqlite3VdbeCurrentAddr(v)); + sqlite3VdbeAddOp1(v, OP_Return, regFlushPart); } } @@ -147414,6 +149756,7 @@ static void disableLookaside(Parse *pParse){ ** SQLITE_LIMIT_COMPOUND_SELECT. */ static void parserDoubleLinkSelect(Parse *pParse, Select *p){ + assert( p!=0 ); if( p->pPrior ){ Select *pNext = 0, *pLoop; int mxSelect, cnt = 0; @@ -147440,7 +149783,7 @@ static void disableLookaside(Parse *pParse){ if( p ){ /* memset(p, 0, sizeof(Expr)); */ p->op = (u8)op; - p->affinity = 0; + p->affExpr = 0; p->flags = EP_Leaf; p->iAgg = -1; p->pLeft = p->pRight = 0; @@ -147500,6 +149843,10 @@ static void disableLookaside(Parse *pParse){ sqlite3ExprListSetName(pParse, p, pIdToken, 1); return p; } + +#if TK_SPAN>255 +# error too many tokens in the grammar +#endif /**************** End of %include directives **********************************/ /* These constants specify the various numeric values for terminal symbols ** in a format understandable to "makeheaders". This section is blank unless @@ -147563,27 +149910,28 @@ static void disableLookaside(Parse *pParse){ #endif /************* Begin control #defines *****************************************/ #define YYCODETYPE unsigned short int -#define YYNOCODE 278 +#define YYNOCODE 307 #define YYACTIONTYPE unsigned short int -#define YYWILDCARD 91 +#define YYWILDCARD 98 #define sqlite3ParserTOKENTYPE Token typedef union { int yyinit; sqlite3ParserTOKENTYPE yy0; - ExprList* yy42; - int yy96; - TriggerStep* yy119; - Window* yy147; - SrcList* yy167; - Upsert* yy266; - struct FrameBound yy317; - IdList* yy336; - struct TrigEvent yy350; - struct {int value; int mask;} yy367; - Select* yy423; - const char* yy464; - Expr* yy490; - With* yy499; + const char* yy8; + Select* yy25; + int yy32; + Expr* yy46; + struct FrameBound yy57; + u8 yy118; + ExprList* yy138; + Upsert* yy288; + With* yy297; + IdList* yy406; + Window* yy455; + struct {int value; int mask;} yy495; + TriggerStep* yy527; + struct TrigEvent yy572; + SrcList* yy609; } YYMINORTYPE; #ifndef YYSTACKDEPTH #define YYSTACKDEPTH 100 @@ -147599,17 +149947,17 @@ typedef union { #define sqlite3ParserCTX_FETCH Parse *pParse=yypParser->pParse; #define sqlite3ParserCTX_STORE yypParser->pParse=pParse; #define YYFALLBACK 1 -#define YYNSTATE 524 -#define YYNRULE 369 -#define YYNTOKEN 155 -#define YY_MAX_SHIFT 523 -#define YY_MIN_SHIFTREDUCE 760 -#define YY_MAX_SHIFTREDUCE 1128 -#define YY_ERROR_ACTION 1129 -#define YY_ACCEPT_ACTION 1130 -#define YY_NO_ACTION 1131 -#define YY_MIN_REDUCE 1132 -#define YY_MAX_REDUCE 1500 +#define YYNSTATE 543 +#define YYNRULE 381 +#define YYNTOKEN 179 +#define YY_MAX_SHIFT 542 +#define YY_MIN_SHIFTREDUCE 790 +#define YY_MAX_SHIFTREDUCE 1170 +#define YY_ERROR_ACTION 1171 +#define YY_ACCEPT_ACTION 1172 +#define YY_NO_ACTION 1173 +#define YY_MIN_REDUCE 1174 +#define YY_MAX_REDUCE 1554 /************* End control #defines *******************************************/ #define YY_NLOOKAHEAD ((int)(sizeof(yy_lookahead)/sizeof(yy_lookahead[0]))) @@ -147676,569 +150024,573 @@ typedef union { ** yy_default[] Default action for each state. ** *********** Begin parsing tables **********************************************/ -#define YY_ACTTAB_COUNT (2009) +#define YY_ACTTAB_COUNT (1913) static const YYACTIONTYPE yy_action[] = { - /* 0 */ 377, 518, 371, 107, 104, 200, 1293, 518, 1130, 1, - /* 10 */ 1, 523, 2, 1134, 518, 1203, 1203, 1262, 277, 373, - /* 20 */ 129, 495, 37, 37, 1397, 1201, 1201, 1211, 65, 65, - /* 30 */ 480, 891, 107, 104, 200, 37, 37, 1043, 1494, 892, - /* 40 */ 346, 1494, 342, 114, 115, 105, 1106, 1106, 957, 960, - /* 50 */ 950, 950, 112, 112, 113, 113, 113, 113, 285, 254, - /* 60 */ 254, 518, 254, 254, 500, 518, 495, 518, 107, 104, - /* 70 */ 200, 1085, 515, 481, 386, 515, 1464, 442, 501, 230, - /* 80 */ 197, 439, 37, 37, 1172, 210, 65, 65, 65, 65, - /* 90 */ 254, 254, 111, 111, 111, 111, 110, 110, 109, 109, - /* 100 */ 109, 108, 404, 515, 404, 155, 1041, 431, 401, 400, - /* 110 */ 254, 254, 373, 1431, 1427, 408, 1110, 1085, 1086, 1087, - /* 120 */ 284, 1112, 500, 515, 500, 368, 1433, 1421, 1428, 1111, - /* 130 */ 1261, 499, 373, 502, 108, 404, 114, 115, 105, 1106, - /* 140 */ 1106, 957, 960, 950, 950, 112, 112, 113, 113, 113, - /* 150 */ 113, 276, 509, 1113, 369, 1113, 114, 115, 105, 1106, - /* 160 */ 1106, 957, 960, 950, 950, 112, 112, 113, 113, 113, - /* 170 */ 113, 496, 1420, 1431, 493, 1468, 1065, 260, 1063, 433, - /* 180 */ 74, 107, 104, 200, 498, 111, 111, 111, 111, 110, - /* 190 */ 110, 109, 109, 109, 108, 404, 373, 113, 113, 113, - /* 200 */ 113, 106, 131, 91, 1361, 111, 111, 111, 111, 110, - /* 210 */ 110, 109, 109, 109, 108, 404, 113, 113, 113, 113, - /* 220 */ 114, 115, 105, 1106, 1106, 957, 960, 950, 950, 112, - /* 230 */ 112, 113, 113, 113, 113, 111, 111, 111, 111, 110, - /* 240 */ 110, 109, 109, 109, 108, 404, 116, 110, 110, 109, - /* 250 */ 109, 109, 108, 404, 111, 111, 111, 111, 110, 110, - /* 260 */ 109, 109, 109, 108, 404, 917, 512, 512, 512, 111, - /* 270 */ 111, 111, 111, 110, 110, 109, 109, 109, 108, 404, - /* 280 */ 517, 1198, 1177, 181, 109, 109, 109, 108, 404, 373, - /* 290 */ 1198, 402, 402, 402, 75, 360, 111, 111, 111, 111, - /* 300 */ 110, 110, 109, 109, 109, 108, 404, 382, 299, 419, - /* 310 */ 287, 170, 518, 114, 115, 105, 1106, 1106, 957, 960, - /* 320 */ 950, 950, 112, 112, 113, 113, 113, 113, 1444, 523, - /* 330 */ 2, 1134, 518, 13, 13, 337, 277, 1085, 129, 226, - /* 340 */ 937, 1058, 1000, 471, 917, 1211, 453, 384, 1085, 395, - /* 350 */ 162, 1057, 155, 45, 45, 416, 928, 401, 400, 479, - /* 360 */ 927, 12, 111, 111, 111, 111, 110, 110, 109, 109, - /* 370 */ 109, 108, 404, 226, 286, 254, 254, 254, 254, 518, - /* 380 */ 16, 16, 373, 1085, 1086, 1087, 314, 299, 515, 472, - /* 390 */ 515, 927, 927, 929, 1085, 1086, 1087, 378, 276, 509, - /* 400 */ 65, 65, 1113, 210, 1113, 1085, 114, 115, 105, 1106, - /* 410 */ 1106, 957, 960, 950, 950, 112, 112, 113, 113, 113, - /* 420 */ 113, 1448, 222, 1134, 1089, 461, 458, 457, 277, 180, - /* 430 */ 129, 378, 392, 408, 423, 456, 500, 1211, 240, 257, - /* 440 */ 324, 464, 319, 463, 227, 470, 12, 317, 424, 300, - /* 450 */ 317, 1085, 1086, 1087, 485, 111, 111, 111, 111, 110, - /* 460 */ 110, 109, 109, 109, 108, 404, 181, 118, 1085, 254, - /* 470 */ 254, 1089, 518, 90, 351, 373, 518, 1181, 365, 798, - /* 480 */ 1440, 339, 515, 248, 248, 77, 325, 133, 1085, 249, - /* 490 */ 424, 300, 794, 49, 49, 210, 515, 65, 65, 114, - /* 500 */ 115, 105, 1106, 1106, 957, 960, 950, 950, 112, 112, - /* 510 */ 113, 113, 113, 113, 1085, 1086, 1087, 222, 1085, 438, - /* 520 */ 461, 458, 457, 937, 787, 408, 171, 857, 362, 1021, - /* 530 */ 456, 136, 198, 486, 1085, 1086, 1087, 448, 794, 928, - /* 540 */ 5, 193, 192, 927, 1022, 107, 104, 200, 111, 111, - /* 550 */ 111, 111, 110, 110, 109, 109, 109, 108, 404, 1023, - /* 560 */ 254, 254, 803, 1085, 1085, 1086, 1087, 437, 373, 1085, - /* 570 */ 344, 787, 791, 515, 927, 927, 929, 1085, 1408, 1396, - /* 580 */ 832, 1085, 176, 3, 852, 1085, 518, 1439, 429, 851, - /* 590 */ 833, 518, 114, 115, 105, 1106, 1106, 957, 960, 950, - /* 600 */ 950, 112, 112, 113, 113, 113, 113, 13, 13, 1085, - /* 610 */ 1086, 1087, 13, 13, 518, 1085, 1086, 1087, 1496, 358, - /* 620 */ 1085, 389, 1234, 1085, 1086, 1087, 391, 1085, 1086, 1087, - /* 630 */ 448, 1085, 1086, 1087, 518, 65, 65, 947, 947, 958, - /* 640 */ 961, 111, 111, 111, 111, 110, 110, 109, 109, 109, - /* 650 */ 108, 404, 518, 382, 878, 13, 13, 518, 877, 518, - /* 660 */ 263, 373, 518, 431, 448, 1070, 1085, 1086, 1087, 267, - /* 670 */ 448, 488, 1360, 64, 64, 431, 812, 155, 50, 50, - /* 680 */ 65, 65, 518, 65, 65, 114, 115, 105, 1106, 1106, - /* 690 */ 957, 960, 950, 950, 112, 112, 113, 113, 113, 113, - /* 700 */ 518, 951, 382, 13, 13, 415, 411, 462, 414, 1085, - /* 710 */ 1366, 777, 1210, 292, 297, 813, 399, 497, 181, 403, - /* 720 */ 261, 15, 15, 276, 509, 414, 413, 1366, 1368, 410, - /* 730 */ 372, 345, 1209, 264, 111, 111, 111, 111, 110, 110, - /* 740 */ 109, 109, 109, 108, 404, 265, 254, 254, 229, 1405, - /* 750 */ 268, 1215, 268, 1103, 373, 1085, 1086, 1087, 938, 515, - /* 760 */ 393, 409, 876, 515, 254, 254, 1152, 482, 473, 262, - /* 770 */ 422, 476, 325, 503, 289, 518, 291, 515, 114, 115, - /* 780 */ 105, 1106, 1106, 957, 960, 950, 950, 112, 112, 113, - /* 790 */ 113, 113, 113, 414, 1021, 1366, 39, 39, 254, 254, - /* 800 */ 254, 254, 980, 254, 254, 254, 254, 255, 255, 1022, - /* 810 */ 279, 515, 516, 515, 846, 846, 515, 138, 515, 518, - /* 820 */ 515, 1043, 1495, 251, 1023, 1495, 876, 111, 111, 111, - /* 830 */ 111, 110, 110, 109, 109, 109, 108, 404, 518, 1353, - /* 840 */ 51, 51, 518, 199, 518, 506, 290, 373, 518, 276, - /* 850 */ 509, 922, 9, 483, 233, 1005, 1005, 445, 189, 52, - /* 860 */ 52, 325, 280, 53, 53, 54, 54, 373, 876, 55, - /* 870 */ 55, 114, 115, 105, 1106, 1106, 957, 960, 950, 950, - /* 880 */ 112, 112, 113, 113, 113, 113, 97, 518, 95, 1104, - /* 890 */ 1041, 114, 115, 105, 1106, 1106, 957, 960, 950, 950, - /* 900 */ 112, 112, 113, 113, 113, 113, 135, 199, 56, 56, - /* 910 */ 765, 766, 767, 225, 224, 223, 518, 283, 437, 233, - /* 920 */ 111, 111, 111, 111, 110, 110, 109, 109, 109, 108, - /* 930 */ 404, 1002, 876, 326, 518, 1002, 1104, 40, 40, 518, - /* 940 */ 111, 111, 111, 111, 110, 110, 109, 109, 109, 108, - /* 950 */ 404, 518, 448, 518, 1104, 41, 41, 518, 17, 518, - /* 960 */ 43, 43, 1155, 379, 518, 448, 518, 443, 518, 390, - /* 970 */ 518, 194, 44, 44, 57, 57, 1247, 518, 58, 58, - /* 980 */ 59, 59, 518, 466, 326, 14, 14, 60, 60, 120, - /* 990 */ 120, 61, 61, 449, 1206, 93, 518, 425, 46, 46, - /* 1000 */ 518, 1104, 518, 62, 62, 518, 437, 305, 518, 852, - /* 1010 */ 518, 298, 518, 1246, 851, 373, 518, 63, 63, 1293, - /* 1020 */ 397, 47, 47, 142, 142, 1467, 143, 143, 821, 70, - /* 1030 */ 70, 48, 48, 66, 66, 373, 518, 121, 121, 114, - /* 1040 */ 115, 105, 1106, 1106, 957, 960, 950, 950, 112, 112, - /* 1050 */ 113, 113, 113, 113, 518, 418, 518, 67, 67, 114, - /* 1060 */ 115, 105, 1106, 1106, 957, 960, 950, 950, 112, 112, - /* 1070 */ 113, 113, 113, 113, 312, 122, 122, 123, 123, 1293, - /* 1080 */ 518, 357, 1126, 88, 518, 435, 325, 387, 111, 111, - /* 1090 */ 111, 111, 110, 110, 109, 109, 109, 108, 404, 266, - /* 1100 */ 518, 119, 119, 518, 1293, 141, 141, 518, 111, 111, - /* 1110 */ 111, 111, 110, 110, 109, 109, 109, 108, 404, 518, - /* 1120 */ 801, 140, 140, 518, 127, 127, 511, 379, 126, 126, - /* 1130 */ 518, 137, 518, 1308, 518, 307, 518, 310, 518, 203, - /* 1140 */ 124, 124, 1307, 96, 125, 125, 207, 388, 1441, 468, - /* 1150 */ 1127, 69, 69, 71, 71, 68, 68, 38, 38, 42, - /* 1160 */ 42, 357, 1042, 373, 1293, 276, 509, 801, 185, 469, - /* 1170 */ 494, 436, 444, 6, 380, 156, 253, 197, 469, 134, - /* 1180 */ 426, 33, 1038, 373, 1121, 359, 1411, 114, 115, 105, - /* 1190 */ 1106, 1106, 957, 960, 950, 950, 112, 112, 113, 113, - /* 1200 */ 113, 113, 914, 296, 27, 293, 90, 114, 103, 105, - /* 1210 */ 1106, 1106, 957, 960, 950, 950, 112, 112, 113, 113, - /* 1220 */ 113, 113, 919, 275, 430, 232, 891, 232, 432, 256, - /* 1230 */ 1127, 232, 398, 370, 892, 28, 111, 111, 111, 111, - /* 1240 */ 110, 110, 109, 109, 109, 108, 404, 301, 454, 1385, - /* 1250 */ 90, 228, 209, 987, 811, 810, 111, 111, 111, 111, - /* 1260 */ 110, 110, 109, 109, 109, 108, 404, 315, 818, 819, - /* 1270 */ 90, 323, 983, 931, 885, 228, 373, 232, 999, 849, - /* 1280 */ 999, 322, 102, 998, 1384, 998, 785, 850, 440, 132, - /* 1290 */ 102, 302, 1243, 306, 309, 311, 373, 313, 1194, 1180, - /* 1300 */ 987, 115, 105, 1106, 1106, 957, 960, 950, 950, 112, - /* 1310 */ 112, 113, 113, 113, 113, 1178, 1179, 318, 327, 328, - /* 1320 */ 931, 1255, 105, 1106, 1106, 957, 960, 950, 950, 112, - /* 1330 */ 112, 113, 113, 113, 113, 1292, 1230, 1457, 273, 1241, - /* 1340 */ 504, 505, 1298, 100, 510, 246, 4, 1161, 1154, 111, - /* 1350 */ 111, 111, 111, 110, 110, 109, 109, 109, 108, 404, - /* 1360 */ 513, 1143, 187, 1142, 202, 1144, 1451, 356, 1227, 111, - /* 1370 */ 111, 111, 111, 110, 110, 109, 109, 109, 108, 404, - /* 1380 */ 11, 1277, 330, 405, 332, 334, 191, 1285, 364, 195, - /* 1390 */ 295, 417, 288, 100, 510, 507, 4, 434, 459, 321, - /* 1400 */ 1177, 349, 1357, 1356, 336, 155, 190, 1454, 1121, 158, - /* 1410 */ 513, 508, 235, 1404, 937, 1402, 1118, 381, 77, 428, - /* 1420 */ 98, 98, 8, 1282, 168, 30, 152, 99, 160, 405, - /* 1430 */ 520, 519, 88, 405, 927, 1362, 1274, 420, 163, 73, - /* 1440 */ 164, 76, 165, 166, 421, 507, 452, 212, 361, 363, - /* 1450 */ 427, 276, 509, 31, 1288, 172, 491, 441, 216, 1351, - /* 1460 */ 82, 490, 447, 1373, 937, 927, 927, 929, 930, 24, - /* 1470 */ 98, 98, 304, 247, 218, 177, 308, 99, 219, 405, - /* 1480 */ 520, 519, 450, 1145, 927, 220, 366, 1197, 100, 510, - /* 1490 */ 465, 4, 1188, 1196, 1195, 394, 803, 1169, 1187, 367, - /* 1500 */ 1168, 396, 484, 320, 1167, 513, 1466, 87, 475, 100, - /* 1510 */ 510, 271, 4, 272, 478, 927, 927, 929, 930, 24, - /* 1520 */ 1443, 1074, 407, 1238, 1239, 258, 513, 329, 405, 331, - /* 1530 */ 355, 355, 354, 243, 352, 234, 489, 774, 498, 184, - /* 1540 */ 507, 338, 1422, 339, 117, 1220, 10, 341, 333, 405, - /* 1550 */ 204, 491, 282, 1219, 1237, 1236, 492, 335, 343, 937, - /* 1560 */ 281, 507, 94, 1337, 186, 98, 98, 347, 89, 487, - /* 1570 */ 348, 241, 99, 29, 405, 520, 519, 274, 1151, 927, - /* 1580 */ 937, 521, 1080, 245, 242, 244, 98, 98, 856, 522, - /* 1590 */ 206, 1140, 1135, 99, 144, 405, 520, 519, 147, 375, - /* 1600 */ 927, 149, 376, 157, 1389, 1390, 1388, 1387, 205, 145, - /* 1610 */ 927, 927, 929, 930, 24, 146, 130, 761, 1165, 1164, - /* 1620 */ 72, 100, 510, 1162, 4, 269, 406, 188, 278, 201, - /* 1630 */ 259, 927, 927, 929, 930, 24, 128, 911, 513, 997, - /* 1640 */ 995, 159, 374, 208, 148, 161, 835, 276, 509, 211, - /* 1650 */ 294, 1011, 915, 167, 150, 383, 169, 78, 385, 79, - /* 1660 */ 80, 405, 81, 151, 1014, 213, 214, 1010, 139, 18, - /* 1670 */ 412, 215, 303, 507, 232, 1115, 1003, 446, 173, 217, - /* 1680 */ 174, 32, 776, 451, 491, 322, 221, 175, 814, 490, - /* 1690 */ 83, 455, 937, 19, 460, 316, 20, 84, 98, 98, - /* 1700 */ 270, 182, 85, 467, 153, 99, 154, 405, 520, 519, - /* 1710 */ 1074, 407, 927, 183, 258, 963, 1046, 86, 34, 355, - /* 1720 */ 355, 354, 243, 352, 474, 1047, 774, 35, 477, 196, - /* 1730 */ 250, 100, 510, 252, 4, 884, 178, 231, 1060, 204, - /* 1740 */ 21, 282, 102, 927, 927, 929, 930, 24, 513, 281, - /* 1750 */ 879, 22, 1064, 1062, 1051, 7, 340, 23, 978, 179, - /* 1760 */ 90, 92, 510, 964, 4, 236, 962, 966, 1020, 1019, - /* 1770 */ 237, 405, 967, 25, 36, 514, 932, 786, 513, 206, - /* 1780 */ 101, 26, 845, 507, 238, 239, 1459, 147, 350, 1458, - /* 1790 */ 149, 353, 1075, 1131, 1131, 1131, 1131, 205, 1131, 1131, - /* 1800 */ 1131, 405, 937, 1131, 1131, 1131, 1131, 1131, 98, 98, - /* 1810 */ 1131, 1131, 1131, 507, 1131, 99, 1131, 405, 520, 519, - /* 1820 */ 1131, 1131, 927, 1131, 1131, 1131, 1131, 1131, 1131, 1131, - /* 1830 */ 1131, 374, 937, 1131, 1131, 1131, 276, 509, 98, 98, - /* 1840 */ 1131, 1131, 1131, 1131, 1131, 99, 1131, 405, 520, 519, - /* 1850 */ 1131, 1131, 927, 927, 927, 929, 930, 24, 1131, 412, - /* 1860 */ 1131, 1131, 1131, 258, 1131, 1131, 1131, 1131, 355, 355, - /* 1870 */ 354, 243, 352, 1131, 1131, 774, 1131, 1131, 1131, 1131, - /* 1880 */ 1131, 1131, 1131, 927, 927, 929, 930, 24, 204, 1131, - /* 1890 */ 282, 1131, 1131, 1131, 1131, 1131, 1131, 1131, 281, 1131, - /* 1900 */ 1131, 1131, 1131, 1131, 1131, 1131, 1131, 1131, 1131, 1131, - /* 1910 */ 1131, 1131, 1131, 1131, 1131, 1131, 1131, 1131, 1131, 1131, - /* 1920 */ 1131, 1131, 1131, 1131, 1131, 1131, 1131, 1131, 206, 1131, - /* 1930 */ 1131, 1131, 1131, 1131, 1131, 1131, 147, 1131, 1131, 149, - /* 1940 */ 1131, 1131, 1131, 1131, 1131, 1131, 205, 1131, 1131, 1131, - /* 1950 */ 1131, 1131, 1131, 1131, 1131, 1131, 1131, 1131, 1131, 1131, - /* 1960 */ 1131, 1131, 1131, 1131, 1131, 1131, 1131, 1131, 1131, 1131, - /* 1970 */ 1131, 1131, 1131, 1131, 1131, 1131, 1131, 1131, 1131, 1131, - /* 1980 */ 374, 1131, 1131, 1131, 1131, 276, 509, 1131, 1131, 1131, - /* 1990 */ 1131, 1131, 1131, 1131, 1131, 1131, 1131, 1131, 1131, 1131, - /* 2000 */ 1131, 1131, 1131, 1131, 1131, 1131, 1131, 1131, 412, + /* 0 */ 537, 339, 537, 1241, 1220, 537, 12, 537, 112, 109, + /* 10 */ 209, 537, 1241, 537, 1205, 462, 112, 109, 209, 386, + /* 20 */ 338, 462, 42, 42, 42, 42, 445, 42, 42, 70, + /* 30 */ 70, 922, 1208, 70, 70, 70, 70, 1443, 403, 923, + /* 40 */ 531, 531, 531, 119, 120, 110, 1148, 1148, 991, 994, + /* 50 */ 984, 984, 117, 117, 118, 118, 118, 118, 425, 386, + /* 60 */ 1498, 542, 2, 1176, 1442, 519, 141, 1518, 289, 519, + /* 70 */ 134, 519, 95, 259, 495, 1215, 189, 1254, 518, 494, + /* 80 */ 484, 437, 296, 119, 120, 110, 1148, 1148, 991, 994, + /* 90 */ 984, 984, 117, 117, 118, 118, 118, 118, 270, 116, + /* 100 */ 116, 116, 116, 115, 115, 114, 114, 114, 113, 418, + /* 110 */ 264, 264, 264, 264, 423, 1479, 352, 1481, 123, 351, + /* 120 */ 1479, 508, 1094, 534, 1034, 534, 1099, 386, 1099, 239, + /* 130 */ 206, 112, 109, 209, 96, 1094, 376, 219, 1094, 116, + /* 140 */ 116, 116, 116, 115, 115, 114, 114, 114, 113, 418, + /* 150 */ 480, 119, 120, 110, 1148, 1148, 991, 994, 984, 984, + /* 160 */ 117, 117, 118, 118, 118, 118, 353, 422, 1407, 264, + /* 170 */ 264, 114, 114, 114, 113, 418, 883, 121, 416, 416, + /* 180 */ 416, 882, 534, 116, 116, 116, 116, 115, 115, 114, + /* 190 */ 114, 114, 113, 418, 212, 415, 414, 386, 443, 383, + /* 200 */ 382, 118, 118, 118, 118, 111, 177, 116, 116, 116, + /* 210 */ 116, 115, 115, 114, 114, 114, 113, 418, 112, 109, + /* 220 */ 209, 119, 120, 110, 1148, 1148, 991, 994, 984, 984, + /* 230 */ 117, 117, 118, 118, 118, 118, 386, 438, 312, 1163, + /* 240 */ 1155, 80, 1155, 1127, 514, 79, 116, 116, 116, 116, + /* 250 */ 115, 115, 114, 114, 114, 113, 418, 514, 428, 418, + /* 260 */ 119, 120, 110, 1148, 1148, 991, 994, 984, 984, 117, + /* 270 */ 117, 118, 118, 118, 118, 428, 427, 116, 116, 116, + /* 280 */ 116, 115, 115, 114, 114, 114, 113, 418, 115, 115, + /* 290 */ 114, 114, 114, 113, 418, 1127, 1127, 1128, 1129, 1094, + /* 300 */ 258, 258, 192, 386, 408, 371, 1168, 326, 118, 118, + /* 310 */ 118, 118, 1094, 534, 374, 1094, 116, 116, 116, 116, + /* 320 */ 115, 115, 114, 114, 114, 113, 418, 119, 120, 110, + /* 330 */ 1148, 1148, 991, 994, 984, 984, 117, 117, 118, 118, + /* 340 */ 118, 118, 386, 354, 445, 428, 829, 238, 1127, 1128, + /* 350 */ 1129, 515, 1466, 116, 116, 116, 116, 115, 115, 114, + /* 360 */ 114, 114, 113, 418, 1127, 1467, 119, 120, 110, 1148, + /* 370 */ 1148, 991, 994, 984, 984, 117, 117, 118, 118, 118, + /* 380 */ 118, 1169, 82, 116, 116, 116, 116, 115, 115, 114, + /* 390 */ 114, 114, 113, 418, 405, 112, 109, 209, 161, 445, + /* 400 */ 250, 267, 336, 478, 331, 477, 236, 951, 1127, 386, + /* 410 */ 888, 1521, 329, 822, 852, 162, 274, 1127, 1128, 1129, + /* 420 */ 338, 169, 116, 116, 116, 116, 115, 115, 114, 114, + /* 430 */ 114, 113, 418, 119, 120, 110, 1148, 1148, 991, 994, + /* 440 */ 984, 984, 117, 117, 118, 118, 118, 118, 386, 438, + /* 450 */ 312, 1502, 1112, 1176, 161, 288, 528, 311, 289, 883, + /* 460 */ 134, 1127, 1128, 1129, 882, 537, 143, 1254, 288, 528, + /* 470 */ 297, 275, 119, 120, 110, 1148, 1148, 991, 994, 984, + /* 480 */ 984, 117, 117, 118, 118, 118, 118, 70, 70, 116, + /* 490 */ 116, 116, 116, 115, 115, 114, 114, 114, 113, 418, + /* 500 */ 264, 264, 12, 264, 264, 395, 1127, 483, 1473, 1094, + /* 510 */ 204, 482, 6, 534, 1258, 386, 534, 1474, 825, 972, + /* 520 */ 504, 6, 1094, 500, 95, 1094, 534, 219, 116, 116, + /* 530 */ 116, 116, 115, 115, 114, 114, 114, 113, 418, 119, + /* 540 */ 120, 110, 1148, 1148, 991, 994, 984, 984, 117, 117, + /* 550 */ 118, 118, 118, 118, 386, 1339, 971, 422, 956, 1127, + /* 560 */ 1128, 1129, 231, 512, 1473, 475, 472, 471, 6, 113, + /* 570 */ 418, 825, 962, 298, 503, 470, 961, 452, 119, 120, + /* 580 */ 110, 1148, 1148, 991, 994, 984, 984, 117, 117, 118, + /* 590 */ 118, 118, 118, 395, 537, 116, 116, 116, 116, 115, + /* 600 */ 115, 114, 114, 114, 113, 418, 202, 961, 961, 963, + /* 610 */ 231, 971, 1127, 475, 472, 471, 13, 13, 951, 1127, + /* 620 */ 834, 386, 1207, 470, 399, 183, 447, 962, 462, 162, + /* 630 */ 397, 961, 1246, 1246, 116, 116, 116, 116, 115, 115, + /* 640 */ 114, 114, 114, 113, 418, 119, 120, 110, 1148, 1148, + /* 650 */ 991, 994, 984, 984, 117, 117, 118, 118, 118, 118, + /* 660 */ 386, 271, 961, 961, 963, 1127, 1128, 1129, 311, 433, + /* 670 */ 299, 1406, 1127, 1128, 1129, 178, 1471, 138, 162, 32, + /* 680 */ 6, 1127, 288, 528, 119, 120, 110, 1148, 1148, 991, + /* 690 */ 994, 984, 984, 117, 117, 118, 118, 118, 118, 909, + /* 700 */ 390, 116, 116, 116, 116, 115, 115, 114, 114, 114, + /* 710 */ 113, 418, 1127, 429, 817, 537, 1127, 265, 265, 981, + /* 720 */ 981, 992, 995, 324, 1055, 93, 520, 5, 338, 537, + /* 730 */ 534, 288, 528, 1522, 1127, 1128, 1129, 70, 70, 1056, + /* 740 */ 116, 116, 116, 116, 115, 115, 114, 114, 114, 113, + /* 750 */ 418, 70, 70, 1495, 1057, 537, 98, 1244, 1244, 264, + /* 760 */ 264, 908, 371, 1076, 1127, 1127, 1128, 1129, 817, 1127, + /* 770 */ 1128, 1129, 534, 519, 140, 863, 386, 13, 13, 456, + /* 780 */ 192, 193, 521, 453, 319, 864, 322, 284, 365, 430, + /* 790 */ 985, 402, 379, 1077, 1548, 101, 386, 1548, 3, 395, + /* 800 */ 119, 120, 110, 1148, 1148, 991, 994, 984, 984, 117, + /* 810 */ 117, 118, 118, 118, 118, 386, 451, 1127, 1128, 1129, + /* 820 */ 119, 120, 110, 1148, 1148, 991, 994, 984, 984, 117, + /* 830 */ 117, 118, 118, 118, 118, 1127, 1354, 1412, 1169, 119, + /* 840 */ 108, 110, 1148, 1148, 991, 994, 984, 984, 117, 117, + /* 850 */ 118, 118, 118, 118, 1412, 1414, 116, 116, 116, 116, + /* 860 */ 115, 115, 114, 114, 114, 113, 418, 272, 535, 1075, + /* 870 */ 877, 877, 337, 1492, 309, 462, 116, 116, 116, 116, + /* 880 */ 115, 115, 114, 114, 114, 113, 418, 537, 1127, 1128, + /* 890 */ 1129, 537, 360, 537, 356, 116, 116, 116, 116, 115, + /* 900 */ 115, 114, 114, 114, 113, 418, 386, 264, 264, 13, + /* 910 */ 13, 273, 1127, 13, 13, 13, 13, 304, 1253, 386, + /* 920 */ 534, 1077, 1549, 404, 1412, 1549, 496, 277, 451, 186, + /* 930 */ 1252, 120, 110, 1148, 1148, 991, 994, 984, 984, 117, + /* 940 */ 117, 118, 118, 118, 118, 110, 1148, 1148, 991, 994, + /* 950 */ 984, 984, 117, 117, 118, 118, 118, 118, 105, 529, + /* 960 */ 537, 4, 1339, 264, 264, 1127, 1128, 1129, 1039, 1039, + /* 970 */ 459, 795, 796, 797, 536, 532, 534, 242, 301, 807, + /* 980 */ 303, 462, 69, 69, 451, 1353, 116, 116, 116, 116, + /* 990 */ 115, 115, 114, 114, 114, 113, 418, 1075, 419, 116, + /* 1000 */ 116, 116, 116, 115, 115, 114, 114, 114, 113, 418, + /* 1010 */ 526, 537, 1146, 192, 350, 105, 529, 537, 4, 497, + /* 1020 */ 162, 337, 1492, 310, 1249, 385, 1550, 372, 9, 462, + /* 1030 */ 242, 400, 532, 13, 13, 499, 971, 843, 436, 70, + /* 1040 */ 70, 359, 103, 103, 8, 339, 278, 187, 278, 104, + /* 1050 */ 1127, 419, 539, 538, 1339, 419, 961, 302, 1339, 1172, + /* 1060 */ 1, 1, 542, 2, 1176, 1146, 1146, 526, 476, 289, + /* 1070 */ 30, 134, 317, 288, 528, 285, 844, 1014, 1254, 276, + /* 1080 */ 1472, 506, 410, 1194, 6, 207, 505, 961, 961, 963, + /* 1090 */ 964, 27, 449, 971, 415, 414, 234, 233, 232, 103, + /* 1100 */ 103, 31, 1152, 1127, 1128, 1129, 104, 1154, 419, 539, + /* 1110 */ 538, 264, 264, 961, 1399, 1153, 264, 264, 1470, 1146, + /* 1120 */ 537, 216, 6, 401, 534, 1197, 392, 458, 406, 534, + /* 1130 */ 537, 485, 358, 537, 261, 537, 1339, 907, 219, 1155, + /* 1140 */ 467, 1155, 50, 50, 961, 961, 963, 964, 27, 1497, + /* 1150 */ 1116, 421, 70, 70, 268, 70, 70, 13, 13, 369, + /* 1160 */ 369, 368, 253, 366, 264, 264, 804, 235, 422, 105, + /* 1170 */ 529, 516, 4, 287, 487, 510, 493, 534, 486, 213, + /* 1180 */ 1055, 294, 490, 384, 1127, 450, 532, 338, 413, 293, + /* 1190 */ 522, 417, 335, 1036, 509, 1056, 107, 1036, 16, 16, + /* 1200 */ 1469, 1094, 334, 1105, 6, 411, 1145, 264, 264, 419, + /* 1210 */ 1057, 102, 511, 100, 1094, 264, 264, 1094, 922, 215, + /* 1220 */ 534, 526, 907, 264, 264, 208, 923, 154, 534, 457, + /* 1230 */ 156, 525, 391, 142, 218, 506, 534, 1127, 1128, 1129, + /* 1240 */ 507, 139, 1131, 38, 214, 530, 392, 971, 329, 1454, + /* 1250 */ 907, 1105, 537, 103, 103, 105, 529, 537, 4, 537, + /* 1260 */ 104, 424, 419, 539, 538, 537, 502, 961, 517, 537, + /* 1270 */ 1072, 537, 532, 373, 54, 54, 288, 528, 387, 55, + /* 1280 */ 55, 15, 15, 288, 528, 17, 136, 44, 44, 1451, + /* 1290 */ 537, 56, 56, 57, 57, 419, 1131, 291, 961, 961, + /* 1300 */ 963, 964, 27, 393, 163, 537, 426, 526, 263, 206, + /* 1310 */ 208, 517, 58, 58, 235, 440, 842, 841, 197, 105, + /* 1320 */ 529, 506, 4, 1033, 439, 1033, 505, 59, 59, 308, + /* 1330 */ 849, 850, 95, 971, 537, 907, 532, 948, 832, 103, + /* 1340 */ 103, 105, 529, 537, 4, 1021, 104, 537, 419, 539, + /* 1350 */ 538, 1116, 421, 961, 537, 268, 60, 60, 532, 419, + /* 1360 */ 369, 369, 368, 253, 366, 61, 61, 804, 965, 45, + /* 1370 */ 45, 526, 537, 1032, 1277, 1032, 46, 46, 537, 391, + /* 1380 */ 213, 419, 294, 266, 961, 961, 963, 964, 27, 292, + /* 1390 */ 293, 295, 832, 526, 48, 48, 1290, 971, 1289, 1021, + /* 1400 */ 49, 49, 432, 103, 103, 887, 953, 537, 1457, 241, + /* 1410 */ 104, 305, 419, 539, 538, 925, 926, 961, 444, 971, + /* 1420 */ 215, 241, 965, 1224, 537, 103, 103, 1431, 154, 62, + /* 1430 */ 62, 156, 104, 1430, 419, 539, 538, 97, 529, 961, + /* 1440 */ 4, 537, 454, 537, 314, 214, 63, 63, 961, 961, + /* 1450 */ 963, 964, 27, 537, 532, 446, 1286, 318, 241, 537, + /* 1460 */ 321, 323, 325, 64, 64, 14, 14, 1237, 537, 1223, + /* 1470 */ 961, 961, 963, 964, 27, 65, 65, 419, 537, 387, + /* 1480 */ 537, 125, 125, 537, 288, 528, 537, 1486, 537, 526, + /* 1490 */ 66, 66, 313, 524, 537, 95, 468, 1221, 1511, 237, + /* 1500 */ 51, 51, 67, 67, 330, 68, 68, 426, 52, 52, + /* 1510 */ 149, 149, 1222, 340, 341, 971, 150, 150, 1298, 463, + /* 1520 */ 327, 103, 103, 95, 537, 1338, 1273, 537, 104, 537, + /* 1530 */ 419, 539, 538, 1284, 537, 961, 268, 283, 523, 1344, + /* 1540 */ 1204, 369, 369, 368, 253, 366, 75, 75, 804, 53, + /* 1550 */ 53, 71, 71, 537, 1196, 537, 126, 126, 537, 1017, + /* 1560 */ 537, 213, 237, 294, 537, 1185, 961, 961, 963, 964, + /* 1570 */ 27, 293, 537, 1184, 537, 72, 72, 127, 127, 1186, + /* 1580 */ 128, 128, 124, 124, 1505, 537, 148, 148, 537, 256, + /* 1590 */ 195, 537, 1270, 537, 147, 147, 132, 132, 537, 11, + /* 1600 */ 537, 215, 537, 199, 343, 345, 347, 131, 131, 154, + /* 1610 */ 129, 129, 156, 130, 130, 74, 74, 537, 370, 1323, + /* 1620 */ 76, 76, 73, 73, 43, 43, 214, 431, 211, 1331, + /* 1630 */ 300, 916, 880, 815, 241, 107, 137, 307, 881, 47, + /* 1640 */ 47, 107, 473, 378, 203, 448, 333, 1403, 1220, 1402, + /* 1650 */ 349, 190, 527, 191, 363, 198, 1508, 1163, 245, 165, + /* 1660 */ 387, 1450, 1448, 1160, 78, 288, 528, 1408, 81, 394, + /* 1670 */ 82, 442, 175, 159, 167, 93, 1328, 35, 1320, 434, + /* 1680 */ 170, 171, 172, 173, 435, 466, 221, 375, 426, 377, + /* 1690 */ 1334, 179, 455, 441, 1397, 225, 87, 36, 461, 1419, + /* 1700 */ 316, 257, 227, 184, 320, 464, 228, 479, 1187, 229, + /* 1710 */ 380, 1240, 1239, 407, 1238, 1212, 834, 332, 1231, 381, + /* 1720 */ 409, 1211, 204, 1210, 1491, 498, 1520, 1281, 92, 281, + /* 1730 */ 1230, 489, 282, 492, 342, 243, 1282, 344, 244, 1280, + /* 1740 */ 346, 412, 1279, 1477, 348, 122, 1476, 517, 10, 357, + /* 1750 */ 286, 1305, 1304, 99, 1383, 94, 501, 251, 1193, 34, + /* 1760 */ 1263, 355, 540, 194, 1262, 361, 362, 1122, 252, 254, + /* 1770 */ 255, 388, 541, 1182, 1177, 151, 1435, 389, 1436, 1434, + /* 1780 */ 1433, 791, 152, 135, 279, 200, 201, 420, 196, 77, + /* 1790 */ 153, 290, 269, 210, 1031, 133, 1029, 945, 166, 155, + /* 1800 */ 217, 168, 866, 306, 220, 1045, 174, 949, 157, 396, + /* 1810 */ 83, 398, 176, 84, 85, 164, 86, 158, 1048, 222, + /* 1820 */ 223, 1044, 144, 18, 224, 315, 1037, 180, 241, 460, + /* 1830 */ 1157, 226, 181, 37, 806, 465, 334, 230, 328, 469, + /* 1840 */ 182, 88, 474, 19, 20, 160, 89, 280, 145, 90, + /* 1850 */ 481, 845, 1110, 146, 997, 205, 1080, 39, 91, 40, + /* 1860 */ 488, 1081, 915, 491, 260, 262, 185, 910, 240, 107, + /* 1870 */ 1100, 1096, 1098, 1104, 21, 1084, 33, 513, 247, 22, + /* 1880 */ 23, 24, 1103, 25, 188, 95, 1012, 998, 996, 26, + /* 1890 */ 1000, 1054, 7, 1053, 1001, 246, 28, 41, 533, 966, + /* 1900 */ 816, 106, 29, 367, 248, 249, 1513, 1512, 364, 1117, + /* 1910 */ 1173, 1173, 876, }; static const YYCODETYPE yy_lookahead[] = { - /* 0 */ 168, 163, 184, 238, 239, 240, 163, 163, 155, 156, - /* 10 */ 157, 158, 159, 160, 163, 202, 203, 187, 165, 19, - /* 20 */ 167, 163, 184, 185, 259, 202, 203, 174, 184, 185, - /* 30 */ 174, 31, 238, 239, 240, 184, 185, 22, 23, 39, - /* 40 */ 216, 26, 218, 43, 44, 45, 46, 47, 48, 49, - /* 50 */ 50, 51, 52, 53, 54, 55, 56, 57, 174, 206, - /* 60 */ 207, 163, 206, 207, 220, 163, 163, 163, 238, 239, - /* 70 */ 240, 59, 219, 229, 231, 219, 183, 245, 174, 223, - /* 80 */ 224, 249, 184, 185, 191, 232, 184, 185, 184, 185, - /* 90 */ 206, 207, 92, 93, 94, 95, 96, 97, 98, 99, - /* 100 */ 100, 101, 102, 219, 102, 81, 91, 163, 96, 97, - /* 110 */ 206, 207, 19, 275, 276, 262, 104, 105, 106, 107, - /* 120 */ 163, 109, 220, 219, 220, 184, 275, 269, 277, 117, - /* 130 */ 187, 229, 19, 229, 101, 102, 43, 44, 45, 46, - /* 140 */ 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, - /* 150 */ 57, 127, 128, 141, 184, 143, 43, 44, 45, 46, - /* 160 */ 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, - /* 170 */ 57, 268, 269, 275, 276, 197, 83, 233, 85, 163, - /* 180 */ 67, 238, 239, 240, 134, 92, 93, 94, 95, 96, - /* 190 */ 97, 98, 99, 100, 101, 102, 19, 54, 55, 56, - /* 200 */ 57, 58, 152, 26, 247, 92, 93, 94, 95, 96, - /* 210 */ 97, 98, 99, 100, 101, 102, 54, 55, 56, 57, - /* 220 */ 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, - /* 230 */ 53, 54, 55, 56, 57, 92, 93, 94, 95, 96, - /* 240 */ 97, 98, 99, 100, 101, 102, 69, 96, 97, 98, - /* 250 */ 99, 100, 101, 102, 92, 93, 94, 95, 96, 97, - /* 260 */ 98, 99, 100, 101, 102, 73, 179, 180, 181, 92, - /* 270 */ 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, - /* 280 */ 163, 191, 192, 163, 98, 99, 100, 101, 102, 19, - /* 290 */ 200, 179, 180, 181, 24, 175, 92, 93, 94, 95, - /* 300 */ 96, 97, 98, 99, 100, 101, 102, 163, 116, 117, - /* 310 */ 118, 22, 163, 43, 44, 45, 46, 47, 48, 49, - /* 320 */ 50, 51, 52, 53, 54, 55, 56, 57, 157, 158, - /* 330 */ 159, 160, 163, 184, 185, 163, 165, 59, 167, 46, - /* 340 */ 90, 76, 11, 174, 73, 174, 19, 198, 59, 19, - /* 350 */ 72, 86, 81, 184, 185, 234, 106, 96, 97, 163, - /* 360 */ 110, 182, 92, 93, 94, 95, 96, 97, 98, 99, - /* 370 */ 100, 101, 102, 46, 230, 206, 207, 206, 207, 163, - /* 380 */ 184, 185, 19, 105, 106, 107, 23, 116, 219, 220, - /* 390 */ 219, 141, 142, 143, 105, 106, 107, 104, 127, 128, - /* 400 */ 184, 185, 141, 232, 143, 59, 43, 44, 45, 46, - /* 410 */ 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, - /* 420 */ 57, 158, 108, 160, 59, 111, 112, 113, 165, 250, - /* 430 */ 167, 104, 102, 262, 255, 121, 220, 174, 108, 109, - /* 440 */ 110, 111, 112, 113, 114, 229, 182, 120, 117, 118, - /* 450 */ 120, 105, 106, 107, 163, 92, 93, 94, 95, 96, - /* 460 */ 97, 98, 99, 100, 101, 102, 163, 22, 59, 206, - /* 470 */ 207, 106, 163, 26, 171, 19, 163, 193, 175, 23, - /* 480 */ 163, 22, 219, 206, 207, 139, 163, 22, 59, 182, - /* 490 */ 117, 118, 59, 184, 185, 232, 219, 184, 185, 43, - /* 500 */ 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, - /* 510 */ 54, 55, 56, 57, 105, 106, 107, 108, 59, 255, - /* 520 */ 111, 112, 113, 90, 59, 262, 22, 98, 174, 12, - /* 530 */ 121, 208, 163, 220, 105, 106, 107, 163, 105, 106, - /* 540 */ 22, 96, 97, 110, 27, 238, 239, 240, 92, 93, - /* 550 */ 94, 95, 96, 97, 98, 99, 100, 101, 102, 42, - /* 560 */ 206, 207, 115, 59, 105, 106, 107, 163, 19, 59, - /* 570 */ 163, 106, 23, 219, 141, 142, 143, 59, 163, 205, - /* 580 */ 63, 59, 72, 22, 124, 59, 163, 270, 234, 129, - /* 590 */ 73, 163, 43, 44, 45, 46, 47, 48, 49, 50, - /* 600 */ 51, 52, 53, 54, 55, 56, 57, 184, 185, 105, - /* 610 */ 106, 107, 184, 185, 163, 105, 106, 107, 265, 266, - /* 620 */ 59, 198, 225, 105, 106, 107, 198, 105, 106, 107, - /* 630 */ 163, 105, 106, 107, 163, 184, 185, 46, 47, 48, - /* 640 */ 49, 92, 93, 94, 95, 96, 97, 98, 99, 100, - /* 650 */ 101, 102, 163, 163, 132, 184, 185, 163, 132, 163, - /* 660 */ 256, 19, 163, 163, 163, 23, 105, 106, 107, 198, - /* 670 */ 163, 220, 205, 184, 185, 163, 35, 81, 184, 185, - /* 680 */ 184, 185, 163, 184, 185, 43, 44, 45, 46, 47, - /* 690 */ 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, - /* 700 */ 163, 110, 163, 184, 185, 109, 205, 66, 163, 59, - /* 710 */ 163, 21, 205, 16, 174, 74, 220, 198, 163, 220, - /* 720 */ 230, 184, 185, 127, 128, 180, 181, 180, 181, 163, - /* 730 */ 175, 242, 174, 233, 92, 93, 94, 95, 96, 97, - /* 740 */ 98, 99, 100, 101, 102, 233, 206, 207, 26, 163, - /* 750 */ 195, 207, 197, 26, 19, 105, 106, 107, 23, 219, - /* 760 */ 119, 260, 26, 219, 206, 207, 174, 19, 174, 230, - /* 770 */ 80, 174, 163, 174, 77, 163, 79, 219, 43, 44, - /* 780 */ 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, - /* 790 */ 55, 56, 57, 248, 12, 248, 184, 185, 206, 207, - /* 800 */ 206, 207, 112, 206, 207, 206, 207, 206, 207, 27, - /* 810 */ 163, 219, 123, 219, 125, 126, 219, 208, 219, 163, - /* 820 */ 219, 22, 23, 23, 42, 26, 26, 92, 93, 94, - /* 830 */ 95, 96, 97, 98, 99, 100, 101, 102, 163, 149, - /* 840 */ 184, 185, 163, 107, 163, 63, 149, 19, 163, 127, - /* 850 */ 128, 23, 22, 105, 24, 116, 117, 118, 131, 184, - /* 860 */ 185, 163, 163, 184, 185, 184, 185, 19, 132, 184, - /* 870 */ 185, 43, 44, 45, 46, 47, 48, 49, 50, 51, - /* 880 */ 52, 53, 54, 55, 56, 57, 146, 163, 148, 59, - /* 890 */ 91, 43, 44, 45, 46, 47, 48, 49, 50, 51, - /* 900 */ 52, 53, 54, 55, 56, 57, 208, 107, 184, 185, - /* 910 */ 7, 8, 9, 116, 117, 118, 163, 163, 163, 24, - /* 920 */ 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, - /* 930 */ 102, 29, 132, 163, 163, 33, 106, 184, 185, 163, - /* 940 */ 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, - /* 950 */ 102, 163, 163, 163, 59, 184, 185, 163, 22, 163, - /* 960 */ 184, 185, 177, 178, 163, 163, 163, 65, 163, 199, - /* 970 */ 163, 26, 184, 185, 184, 185, 163, 163, 184, 185, - /* 980 */ 184, 185, 163, 98, 163, 184, 185, 184, 185, 184, - /* 990 */ 185, 184, 185, 252, 205, 147, 163, 61, 184, 185, - /* 1000 */ 163, 106, 163, 184, 185, 163, 163, 205, 163, 124, - /* 1010 */ 163, 256, 163, 163, 129, 19, 163, 184, 185, 163, - /* 1020 */ 199, 184, 185, 184, 185, 23, 184, 185, 26, 184, - /* 1030 */ 185, 184, 185, 184, 185, 19, 163, 184, 185, 43, - /* 1040 */ 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, - /* 1050 */ 54, 55, 56, 57, 163, 163, 163, 184, 185, 43, - /* 1060 */ 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, - /* 1070 */ 54, 55, 56, 57, 16, 184, 185, 184, 185, 163, - /* 1080 */ 163, 22, 23, 138, 163, 19, 163, 231, 92, 93, - /* 1090 */ 94, 95, 96, 97, 98, 99, 100, 101, 102, 256, - /* 1100 */ 163, 184, 185, 163, 163, 184, 185, 163, 92, 93, - /* 1110 */ 94, 95, 96, 97, 98, 99, 100, 101, 102, 163, - /* 1120 */ 59, 184, 185, 163, 184, 185, 177, 178, 184, 185, - /* 1130 */ 163, 208, 163, 237, 163, 77, 163, 79, 163, 15, - /* 1140 */ 184, 185, 237, 147, 184, 185, 24, 231, 153, 154, - /* 1150 */ 91, 184, 185, 184, 185, 184, 185, 184, 185, 184, - /* 1160 */ 185, 22, 23, 19, 163, 127, 128, 106, 24, 273, - /* 1170 */ 271, 105, 231, 274, 263, 264, 223, 224, 273, 22, - /* 1180 */ 118, 24, 23, 19, 60, 26, 163, 43, 44, 45, - /* 1190 */ 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, - /* 1200 */ 56, 57, 140, 23, 22, 163, 26, 43, 44, 45, - /* 1210 */ 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, - /* 1220 */ 56, 57, 23, 211, 23, 26, 31, 26, 23, 22, - /* 1230 */ 91, 26, 231, 221, 39, 53, 92, 93, 94, 95, - /* 1240 */ 96, 97, 98, 99, 100, 101, 102, 23, 23, 163, - /* 1250 */ 26, 26, 130, 59, 109, 110, 92, 93, 94, 95, - /* 1260 */ 96, 97, 98, 99, 100, 101, 102, 23, 7, 8, - /* 1270 */ 26, 110, 23, 59, 23, 26, 19, 26, 141, 23, - /* 1280 */ 143, 120, 26, 141, 163, 143, 23, 23, 163, 26, - /* 1290 */ 26, 163, 163, 163, 163, 163, 19, 163, 163, 193, - /* 1300 */ 106, 44, 45, 46, 47, 48, 49, 50, 51, 52, - /* 1310 */ 53, 54, 55, 56, 57, 163, 193, 163, 163, 163, - /* 1320 */ 106, 163, 45, 46, 47, 48, 49, 50, 51, 52, - /* 1330 */ 53, 54, 55, 56, 57, 163, 163, 130, 222, 163, - /* 1340 */ 163, 203, 163, 19, 20, 251, 22, 163, 163, 92, - /* 1350 */ 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, - /* 1360 */ 36, 163, 209, 163, 261, 163, 163, 161, 222, 92, - /* 1370 */ 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, - /* 1380 */ 210, 213, 222, 59, 222, 222, 182, 213, 213, 196, - /* 1390 */ 257, 226, 226, 19, 20, 71, 22, 257, 188, 187, - /* 1400 */ 192, 212, 187, 187, 226, 81, 210, 166, 60, 261, - /* 1410 */ 36, 244, 130, 170, 90, 170, 38, 170, 139, 104, - /* 1420 */ 96, 97, 48, 236, 22, 235, 43, 103, 201, 105, - /* 1430 */ 106, 107, 138, 59, 110, 247, 213, 18, 204, 258, - /* 1440 */ 204, 258, 204, 204, 170, 71, 18, 169, 213, 236, - /* 1450 */ 213, 127, 128, 235, 201, 201, 82, 170, 169, 213, - /* 1460 */ 146, 87, 62, 254, 90, 141, 142, 143, 144, 145, - /* 1470 */ 96, 97, 253, 170, 169, 22, 170, 103, 169, 105, - /* 1480 */ 106, 107, 189, 170, 110, 169, 189, 186, 19, 20, - /* 1490 */ 104, 22, 194, 186, 186, 64, 115, 186, 194, 189, - /* 1500 */ 188, 102, 133, 186, 186, 36, 186, 104, 189, 19, - /* 1510 */ 20, 246, 22, 246, 189, 141, 142, 143, 144, 145, - /* 1520 */ 0, 1, 2, 228, 228, 5, 36, 227, 59, 227, - /* 1530 */ 10, 11, 12, 13, 14, 170, 84, 17, 134, 216, - /* 1540 */ 71, 272, 270, 22, 137, 217, 22, 216, 227, 59, - /* 1550 */ 30, 82, 32, 217, 228, 228, 87, 227, 170, 90, - /* 1560 */ 40, 71, 146, 241, 215, 96, 97, 214, 136, 135, - /* 1570 */ 213, 25, 103, 26, 105, 106, 107, 243, 173, 110, - /* 1580 */ 90, 172, 13, 6, 164, 164, 96, 97, 98, 162, - /* 1590 */ 70, 162, 162, 103, 176, 105, 106, 107, 78, 267, - /* 1600 */ 110, 81, 267, 264, 182, 182, 182, 182, 88, 176, - /* 1610 */ 141, 142, 143, 144, 145, 176, 190, 4, 182, 182, - /* 1620 */ 182, 19, 20, 182, 22, 190, 3, 22, 151, 15, - /* 1630 */ 89, 141, 142, 143, 144, 145, 16, 128, 36, 23, - /* 1640 */ 23, 139, 122, 24, 119, 131, 20, 127, 128, 133, - /* 1650 */ 16, 1, 140, 131, 119, 61, 139, 53, 37, 53, - /* 1660 */ 53, 59, 53, 119, 105, 34, 130, 1, 5, 22, - /* 1670 */ 150, 104, 149, 71, 26, 75, 68, 41, 68, 130, - /* 1680 */ 104, 24, 20, 19, 82, 120, 114, 22, 28, 87, - /* 1690 */ 22, 67, 90, 22, 67, 23, 22, 22, 96, 97, - /* 1700 */ 67, 23, 138, 22, 37, 103, 153, 105, 106, 107, - /* 1710 */ 1, 2, 110, 23, 5, 23, 23, 26, 22, 10, - /* 1720 */ 11, 12, 13, 14, 24, 23, 17, 22, 24, 130, - /* 1730 */ 23, 19, 20, 23, 22, 105, 22, 34, 85, 30, - /* 1740 */ 34, 32, 26, 141, 142, 143, 144, 145, 36, 40, - /* 1750 */ 132, 34, 75, 83, 23, 44, 24, 34, 23, 26, - /* 1760 */ 26, 19, 20, 23, 22, 26, 23, 23, 23, 23, - /* 1770 */ 22, 59, 11, 22, 22, 26, 23, 23, 36, 70, - /* 1780 */ 22, 22, 124, 71, 130, 130, 130, 78, 23, 130, - /* 1790 */ 81, 15, 1, 278, 278, 278, 278, 88, 278, 278, - /* 1800 */ 278, 59, 90, 278, 278, 278, 278, 278, 96, 97, - /* 1810 */ 278, 278, 278, 71, 278, 103, 278, 105, 106, 107, - /* 1820 */ 278, 278, 110, 278, 278, 278, 278, 278, 278, 278, - /* 1830 */ 278, 122, 90, 278, 278, 278, 127, 128, 96, 97, - /* 1840 */ 278, 278, 278, 278, 278, 103, 278, 105, 106, 107, - /* 1850 */ 278, 278, 110, 141, 142, 143, 144, 145, 278, 150, - /* 1860 */ 278, 278, 278, 5, 278, 278, 278, 278, 10, 11, - /* 1870 */ 12, 13, 14, 278, 278, 17, 278, 278, 278, 278, - /* 1880 */ 278, 278, 278, 141, 142, 143, 144, 145, 30, 278, - /* 1890 */ 32, 278, 278, 278, 278, 278, 278, 278, 40, 278, - /* 1900 */ 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, - /* 1910 */ 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, - /* 1920 */ 278, 278, 278, 278, 278, 278, 278, 278, 70, 278, - /* 1930 */ 278, 278, 278, 278, 278, 278, 78, 278, 278, 81, - /* 1940 */ 278, 278, 278, 278, 278, 278, 88, 278, 278, 278, - /* 1950 */ 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, - /* 1960 */ 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, - /* 1970 */ 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, - /* 1980 */ 122, 278, 278, 278, 278, 127, 128, 278, 278, 278, - /* 1990 */ 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, - /* 2000 */ 278, 278, 278, 278, 278, 278, 278, 278, 150, 278, - /* 2010 */ 278, 278, 278, 278, 278, 278, 278, 278, 278, + /* 0 */ 187, 187, 187, 216, 217, 187, 206, 187, 264, 265, + /* 10 */ 266, 187, 225, 187, 209, 187, 264, 265, 266, 19, + /* 20 */ 187, 187, 209, 210, 209, 210, 187, 209, 210, 209, + /* 30 */ 210, 31, 209, 209, 210, 209, 210, 285, 224, 39, + /* 40 */ 203, 204, 205, 43, 44, 45, 46, 47, 48, 49, + /* 50 */ 50, 51, 52, 53, 54, 55, 56, 57, 230, 19, + /* 60 */ 181, 182, 183, 184, 230, 245, 233, 208, 189, 245, + /* 70 */ 191, 245, 26, 206, 254, 216, 276, 198, 254, 198, + /* 80 */ 254, 281, 187, 43, 44, 45, 46, 47, 48, 49, + /* 90 */ 50, 51, 52, 53, 54, 55, 56, 57, 259, 99, + /* 100 */ 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, + /* 110 */ 231, 232, 231, 232, 286, 302, 303, 302, 22, 304, + /* 120 */ 302, 303, 76, 244, 11, 244, 86, 19, 88, 248, + /* 130 */ 249, 264, 265, 266, 26, 89, 198, 258, 92, 99, + /* 140 */ 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, + /* 150 */ 105, 43, 44, 45, 46, 47, 48, 49, 50, 51, + /* 160 */ 52, 53, 54, 55, 56, 57, 212, 288, 273, 231, + /* 170 */ 232, 105, 106, 107, 108, 109, 131, 69, 203, 204, + /* 180 */ 205, 136, 244, 99, 100, 101, 102, 103, 104, 105, + /* 190 */ 106, 107, 108, 109, 15, 103, 104, 19, 260, 103, + /* 200 */ 104, 54, 55, 56, 57, 58, 22, 99, 100, 101, + /* 210 */ 102, 103, 104, 105, 106, 107, 108, 109, 264, 265, + /* 220 */ 266, 43, 44, 45, 46, 47, 48, 49, 50, 51, + /* 230 */ 52, 53, 54, 55, 56, 57, 19, 124, 125, 60, + /* 240 */ 148, 24, 150, 59, 187, 67, 99, 100, 101, 102, + /* 250 */ 103, 104, 105, 106, 107, 108, 109, 187, 187, 109, + /* 260 */ 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, + /* 270 */ 53, 54, 55, 56, 57, 204, 205, 99, 100, 101, + /* 280 */ 102, 103, 104, 105, 106, 107, 108, 109, 103, 104, + /* 290 */ 105, 106, 107, 108, 109, 59, 112, 113, 114, 76, + /* 300 */ 231, 232, 187, 19, 19, 22, 23, 23, 54, 55, + /* 310 */ 56, 57, 89, 244, 199, 92, 99, 100, 101, 102, + /* 320 */ 103, 104, 105, 106, 107, 108, 109, 43, 44, 45, + /* 330 */ 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, + /* 340 */ 56, 57, 19, 212, 187, 274, 23, 26, 112, 113, + /* 350 */ 114, 294, 295, 99, 100, 101, 102, 103, 104, 105, + /* 360 */ 106, 107, 108, 109, 59, 295, 43, 44, 45, 46, + /* 370 */ 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, + /* 380 */ 57, 98, 146, 99, 100, 101, 102, 103, 104, 105, + /* 390 */ 106, 107, 108, 109, 109, 264, 265, 266, 187, 187, + /* 400 */ 115, 116, 117, 118, 119, 120, 121, 73, 59, 19, + /* 410 */ 105, 23, 127, 23, 26, 81, 259, 112, 113, 114, + /* 420 */ 187, 72, 99, 100, 101, 102, 103, 104, 105, 106, + /* 430 */ 107, 108, 109, 43, 44, 45, 46, 47, 48, 49, + /* 440 */ 50, 51, 52, 53, 54, 55, 56, 57, 19, 124, + /* 450 */ 125, 182, 23, 184, 187, 134, 135, 123, 189, 131, + /* 460 */ 191, 112, 113, 114, 136, 187, 233, 198, 134, 135, + /* 470 */ 198, 259, 43, 44, 45, 46, 47, 48, 49, 50, + /* 480 */ 51, 52, 53, 54, 55, 56, 57, 209, 210, 99, + /* 490 */ 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, + /* 500 */ 231, 232, 206, 231, 232, 187, 59, 296, 297, 76, + /* 510 */ 160, 161, 301, 244, 232, 19, 244, 297, 59, 23, + /* 520 */ 87, 301, 89, 245, 26, 92, 244, 258, 99, 100, + /* 530 */ 101, 102, 103, 104, 105, 106, 107, 108, 109, 43, + /* 540 */ 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, + /* 550 */ 54, 55, 56, 57, 19, 187, 97, 288, 23, 112, + /* 560 */ 113, 114, 115, 296, 297, 118, 119, 120, 301, 108, + /* 570 */ 109, 112, 113, 255, 141, 128, 117, 281, 43, 44, + /* 580 */ 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, + /* 590 */ 55, 56, 57, 187, 187, 99, 100, 101, 102, 103, + /* 600 */ 104, 105, 106, 107, 108, 109, 26, 148, 149, 150, + /* 610 */ 115, 97, 59, 118, 119, 120, 209, 210, 73, 59, + /* 620 */ 122, 19, 209, 128, 256, 72, 187, 113, 187, 81, + /* 630 */ 223, 117, 227, 228, 99, 100, 101, 102, 103, 104, + /* 640 */ 105, 106, 107, 108, 109, 43, 44, 45, 46, 47, + /* 650 */ 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, + /* 660 */ 19, 255, 148, 149, 150, 112, 113, 114, 123, 124, + /* 670 */ 125, 230, 112, 113, 114, 22, 297, 22, 81, 22, + /* 680 */ 301, 59, 134, 135, 43, 44, 45, 46, 47, 48, + /* 690 */ 49, 50, 51, 52, 53, 54, 55, 56, 57, 139, + /* 700 */ 192, 99, 100, 101, 102, 103, 104, 105, 106, 107, + /* 710 */ 108, 109, 59, 116, 59, 187, 59, 231, 232, 46, + /* 720 */ 47, 48, 49, 16, 12, 145, 198, 22, 187, 187, + /* 730 */ 244, 134, 135, 222, 112, 113, 114, 209, 210, 27, + /* 740 */ 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, + /* 750 */ 109, 209, 210, 187, 42, 187, 154, 227, 228, 231, + /* 760 */ 232, 139, 22, 23, 59, 112, 113, 114, 113, 112, + /* 770 */ 113, 114, 244, 245, 233, 63, 19, 209, 210, 271, + /* 780 */ 187, 24, 254, 275, 77, 73, 79, 245, 195, 260, + /* 790 */ 117, 223, 199, 22, 23, 154, 19, 26, 22, 187, + /* 800 */ 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, + /* 810 */ 53, 54, 55, 56, 57, 19, 187, 112, 113, 114, + /* 820 */ 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, + /* 830 */ 53, 54, 55, 56, 57, 59, 263, 187, 98, 43, + /* 840 */ 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, + /* 850 */ 54, 55, 56, 57, 204, 205, 99, 100, 101, 102, + /* 860 */ 103, 104, 105, 106, 107, 108, 109, 255, 130, 98, + /* 870 */ 132, 133, 299, 300, 198, 187, 99, 100, 101, 102, + /* 880 */ 103, 104, 105, 106, 107, 108, 109, 187, 112, 113, + /* 890 */ 114, 187, 241, 187, 243, 99, 100, 101, 102, 103, + /* 900 */ 104, 105, 106, 107, 108, 109, 19, 231, 232, 209, + /* 910 */ 210, 282, 59, 209, 210, 209, 210, 16, 230, 19, + /* 920 */ 244, 22, 23, 223, 274, 26, 19, 223, 187, 223, + /* 930 */ 198, 44, 45, 46, 47, 48, 49, 50, 51, 52, + /* 940 */ 53, 54, 55, 56, 57, 45, 46, 47, 48, 49, + /* 950 */ 50, 51, 52, 53, 54, 55, 56, 57, 19, 20, + /* 960 */ 187, 22, 187, 231, 232, 112, 113, 114, 123, 124, + /* 970 */ 125, 7, 8, 9, 187, 36, 244, 24, 77, 21, + /* 980 */ 79, 187, 209, 210, 187, 263, 99, 100, 101, 102, + /* 990 */ 103, 104, 105, 106, 107, 108, 109, 98, 59, 99, + /* 1000 */ 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, + /* 1010 */ 71, 187, 59, 187, 187, 19, 20, 187, 22, 112, + /* 1020 */ 81, 299, 300, 282, 230, 199, 291, 292, 22, 187, + /* 1030 */ 24, 256, 36, 209, 210, 187, 97, 35, 80, 209, + /* 1040 */ 210, 268, 103, 104, 48, 187, 220, 223, 222, 110, + /* 1050 */ 59, 112, 113, 114, 187, 59, 117, 156, 187, 179, + /* 1060 */ 180, 181, 182, 183, 184, 59, 113, 71, 66, 189, + /* 1070 */ 22, 191, 230, 134, 135, 245, 74, 119, 198, 282, + /* 1080 */ 297, 85, 224, 198, 301, 187, 90, 148, 149, 150, + /* 1090 */ 151, 152, 19, 97, 103, 104, 123, 124, 125, 103, + /* 1100 */ 104, 53, 111, 112, 113, 114, 110, 116, 112, 113, + /* 1110 */ 114, 231, 232, 117, 156, 124, 231, 232, 297, 113, + /* 1120 */ 187, 24, 301, 256, 244, 201, 202, 256, 126, 244, + /* 1130 */ 187, 198, 187, 187, 23, 187, 187, 26, 258, 148, + /* 1140 */ 19, 150, 209, 210, 148, 149, 150, 151, 152, 0, + /* 1150 */ 1, 2, 209, 210, 5, 209, 210, 209, 210, 10, + /* 1160 */ 11, 12, 13, 14, 231, 232, 17, 46, 288, 19, + /* 1170 */ 20, 223, 22, 236, 198, 66, 187, 244, 245, 30, + /* 1180 */ 12, 32, 198, 246, 59, 112, 36, 187, 245, 40, + /* 1190 */ 198, 245, 117, 29, 85, 27, 26, 33, 209, 210, + /* 1200 */ 297, 76, 127, 94, 301, 256, 26, 231, 232, 59, + /* 1210 */ 42, 153, 87, 155, 89, 231, 232, 92, 31, 70, + /* 1220 */ 244, 71, 26, 231, 232, 114, 39, 78, 244, 65, + /* 1230 */ 81, 63, 111, 233, 137, 85, 244, 112, 113, 114, + /* 1240 */ 90, 22, 59, 24, 95, 201, 202, 97, 127, 187, + /* 1250 */ 139, 142, 187, 103, 104, 19, 20, 187, 22, 187, + /* 1260 */ 110, 187, 112, 113, 114, 187, 141, 117, 141, 187, + /* 1270 */ 23, 187, 36, 26, 209, 210, 134, 135, 129, 209, + /* 1280 */ 210, 209, 210, 134, 135, 22, 159, 209, 210, 187, + /* 1290 */ 187, 209, 210, 209, 210, 59, 113, 187, 148, 149, + /* 1300 */ 150, 151, 152, 289, 290, 187, 157, 71, 248, 249, + /* 1310 */ 114, 141, 209, 210, 46, 125, 116, 117, 138, 19, + /* 1320 */ 20, 85, 22, 148, 61, 150, 90, 209, 210, 23, + /* 1330 */ 7, 8, 26, 97, 187, 139, 36, 147, 59, 103, + /* 1340 */ 104, 19, 20, 187, 22, 59, 110, 187, 112, 113, + /* 1350 */ 114, 1, 2, 117, 187, 5, 209, 210, 36, 59, + /* 1360 */ 10, 11, 12, 13, 14, 209, 210, 17, 59, 209, + /* 1370 */ 210, 71, 187, 148, 250, 150, 209, 210, 187, 111, + /* 1380 */ 30, 59, 32, 22, 148, 149, 150, 151, 152, 187, + /* 1390 */ 40, 187, 113, 71, 209, 210, 187, 97, 187, 113, + /* 1400 */ 209, 210, 187, 103, 104, 105, 23, 187, 187, 26, + /* 1410 */ 110, 187, 112, 113, 114, 83, 84, 117, 23, 97, + /* 1420 */ 70, 26, 113, 218, 187, 103, 104, 187, 78, 209, + /* 1430 */ 210, 81, 110, 187, 112, 113, 114, 19, 20, 117, + /* 1440 */ 22, 187, 187, 187, 187, 95, 209, 210, 148, 149, + /* 1450 */ 150, 151, 152, 187, 36, 23, 187, 187, 26, 187, + /* 1460 */ 187, 187, 187, 209, 210, 209, 210, 187, 187, 218, + /* 1470 */ 148, 149, 150, 151, 152, 209, 210, 59, 187, 129, + /* 1480 */ 187, 209, 210, 187, 134, 135, 187, 306, 187, 71, + /* 1490 */ 209, 210, 23, 228, 187, 26, 23, 187, 137, 26, + /* 1500 */ 209, 210, 209, 210, 187, 209, 210, 157, 209, 210, + /* 1510 */ 209, 210, 218, 187, 187, 97, 209, 210, 187, 278, + /* 1520 */ 23, 103, 104, 26, 187, 187, 187, 187, 110, 187, + /* 1530 */ 112, 113, 114, 187, 187, 117, 5, 247, 187, 187, + /* 1540 */ 187, 10, 11, 12, 13, 14, 209, 210, 17, 209, + /* 1550 */ 210, 209, 210, 187, 187, 187, 209, 210, 187, 23, + /* 1560 */ 187, 30, 26, 32, 187, 187, 148, 149, 150, 151, + /* 1570 */ 152, 40, 187, 187, 187, 209, 210, 209, 210, 187, + /* 1580 */ 209, 210, 209, 210, 187, 187, 209, 210, 187, 277, + /* 1590 */ 234, 187, 247, 187, 209, 210, 209, 210, 187, 235, + /* 1600 */ 187, 70, 187, 207, 247, 247, 247, 209, 210, 78, + /* 1610 */ 209, 210, 81, 209, 210, 209, 210, 187, 185, 238, + /* 1620 */ 209, 210, 209, 210, 209, 210, 95, 251, 287, 238, + /* 1630 */ 251, 23, 23, 23, 26, 26, 26, 283, 23, 209, + /* 1640 */ 210, 26, 213, 238, 221, 283, 212, 212, 217, 212, + /* 1650 */ 251, 241, 270, 241, 237, 235, 190, 60, 137, 287, + /* 1660 */ 129, 194, 194, 38, 284, 134, 135, 273, 284, 194, + /* 1670 */ 146, 111, 22, 43, 226, 145, 262, 261, 238, 18, + /* 1680 */ 229, 229, 229, 229, 194, 18, 193, 238, 157, 262, + /* 1690 */ 226, 226, 194, 238, 238, 193, 153, 261, 62, 280, + /* 1700 */ 279, 194, 193, 22, 194, 214, 193, 111, 194, 193, + /* 1710 */ 214, 211, 211, 64, 211, 211, 122, 211, 219, 214, + /* 1720 */ 109, 213, 160, 211, 300, 140, 211, 253, 111, 272, + /* 1730 */ 219, 214, 272, 214, 252, 194, 253, 252, 91, 253, + /* 1740 */ 252, 82, 253, 305, 252, 144, 305, 141, 22, 194, + /* 1750 */ 269, 257, 257, 153, 267, 143, 142, 25, 197, 26, + /* 1760 */ 242, 241, 196, 240, 242, 239, 238, 13, 188, 188, + /* 1770 */ 6, 293, 186, 186, 186, 200, 206, 293, 206, 206, + /* 1780 */ 206, 4, 200, 215, 215, 207, 207, 3, 22, 206, + /* 1790 */ 200, 158, 96, 15, 23, 16, 23, 135, 146, 126, + /* 1800 */ 24, 138, 20, 16, 140, 1, 138, 147, 126, 61, + /* 1810 */ 53, 37, 146, 53, 53, 290, 53, 126, 112, 34, + /* 1820 */ 137, 1, 5, 22, 111, 156, 68, 68, 26, 41, + /* 1830 */ 75, 137, 111, 24, 20, 19, 127, 121, 23, 67, + /* 1840 */ 22, 22, 67, 22, 22, 37, 22, 67, 23, 145, + /* 1850 */ 22, 28, 23, 23, 23, 137, 23, 22, 26, 22, + /* 1860 */ 24, 23, 112, 24, 23, 23, 22, 139, 34, 26, + /* 1870 */ 75, 88, 86, 75, 34, 23, 22, 24, 22, 34, + /* 1880 */ 34, 34, 93, 34, 26, 26, 23, 23, 23, 34, + /* 1890 */ 23, 23, 44, 23, 11, 26, 22, 22, 26, 23, + /* 1900 */ 23, 22, 22, 15, 137, 137, 137, 137, 23, 1, + /* 1910 */ 307, 307, 131, 307, 307, 307, 307, 307, 307, 307, + /* 1920 */ 307, 307, 307, 307, 307, 307, 307, 307, 307, 307, + /* 1930 */ 307, 307, 307, 307, 307, 307, 307, 307, 307, 307, + /* 1940 */ 307, 307, 307, 307, 307, 307, 307, 307, 307, 307, + /* 1950 */ 307, 307, 307, 307, 307, 307, 307, 307, 307, 307, + /* 1960 */ 307, 307, 307, 307, 307, 307, 307, 307, 307, 307, + /* 1970 */ 307, 307, 307, 307, 307, 307, 307, 307, 307, 307, + /* 1980 */ 307, 307, 307, 307, 307, 307, 307, 307, 307, 307, + /* 1990 */ 307, 307, 307, 307, 307, 307, 307, 307, 307, 307, + /* 2000 */ 307, 307, 307, 307, 307, 307, 307, 307, 307, 307, + /* 2010 */ 307, 307, 307, 307, 307, 307, 307, 307, 307, 307, + /* 2020 */ 307, 307, 307, 307, 307, 307, 307, 307, 307, 307, + /* 2030 */ 307, 307, 307, 307, 307, 307, 307, 307, 307, 307, + /* 2040 */ 307, 307, 307, 307, 307, 307, 307, 307, 307, 307, + /* 2050 */ 307, 307, 307, 307, 307, 307, 307, 307, 307, 307, + /* 2060 */ 307, 307, 307, 307, 307, 307, 307, 307, 307, 307, + /* 2070 */ 307, 307, 307, 307, 307, 307, 307, 307, 307, 307, + /* 2080 */ 307, 307, 307, 307, 307, 307, 307, 307, 307, 307, + /* 2090 */ 307, 307, }; -#define YY_SHIFT_COUNT (523) +#define YY_SHIFT_COUNT (542) #define YY_SHIFT_MIN (0) -#define YY_SHIFT_MAX (1858) +#define YY_SHIFT_MAX (1908) static const unsigned short int yy_shift_ofst[] = { - /* 0 */ 1709, 1520, 1858, 1324, 1324, 24, 1374, 1469, 1602, 1712, - /* 10 */ 1712, 1712, 271, 0, 0, 113, 1016, 1712, 1712, 1712, - /* 20 */ 1712, 1712, 1712, 1712, 1712, 1712, 1712, 12, 12, 409, - /* 30 */ 596, 24, 24, 24, 24, 24, 24, 93, 177, 270, - /* 40 */ 363, 456, 549, 642, 735, 828, 848, 996, 1144, 1016, - /* 50 */ 1016, 1016, 1016, 1016, 1016, 1016, 1016, 1016, 1016, 1016, - /* 60 */ 1016, 1016, 1016, 1016, 1016, 1016, 1016, 1164, 1016, 1257, - /* 70 */ 1277, 1277, 1490, 1712, 1712, 1712, 1712, 1712, 1712, 1712, - /* 80 */ 1712, 1712, 1712, 1712, 1712, 1712, 1712, 1712, 1712, 1712, - /* 90 */ 1712, 1712, 1712, 1712, 1712, 1712, 1712, 1712, 1712, 1712, - /* 100 */ 1712, 1712, 1712, 1712, 1712, 1742, 1712, 1712, 1712, 1712, - /* 110 */ 1712, 1712, 1712, 1712, 1712, 1712, 1712, 1712, 1712, 143, - /* 120 */ 162, 162, 162, 162, 162, 204, 151, 186, 650, 690, - /* 130 */ 327, 650, 261, 261, 650, 722, 722, 722, 722, 373, - /* 140 */ 33, 2, 2009, 2009, 330, 330, 330, 346, 289, 278, - /* 150 */ 289, 289, 517, 517, 459, 510, 15, 799, 650, 650, - /* 160 */ 650, 650, 650, 650, 650, 650, 650, 650, 650, 650, - /* 170 */ 650, 650, 650, 650, 650, 650, 650, 650, 650, 650, - /* 180 */ 331, 365, 995, 995, 265, 365, 50, 1038, 2009, 2009, - /* 190 */ 2009, 433, 250, 250, 504, 314, 429, 518, 522, 526, - /* 200 */ 561, 650, 650, 650, 650, 650, 650, 650, 650, 650, - /* 210 */ 192, 650, 650, 650, 650, 650, 650, 650, 650, 650, - /* 220 */ 650, 650, 650, 641, 641, 641, 650, 650, 650, 650, - /* 230 */ 800, 650, 650, 650, 830, 650, 650, 782, 650, 650, - /* 240 */ 650, 650, 650, 650, 650, 650, 739, 902, 689, 895, - /* 250 */ 895, 895, 895, 736, 689, 689, 885, 445, 903, 1124, - /* 260 */ 945, 748, 748, 1066, 945, 945, 1066, 447, 1002, 293, - /* 270 */ 1195, 1195, 1195, 748, 740, 727, 460, 1157, 1348, 1282, - /* 280 */ 1282, 1378, 1378, 1282, 1279, 1315, 1402, 1383, 1294, 1419, - /* 290 */ 1419, 1419, 1419, 1282, 1428, 1294, 1294, 1315, 1402, 1383, - /* 300 */ 1383, 1294, 1282, 1428, 1314, 1400, 1282, 1428, 1453, 1282, - /* 310 */ 1428, 1282, 1428, 1453, 1386, 1386, 1386, 1431, 1453, 1386, - /* 320 */ 1381, 1386, 1431, 1386, 1386, 1453, 1399, 1399, 1453, 1369, - /* 330 */ 1403, 1369, 1403, 1369, 1403, 1369, 1403, 1282, 1404, 1452, - /* 340 */ 1521, 1407, 1404, 1524, 1282, 1416, 1407, 1432, 1434, 1294, - /* 350 */ 1546, 1547, 1569, 1569, 1577, 1577, 1577, 2009, 2009, 2009, - /* 360 */ 2009, 2009, 2009, 2009, 2009, 2009, 2009, 2009, 2009, 2009, - /* 370 */ 2009, 2009, 2009, 591, 697, 1059, 1139, 1058, 797, 465, - /* 380 */ 1159, 1182, 1122, 1062, 1180, 936, 1199, 1201, 1205, 1224, - /* 390 */ 1225, 1244, 1061, 1145, 1261, 1161, 1194, 1249, 1251, 1256, - /* 400 */ 1137, 1142, 1263, 1264, 1214, 1207, 1613, 1623, 1605, 1477, - /* 410 */ 1614, 1541, 1620, 1616, 1617, 1509, 1502, 1525, 1619, 1514, - /* 420 */ 1626, 1516, 1634, 1650, 1522, 1512, 1535, 1594, 1621, 1517, - /* 430 */ 1604, 1606, 1607, 1609, 1544, 1559, 1631, 1536, 1666, 1663, - /* 440 */ 1647, 1567, 1523, 1608, 1648, 1610, 1600, 1636, 1549, 1576, - /* 450 */ 1657, 1662, 1664, 1565, 1572, 1665, 1624, 1668, 1671, 1672, - /* 460 */ 1674, 1627, 1660, 1675, 1633, 1667, 1678, 1564, 1681, 1553, - /* 470 */ 1690, 1692, 1691, 1693, 1696, 1700, 1702, 1705, 1704, 1599, - /* 480 */ 1707, 1710, 1630, 1703, 1714, 1618, 1716, 1706, 1716, 1717, - /* 490 */ 1653, 1677, 1670, 1711, 1731, 1732, 1733, 1734, 1723, 1735, - /* 500 */ 1716, 1740, 1743, 1744, 1745, 1739, 1746, 1748, 1761, 1751, - /* 510 */ 1752, 1753, 1754, 1758, 1759, 1749, 1658, 1654, 1655, 1656, - /* 520 */ 1659, 1765, 1776, 1791, + /* 0 */ 1350, 1149, 1531, 939, 939, 548, 996, 1150, 1236, 1322, + /* 10 */ 1322, 1322, 334, 0, 0, 178, 777, 1322, 1322, 1322, + /* 20 */ 1322, 1322, 1322, 1322, 1322, 1322, 1322, 1322, 1322, 1322, + /* 30 */ 991, 991, 1125, 1125, 447, 597, 548, 548, 548, 548, + /* 40 */ 548, 548, 40, 108, 217, 284, 323, 390, 429, 496, + /* 50 */ 535, 602, 641, 757, 777, 777, 777, 777, 777, 777, + /* 60 */ 777, 777, 777, 777, 777, 777, 777, 777, 777, 777, + /* 70 */ 777, 777, 796, 777, 887, 900, 900, 1300, 1322, 1322, + /* 80 */ 1322, 1322, 1322, 1322, 1322, 1322, 1322, 1322, 1322, 1322, + /* 90 */ 1322, 1322, 1322, 1322, 1322, 1322, 1322, 1322, 1322, 1322, + /* 100 */ 1322, 1322, 1322, 1322, 1322, 1322, 1322, 1322, 1322, 1322, + /* 110 */ 1418, 1322, 1322, 1322, 1322, 1322, 1322, 1322, 1322, 1322, + /* 120 */ 1322, 1322, 1322, 1322, 147, 254, 254, 254, 254, 254, + /* 130 */ 84, 185, 66, 853, 958, 1121, 853, 92, 92, 853, + /* 140 */ 321, 321, 321, 321, 325, 350, 350, 461, 150, 1913, + /* 150 */ 1913, 285, 285, 285, 236, 184, 349, 184, 184, 712, + /* 160 */ 712, 433, 553, 771, 899, 853, 853, 853, 853, 853, + /* 170 */ 853, 853, 853, 853, 853, 853, 853, 853, 853, 853, + /* 180 */ 853, 853, 853, 853, 853, 853, 46, 46, 853, 113, + /* 190 */ 223, 223, 1183, 1183, 1127, 1142, 1913, 1913, 1913, 459, + /* 200 */ 514, 514, 653, 495, 657, 305, 705, 560, 622, 776, + /* 210 */ 853, 853, 853, 853, 853, 853, 853, 853, 853, 545, + /* 220 */ 853, 853, 853, 853, 853, 853, 853, 853, 853, 853, + /* 230 */ 853, 853, 1002, 1002, 1002, 853, 853, 853, 853, 1111, + /* 240 */ 853, 853, 853, 1006, 1109, 853, 853, 1168, 853, 853, + /* 250 */ 853, 853, 853, 853, 853, 853, 845, 1164, 738, 953, + /* 260 */ 953, 953, 953, 1196, 738, 738, 45, 96, 964, 179, + /* 270 */ 580, 907, 907, 1073, 580, 580, 1073, 498, 388, 1268, + /* 280 */ 1187, 1187, 1187, 907, 1170, 1170, 1058, 1180, 328, 1219, + /* 290 */ 1597, 1521, 1521, 1625, 1625, 1521, 1524, 1560, 1650, 1630, + /* 300 */ 1530, 1661, 1661, 1661, 1661, 1521, 1667, 1530, 1530, 1560, + /* 310 */ 1650, 1630, 1630, 1530, 1521, 1667, 1543, 1636, 1521, 1667, + /* 320 */ 1681, 1521, 1667, 1521, 1667, 1681, 1596, 1596, 1596, 1649, + /* 330 */ 1681, 1596, 1594, 1596, 1649, 1596, 1596, 1562, 1681, 1611, + /* 340 */ 1611, 1681, 1585, 1617, 1585, 1617, 1585, 1617, 1585, 1617, + /* 350 */ 1521, 1647, 1647, 1659, 1659, 1601, 1606, 1726, 1521, 1600, + /* 360 */ 1601, 1612, 1614, 1530, 1732, 1733, 1754, 1754, 1764, 1764, + /* 370 */ 1764, 1913, 1913, 1913, 1913, 1913, 1913, 1913, 1913, 1913, + /* 380 */ 1913, 1913, 1913, 1913, 1913, 1913, 673, 901, 283, 740, + /* 390 */ 707, 973, 655, 1247, 1048, 1097, 1190, 1306, 1263, 1383, + /* 400 */ 1395, 1432, 1469, 1473, 1497, 1279, 1200, 1323, 1075, 1286, + /* 410 */ 1536, 1608, 1332, 1609, 1175, 1225, 1610, 1615, 1309, 1361, + /* 420 */ 1777, 1784, 1766, 1633, 1778, 1696, 1779, 1771, 1773, 1662, + /* 430 */ 1652, 1673, 1776, 1663, 1782, 1664, 1787, 1804, 1668, 1660, + /* 440 */ 1682, 1748, 1774, 1666, 1757, 1760, 1761, 1763, 1691, 1706, + /* 450 */ 1785, 1683, 1820, 1817, 1801, 1713, 1669, 1758, 1802, 1759, + /* 460 */ 1755, 1788, 1694, 1721, 1809, 1814, 1816, 1709, 1716, 1818, + /* 470 */ 1772, 1819, 1821, 1815, 1822, 1775, 1823, 1824, 1780, 1808, + /* 480 */ 1825, 1704, 1828, 1829, 1830, 1831, 1832, 1833, 1835, 1836, + /* 490 */ 1838, 1837, 1839, 1718, 1841, 1842, 1750, 1834, 1844, 1728, + /* 500 */ 1843, 1840, 1845, 1846, 1847, 1783, 1795, 1786, 1848, 1798, + /* 510 */ 1789, 1849, 1852, 1854, 1853, 1858, 1859, 1855, 1863, 1843, + /* 520 */ 1864, 1865, 1867, 1868, 1869, 1870, 1856, 1883, 1874, 1875, + /* 530 */ 1876, 1877, 1879, 1880, 1872, 1781, 1767, 1768, 1769, 1770, + /* 540 */ 1885, 1888, 1908, }; -#define YY_REDUCE_COUNT (372) -#define YY_REDUCE_MIN (-235) -#define YY_REDUCE_MAX (1441) +#define YY_REDUCE_COUNT (385) +#define YY_REDUCE_MIN (-256) +#define YY_REDUCE_MAX (1590) static const short yy_reduce_ofst[] = { - /* 0 */ -147, 171, 263, -96, 169, -144, -162, -149, -102, -156, - /* 10 */ -98, 216, 354, -170, -57, -235, 307, 149, 423, 428, - /* 20 */ 471, 313, 451, 519, 489, 496, 499, 545, 547, 555, - /* 30 */ -116, 540, 558, 592, 594, 597, 599, -206, -206, -206, - /* 40 */ -206, -206, -206, -206, -206, -206, -206, -206, -206, -206, - /* 50 */ -206, -206, -206, -206, -206, -206, -206, -206, -206, -206, - /* 60 */ -206, -206, -206, -206, -206, -206, -206, -206, -206, -206, - /* 70 */ -206, -206, 196, 309, 494, 537, 612, 656, 675, 679, - /* 80 */ 681, 685, 724, 753, 771, 776, 788, 790, 794, 796, - /* 90 */ 801, 803, 805, 807, 814, 819, 833, 837, 839, 842, - /* 100 */ 845, 847, 849, 853, 873, 891, 893, 917, 921, 937, - /* 110 */ 940, 944, 956, 960, 967, 969, 971, 973, 975, -206, - /* 120 */ -206, -206, -206, -206, -206, -206, -206, -206, 501, -168, - /* 130 */ 90, -97, 87, 112, 303, 277, 601, 277, 601, 179, - /* 140 */ -206, -206, -206, -206, -107, -107, -107, -43, -56, 323, - /* 150 */ 500, 512, -187, -177, 317, 609, 353, 353, 120, 144, - /* 160 */ 490, 539, 698, 374, 467, 507, 789, 404, -157, 755, - /* 170 */ 856, 916, 843, 941, 802, 770, 923, 821, 1001, -142, - /* 180 */ 264, 785, 896, 905, 899, 949, -176, 544, 911, 953, - /* 190 */ 1012, -182, -59, -30, 16, -22, 117, 172, 291, 369, - /* 200 */ 407, 415, 566, 586, 647, 699, 754, 813, 850, 892, - /* 210 */ 121, 1023, 1042, 1086, 1121, 1125, 1128, 1129, 1130, 1131, - /* 220 */ 1132, 1134, 1135, 284, 1106, 1123, 1152, 1154, 1155, 1156, - /* 230 */ 397, 1158, 1172, 1173, 1116, 1176, 1177, 1138, 1179, 117, - /* 240 */ 1184, 1185, 1198, 1200, 1202, 1203, 741, 1094, 1153, 1146, - /* 250 */ 1160, 1162, 1163, 397, 1153, 1153, 1170, 1204, 1206, 1103, - /* 260 */ 1168, 1165, 1166, 1133, 1174, 1175, 1140, 1210, 1193, 1208, - /* 270 */ 1212, 1215, 1216, 1178, 1167, 1189, 1196, 1241, 1148, 1243, - /* 280 */ 1245, 1181, 1183, 1247, 1188, 1187, 1190, 1227, 1223, 1234, - /* 290 */ 1236, 1238, 1239, 1274, 1278, 1235, 1237, 1213, 1218, 1253, - /* 300 */ 1254, 1246, 1287, 1289, 1209, 1219, 1303, 1305, 1293, 1306, - /* 310 */ 1309, 1313, 1316, 1297, 1301, 1307, 1308, 1298, 1310, 1311, - /* 320 */ 1312, 1317, 1304, 1318, 1320, 1319, 1265, 1267, 1325, 1295, - /* 330 */ 1300, 1296, 1302, 1326, 1321, 1327, 1330, 1365, 1323, 1269, - /* 340 */ 1272, 1328, 1331, 1322, 1388, 1334, 1336, 1349, 1353, 1357, - /* 350 */ 1405, 1409, 1420, 1421, 1427, 1429, 1430, 1332, 1335, 1339, - /* 360 */ 1418, 1422, 1423, 1424, 1425, 1433, 1426, 1435, 1436, 1437, - /* 370 */ 1438, 1441, 1439, + /* 0 */ 880, -121, 269, 528, 933, -119, -187, -185, -182, -180, + /* 10 */ -176, -174, -62, -46, 131, -248, -133, 407, 568, 700, + /* 20 */ 704, 278, 706, 824, 542, 830, 948, 773, 943, 946, + /* 30 */ 71, 650, 211, 267, 826, 272, 676, 732, 885, 976, + /* 40 */ 984, 992, -256, -256, -256, -256, -256, -256, -256, -256, + /* 50 */ -256, -256, -256, -256, -256, -256, -256, -256, -256, -256, + /* 60 */ -256, -256, -256, -256, -256, -256, -256, -256, -256, -256, + /* 70 */ -256, -256, -256, -256, -256, -256, -256, 989, 1065, 1070, + /* 80 */ 1072, 1078, 1082, 1084, 1103, 1118, 1147, 1156, 1160, 1167, + /* 90 */ 1185, 1191, 1220, 1237, 1254, 1256, 1266, 1272, 1281, 1291, + /* 100 */ 1293, 1296, 1299, 1301, 1307, 1337, 1340, 1342, 1347, 1366, + /* 110 */ 1368, 1371, 1373, 1377, 1385, 1387, 1398, 1401, 1404, 1406, + /* 120 */ 1411, 1413, 1415, 1430, -256, -256, -256, -256, -256, -256, + /* 130 */ -256, -256, -256, -172, 508, -213, 57, -163, -25, 593, + /* 140 */ 69, 486, 69, 486, -200, 573, 722, -256, -256, -256, + /* 150 */ -256, -141, -141, -141, -105, -161, -167, 157, 212, 405, + /* 160 */ 530, 220, 233, 735, 735, 115, 318, 406, 612, 541, + /* 170 */ -166, 441, 688, 794, 629, 368, 741, 775, 867, 797, + /* 180 */ 871, 842, -186, 1000, 858, 949, 379, 783, 70, 296, + /* 190 */ 821, 903, 924, 1044, 651, 282, 1014, 1060, 937, -195, + /* 200 */ -177, 413, 439, 511, 566, 787, 827, 848, 898, 945, + /* 210 */ 1062, 1074, 1102, 1110, 1202, 1204, 1209, 1211, 1215, 529, + /* 220 */ 1221, 1224, 1240, 1246, 1255, 1257, 1269, 1270, 1273, 1274, + /* 230 */ 1275, 1280, 1205, 1251, 1294, 1310, 1317, 1326, 1327, 1124, + /* 240 */ 1331, 1338, 1339, 1290, 1181, 1346, 1351, 1265, 1352, 787, + /* 250 */ 1353, 1367, 1378, 1386, 1392, 1397, 1241, 1312, 1356, 1345, + /* 260 */ 1357, 1358, 1359, 1124, 1356, 1356, 1364, 1396, 1433, 1341, + /* 270 */ 1381, 1376, 1379, 1354, 1391, 1405, 1362, 1429, 1423, 1431, + /* 280 */ 1434, 1435, 1437, 1399, 1410, 1412, 1382, 1417, 1420, 1466, + /* 290 */ 1372, 1467, 1468, 1380, 1384, 1475, 1394, 1414, 1416, 1448, + /* 300 */ 1440, 1451, 1452, 1453, 1454, 1490, 1493, 1449, 1455, 1427, + /* 310 */ 1436, 1464, 1465, 1456, 1498, 1502, 1419, 1421, 1507, 1509, + /* 320 */ 1491, 1510, 1513, 1514, 1516, 1496, 1500, 1501, 1503, 1499, + /* 330 */ 1505, 1504, 1508, 1506, 1511, 1512, 1515, 1424, 1517, 1457, + /* 340 */ 1460, 1519, 1474, 1482, 1483, 1485, 1486, 1488, 1489, 1492, + /* 350 */ 1541, 1438, 1441, 1494, 1495, 1518, 1520, 1487, 1555, 1481, + /* 360 */ 1522, 1523, 1526, 1528, 1561, 1566, 1580, 1581, 1586, 1587, + /* 370 */ 1588, 1478, 1484, 1525, 1575, 1570, 1572, 1573, 1574, 1582, + /* 380 */ 1568, 1569, 1578, 1579, 1583, 1590, }; static const YYACTIONTYPE yy_default[] = { - /* 0 */ 1500, 1500, 1500, 1346, 1129, 1235, 1129, 1129, 1129, 1346, - /* 10 */ 1346, 1346, 1129, 1265, 1265, 1399, 1160, 1129, 1129, 1129, - /* 20 */ 1129, 1129, 1129, 1129, 1345, 1129, 1129, 1129, 1129, 1129, - /* 30 */ 1129, 1129, 1129, 1129, 1129, 1129, 1129, 1129, 1271, 1129, - /* 40 */ 1129, 1129, 1129, 1129, 1347, 1348, 1129, 1129, 1129, 1398, - /* 50 */ 1400, 1363, 1281, 1280, 1279, 1278, 1381, 1252, 1276, 1269, - /* 60 */ 1273, 1341, 1342, 1340, 1344, 1348, 1347, 1129, 1272, 1312, - /* 70 */ 1326, 1311, 1129, 1129, 1129, 1129, 1129, 1129, 1129, 1129, - /* 80 */ 1129, 1129, 1129, 1129, 1129, 1129, 1129, 1129, 1129, 1129, - /* 90 */ 1129, 1129, 1129, 1129, 1129, 1129, 1129, 1129, 1129, 1129, - /* 100 */ 1129, 1129, 1129, 1129, 1129, 1129, 1129, 1129, 1129, 1129, - /* 110 */ 1129, 1129, 1129, 1129, 1129, 1129, 1129, 1129, 1129, 1320, - /* 120 */ 1325, 1331, 1324, 1321, 1314, 1313, 1315, 1316, 1129, 1150, - /* 130 */ 1199, 1129, 1129, 1129, 1129, 1417, 1416, 1129, 1129, 1160, - /* 140 */ 1317, 1318, 1328, 1327, 1406, 1456, 1455, 1364, 1129, 1129, - /* 150 */ 1129, 1129, 1129, 1129, 1129, 1129, 1129, 1129, 1129, 1129, - /* 160 */ 1129, 1129, 1129, 1129, 1129, 1129, 1129, 1129, 1129, 1129, - /* 170 */ 1129, 1129, 1129, 1129, 1129, 1129, 1129, 1129, 1129, 1129, - /* 180 */ 1160, 1156, 1306, 1305, 1426, 1156, 1259, 1129, 1412, 1235, - /* 190 */ 1226, 1129, 1129, 1129, 1129, 1129, 1129, 1129, 1129, 1129, - /* 200 */ 1129, 1129, 1129, 1129, 1403, 1401, 1129, 1129, 1129, 1129, - /* 210 */ 1129, 1129, 1129, 1129, 1129, 1129, 1129, 1129, 1129, 1129, - /* 220 */ 1129, 1129, 1129, 1129, 1129, 1129, 1129, 1129, 1129, 1129, - /* 230 */ 1129, 1129, 1129, 1129, 1231, 1129, 1129, 1129, 1129, 1129, - /* 240 */ 1129, 1129, 1129, 1129, 1129, 1450, 1129, 1376, 1213, 1231, - /* 250 */ 1231, 1231, 1231, 1233, 1214, 1212, 1225, 1160, 1136, 1492, - /* 260 */ 1275, 1254, 1254, 1489, 1275, 1275, 1489, 1174, 1470, 1171, - /* 270 */ 1265, 1265, 1265, 1254, 1343, 1232, 1225, 1129, 1492, 1240, - /* 280 */ 1240, 1491, 1491, 1240, 1364, 1284, 1290, 1202, 1275, 1208, - /* 290 */ 1208, 1208, 1208, 1240, 1147, 1275, 1275, 1284, 1290, 1202, - /* 300 */ 1202, 1275, 1240, 1147, 1380, 1486, 1240, 1147, 1354, 1240, - /* 310 */ 1147, 1240, 1147, 1354, 1200, 1200, 1200, 1189, 1354, 1200, - /* 320 */ 1174, 1200, 1189, 1200, 1200, 1354, 1358, 1358, 1354, 1258, - /* 330 */ 1253, 1258, 1253, 1258, 1253, 1258, 1253, 1240, 1259, 1425, - /* 340 */ 1129, 1270, 1259, 1349, 1240, 1129, 1270, 1268, 1266, 1275, - /* 350 */ 1153, 1192, 1453, 1453, 1449, 1449, 1449, 1497, 1497, 1412, - /* 360 */ 1465, 1160, 1160, 1160, 1160, 1465, 1176, 1176, 1160, 1160, - /* 370 */ 1160, 1160, 1465, 1129, 1129, 1129, 1129, 1129, 1129, 1460, - /* 380 */ 1129, 1365, 1244, 1129, 1129, 1129, 1129, 1129, 1129, 1129, - /* 390 */ 1129, 1129, 1129, 1129, 1129, 1129, 1129, 1129, 1129, 1129, - /* 400 */ 1129, 1129, 1129, 1129, 1129, 1295, 1129, 1132, 1409, 1129, - /* 410 */ 1129, 1407, 1129, 1129, 1129, 1129, 1129, 1129, 1245, 1129, - /* 420 */ 1129, 1129, 1129, 1129, 1129, 1129, 1129, 1129, 1129, 1129, - /* 430 */ 1129, 1129, 1129, 1129, 1129, 1129, 1129, 1488, 1129, 1129, - /* 440 */ 1129, 1129, 1129, 1129, 1379, 1378, 1129, 1129, 1242, 1129, - /* 450 */ 1129, 1129, 1129, 1129, 1129, 1129, 1129, 1129, 1129, 1129, - /* 460 */ 1129, 1129, 1129, 1129, 1129, 1129, 1129, 1129, 1129, 1129, - /* 470 */ 1129, 1129, 1129, 1129, 1129, 1129, 1129, 1129, 1129, 1129, - /* 480 */ 1129, 1129, 1129, 1129, 1129, 1129, 1267, 1129, 1424, 1129, - /* 490 */ 1129, 1129, 1129, 1129, 1129, 1129, 1438, 1260, 1129, 1129, - /* 500 */ 1479, 1129, 1129, 1129, 1129, 1129, 1129, 1129, 1129, 1129, - /* 510 */ 1129, 1129, 1129, 1129, 1129, 1474, 1216, 1297, 1129, 1296, - /* 520 */ 1300, 1129, 1141, 1129, + /* 0 */ 1554, 1554, 1554, 1392, 1171, 1278, 1171, 1171, 1171, 1392, + /* 10 */ 1392, 1392, 1171, 1308, 1308, 1445, 1202, 1171, 1171, 1171, + /* 20 */ 1171, 1171, 1171, 1171, 1171, 1171, 1171, 1391, 1171, 1171, + /* 30 */ 1171, 1171, 1475, 1475, 1171, 1171, 1171, 1171, 1171, 1171, + /* 40 */ 1171, 1171, 1171, 1317, 1171, 1171, 1171, 1171, 1171, 1393, + /* 50 */ 1394, 1171, 1171, 1171, 1444, 1446, 1409, 1327, 1326, 1325, + /* 60 */ 1324, 1427, 1295, 1322, 1315, 1319, 1387, 1388, 1386, 1390, + /* 70 */ 1394, 1393, 1171, 1318, 1358, 1372, 1357, 1171, 1171, 1171, + /* 80 */ 1171, 1171, 1171, 1171, 1171, 1171, 1171, 1171, 1171, 1171, + /* 90 */ 1171, 1171, 1171, 1171, 1171, 1171, 1171, 1171, 1171, 1171, + /* 100 */ 1171, 1171, 1171, 1171, 1171, 1171, 1171, 1171, 1171, 1171, + /* 110 */ 1171, 1171, 1171, 1171, 1171, 1171, 1171, 1171, 1171, 1171, + /* 120 */ 1171, 1171, 1171, 1171, 1366, 1371, 1377, 1370, 1367, 1360, + /* 130 */ 1359, 1361, 1362, 1171, 1192, 1242, 1171, 1171, 1171, 1171, + /* 140 */ 1463, 1462, 1171, 1171, 1202, 1352, 1351, 1363, 1364, 1374, + /* 150 */ 1373, 1452, 1510, 1509, 1410, 1171, 1171, 1171, 1171, 1171, + /* 160 */ 1171, 1475, 1171, 1171, 1171, 1171, 1171, 1171, 1171, 1171, + /* 170 */ 1171, 1171, 1171, 1171, 1171, 1171, 1171, 1171, 1171, 1171, + /* 180 */ 1171, 1171, 1171, 1171, 1171, 1171, 1475, 1475, 1171, 1202, + /* 190 */ 1475, 1475, 1198, 1198, 1302, 1171, 1458, 1278, 1269, 1171, + /* 200 */ 1171, 1171, 1171, 1171, 1171, 1171, 1171, 1171, 1171, 1171, + /* 210 */ 1171, 1171, 1171, 1449, 1447, 1171, 1171, 1171, 1171, 1171, + /* 220 */ 1171, 1171, 1171, 1171, 1171, 1171, 1171, 1171, 1171, 1171, + /* 230 */ 1171, 1171, 1171, 1171, 1171, 1171, 1171, 1171, 1171, 1171, + /* 240 */ 1171, 1171, 1171, 1274, 1171, 1171, 1171, 1171, 1171, 1171, + /* 250 */ 1171, 1171, 1171, 1171, 1171, 1504, 1171, 1422, 1256, 1274, + /* 260 */ 1274, 1274, 1274, 1276, 1257, 1255, 1268, 1203, 1178, 1546, + /* 270 */ 1321, 1297, 1297, 1543, 1321, 1321, 1543, 1217, 1524, 1214, + /* 280 */ 1308, 1308, 1308, 1297, 1302, 1302, 1389, 1275, 1268, 1171, + /* 290 */ 1546, 1283, 1283, 1545, 1545, 1283, 1410, 1330, 1336, 1245, + /* 300 */ 1321, 1251, 1251, 1251, 1251, 1283, 1189, 1321, 1321, 1330, + /* 310 */ 1336, 1245, 1245, 1321, 1283, 1189, 1426, 1540, 1283, 1189, + /* 320 */ 1400, 1283, 1189, 1283, 1189, 1400, 1243, 1243, 1243, 1232, + /* 330 */ 1400, 1243, 1217, 1243, 1232, 1243, 1243, 1493, 1400, 1404, + /* 340 */ 1404, 1400, 1301, 1296, 1301, 1296, 1301, 1296, 1301, 1296, + /* 350 */ 1283, 1485, 1485, 1311, 1311, 1316, 1302, 1395, 1283, 1171, + /* 360 */ 1316, 1314, 1312, 1321, 1195, 1235, 1507, 1507, 1503, 1503, + /* 370 */ 1503, 1551, 1551, 1458, 1519, 1202, 1202, 1202, 1202, 1519, + /* 380 */ 1219, 1219, 1203, 1203, 1202, 1519, 1171, 1171, 1171, 1171, + /* 390 */ 1171, 1171, 1514, 1171, 1411, 1287, 1171, 1171, 1171, 1171, + /* 400 */ 1171, 1171, 1171, 1171, 1171, 1171, 1171, 1171, 1171, 1171, + /* 410 */ 1171, 1171, 1171, 1171, 1171, 1171, 1171, 1171, 1171, 1341, + /* 420 */ 1171, 1174, 1455, 1171, 1171, 1453, 1171, 1171, 1171, 1171, + /* 430 */ 1171, 1171, 1288, 1171, 1171, 1171, 1171, 1171, 1171, 1171, + /* 440 */ 1171, 1171, 1171, 1171, 1171, 1171, 1171, 1171, 1171, 1171, + /* 450 */ 1171, 1542, 1171, 1171, 1171, 1171, 1171, 1171, 1425, 1424, + /* 460 */ 1171, 1171, 1285, 1171, 1171, 1171, 1171, 1171, 1171, 1171, + /* 470 */ 1171, 1171, 1171, 1171, 1171, 1171, 1171, 1171, 1171, 1171, + /* 480 */ 1171, 1171, 1171, 1171, 1171, 1171, 1171, 1171, 1171, 1171, + /* 490 */ 1171, 1171, 1171, 1171, 1171, 1171, 1171, 1171, 1171, 1171, + /* 500 */ 1313, 1171, 1171, 1171, 1171, 1171, 1171, 1171, 1171, 1171, + /* 510 */ 1171, 1171, 1171, 1171, 1171, 1490, 1303, 1171, 1171, 1533, + /* 520 */ 1171, 1171, 1171, 1171, 1171, 1171, 1171, 1171, 1171, 1171, + /* 530 */ 1171, 1171, 1171, 1171, 1528, 1259, 1343, 1171, 1342, 1346, + /* 540 */ 1171, 1183, 1171, }; /********** End of lemon-generated parsing tables *****************************/ @@ -148340,15 +150692,103 @@ static const YYCODETYPE yyFallback[] = { 59, /* VIEW => ID */ 59, /* VIRTUAL => ID */ 59, /* WITH => ID */ + 59, /* NULLS => ID */ + 59, /* FIRST => ID */ + 59, /* LAST => ID */ 59, /* CURRENT => ID */ 59, /* FOLLOWING => ID */ 59, /* PARTITION => ID */ 59, /* PRECEDING => ID */ 59, /* RANGE => ID */ 59, /* UNBOUNDED => ID */ + 59, /* EXCLUDE => ID */ + 59, /* GROUPS => ID */ + 59, /* OTHERS => ID */ + 59, /* TIES => ID */ 59, /* REINDEX => ID */ 59, /* RENAME => ID */ 59, /* CTIME_KW => ID */ + 0, /* ANY => nothing */ + 0, /* BITAND => nothing */ + 0, /* BITOR => nothing */ + 0, /* LSHIFT => nothing */ + 0, /* RSHIFT => nothing */ + 0, /* PLUS => nothing */ + 0, /* MINUS => nothing */ + 0, /* STAR => nothing */ + 0, /* SLASH => nothing */ + 0, /* REM => nothing */ + 0, /* CONCAT => nothing */ + 0, /* COLLATE => nothing */ + 0, /* BITNOT => nothing */ + 0, /* ON => nothing */ + 0, /* INDEXED => nothing */ + 0, /* STRING => nothing */ + 0, /* JOIN_KW => nothing */ + 0, /* CONSTRAINT => nothing */ + 0, /* DEFAULT => nothing */ + 0, /* NULL => nothing */ + 0, /* PRIMARY => nothing */ + 0, /* UNIQUE => nothing */ + 0, /* CHECK => nothing */ + 0, /* REFERENCES => nothing */ + 0, /* AUTOINCR => nothing */ + 0, /* INSERT => nothing */ + 0, /* DELETE => nothing */ + 0, /* UPDATE => nothing */ + 0, /* SET => nothing */ + 0, /* DEFERRABLE => nothing */ + 0, /* FOREIGN => nothing */ + 0, /* DROP => nothing */ + 0, /* UNION => nothing */ + 0, /* ALL => nothing */ + 0, /* EXCEPT => nothing */ + 0, /* INTERSECT => nothing */ + 0, /* SELECT => nothing */ + 0, /* VALUES => nothing */ + 0, /* DISTINCT => nothing */ + 0, /* DOT => nothing */ + 0, /* FROM => nothing */ + 0, /* JOIN => nothing */ + 0, /* USING => nothing */ + 0, /* ORDER => nothing */ + 0, /* GROUP => nothing */ + 0, /* HAVING => nothing */ + 0, /* LIMIT => nothing */ + 0, /* WHERE => nothing */ + 0, /* INTO => nothing */ + 0, /* NOTHING => nothing */ + 0, /* FLOAT => nothing */ + 0, /* BLOB => nothing */ + 0, /* INTEGER => nothing */ + 0, /* VARIABLE => nothing */ + 0, /* CASE => nothing */ + 0, /* WHEN => nothing */ + 0, /* THEN => nothing */ + 0, /* ELSE => nothing */ + 0, /* INDEX => nothing */ + 0, /* ALTER => nothing */ + 0, /* ADD => nothing */ + 0, /* WINDOW => nothing */ + 0, /* OVER => nothing */ + 0, /* FILTER => nothing */ + 0, /* COLUMN => nothing */ + 0, /* AGG_FUNCTION => nothing */ + 0, /* AGG_COLUMN => nothing */ + 0, /* TRUEFALSE => nothing */ + 0, /* ISNOT => nothing */ + 0, /* FUNCTION => nothing */ + 0, /* UMINUS => nothing */ + 0, /* UPLUS => nothing */ + 0, /* TRUTH => nothing */ + 0, /* REGISTER => nothing */ + 0, /* VECTOR => nothing */ + 0, /* SELECT_COLUMN => nothing */ + 0, /* IF_NULL_ROW => nothing */ + 0, /* ASTERISK => nothing */ + 0, /* SPAN => nothing */ + 0, /* SPACE => nothing */ + 0, /* ILLEGAL => nothing */ }; #endif /* YYFALLBACK */ @@ -148518,202 +150958,231 @@ static const char *const yyTokenName[] = { /* 79 */ "VIEW", /* 80 */ "VIRTUAL", /* 81 */ "WITH", - /* 82 */ "CURRENT", - /* 83 */ "FOLLOWING", - /* 84 */ "PARTITION", - /* 85 */ "PRECEDING", - /* 86 */ "RANGE", - /* 87 */ "UNBOUNDED", - /* 88 */ "REINDEX", - /* 89 */ "RENAME", - /* 90 */ "CTIME_KW", - /* 91 */ "ANY", - /* 92 */ "BITAND", - /* 93 */ "BITOR", - /* 94 */ "LSHIFT", - /* 95 */ "RSHIFT", - /* 96 */ "PLUS", - /* 97 */ "MINUS", - /* 98 */ "STAR", - /* 99 */ "SLASH", - /* 100 */ "REM", - /* 101 */ "CONCAT", - /* 102 */ "COLLATE", - /* 103 */ "BITNOT", - /* 104 */ "ON", - /* 105 */ "INDEXED", - /* 106 */ "STRING", - /* 107 */ "JOIN_KW", - /* 108 */ "CONSTRAINT", - /* 109 */ "DEFAULT", - /* 110 */ "NULL", - /* 111 */ "PRIMARY", - /* 112 */ "UNIQUE", - /* 113 */ "CHECK", - /* 114 */ "REFERENCES", - /* 115 */ "AUTOINCR", - /* 116 */ "INSERT", - /* 117 */ "DELETE", - /* 118 */ "UPDATE", - /* 119 */ "SET", - /* 120 */ "DEFERRABLE", - /* 121 */ "FOREIGN", - /* 122 */ "DROP", - /* 123 */ "UNION", - /* 124 */ "ALL", - /* 125 */ "EXCEPT", - /* 126 */ "INTERSECT", - /* 127 */ "SELECT", - /* 128 */ "VALUES", - /* 129 */ "DISTINCT", - /* 130 */ "DOT", - /* 131 */ "FROM", - /* 132 */ "JOIN", - /* 133 */ "USING", - /* 134 */ "ORDER", - /* 135 */ "GROUP", - /* 136 */ "HAVING", - /* 137 */ "LIMIT", - /* 138 */ "WHERE", - /* 139 */ "INTO", - /* 140 */ "NOTHING", - /* 141 */ "FLOAT", - /* 142 */ "BLOB", - /* 143 */ "INTEGER", - /* 144 */ "VARIABLE", - /* 145 */ "CASE", - /* 146 */ "WHEN", - /* 147 */ "THEN", - /* 148 */ "ELSE", - /* 149 */ "INDEX", - /* 150 */ "ALTER", - /* 151 */ "ADD", - /* 152 */ "WINDOW", - /* 153 */ "OVER", - /* 154 */ "FILTER", - /* 155 */ "input", - /* 156 */ "cmdlist", - /* 157 */ "ecmd", - /* 158 */ "cmdx", - /* 159 */ "explain", - /* 160 */ "cmd", - /* 161 */ "transtype", - /* 162 */ "trans_opt", - /* 163 */ "nm", - /* 164 */ "savepoint_opt", - /* 165 */ "create_table", - /* 166 */ "create_table_args", - /* 167 */ "createkw", - /* 168 */ "temp", - /* 169 */ "ifnotexists", - /* 170 */ "dbnm", - /* 171 */ "columnlist", - /* 172 */ "conslist_opt", - /* 173 */ "table_options", - /* 174 */ "select", - /* 175 */ "columnname", - /* 176 */ "carglist", - /* 177 */ "typetoken", - /* 178 */ "typename", - /* 179 */ "signed", - /* 180 */ "plus_num", - /* 181 */ "minus_num", - /* 182 */ "scanpt", - /* 183 */ "ccons", - /* 184 */ "term", - /* 185 */ "expr", - /* 186 */ "onconf", - /* 187 */ "sortorder", - /* 188 */ "autoinc", - /* 189 */ "eidlist_opt", - /* 190 */ "refargs", - /* 191 */ "defer_subclause", - /* 192 */ "refarg", - /* 193 */ "refact", - /* 194 */ "init_deferred_pred_opt", - /* 195 */ "conslist", - /* 196 */ "tconscomma", - /* 197 */ "tcons", - /* 198 */ "sortlist", - /* 199 */ "eidlist", - /* 200 */ "defer_subclause_opt", - /* 201 */ "orconf", - /* 202 */ "resolvetype", - /* 203 */ "raisetype", - /* 204 */ "ifexists", - /* 205 */ "fullname", - /* 206 */ "selectnowith", - /* 207 */ "oneselect", - /* 208 */ "wqlist", - /* 209 */ "multiselect_op", - /* 210 */ "distinct", - /* 211 */ "selcollist", - /* 212 */ "from", - /* 213 */ "where_opt", - /* 214 */ "groupby_opt", - /* 215 */ "having_opt", - /* 216 */ "orderby_opt", - /* 217 */ "limit_opt", - /* 218 */ "window_clause", - /* 219 */ "values", - /* 220 */ "nexprlist", - /* 221 */ "sclp", - /* 222 */ "as", - /* 223 */ "seltablist", - /* 224 */ "stl_prefix", - /* 225 */ "joinop", - /* 226 */ "indexed_opt", - /* 227 */ "on_opt", - /* 228 */ "using_opt", - /* 229 */ "exprlist", - /* 230 */ "xfullname", - /* 231 */ "idlist", - /* 232 */ "with", - /* 233 */ "setlist", - /* 234 */ "insert_cmd", - /* 235 */ "idlist_opt", - /* 236 */ "upsert", - /* 237 */ "over_clause", - /* 238 */ "likeop", - /* 239 */ "between_op", - /* 240 */ "in_op", - /* 241 */ "paren_exprlist", - /* 242 */ "case_operand", - /* 243 */ "case_exprlist", - /* 244 */ "case_else", - /* 245 */ "uniqueflag", - /* 246 */ "collate", - /* 247 */ "vinto", - /* 248 */ "nmnum", - /* 249 */ "trigger_decl", - /* 250 */ "trigger_cmd_list", - /* 251 */ "trigger_time", - /* 252 */ "trigger_event", - /* 253 */ "foreach_clause", - /* 254 */ "when_clause", - /* 255 */ "trigger_cmd", - /* 256 */ "trnm", - /* 257 */ "tridxby", - /* 258 */ "database_kw_opt", - /* 259 */ "key_opt", - /* 260 */ "add_column_fullname", - /* 261 */ "kwcolumn_opt", - /* 262 */ "create_vtab", - /* 263 */ "vtabarglist", - /* 264 */ "vtabarg", - /* 265 */ "vtabargtoken", - /* 266 */ "lp", - /* 267 */ "anylist", - /* 268 */ "windowdefn_list", - /* 269 */ "windowdefn", - /* 270 */ "window", - /* 271 */ "frame_opt", - /* 272 */ "part_opt", - /* 273 */ "filter_opt", - /* 274 */ "range_or_rows", - /* 275 */ "frame_bound", - /* 276 */ "frame_bound_s", - /* 277 */ "frame_bound_e", + /* 82 */ "NULLS", + /* 83 */ "FIRST", + /* 84 */ "LAST", + /* 85 */ "CURRENT", + /* 86 */ "FOLLOWING", + /* 87 */ "PARTITION", + /* 88 */ "PRECEDING", + /* 89 */ "RANGE", + /* 90 */ "UNBOUNDED", + /* 91 */ "EXCLUDE", + /* 92 */ "GROUPS", + /* 93 */ "OTHERS", + /* 94 */ "TIES", + /* 95 */ "REINDEX", + /* 96 */ "RENAME", + /* 97 */ "CTIME_KW", + /* 98 */ "ANY", + /* 99 */ "BITAND", + /* 100 */ "BITOR", + /* 101 */ "LSHIFT", + /* 102 */ "RSHIFT", + /* 103 */ "PLUS", + /* 104 */ "MINUS", + /* 105 */ "STAR", + /* 106 */ "SLASH", + /* 107 */ "REM", + /* 108 */ "CONCAT", + /* 109 */ "COLLATE", + /* 110 */ "BITNOT", + /* 111 */ "ON", + /* 112 */ "INDEXED", + /* 113 */ "STRING", + /* 114 */ "JOIN_KW", + /* 115 */ "CONSTRAINT", + /* 116 */ "DEFAULT", + /* 117 */ "NULL", + /* 118 */ "PRIMARY", + /* 119 */ "UNIQUE", + /* 120 */ "CHECK", + /* 121 */ "REFERENCES", + /* 122 */ "AUTOINCR", + /* 123 */ "INSERT", + /* 124 */ "DELETE", + /* 125 */ "UPDATE", + /* 126 */ "SET", + /* 127 */ "DEFERRABLE", + /* 128 */ "FOREIGN", + /* 129 */ "DROP", + /* 130 */ "UNION", + /* 131 */ "ALL", + /* 132 */ "EXCEPT", + /* 133 */ "INTERSECT", + /* 134 */ "SELECT", + /* 135 */ "VALUES", + /* 136 */ "DISTINCT", + /* 137 */ "DOT", + /* 138 */ "FROM", + /* 139 */ "JOIN", + /* 140 */ "USING", + /* 141 */ "ORDER", + /* 142 */ "GROUP", + /* 143 */ "HAVING", + /* 144 */ "LIMIT", + /* 145 */ "WHERE", + /* 146 */ "INTO", + /* 147 */ "NOTHING", + /* 148 */ "FLOAT", + /* 149 */ "BLOB", + /* 150 */ "INTEGER", + /* 151 */ "VARIABLE", + /* 152 */ "CASE", + /* 153 */ "WHEN", + /* 154 */ "THEN", + /* 155 */ "ELSE", + /* 156 */ "INDEX", + /* 157 */ "ALTER", + /* 158 */ "ADD", + /* 159 */ "WINDOW", + /* 160 */ "OVER", + /* 161 */ "FILTER", + /* 162 */ "COLUMN", + /* 163 */ "AGG_FUNCTION", + /* 164 */ "AGG_COLUMN", + /* 165 */ "TRUEFALSE", + /* 166 */ "ISNOT", + /* 167 */ "FUNCTION", + /* 168 */ "UMINUS", + /* 169 */ "UPLUS", + /* 170 */ "TRUTH", + /* 171 */ "REGISTER", + /* 172 */ "VECTOR", + /* 173 */ "SELECT_COLUMN", + /* 174 */ "IF_NULL_ROW", + /* 175 */ "ASTERISK", + /* 176 */ "SPAN", + /* 177 */ "SPACE", + /* 178 */ "ILLEGAL", + /* 179 */ "input", + /* 180 */ "cmdlist", + /* 181 */ "ecmd", + /* 182 */ "cmdx", + /* 183 */ "explain", + /* 184 */ "cmd", + /* 185 */ "transtype", + /* 186 */ "trans_opt", + /* 187 */ "nm", + /* 188 */ "savepoint_opt", + /* 189 */ "create_table", + /* 190 */ "create_table_args", + /* 191 */ "createkw", + /* 192 */ "temp", + /* 193 */ "ifnotexists", + /* 194 */ "dbnm", + /* 195 */ "columnlist", + /* 196 */ "conslist_opt", + /* 197 */ "table_options", + /* 198 */ "select", + /* 199 */ "columnname", + /* 200 */ "carglist", + /* 201 */ "typetoken", + /* 202 */ "typename", + /* 203 */ "signed", + /* 204 */ "plus_num", + /* 205 */ "minus_num", + /* 206 */ "scanpt", + /* 207 */ "scantok", + /* 208 */ "ccons", + /* 209 */ "term", + /* 210 */ "expr", + /* 211 */ "onconf", + /* 212 */ "sortorder", + /* 213 */ "autoinc", + /* 214 */ "eidlist_opt", + /* 215 */ "refargs", + /* 216 */ "defer_subclause", + /* 217 */ "refarg", + /* 218 */ "refact", + /* 219 */ "init_deferred_pred_opt", + /* 220 */ "conslist", + /* 221 */ "tconscomma", + /* 222 */ "tcons", + /* 223 */ "sortlist", + /* 224 */ "eidlist", + /* 225 */ "defer_subclause_opt", + /* 226 */ "orconf", + /* 227 */ "resolvetype", + /* 228 */ "raisetype", + /* 229 */ "ifexists", + /* 230 */ "fullname", + /* 231 */ "selectnowith", + /* 232 */ "oneselect", + /* 233 */ "wqlist", + /* 234 */ "multiselect_op", + /* 235 */ "distinct", + /* 236 */ "selcollist", + /* 237 */ "from", + /* 238 */ "where_opt", + /* 239 */ "groupby_opt", + /* 240 */ "having_opt", + /* 241 */ "orderby_opt", + /* 242 */ "limit_opt", + /* 243 */ "window_clause", + /* 244 */ "values", + /* 245 */ "nexprlist", + /* 246 */ "sclp", + /* 247 */ "as", + /* 248 */ "seltablist", + /* 249 */ "stl_prefix", + /* 250 */ "joinop", + /* 251 */ "indexed_opt", + /* 252 */ "on_opt", + /* 253 */ "using_opt", + /* 254 */ "exprlist", + /* 255 */ "xfullname", + /* 256 */ "idlist", + /* 257 */ "nulls", + /* 258 */ "with", + /* 259 */ "setlist", + /* 260 */ "insert_cmd", + /* 261 */ "idlist_opt", + /* 262 */ "upsert", + /* 263 */ "filter_over", + /* 264 */ "likeop", + /* 265 */ "between_op", + /* 266 */ "in_op", + /* 267 */ "paren_exprlist", + /* 268 */ "case_operand", + /* 269 */ "case_exprlist", + /* 270 */ "case_else", + /* 271 */ "uniqueflag", + /* 272 */ "collate", + /* 273 */ "vinto", + /* 274 */ "nmnum", + /* 275 */ "trigger_decl", + /* 276 */ "trigger_cmd_list", + /* 277 */ "trigger_time", + /* 278 */ "trigger_event", + /* 279 */ "foreach_clause", + /* 280 */ "when_clause", + /* 281 */ "trigger_cmd", + /* 282 */ "trnm", + /* 283 */ "tridxby", + /* 284 */ "database_kw_opt", + /* 285 */ "key_opt", + /* 286 */ "add_column_fullname", + /* 287 */ "kwcolumn_opt", + /* 288 */ "create_vtab", + /* 289 */ "vtabarglist", + /* 290 */ "vtabarg", + /* 291 */ "vtabargtoken", + /* 292 */ "lp", + /* 293 */ "anylist", + /* 294 */ "windowdefn_list", + /* 295 */ "windowdefn", + /* 296 */ "window", + /* 297 */ "frame_opt", + /* 298 */ "part_opt", + /* 299 */ "filter_clause", + /* 300 */ "over_clause", + /* 301 */ "range_or_rows", + /* 302 */ "frame_bound", + /* 303 */ "frame_bound_s", + /* 304 */ "frame_bound_e", + /* 305 */ "frame_exclude_opt", + /* 306 */ "frame_exclude", }; #endif /* defined(YYCOVERAGE) || !defined(NDEBUG) */ @@ -148750,346 +151219,358 @@ static const char *const yyRuleName[] = { /* 26 */ "typetoken ::= typename LP signed COMMA signed RP", /* 27 */ "typename ::= typename ID|STRING", /* 28 */ "scanpt ::=", - /* 29 */ "ccons ::= CONSTRAINT nm", - /* 30 */ "ccons ::= DEFAULT scanpt term scanpt", - /* 31 */ "ccons ::= DEFAULT LP expr RP", - /* 32 */ "ccons ::= DEFAULT PLUS term scanpt", - /* 33 */ "ccons ::= DEFAULT MINUS term scanpt", - /* 34 */ "ccons ::= DEFAULT scanpt ID|INDEXED", - /* 35 */ "ccons ::= NOT NULL onconf", - /* 36 */ "ccons ::= PRIMARY KEY sortorder onconf autoinc", - /* 37 */ "ccons ::= UNIQUE onconf", - /* 38 */ "ccons ::= CHECK LP expr RP", - /* 39 */ "ccons ::= REFERENCES nm eidlist_opt refargs", - /* 40 */ "ccons ::= defer_subclause", - /* 41 */ "ccons ::= COLLATE ID|STRING", - /* 42 */ "autoinc ::=", - /* 43 */ "autoinc ::= AUTOINCR", - /* 44 */ "refargs ::=", - /* 45 */ "refargs ::= refargs refarg", - /* 46 */ "refarg ::= MATCH nm", - /* 47 */ "refarg ::= ON INSERT refact", - /* 48 */ "refarg ::= ON DELETE refact", - /* 49 */ "refarg ::= ON UPDATE refact", - /* 50 */ "refact ::= SET NULL", - /* 51 */ "refact ::= SET DEFAULT", - /* 52 */ "refact ::= CASCADE", - /* 53 */ "refact ::= RESTRICT", - /* 54 */ "refact ::= NO ACTION", - /* 55 */ "defer_subclause ::= NOT DEFERRABLE init_deferred_pred_opt", - /* 56 */ "defer_subclause ::= DEFERRABLE init_deferred_pred_opt", - /* 57 */ "init_deferred_pred_opt ::=", - /* 58 */ "init_deferred_pred_opt ::= INITIALLY DEFERRED", - /* 59 */ "init_deferred_pred_opt ::= INITIALLY IMMEDIATE", - /* 60 */ "conslist_opt ::=", - /* 61 */ "tconscomma ::= COMMA", - /* 62 */ "tcons ::= CONSTRAINT nm", - /* 63 */ "tcons ::= PRIMARY KEY LP sortlist autoinc RP onconf", - /* 64 */ "tcons ::= UNIQUE LP sortlist RP onconf", - /* 65 */ "tcons ::= CHECK LP expr RP onconf", - /* 66 */ "tcons ::= FOREIGN KEY LP eidlist RP REFERENCES nm eidlist_opt refargs defer_subclause_opt", - /* 67 */ "defer_subclause_opt ::=", - /* 68 */ "onconf ::=", - /* 69 */ "onconf ::= ON CONFLICT resolvetype", - /* 70 */ "orconf ::=", - /* 71 */ "orconf ::= OR resolvetype", - /* 72 */ "resolvetype ::= IGNORE", - /* 73 */ "resolvetype ::= REPLACE", - /* 74 */ "cmd ::= DROP TABLE ifexists fullname", - /* 75 */ "ifexists ::= IF EXISTS", - /* 76 */ "ifexists ::=", - /* 77 */ "cmd ::= createkw temp VIEW ifnotexists nm dbnm eidlist_opt AS select", - /* 78 */ "cmd ::= DROP VIEW ifexists fullname", - /* 79 */ "cmd ::= select", - /* 80 */ "select ::= WITH wqlist selectnowith", - /* 81 */ "select ::= WITH RECURSIVE wqlist selectnowith", - /* 82 */ "select ::= selectnowith", - /* 83 */ "selectnowith ::= selectnowith multiselect_op oneselect", - /* 84 */ "multiselect_op ::= UNION", - /* 85 */ "multiselect_op ::= UNION ALL", - /* 86 */ "multiselect_op ::= EXCEPT|INTERSECT", - /* 87 */ "oneselect ::= SELECT distinct selcollist from where_opt groupby_opt having_opt orderby_opt limit_opt", - /* 88 */ "oneselect ::= SELECT distinct selcollist from where_opt groupby_opt having_opt window_clause orderby_opt limit_opt", - /* 89 */ "values ::= VALUES LP nexprlist RP", - /* 90 */ "values ::= values COMMA LP nexprlist RP", - /* 91 */ "distinct ::= DISTINCT", - /* 92 */ "distinct ::= ALL", - /* 93 */ "distinct ::=", - /* 94 */ "sclp ::=", - /* 95 */ "selcollist ::= sclp scanpt expr scanpt as", - /* 96 */ "selcollist ::= sclp scanpt STAR", - /* 97 */ "selcollist ::= sclp scanpt nm DOT STAR", - /* 98 */ "as ::= AS nm", - /* 99 */ "as ::=", - /* 100 */ "from ::=", - /* 101 */ "from ::= FROM seltablist", - /* 102 */ "stl_prefix ::= seltablist joinop", - /* 103 */ "stl_prefix ::=", - /* 104 */ "seltablist ::= stl_prefix nm dbnm as indexed_opt on_opt using_opt", - /* 105 */ "seltablist ::= stl_prefix nm dbnm LP exprlist RP as on_opt using_opt", - /* 106 */ "seltablist ::= stl_prefix LP select RP as on_opt using_opt", - /* 107 */ "seltablist ::= stl_prefix LP seltablist RP as on_opt using_opt", - /* 108 */ "dbnm ::=", - /* 109 */ "dbnm ::= DOT nm", - /* 110 */ "fullname ::= nm", - /* 111 */ "fullname ::= nm DOT nm", - /* 112 */ "xfullname ::= nm", - /* 113 */ "xfullname ::= nm DOT nm", - /* 114 */ "xfullname ::= nm DOT nm AS nm", - /* 115 */ "xfullname ::= nm AS nm", - /* 116 */ "joinop ::= COMMA|JOIN", - /* 117 */ "joinop ::= JOIN_KW JOIN", - /* 118 */ "joinop ::= JOIN_KW nm JOIN", - /* 119 */ "joinop ::= JOIN_KW nm nm JOIN", - /* 120 */ "on_opt ::= ON expr", - /* 121 */ "on_opt ::=", - /* 122 */ "indexed_opt ::=", - /* 123 */ "indexed_opt ::= INDEXED BY nm", - /* 124 */ "indexed_opt ::= NOT INDEXED", - /* 125 */ "using_opt ::= USING LP idlist RP", - /* 126 */ "using_opt ::=", - /* 127 */ "orderby_opt ::=", - /* 128 */ "orderby_opt ::= ORDER BY sortlist", - /* 129 */ "sortlist ::= sortlist COMMA expr sortorder", - /* 130 */ "sortlist ::= expr sortorder", - /* 131 */ "sortorder ::= ASC", - /* 132 */ "sortorder ::= DESC", - /* 133 */ "sortorder ::=", - /* 134 */ "groupby_opt ::=", - /* 135 */ "groupby_opt ::= GROUP BY nexprlist", - /* 136 */ "having_opt ::=", - /* 137 */ "having_opt ::= HAVING expr", - /* 138 */ "limit_opt ::=", - /* 139 */ "limit_opt ::= LIMIT expr", - /* 140 */ "limit_opt ::= LIMIT expr OFFSET expr", - /* 141 */ "limit_opt ::= LIMIT expr COMMA expr", - /* 142 */ "cmd ::= with DELETE FROM xfullname indexed_opt where_opt", - /* 143 */ "where_opt ::=", - /* 144 */ "where_opt ::= WHERE expr", - /* 145 */ "cmd ::= with UPDATE orconf xfullname indexed_opt SET setlist where_opt", - /* 146 */ "setlist ::= setlist COMMA nm EQ expr", - /* 147 */ "setlist ::= setlist COMMA LP idlist RP EQ expr", - /* 148 */ "setlist ::= nm EQ expr", - /* 149 */ "setlist ::= LP idlist RP EQ expr", - /* 150 */ "cmd ::= with insert_cmd INTO xfullname idlist_opt select upsert", - /* 151 */ "cmd ::= with insert_cmd INTO xfullname idlist_opt DEFAULT VALUES", - /* 152 */ "upsert ::=", - /* 153 */ "upsert ::= ON CONFLICT LP sortlist RP where_opt DO UPDATE SET setlist where_opt", - /* 154 */ "upsert ::= ON CONFLICT LP sortlist RP where_opt DO NOTHING", - /* 155 */ "upsert ::= ON CONFLICT DO NOTHING", - /* 156 */ "insert_cmd ::= INSERT orconf", - /* 157 */ "insert_cmd ::= REPLACE", - /* 158 */ "idlist_opt ::=", - /* 159 */ "idlist_opt ::= LP idlist RP", - /* 160 */ "idlist ::= idlist COMMA nm", - /* 161 */ "idlist ::= nm", - /* 162 */ "expr ::= LP expr RP", - /* 163 */ "expr ::= ID|INDEXED", - /* 164 */ "expr ::= JOIN_KW", - /* 165 */ "expr ::= nm DOT nm", - /* 166 */ "expr ::= nm DOT nm DOT nm", - /* 167 */ "term ::= NULL|FLOAT|BLOB", - /* 168 */ "term ::= STRING", - /* 169 */ "term ::= INTEGER", - /* 170 */ "expr ::= VARIABLE", - /* 171 */ "expr ::= expr COLLATE ID|STRING", - /* 172 */ "expr ::= CAST LP expr AS typetoken RP", - /* 173 */ "expr ::= ID|INDEXED LP distinct exprlist RP", - /* 174 */ "expr ::= ID|INDEXED LP STAR RP", - /* 175 */ "expr ::= ID|INDEXED LP distinct exprlist RP over_clause", - /* 176 */ "expr ::= ID|INDEXED LP STAR RP over_clause", - /* 177 */ "term ::= CTIME_KW", - /* 178 */ "expr ::= LP nexprlist COMMA expr RP", - /* 179 */ "expr ::= expr AND expr", - /* 180 */ "expr ::= expr OR expr", - /* 181 */ "expr ::= expr LT|GT|GE|LE expr", - /* 182 */ "expr ::= expr EQ|NE expr", - /* 183 */ "expr ::= expr BITAND|BITOR|LSHIFT|RSHIFT expr", - /* 184 */ "expr ::= expr PLUS|MINUS expr", - /* 185 */ "expr ::= expr STAR|SLASH|REM expr", - /* 186 */ "expr ::= expr CONCAT expr", - /* 187 */ "likeop ::= NOT LIKE_KW|MATCH", - /* 188 */ "expr ::= expr likeop expr", - /* 189 */ "expr ::= expr likeop expr ESCAPE expr", - /* 190 */ "expr ::= expr ISNULL|NOTNULL", - /* 191 */ "expr ::= expr NOT NULL", - /* 192 */ "expr ::= expr IS expr", - /* 193 */ "expr ::= expr IS NOT expr", - /* 194 */ "expr ::= NOT expr", - /* 195 */ "expr ::= BITNOT expr", - /* 196 */ "expr ::= PLUS|MINUS expr", - /* 197 */ "between_op ::= BETWEEN", - /* 198 */ "between_op ::= NOT BETWEEN", - /* 199 */ "expr ::= expr between_op expr AND expr", - /* 200 */ "in_op ::= IN", - /* 201 */ "in_op ::= NOT IN", - /* 202 */ "expr ::= expr in_op LP exprlist RP", - /* 203 */ "expr ::= LP select RP", - /* 204 */ "expr ::= expr in_op LP select RP", - /* 205 */ "expr ::= expr in_op nm dbnm paren_exprlist", - /* 206 */ "expr ::= EXISTS LP select RP", - /* 207 */ "expr ::= CASE case_operand case_exprlist case_else END", - /* 208 */ "case_exprlist ::= case_exprlist WHEN expr THEN expr", - /* 209 */ "case_exprlist ::= WHEN expr THEN expr", - /* 210 */ "case_else ::= ELSE expr", - /* 211 */ "case_else ::=", - /* 212 */ "case_operand ::= expr", - /* 213 */ "case_operand ::=", - /* 214 */ "exprlist ::=", - /* 215 */ "nexprlist ::= nexprlist COMMA expr", - /* 216 */ "nexprlist ::= expr", - /* 217 */ "paren_exprlist ::=", - /* 218 */ "paren_exprlist ::= LP exprlist RP", - /* 219 */ "cmd ::= createkw uniqueflag INDEX ifnotexists nm dbnm ON nm LP sortlist RP where_opt", - /* 220 */ "uniqueflag ::= UNIQUE", - /* 221 */ "uniqueflag ::=", - /* 222 */ "eidlist_opt ::=", - /* 223 */ "eidlist_opt ::= LP eidlist RP", - /* 224 */ "eidlist ::= eidlist COMMA nm collate sortorder", - /* 225 */ "eidlist ::= nm collate sortorder", - /* 226 */ "collate ::=", - /* 227 */ "collate ::= COLLATE ID|STRING", - /* 228 */ "cmd ::= DROP INDEX ifexists fullname", - /* 229 */ "cmd ::= VACUUM vinto", - /* 230 */ "cmd ::= VACUUM nm vinto", - /* 231 */ "vinto ::= INTO expr", - /* 232 */ "vinto ::=", - /* 233 */ "cmd ::= PRAGMA nm dbnm", - /* 234 */ "cmd ::= PRAGMA nm dbnm EQ nmnum", - /* 235 */ "cmd ::= PRAGMA nm dbnm LP nmnum RP", - /* 236 */ "cmd ::= PRAGMA nm dbnm EQ minus_num", - /* 237 */ "cmd ::= PRAGMA nm dbnm LP minus_num RP", - /* 238 */ "plus_num ::= PLUS INTEGER|FLOAT", - /* 239 */ "minus_num ::= MINUS INTEGER|FLOAT", - /* 240 */ "cmd ::= createkw trigger_decl BEGIN trigger_cmd_list END", - /* 241 */ "trigger_decl ::= temp TRIGGER ifnotexists nm dbnm trigger_time trigger_event ON fullname foreach_clause when_clause", - /* 242 */ "trigger_time ::= BEFORE|AFTER", - /* 243 */ "trigger_time ::= INSTEAD OF", - /* 244 */ "trigger_time ::=", - /* 245 */ "trigger_event ::= DELETE|INSERT", - /* 246 */ "trigger_event ::= UPDATE", - /* 247 */ "trigger_event ::= UPDATE OF idlist", - /* 248 */ "when_clause ::=", - /* 249 */ "when_clause ::= WHEN expr", - /* 250 */ "trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI", - /* 251 */ "trigger_cmd_list ::= trigger_cmd SEMI", - /* 252 */ "trnm ::= nm DOT nm", - /* 253 */ "tridxby ::= INDEXED BY nm", - /* 254 */ "tridxby ::= NOT INDEXED", - /* 255 */ "trigger_cmd ::= UPDATE orconf trnm tridxby SET setlist where_opt scanpt", - /* 256 */ "trigger_cmd ::= scanpt insert_cmd INTO trnm idlist_opt select upsert scanpt", - /* 257 */ "trigger_cmd ::= DELETE FROM trnm tridxby where_opt scanpt", - /* 258 */ "trigger_cmd ::= scanpt select scanpt", - /* 259 */ "expr ::= RAISE LP IGNORE RP", - /* 260 */ "expr ::= RAISE LP raisetype COMMA nm RP", - /* 261 */ "raisetype ::= ROLLBACK", - /* 262 */ "raisetype ::= ABORT", - /* 263 */ "raisetype ::= FAIL", - /* 264 */ "cmd ::= DROP TRIGGER ifexists fullname", - /* 265 */ "cmd ::= ATTACH database_kw_opt expr AS expr key_opt", - /* 266 */ "cmd ::= DETACH database_kw_opt expr", - /* 267 */ "key_opt ::=", - /* 268 */ "key_opt ::= KEY expr", - /* 269 */ "cmd ::= REINDEX", - /* 270 */ "cmd ::= REINDEX nm dbnm", - /* 271 */ "cmd ::= ANALYZE", - /* 272 */ "cmd ::= ANALYZE nm dbnm", - /* 273 */ "cmd ::= ALTER TABLE fullname RENAME TO nm", - /* 274 */ "cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt columnname carglist", - /* 275 */ "add_column_fullname ::= fullname", - /* 276 */ "cmd ::= ALTER TABLE fullname RENAME kwcolumn_opt nm TO nm", - /* 277 */ "cmd ::= create_vtab", - /* 278 */ "cmd ::= create_vtab LP vtabarglist RP", - /* 279 */ "create_vtab ::= createkw VIRTUAL TABLE ifnotexists nm dbnm USING nm", - /* 280 */ "vtabarg ::=", - /* 281 */ "vtabargtoken ::= ANY", - /* 282 */ "vtabargtoken ::= lp anylist RP", - /* 283 */ "lp ::= LP", - /* 284 */ "with ::= WITH wqlist", - /* 285 */ "with ::= WITH RECURSIVE wqlist", - /* 286 */ "wqlist ::= nm eidlist_opt AS LP select RP", - /* 287 */ "wqlist ::= wqlist COMMA nm eidlist_opt AS LP select RP", - /* 288 */ "windowdefn_list ::= windowdefn", - /* 289 */ "windowdefn_list ::= windowdefn_list COMMA windowdefn", - /* 290 */ "windowdefn ::= nm AS window", - /* 291 */ "window ::= LP part_opt orderby_opt frame_opt RP", - /* 292 */ "part_opt ::= PARTITION BY nexprlist", - /* 293 */ "part_opt ::=", - /* 294 */ "frame_opt ::=", - /* 295 */ "frame_opt ::= range_or_rows frame_bound_s", - /* 296 */ "frame_opt ::= range_or_rows BETWEEN frame_bound_s AND frame_bound_e", - /* 297 */ "range_or_rows ::= RANGE", - /* 298 */ "range_or_rows ::= ROWS", - /* 299 */ "frame_bound_s ::= frame_bound", - /* 300 */ "frame_bound_s ::= UNBOUNDED PRECEDING", - /* 301 */ "frame_bound_e ::= frame_bound", - /* 302 */ "frame_bound_e ::= UNBOUNDED FOLLOWING", - /* 303 */ "frame_bound ::= expr PRECEDING", - /* 304 */ "frame_bound ::= CURRENT ROW", - /* 305 */ "frame_bound ::= expr FOLLOWING", - /* 306 */ "window_clause ::= WINDOW windowdefn_list", - /* 307 */ "over_clause ::= filter_opt OVER window", - /* 308 */ "over_clause ::= filter_opt OVER nm", - /* 309 */ "filter_opt ::=", - /* 310 */ "filter_opt ::= FILTER LP WHERE expr RP", - /* 311 */ "input ::= cmdlist", - /* 312 */ "cmdlist ::= cmdlist ecmd", - /* 313 */ "cmdlist ::= ecmd", - /* 314 */ "ecmd ::= SEMI", - /* 315 */ "ecmd ::= cmdx SEMI", - /* 316 */ "ecmd ::= explain cmdx", - /* 317 */ "trans_opt ::=", - /* 318 */ "trans_opt ::= TRANSACTION", - /* 319 */ "trans_opt ::= TRANSACTION nm", - /* 320 */ "savepoint_opt ::= SAVEPOINT", - /* 321 */ "savepoint_opt ::=", - /* 322 */ "cmd ::= create_table create_table_args", - /* 323 */ "columnlist ::= columnlist COMMA columnname carglist", - /* 324 */ "columnlist ::= columnname carglist", - /* 325 */ "nm ::= ID|INDEXED", - /* 326 */ "nm ::= STRING", - /* 327 */ "nm ::= JOIN_KW", - /* 328 */ "typetoken ::= typename", - /* 329 */ "typename ::= ID|STRING", - /* 330 */ "signed ::= plus_num", - /* 331 */ "signed ::= minus_num", - /* 332 */ "carglist ::= carglist ccons", - /* 333 */ "carglist ::=", - /* 334 */ "ccons ::= NULL onconf", - /* 335 */ "conslist_opt ::= COMMA conslist", - /* 336 */ "conslist ::= conslist tconscomma tcons", - /* 337 */ "conslist ::= tcons", - /* 338 */ "tconscomma ::=", - /* 339 */ "defer_subclause_opt ::= defer_subclause", - /* 340 */ "resolvetype ::= raisetype", - /* 341 */ "selectnowith ::= oneselect", - /* 342 */ "oneselect ::= values", - /* 343 */ "sclp ::= selcollist COMMA", - /* 344 */ "as ::= ID|STRING", - /* 345 */ "expr ::= term", - /* 346 */ "likeop ::= LIKE_KW|MATCH", - /* 347 */ "exprlist ::= nexprlist", - /* 348 */ "nmnum ::= plus_num", - /* 349 */ "nmnum ::= nm", - /* 350 */ "nmnum ::= ON", - /* 351 */ "nmnum ::= DELETE", - /* 352 */ "nmnum ::= DEFAULT", - /* 353 */ "plus_num ::= INTEGER|FLOAT", - /* 354 */ "foreach_clause ::=", - /* 355 */ "foreach_clause ::= FOR EACH ROW", - /* 356 */ "trnm ::= nm", - /* 357 */ "tridxby ::=", - /* 358 */ "database_kw_opt ::= DATABASE", - /* 359 */ "database_kw_opt ::=", - /* 360 */ "kwcolumn_opt ::=", - /* 361 */ "kwcolumn_opt ::= COLUMNKW", - /* 362 */ "vtabarglist ::= vtabarg", - /* 363 */ "vtabarglist ::= vtabarglist COMMA vtabarg", - /* 364 */ "vtabarg ::= vtabarg vtabargtoken", - /* 365 */ "anylist ::=", - /* 366 */ "anylist ::= anylist LP anylist RP", - /* 367 */ "anylist ::= anylist ANY", - /* 368 */ "with ::=", + /* 29 */ "scantok ::=", + /* 30 */ "ccons ::= CONSTRAINT nm", + /* 31 */ "ccons ::= DEFAULT scantok term", + /* 32 */ "ccons ::= DEFAULT LP expr RP", + /* 33 */ "ccons ::= DEFAULT PLUS scantok term", + /* 34 */ "ccons ::= DEFAULT MINUS scantok term", + /* 35 */ "ccons ::= DEFAULT scantok ID|INDEXED", + /* 36 */ "ccons ::= NOT NULL onconf", + /* 37 */ "ccons ::= PRIMARY KEY sortorder onconf autoinc", + /* 38 */ "ccons ::= UNIQUE onconf", + /* 39 */ "ccons ::= CHECK LP expr RP", + /* 40 */ "ccons ::= REFERENCES nm eidlist_opt refargs", + /* 41 */ "ccons ::= defer_subclause", + /* 42 */ "ccons ::= COLLATE ID|STRING", + /* 43 */ "autoinc ::=", + /* 44 */ "autoinc ::= AUTOINCR", + /* 45 */ "refargs ::=", + /* 46 */ "refargs ::= refargs refarg", + /* 47 */ "refarg ::= MATCH nm", + /* 48 */ "refarg ::= ON INSERT refact", + /* 49 */ "refarg ::= ON DELETE refact", + /* 50 */ "refarg ::= ON UPDATE refact", + /* 51 */ "refact ::= SET NULL", + /* 52 */ "refact ::= SET DEFAULT", + /* 53 */ "refact ::= CASCADE", + /* 54 */ "refact ::= RESTRICT", + /* 55 */ "refact ::= NO ACTION", + /* 56 */ "defer_subclause ::= NOT DEFERRABLE init_deferred_pred_opt", + /* 57 */ "defer_subclause ::= DEFERRABLE init_deferred_pred_opt", + /* 58 */ "init_deferred_pred_opt ::=", + /* 59 */ "init_deferred_pred_opt ::= INITIALLY DEFERRED", + /* 60 */ "init_deferred_pred_opt ::= INITIALLY IMMEDIATE", + /* 61 */ "conslist_opt ::=", + /* 62 */ "tconscomma ::= COMMA", + /* 63 */ "tcons ::= CONSTRAINT nm", + /* 64 */ "tcons ::= PRIMARY KEY LP sortlist autoinc RP onconf", + /* 65 */ "tcons ::= UNIQUE LP sortlist RP onconf", + /* 66 */ "tcons ::= CHECK LP expr RP onconf", + /* 67 */ "tcons ::= FOREIGN KEY LP eidlist RP REFERENCES nm eidlist_opt refargs defer_subclause_opt", + /* 68 */ "defer_subclause_opt ::=", + /* 69 */ "onconf ::=", + /* 70 */ "onconf ::= ON CONFLICT resolvetype", + /* 71 */ "orconf ::=", + /* 72 */ "orconf ::= OR resolvetype", + /* 73 */ "resolvetype ::= IGNORE", + /* 74 */ "resolvetype ::= REPLACE", + /* 75 */ "cmd ::= DROP TABLE ifexists fullname", + /* 76 */ "ifexists ::= IF EXISTS", + /* 77 */ "ifexists ::=", + /* 78 */ "cmd ::= createkw temp VIEW ifnotexists nm dbnm eidlist_opt AS select", + /* 79 */ "cmd ::= DROP VIEW ifexists fullname", + /* 80 */ "cmd ::= select", + /* 81 */ "select ::= WITH wqlist selectnowith", + /* 82 */ "select ::= WITH RECURSIVE wqlist selectnowith", + /* 83 */ "select ::= selectnowith", + /* 84 */ "selectnowith ::= selectnowith multiselect_op oneselect", + /* 85 */ "multiselect_op ::= UNION", + /* 86 */ "multiselect_op ::= UNION ALL", + /* 87 */ "multiselect_op ::= EXCEPT|INTERSECT", + /* 88 */ "oneselect ::= SELECT distinct selcollist from where_opt groupby_opt having_opt orderby_opt limit_opt", + /* 89 */ "oneselect ::= SELECT distinct selcollist from where_opt groupby_opt having_opt window_clause orderby_opt limit_opt", + /* 90 */ "values ::= VALUES LP nexprlist RP", + /* 91 */ "values ::= values COMMA LP nexprlist RP", + /* 92 */ "distinct ::= DISTINCT", + /* 93 */ "distinct ::= ALL", + /* 94 */ "distinct ::=", + /* 95 */ "sclp ::=", + /* 96 */ "selcollist ::= sclp scanpt expr scanpt as", + /* 97 */ "selcollist ::= sclp scanpt STAR", + /* 98 */ "selcollist ::= sclp scanpt nm DOT STAR", + /* 99 */ "as ::= AS nm", + /* 100 */ "as ::=", + /* 101 */ "from ::=", + /* 102 */ "from ::= FROM seltablist", + /* 103 */ "stl_prefix ::= seltablist joinop", + /* 104 */ "stl_prefix ::=", + /* 105 */ "seltablist ::= stl_prefix nm dbnm as indexed_opt on_opt using_opt", + /* 106 */ "seltablist ::= stl_prefix nm dbnm LP exprlist RP as on_opt using_opt", + /* 107 */ "seltablist ::= stl_prefix LP select RP as on_opt using_opt", + /* 108 */ "seltablist ::= stl_prefix LP seltablist RP as on_opt using_opt", + /* 109 */ "dbnm ::=", + /* 110 */ "dbnm ::= DOT nm", + /* 111 */ "fullname ::= nm", + /* 112 */ "fullname ::= nm DOT nm", + /* 113 */ "xfullname ::= nm", + /* 114 */ "xfullname ::= nm DOT nm", + /* 115 */ "xfullname ::= nm DOT nm AS nm", + /* 116 */ "xfullname ::= nm AS nm", + /* 117 */ "joinop ::= COMMA|JOIN", + /* 118 */ "joinop ::= JOIN_KW JOIN", + /* 119 */ "joinop ::= JOIN_KW nm JOIN", + /* 120 */ "joinop ::= JOIN_KW nm nm JOIN", + /* 121 */ "on_opt ::= ON expr", + /* 122 */ "on_opt ::=", + /* 123 */ "indexed_opt ::=", + /* 124 */ "indexed_opt ::= INDEXED BY nm", + /* 125 */ "indexed_opt ::= NOT INDEXED", + /* 126 */ "using_opt ::= USING LP idlist RP", + /* 127 */ "using_opt ::=", + /* 128 */ "orderby_opt ::=", + /* 129 */ "orderby_opt ::= ORDER BY sortlist", + /* 130 */ "sortlist ::= sortlist COMMA expr sortorder nulls", + /* 131 */ "sortlist ::= expr sortorder nulls", + /* 132 */ "sortorder ::= ASC", + /* 133 */ "sortorder ::= DESC", + /* 134 */ "sortorder ::=", + /* 135 */ "nulls ::= NULLS FIRST", + /* 136 */ "nulls ::= NULLS LAST", + /* 137 */ "nulls ::=", + /* 138 */ "groupby_opt ::=", + /* 139 */ "groupby_opt ::= GROUP BY nexprlist", + /* 140 */ "having_opt ::=", + /* 141 */ "having_opt ::= HAVING expr", + /* 142 */ "limit_opt ::=", + /* 143 */ "limit_opt ::= LIMIT expr", + /* 144 */ "limit_opt ::= LIMIT expr OFFSET expr", + /* 145 */ "limit_opt ::= LIMIT expr COMMA expr", + /* 146 */ "cmd ::= with DELETE FROM xfullname indexed_opt where_opt", + /* 147 */ "where_opt ::=", + /* 148 */ "where_opt ::= WHERE expr", + /* 149 */ "cmd ::= with UPDATE orconf xfullname indexed_opt SET setlist where_opt", + /* 150 */ "setlist ::= setlist COMMA nm EQ expr", + /* 151 */ "setlist ::= setlist COMMA LP idlist RP EQ expr", + /* 152 */ "setlist ::= nm EQ expr", + /* 153 */ "setlist ::= LP idlist RP EQ expr", + /* 154 */ "cmd ::= with insert_cmd INTO xfullname idlist_opt select upsert", + /* 155 */ "cmd ::= with insert_cmd INTO xfullname idlist_opt DEFAULT VALUES", + /* 156 */ "upsert ::=", + /* 157 */ "upsert ::= ON CONFLICT LP sortlist RP where_opt DO UPDATE SET setlist where_opt", + /* 158 */ "upsert ::= ON CONFLICT LP sortlist RP where_opt DO NOTHING", + /* 159 */ "upsert ::= ON CONFLICT DO NOTHING", + /* 160 */ "insert_cmd ::= INSERT orconf", + /* 161 */ "insert_cmd ::= REPLACE", + /* 162 */ "idlist_opt ::=", + /* 163 */ "idlist_opt ::= LP idlist RP", + /* 164 */ "idlist ::= idlist COMMA nm", + /* 165 */ "idlist ::= nm", + /* 166 */ "expr ::= LP expr RP", + /* 167 */ "expr ::= ID|INDEXED", + /* 168 */ "expr ::= JOIN_KW", + /* 169 */ "expr ::= nm DOT nm", + /* 170 */ "expr ::= nm DOT nm DOT nm", + /* 171 */ "term ::= NULL|FLOAT|BLOB", + /* 172 */ "term ::= STRING", + /* 173 */ "term ::= INTEGER", + /* 174 */ "expr ::= VARIABLE", + /* 175 */ "expr ::= expr COLLATE ID|STRING", + /* 176 */ "expr ::= CAST LP expr AS typetoken RP", + /* 177 */ "expr ::= ID|INDEXED LP distinct exprlist RP", + /* 178 */ "expr ::= ID|INDEXED LP STAR RP", + /* 179 */ "expr ::= ID|INDEXED LP distinct exprlist RP filter_over", + /* 180 */ "expr ::= ID|INDEXED LP STAR RP filter_over", + /* 181 */ "term ::= CTIME_KW", + /* 182 */ "expr ::= LP nexprlist COMMA expr RP", + /* 183 */ "expr ::= expr AND expr", + /* 184 */ "expr ::= expr OR expr", + /* 185 */ "expr ::= expr LT|GT|GE|LE expr", + /* 186 */ "expr ::= expr EQ|NE expr", + /* 187 */ "expr ::= expr BITAND|BITOR|LSHIFT|RSHIFT expr", + /* 188 */ "expr ::= expr PLUS|MINUS expr", + /* 189 */ "expr ::= expr STAR|SLASH|REM expr", + /* 190 */ "expr ::= expr CONCAT expr", + /* 191 */ "likeop ::= NOT LIKE_KW|MATCH", + /* 192 */ "expr ::= expr likeop expr", + /* 193 */ "expr ::= expr likeop expr ESCAPE expr", + /* 194 */ "expr ::= expr ISNULL|NOTNULL", + /* 195 */ "expr ::= expr NOT NULL", + /* 196 */ "expr ::= expr IS expr", + /* 197 */ "expr ::= expr IS NOT expr", + /* 198 */ "expr ::= NOT expr", + /* 199 */ "expr ::= BITNOT expr", + /* 200 */ "expr ::= PLUS|MINUS expr", + /* 201 */ "between_op ::= BETWEEN", + /* 202 */ "between_op ::= NOT BETWEEN", + /* 203 */ "expr ::= expr between_op expr AND expr", + /* 204 */ "in_op ::= IN", + /* 205 */ "in_op ::= NOT IN", + /* 206 */ "expr ::= expr in_op LP exprlist RP", + /* 207 */ "expr ::= LP select RP", + /* 208 */ "expr ::= expr in_op LP select RP", + /* 209 */ "expr ::= expr in_op nm dbnm paren_exprlist", + /* 210 */ "expr ::= EXISTS LP select RP", + /* 211 */ "expr ::= CASE case_operand case_exprlist case_else END", + /* 212 */ "case_exprlist ::= case_exprlist WHEN expr THEN expr", + /* 213 */ "case_exprlist ::= WHEN expr THEN expr", + /* 214 */ "case_else ::= ELSE expr", + /* 215 */ "case_else ::=", + /* 216 */ "case_operand ::= expr", + /* 217 */ "case_operand ::=", + /* 218 */ "exprlist ::=", + /* 219 */ "nexprlist ::= nexprlist COMMA expr", + /* 220 */ "nexprlist ::= expr", + /* 221 */ "paren_exprlist ::=", + /* 222 */ "paren_exprlist ::= LP exprlist RP", + /* 223 */ "cmd ::= createkw uniqueflag INDEX ifnotexists nm dbnm ON nm LP sortlist RP where_opt", + /* 224 */ "uniqueflag ::= UNIQUE", + /* 225 */ "uniqueflag ::=", + /* 226 */ "eidlist_opt ::=", + /* 227 */ "eidlist_opt ::= LP eidlist RP", + /* 228 */ "eidlist ::= eidlist COMMA nm collate sortorder", + /* 229 */ "eidlist ::= nm collate sortorder", + /* 230 */ "collate ::=", + /* 231 */ "collate ::= COLLATE ID|STRING", + /* 232 */ "cmd ::= DROP INDEX ifexists fullname", + /* 233 */ "cmd ::= VACUUM vinto", + /* 234 */ "cmd ::= VACUUM nm vinto", + /* 235 */ "vinto ::= INTO expr", + /* 236 */ "vinto ::=", + /* 237 */ "cmd ::= PRAGMA nm dbnm", + /* 238 */ "cmd ::= PRAGMA nm dbnm EQ nmnum", + /* 239 */ "cmd ::= PRAGMA nm dbnm LP nmnum RP", + /* 240 */ "cmd ::= PRAGMA nm dbnm EQ minus_num", + /* 241 */ "cmd ::= PRAGMA nm dbnm LP minus_num RP", + /* 242 */ "plus_num ::= PLUS INTEGER|FLOAT", + /* 243 */ "minus_num ::= MINUS INTEGER|FLOAT", + /* 244 */ "cmd ::= createkw trigger_decl BEGIN trigger_cmd_list END", + /* 245 */ "trigger_decl ::= temp TRIGGER ifnotexists nm dbnm trigger_time trigger_event ON fullname foreach_clause when_clause", + /* 246 */ "trigger_time ::= BEFORE|AFTER", + /* 247 */ "trigger_time ::= INSTEAD OF", + /* 248 */ "trigger_time ::=", + /* 249 */ "trigger_event ::= DELETE|INSERT", + /* 250 */ "trigger_event ::= UPDATE", + /* 251 */ "trigger_event ::= UPDATE OF idlist", + /* 252 */ "when_clause ::=", + /* 253 */ "when_clause ::= WHEN expr", + /* 254 */ "trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI", + /* 255 */ "trigger_cmd_list ::= trigger_cmd SEMI", + /* 256 */ "trnm ::= nm DOT nm", + /* 257 */ "tridxby ::= INDEXED BY nm", + /* 258 */ "tridxby ::= NOT INDEXED", + /* 259 */ "trigger_cmd ::= UPDATE orconf trnm tridxby SET setlist where_opt scanpt", + /* 260 */ "trigger_cmd ::= scanpt insert_cmd INTO trnm idlist_opt select upsert scanpt", + /* 261 */ "trigger_cmd ::= DELETE FROM trnm tridxby where_opt scanpt", + /* 262 */ "trigger_cmd ::= scanpt select scanpt", + /* 263 */ "expr ::= RAISE LP IGNORE RP", + /* 264 */ "expr ::= RAISE LP raisetype COMMA nm RP", + /* 265 */ "raisetype ::= ROLLBACK", + /* 266 */ "raisetype ::= ABORT", + /* 267 */ "raisetype ::= FAIL", + /* 268 */ "cmd ::= DROP TRIGGER ifexists fullname", + /* 269 */ "cmd ::= ATTACH database_kw_opt expr AS expr key_opt", + /* 270 */ "cmd ::= DETACH database_kw_opt expr", + /* 271 */ "key_opt ::=", + /* 272 */ "key_opt ::= KEY expr", + /* 273 */ "cmd ::= REINDEX", + /* 274 */ "cmd ::= REINDEX nm dbnm", + /* 275 */ "cmd ::= ANALYZE", + /* 276 */ "cmd ::= ANALYZE nm dbnm", + /* 277 */ "cmd ::= ALTER TABLE fullname RENAME TO nm", + /* 278 */ "cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt columnname carglist", + /* 279 */ "add_column_fullname ::= fullname", + /* 280 */ "cmd ::= ALTER TABLE fullname RENAME kwcolumn_opt nm TO nm", + /* 281 */ "cmd ::= create_vtab", + /* 282 */ "cmd ::= create_vtab LP vtabarglist RP", + /* 283 */ "create_vtab ::= createkw VIRTUAL TABLE ifnotexists nm dbnm USING nm", + /* 284 */ "vtabarg ::=", + /* 285 */ "vtabargtoken ::= ANY", + /* 286 */ "vtabargtoken ::= lp anylist RP", + /* 287 */ "lp ::= LP", + /* 288 */ "with ::= WITH wqlist", + /* 289 */ "with ::= WITH RECURSIVE wqlist", + /* 290 */ "wqlist ::= nm eidlist_opt AS LP select RP", + /* 291 */ "wqlist ::= wqlist COMMA nm eidlist_opt AS LP select RP", + /* 292 */ "windowdefn_list ::= windowdefn", + /* 293 */ "windowdefn_list ::= windowdefn_list COMMA windowdefn", + /* 294 */ "windowdefn ::= nm AS LP window RP", + /* 295 */ "window ::= PARTITION BY nexprlist orderby_opt frame_opt", + /* 296 */ "window ::= nm PARTITION BY nexprlist orderby_opt frame_opt", + /* 297 */ "window ::= ORDER BY sortlist frame_opt", + /* 298 */ "window ::= nm ORDER BY sortlist frame_opt", + /* 299 */ "window ::= frame_opt", + /* 300 */ "window ::= nm frame_opt", + /* 301 */ "frame_opt ::=", + /* 302 */ "frame_opt ::= range_or_rows frame_bound_s frame_exclude_opt", + /* 303 */ "frame_opt ::= range_or_rows BETWEEN frame_bound_s AND frame_bound_e frame_exclude_opt", + /* 304 */ "range_or_rows ::= RANGE|ROWS|GROUPS", + /* 305 */ "frame_bound_s ::= frame_bound", + /* 306 */ "frame_bound_s ::= UNBOUNDED PRECEDING", + /* 307 */ "frame_bound_e ::= frame_bound", + /* 308 */ "frame_bound_e ::= UNBOUNDED FOLLOWING", + /* 309 */ "frame_bound ::= expr PRECEDING|FOLLOWING", + /* 310 */ "frame_bound ::= CURRENT ROW", + /* 311 */ "frame_exclude_opt ::=", + /* 312 */ "frame_exclude_opt ::= EXCLUDE frame_exclude", + /* 313 */ "frame_exclude ::= NO OTHERS", + /* 314 */ "frame_exclude ::= CURRENT ROW", + /* 315 */ "frame_exclude ::= GROUP|TIES", + /* 316 */ "window_clause ::= WINDOW windowdefn_list", + /* 317 */ "filter_over ::= filter_clause over_clause", + /* 318 */ "filter_over ::= over_clause", + /* 319 */ "filter_over ::= filter_clause", + /* 320 */ "over_clause ::= OVER LP window RP", + /* 321 */ "over_clause ::= OVER nm", + /* 322 */ "filter_clause ::= FILTER LP WHERE expr RP", + /* 323 */ "input ::= cmdlist", + /* 324 */ "cmdlist ::= cmdlist ecmd", + /* 325 */ "cmdlist ::= ecmd", + /* 326 */ "ecmd ::= SEMI", + /* 327 */ "ecmd ::= cmdx SEMI", + /* 328 */ "ecmd ::= explain cmdx", + /* 329 */ "trans_opt ::=", + /* 330 */ "trans_opt ::= TRANSACTION", + /* 331 */ "trans_opt ::= TRANSACTION nm", + /* 332 */ "savepoint_opt ::= SAVEPOINT", + /* 333 */ "savepoint_opt ::=", + /* 334 */ "cmd ::= create_table create_table_args", + /* 335 */ "columnlist ::= columnlist COMMA columnname carglist", + /* 336 */ "columnlist ::= columnname carglist", + /* 337 */ "nm ::= ID|INDEXED", + /* 338 */ "nm ::= STRING", + /* 339 */ "nm ::= JOIN_KW", + /* 340 */ "typetoken ::= typename", + /* 341 */ "typename ::= ID|STRING", + /* 342 */ "signed ::= plus_num", + /* 343 */ "signed ::= minus_num", + /* 344 */ "carglist ::= carglist ccons", + /* 345 */ "carglist ::=", + /* 346 */ "ccons ::= NULL onconf", + /* 347 */ "conslist_opt ::= COMMA conslist", + /* 348 */ "conslist ::= conslist tconscomma tcons", + /* 349 */ "conslist ::= tcons", + /* 350 */ "tconscomma ::=", + /* 351 */ "defer_subclause_opt ::= defer_subclause", + /* 352 */ "resolvetype ::= raisetype", + /* 353 */ "selectnowith ::= oneselect", + /* 354 */ "oneselect ::= values", + /* 355 */ "sclp ::= selcollist COMMA", + /* 356 */ "as ::= ID|STRING", + /* 357 */ "expr ::= term", + /* 358 */ "likeop ::= LIKE_KW|MATCH", + /* 359 */ "exprlist ::= nexprlist", + /* 360 */ "nmnum ::= plus_num", + /* 361 */ "nmnum ::= nm", + /* 362 */ "nmnum ::= ON", + /* 363 */ "nmnum ::= DELETE", + /* 364 */ "nmnum ::= DEFAULT", + /* 365 */ "plus_num ::= INTEGER|FLOAT", + /* 366 */ "foreach_clause ::=", + /* 367 */ "foreach_clause ::= FOR EACH ROW", + /* 368 */ "trnm ::= nm", + /* 369 */ "tridxby ::=", + /* 370 */ "database_kw_opt ::= DATABASE", + /* 371 */ "database_kw_opt ::=", + /* 372 */ "kwcolumn_opt ::=", + /* 373 */ "kwcolumn_opt ::= COLUMNKW", + /* 374 */ "vtabarglist ::= vtabarg", + /* 375 */ "vtabarglist ::= vtabarglist COMMA vtabarg", + /* 376 */ "vtabarg ::= vtabarg vtabargtoken", + /* 377 */ "anylist ::=", + /* 378 */ "anylist ::= anylist LP anylist RP", + /* 379 */ "anylist ::= anylist ANY", + /* 380 */ "with ::=", }; #endif /* NDEBUG */ @@ -149215,97 +151696,98 @@ static void yy_destructor( ** inside the C code. */ /********* Begin destructor definitions ***************************************/ - case 174: /* select */ - case 206: /* selectnowith */ - case 207: /* oneselect */ - case 219: /* values */ + case 198: /* select */ + case 231: /* selectnowith */ + case 232: /* oneselect */ + case 244: /* values */ { -sqlite3SelectDelete(pParse->db, (yypminor->yy423)); +sqlite3SelectDelete(pParse->db, (yypminor->yy25)); } break; - case 184: /* term */ - case 185: /* expr */ - case 213: /* where_opt */ - case 215: /* having_opt */ - case 227: /* on_opt */ - case 242: /* case_operand */ - case 244: /* case_else */ - case 247: /* vinto */ - case 254: /* when_clause */ - case 259: /* key_opt */ - case 273: /* filter_opt */ + case 209: /* term */ + case 210: /* expr */ + case 238: /* where_opt */ + case 240: /* having_opt */ + case 252: /* on_opt */ + case 268: /* case_operand */ + case 270: /* case_else */ + case 273: /* vinto */ + case 280: /* when_clause */ + case 285: /* key_opt */ + case 299: /* filter_clause */ { -sqlite3ExprDelete(pParse->db, (yypminor->yy490)); +sqlite3ExprDelete(pParse->db, (yypminor->yy46)); } break; - case 189: /* eidlist_opt */ - case 198: /* sortlist */ - case 199: /* eidlist */ - case 211: /* selcollist */ - case 214: /* groupby_opt */ - case 216: /* orderby_opt */ - case 220: /* nexprlist */ - case 221: /* sclp */ - case 229: /* exprlist */ - case 233: /* setlist */ - case 241: /* paren_exprlist */ - case 243: /* case_exprlist */ - case 272: /* part_opt */ + case 214: /* eidlist_opt */ + case 223: /* sortlist */ + case 224: /* eidlist */ + case 236: /* selcollist */ + case 239: /* groupby_opt */ + case 241: /* orderby_opt */ + case 245: /* nexprlist */ + case 246: /* sclp */ + case 254: /* exprlist */ + case 259: /* setlist */ + case 267: /* paren_exprlist */ + case 269: /* case_exprlist */ + case 298: /* part_opt */ { -sqlite3ExprListDelete(pParse->db, (yypminor->yy42)); +sqlite3ExprListDelete(pParse->db, (yypminor->yy138)); } break; - case 205: /* fullname */ - case 212: /* from */ - case 223: /* seltablist */ - case 224: /* stl_prefix */ - case 230: /* xfullname */ + case 230: /* fullname */ + case 237: /* from */ + case 248: /* seltablist */ + case 249: /* stl_prefix */ + case 255: /* xfullname */ { -sqlite3SrcListDelete(pParse->db, (yypminor->yy167)); +sqlite3SrcListDelete(pParse->db, (yypminor->yy609)); } break; - case 208: /* wqlist */ + case 233: /* wqlist */ { -sqlite3WithDelete(pParse->db, (yypminor->yy499)); +sqlite3WithDelete(pParse->db, (yypminor->yy297)); } break; - case 218: /* window_clause */ - case 268: /* windowdefn_list */ + case 243: /* window_clause */ + case 294: /* windowdefn_list */ { -sqlite3WindowListDelete(pParse->db, (yypminor->yy147)); +sqlite3WindowListDelete(pParse->db, (yypminor->yy455)); } break; - case 228: /* using_opt */ - case 231: /* idlist */ - case 235: /* idlist_opt */ + case 253: /* using_opt */ + case 256: /* idlist */ + case 261: /* idlist_opt */ { -sqlite3IdListDelete(pParse->db, (yypminor->yy336)); +sqlite3IdListDelete(pParse->db, (yypminor->yy406)); } break; - case 237: /* over_clause */ - case 269: /* windowdefn */ - case 270: /* window */ - case 271: /* frame_opt */ + case 263: /* filter_over */ + case 295: /* windowdefn */ + case 296: /* window */ + case 297: /* frame_opt */ + case 300: /* over_clause */ { -sqlite3WindowDelete(pParse->db, (yypminor->yy147)); +sqlite3WindowDelete(pParse->db, (yypminor->yy455)); } break; - case 250: /* trigger_cmd_list */ - case 255: /* trigger_cmd */ + case 276: /* trigger_cmd_list */ + case 281: /* trigger_cmd */ { -sqlite3DeleteTriggerStep(pParse->db, (yypminor->yy119)); +sqlite3DeleteTriggerStep(pParse->db, (yypminor->yy527)); } break; - case 252: /* trigger_event */ + case 278: /* trigger_event */ { -sqlite3IdListDelete(pParse->db, (yypminor->yy350).b); +sqlite3IdListDelete(pParse->db, (yypminor->yy572).b); } break; - case 275: /* frame_bound */ - case 276: /* frame_bound_s */ - case 277: /* frame_bound_e */ + case 302: /* frame_bound */ + case 303: /* frame_bound_s */ + case 304: /* frame_bound_e */ { -sqlite3ExprDelete(pParse->db, (yypminor->yy317).pExpr); +sqlite3ExprDelete(pParse->db, (yypminor->yy57).pExpr); } break; /********* End destructor definitions *****************************************/ @@ -149431,15 +151913,18 @@ static YYACTIONTYPE yy_find_shift_action( do{ i = yy_shift_ofst[stateno]; assert( i>=0 ); - /* assert( i+YYNTOKEN<=(int)YY_NLOOKAHEAD ); */ + assert( i<=YY_ACTTAB_COUNT ); + assert( i+YYNTOKEN<=(int)YY_NLOOKAHEAD ); assert( iLookAhead!=YYNOCODE ); assert( iLookAhead < YYNTOKEN ); i += iLookAhead; - if( i>=YY_NLOOKAHEAD || yy_lookahead[i]!=iLookAhead ){ + assert( i<(int)YY_NLOOKAHEAD ); + if( yy_lookahead[i]!=iLookAhead ){ #ifdef YYFALLBACK YYCODETYPE iFallback; /* Fallback token */ - if( iLookAhead %s\n", @@ -149454,16 +151939,8 @@ static YYACTIONTYPE yy_find_shift_action( #ifdef YYWILDCARD { int j = i - iLookAhead + YYWILDCARD; - if( -#if YY_SHIFT_MIN+YYWILDCARD<0 - j>=0 && -#endif -#if YY_SHIFT_MAX+YYWILDCARD>=YY_ACTTAB_COUNT - j0 - ){ + assert( j<(int)(sizeof(yy_lookahead)/sizeof(yy_lookahead[0])) ); + if( yy_lookahead[j]==YYWILDCARD && iLookAhead>0 ){ #ifndef NDEBUG if( yyTraceFILE ){ fprintf(yyTraceFILE, "%sWILDCARD %s => %s\n", @@ -149477,6 +151954,7 @@ static YYACTIONTYPE yy_find_shift_action( #endif /* YYWILDCARD */ return yy_default[stateno]; }else{ + assert( i>=0 && idb, yymsp[0].minor.yy423); + sqlite3EndTable(pParse,0,0,0,yymsp[0].minor.yy25); + sqlite3SelectDelete(pParse->db, yymsp[0].minor.yy25); } break; case 22: /* table_options ::= WITHOUT nm */ { if( yymsp[0].minor.yy0.n==5 && sqlite3_strnicmp(yymsp[0].minor.yy0.z,"rowid",5)==0 ){ - yymsp[-1].minor.yy96 = TF_WithoutRowid | TF_NoVisibleRowid; + yymsp[-1].minor.yy32 = TF_WithoutRowid | TF_NoVisibleRowid; }else{ - yymsp[-1].minor.yy96 = 0; + yymsp[-1].minor.yy32 = 0; sqlite3ErrorMsg(pParse, "unknown table option: %.*s", yymsp[0].minor.yy0.n, yymsp[0].minor.yy0.z); } } @@ -150519,8 +153022,8 @@ static YYACTIONTYPE yy_reduce( {sqlite3AddColumn(pParse,&yymsp[-1].minor.yy0,&yymsp[0].minor.yy0);} break; case 24: /* typetoken ::= */ - case 60: /* conslist_opt ::= */ yytestcase(yyruleno==60); - case 99: /* as ::= */ yytestcase(yyruleno==99); + case 61: /* conslist_opt ::= */ yytestcase(yyruleno==61); + case 100: /* as ::= */ yytestcase(yyruleno==100); {yymsp[1].minor.yy0.n = 0; yymsp[1].minor.yy0.z = 0;} break; case 25: /* typetoken ::= typename LP signed RP */ @@ -150539,29 +153042,35 @@ static YYACTIONTYPE yy_reduce( case 28: /* scanpt ::= */ { assert( yyLookahead!=YYNOCODE ); - yymsp[1].minor.yy464 = yyLookaheadToken.z; + yymsp[1].minor.yy8 = yyLookaheadToken.z; +} + break; + case 29: /* scantok ::= */ +{ + assert( yyLookahead!=YYNOCODE ); + yymsp[1].minor.yy0 = yyLookaheadToken; } break; - case 29: /* ccons ::= CONSTRAINT nm */ - case 62: /* tcons ::= CONSTRAINT nm */ yytestcase(yyruleno==62); + case 30: /* ccons ::= CONSTRAINT nm */ + case 63: /* tcons ::= CONSTRAINT nm */ yytestcase(yyruleno==63); {pParse->constraintName = yymsp[0].minor.yy0;} break; - case 30: /* ccons ::= DEFAULT scanpt term scanpt */ -{sqlite3AddDefaultValue(pParse,yymsp[-1].minor.yy490,yymsp[-2].minor.yy464,yymsp[0].minor.yy464);} + case 31: /* ccons ::= DEFAULT scantok term */ +{sqlite3AddDefaultValue(pParse,yymsp[0].minor.yy46,yymsp[-1].minor.yy0.z,&yymsp[-1].minor.yy0.z[yymsp[-1].minor.yy0.n]);} break; - case 31: /* ccons ::= DEFAULT LP expr RP */ -{sqlite3AddDefaultValue(pParse,yymsp[-1].minor.yy490,yymsp[-2].minor.yy0.z+1,yymsp[0].minor.yy0.z);} + case 32: /* ccons ::= DEFAULT LP expr RP */ +{sqlite3AddDefaultValue(pParse,yymsp[-1].minor.yy46,yymsp[-2].minor.yy0.z+1,yymsp[0].minor.yy0.z);} break; - case 32: /* ccons ::= DEFAULT PLUS term scanpt */ -{sqlite3AddDefaultValue(pParse,yymsp[-1].minor.yy490,yymsp[-2].minor.yy0.z,yymsp[0].minor.yy464);} + case 33: /* ccons ::= DEFAULT PLUS scantok term */ +{sqlite3AddDefaultValue(pParse,yymsp[0].minor.yy46,yymsp[-2].minor.yy0.z,&yymsp[-1].minor.yy0.z[yymsp[-1].minor.yy0.n]);} break; - case 33: /* ccons ::= DEFAULT MINUS term scanpt */ + case 34: /* ccons ::= DEFAULT MINUS scantok term */ { - Expr *p = sqlite3PExpr(pParse, TK_UMINUS, yymsp[-1].minor.yy490, 0); - sqlite3AddDefaultValue(pParse,p,yymsp[-2].minor.yy0.z,yymsp[0].minor.yy464); + Expr *p = sqlite3PExpr(pParse, TK_UMINUS, yymsp[0].minor.yy46, 0); + sqlite3AddDefaultValue(pParse,p,yymsp[-2].minor.yy0.z,&yymsp[-1].minor.yy0.z[yymsp[-1].minor.yy0.n]); } break; - case 34: /* ccons ::= DEFAULT scanpt ID|INDEXED */ + case 35: /* ccons ::= DEFAULT scantok ID|INDEXED */ { Expr *p = tokenExpr(pParse, TK_STRING, yymsp[0].minor.yy0); if( p ){ @@ -150571,171 +153080,171 @@ static YYACTIONTYPE yy_reduce( sqlite3AddDefaultValue(pParse,p,yymsp[0].minor.yy0.z,yymsp[0].minor.yy0.z+yymsp[0].minor.yy0.n); } break; - case 35: /* ccons ::= NOT NULL onconf */ -{sqlite3AddNotNull(pParse, yymsp[0].minor.yy96);} + case 36: /* ccons ::= NOT NULL onconf */ +{sqlite3AddNotNull(pParse, yymsp[0].minor.yy32);} break; - case 36: /* ccons ::= PRIMARY KEY sortorder onconf autoinc */ -{sqlite3AddPrimaryKey(pParse,0,yymsp[-1].minor.yy96,yymsp[0].minor.yy96,yymsp[-2].minor.yy96);} + case 37: /* ccons ::= PRIMARY KEY sortorder onconf autoinc */ +{sqlite3AddPrimaryKey(pParse,0,yymsp[-1].minor.yy32,yymsp[0].minor.yy32,yymsp[-2].minor.yy32);} break; - case 37: /* ccons ::= UNIQUE onconf */ -{sqlite3CreateIndex(pParse,0,0,0,0,yymsp[0].minor.yy96,0,0,0,0, + case 38: /* ccons ::= UNIQUE onconf */ +{sqlite3CreateIndex(pParse,0,0,0,0,yymsp[0].minor.yy32,0,0,0,0, SQLITE_IDXTYPE_UNIQUE);} break; - case 38: /* ccons ::= CHECK LP expr RP */ -{sqlite3AddCheckConstraint(pParse,yymsp[-1].minor.yy490);} + case 39: /* ccons ::= CHECK LP expr RP */ +{sqlite3AddCheckConstraint(pParse,yymsp[-1].minor.yy46);} break; - case 39: /* ccons ::= REFERENCES nm eidlist_opt refargs */ -{sqlite3CreateForeignKey(pParse,0,&yymsp[-2].minor.yy0,yymsp[-1].minor.yy42,yymsp[0].minor.yy96);} + case 40: /* ccons ::= REFERENCES nm eidlist_opt refargs */ +{sqlite3CreateForeignKey(pParse,0,&yymsp[-2].minor.yy0,yymsp[-1].minor.yy138,yymsp[0].minor.yy32);} break; - case 40: /* ccons ::= defer_subclause */ -{sqlite3DeferForeignKey(pParse,yymsp[0].minor.yy96);} + case 41: /* ccons ::= defer_subclause */ +{sqlite3DeferForeignKey(pParse,yymsp[0].minor.yy32);} break; - case 41: /* ccons ::= COLLATE ID|STRING */ + case 42: /* ccons ::= COLLATE ID|STRING */ {sqlite3AddCollateType(pParse, &yymsp[0].minor.yy0);} break; - case 44: /* refargs ::= */ -{ yymsp[1].minor.yy96 = OE_None*0x0101; /* EV: R-19803-45884 */} + case 45: /* refargs ::= */ +{ yymsp[1].minor.yy32 = OE_None*0x0101; /* EV: R-19803-45884 */} break; - case 45: /* refargs ::= refargs refarg */ -{ yymsp[-1].minor.yy96 = (yymsp[-1].minor.yy96 & ~yymsp[0].minor.yy367.mask) | yymsp[0].minor.yy367.value; } + case 46: /* refargs ::= refargs refarg */ +{ yymsp[-1].minor.yy32 = (yymsp[-1].minor.yy32 & ~yymsp[0].minor.yy495.mask) | yymsp[0].minor.yy495.value; } break; - case 46: /* refarg ::= MATCH nm */ -{ yymsp[-1].minor.yy367.value = 0; yymsp[-1].minor.yy367.mask = 0x000000; } + case 47: /* refarg ::= MATCH nm */ +{ yymsp[-1].minor.yy495.value = 0; yymsp[-1].minor.yy495.mask = 0x000000; } break; - case 47: /* refarg ::= ON INSERT refact */ -{ yymsp[-2].minor.yy367.value = 0; yymsp[-2].minor.yy367.mask = 0x000000; } + case 48: /* refarg ::= ON INSERT refact */ +{ yymsp[-2].minor.yy495.value = 0; yymsp[-2].minor.yy495.mask = 0x000000; } break; - case 48: /* refarg ::= ON DELETE refact */ -{ yymsp[-2].minor.yy367.value = yymsp[0].minor.yy96; yymsp[-2].minor.yy367.mask = 0x0000ff; } + case 49: /* refarg ::= ON DELETE refact */ +{ yymsp[-2].minor.yy495.value = yymsp[0].minor.yy32; yymsp[-2].minor.yy495.mask = 0x0000ff; } break; - case 49: /* refarg ::= ON UPDATE refact */ -{ yymsp[-2].minor.yy367.value = yymsp[0].minor.yy96<<8; yymsp[-2].minor.yy367.mask = 0x00ff00; } + case 50: /* refarg ::= ON UPDATE refact */ +{ yymsp[-2].minor.yy495.value = yymsp[0].minor.yy32<<8; yymsp[-2].minor.yy495.mask = 0x00ff00; } break; - case 50: /* refact ::= SET NULL */ -{ yymsp[-1].minor.yy96 = OE_SetNull; /* EV: R-33326-45252 */} + case 51: /* refact ::= SET NULL */ +{ yymsp[-1].minor.yy32 = OE_SetNull; /* EV: R-33326-45252 */} break; - case 51: /* refact ::= SET DEFAULT */ -{ yymsp[-1].minor.yy96 = OE_SetDflt; /* EV: R-33326-45252 */} + case 52: /* refact ::= SET DEFAULT */ +{ yymsp[-1].minor.yy32 = OE_SetDflt; /* EV: R-33326-45252 */} break; - case 52: /* refact ::= CASCADE */ -{ yymsp[0].minor.yy96 = OE_Cascade; /* EV: R-33326-45252 */} + case 53: /* refact ::= CASCADE */ +{ yymsp[0].minor.yy32 = OE_Cascade; /* EV: R-33326-45252 */} break; - case 53: /* refact ::= RESTRICT */ -{ yymsp[0].minor.yy96 = OE_Restrict; /* EV: R-33326-45252 */} + case 54: /* refact ::= RESTRICT */ +{ yymsp[0].minor.yy32 = OE_Restrict; /* EV: R-33326-45252 */} break; - case 54: /* refact ::= NO ACTION */ -{ yymsp[-1].minor.yy96 = OE_None; /* EV: R-33326-45252 */} + case 55: /* refact ::= NO ACTION */ +{ yymsp[-1].minor.yy32 = OE_None; /* EV: R-33326-45252 */} break; - case 55: /* defer_subclause ::= NOT DEFERRABLE init_deferred_pred_opt */ -{yymsp[-2].minor.yy96 = 0;} + case 56: /* defer_subclause ::= NOT DEFERRABLE init_deferred_pred_opt */ +{yymsp[-2].minor.yy32 = 0;} break; - case 56: /* defer_subclause ::= DEFERRABLE init_deferred_pred_opt */ - case 71: /* orconf ::= OR resolvetype */ yytestcase(yyruleno==71); - case 156: /* insert_cmd ::= INSERT orconf */ yytestcase(yyruleno==156); -{yymsp[-1].minor.yy96 = yymsp[0].minor.yy96;} + case 57: /* defer_subclause ::= DEFERRABLE init_deferred_pred_opt */ + case 72: /* orconf ::= OR resolvetype */ yytestcase(yyruleno==72); + case 160: /* insert_cmd ::= INSERT orconf */ yytestcase(yyruleno==160); +{yymsp[-1].minor.yy32 = yymsp[0].minor.yy32;} break; - case 58: /* init_deferred_pred_opt ::= INITIALLY DEFERRED */ - case 75: /* ifexists ::= IF EXISTS */ yytestcase(yyruleno==75); - case 198: /* between_op ::= NOT BETWEEN */ yytestcase(yyruleno==198); - case 201: /* in_op ::= NOT IN */ yytestcase(yyruleno==201); - case 227: /* collate ::= COLLATE ID|STRING */ yytestcase(yyruleno==227); -{yymsp[-1].minor.yy96 = 1;} + case 59: /* init_deferred_pred_opt ::= INITIALLY DEFERRED */ + case 76: /* ifexists ::= IF EXISTS */ yytestcase(yyruleno==76); + case 202: /* between_op ::= NOT BETWEEN */ yytestcase(yyruleno==202); + case 205: /* in_op ::= NOT IN */ yytestcase(yyruleno==205); + case 231: /* collate ::= COLLATE ID|STRING */ yytestcase(yyruleno==231); +{yymsp[-1].minor.yy32 = 1;} break; - case 59: /* init_deferred_pred_opt ::= INITIALLY IMMEDIATE */ -{yymsp[-1].minor.yy96 = 0;} + case 60: /* init_deferred_pred_opt ::= INITIALLY IMMEDIATE */ +{yymsp[-1].minor.yy32 = 0;} break; - case 61: /* tconscomma ::= COMMA */ + case 62: /* tconscomma ::= COMMA */ {pParse->constraintName.n = 0;} break; - case 63: /* tcons ::= PRIMARY KEY LP sortlist autoinc RP onconf */ -{sqlite3AddPrimaryKey(pParse,yymsp[-3].minor.yy42,yymsp[0].minor.yy96,yymsp[-2].minor.yy96,0);} + case 64: /* tcons ::= PRIMARY KEY LP sortlist autoinc RP onconf */ +{sqlite3AddPrimaryKey(pParse,yymsp[-3].minor.yy138,yymsp[0].minor.yy32,yymsp[-2].minor.yy32,0);} break; - case 64: /* tcons ::= UNIQUE LP sortlist RP onconf */ -{sqlite3CreateIndex(pParse,0,0,0,yymsp[-2].minor.yy42,yymsp[0].minor.yy96,0,0,0,0, + case 65: /* tcons ::= UNIQUE LP sortlist RP onconf */ +{sqlite3CreateIndex(pParse,0,0,0,yymsp[-2].minor.yy138,yymsp[0].minor.yy32,0,0,0,0, SQLITE_IDXTYPE_UNIQUE);} break; - case 65: /* tcons ::= CHECK LP expr RP onconf */ -{sqlite3AddCheckConstraint(pParse,yymsp[-2].minor.yy490);} + case 66: /* tcons ::= CHECK LP expr RP onconf */ +{sqlite3AddCheckConstraint(pParse,yymsp[-2].minor.yy46);} break; - case 66: /* tcons ::= FOREIGN KEY LP eidlist RP REFERENCES nm eidlist_opt refargs defer_subclause_opt */ + case 67: /* tcons ::= FOREIGN KEY LP eidlist RP REFERENCES nm eidlist_opt refargs defer_subclause_opt */ { - sqlite3CreateForeignKey(pParse, yymsp[-6].minor.yy42, &yymsp[-3].minor.yy0, yymsp[-2].minor.yy42, yymsp[-1].minor.yy96); - sqlite3DeferForeignKey(pParse, yymsp[0].minor.yy96); + sqlite3CreateForeignKey(pParse, yymsp[-6].minor.yy138, &yymsp[-3].minor.yy0, yymsp[-2].minor.yy138, yymsp[-1].minor.yy32); + sqlite3DeferForeignKey(pParse, yymsp[0].minor.yy32); } break; - case 68: /* onconf ::= */ - case 70: /* orconf ::= */ yytestcase(yyruleno==70); -{yymsp[1].minor.yy96 = OE_Default;} + case 69: /* onconf ::= */ + case 71: /* orconf ::= */ yytestcase(yyruleno==71); +{yymsp[1].minor.yy32 = OE_Default;} break; - case 69: /* onconf ::= ON CONFLICT resolvetype */ -{yymsp[-2].minor.yy96 = yymsp[0].minor.yy96;} + case 70: /* onconf ::= ON CONFLICT resolvetype */ +{yymsp[-2].minor.yy32 = yymsp[0].minor.yy32;} break; - case 72: /* resolvetype ::= IGNORE */ -{yymsp[0].minor.yy96 = OE_Ignore;} + case 73: /* resolvetype ::= IGNORE */ +{yymsp[0].minor.yy32 = OE_Ignore;} break; - case 73: /* resolvetype ::= REPLACE */ - case 157: /* insert_cmd ::= REPLACE */ yytestcase(yyruleno==157); -{yymsp[0].minor.yy96 = OE_Replace;} + case 74: /* resolvetype ::= REPLACE */ + case 161: /* insert_cmd ::= REPLACE */ yytestcase(yyruleno==161); +{yymsp[0].minor.yy32 = OE_Replace;} break; - case 74: /* cmd ::= DROP TABLE ifexists fullname */ + case 75: /* cmd ::= DROP TABLE ifexists fullname */ { - sqlite3DropTable(pParse, yymsp[0].minor.yy167, 0, yymsp[-1].minor.yy96); + sqlite3DropTable(pParse, yymsp[0].minor.yy609, 0, yymsp[-1].minor.yy32); } break; - case 77: /* cmd ::= createkw temp VIEW ifnotexists nm dbnm eidlist_opt AS select */ + case 78: /* cmd ::= createkw temp VIEW ifnotexists nm dbnm eidlist_opt AS select */ { - sqlite3CreateView(pParse, &yymsp[-8].minor.yy0, &yymsp[-4].minor.yy0, &yymsp[-3].minor.yy0, yymsp[-2].minor.yy42, yymsp[0].minor.yy423, yymsp[-7].minor.yy96, yymsp[-5].minor.yy96); + sqlite3CreateView(pParse, &yymsp[-8].minor.yy0, &yymsp[-4].minor.yy0, &yymsp[-3].minor.yy0, yymsp[-2].minor.yy138, yymsp[0].minor.yy25, yymsp[-7].minor.yy32, yymsp[-5].minor.yy32); } break; - case 78: /* cmd ::= DROP VIEW ifexists fullname */ + case 79: /* cmd ::= DROP VIEW ifexists fullname */ { - sqlite3DropTable(pParse, yymsp[0].minor.yy167, 1, yymsp[-1].minor.yy96); + sqlite3DropTable(pParse, yymsp[0].minor.yy609, 1, yymsp[-1].minor.yy32); } break; - case 79: /* cmd ::= select */ + case 80: /* cmd ::= select */ { SelectDest dest = {SRT_Output, 0, 0, 0, 0, 0}; - sqlite3Select(pParse, yymsp[0].minor.yy423, &dest); - sqlite3SelectDelete(pParse->db, yymsp[0].minor.yy423); + sqlite3Select(pParse, yymsp[0].minor.yy25, &dest); + sqlite3SelectDelete(pParse->db, yymsp[0].minor.yy25); } break; - case 80: /* select ::= WITH wqlist selectnowith */ + case 81: /* select ::= WITH wqlist selectnowith */ { - Select *p = yymsp[0].minor.yy423; + Select *p = yymsp[0].minor.yy25; if( p ){ - p->pWith = yymsp[-1].minor.yy499; + p->pWith = yymsp[-1].minor.yy297; parserDoubleLinkSelect(pParse, p); }else{ - sqlite3WithDelete(pParse->db, yymsp[-1].minor.yy499); + sqlite3WithDelete(pParse->db, yymsp[-1].minor.yy297); } - yymsp[-2].minor.yy423 = p; + yymsp[-2].minor.yy25 = p; } break; - case 81: /* select ::= WITH RECURSIVE wqlist selectnowith */ + case 82: /* select ::= WITH RECURSIVE wqlist selectnowith */ { - Select *p = yymsp[0].minor.yy423; + Select *p = yymsp[0].minor.yy25; if( p ){ - p->pWith = yymsp[-1].minor.yy499; + p->pWith = yymsp[-1].minor.yy297; parserDoubleLinkSelect(pParse, p); }else{ - sqlite3WithDelete(pParse->db, yymsp[-1].minor.yy499); + sqlite3WithDelete(pParse->db, yymsp[-1].minor.yy297); } - yymsp[-3].minor.yy423 = p; + yymsp[-3].minor.yy25 = p; } break; - case 82: /* select ::= selectnowith */ + case 83: /* select ::= selectnowith */ { - Select *p = yymsp[0].minor.yy423; + Select *p = yymsp[0].minor.yy25; if( p ){ parserDoubleLinkSelect(pParse, p); } - yymsp[0].minor.yy423 = p; /*A-overwrites-X*/ + yymsp[0].minor.yy25 = p; /*A-overwrites-X*/ } break; - case 83: /* selectnowith ::= selectnowith multiselect_op oneselect */ + case 84: /* selectnowith ::= selectnowith multiselect_op oneselect */ { - Select *pRhs = yymsp[0].minor.yy423; - Select *pLhs = yymsp[-2].minor.yy423; + Select *pRhs = yymsp[0].minor.yy25; + Select *pLhs = yymsp[-2].minor.yy25; if( pRhs && pRhs->pPrior ){ SrcList *pFrom; Token x; @@ -150745,142 +153254,142 @@ static YYACTIONTYPE yy_reduce( pRhs = sqlite3SelectNew(pParse,0,pFrom,0,0,0,0,0,0); } if( pRhs ){ - pRhs->op = (u8)yymsp[-1].minor.yy96; + pRhs->op = (u8)yymsp[-1].minor.yy32; pRhs->pPrior = pLhs; if( ALWAYS(pLhs) ) pLhs->selFlags &= ~SF_MultiValue; pRhs->selFlags &= ~SF_MultiValue; - if( yymsp[-1].minor.yy96!=TK_ALL ) pParse->hasCompound = 1; + if( yymsp[-1].minor.yy32!=TK_ALL ) pParse->hasCompound = 1; }else{ sqlite3SelectDelete(pParse->db, pLhs); } - yymsp[-2].minor.yy423 = pRhs; + yymsp[-2].minor.yy25 = pRhs; } break; - case 84: /* multiselect_op ::= UNION */ - case 86: /* multiselect_op ::= EXCEPT|INTERSECT */ yytestcase(yyruleno==86); -{yymsp[0].minor.yy96 = yymsp[0].major; /*A-overwrites-OP*/} + case 85: /* multiselect_op ::= UNION */ + case 87: /* multiselect_op ::= EXCEPT|INTERSECT */ yytestcase(yyruleno==87); +{yymsp[0].minor.yy32 = yymsp[0].major; /*A-overwrites-OP*/} break; - case 85: /* multiselect_op ::= UNION ALL */ -{yymsp[-1].minor.yy96 = TK_ALL;} + case 86: /* multiselect_op ::= UNION ALL */ +{yymsp[-1].minor.yy32 = TK_ALL;} break; - case 87: /* oneselect ::= SELECT distinct selcollist from where_opt groupby_opt having_opt orderby_opt limit_opt */ + case 88: /* oneselect ::= SELECT distinct selcollist from where_opt groupby_opt having_opt orderby_opt limit_opt */ { - yymsp[-8].minor.yy423 = sqlite3SelectNew(pParse,yymsp[-6].minor.yy42,yymsp[-5].minor.yy167,yymsp[-4].minor.yy490,yymsp[-3].minor.yy42,yymsp[-2].minor.yy490,yymsp[-1].minor.yy42,yymsp[-7].minor.yy96,yymsp[0].minor.yy490); + yymsp[-8].minor.yy25 = sqlite3SelectNew(pParse,yymsp[-6].minor.yy138,yymsp[-5].minor.yy609,yymsp[-4].minor.yy46,yymsp[-3].minor.yy138,yymsp[-2].minor.yy46,yymsp[-1].minor.yy138,yymsp[-7].minor.yy32,yymsp[0].minor.yy46); } break; - case 88: /* oneselect ::= SELECT distinct selcollist from where_opt groupby_opt having_opt window_clause orderby_opt limit_opt */ + case 89: /* oneselect ::= SELECT distinct selcollist from where_opt groupby_opt having_opt window_clause orderby_opt limit_opt */ { - yymsp[-9].minor.yy423 = sqlite3SelectNew(pParse,yymsp[-7].minor.yy42,yymsp[-6].minor.yy167,yymsp[-5].minor.yy490,yymsp[-4].minor.yy42,yymsp[-3].minor.yy490,yymsp[-1].minor.yy42,yymsp[-8].minor.yy96,yymsp[0].minor.yy490); - if( yymsp[-9].minor.yy423 ){ - yymsp[-9].minor.yy423->pWinDefn = yymsp[-2].minor.yy147; + yymsp[-9].minor.yy25 = sqlite3SelectNew(pParse,yymsp[-7].minor.yy138,yymsp[-6].minor.yy609,yymsp[-5].minor.yy46,yymsp[-4].minor.yy138,yymsp[-3].minor.yy46,yymsp[-1].minor.yy138,yymsp[-8].minor.yy32,yymsp[0].minor.yy46); + if( yymsp[-9].minor.yy25 ){ + yymsp[-9].minor.yy25->pWinDefn = yymsp[-2].minor.yy455; }else{ - sqlite3WindowListDelete(pParse->db, yymsp[-2].minor.yy147); + sqlite3WindowListDelete(pParse->db, yymsp[-2].minor.yy455); } } break; - case 89: /* values ::= VALUES LP nexprlist RP */ + case 90: /* values ::= VALUES LP nexprlist RP */ { - yymsp[-3].minor.yy423 = sqlite3SelectNew(pParse,yymsp[-1].minor.yy42,0,0,0,0,0,SF_Values,0); + yymsp[-3].minor.yy25 = sqlite3SelectNew(pParse,yymsp[-1].minor.yy138,0,0,0,0,0,SF_Values,0); } break; - case 90: /* values ::= values COMMA LP nexprlist RP */ + case 91: /* values ::= values COMMA LP nexprlist RP */ { - Select *pRight, *pLeft = yymsp[-4].minor.yy423; - pRight = sqlite3SelectNew(pParse,yymsp[-1].minor.yy42,0,0,0,0,0,SF_Values|SF_MultiValue,0); + Select *pRight, *pLeft = yymsp[-4].minor.yy25; + pRight = sqlite3SelectNew(pParse,yymsp[-1].minor.yy138,0,0,0,0,0,SF_Values|SF_MultiValue,0); if( ALWAYS(pLeft) ) pLeft->selFlags &= ~SF_MultiValue; if( pRight ){ pRight->op = TK_ALL; pRight->pPrior = pLeft; - yymsp[-4].minor.yy423 = pRight; + yymsp[-4].minor.yy25 = pRight; }else{ - yymsp[-4].minor.yy423 = pLeft; + yymsp[-4].minor.yy25 = pLeft; } } break; - case 91: /* distinct ::= DISTINCT */ -{yymsp[0].minor.yy96 = SF_Distinct;} + case 92: /* distinct ::= DISTINCT */ +{yymsp[0].minor.yy32 = SF_Distinct;} break; - case 92: /* distinct ::= ALL */ -{yymsp[0].minor.yy96 = SF_All;} + case 93: /* distinct ::= ALL */ +{yymsp[0].minor.yy32 = SF_All;} break; - case 94: /* sclp ::= */ - case 127: /* orderby_opt ::= */ yytestcase(yyruleno==127); - case 134: /* groupby_opt ::= */ yytestcase(yyruleno==134); - case 214: /* exprlist ::= */ yytestcase(yyruleno==214); - case 217: /* paren_exprlist ::= */ yytestcase(yyruleno==217); - case 222: /* eidlist_opt ::= */ yytestcase(yyruleno==222); -{yymsp[1].minor.yy42 = 0;} + case 95: /* sclp ::= */ + case 128: /* orderby_opt ::= */ yytestcase(yyruleno==128); + case 138: /* groupby_opt ::= */ yytestcase(yyruleno==138); + case 218: /* exprlist ::= */ yytestcase(yyruleno==218); + case 221: /* paren_exprlist ::= */ yytestcase(yyruleno==221); + case 226: /* eidlist_opt ::= */ yytestcase(yyruleno==226); +{yymsp[1].minor.yy138 = 0;} break; - case 95: /* selcollist ::= sclp scanpt expr scanpt as */ + case 96: /* selcollist ::= sclp scanpt expr scanpt as */ { - yymsp[-4].minor.yy42 = sqlite3ExprListAppend(pParse, yymsp[-4].minor.yy42, yymsp[-2].minor.yy490); - if( yymsp[0].minor.yy0.n>0 ) sqlite3ExprListSetName(pParse, yymsp[-4].minor.yy42, &yymsp[0].minor.yy0, 1); - sqlite3ExprListSetSpan(pParse,yymsp[-4].minor.yy42,yymsp[-3].minor.yy464,yymsp[-1].minor.yy464); + yymsp[-4].minor.yy138 = sqlite3ExprListAppend(pParse, yymsp[-4].minor.yy138, yymsp[-2].minor.yy46); + if( yymsp[0].minor.yy0.n>0 ) sqlite3ExprListSetName(pParse, yymsp[-4].minor.yy138, &yymsp[0].minor.yy0, 1); + sqlite3ExprListSetSpan(pParse,yymsp[-4].minor.yy138,yymsp[-3].minor.yy8,yymsp[-1].minor.yy8); } break; - case 96: /* selcollist ::= sclp scanpt STAR */ + case 97: /* selcollist ::= sclp scanpt STAR */ { Expr *p = sqlite3Expr(pParse->db, TK_ASTERISK, 0); - yymsp[-2].minor.yy42 = sqlite3ExprListAppend(pParse, yymsp[-2].minor.yy42, p); + yymsp[-2].minor.yy138 = sqlite3ExprListAppend(pParse, yymsp[-2].minor.yy138, p); } break; - case 97: /* selcollist ::= sclp scanpt nm DOT STAR */ + case 98: /* selcollist ::= sclp scanpt nm DOT STAR */ { Expr *pRight = sqlite3PExpr(pParse, TK_ASTERISK, 0, 0); Expr *pLeft = sqlite3ExprAlloc(pParse->db, TK_ID, &yymsp[-2].minor.yy0, 1); Expr *pDot = sqlite3PExpr(pParse, TK_DOT, pLeft, pRight); - yymsp[-4].minor.yy42 = sqlite3ExprListAppend(pParse,yymsp[-4].minor.yy42, pDot); + yymsp[-4].minor.yy138 = sqlite3ExprListAppend(pParse,yymsp[-4].minor.yy138, pDot); } break; - case 98: /* as ::= AS nm */ - case 109: /* dbnm ::= DOT nm */ yytestcase(yyruleno==109); - case 238: /* plus_num ::= PLUS INTEGER|FLOAT */ yytestcase(yyruleno==238); - case 239: /* minus_num ::= MINUS INTEGER|FLOAT */ yytestcase(yyruleno==239); + case 99: /* as ::= AS nm */ + case 110: /* dbnm ::= DOT nm */ yytestcase(yyruleno==110); + case 242: /* plus_num ::= PLUS INTEGER|FLOAT */ yytestcase(yyruleno==242); + case 243: /* minus_num ::= MINUS INTEGER|FLOAT */ yytestcase(yyruleno==243); {yymsp[-1].minor.yy0 = yymsp[0].minor.yy0;} break; - case 100: /* from ::= */ -{yymsp[1].minor.yy167 = sqlite3DbMallocZero(pParse->db, sizeof(*yymsp[1].minor.yy167));} + case 101: /* from ::= */ +{yymsp[1].minor.yy609 = sqlite3DbMallocZero(pParse->db, sizeof(*yymsp[1].minor.yy609));} break; - case 101: /* from ::= FROM seltablist */ + case 102: /* from ::= FROM seltablist */ { - yymsp[-1].minor.yy167 = yymsp[0].minor.yy167; - sqlite3SrcListShiftJoinType(yymsp[-1].minor.yy167); + yymsp[-1].minor.yy609 = yymsp[0].minor.yy609; + sqlite3SrcListShiftJoinType(yymsp[-1].minor.yy609); } break; - case 102: /* stl_prefix ::= seltablist joinop */ + case 103: /* stl_prefix ::= seltablist joinop */ { - if( ALWAYS(yymsp[-1].minor.yy167 && yymsp[-1].minor.yy167->nSrc>0) ) yymsp[-1].minor.yy167->a[yymsp[-1].minor.yy167->nSrc-1].fg.jointype = (u8)yymsp[0].minor.yy96; + if( ALWAYS(yymsp[-1].minor.yy609 && yymsp[-1].minor.yy609->nSrc>0) ) yymsp[-1].minor.yy609->a[yymsp[-1].minor.yy609->nSrc-1].fg.jointype = (u8)yymsp[0].minor.yy32; } break; - case 103: /* stl_prefix ::= */ -{yymsp[1].minor.yy167 = 0;} + case 104: /* stl_prefix ::= */ +{yymsp[1].minor.yy609 = 0;} break; - case 104: /* seltablist ::= stl_prefix nm dbnm as indexed_opt on_opt using_opt */ + case 105: /* seltablist ::= stl_prefix nm dbnm as indexed_opt on_opt using_opt */ { - yymsp[-6].minor.yy167 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-6].minor.yy167,&yymsp[-5].minor.yy0,&yymsp[-4].minor.yy0,&yymsp[-3].minor.yy0,0,yymsp[-1].minor.yy490,yymsp[0].minor.yy336); - sqlite3SrcListIndexedBy(pParse, yymsp[-6].minor.yy167, &yymsp[-2].minor.yy0); + yymsp[-6].minor.yy609 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-6].minor.yy609,&yymsp[-5].minor.yy0,&yymsp[-4].minor.yy0,&yymsp[-3].minor.yy0,0,yymsp[-1].minor.yy46,yymsp[0].minor.yy406); + sqlite3SrcListIndexedBy(pParse, yymsp[-6].minor.yy609, &yymsp[-2].minor.yy0); } break; - case 105: /* seltablist ::= stl_prefix nm dbnm LP exprlist RP as on_opt using_opt */ + case 106: /* seltablist ::= stl_prefix nm dbnm LP exprlist RP as on_opt using_opt */ { - yymsp[-8].minor.yy167 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-8].minor.yy167,&yymsp[-7].minor.yy0,&yymsp[-6].minor.yy0,&yymsp[-2].minor.yy0,0,yymsp[-1].minor.yy490,yymsp[0].minor.yy336); - sqlite3SrcListFuncArgs(pParse, yymsp[-8].minor.yy167, yymsp[-4].minor.yy42); + yymsp[-8].minor.yy609 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-8].minor.yy609,&yymsp[-7].minor.yy0,&yymsp[-6].minor.yy0,&yymsp[-2].minor.yy0,0,yymsp[-1].minor.yy46,yymsp[0].minor.yy406); + sqlite3SrcListFuncArgs(pParse, yymsp[-8].minor.yy609, yymsp[-4].minor.yy138); } break; - case 106: /* seltablist ::= stl_prefix LP select RP as on_opt using_opt */ + case 107: /* seltablist ::= stl_prefix LP select RP as on_opt using_opt */ { - yymsp[-6].minor.yy167 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-6].minor.yy167,0,0,&yymsp[-2].minor.yy0,yymsp[-4].minor.yy423,yymsp[-1].minor.yy490,yymsp[0].minor.yy336); + yymsp[-6].minor.yy609 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-6].minor.yy609,0,0,&yymsp[-2].minor.yy0,yymsp[-4].minor.yy25,yymsp[-1].minor.yy46,yymsp[0].minor.yy406); } break; - case 107: /* seltablist ::= stl_prefix LP seltablist RP as on_opt using_opt */ + case 108: /* seltablist ::= stl_prefix LP seltablist RP as on_opt using_opt */ { - if( yymsp[-6].minor.yy167==0 && yymsp[-2].minor.yy0.n==0 && yymsp[-1].minor.yy490==0 && yymsp[0].minor.yy336==0 ){ - yymsp[-6].minor.yy167 = yymsp[-4].minor.yy167; - }else if( yymsp[-4].minor.yy167->nSrc==1 ){ - yymsp[-6].minor.yy167 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-6].minor.yy167,0,0,&yymsp[-2].minor.yy0,0,yymsp[-1].minor.yy490,yymsp[0].minor.yy336); - if( yymsp[-6].minor.yy167 ){ - struct SrcList_item *pNew = &yymsp[-6].minor.yy167->a[yymsp[-6].minor.yy167->nSrc-1]; - struct SrcList_item *pOld = yymsp[-4].minor.yy167->a; + if( yymsp[-6].minor.yy609==0 && yymsp[-2].minor.yy0.n==0 && yymsp[-1].minor.yy46==0 && yymsp[0].minor.yy406==0 ){ + yymsp[-6].minor.yy609 = yymsp[-4].minor.yy609; + }else if( yymsp[-4].minor.yy609->nSrc==1 ){ + yymsp[-6].minor.yy609 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-6].minor.yy609,0,0,&yymsp[-2].minor.yy0,0,yymsp[-1].minor.yy46,yymsp[0].minor.yy406); + if( yymsp[-6].minor.yy609 ){ + struct SrcList_item *pNew = &yymsp[-6].minor.yy609->a[yymsp[-6].minor.yy609->nSrc-1]; + struct SrcList_item *pOld = yymsp[-4].minor.yy609->a; pNew->zName = pOld->zName; pNew->zDatabase = pOld->zDatabase; pNew->pSelect = pOld->pSelect; @@ -150893,201 +153402,208 @@ static YYACTIONTYPE yy_reduce( pOld->zName = pOld->zDatabase = 0; pOld->pSelect = 0; } - sqlite3SrcListDelete(pParse->db, yymsp[-4].minor.yy167); + sqlite3SrcListDelete(pParse->db, yymsp[-4].minor.yy609); }else{ Select *pSubquery; - sqlite3SrcListShiftJoinType(yymsp[-4].minor.yy167); - pSubquery = sqlite3SelectNew(pParse,0,yymsp[-4].minor.yy167,0,0,0,0,SF_NestedFrom,0); - yymsp[-6].minor.yy167 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-6].minor.yy167,0,0,&yymsp[-2].minor.yy0,pSubquery,yymsp[-1].minor.yy490,yymsp[0].minor.yy336); + sqlite3SrcListShiftJoinType(yymsp[-4].minor.yy609); + pSubquery = sqlite3SelectNew(pParse,0,yymsp[-4].minor.yy609,0,0,0,0,SF_NestedFrom,0); + yymsp[-6].minor.yy609 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-6].minor.yy609,0,0,&yymsp[-2].minor.yy0,pSubquery,yymsp[-1].minor.yy46,yymsp[0].minor.yy406); } } break; - case 108: /* dbnm ::= */ - case 122: /* indexed_opt ::= */ yytestcase(yyruleno==122); + case 109: /* dbnm ::= */ + case 123: /* indexed_opt ::= */ yytestcase(yyruleno==123); {yymsp[1].minor.yy0.z=0; yymsp[1].minor.yy0.n=0;} break; - case 110: /* fullname ::= nm */ + case 111: /* fullname ::= nm */ { - yylhsminor.yy167 = sqlite3SrcListAppend(pParse,0,&yymsp[0].minor.yy0,0); - if( IN_RENAME_OBJECT && yylhsminor.yy167 ) sqlite3RenameTokenMap(pParse, yylhsminor.yy167->a[0].zName, &yymsp[0].minor.yy0); + yylhsminor.yy609 = sqlite3SrcListAppend(pParse,0,&yymsp[0].minor.yy0,0); + if( IN_RENAME_OBJECT && yylhsminor.yy609 ) sqlite3RenameTokenMap(pParse, yylhsminor.yy609->a[0].zName, &yymsp[0].minor.yy0); } - yymsp[0].minor.yy167 = yylhsminor.yy167; + yymsp[0].minor.yy609 = yylhsminor.yy609; break; - case 111: /* fullname ::= nm DOT nm */ + case 112: /* fullname ::= nm DOT nm */ { - yylhsminor.yy167 = sqlite3SrcListAppend(pParse,0,&yymsp[-2].minor.yy0,&yymsp[0].minor.yy0); - if( IN_RENAME_OBJECT && yylhsminor.yy167 ) sqlite3RenameTokenMap(pParse, yylhsminor.yy167->a[0].zName, &yymsp[0].minor.yy0); + yylhsminor.yy609 = sqlite3SrcListAppend(pParse,0,&yymsp[-2].minor.yy0,&yymsp[0].minor.yy0); + if( IN_RENAME_OBJECT && yylhsminor.yy609 ) sqlite3RenameTokenMap(pParse, yylhsminor.yy609->a[0].zName, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy167 = yylhsminor.yy167; + yymsp[-2].minor.yy609 = yylhsminor.yy609; break; - case 112: /* xfullname ::= nm */ -{yymsp[0].minor.yy167 = sqlite3SrcListAppend(pParse,0,&yymsp[0].minor.yy0,0); /*A-overwrites-X*/} + case 113: /* xfullname ::= nm */ +{yymsp[0].minor.yy609 = sqlite3SrcListAppend(pParse,0,&yymsp[0].minor.yy0,0); /*A-overwrites-X*/} break; - case 113: /* xfullname ::= nm DOT nm */ -{yymsp[-2].minor.yy167 = sqlite3SrcListAppend(pParse,0,&yymsp[-2].minor.yy0,&yymsp[0].minor.yy0); /*A-overwrites-X*/} + case 114: /* xfullname ::= nm DOT nm */ +{yymsp[-2].minor.yy609 = sqlite3SrcListAppend(pParse,0,&yymsp[-2].minor.yy0,&yymsp[0].minor.yy0); /*A-overwrites-X*/} break; - case 114: /* xfullname ::= nm DOT nm AS nm */ + case 115: /* xfullname ::= nm DOT nm AS nm */ { - yymsp[-4].minor.yy167 = sqlite3SrcListAppend(pParse,0,&yymsp[-4].minor.yy0,&yymsp[-2].minor.yy0); /*A-overwrites-X*/ - if( yymsp[-4].minor.yy167 ) yymsp[-4].minor.yy167->a[0].zAlias = sqlite3NameFromToken(pParse->db, &yymsp[0].minor.yy0); + yymsp[-4].minor.yy609 = sqlite3SrcListAppend(pParse,0,&yymsp[-4].minor.yy0,&yymsp[-2].minor.yy0); /*A-overwrites-X*/ + if( yymsp[-4].minor.yy609 ) yymsp[-4].minor.yy609->a[0].zAlias = sqlite3NameFromToken(pParse->db, &yymsp[0].minor.yy0); } break; - case 115: /* xfullname ::= nm AS nm */ + case 116: /* xfullname ::= nm AS nm */ { - yymsp[-2].minor.yy167 = sqlite3SrcListAppend(pParse,0,&yymsp[-2].minor.yy0,0); /*A-overwrites-X*/ - if( yymsp[-2].minor.yy167 ) yymsp[-2].minor.yy167->a[0].zAlias = sqlite3NameFromToken(pParse->db, &yymsp[0].minor.yy0); + yymsp[-2].minor.yy609 = sqlite3SrcListAppend(pParse,0,&yymsp[-2].minor.yy0,0); /*A-overwrites-X*/ + if( yymsp[-2].minor.yy609 ) yymsp[-2].minor.yy609->a[0].zAlias = sqlite3NameFromToken(pParse->db, &yymsp[0].minor.yy0); } break; - case 116: /* joinop ::= COMMA|JOIN */ -{ yymsp[0].minor.yy96 = JT_INNER; } + case 117: /* joinop ::= COMMA|JOIN */ +{ yymsp[0].minor.yy32 = JT_INNER; } break; - case 117: /* joinop ::= JOIN_KW JOIN */ -{yymsp[-1].minor.yy96 = sqlite3JoinType(pParse,&yymsp[-1].minor.yy0,0,0); /*X-overwrites-A*/} + case 118: /* joinop ::= JOIN_KW JOIN */ +{yymsp[-1].minor.yy32 = sqlite3JoinType(pParse,&yymsp[-1].minor.yy0,0,0); /*X-overwrites-A*/} break; - case 118: /* joinop ::= JOIN_KW nm JOIN */ -{yymsp[-2].minor.yy96 = sqlite3JoinType(pParse,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy0,0); /*X-overwrites-A*/} + case 119: /* joinop ::= JOIN_KW nm JOIN */ +{yymsp[-2].minor.yy32 = sqlite3JoinType(pParse,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy0,0); /*X-overwrites-A*/} break; - case 119: /* joinop ::= JOIN_KW nm nm JOIN */ -{yymsp[-3].minor.yy96 = sqlite3JoinType(pParse,&yymsp[-3].minor.yy0,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy0);/*X-overwrites-A*/} + case 120: /* joinop ::= JOIN_KW nm nm JOIN */ +{yymsp[-3].minor.yy32 = sqlite3JoinType(pParse,&yymsp[-3].minor.yy0,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy0);/*X-overwrites-A*/} break; - case 120: /* on_opt ::= ON expr */ - case 137: /* having_opt ::= HAVING expr */ yytestcase(yyruleno==137); - case 144: /* where_opt ::= WHERE expr */ yytestcase(yyruleno==144); - case 210: /* case_else ::= ELSE expr */ yytestcase(yyruleno==210); - case 231: /* vinto ::= INTO expr */ yytestcase(yyruleno==231); -{yymsp[-1].minor.yy490 = yymsp[0].minor.yy490;} + case 121: /* on_opt ::= ON expr */ + case 141: /* having_opt ::= HAVING expr */ yytestcase(yyruleno==141); + case 148: /* where_opt ::= WHERE expr */ yytestcase(yyruleno==148); + case 214: /* case_else ::= ELSE expr */ yytestcase(yyruleno==214); + case 235: /* vinto ::= INTO expr */ yytestcase(yyruleno==235); +{yymsp[-1].minor.yy46 = yymsp[0].minor.yy46;} break; - case 121: /* on_opt ::= */ - case 136: /* having_opt ::= */ yytestcase(yyruleno==136); - case 138: /* limit_opt ::= */ yytestcase(yyruleno==138); - case 143: /* where_opt ::= */ yytestcase(yyruleno==143); - case 211: /* case_else ::= */ yytestcase(yyruleno==211); - case 213: /* case_operand ::= */ yytestcase(yyruleno==213); - case 232: /* vinto ::= */ yytestcase(yyruleno==232); -{yymsp[1].minor.yy490 = 0;} + case 122: /* on_opt ::= */ + case 140: /* having_opt ::= */ yytestcase(yyruleno==140); + case 142: /* limit_opt ::= */ yytestcase(yyruleno==142); + case 147: /* where_opt ::= */ yytestcase(yyruleno==147); + case 215: /* case_else ::= */ yytestcase(yyruleno==215); + case 217: /* case_operand ::= */ yytestcase(yyruleno==217); + case 236: /* vinto ::= */ yytestcase(yyruleno==236); +{yymsp[1].minor.yy46 = 0;} break; - case 123: /* indexed_opt ::= INDEXED BY nm */ + case 124: /* indexed_opt ::= INDEXED BY nm */ {yymsp[-2].minor.yy0 = yymsp[0].minor.yy0;} break; - case 124: /* indexed_opt ::= NOT INDEXED */ + case 125: /* indexed_opt ::= NOT INDEXED */ {yymsp[-1].minor.yy0.z=0; yymsp[-1].minor.yy0.n=1;} break; - case 125: /* using_opt ::= USING LP idlist RP */ -{yymsp[-3].minor.yy336 = yymsp[-1].minor.yy336;} + case 126: /* using_opt ::= USING LP idlist RP */ +{yymsp[-3].minor.yy406 = yymsp[-1].minor.yy406;} break; - case 126: /* using_opt ::= */ - case 158: /* idlist_opt ::= */ yytestcase(yyruleno==158); -{yymsp[1].minor.yy336 = 0;} + case 127: /* using_opt ::= */ + case 162: /* idlist_opt ::= */ yytestcase(yyruleno==162); +{yymsp[1].minor.yy406 = 0;} break; - case 128: /* orderby_opt ::= ORDER BY sortlist */ - case 135: /* groupby_opt ::= GROUP BY nexprlist */ yytestcase(yyruleno==135); -{yymsp[-2].minor.yy42 = yymsp[0].minor.yy42;} + case 129: /* orderby_opt ::= ORDER BY sortlist */ + case 139: /* groupby_opt ::= GROUP BY nexprlist */ yytestcase(yyruleno==139); +{yymsp[-2].minor.yy138 = yymsp[0].minor.yy138;} break; - case 129: /* sortlist ::= sortlist COMMA expr sortorder */ + case 130: /* sortlist ::= sortlist COMMA expr sortorder nulls */ { - yymsp[-3].minor.yy42 = sqlite3ExprListAppend(pParse,yymsp[-3].minor.yy42,yymsp[-1].minor.yy490); - sqlite3ExprListSetSortOrder(yymsp[-3].minor.yy42,yymsp[0].minor.yy96); + yymsp[-4].minor.yy138 = sqlite3ExprListAppend(pParse,yymsp[-4].minor.yy138,yymsp[-2].minor.yy46); + sqlite3ExprListSetSortOrder(yymsp[-4].minor.yy138,yymsp[-1].minor.yy32,yymsp[0].minor.yy32); } break; - case 130: /* sortlist ::= expr sortorder */ + case 131: /* sortlist ::= expr sortorder nulls */ { - yymsp[-1].minor.yy42 = sqlite3ExprListAppend(pParse,0,yymsp[-1].minor.yy490); /*A-overwrites-Y*/ - sqlite3ExprListSetSortOrder(yymsp[-1].minor.yy42,yymsp[0].minor.yy96); + yymsp[-2].minor.yy138 = sqlite3ExprListAppend(pParse,0,yymsp[-2].minor.yy46); /*A-overwrites-Y*/ + sqlite3ExprListSetSortOrder(yymsp[-2].minor.yy138,yymsp[-1].minor.yy32,yymsp[0].minor.yy32); } break; - case 131: /* sortorder ::= ASC */ -{yymsp[0].minor.yy96 = SQLITE_SO_ASC;} + case 132: /* sortorder ::= ASC */ +{yymsp[0].minor.yy32 = SQLITE_SO_ASC;} + break; + case 133: /* sortorder ::= DESC */ +{yymsp[0].minor.yy32 = SQLITE_SO_DESC;} break; - case 132: /* sortorder ::= DESC */ -{yymsp[0].minor.yy96 = SQLITE_SO_DESC;} + case 134: /* sortorder ::= */ + case 137: /* nulls ::= */ yytestcase(yyruleno==137); +{yymsp[1].minor.yy32 = SQLITE_SO_UNDEFINED;} break; - case 133: /* sortorder ::= */ -{yymsp[1].minor.yy96 = SQLITE_SO_UNDEFINED;} + case 135: /* nulls ::= NULLS FIRST */ +{yymsp[-1].minor.yy32 = SQLITE_SO_ASC;} break; - case 139: /* limit_opt ::= LIMIT expr */ -{yymsp[-1].minor.yy490 = sqlite3PExpr(pParse,TK_LIMIT,yymsp[0].minor.yy490,0);} + case 136: /* nulls ::= NULLS LAST */ +{yymsp[-1].minor.yy32 = SQLITE_SO_DESC;} break; - case 140: /* limit_opt ::= LIMIT expr OFFSET expr */ -{yymsp[-3].minor.yy490 = sqlite3PExpr(pParse,TK_LIMIT,yymsp[-2].minor.yy490,yymsp[0].minor.yy490);} + case 143: /* limit_opt ::= LIMIT expr */ +{yymsp[-1].minor.yy46 = sqlite3PExpr(pParse,TK_LIMIT,yymsp[0].minor.yy46,0);} break; - case 141: /* limit_opt ::= LIMIT expr COMMA expr */ -{yymsp[-3].minor.yy490 = sqlite3PExpr(pParse,TK_LIMIT,yymsp[0].minor.yy490,yymsp[-2].minor.yy490);} + case 144: /* limit_opt ::= LIMIT expr OFFSET expr */ +{yymsp[-3].minor.yy46 = sqlite3PExpr(pParse,TK_LIMIT,yymsp[-2].minor.yy46,yymsp[0].minor.yy46);} break; - case 142: /* cmd ::= with DELETE FROM xfullname indexed_opt where_opt */ + case 145: /* limit_opt ::= LIMIT expr COMMA expr */ +{yymsp[-3].minor.yy46 = sqlite3PExpr(pParse,TK_LIMIT,yymsp[0].minor.yy46,yymsp[-2].minor.yy46);} + break; + case 146: /* cmd ::= with DELETE FROM xfullname indexed_opt where_opt */ { - sqlite3SrcListIndexedBy(pParse, yymsp[-2].minor.yy167, &yymsp[-1].minor.yy0); - sqlite3DeleteFrom(pParse,yymsp[-2].minor.yy167,yymsp[0].minor.yy490,0,0); + sqlite3SrcListIndexedBy(pParse, yymsp[-2].minor.yy609, &yymsp[-1].minor.yy0); + sqlite3DeleteFrom(pParse,yymsp[-2].minor.yy609,yymsp[0].minor.yy46,0,0); } break; - case 145: /* cmd ::= with UPDATE orconf xfullname indexed_opt SET setlist where_opt */ + case 149: /* cmd ::= with UPDATE orconf xfullname indexed_opt SET setlist where_opt */ { - sqlite3SrcListIndexedBy(pParse, yymsp[-4].minor.yy167, &yymsp[-3].minor.yy0); - sqlite3ExprListCheckLength(pParse,yymsp[-1].minor.yy42,"set list"); - sqlite3Update(pParse,yymsp[-4].minor.yy167,yymsp[-1].minor.yy42,yymsp[0].minor.yy490,yymsp[-5].minor.yy96,0,0,0); + sqlite3SrcListIndexedBy(pParse, yymsp[-4].minor.yy609, &yymsp[-3].minor.yy0); + sqlite3ExprListCheckLength(pParse,yymsp[-1].minor.yy138,"set list"); + sqlite3Update(pParse,yymsp[-4].minor.yy609,yymsp[-1].minor.yy138,yymsp[0].minor.yy46,yymsp[-5].minor.yy32,0,0,0); } break; - case 146: /* setlist ::= setlist COMMA nm EQ expr */ + case 150: /* setlist ::= setlist COMMA nm EQ expr */ { - yymsp[-4].minor.yy42 = sqlite3ExprListAppend(pParse, yymsp[-4].minor.yy42, yymsp[0].minor.yy490); - sqlite3ExprListSetName(pParse, yymsp[-4].minor.yy42, &yymsp[-2].minor.yy0, 1); + yymsp[-4].minor.yy138 = sqlite3ExprListAppend(pParse, yymsp[-4].minor.yy138, yymsp[0].minor.yy46); + sqlite3ExprListSetName(pParse, yymsp[-4].minor.yy138, &yymsp[-2].minor.yy0, 1); } break; - case 147: /* setlist ::= setlist COMMA LP idlist RP EQ expr */ + case 151: /* setlist ::= setlist COMMA LP idlist RP EQ expr */ { - yymsp[-6].minor.yy42 = sqlite3ExprListAppendVector(pParse, yymsp[-6].minor.yy42, yymsp[-3].minor.yy336, yymsp[0].minor.yy490); + yymsp[-6].minor.yy138 = sqlite3ExprListAppendVector(pParse, yymsp[-6].minor.yy138, yymsp[-3].minor.yy406, yymsp[0].minor.yy46); } break; - case 148: /* setlist ::= nm EQ expr */ + case 152: /* setlist ::= nm EQ expr */ { - yylhsminor.yy42 = sqlite3ExprListAppend(pParse, 0, yymsp[0].minor.yy490); - sqlite3ExprListSetName(pParse, yylhsminor.yy42, &yymsp[-2].minor.yy0, 1); + yylhsminor.yy138 = sqlite3ExprListAppend(pParse, 0, yymsp[0].minor.yy46); + sqlite3ExprListSetName(pParse, yylhsminor.yy138, &yymsp[-2].minor.yy0, 1); } - yymsp[-2].minor.yy42 = yylhsminor.yy42; + yymsp[-2].minor.yy138 = yylhsminor.yy138; break; - case 149: /* setlist ::= LP idlist RP EQ expr */ + case 153: /* setlist ::= LP idlist RP EQ expr */ { - yymsp[-4].minor.yy42 = sqlite3ExprListAppendVector(pParse, 0, yymsp[-3].minor.yy336, yymsp[0].minor.yy490); + yymsp[-4].minor.yy138 = sqlite3ExprListAppendVector(pParse, 0, yymsp[-3].minor.yy406, yymsp[0].minor.yy46); } break; - case 150: /* cmd ::= with insert_cmd INTO xfullname idlist_opt select upsert */ + case 154: /* cmd ::= with insert_cmd INTO xfullname idlist_opt select upsert */ { - sqlite3Insert(pParse, yymsp[-3].minor.yy167, yymsp[-1].minor.yy423, yymsp[-2].minor.yy336, yymsp[-5].minor.yy96, yymsp[0].minor.yy266); + sqlite3Insert(pParse, yymsp[-3].minor.yy609, yymsp[-1].minor.yy25, yymsp[-2].minor.yy406, yymsp[-5].minor.yy32, yymsp[0].minor.yy288); } break; - case 151: /* cmd ::= with insert_cmd INTO xfullname idlist_opt DEFAULT VALUES */ + case 155: /* cmd ::= with insert_cmd INTO xfullname idlist_opt DEFAULT VALUES */ { - sqlite3Insert(pParse, yymsp[-3].minor.yy167, 0, yymsp[-2].minor.yy336, yymsp[-5].minor.yy96, 0); + sqlite3Insert(pParse, yymsp[-3].minor.yy609, 0, yymsp[-2].minor.yy406, yymsp[-5].minor.yy32, 0); } break; - case 152: /* upsert ::= */ -{ yymsp[1].minor.yy266 = 0; } + case 156: /* upsert ::= */ +{ yymsp[1].minor.yy288 = 0; } break; - case 153: /* upsert ::= ON CONFLICT LP sortlist RP where_opt DO UPDATE SET setlist where_opt */ -{ yymsp[-10].minor.yy266 = sqlite3UpsertNew(pParse->db,yymsp[-7].minor.yy42,yymsp[-5].minor.yy490,yymsp[-1].minor.yy42,yymsp[0].minor.yy490);} + case 157: /* upsert ::= ON CONFLICT LP sortlist RP where_opt DO UPDATE SET setlist where_opt */ +{ yymsp[-10].minor.yy288 = sqlite3UpsertNew(pParse->db,yymsp[-7].minor.yy138,yymsp[-5].minor.yy46,yymsp[-1].minor.yy138,yymsp[0].minor.yy46);} break; - case 154: /* upsert ::= ON CONFLICT LP sortlist RP where_opt DO NOTHING */ -{ yymsp[-7].minor.yy266 = sqlite3UpsertNew(pParse->db,yymsp[-4].minor.yy42,yymsp[-2].minor.yy490,0,0); } + case 158: /* upsert ::= ON CONFLICT LP sortlist RP where_opt DO NOTHING */ +{ yymsp[-7].minor.yy288 = sqlite3UpsertNew(pParse->db,yymsp[-4].minor.yy138,yymsp[-2].minor.yy46,0,0); } break; - case 155: /* upsert ::= ON CONFLICT DO NOTHING */ -{ yymsp[-3].minor.yy266 = sqlite3UpsertNew(pParse->db,0,0,0,0); } + case 159: /* upsert ::= ON CONFLICT DO NOTHING */ +{ yymsp[-3].minor.yy288 = sqlite3UpsertNew(pParse->db,0,0,0,0); } break; - case 159: /* idlist_opt ::= LP idlist RP */ -{yymsp[-2].minor.yy336 = yymsp[-1].minor.yy336;} + case 163: /* idlist_opt ::= LP idlist RP */ +{yymsp[-2].minor.yy406 = yymsp[-1].minor.yy406;} break; - case 160: /* idlist ::= idlist COMMA nm */ -{yymsp[-2].minor.yy336 = sqlite3IdListAppend(pParse,yymsp[-2].minor.yy336,&yymsp[0].minor.yy0);} + case 164: /* idlist ::= idlist COMMA nm */ +{yymsp[-2].minor.yy406 = sqlite3IdListAppend(pParse,yymsp[-2].minor.yy406,&yymsp[0].minor.yy0);} break; - case 161: /* idlist ::= nm */ -{yymsp[0].minor.yy336 = sqlite3IdListAppend(pParse,0,&yymsp[0].minor.yy0); /*A-overwrites-Y*/} + case 165: /* idlist ::= nm */ +{yymsp[0].minor.yy406 = sqlite3IdListAppend(pParse,0,&yymsp[0].minor.yy0); /*A-overwrites-Y*/} break; - case 162: /* expr ::= LP expr RP */ -{yymsp[-2].minor.yy490 = yymsp[-1].minor.yy490;} + case 166: /* expr ::= LP expr RP */ +{yymsp[-2].minor.yy46 = yymsp[-1].minor.yy46;} break; - case 163: /* expr ::= ID|INDEXED */ - case 164: /* expr ::= JOIN_KW */ yytestcase(yyruleno==164); -{yymsp[0].minor.yy490=tokenExpr(pParse,TK_ID,yymsp[0].minor.yy0); /*A-overwrites-X*/} + case 167: /* expr ::= ID|INDEXED */ + case 168: /* expr ::= JOIN_KW */ yytestcase(yyruleno==168); +{yymsp[0].minor.yy46=tokenExpr(pParse,TK_ID,yymsp[0].minor.yy0); /*A-overwrites-X*/} break; - case 165: /* expr ::= nm DOT nm */ + case 169: /* expr ::= nm DOT nm */ { Expr *temp1 = sqlite3ExprAlloc(pParse->db, TK_ID, &yymsp[-2].minor.yy0, 1); Expr *temp2 = sqlite3ExprAlloc(pParse->db, TK_ID, &yymsp[0].minor.yy0, 1); @@ -151095,11 +153611,11 @@ static YYACTIONTYPE yy_reduce( sqlite3RenameTokenMap(pParse, (void*)temp2, &yymsp[0].minor.yy0); sqlite3RenameTokenMap(pParse, (void*)temp1, &yymsp[-2].minor.yy0); } - yylhsminor.yy490 = sqlite3PExpr(pParse, TK_DOT, temp1, temp2); + yylhsminor.yy46 = sqlite3PExpr(pParse, TK_DOT, temp1, temp2); } - yymsp[-2].minor.yy490 = yylhsminor.yy490; + yymsp[-2].minor.yy46 = yylhsminor.yy46; break; - case 166: /* expr ::= nm DOT nm DOT nm */ + case 170: /* expr ::= nm DOT nm DOT nm */ { Expr *temp1 = sqlite3ExprAlloc(pParse->db, TK_ID, &yymsp[-4].minor.yy0, 1); Expr *temp2 = sqlite3ExprAlloc(pParse->db, TK_ID, &yymsp[-2].minor.yy0, 1); @@ -151109,26 +153625,26 @@ static YYACTIONTYPE yy_reduce( sqlite3RenameTokenMap(pParse, (void*)temp3, &yymsp[0].minor.yy0); sqlite3RenameTokenMap(pParse, (void*)temp2, &yymsp[-2].minor.yy0); } - yylhsminor.yy490 = sqlite3PExpr(pParse, TK_DOT, temp1, temp4); + yylhsminor.yy46 = sqlite3PExpr(pParse, TK_DOT, temp1, temp4); } - yymsp[-4].minor.yy490 = yylhsminor.yy490; + yymsp[-4].minor.yy46 = yylhsminor.yy46; break; - case 167: /* term ::= NULL|FLOAT|BLOB */ - case 168: /* term ::= STRING */ yytestcase(yyruleno==168); -{yymsp[0].minor.yy490=tokenExpr(pParse,yymsp[0].major,yymsp[0].minor.yy0); /*A-overwrites-X*/} + case 171: /* term ::= NULL|FLOAT|BLOB */ + case 172: /* term ::= STRING */ yytestcase(yyruleno==172); +{yymsp[0].minor.yy46=tokenExpr(pParse,yymsp[0].major,yymsp[0].minor.yy0); /*A-overwrites-X*/} break; - case 169: /* term ::= INTEGER */ + case 173: /* term ::= INTEGER */ { - yylhsminor.yy490 = sqlite3ExprAlloc(pParse->db, TK_INTEGER, &yymsp[0].minor.yy0, 1); + yylhsminor.yy46 = sqlite3ExprAlloc(pParse->db, TK_INTEGER, &yymsp[0].minor.yy0, 1); } - yymsp[0].minor.yy490 = yylhsminor.yy490; + yymsp[0].minor.yy46 = yylhsminor.yy46; break; - case 170: /* expr ::= VARIABLE */ + case 174: /* expr ::= VARIABLE */ { if( !(yymsp[0].minor.yy0.z[0]=='#' && sqlite3Isdigit(yymsp[0].minor.yy0.z[1])) ){ u32 n = yymsp[0].minor.yy0.n; - yymsp[0].minor.yy490 = tokenExpr(pParse, TK_VARIABLE, yymsp[0].minor.yy0); - sqlite3ExprAssignVarNumber(pParse, yymsp[0].minor.yy490, n); + yymsp[0].minor.yy46 = tokenExpr(pParse, TK_VARIABLE, yymsp[0].minor.yy0); + sqlite3ExprAssignVarNumber(pParse, yymsp[0].minor.yy46, n); }else{ /* When doing a nested parse, one can include terms in an expression ** that look like this: #1 #2 ... These terms refer to registers @@ -151137,154 +153653,156 @@ static YYACTIONTYPE yy_reduce( assert( t.n>=2 ); if( pParse->nested==0 ){ sqlite3ErrorMsg(pParse, "near \"%T\": syntax error", &t); - yymsp[0].minor.yy490 = 0; + yymsp[0].minor.yy46 = 0; }else{ - yymsp[0].minor.yy490 = sqlite3PExpr(pParse, TK_REGISTER, 0, 0); - if( yymsp[0].minor.yy490 ) sqlite3GetInt32(&t.z[1], &yymsp[0].minor.yy490->iTable); + yymsp[0].minor.yy46 = sqlite3PExpr(pParse, TK_REGISTER, 0, 0); + if( yymsp[0].minor.yy46 ) sqlite3GetInt32(&t.z[1], &yymsp[0].minor.yy46->iTable); } } } break; - case 171: /* expr ::= expr COLLATE ID|STRING */ + case 175: /* expr ::= expr COLLATE ID|STRING */ { - yymsp[-2].minor.yy490 = sqlite3ExprAddCollateToken(pParse, yymsp[-2].minor.yy490, &yymsp[0].minor.yy0, 1); + yymsp[-2].minor.yy46 = sqlite3ExprAddCollateToken(pParse, yymsp[-2].minor.yy46, &yymsp[0].minor.yy0, 1); } break; - case 172: /* expr ::= CAST LP expr AS typetoken RP */ + case 176: /* expr ::= CAST LP expr AS typetoken RP */ { - yymsp[-5].minor.yy490 = sqlite3ExprAlloc(pParse->db, TK_CAST, &yymsp[-1].minor.yy0, 1); - sqlite3ExprAttachSubtrees(pParse->db, yymsp[-5].minor.yy490, yymsp[-3].minor.yy490, 0); + yymsp[-5].minor.yy46 = sqlite3ExprAlloc(pParse->db, TK_CAST, &yymsp[-1].minor.yy0, 1); + sqlite3ExprAttachSubtrees(pParse->db, yymsp[-5].minor.yy46, yymsp[-3].minor.yy46, 0); } break; - case 173: /* expr ::= ID|INDEXED LP distinct exprlist RP */ + case 177: /* expr ::= ID|INDEXED LP distinct exprlist RP */ { - yylhsminor.yy490 = sqlite3ExprFunction(pParse, yymsp[-1].minor.yy42, &yymsp[-4].minor.yy0, yymsp[-2].minor.yy96); + yylhsminor.yy46 = sqlite3ExprFunction(pParse, yymsp[-1].minor.yy138, &yymsp[-4].minor.yy0, yymsp[-2].minor.yy32); } - yymsp[-4].minor.yy490 = yylhsminor.yy490; + yymsp[-4].minor.yy46 = yylhsminor.yy46; break; - case 174: /* expr ::= ID|INDEXED LP STAR RP */ + case 178: /* expr ::= ID|INDEXED LP STAR RP */ { - yylhsminor.yy490 = sqlite3ExprFunction(pParse, 0, &yymsp[-3].minor.yy0, 0); + yylhsminor.yy46 = sqlite3ExprFunction(pParse, 0, &yymsp[-3].minor.yy0, 0); } - yymsp[-3].minor.yy490 = yylhsminor.yy490; + yymsp[-3].minor.yy46 = yylhsminor.yy46; break; - case 175: /* expr ::= ID|INDEXED LP distinct exprlist RP over_clause */ + case 179: /* expr ::= ID|INDEXED LP distinct exprlist RP filter_over */ { - yylhsminor.yy490 = sqlite3ExprFunction(pParse, yymsp[-2].minor.yy42, &yymsp[-5].minor.yy0, yymsp[-3].minor.yy96); - sqlite3WindowAttach(pParse, yylhsminor.yy490, yymsp[0].minor.yy147); + yylhsminor.yy46 = sqlite3ExprFunction(pParse, yymsp[-2].minor.yy138, &yymsp[-5].minor.yy0, yymsp[-3].minor.yy32); + sqlite3WindowAttach(pParse, yylhsminor.yy46, yymsp[0].minor.yy455); } - yymsp[-5].minor.yy490 = yylhsminor.yy490; + yymsp[-5].minor.yy46 = yylhsminor.yy46; break; - case 176: /* expr ::= ID|INDEXED LP STAR RP over_clause */ + case 180: /* expr ::= ID|INDEXED LP STAR RP filter_over */ { - yylhsminor.yy490 = sqlite3ExprFunction(pParse, 0, &yymsp[-4].minor.yy0, 0); - sqlite3WindowAttach(pParse, yylhsminor.yy490, yymsp[0].minor.yy147); + yylhsminor.yy46 = sqlite3ExprFunction(pParse, 0, &yymsp[-4].minor.yy0, 0); + sqlite3WindowAttach(pParse, yylhsminor.yy46, yymsp[0].minor.yy455); } - yymsp[-4].minor.yy490 = yylhsminor.yy490; + yymsp[-4].minor.yy46 = yylhsminor.yy46; break; - case 177: /* term ::= CTIME_KW */ + case 181: /* term ::= CTIME_KW */ { - yylhsminor.yy490 = sqlite3ExprFunction(pParse, 0, &yymsp[0].minor.yy0, 0); + yylhsminor.yy46 = sqlite3ExprFunction(pParse, 0, &yymsp[0].minor.yy0, 0); } - yymsp[0].minor.yy490 = yylhsminor.yy490; + yymsp[0].minor.yy46 = yylhsminor.yy46; break; - case 178: /* expr ::= LP nexprlist COMMA expr RP */ + case 182: /* expr ::= LP nexprlist COMMA expr RP */ { - ExprList *pList = sqlite3ExprListAppend(pParse, yymsp[-3].minor.yy42, yymsp[-1].minor.yy490); - yymsp[-4].minor.yy490 = sqlite3PExpr(pParse, TK_VECTOR, 0, 0); - if( yymsp[-4].minor.yy490 ){ - yymsp[-4].minor.yy490->x.pList = pList; + ExprList *pList = sqlite3ExprListAppend(pParse, yymsp[-3].minor.yy138, yymsp[-1].minor.yy46); + yymsp[-4].minor.yy46 = sqlite3PExpr(pParse, TK_VECTOR, 0, 0); + if( yymsp[-4].minor.yy46 ){ + yymsp[-4].minor.yy46->x.pList = pList; }else{ sqlite3ExprListDelete(pParse->db, pList); } } break; - case 179: /* expr ::= expr AND expr */ - case 180: /* expr ::= expr OR expr */ yytestcase(yyruleno==180); - case 181: /* expr ::= expr LT|GT|GE|LE expr */ yytestcase(yyruleno==181); - case 182: /* expr ::= expr EQ|NE expr */ yytestcase(yyruleno==182); - case 183: /* expr ::= expr BITAND|BITOR|LSHIFT|RSHIFT expr */ yytestcase(yyruleno==183); - case 184: /* expr ::= expr PLUS|MINUS expr */ yytestcase(yyruleno==184); - case 185: /* expr ::= expr STAR|SLASH|REM expr */ yytestcase(yyruleno==185); - case 186: /* expr ::= expr CONCAT expr */ yytestcase(yyruleno==186); -{yymsp[-2].minor.yy490=sqlite3PExpr(pParse,yymsp[-1].major,yymsp[-2].minor.yy490,yymsp[0].minor.yy490);} + case 183: /* expr ::= expr AND expr */ +{yymsp[-2].minor.yy46=sqlite3ExprAnd(pParse,yymsp[-2].minor.yy46,yymsp[0].minor.yy46);} + break; + case 184: /* expr ::= expr OR expr */ + case 185: /* expr ::= expr LT|GT|GE|LE expr */ yytestcase(yyruleno==185); + case 186: /* expr ::= expr EQ|NE expr */ yytestcase(yyruleno==186); + case 187: /* expr ::= expr BITAND|BITOR|LSHIFT|RSHIFT expr */ yytestcase(yyruleno==187); + case 188: /* expr ::= expr PLUS|MINUS expr */ yytestcase(yyruleno==188); + case 189: /* expr ::= expr STAR|SLASH|REM expr */ yytestcase(yyruleno==189); + case 190: /* expr ::= expr CONCAT expr */ yytestcase(yyruleno==190); +{yymsp[-2].minor.yy46=sqlite3PExpr(pParse,yymsp[-1].major,yymsp[-2].minor.yy46,yymsp[0].minor.yy46);} break; - case 187: /* likeop ::= NOT LIKE_KW|MATCH */ + case 191: /* likeop ::= NOT LIKE_KW|MATCH */ {yymsp[-1].minor.yy0=yymsp[0].minor.yy0; yymsp[-1].minor.yy0.n|=0x80000000; /*yymsp[-1].minor.yy0-overwrite-yymsp[0].minor.yy0*/} break; - case 188: /* expr ::= expr likeop expr */ + case 192: /* expr ::= expr likeop expr */ { ExprList *pList; int bNot = yymsp[-1].minor.yy0.n & 0x80000000; yymsp[-1].minor.yy0.n &= 0x7fffffff; - pList = sqlite3ExprListAppend(pParse,0, yymsp[0].minor.yy490); - pList = sqlite3ExprListAppend(pParse,pList, yymsp[-2].minor.yy490); - yymsp[-2].minor.yy490 = sqlite3ExprFunction(pParse, pList, &yymsp[-1].minor.yy0, 0); - if( bNot ) yymsp[-2].minor.yy490 = sqlite3PExpr(pParse, TK_NOT, yymsp[-2].minor.yy490, 0); - if( yymsp[-2].minor.yy490 ) yymsp[-2].minor.yy490->flags |= EP_InfixFunc; + pList = sqlite3ExprListAppend(pParse,0, yymsp[0].minor.yy46); + pList = sqlite3ExprListAppend(pParse,pList, yymsp[-2].minor.yy46); + yymsp[-2].minor.yy46 = sqlite3ExprFunction(pParse, pList, &yymsp[-1].minor.yy0, 0); + if( bNot ) yymsp[-2].minor.yy46 = sqlite3PExpr(pParse, TK_NOT, yymsp[-2].minor.yy46, 0); + if( yymsp[-2].minor.yy46 ) yymsp[-2].minor.yy46->flags |= EP_InfixFunc; } break; - case 189: /* expr ::= expr likeop expr ESCAPE expr */ + case 193: /* expr ::= expr likeop expr ESCAPE expr */ { ExprList *pList; int bNot = yymsp[-3].minor.yy0.n & 0x80000000; yymsp[-3].minor.yy0.n &= 0x7fffffff; - pList = sqlite3ExprListAppend(pParse,0, yymsp[-2].minor.yy490); - pList = sqlite3ExprListAppend(pParse,pList, yymsp[-4].minor.yy490); - pList = sqlite3ExprListAppend(pParse,pList, yymsp[0].minor.yy490); - yymsp[-4].minor.yy490 = sqlite3ExprFunction(pParse, pList, &yymsp[-3].minor.yy0, 0); - if( bNot ) yymsp[-4].minor.yy490 = sqlite3PExpr(pParse, TK_NOT, yymsp[-4].minor.yy490, 0); - if( yymsp[-4].minor.yy490 ) yymsp[-4].minor.yy490->flags |= EP_InfixFunc; + pList = sqlite3ExprListAppend(pParse,0, yymsp[-2].minor.yy46); + pList = sqlite3ExprListAppend(pParse,pList, yymsp[-4].minor.yy46); + pList = sqlite3ExprListAppend(pParse,pList, yymsp[0].minor.yy46); + yymsp[-4].minor.yy46 = sqlite3ExprFunction(pParse, pList, &yymsp[-3].minor.yy0, 0); + if( bNot ) yymsp[-4].minor.yy46 = sqlite3PExpr(pParse, TK_NOT, yymsp[-4].minor.yy46, 0); + if( yymsp[-4].minor.yy46 ) yymsp[-4].minor.yy46->flags |= EP_InfixFunc; } break; - case 190: /* expr ::= expr ISNULL|NOTNULL */ -{yymsp[-1].minor.yy490 = sqlite3PExpr(pParse,yymsp[0].major,yymsp[-1].minor.yy490,0);} + case 194: /* expr ::= expr ISNULL|NOTNULL */ +{yymsp[-1].minor.yy46 = sqlite3PExpr(pParse,yymsp[0].major,yymsp[-1].minor.yy46,0);} break; - case 191: /* expr ::= expr NOT NULL */ -{yymsp[-2].minor.yy490 = sqlite3PExpr(pParse,TK_NOTNULL,yymsp[-2].minor.yy490,0);} + case 195: /* expr ::= expr NOT NULL */ +{yymsp[-2].minor.yy46 = sqlite3PExpr(pParse,TK_NOTNULL,yymsp[-2].minor.yy46,0);} break; - case 192: /* expr ::= expr IS expr */ + case 196: /* expr ::= expr IS expr */ { - yymsp[-2].minor.yy490 = sqlite3PExpr(pParse,TK_IS,yymsp[-2].minor.yy490,yymsp[0].minor.yy490); - binaryToUnaryIfNull(pParse, yymsp[0].minor.yy490, yymsp[-2].minor.yy490, TK_ISNULL); + yymsp[-2].minor.yy46 = sqlite3PExpr(pParse,TK_IS,yymsp[-2].minor.yy46,yymsp[0].minor.yy46); + binaryToUnaryIfNull(pParse, yymsp[0].minor.yy46, yymsp[-2].minor.yy46, TK_ISNULL); } break; - case 193: /* expr ::= expr IS NOT expr */ + case 197: /* expr ::= expr IS NOT expr */ { - yymsp[-3].minor.yy490 = sqlite3PExpr(pParse,TK_ISNOT,yymsp[-3].minor.yy490,yymsp[0].minor.yy490); - binaryToUnaryIfNull(pParse, yymsp[0].minor.yy490, yymsp[-3].minor.yy490, TK_NOTNULL); + yymsp[-3].minor.yy46 = sqlite3PExpr(pParse,TK_ISNOT,yymsp[-3].minor.yy46,yymsp[0].minor.yy46); + binaryToUnaryIfNull(pParse, yymsp[0].minor.yy46, yymsp[-3].minor.yy46, TK_NOTNULL); } break; - case 194: /* expr ::= NOT expr */ - case 195: /* expr ::= BITNOT expr */ yytestcase(yyruleno==195); -{yymsp[-1].minor.yy490 = sqlite3PExpr(pParse, yymsp[-1].major, yymsp[0].minor.yy490, 0);/*A-overwrites-B*/} + case 198: /* expr ::= NOT expr */ + case 199: /* expr ::= BITNOT expr */ yytestcase(yyruleno==199); +{yymsp[-1].minor.yy46 = sqlite3PExpr(pParse, yymsp[-1].major, yymsp[0].minor.yy46, 0);/*A-overwrites-B*/} break; - case 196: /* expr ::= PLUS|MINUS expr */ + case 200: /* expr ::= PLUS|MINUS expr */ { - yymsp[-1].minor.yy490 = sqlite3PExpr(pParse, yymsp[-1].major==TK_PLUS ? TK_UPLUS : TK_UMINUS, yymsp[0].minor.yy490, 0); + yymsp[-1].minor.yy46 = sqlite3PExpr(pParse, yymsp[-1].major==TK_PLUS ? TK_UPLUS : TK_UMINUS, yymsp[0].minor.yy46, 0); /*A-overwrites-B*/ } break; - case 197: /* between_op ::= BETWEEN */ - case 200: /* in_op ::= IN */ yytestcase(yyruleno==200); -{yymsp[0].minor.yy96 = 0;} + case 201: /* between_op ::= BETWEEN */ + case 204: /* in_op ::= IN */ yytestcase(yyruleno==204); +{yymsp[0].minor.yy32 = 0;} break; - case 199: /* expr ::= expr between_op expr AND expr */ + case 203: /* expr ::= expr between_op expr AND expr */ { - ExprList *pList = sqlite3ExprListAppend(pParse,0, yymsp[-2].minor.yy490); - pList = sqlite3ExprListAppend(pParse,pList, yymsp[0].minor.yy490); - yymsp[-4].minor.yy490 = sqlite3PExpr(pParse, TK_BETWEEN, yymsp[-4].minor.yy490, 0); - if( yymsp[-4].minor.yy490 ){ - yymsp[-4].minor.yy490->x.pList = pList; + ExprList *pList = sqlite3ExprListAppend(pParse,0, yymsp[-2].minor.yy46); + pList = sqlite3ExprListAppend(pParse,pList, yymsp[0].minor.yy46); + yymsp[-4].minor.yy46 = sqlite3PExpr(pParse, TK_BETWEEN, yymsp[-4].minor.yy46, 0); + if( yymsp[-4].minor.yy46 ){ + yymsp[-4].minor.yy46->x.pList = pList; }else{ sqlite3ExprListDelete(pParse->db, pList); } - if( yymsp[-3].minor.yy96 ) yymsp[-4].minor.yy490 = sqlite3PExpr(pParse, TK_NOT, yymsp[-4].minor.yy490, 0); + if( yymsp[-3].minor.yy32 ) yymsp[-4].minor.yy46 = sqlite3PExpr(pParse, TK_NOT, yymsp[-4].minor.yy46, 0); } break; - case 202: /* expr ::= expr in_op LP exprlist RP */ + case 206: /* expr ::= expr in_op LP exprlist RP */ { - if( yymsp[-1].minor.yy42==0 ){ + if( yymsp[-1].minor.yy138==0 ){ /* Expressions of the form ** ** expr1 IN () @@ -151293,220 +153811,190 @@ static YYACTIONTYPE yy_reduce( ** simplify to constants 0 (false) and 1 (true), respectively, ** regardless of the value of expr1. */ - if( IN_RENAME_OBJECT==0 ){ - sqlite3ExprDelete(pParse->db, yymsp[-4].minor.yy490); - yymsp[-4].minor.yy490 = sqlite3ExprAlloc(pParse->db, TK_INTEGER,&sqlite3IntTokens[yymsp[-3].minor.yy96],1); - } - }else if( yymsp[-1].minor.yy42->nExpr==1 ){ - /* Expressions of the form: - ** - ** expr1 IN (?1) - ** expr1 NOT IN (?2) - ** - ** with exactly one value on the RHS can be simplified to something - ** like this: - ** - ** expr1 == ?1 - ** expr1 <> ?2 - ** - ** But, the RHS of the == or <> is marked with the EP_Generic flag - ** so that it may not contribute to the computation of comparison - ** affinity or the collating sequence to use for comparison. Otherwise, - ** the semantics would be subtly different from IN or NOT IN. - */ - Expr *pRHS = yymsp[-1].minor.yy42->a[0].pExpr; - yymsp[-1].minor.yy42->a[0].pExpr = 0; - sqlite3ExprListDelete(pParse->db, yymsp[-1].minor.yy42); - /* pRHS cannot be NULL because a malloc error would have been detected - ** before now and control would have never reached this point */ - if( ALWAYS(pRHS) ){ - pRHS->flags &= ~EP_Collate; - pRHS->flags |= EP_Generic; - } - yymsp[-4].minor.yy490 = sqlite3PExpr(pParse, yymsp[-3].minor.yy96 ? TK_NE : TK_EQ, yymsp[-4].minor.yy490, pRHS); - }else{ - yymsp[-4].minor.yy490 = sqlite3PExpr(pParse, TK_IN, yymsp[-4].minor.yy490, 0); - if( yymsp[-4].minor.yy490 ){ - yymsp[-4].minor.yy490->x.pList = yymsp[-1].minor.yy42; - sqlite3ExprSetHeightAndFlags(pParse, yymsp[-4].minor.yy490); + sqlite3ExprUnmapAndDelete(pParse, yymsp[-4].minor.yy46); + yymsp[-4].minor.yy46 = sqlite3Expr(pParse->db, TK_INTEGER, yymsp[-3].minor.yy32 ? "1" : "0"); + }else{ + yymsp[-4].minor.yy46 = sqlite3PExpr(pParse, TK_IN, yymsp[-4].minor.yy46, 0); + if( yymsp[-4].minor.yy46 ){ + yymsp[-4].minor.yy46->x.pList = yymsp[-1].minor.yy138; + sqlite3ExprSetHeightAndFlags(pParse, yymsp[-4].minor.yy46); }else{ - sqlite3ExprListDelete(pParse->db, yymsp[-1].minor.yy42); + sqlite3ExprListDelete(pParse->db, yymsp[-1].minor.yy138); } - if( yymsp[-3].minor.yy96 ) yymsp[-4].minor.yy490 = sqlite3PExpr(pParse, TK_NOT, yymsp[-4].minor.yy490, 0); + if( yymsp[-3].minor.yy32 ) yymsp[-4].minor.yy46 = sqlite3PExpr(pParse, TK_NOT, yymsp[-4].minor.yy46, 0); } } break; - case 203: /* expr ::= LP select RP */ + case 207: /* expr ::= LP select RP */ { - yymsp[-2].minor.yy490 = sqlite3PExpr(pParse, TK_SELECT, 0, 0); - sqlite3PExprAddSelect(pParse, yymsp[-2].minor.yy490, yymsp[-1].minor.yy423); + yymsp[-2].minor.yy46 = sqlite3PExpr(pParse, TK_SELECT, 0, 0); + sqlite3PExprAddSelect(pParse, yymsp[-2].minor.yy46, yymsp[-1].minor.yy25); } break; - case 204: /* expr ::= expr in_op LP select RP */ + case 208: /* expr ::= expr in_op LP select RP */ { - yymsp[-4].minor.yy490 = sqlite3PExpr(pParse, TK_IN, yymsp[-4].minor.yy490, 0); - sqlite3PExprAddSelect(pParse, yymsp[-4].minor.yy490, yymsp[-1].minor.yy423); - if( yymsp[-3].minor.yy96 ) yymsp[-4].minor.yy490 = sqlite3PExpr(pParse, TK_NOT, yymsp[-4].minor.yy490, 0); + yymsp[-4].minor.yy46 = sqlite3PExpr(pParse, TK_IN, yymsp[-4].minor.yy46, 0); + sqlite3PExprAddSelect(pParse, yymsp[-4].minor.yy46, yymsp[-1].minor.yy25); + if( yymsp[-3].minor.yy32 ) yymsp[-4].minor.yy46 = sqlite3PExpr(pParse, TK_NOT, yymsp[-4].minor.yy46, 0); } break; - case 205: /* expr ::= expr in_op nm dbnm paren_exprlist */ + case 209: /* expr ::= expr in_op nm dbnm paren_exprlist */ { SrcList *pSrc = sqlite3SrcListAppend(pParse, 0,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy0); Select *pSelect = sqlite3SelectNew(pParse, 0,pSrc,0,0,0,0,0,0); - if( yymsp[0].minor.yy42 ) sqlite3SrcListFuncArgs(pParse, pSelect ? pSrc : 0, yymsp[0].minor.yy42); - yymsp[-4].minor.yy490 = sqlite3PExpr(pParse, TK_IN, yymsp[-4].minor.yy490, 0); - sqlite3PExprAddSelect(pParse, yymsp[-4].minor.yy490, pSelect); - if( yymsp[-3].minor.yy96 ) yymsp[-4].minor.yy490 = sqlite3PExpr(pParse, TK_NOT, yymsp[-4].minor.yy490, 0); + if( yymsp[0].minor.yy138 ) sqlite3SrcListFuncArgs(pParse, pSelect ? pSrc : 0, yymsp[0].minor.yy138); + yymsp[-4].minor.yy46 = sqlite3PExpr(pParse, TK_IN, yymsp[-4].minor.yy46, 0); + sqlite3PExprAddSelect(pParse, yymsp[-4].minor.yy46, pSelect); + if( yymsp[-3].minor.yy32 ) yymsp[-4].minor.yy46 = sqlite3PExpr(pParse, TK_NOT, yymsp[-4].minor.yy46, 0); } break; - case 206: /* expr ::= EXISTS LP select RP */ + case 210: /* expr ::= EXISTS LP select RP */ { Expr *p; - p = yymsp[-3].minor.yy490 = sqlite3PExpr(pParse, TK_EXISTS, 0, 0); - sqlite3PExprAddSelect(pParse, p, yymsp[-1].minor.yy423); + p = yymsp[-3].minor.yy46 = sqlite3PExpr(pParse, TK_EXISTS, 0, 0); + sqlite3PExprAddSelect(pParse, p, yymsp[-1].minor.yy25); } break; - case 207: /* expr ::= CASE case_operand case_exprlist case_else END */ + case 211: /* expr ::= CASE case_operand case_exprlist case_else END */ { - yymsp[-4].minor.yy490 = sqlite3PExpr(pParse, TK_CASE, yymsp[-3].minor.yy490, 0); - if( yymsp[-4].minor.yy490 ){ - yymsp[-4].minor.yy490->x.pList = yymsp[-1].minor.yy490 ? sqlite3ExprListAppend(pParse,yymsp[-2].minor.yy42,yymsp[-1].minor.yy490) : yymsp[-2].minor.yy42; - sqlite3ExprSetHeightAndFlags(pParse, yymsp[-4].minor.yy490); + yymsp[-4].minor.yy46 = sqlite3PExpr(pParse, TK_CASE, yymsp[-3].minor.yy46, 0); + if( yymsp[-4].minor.yy46 ){ + yymsp[-4].minor.yy46->x.pList = yymsp[-1].minor.yy46 ? sqlite3ExprListAppend(pParse,yymsp[-2].minor.yy138,yymsp[-1].minor.yy46) : yymsp[-2].minor.yy138; + sqlite3ExprSetHeightAndFlags(pParse, yymsp[-4].minor.yy46); }else{ - sqlite3ExprListDelete(pParse->db, yymsp[-2].minor.yy42); - sqlite3ExprDelete(pParse->db, yymsp[-1].minor.yy490); + sqlite3ExprListDelete(pParse->db, yymsp[-2].minor.yy138); + sqlite3ExprDelete(pParse->db, yymsp[-1].minor.yy46); } } break; - case 208: /* case_exprlist ::= case_exprlist WHEN expr THEN expr */ + case 212: /* case_exprlist ::= case_exprlist WHEN expr THEN expr */ { - yymsp[-4].minor.yy42 = sqlite3ExprListAppend(pParse,yymsp[-4].minor.yy42, yymsp[-2].minor.yy490); - yymsp[-4].minor.yy42 = sqlite3ExprListAppend(pParse,yymsp[-4].minor.yy42, yymsp[0].minor.yy490); + yymsp[-4].minor.yy138 = sqlite3ExprListAppend(pParse,yymsp[-4].minor.yy138, yymsp[-2].minor.yy46); + yymsp[-4].minor.yy138 = sqlite3ExprListAppend(pParse,yymsp[-4].minor.yy138, yymsp[0].minor.yy46); } break; - case 209: /* case_exprlist ::= WHEN expr THEN expr */ + case 213: /* case_exprlist ::= WHEN expr THEN expr */ { - yymsp[-3].minor.yy42 = sqlite3ExprListAppend(pParse,0, yymsp[-2].minor.yy490); - yymsp[-3].minor.yy42 = sqlite3ExprListAppend(pParse,yymsp[-3].minor.yy42, yymsp[0].minor.yy490); + yymsp[-3].minor.yy138 = sqlite3ExprListAppend(pParse,0, yymsp[-2].minor.yy46); + yymsp[-3].minor.yy138 = sqlite3ExprListAppend(pParse,yymsp[-3].minor.yy138, yymsp[0].minor.yy46); } break; - case 212: /* case_operand ::= expr */ -{yymsp[0].minor.yy490 = yymsp[0].minor.yy490; /*A-overwrites-X*/} + case 216: /* case_operand ::= expr */ +{yymsp[0].minor.yy46 = yymsp[0].minor.yy46; /*A-overwrites-X*/} break; - case 215: /* nexprlist ::= nexprlist COMMA expr */ -{yymsp[-2].minor.yy42 = sqlite3ExprListAppend(pParse,yymsp[-2].minor.yy42,yymsp[0].minor.yy490);} + case 219: /* nexprlist ::= nexprlist COMMA expr */ +{yymsp[-2].minor.yy138 = sqlite3ExprListAppend(pParse,yymsp[-2].minor.yy138,yymsp[0].minor.yy46);} break; - case 216: /* nexprlist ::= expr */ -{yymsp[0].minor.yy42 = sqlite3ExprListAppend(pParse,0,yymsp[0].minor.yy490); /*A-overwrites-Y*/} + case 220: /* nexprlist ::= expr */ +{yymsp[0].minor.yy138 = sqlite3ExprListAppend(pParse,0,yymsp[0].minor.yy46); /*A-overwrites-Y*/} break; - case 218: /* paren_exprlist ::= LP exprlist RP */ - case 223: /* eidlist_opt ::= LP eidlist RP */ yytestcase(yyruleno==223); -{yymsp[-2].minor.yy42 = yymsp[-1].minor.yy42;} + case 222: /* paren_exprlist ::= LP exprlist RP */ + case 227: /* eidlist_opt ::= LP eidlist RP */ yytestcase(yyruleno==227); +{yymsp[-2].minor.yy138 = yymsp[-1].minor.yy138;} break; - case 219: /* cmd ::= createkw uniqueflag INDEX ifnotexists nm dbnm ON nm LP sortlist RP where_opt */ + case 223: /* cmd ::= createkw uniqueflag INDEX ifnotexists nm dbnm ON nm LP sortlist RP where_opt */ { sqlite3CreateIndex(pParse, &yymsp[-7].minor.yy0, &yymsp[-6].minor.yy0, - sqlite3SrcListAppend(pParse,0,&yymsp[-4].minor.yy0,0), yymsp[-2].minor.yy42, yymsp[-10].minor.yy96, - &yymsp[-11].minor.yy0, yymsp[0].minor.yy490, SQLITE_SO_ASC, yymsp[-8].minor.yy96, SQLITE_IDXTYPE_APPDEF); + sqlite3SrcListAppend(pParse,0,&yymsp[-4].minor.yy0,0), yymsp[-2].minor.yy138, yymsp[-10].minor.yy32, + &yymsp[-11].minor.yy0, yymsp[0].minor.yy46, SQLITE_SO_ASC, yymsp[-8].minor.yy32, SQLITE_IDXTYPE_APPDEF); if( IN_RENAME_OBJECT && pParse->pNewIndex ){ sqlite3RenameTokenMap(pParse, pParse->pNewIndex->zName, &yymsp[-4].minor.yy0); } } break; - case 220: /* uniqueflag ::= UNIQUE */ - case 262: /* raisetype ::= ABORT */ yytestcase(yyruleno==262); -{yymsp[0].minor.yy96 = OE_Abort;} + case 224: /* uniqueflag ::= UNIQUE */ + case 266: /* raisetype ::= ABORT */ yytestcase(yyruleno==266); +{yymsp[0].minor.yy32 = OE_Abort;} break; - case 221: /* uniqueflag ::= */ -{yymsp[1].minor.yy96 = OE_None;} + case 225: /* uniqueflag ::= */ +{yymsp[1].minor.yy32 = OE_None;} break; - case 224: /* eidlist ::= eidlist COMMA nm collate sortorder */ + case 228: /* eidlist ::= eidlist COMMA nm collate sortorder */ { - yymsp[-4].minor.yy42 = parserAddExprIdListTerm(pParse, yymsp[-4].minor.yy42, &yymsp[-2].minor.yy0, yymsp[-1].minor.yy96, yymsp[0].minor.yy96); + yymsp[-4].minor.yy138 = parserAddExprIdListTerm(pParse, yymsp[-4].minor.yy138, &yymsp[-2].minor.yy0, yymsp[-1].minor.yy32, yymsp[0].minor.yy32); } break; - case 225: /* eidlist ::= nm collate sortorder */ + case 229: /* eidlist ::= nm collate sortorder */ { - yymsp[-2].minor.yy42 = parserAddExprIdListTerm(pParse, 0, &yymsp[-2].minor.yy0, yymsp[-1].minor.yy96, yymsp[0].minor.yy96); /*A-overwrites-Y*/ + yymsp[-2].minor.yy138 = parserAddExprIdListTerm(pParse, 0, &yymsp[-2].minor.yy0, yymsp[-1].minor.yy32, yymsp[0].minor.yy32); /*A-overwrites-Y*/ } break; - case 228: /* cmd ::= DROP INDEX ifexists fullname */ -{sqlite3DropIndex(pParse, yymsp[0].minor.yy167, yymsp[-1].minor.yy96);} + case 232: /* cmd ::= DROP INDEX ifexists fullname */ +{sqlite3DropIndex(pParse, yymsp[0].minor.yy609, yymsp[-1].minor.yy32);} break; - case 229: /* cmd ::= VACUUM vinto */ -{sqlite3Vacuum(pParse,0,yymsp[0].minor.yy490);} + case 233: /* cmd ::= VACUUM vinto */ +{sqlite3Vacuum(pParse,0,yymsp[0].minor.yy46);} break; - case 230: /* cmd ::= VACUUM nm vinto */ -{sqlite3Vacuum(pParse,&yymsp[-1].minor.yy0,yymsp[0].minor.yy490);} + case 234: /* cmd ::= VACUUM nm vinto */ +{sqlite3Vacuum(pParse,&yymsp[-1].minor.yy0,yymsp[0].minor.yy46);} break; - case 233: /* cmd ::= PRAGMA nm dbnm */ + case 237: /* cmd ::= PRAGMA nm dbnm */ {sqlite3Pragma(pParse,&yymsp[-1].minor.yy0,&yymsp[0].minor.yy0,0,0);} break; - case 234: /* cmd ::= PRAGMA nm dbnm EQ nmnum */ + case 238: /* cmd ::= PRAGMA nm dbnm EQ nmnum */ {sqlite3Pragma(pParse,&yymsp[-3].minor.yy0,&yymsp[-2].minor.yy0,&yymsp[0].minor.yy0,0);} break; - case 235: /* cmd ::= PRAGMA nm dbnm LP nmnum RP */ + case 239: /* cmd ::= PRAGMA nm dbnm LP nmnum RP */ {sqlite3Pragma(pParse,&yymsp[-4].minor.yy0,&yymsp[-3].minor.yy0,&yymsp[-1].minor.yy0,0);} break; - case 236: /* cmd ::= PRAGMA nm dbnm EQ minus_num */ + case 240: /* cmd ::= PRAGMA nm dbnm EQ minus_num */ {sqlite3Pragma(pParse,&yymsp[-3].minor.yy0,&yymsp[-2].minor.yy0,&yymsp[0].minor.yy0,1);} break; - case 237: /* cmd ::= PRAGMA nm dbnm LP minus_num RP */ + case 241: /* cmd ::= PRAGMA nm dbnm LP minus_num RP */ {sqlite3Pragma(pParse,&yymsp[-4].minor.yy0,&yymsp[-3].minor.yy0,&yymsp[-1].minor.yy0,1);} break; - case 240: /* cmd ::= createkw trigger_decl BEGIN trigger_cmd_list END */ + case 244: /* cmd ::= createkw trigger_decl BEGIN trigger_cmd_list END */ { Token all; all.z = yymsp[-3].minor.yy0.z; all.n = (int)(yymsp[0].minor.yy0.z - yymsp[-3].minor.yy0.z) + yymsp[0].minor.yy0.n; - sqlite3FinishTrigger(pParse, yymsp[-1].minor.yy119, &all); + sqlite3FinishTrigger(pParse, yymsp[-1].minor.yy527, &all); } break; - case 241: /* trigger_decl ::= temp TRIGGER ifnotexists nm dbnm trigger_time trigger_event ON fullname foreach_clause when_clause */ + case 245: /* trigger_decl ::= temp TRIGGER ifnotexists nm dbnm trigger_time trigger_event ON fullname foreach_clause when_clause */ { - sqlite3BeginTrigger(pParse, &yymsp[-7].minor.yy0, &yymsp[-6].minor.yy0, yymsp[-5].minor.yy96, yymsp[-4].minor.yy350.a, yymsp[-4].minor.yy350.b, yymsp[-2].minor.yy167, yymsp[0].minor.yy490, yymsp[-10].minor.yy96, yymsp[-8].minor.yy96); + sqlite3BeginTrigger(pParse, &yymsp[-7].minor.yy0, &yymsp[-6].minor.yy0, yymsp[-5].minor.yy32, yymsp[-4].minor.yy572.a, yymsp[-4].minor.yy572.b, yymsp[-2].minor.yy609, yymsp[0].minor.yy46, yymsp[-10].minor.yy32, yymsp[-8].minor.yy32); yymsp[-10].minor.yy0 = (yymsp[-6].minor.yy0.n==0?yymsp[-7].minor.yy0:yymsp[-6].minor.yy0); /*A-overwrites-T*/ } break; - case 242: /* trigger_time ::= BEFORE|AFTER */ -{ yymsp[0].minor.yy96 = yymsp[0].major; /*A-overwrites-X*/ } + case 246: /* trigger_time ::= BEFORE|AFTER */ +{ yymsp[0].minor.yy32 = yymsp[0].major; /*A-overwrites-X*/ } break; - case 243: /* trigger_time ::= INSTEAD OF */ -{ yymsp[-1].minor.yy96 = TK_INSTEAD;} + case 247: /* trigger_time ::= INSTEAD OF */ +{ yymsp[-1].minor.yy32 = TK_INSTEAD;} break; - case 244: /* trigger_time ::= */ -{ yymsp[1].minor.yy96 = TK_BEFORE; } + case 248: /* trigger_time ::= */ +{ yymsp[1].minor.yy32 = TK_BEFORE; } break; - case 245: /* trigger_event ::= DELETE|INSERT */ - case 246: /* trigger_event ::= UPDATE */ yytestcase(yyruleno==246); -{yymsp[0].minor.yy350.a = yymsp[0].major; /*A-overwrites-X*/ yymsp[0].minor.yy350.b = 0;} + case 249: /* trigger_event ::= DELETE|INSERT */ + case 250: /* trigger_event ::= UPDATE */ yytestcase(yyruleno==250); +{yymsp[0].minor.yy572.a = yymsp[0].major; /*A-overwrites-X*/ yymsp[0].minor.yy572.b = 0;} break; - case 247: /* trigger_event ::= UPDATE OF idlist */ -{yymsp[-2].minor.yy350.a = TK_UPDATE; yymsp[-2].minor.yy350.b = yymsp[0].minor.yy336;} + case 251: /* trigger_event ::= UPDATE OF idlist */ +{yymsp[-2].minor.yy572.a = TK_UPDATE; yymsp[-2].minor.yy572.b = yymsp[0].minor.yy406;} break; - case 248: /* when_clause ::= */ - case 267: /* key_opt ::= */ yytestcase(yyruleno==267); - case 309: /* filter_opt ::= */ yytestcase(yyruleno==309); -{ yymsp[1].minor.yy490 = 0; } + case 252: /* when_clause ::= */ + case 271: /* key_opt ::= */ yytestcase(yyruleno==271); +{ yymsp[1].minor.yy46 = 0; } break; - case 249: /* when_clause ::= WHEN expr */ - case 268: /* key_opt ::= KEY expr */ yytestcase(yyruleno==268); -{ yymsp[-1].minor.yy490 = yymsp[0].minor.yy490; } + case 253: /* when_clause ::= WHEN expr */ + case 272: /* key_opt ::= KEY expr */ yytestcase(yyruleno==272); +{ yymsp[-1].minor.yy46 = yymsp[0].minor.yy46; } break; - case 250: /* trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI */ + case 254: /* trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI */ { - assert( yymsp[-2].minor.yy119!=0 ); - yymsp[-2].minor.yy119->pLast->pNext = yymsp[-1].minor.yy119; - yymsp[-2].minor.yy119->pLast = yymsp[-1].minor.yy119; + assert( yymsp[-2].minor.yy527!=0 ); + yymsp[-2].minor.yy527->pLast->pNext = yymsp[-1].minor.yy527; + yymsp[-2].minor.yy527->pLast = yymsp[-1].minor.yy527; } break; - case 251: /* trigger_cmd_list ::= trigger_cmd SEMI */ + case 255: /* trigger_cmd_list ::= trigger_cmd SEMI */ { - assert( yymsp[-1].minor.yy119!=0 ); - yymsp[-1].minor.yy119->pLast = yymsp[-1].minor.yy119; + assert( yymsp[-1].minor.yy527!=0 ); + yymsp[-1].minor.yy527->pLast = yymsp[-1].minor.yy527; } break; - case 252: /* trnm ::= nm DOT nm */ + case 256: /* trnm ::= nm DOT nm */ { yymsp[-2].minor.yy0 = yymsp[0].minor.yy0; sqlite3ErrorMsg(pParse, @@ -151514,306 +154002,342 @@ static YYACTIONTYPE yy_reduce( "statements within triggers"); } break; - case 253: /* tridxby ::= INDEXED BY nm */ + case 257: /* tridxby ::= INDEXED BY nm */ { sqlite3ErrorMsg(pParse, "the INDEXED BY clause is not allowed on UPDATE or DELETE statements " "within triggers"); } break; - case 254: /* tridxby ::= NOT INDEXED */ + case 258: /* tridxby ::= NOT INDEXED */ { sqlite3ErrorMsg(pParse, "the NOT INDEXED clause is not allowed on UPDATE or DELETE statements " "within triggers"); } break; - case 255: /* trigger_cmd ::= UPDATE orconf trnm tridxby SET setlist where_opt scanpt */ -{yylhsminor.yy119 = sqlite3TriggerUpdateStep(pParse, &yymsp[-5].minor.yy0, yymsp[-2].minor.yy42, yymsp[-1].minor.yy490, yymsp[-6].minor.yy96, yymsp[-7].minor.yy0.z, yymsp[0].minor.yy464);} - yymsp[-7].minor.yy119 = yylhsminor.yy119; + case 259: /* trigger_cmd ::= UPDATE orconf trnm tridxby SET setlist where_opt scanpt */ +{yylhsminor.yy527 = sqlite3TriggerUpdateStep(pParse, &yymsp[-5].minor.yy0, yymsp[-2].minor.yy138, yymsp[-1].minor.yy46, yymsp[-6].minor.yy32, yymsp[-7].minor.yy0.z, yymsp[0].minor.yy8);} + yymsp[-7].minor.yy527 = yylhsminor.yy527; break; - case 256: /* trigger_cmd ::= scanpt insert_cmd INTO trnm idlist_opt select upsert scanpt */ + case 260: /* trigger_cmd ::= scanpt insert_cmd INTO trnm idlist_opt select upsert scanpt */ { - yylhsminor.yy119 = sqlite3TriggerInsertStep(pParse,&yymsp[-4].minor.yy0,yymsp[-3].minor.yy336,yymsp[-2].minor.yy423,yymsp[-6].minor.yy96,yymsp[-1].minor.yy266,yymsp[-7].minor.yy464,yymsp[0].minor.yy464);/*yylhsminor.yy119-overwrites-yymsp[-6].minor.yy96*/ + yylhsminor.yy527 = sqlite3TriggerInsertStep(pParse,&yymsp[-4].minor.yy0,yymsp[-3].minor.yy406,yymsp[-2].minor.yy25,yymsp[-6].minor.yy32,yymsp[-1].minor.yy288,yymsp[-7].minor.yy8,yymsp[0].minor.yy8);/*yylhsminor.yy527-overwrites-yymsp[-6].minor.yy32*/ } - yymsp[-7].minor.yy119 = yylhsminor.yy119; + yymsp[-7].minor.yy527 = yylhsminor.yy527; break; - case 257: /* trigger_cmd ::= DELETE FROM trnm tridxby where_opt scanpt */ -{yylhsminor.yy119 = sqlite3TriggerDeleteStep(pParse, &yymsp[-3].minor.yy0, yymsp[-1].minor.yy490, yymsp[-5].minor.yy0.z, yymsp[0].minor.yy464);} - yymsp[-5].minor.yy119 = yylhsminor.yy119; + case 261: /* trigger_cmd ::= DELETE FROM trnm tridxby where_opt scanpt */ +{yylhsminor.yy527 = sqlite3TriggerDeleteStep(pParse, &yymsp[-3].minor.yy0, yymsp[-1].minor.yy46, yymsp[-5].minor.yy0.z, yymsp[0].minor.yy8);} + yymsp[-5].minor.yy527 = yylhsminor.yy527; break; - case 258: /* trigger_cmd ::= scanpt select scanpt */ -{yylhsminor.yy119 = sqlite3TriggerSelectStep(pParse->db, yymsp[-1].minor.yy423, yymsp[-2].minor.yy464, yymsp[0].minor.yy464); /*yylhsminor.yy119-overwrites-yymsp[-1].minor.yy423*/} - yymsp[-2].minor.yy119 = yylhsminor.yy119; + case 262: /* trigger_cmd ::= scanpt select scanpt */ +{yylhsminor.yy527 = sqlite3TriggerSelectStep(pParse->db, yymsp[-1].minor.yy25, yymsp[-2].minor.yy8, yymsp[0].minor.yy8); /*yylhsminor.yy527-overwrites-yymsp[-1].minor.yy25*/} + yymsp[-2].minor.yy527 = yylhsminor.yy527; break; - case 259: /* expr ::= RAISE LP IGNORE RP */ + case 263: /* expr ::= RAISE LP IGNORE RP */ { - yymsp[-3].minor.yy490 = sqlite3PExpr(pParse, TK_RAISE, 0, 0); - if( yymsp[-3].minor.yy490 ){ - yymsp[-3].minor.yy490->affinity = OE_Ignore; + yymsp[-3].minor.yy46 = sqlite3PExpr(pParse, TK_RAISE, 0, 0); + if( yymsp[-3].minor.yy46 ){ + yymsp[-3].minor.yy46->affExpr = OE_Ignore; } } break; - case 260: /* expr ::= RAISE LP raisetype COMMA nm RP */ + case 264: /* expr ::= RAISE LP raisetype COMMA nm RP */ { - yymsp[-5].minor.yy490 = sqlite3ExprAlloc(pParse->db, TK_RAISE, &yymsp[-1].minor.yy0, 1); - if( yymsp[-5].minor.yy490 ) { - yymsp[-5].minor.yy490->affinity = (char)yymsp[-3].minor.yy96; + yymsp[-5].minor.yy46 = sqlite3ExprAlloc(pParse->db, TK_RAISE, &yymsp[-1].minor.yy0, 1); + if( yymsp[-5].minor.yy46 ) { + yymsp[-5].minor.yy46->affExpr = (char)yymsp[-3].minor.yy32; } } break; - case 261: /* raisetype ::= ROLLBACK */ -{yymsp[0].minor.yy96 = OE_Rollback;} + case 265: /* raisetype ::= ROLLBACK */ +{yymsp[0].minor.yy32 = OE_Rollback;} break; - case 263: /* raisetype ::= FAIL */ -{yymsp[0].minor.yy96 = OE_Fail;} + case 267: /* raisetype ::= FAIL */ +{yymsp[0].minor.yy32 = OE_Fail;} break; - case 264: /* cmd ::= DROP TRIGGER ifexists fullname */ + case 268: /* cmd ::= DROP TRIGGER ifexists fullname */ { - sqlite3DropTrigger(pParse,yymsp[0].minor.yy167,yymsp[-1].minor.yy96); + sqlite3DropTrigger(pParse,yymsp[0].minor.yy609,yymsp[-1].minor.yy32); } break; - case 265: /* cmd ::= ATTACH database_kw_opt expr AS expr key_opt */ + case 269: /* cmd ::= ATTACH database_kw_opt expr AS expr key_opt */ { - sqlite3Attach(pParse, yymsp[-3].minor.yy490, yymsp[-1].minor.yy490, yymsp[0].minor.yy490); + sqlite3Attach(pParse, yymsp[-3].minor.yy46, yymsp[-1].minor.yy46, yymsp[0].minor.yy46); } break; - case 266: /* cmd ::= DETACH database_kw_opt expr */ + case 270: /* cmd ::= DETACH database_kw_opt expr */ { - sqlite3Detach(pParse, yymsp[0].minor.yy490); + sqlite3Detach(pParse, yymsp[0].minor.yy46); } break; - case 269: /* cmd ::= REINDEX */ + case 273: /* cmd ::= REINDEX */ {sqlite3Reindex(pParse, 0, 0);} break; - case 270: /* cmd ::= REINDEX nm dbnm */ + case 274: /* cmd ::= REINDEX nm dbnm */ {sqlite3Reindex(pParse, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0);} break; - case 271: /* cmd ::= ANALYZE */ + case 275: /* cmd ::= ANALYZE */ {sqlite3Analyze(pParse, 0, 0);} break; - case 272: /* cmd ::= ANALYZE nm dbnm */ + case 276: /* cmd ::= ANALYZE nm dbnm */ {sqlite3Analyze(pParse, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0);} break; - case 273: /* cmd ::= ALTER TABLE fullname RENAME TO nm */ + case 277: /* cmd ::= ALTER TABLE fullname RENAME TO nm */ { - sqlite3AlterRenameTable(pParse,yymsp[-3].minor.yy167,&yymsp[0].minor.yy0); + sqlite3AlterRenameTable(pParse,yymsp[-3].minor.yy609,&yymsp[0].minor.yy0); } break; - case 274: /* cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt columnname carglist */ + case 278: /* cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt columnname carglist */ { yymsp[-1].minor.yy0.n = (int)(pParse->sLastToken.z-yymsp[-1].minor.yy0.z) + pParse->sLastToken.n; sqlite3AlterFinishAddColumn(pParse, &yymsp[-1].minor.yy0); } break; - case 275: /* add_column_fullname ::= fullname */ + case 279: /* add_column_fullname ::= fullname */ { disableLookaside(pParse); - sqlite3AlterBeginAddColumn(pParse, yymsp[0].minor.yy167); + sqlite3AlterBeginAddColumn(pParse, yymsp[0].minor.yy609); } break; - case 276: /* cmd ::= ALTER TABLE fullname RENAME kwcolumn_opt nm TO nm */ + case 280: /* cmd ::= ALTER TABLE fullname RENAME kwcolumn_opt nm TO nm */ { - sqlite3AlterRenameColumn(pParse, yymsp[-5].minor.yy167, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0); + sqlite3AlterRenameColumn(pParse, yymsp[-5].minor.yy609, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0); } break; - case 277: /* cmd ::= create_vtab */ + case 281: /* cmd ::= create_vtab */ {sqlite3VtabFinishParse(pParse,0);} break; - case 278: /* cmd ::= create_vtab LP vtabarglist RP */ + case 282: /* cmd ::= create_vtab LP vtabarglist RP */ {sqlite3VtabFinishParse(pParse,&yymsp[0].minor.yy0);} break; - case 279: /* create_vtab ::= createkw VIRTUAL TABLE ifnotexists nm dbnm USING nm */ + case 283: /* create_vtab ::= createkw VIRTUAL TABLE ifnotexists nm dbnm USING nm */ { - sqlite3VtabBeginParse(pParse, &yymsp[-3].minor.yy0, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, yymsp[-4].minor.yy96); + sqlite3VtabBeginParse(pParse, &yymsp[-3].minor.yy0, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, yymsp[-4].minor.yy32); } break; - case 280: /* vtabarg ::= */ + case 284: /* vtabarg ::= */ {sqlite3VtabArgInit(pParse);} break; - case 281: /* vtabargtoken ::= ANY */ - case 282: /* vtabargtoken ::= lp anylist RP */ yytestcase(yyruleno==282); - case 283: /* lp ::= LP */ yytestcase(yyruleno==283); + case 285: /* vtabargtoken ::= ANY */ + case 286: /* vtabargtoken ::= lp anylist RP */ yytestcase(yyruleno==286); + case 287: /* lp ::= LP */ yytestcase(yyruleno==287); {sqlite3VtabArgExtend(pParse,&yymsp[0].minor.yy0);} break; - case 284: /* with ::= WITH wqlist */ - case 285: /* with ::= WITH RECURSIVE wqlist */ yytestcase(yyruleno==285); -{ sqlite3WithPush(pParse, yymsp[0].minor.yy499, 1); } + case 288: /* with ::= WITH wqlist */ + case 289: /* with ::= WITH RECURSIVE wqlist */ yytestcase(yyruleno==289); +{ sqlite3WithPush(pParse, yymsp[0].minor.yy297, 1); } break; - case 286: /* wqlist ::= nm eidlist_opt AS LP select RP */ + case 290: /* wqlist ::= nm eidlist_opt AS LP select RP */ { - yymsp[-5].minor.yy499 = sqlite3WithAdd(pParse, 0, &yymsp[-5].minor.yy0, yymsp[-4].minor.yy42, yymsp[-1].minor.yy423); /*A-overwrites-X*/ + yymsp[-5].minor.yy297 = sqlite3WithAdd(pParse, 0, &yymsp[-5].minor.yy0, yymsp[-4].minor.yy138, yymsp[-1].minor.yy25); /*A-overwrites-X*/ } break; - case 287: /* wqlist ::= wqlist COMMA nm eidlist_opt AS LP select RP */ + case 291: /* wqlist ::= wqlist COMMA nm eidlist_opt AS LP select RP */ { - yymsp[-7].minor.yy499 = sqlite3WithAdd(pParse, yymsp[-7].minor.yy499, &yymsp[-5].minor.yy0, yymsp[-4].minor.yy42, yymsp[-1].minor.yy423); + yymsp[-7].minor.yy297 = sqlite3WithAdd(pParse, yymsp[-7].minor.yy297, &yymsp[-5].minor.yy0, yymsp[-4].minor.yy138, yymsp[-1].minor.yy25); } break; - case 288: /* windowdefn_list ::= windowdefn */ -{ yylhsminor.yy147 = yymsp[0].minor.yy147; } - yymsp[0].minor.yy147 = yylhsminor.yy147; + case 292: /* windowdefn_list ::= windowdefn */ +{ yylhsminor.yy455 = yymsp[0].minor.yy455; } + yymsp[0].minor.yy455 = yylhsminor.yy455; break; - case 289: /* windowdefn_list ::= windowdefn_list COMMA windowdefn */ + case 293: /* windowdefn_list ::= windowdefn_list COMMA windowdefn */ { - assert( yymsp[0].minor.yy147!=0 ); - yymsp[0].minor.yy147->pNextWin = yymsp[-2].minor.yy147; - yylhsminor.yy147 = yymsp[0].minor.yy147; + assert( yymsp[0].minor.yy455!=0 ); + sqlite3WindowChain(pParse, yymsp[0].minor.yy455, yymsp[-2].minor.yy455); + yymsp[0].minor.yy455->pNextWin = yymsp[-2].minor.yy455; + yylhsminor.yy455 = yymsp[0].minor.yy455; } - yymsp[-2].minor.yy147 = yylhsminor.yy147; + yymsp[-2].minor.yy455 = yylhsminor.yy455; break; - case 290: /* windowdefn ::= nm AS window */ + case 294: /* windowdefn ::= nm AS LP window RP */ { - if( ALWAYS(yymsp[0].minor.yy147) ){ - yymsp[0].minor.yy147->zName = sqlite3DbStrNDup(pParse->db, yymsp[-2].minor.yy0.z, yymsp[-2].minor.yy0.n); + if( ALWAYS(yymsp[-1].minor.yy455) ){ + yymsp[-1].minor.yy455->zName = sqlite3DbStrNDup(pParse->db, yymsp[-4].minor.yy0.z, yymsp[-4].minor.yy0.n); } - yylhsminor.yy147 = yymsp[0].minor.yy147; + yylhsminor.yy455 = yymsp[-1].minor.yy455; } - yymsp[-2].minor.yy147 = yylhsminor.yy147; + yymsp[-4].minor.yy455 = yylhsminor.yy455; break; - case 291: /* window ::= LP part_opt orderby_opt frame_opt RP */ + case 295: /* window ::= PARTITION BY nexprlist orderby_opt frame_opt */ { - yymsp[-4].minor.yy147 = yymsp[-1].minor.yy147; - if( ALWAYS(yymsp[-4].minor.yy147) ){ - yymsp[-4].minor.yy147->pPartition = yymsp[-3].minor.yy42; - yymsp[-4].minor.yy147->pOrderBy = yymsp[-2].minor.yy42; - } + yymsp[-4].minor.yy455 = sqlite3WindowAssemble(pParse, yymsp[0].minor.yy455, yymsp[-2].minor.yy138, yymsp[-1].minor.yy138, 0); } break; - case 292: /* part_opt ::= PARTITION BY nexprlist */ -{ yymsp[-2].minor.yy42 = yymsp[0].minor.yy42; } + case 296: /* window ::= nm PARTITION BY nexprlist orderby_opt frame_opt */ +{ + yylhsminor.yy455 = sqlite3WindowAssemble(pParse, yymsp[0].minor.yy455, yymsp[-2].minor.yy138, yymsp[-1].minor.yy138, &yymsp[-5].minor.yy0); +} + yymsp[-5].minor.yy455 = yylhsminor.yy455; break; - case 293: /* part_opt ::= */ -{ yymsp[1].minor.yy42 = 0; } + case 297: /* window ::= ORDER BY sortlist frame_opt */ +{ + yymsp[-3].minor.yy455 = sqlite3WindowAssemble(pParse, yymsp[0].minor.yy455, 0, yymsp[-1].minor.yy138, 0); +} break; - case 294: /* frame_opt ::= */ + case 298: /* window ::= nm ORDER BY sortlist frame_opt */ +{ + yylhsminor.yy455 = sqlite3WindowAssemble(pParse, yymsp[0].minor.yy455, 0, yymsp[-1].minor.yy138, &yymsp[-4].minor.yy0); +} + yymsp[-4].minor.yy455 = yylhsminor.yy455; + break; + case 299: /* window ::= frame_opt */ + case 318: /* filter_over ::= over_clause */ yytestcase(yyruleno==318); +{ + yylhsminor.yy455 = yymsp[0].minor.yy455; +} + yymsp[0].minor.yy455 = yylhsminor.yy455; + break; + case 300: /* window ::= nm frame_opt */ +{ + yylhsminor.yy455 = sqlite3WindowAssemble(pParse, yymsp[0].minor.yy455, 0, 0, &yymsp[-1].minor.yy0); +} + yymsp[-1].minor.yy455 = yylhsminor.yy455; + break; + case 301: /* frame_opt ::= */ { - yymsp[1].minor.yy147 = sqlite3WindowAlloc(pParse, TK_RANGE, TK_UNBOUNDED, 0, TK_CURRENT, 0); + yymsp[1].minor.yy455 = sqlite3WindowAlloc(pParse, 0, TK_UNBOUNDED, 0, TK_CURRENT, 0, 0); } break; - case 295: /* frame_opt ::= range_or_rows frame_bound_s */ + case 302: /* frame_opt ::= range_or_rows frame_bound_s frame_exclude_opt */ { - yylhsminor.yy147 = sqlite3WindowAlloc(pParse, yymsp[-1].minor.yy96, yymsp[0].minor.yy317.eType, yymsp[0].minor.yy317.pExpr, TK_CURRENT, 0); + yylhsminor.yy455 = sqlite3WindowAlloc(pParse, yymsp[-2].minor.yy32, yymsp[-1].minor.yy57.eType, yymsp[-1].minor.yy57.pExpr, TK_CURRENT, 0, yymsp[0].minor.yy118); } - yymsp[-1].minor.yy147 = yylhsminor.yy147; + yymsp[-2].minor.yy455 = yylhsminor.yy455; break; - case 296: /* frame_opt ::= range_or_rows BETWEEN frame_bound_s AND frame_bound_e */ + case 303: /* frame_opt ::= range_or_rows BETWEEN frame_bound_s AND frame_bound_e frame_exclude_opt */ { - yylhsminor.yy147 = sqlite3WindowAlloc(pParse, yymsp[-4].minor.yy96, yymsp[-2].minor.yy317.eType, yymsp[-2].minor.yy317.pExpr, yymsp[0].minor.yy317.eType, yymsp[0].minor.yy317.pExpr); + yylhsminor.yy455 = sqlite3WindowAlloc(pParse, yymsp[-5].minor.yy32, yymsp[-3].minor.yy57.eType, yymsp[-3].minor.yy57.pExpr, yymsp[-1].minor.yy57.eType, yymsp[-1].minor.yy57.pExpr, yymsp[0].minor.yy118); } - yymsp[-4].minor.yy147 = yylhsminor.yy147; + yymsp[-5].minor.yy455 = yylhsminor.yy455; break; - case 297: /* range_or_rows ::= RANGE */ -{ yymsp[0].minor.yy96 = TK_RANGE; } + case 305: /* frame_bound_s ::= frame_bound */ + case 307: /* frame_bound_e ::= frame_bound */ yytestcase(yyruleno==307); +{yylhsminor.yy57 = yymsp[0].minor.yy57;} + yymsp[0].minor.yy57 = yylhsminor.yy57; break; - case 298: /* range_or_rows ::= ROWS */ -{ yymsp[0].minor.yy96 = TK_ROWS; } + case 306: /* frame_bound_s ::= UNBOUNDED PRECEDING */ + case 308: /* frame_bound_e ::= UNBOUNDED FOLLOWING */ yytestcase(yyruleno==308); + case 310: /* frame_bound ::= CURRENT ROW */ yytestcase(yyruleno==310); +{yylhsminor.yy57.eType = yymsp[-1].major; yylhsminor.yy57.pExpr = 0;} + yymsp[-1].minor.yy57 = yylhsminor.yy57; break; - case 299: /* frame_bound_s ::= frame_bound */ - case 301: /* frame_bound_e ::= frame_bound */ yytestcase(yyruleno==301); -{ yylhsminor.yy317 = yymsp[0].minor.yy317; } - yymsp[0].minor.yy317 = yylhsminor.yy317; + case 309: /* frame_bound ::= expr PRECEDING|FOLLOWING */ +{yylhsminor.yy57.eType = yymsp[0].major; yylhsminor.yy57.pExpr = yymsp[-1].minor.yy46;} + yymsp[-1].minor.yy57 = yylhsminor.yy57; break; - case 300: /* frame_bound_s ::= UNBOUNDED PRECEDING */ - case 302: /* frame_bound_e ::= UNBOUNDED FOLLOWING */ yytestcase(yyruleno==302); -{yymsp[-1].minor.yy317.eType = TK_UNBOUNDED; yymsp[-1].minor.yy317.pExpr = 0;} + case 311: /* frame_exclude_opt ::= */ +{yymsp[1].minor.yy118 = 0;} break; - case 303: /* frame_bound ::= expr PRECEDING */ -{ yylhsminor.yy317.eType = TK_PRECEDING; yylhsminor.yy317.pExpr = yymsp[-1].minor.yy490; } - yymsp[-1].minor.yy317 = yylhsminor.yy317; + case 312: /* frame_exclude_opt ::= EXCLUDE frame_exclude */ +{yymsp[-1].minor.yy118 = yymsp[0].minor.yy118;} break; - case 304: /* frame_bound ::= CURRENT ROW */ -{ yymsp[-1].minor.yy317.eType = TK_CURRENT ; yymsp[-1].minor.yy317.pExpr = 0; } + case 313: /* frame_exclude ::= NO OTHERS */ + case 314: /* frame_exclude ::= CURRENT ROW */ yytestcase(yyruleno==314); +{yymsp[-1].minor.yy118 = yymsp[-1].major; /*A-overwrites-X*/} break; - case 305: /* frame_bound ::= expr FOLLOWING */ -{ yylhsminor.yy317.eType = TK_FOLLOWING; yylhsminor.yy317.pExpr = yymsp[-1].minor.yy490; } - yymsp[-1].minor.yy317 = yylhsminor.yy317; + case 315: /* frame_exclude ::= GROUP|TIES */ +{yymsp[0].minor.yy118 = yymsp[0].major; /*A-overwrites-X*/} break; - case 306: /* window_clause ::= WINDOW windowdefn_list */ -{ yymsp[-1].minor.yy147 = yymsp[0].minor.yy147; } + case 316: /* window_clause ::= WINDOW windowdefn_list */ +{ yymsp[-1].minor.yy455 = yymsp[0].minor.yy455; } break; - case 307: /* over_clause ::= filter_opt OVER window */ + case 317: /* filter_over ::= filter_clause over_clause */ { - yylhsminor.yy147 = yymsp[0].minor.yy147; - assert( yylhsminor.yy147!=0 ); - yylhsminor.yy147->pFilter = yymsp[-2].minor.yy490; + yymsp[0].minor.yy455->pFilter = yymsp[-1].minor.yy46; + yylhsminor.yy455 = yymsp[0].minor.yy455; } - yymsp[-2].minor.yy147 = yylhsminor.yy147; + yymsp[-1].minor.yy455 = yylhsminor.yy455; break; - case 308: /* over_clause ::= filter_opt OVER nm */ + case 319: /* filter_over ::= filter_clause */ { - yylhsminor.yy147 = (Window*)sqlite3DbMallocZero(pParse->db, sizeof(Window)); - if( yylhsminor.yy147 ){ - yylhsminor.yy147->zName = sqlite3DbStrNDup(pParse->db, yymsp[0].minor.yy0.z, yymsp[0].minor.yy0.n); - yylhsminor.yy147->pFilter = yymsp[-2].minor.yy490; + yylhsminor.yy455 = (Window*)sqlite3DbMallocZero(pParse->db, sizeof(Window)); + if( yylhsminor.yy455 ){ + yylhsminor.yy455->eFrmType = TK_FILTER; + yylhsminor.yy455->pFilter = yymsp[0].minor.yy46; }else{ - sqlite3ExprDelete(pParse->db, yymsp[-2].minor.yy490); + sqlite3ExprDelete(pParse->db, yymsp[0].minor.yy46); } } - yymsp[-2].minor.yy147 = yylhsminor.yy147; + yymsp[0].minor.yy455 = yylhsminor.yy455; + break; + case 320: /* over_clause ::= OVER LP window RP */ +{ + yymsp[-3].minor.yy455 = yymsp[-1].minor.yy455; + assert( yymsp[-3].minor.yy455!=0 ); +} break; - case 310: /* filter_opt ::= FILTER LP WHERE expr RP */ -{ yymsp[-4].minor.yy490 = yymsp[-1].minor.yy490; } + case 321: /* over_clause ::= OVER nm */ +{ + yymsp[-1].minor.yy455 = (Window*)sqlite3DbMallocZero(pParse->db, sizeof(Window)); + if( yymsp[-1].minor.yy455 ){ + yymsp[-1].minor.yy455->zName = sqlite3DbStrNDup(pParse->db, yymsp[0].minor.yy0.z, yymsp[0].minor.yy0.n); + } +} + break; + case 322: /* filter_clause ::= FILTER LP WHERE expr RP */ +{ yymsp[-4].minor.yy46 = yymsp[-1].minor.yy46; } break; default: - /* (311) input ::= cmdlist */ yytestcase(yyruleno==311); - /* (312) cmdlist ::= cmdlist ecmd */ yytestcase(yyruleno==312); - /* (313) cmdlist ::= ecmd (OPTIMIZED OUT) */ assert(yyruleno!=313); - /* (314) ecmd ::= SEMI */ yytestcase(yyruleno==314); - /* (315) ecmd ::= cmdx SEMI */ yytestcase(yyruleno==315); - /* (316) ecmd ::= explain cmdx */ yytestcase(yyruleno==316); - /* (317) trans_opt ::= */ yytestcase(yyruleno==317); - /* (318) trans_opt ::= TRANSACTION */ yytestcase(yyruleno==318); - /* (319) trans_opt ::= TRANSACTION nm */ yytestcase(yyruleno==319); - /* (320) savepoint_opt ::= SAVEPOINT */ yytestcase(yyruleno==320); - /* (321) savepoint_opt ::= */ yytestcase(yyruleno==321); - /* (322) cmd ::= create_table create_table_args */ yytestcase(yyruleno==322); - /* (323) columnlist ::= columnlist COMMA columnname carglist */ yytestcase(yyruleno==323); - /* (324) columnlist ::= columnname carglist */ yytestcase(yyruleno==324); - /* (325) nm ::= ID|INDEXED */ yytestcase(yyruleno==325); - /* (326) nm ::= STRING */ yytestcase(yyruleno==326); - /* (327) nm ::= JOIN_KW */ yytestcase(yyruleno==327); - /* (328) typetoken ::= typename */ yytestcase(yyruleno==328); - /* (329) typename ::= ID|STRING */ yytestcase(yyruleno==329); - /* (330) signed ::= plus_num (OPTIMIZED OUT) */ assert(yyruleno!=330); - /* (331) signed ::= minus_num (OPTIMIZED OUT) */ assert(yyruleno!=331); - /* (332) carglist ::= carglist ccons */ yytestcase(yyruleno==332); - /* (333) carglist ::= */ yytestcase(yyruleno==333); - /* (334) ccons ::= NULL onconf */ yytestcase(yyruleno==334); - /* (335) conslist_opt ::= COMMA conslist */ yytestcase(yyruleno==335); - /* (336) conslist ::= conslist tconscomma tcons */ yytestcase(yyruleno==336); - /* (337) conslist ::= tcons (OPTIMIZED OUT) */ assert(yyruleno!=337); - /* (338) tconscomma ::= */ yytestcase(yyruleno==338); - /* (339) defer_subclause_opt ::= defer_subclause (OPTIMIZED OUT) */ assert(yyruleno!=339); - /* (340) resolvetype ::= raisetype (OPTIMIZED OUT) */ assert(yyruleno!=340); - /* (341) selectnowith ::= oneselect (OPTIMIZED OUT) */ assert(yyruleno!=341); - /* (342) oneselect ::= values */ yytestcase(yyruleno==342); - /* (343) sclp ::= selcollist COMMA */ yytestcase(yyruleno==343); - /* (344) as ::= ID|STRING */ yytestcase(yyruleno==344); - /* (345) expr ::= term (OPTIMIZED OUT) */ assert(yyruleno!=345); - /* (346) likeop ::= LIKE_KW|MATCH */ yytestcase(yyruleno==346); - /* (347) exprlist ::= nexprlist */ yytestcase(yyruleno==347); - /* (348) nmnum ::= plus_num (OPTIMIZED OUT) */ assert(yyruleno!=348); - /* (349) nmnum ::= nm (OPTIMIZED OUT) */ assert(yyruleno!=349); - /* (350) nmnum ::= ON */ yytestcase(yyruleno==350); - /* (351) nmnum ::= DELETE */ yytestcase(yyruleno==351); - /* (352) nmnum ::= DEFAULT */ yytestcase(yyruleno==352); - /* (353) plus_num ::= INTEGER|FLOAT */ yytestcase(yyruleno==353); - /* (354) foreach_clause ::= */ yytestcase(yyruleno==354); - /* (355) foreach_clause ::= FOR EACH ROW */ yytestcase(yyruleno==355); - /* (356) trnm ::= nm */ yytestcase(yyruleno==356); - /* (357) tridxby ::= */ yytestcase(yyruleno==357); - /* (358) database_kw_opt ::= DATABASE */ yytestcase(yyruleno==358); - /* (359) database_kw_opt ::= */ yytestcase(yyruleno==359); - /* (360) kwcolumn_opt ::= */ yytestcase(yyruleno==360); - /* (361) kwcolumn_opt ::= COLUMNKW */ yytestcase(yyruleno==361); - /* (362) vtabarglist ::= vtabarg */ yytestcase(yyruleno==362); - /* (363) vtabarglist ::= vtabarglist COMMA vtabarg */ yytestcase(yyruleno==363); - /* (364) vtabarg ::= vtabarg vtabargtoken */ yytestcase(yyruleno==364); - /* (365) anylist ::= */ yytestcase(yyruleno==365); - /* (366) anylist ::= anylist LP anylist RP */ yytestcase(yyruleno==366); - /* (367) anylist ::= anylist ANY */ yytestcase(yyruleno==367); - /* (368) with ::= */ yytestcase(yyruleno==368); + /* (323) input ::= cmdlist */ yytestcase(yyruleno==323); + /* (324) cmdlist ::= cmdlist ecmd */ yytestcase(yyruleno==324); + /* (325) cmdlist ::= ecmd (OPTIMIZED OUT) */ assert(yyruleno!=325); + /* (326) ecmd ::= SEMI */ yytestcase(yyruleno==326); + /* (327) ecmd ::= cmdx SEMI */ yytestcase(yyruleno==327); + /* (328) ecmd ::= explain cmdx */ yytestcase(yyruleno==328); + /* (329) trans_opt ::= */ yytestcase(yyruleno==329); + /* (330) trans_opt ::= TRANSACTION */ yytestcase(yyruleno==330); + /* (331) trans_opt ::= TRANSACTION nm */ yytestcase(yyruleno==331); + /* (332) savepoint_opt ::= SAVEPOINT */ yytestcase(yyruleno==332); + /* (333) savepoint_opt ::= */ yytestcase(yyruleno==333); + /* (334) cmd ::= create_table create_table_args */ yytestcase(yyruleno==334); + /* (335) columnlist ::= columnlist COMMA columnname carglist */ yytestcase(yyruleno==335); + /* (336) columnlist ::= columnname carglist */ yytestcase(yyruleno==336); + /* (337) nm ::= ID|INDEXED */ yytestcase(yyruleno==337); + /* (338) nm ::= STRING */ yytestcase(yyruleno==338); + /* (339) nm ::= JOIN_KW */ yytestcase(yyruleno==339); + /* (340) typetoken ::= typename */ yytestcase(yyruleno==340); + /* (341) typename ::= ID|STRING */ yytestcase(yyruleno==341); + /* (342) signed ::= plus_num (OPTIMIZED OUT) */ assert(yyruleno!=342); + /* (343) signed ::= minus_num (OPTIMIZED OUT) */ assert(yyruleno!=343); + /* (344) carglist ::= carglist ccons */ yytestcase(yyruleno==344); + /* (345) carglist ::= */ yytestcase(yyruleno==345); + /* (346) ccons ::= NULL onconf */ yytestcase(yyruleno==346); + /* (347) conslist_opt ::= COMMA conslist */ yytestcase(yyruleno==347); + /* (348) conslist ::= conslist tconscomma tcons */ yytestcase(yyruleno==348); + /* (349) conslist ::= tcons (OPTIMIZED OUT) */ assert(yyruleno!=349); + /* (350) tconscomma ::= */ yytestcase(yyruleno==350); + /* (351) defer_subclause_opt ::= defer_subclause (OPTIMIZED OUT) */ assert(yyruleno!=351); + /* (352) resolvetype ::= raisetype (OPTIMIZED OUT) */ assert(yyruleno!=352); + /* (353) selectnowith ::= oneselect (OPTIMIZED OUT) */ assert(yyruleno!=353); + /* (354) oneselect ::= values */ yytestcase(yyruleno==354); + /* (355) sclp ::= selcollist COMMA */ yytestcase(yyruleno==355); + /* (356) as ::= ID|STRING */ yytestcase(yyruleno==356); + /* (357) expr ::= term (OPTIMIZED OUT) */ assert(yyruleno!=357); + /* (358) likeop ::= LIKE_KW|MATCH */ yytestcase(yyruleno==358); + /* (359) exprlist ::= nexprlist */ yytestcase(yyruleno==359); + /* (360) nmnum ::= plus_num (OPTIMIZED OUT) */ assert(yyruleno!=360); + /* (361) nmnum ::= nm (OPTIMIZED OUT) */ assert(yyruleno!=361); + /* (362) nmnum ::= ON */ yytestcase(yyruleno==362); + /* (363) nmnum ::= DELETE */ yytestcase(yyruleno==363); + /* (364) nmnum ::= DEFAULT */ yytestcase(yyruleno==364); + /* (365) plus_num ::= INTEGER|FLOAT */ yytestcase(yyruleno==365); + /* (366) foreach_clause ::= */ yytestcase(yyruleno==366); + /* (367) foreach_clause ::= FOR EACH ROW */ yytestcase(yyruleno==367); + /* (368) trnm ::= nm */ yytestcase(yyruleno==368); + /* (369) tridxby ::= */ yytestcase(yyruleno==369); + /* (370) database_kw_opt ::= DATABASE */ yytestcase(yyruleno==370); + /* (371) database_kw_opt ::= */ yytestcase(yyruleno==371); + /* (372) kwcolumn_opt ::= */ yytestcase(yyruleno==372); + /* (373) kwcolumn_opt ::= COLUMNKW */ yytestcase(yyruleno==373); + /* (374) vtabarglist ::= vtabarg */ yytestcase(yyruleno==374); + /* (375) vtabarglist ::= vtabarglist COMMA vtabarg */ yytestcase(yyruleno==375); + /* (376) vtabarg ::= vtabarg vtabargtoken */ yytestcase(yyruleno==376); + /* (377) anylist ::= */ yytestcase(yyruleno==377); + /* (378) anylist ::= anylist LP anylist RP */ yytestcase(yyruleno==378); + /* (379) anylist ::= anylist ANY */ yytestcase(yyruleno==379); + /* (380) with ::= */ yytestcase(yyruleno==380); break; /********** End reduce actions ************************************************/ }; @@ -152105,9 +154629,8 @@ SQLITE_PRIVATE void sqlite3Parser( */ SQLITE_PRIVATE int sqlite3ParserFallback(int iToken){ #ifdef YYFALLBACK - if( iToken<(int)(sizeof(yyFallback)/sizeof(yyFallback[0])) ){ - return yyFallback[iToken]; - } + assert( iToken<(int)(sizeof(yyFallback)/sizeof(yyFallback[0])) ); + return yyFallback[iToken]; #else (void)iToken; #endif @@ -152276,144 +154799,146 @@ const unsigned char ebcdicToAscii[] = { ** is substantially reduced. This is important for embedded applications ** on platforms with limited memory. */ -/* Hash score: 208 */ -/* zKWText[] encodes 923 bytes of keyword text in 614 bytes */ +/* Hash score: 221 */ +/* zKWText[] encodes 967 bytes of keyword text in 638 bytes */ /* REINDEXEDESCAPEACHECKEYBEFOREIGNOREGEXPLAINSTEADDATABASELECT */ -/* ABLEFTHENDEFERRABLELSEXCEPTRANSACTIONATURALTERAISEXCLUSIVE */ -/* XISTSAVEPOINTERSECTRIGGEREFERENCESCONSTRAINTOFFSETEMPORARY */ -/* UNIQUERYWITHOUTERELEASEATTACHAVINGROUPDATEBEGINNERANGEBETWEEN */ -/* OTHINGLOBYCASCADELETECASECOLLATECREATECURRENT_DATEDETACH */ -/* IMMEDIATEJOINSERTLIKEMATCHPLANALYZEPRAGMABORTVALUESVIRTUALIMIT */ -/* WHENOTNULLWHERECURSIVEAFTERENAMEANDEFAULTAUTOINCREMENTCAST */ -/* COLUMNCOMMITCONFLICTCROSSCURRENT_TIMESTAMPARTITIONDEFERRED */ -/* ISTINCTDROPRECEDINGFAILFILTEREPLACEFOLLOWINGFROMFULLIFISNULL */ -/* ORDERESTRICTOVERIGHTROLLBACKROWSUNBOUNDEDUNIONUSINGVACUUMVIEW */ -/* INDOWINITIALLYPRIMARY */ -static const char zKWText[613] = { +/* ABLEFTHENDEFERRABLELSEXCLUDELETEMPORARYISNULLSAVEPOINTERSECT */ +/* IESNOTNULLIKEXCEPTRANSACTIONATURALTERAISEXCLUSIVEXISTS */ +/* CONSTRAINTOFFSETRIGGEREFERENCESUNIQUERYWITHOUTERELEASEATTACH */ +/* AVINGLOBEGINNERANGEBETWEENOTHINGROUPSCASCADETACHCASECOLLATE */ +/* CREATECURRENT_DATEIMMEDIATEJOINSERTMATCHPLANALYZEPRAGMABORT */ +/* UPDATEVALUESVIRTUALASTWHENWHERECURSIVEAFTERENAMEANDEFAULT */ +/* AUTOINCREMENTCASTCOLUMNCOMMITCONFLICTCROSSCURRENT_TIMESTAMP */ +/* ARTITIONDEFERREDISTINCTDROPRECEDINGFAILIMITFILTEREPLACEFIRST */ +/* FOLLOWINGFROMFULLIFORDERESTRICTOTHERSOVERIGHTROLLBACKROWS */ +/* UNBOUNDEDUNIONUSINGVACUUMVIEWINDOWBYINITIALLYPRIMARY */ +static const char zKWText[637] = { 'R','E','I','N','D','E','X','E','D','E','S','C','A','P','E','A','C','H', 'E','C','K','E','Y','B','E','F','O','R','E','I','G','N','O','R','E','G', 'E','X','P','L','A','I','N','S','T','E','A','D','D','A','T','A','B','A', 'S','E','L','E','C','T','A','B','L','E','F','T','H','E','N','D','E','F', - 'E','R','R','A','B','L','E','L','S','E','X','C','E','P','T','R','A','N', - 'S','A','C','T','I','O','N','A','T','U','R','A','L','T','E','R','A','I', - 'S','E','X','C','L','U','S','I','V','E','X','I','S','T','S','A','V','E', - 'P','O','I','N','T','E','R','S','E','C','T','R','I','G','G','E','R','E', - 'F','E','R','E','N','C','E','S','C','O','N','S','T','R','A','I','N','T', - 'O','F','F','S','E','T','E','M','P','O','R','A','R','Y','U','N','I','Q', - 'U','E','R','Y','W','I','T','H','O','U','T','E','R','E','L','E','A','S', - 'E','A','T','T','A','C','H','A','V','I','N','G','R','O','U','P','D','A', - 'T','E','B','E','G','I','N','N','E','R','A','N','G','E','B','E','T','W', - 'E','E','N','O','T','H','I','N','G','L','O','B','Y','C','A','S','C','A', - 'D','E','L','E','T','E','C','A','S','E','C','O','L','L','A','T','E','C', - 'R','E','A','T','E','C','U','R','R','E','N','T','_','D','A','T','E','D', - 'E','T','A','C','H','I','M','M','E','D','I','A','T','E','J','O','I','N', - 'S','E','R','T','L','I','K','E','M','A','T','C','H','P','L','A','N','A', - 'L','Y','Z','E','P','R','A','G','M','A','B','O','R','T','V','A','L','U', - 'E','S','V','I','R','T','U','A','L','I','M','I','T','W','H','E','N','O', - 'T','N','U','L','L','W','H','E','R','E','C','U','R','S','I','V','E','A', - 'F','T','E','R','E','N','A','M','E','A','N','D','E','F','A','U','L','T', - 'A','U','T','O','I','N','C','R','E','M','E','N','T','C','A','S','T','C', - 'O','L','U','M','N','C','O','M','M','I','T','C','O','N','F','L','I','C', - 'T','C','R','O','S','S','C','U','R','R','E','N','T','_','T','I','M','E', - 'S','T','A','M','P','A','R','T','I','T','I','O','N','D','E','F','E','R', - 'R','E','D','I','S','T','I','N','C','T','D','R','O','P','R','E','C','E', - 'D','I','N','G','F','A','I','L','F','I','L','T','E','R','E','P','L','A', - 'C','E','F','O','L','L','O','W','I','N','G','F','R','O','M','F','U','L', - 'L','I','F','I','S','N','U','L','L','O','R','D','E','R','E','S','T','R', - 'I','C','T','O','V','E','R','I','G','H','T','R','O','L','L','B','A','C', - 'K','R','O','W','S','U','N','B','O','U','N','D','E','D','U','N','I','O', - 'N','U','S','I','N','G','V','A','C','U','U','M','V','I','E','W','I','N', - 'D','O','W','I','N','I','T','I','A','L','L','Y','P','R','I','M','A','R', - 'Y', + 'E','R','R','A','B','L','E','L','S','E','X','C','L','U','D','E','L','E', + 'T','E','M','P','O','R','A','R','Y','I','S','N','U','L','L','S','A','V', + 'E','P','O','I','N','T','E','R','S','E','C','T','I','E','S','N','O','T', + 'N','U','L','L','I','K','E','X','C','E','P','T','R','A','N','S','A','C', + 'T','I','O','N','A','T','U','R','A','L','T','E','R','A','I','S','E','X', + 'C','L','U','S','I','V','E','X','I','S','T','S','C','O','N','S','T','R', + 'A','I','N','T','O','F','F','S','E','T','R','I','G','G','E','R','E','F', + 'E','R','E','N','C','E','S','U','N','I','Q','U','E','R','Y','W','I','T', + 'H','O','U','T','E','R','E','L','E','A','S','E','A','T','T','A','C','H', + 'A','V','I','N','G','L','O','B','E','G','I','N','N','E','R','A','N','G', + 'E','B','E','T','W','E','E','N','O','T','H','I','N','G','R','O','U','P', + 'S','C','A','S','C','A','D','E','T','A','C','H','C','A','S','E','C','O', + 'L','L','A','T','E','C','R','E','A','T','E','C','U','R','R','E','N','T', + '_','D','A','T','E','I','M','M','E','D','I','A','T','E','J','O','I','N', + 'S','E','R','T','M','A','T','C','H','P','L','A','N','A','L','Y','Z','E', + 'P','R','A','G','M','A','B','O','R','T','U','P','D','A','T','E','V','A', + 'L','U','E','S','V','I','R','T','U','A','L','A','S','T','W','H','E','N', + 'W','H','E','R','E','C','U','R','S','I','V','E','A','F','T','E','R','E', + 'N','A','M','E','A','N','D','E','F','A','U','L','T','A','U','T','O','I', + 'N','C','R','E','M','E','N','T','C','A','S','T','C','O','L','U','M','N', + 'C','O','M','M','I','T','C','O','N','F','L','I','C','T','C','R','O','S', + 'S','C','U','R','R','E','N','T','_','T','I','M','E','S','T','A','M','P', + 'A','R','T','I','T','I','O','N','D','E','F','E','R','R','E','D','I','S', + 'T','I','N','C','T','D','R','O','P','R','E','C','E','D','I','N','G','F', + 'A','I','L','I','M','I','T','F','I','L','T','E','R','E','P','L','A','C', + 'E','F','I','R','S','T','F','O','L','L','O','W','I','N','G','F','R','O', + 'M','F','U','L','L','I','F','O','R','D','E','R','E','S','T','R','I','C', + 'T','O','T','H','E','R','S','O','V','E','R','I','G','H','T','R','O','L', + 'L','B','A','C','K','R','O','W','S','U','N','B','O','U','N','D','E','D', + 'U','N','I','O','N','U','S','I','N','G','V','A','C','U','U','M','V','I', + 'E','W','I','N','D','O','W','B','Y','I','N','I','T','I','A','L','L','Y', + 'P','R','I','M','A','R','Y', }; /* aKWHash[i] is the hash value for the i-th keyword */ static const unsigned char aKWHash[127] = { - 74, 109, 124, 72, 106, 45, 0, 0, 81, 0, 76, 61, 0, - 42, 12, 77, 15, 0, 123, 84, 54, 118, 125, 19, 0, 0, - 130, 0, 128, 121, 0, 22, 96, 0, 9, 0, 0, 115, 69, - 0, 67, 6, 0, 48, 93, 136, 0, 126, 104, 0, 0, 44, - 0, 107, 24, 0, 17, 0, 131, 53, 23, 0, 5, 62, 132, - 99, 0, 0, 135, 110, 60, 134, 57, 113, 55, 0, 94, 0, - 103, 26, 0, 102, 0, 0, 0, 98, 95, 100, 105, 117, 14, - 39, 116, 0, 80, 0, 133, 114, 92, 59, 0, 129, 79, 119, - 86, 46, 83, 0, 0, 97, 40, 122, 120, 0, 127, 0, 0, - 29, 0, 89, 87, 88, 0, 20, 85, 111, 56, + 82, 113, 130, 80, 110, 29, 0, 0, 89, 0, 83, 70, 0, + 53, 35, 84, 15, 0, 129, 92, 64, 124, 131, 19, 0, 0, + 136, 0, 134, 126, 0, 22, 100, 0, 9, 0, 0, 121, 78, + 0, 76, 6, 0, 58, 97, 143, 0, 132, 108, 0, 0, 48, + 0, 111, 24, 0, 17, 0, 137, 63, 23, 26, 5, 65, 138, + 103, 120, 0, 142, 114, 69, 141, 66, 118, 72, 0, 98, 0, + 107, 41, 0, 106, 0, 0, 0, 102, 99, 104, 109, 123, 14, + 50, 122, 0, 87, 0, 139, 119, 140, 68, 127, 135, 86, 81, + 37, 91, 117, 0, 0, 101, 51, 128, 125, 0, 133, 0, 0, + 44, 0, 93, 67, 39, 0, 20, 45, 115, 88, }; /* aKWNext[] forms the hash collision chain. If aKWHash[i]==0 ** then the i-th keyword has no more hash collisions. Otherwise, ** the next keyword with the same hash is aKWHash[i]-1. */ -static const unsigned char aKWNext[136] = { +static const unsigned char aKWNext[143] = { 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, - 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 33, 0, 21, 0, 0, 0, 0, 0, 50, - 0, 43, 3, 47, 0, 0, 32, 0, 0, 0, 0, 0, 0, - 0, 1, 64, 0, 0, 65, 0, 41, 0, 38, 0, 0, 0, - 0, 0, 49, 75, 0, 0, 30, 0, 58, 0, 0, 0, 31, - 63, 16, 34, 10, 0, 0, 0, 0, 0, 0, 0, 11, 70, - 91, 0, 0, 8, 0, 108, 0, 101, 28, 52, 68, 0, 112, - 0, 73, 51, 0, 90, 27, 37, 0, 71, 36, 82, 0, 35, - 66, 25, 18, 0, 0, 78, + 0, 0, 0, 21, 0, 0, 0, 0, 12, 0, 0, 0, 0, + 0, 0, 0, 7, 0, 36, 0, 0, 28, 0, 0, 0, 31, + 0, 0, 0, 40, 0, 0, 0, 0, 0, 60, 0, 54, 0, + 0, 38, 47, 0, 0, 0, 3, 0, 0, 74, 1, 73, 0, + 0, 0, 52, 0, 0, 0, 0, 0, 0, 57, 59, 56, 30, + 0, 0, 0, 46, 0, 16, 49, 10, 0, 0, 0, 0, 0, + 0, 0, 11, 79, 95, 0, 0, 8, 0, 112, 0, 105, 0, + 43, 62, 0, 77, 0, 116, 0, 61, 0, 0, 94, 42, 55, + 0, 75, 34, 90, 32, 33, 27, 25, 18, 96, 0, 71, 85, }; /* aKWLen[i] is the length (in bytes) of the i-th keyword */ -static const unsigned char aKWLen[136] = { +static const unsigned char aKWLen[143] = { 7, 7, 5, 4, 6, 4, 5, 3, 6, 7, 3, 6, 6, - 7, 7, 3, 8, 2, 6, 5, 4, 4, 3, 10, 4, 6, - 11, 6, 2, 7, 5, 5, 9, 6, 9, 9, 7, 10, 10, - 4, 6, 2, 3, 9, 4, 2, 6, 5, 7, 4, 5, 7, - 6, 6, 5, 6, 5, 5, 5, 7, 7, 4, 2, 7, 3, - 6, 4, 7, 6, 12, 6, 9, 4, 6, 4, 5, 4, 7, - 6, 5, 6, 7, 5, 4, 7, 3, 2, 4, 5, 9, 5, - 6, 3, 7, 13, 2, 2, 4, 6, 6, 8, 5, 17, 12, - 7, 9, 8, 8, 2, 4, 9, 4, 6, 7, 9, 4, 4, - 2, 6, 5, 8, 4, 5, 8, 4, 3, 9, 5, 5, 6, - 4, 6, 2, 9, 3, 7, + 7, 7, 3, 8, 2, 6, 5, 4, 4, 3, 10, 4, 7, + 6, 9, 4, 2, 6, 5, 9, 9, 4, 7, 3, 2, 4, + 4, 6, 11, 6, 2, 7, 5, 5, 9, 6, 10, 4, 6, + 2, 3, 7, 10, 6, 5, 7, 4, 5, 7, 6, 6, 4, + 5, 5, 5, 7, 7, 6, 5, 7, 3, 6, 4, 7, 6, + 12, 9, 4, 6, 5, 4, 7, 6, 5, 6, 6, 7, 4, + 4, 5, 9, 5, 6, 3, 7, 13, 2, 2, 4, 6, 6, + 8, 5, 17, 12, 7, 9, 8, 8, 2, 4, 9, 4, 5, + 6, 7, 5, 9, 4, 4, 2, 5, 8, 6, 4, 5, 8, + 4, 3, 9, 5, 5, 6, 4, 6, 2, 2, 9, 3, 7, }; /* aKWOffset[i] is the index into zKWText[] of the start of ** the text for the i-th keyword. */ -static const unsigned short int aKWOffset[136] = { +static const unsigned short int aKWOffset[143] = { 0, 2, 2, 8, 9, 14, 16, 20, 23, 25, 25, 29, 33, 36, 41, 46, 48, 53, 54, 59, 62, 65, 67, 69, 78, 81, - 86, 91, 95, 96, 101, 105, 109, 117, 122, 128, 136, 142, 152, - 159, 162, 162, 165, 167, 167, 171, 176, 179, 184, 184, 188, 192, - 199, 204, 209, 212, 218, 221, 225, 230, 236, 242, 245, 247, 248, - 252, 258, 262, 269, 275, 287, 293, 302, 304, 310, 314, 319, 321, - 328, 333, 338, 344, 350, 355, 358, 358, 358, 361, 365, 368, 377, - 381, 387, 389, 396, 398, 400, 409, 413, 419, 425, 433, 438, 438, - 438, 454, 463, 470, 471, 478, 481, 490, 494, 499, 506, 515, 519, - 523, 525, 531, 535, 543, 546, 551, 559, 559, 563, 572, 577, 582, - 588, 591, 594, 597, 602, 606, + 86, 90, 90, 94, 99, 101, 105, 111, 119, 123, 123, 123, 126, + 129, 132, 137, 142, 146, 147, 152, 156, 160, 168, 174, 181, 184, + 184, 187, 189, 195, 205, 208, 213, 213, 217, 221, 228, 233, 238, + 241, 244, 248, 253, 259, 265, 265, 271, 272, 276, 282, 286, 293, + 299, 311, 320, 322, 328, 333, 335, 342, 347, 352, 358, 364, 370, + 374, 378, 381, 390, 394, 400, 402, 409, 411, 413, 422, 426, 432, + 438, 446, 451, 451, 451, 467, 476, 483, 484, 491, 494, 503, 506, + 511, 516, 523, 528, 537, 541, 545, 547, 551, 559, 565, 568, 573, + 581, 581, 585, 594, 599, 604, 610, 613, 616, 619, 621, 626, 630, }; /* aKWCode[i] is the parser symbol code for the i-th keyword */ -static const unsigned char aKWCode[136] = { +static const unsigned char aKWCode[143] = { TK_REINDEX, TK_INDEXED, TK_INDEX, TK_DESC, TK_ESCAPE, TK_EACH, TK_CHECK, TK_KEY, TK_BEFORE, TK_FOREIGN, TK_FOR, TK_IGNORE, TK_LIKE_KW, TK_EXPLAIN, TK_INSTEAD, TK_ADD, TK_DATABASE, TK_AS, TK_SELECT, TK_TABLE, TK_JOIN_KW, TK_THEN, TK_END, TK_DEFERRABLE, TK_ELSE, + TK_EXCLUDE, TK_DELETE, TK_TEMP, TK_TEMP, TK_OR, + TK_ISNULL, TK_NULLS, TK_SAVEPOINT, TK_INTERSECT, TK_TIES, + TK_NOTNULL, TK_NOT, TK_NO, TK_NULL, TK_LIKE_KW, TK_EXCEPT, TK_TRANSACTION,TK_ACTION, TK_ON, TK_JOIN_KW, - TK_ALTER, TK_RAISE, TK_EXCLUSIVE, TK_EXISTS, TK_SAVEPOINT, - TK_INTERSECT, TK_TRIGGER, TK_REFERENCES, TK_CONSTRAINT, TK_INTO, - TK_OFFSET, TK_OF, TK_SET, TK_TEMP, TK_TEMP, - TK_OR, TK_UNIQUE, TK_QUERY, TK_WITHOUT, TK_WITH, - TK_JOIN_KW, TK_RELEASE, TK_ATTACH, TK_HAVING, TK_GROUP, - TK_UPDATE, TK_BEGIN, TK_JOIN_KW, TK_RANGE, TK_BETWEEN, - TK_NOTHING, TK_LIKE_KW, TK_BY, TK_CASCADE, TK_ASC, - TK_DELETE, TK_CASE, TK_COLLATE, TK_CREATE, TK_CTIME_KW, - TK_DETACH, TK_IMMEDIATE, TK_JOIN, TK_INSERT, TK_LIKE_KW, - TK_MATCH, TK_PLAN, TK_ANALYZE, TK_PRAGMA, TK_ABORT, - TK_VALUES, TK_VIRTUAL, TK_LIMIT, TK_WHEN, TK_NOTNULL, - TK_NOT, TK_NO, TK_NULL, TK_WHERE, TK_RECURSIVE, - TK_AFTER, TK_RENAME, TK_AND, TK_DEFAULT, TK_AUTOINCR, - TK_TO, TK_IN, TK_CAST, TK_COLUMNKW, TK_COMMIT, - TK_CONFLICT, TK_JOIN_KW, TK_CTIME_KW, TK_CTIME_KW, TK_CURRENT, - TK_PARTITION, TK_DEFERRED, TK_DISTINCT, TK_IS, TK_DROP, - TK_PRECEDING, TK_FAIL, TK_FILTER, TK_REPLACE, TK_FOLLOWING, - TK_FROM, TK_JOIN_KW, TK_IF, TK_ISNULL, TK_ORDER, - TK_RESTRICT, TK_OVER, TK_JOIN_KW, TK_ROLLBACK, TK_ROWS, - TK_ROW, TK_UNBOUNDED, TK_UNION, TK_USING, TK_VACUUM, - TK_VIEW, TK_WINDOW, TK_DO, TK_INITIALLY, TK_ALL, - TK_PRIMARY, + TK_ALTER, TK_RAISE, TK_EXCLUSIVE, TK_EXISTS, TK_CONSTRAINT, + TK_INTO, TK_OFFSET, TK_OF, TK_SET, TK_TRIGGER, + TK_REFERENCES, TK_UNIQUE, TK_QUERY, TK_WITHOUT, TK_WITH, + TK_JOIN_KW, TK_RELEASE, TK_ATTACH, TK_HAVING, TK_LIKE_KW, + TK_BEGIN, TK_JOIN_KW, TK_RANGE, TK_BETWEEN, TK_NOTHING, + TK_GROUPS, TK_GROUP, TK_CASCADE, TK_ASC, TK_DETACH, + TK_CASE, TK_COLLATE, TK_CREATE, TK_CTIME_KW, TK_IMMEDIATE, + TK_JOIN, TK_INSERT, TK_MATCH, TK_PLAN, TK_ANALYZE, + TK_PRAGMA, TK_ABORT, TK_UPDATE, TK_VALUES, TK_VIRTUAL, + TK_LAST, TK_WHEN, TK_WHERE, TK_RECURSIVE, TK_AFTER, + TK_RENAME, TK_AND, TK_DEFAULT, TK_AUTOINCR, TK_TO, + TK_IN, TK_CAST, TK_COLUMNKW, TK_COMMIT, TK_CONFLICT, + TK_JOIN_KW, TK_CTIME_KW, TK_CTIME_KW, TK_CURRENT, TK_PARTITION, + TK_DEFERRED, TK_DISTINCT, TK_IS, TK_DROP, TK_PRECEDING, + TK_FAIL, TK_LIMIT, TK_FILTER, TK_REPLACE, TK_FIRST, + TK_FOLLOWING, TK_FROM, TK_JOIN_KW, TK_IF, TK_ORDER, + TK_RESTRICT, TK_OTHERS, TK_OVER, TK_JOIN_KW, TK_ROLLBACK, + TK_ROWS, TK_ROW, TK_UNBOUNDED, TK_UNION, TK_USING, + TK_VACUUM, TK_VIEW, TK_WINDOW, TK_DO, TK_BY, + TK_INITIALLY, TK_ALL, TK_PRIMARY, }; /* Check to see if z[0..n-1] is a keyword. If it is, write the ** parser symbol code for that keyword into *pType. Always @@ -152459,117 +154984,124 @@ static int keywordCode(const char *z, int n, int *pType){ testcase( i==22 ); /* END */ testcase( i==23 ); /* DEFERRABLE */ testcase( i==24 ); /* ELSE */ - testcase( i==25 ); /* EXCEPT */ - testcase( i==26 ); /* TRANSACTION */ - testcase( i==27 ); /* ACTION */ - testcase( i==28 ); /* ON */ - testcase( i==29 ); /* NATURAL */ - testcase( i==30 ); /* ALTER */ - testcase( i==31 ); /* RAISE */ - testcase( i==32 ); /* EXCLUSIVE */ - testcase( i==33 ); /* EXISTS */ - testcase( i==34 ); /* SAVEPOINT */ - testcase( i==35 ); /* INTERSECT */ - testcase( i==36 ); /* TRIGGER */ - testcase( i==37 ); /* REFERENCES */ - testcase( i==38 ); /* CONSTRAINT */ - testcase( i==39 ); /* INTO */ - testcase( i==40 ); /* OFFSET */ - testcase( i==41 ); /* OF */ - testcase( i==42 ); /* SET */ - testcase( i==43 ); /* TEMPORARY */ - testcase( i==44 ); /* TEMP */ - testcase( i==45 ); /* OR */ - testcase( i==46 ); /* UNIQUE */ - testcase( i==47 ); /* QUERY */ - testcase( i==48 ); /* WITHOUT */ - testcase( i==49 ); /* WITH */ - testcase( i==50 ); /* OUTER */ - testcase( i==51 ); /* RELEASE */ - testcase( i==52 ); /* ATTACH */ - testcase( i==53 ); /* HAVING */ - testcase( i==54 ); /* GROUP */ - testcase( i==55 ); /* UPDATE */ - testcase( i==56 ); /* BEGIN */ - testcase( i==57 ); /* INNER */ - testcase( i==58 ); /* RANGE */ - testcase( i==59 ); /* BETWEEN */ - testcase( i==60 ); /* NOTHING */ - testcase( i==61 ); /* GLOB */ - testcase( i==62 ); /* BY */ - testcase( i==63 ); /* CASCADE */ - testcase( i==64 ); /* ASC */ - testcase( i==65 ); /* DELETE */ - testcase( i==66 ); /* CASE */ - testcase( i==67 ); /* COLLATE */ - testcase( i==68 ); /* CREATE */ - testcase( i==69 ); /* CURRENT_DATE */ - testcase( i==70 ); /* DETACH */ - testcase( i==71 ); /* IMMEDIATE */ - testcase( i==72 ); /* JOIN */ - testcase( i==73 ); /* INSERT */ - testcase( i==74 ); /* LIKE */ - testcase( i==75 ); /* MATCH */ - testcase( i==76 ); /* PLAN */ - testcase( i==77 ); /* ANALYZE */ - testcase( i==78 ); /* PRAGMA */ - testcase( i==79 ); /* ABORT */ - testcase( i==80 ); /* VALUES */ - testcase( i==81 ); /* VIRTUAL */ - testcase( i==82 ); /* LIMIT */ - testcase( i==83 ); /* WHEN */ - testcase( i==84 ); /* NOTNULL */ - testcase( i==85 ); /* NOT */ - testcase( i==86 ); /* NO */ - testcase( i==87 ); /* NULL */ - testcase( i==88 ); /* WHERE */ - testcase( i==89 ); /* RECURSIVE */ - testcase( i==90 ); /* AFTER */ - testcase( i==91 ); /* RENAME */ - testcase( i==92 ); /* AND */ - testcase( i==93 ); /* DEFAULT */ - testcase( i==94 ); /* AUTOINCREMENT */ - testcase( i==95 ); /* TO */ - testcase( i==96 ); /* IN */ - testcase( i==97 ); /* CAST */ - testcase( i==98 ); /* COLUMN */ - testcase( i==99 ); /* COMMIT */ - testcase( i==100 ); /* CONFLICT */ - testcase( i==101 ); /* CROSS */ - testcase( i==102 ); /* CURRENT_TIMESTAMP */ - testcase( i==103 ); /* CURRENT_TIME */ - testcase( i==104 ); /* CURRENT */ - testcase( i==105 ); /* PARTITION */ - testcase( i==106 ); /* DEFERRED */ - testcase( i==107 ); /* DISTINCT */ - testcase( i==108 ); /* IS */ - testcase( i==109 ); /* DROP */ - testcase( i==110 ); /* PRECEDING */ - testcase( i==111 ); /* FAIL */ - testcase( i==112 ); /* FILTER */ - testcase( i==113 ); /* REPLACE */ - testcase( i==114 ); /* FOLLOWING */ - testcase( i==115 ); /* FROM */ - testcase( i==116 ); /* FULL */ - testcase( i==117 ); /* IF */ - testcase( i==118 ); /* ISNULL */ - testcase( i==119 ); /* ORDER */ - testcase( i==120 ); /* RESTRICT */ - testcase( i==121 ); /* OVER */ - testcase( i==122 ); /* RIGHT */ - testcase( i==123 ); /* ROLLBACK */ - testcase( i==124 ); /* ROWS */ - testcase( i==125 ); /* ROW */ - testcase( i==126 ); /* UNBOUNDED */ - testcase( i==127 ); /* UNION */ - testcase( i==128 ); /* USING */ - testcase( i==129 ); /* VACUUM */ - testcase( i==130 ); /* VIEW */ - testcase( i==131 ); /* WINDOW */ - testcase( i==132 ); /* DO */ - testcase( i==133 ); /* INITIALLY */ - testcase( i==134 ); /* ALL */ - testcase( i==135 ); /* PRIMARY */ + testcase( i==25 ); /* EXCLUDE */ + testcase( i==26 ); /* DELETE */ + testcase( i==27 ); /* TEMPORARY */ + testcase( i==28 ); /* TEMP */ + testcase( i==29 ); /* OR */ + testcase( i==30 ); /* ISNULL */ + testcase( i==31 ); /* NULLS */ + testcase( i==32 ); /* SAVEPOINT */ + testcase( i==33 ); /* INTERSECT */ + testcase( i==34 ); /* TIES */ + testcase( i==35 ); /* NOTNULL */ + testcase( i==36 ); /* NOT */ + testcase( i==37 ); /* NO */ + testcase( i==38 ); /* NULL */ + testcase( i==39 ); /* LIKE */ + testcase( i==40 ); /* EXCEPT */ + testcase( i==41 ); /* TRANSACTION */ + testcase( i==42 ); /* ACTION */ + testcase( i==43 ); /* ON */ + testcase( i==44 ); /* NATURAL */ + testcase( i==45 ); /* ALTER */ + testcase( i==46 ); /* RAISE */ + testcase( i==47 ); /* EXCLUSIVE */ + testcase( i==48 ); /* EXISTS */ + testcase( i==49 ); /* CONSTRAINT */ + testcase( i==50 ); /* INTO */ + testcase( i==51 ); /* OFFSET */ + testcase( i==52 ); /* OF */ + testcase( i==53 ); /* SET */ + testcase( i==54 ); /* TRIGGER */ + testcase( i==55 ); /* REFERENCES */ + testcase( i==56 ); /* UNIQUE */ + testcase( i==57 ); /* QUERY */ + testcase( i==58 ); /* WITHOUT */ + testcase( i==59 ); /* WITH */ + testcase( i==60 ); /* OUTER */ + testcase( i==61 ); /* RELEASE */ + testcase( i==62 ); /* ATTACH */ + testcase( i==63 ); /* HAVING */ + testcase( i==64 ); /* GLOB */ + testcase( i==65 ); /* BEGIN */ + testcase( i==66 ); /* INNER */ + testcase( i==67 ); /* RANGE */ + testcase( i==68 ); /* BETWEEN */ + testcase( i==69 ); /* NOTHING */ + testcase( i==70 ); /* GROUPS */ + testcase( i==71 ); /* GROUP */ + testcase( i==72 ); /* CASCADE */ + testcase( i==73 ); /* ASC */ + testcase( i==74 ); /* DETACH */ + testcase( i==75 ); /* CASE */ + testcase( i==76 ); /* COLLATE */ + testcase( i==77 ); /* CREATE */ + testcase( i==78 ); /* CURRENT_DATE */ + testcase( i==79 ); /* IMMEDIATE */ + testcase( i==80 ); /* JOIN */ + testcase( i==81 ); /* INSERT */ + testcase( i==82 ); /* MATCH */ + testcase( i==83 ); /* PLAN */ + testcase( i==84 ); /* ANALYZE */ + testcase( i==85 ); /* PRAGMA */ + testcase( i==86 ); /* ABORT */ + testcase( i==87 ); /* UPDATE */ + testcase( i==88 ); /* VALUES */ + testcase( i==89 ); /* VIRTUAL */ + testcase( i==90 ); /* LAST */ + testcase( i==91 ); /* WHEN */ + testcase( i==92 ); /* WHERE */ + testcase( i==93 ); /* RECURSIVE */ + testcase( i==94 ); /* AFTER */ + testcase( i==95 ); /* RENAME */ + testcase( i==96 ); /* AND */ + testcase( i==97 ); /* DEFAULT */ + testcase( i==98 ); /* AUTOINCREMENT */ + testcase( i==99 ); /* TO */ + testcase( i==100 ); /* IN */ + testcase( i==101 ); /* CAST */ + testcase( i==102 ); /* COLUMN */ + testcase( i==103 ); /* COMMIT */ + testcase( i==104 ); /* CONFLICT */ + testcase( i==105 ); /* CROSS */ + testcase( i==106 ); /* CURRENT_TIMESTAMP */ + testcase( i==107 ); /* CURRENT_TIME */ + testcase( i==108 ); /* CURRENT */ + testcase( i==109 ); /* PARTITION */ + testcase( i==110 ); /* DEFERRED */ + testcase( i==111 ); /* DISTINCT */ + testcase( i==112 ); /* IS */ + testcase( i==113 ); /* DROP */ + testcase( i==114 ); /* PRECEDING */ + testcase( i==115 ); /* FAIL */ + testcase( i==116 ); /* LIMIT */ + testcase( i==117 ); /* FILTER */ + testcase( i==118 ); /* REPLACE */ + testcase( i==119 ); /* FIRST */ + testcase( i==120 ); /* FOLLOWING */ + testcase( i==121 ); /* FROM */ + testcase( i==122 ); /* FULL */ + testcase( i==123 ); /* IF */ + testcase( i==124 ); /* ORDER */ + testcase( i==125 ); /* RESTRICT */ + testcase( i==126 ); /* OTHERS */ + testcase( i==127 ); /* OVER */ + testcase( i==128 ); /* RIGHT */ + testcase( i==129 ); /* ROLLBACK */ + testcase( i==130 ); /* ROWS */ + testcase( i==131 ); /* ROW */ + testcase( i==132 ); /* UNBOUNDED */ + testcase( i==133 ); /* UNION */ + testcase( i==134 ); /* USING */ + testcase( i==135 ); /* VACUUM */ + testcase( i==136 ); /* VIEW */ + testcase( i==137 ); /* WINDOW */ + testcase( i==138 ); /* DO */ + testcase( i==139 ); /* BY */ + testcase( i==140 ); /* INITIALLY */ + testcase( i==141 ); /* ALL */ + testcase( i==142 ); /* PRIMARY */ *pType = aKWCode[i]; break; } @@ -152581,7 +155113,7 @@ SQLITE_PRIVATE int sqlite3KeywordCode(const unsigned char *z, int n){ keywordCode((char*)z, n, &id); return id; } -#define SQLITE_N_KEYWORD 136 +#define SQLITE_N_KEYWORD 143 SQLITE_API int sqlite3_keyword_name(int i,const char **pzName,int *pnName){ if( i<0 || i>=SQLITE_N_KEYWORD ) return SQLITE_ERROR; *pzName = zKWText + aKWOffset[i]; @@ -153014,6 +155546,7 @@ SQLITE_PRIVATE int sqlite3RunParser(Parse *pParse, const char *zSql, char **pzEr #ifdef sqlite3Parser_ENGINEALWAYSONSTACK yyParser sEngine; /* Space to hold the Lemon-generated Parser object */ #endif + VVA_ONLY( u8 startedWithOom = db->mallocFailed ); assert( zSql!=0 ); mxSqlLen = db->aLimit[SQLITE_LIMIT_SQL_LENGTH]; @@ -153045,6 +155578,8 @@ SQLITE_PRIVATE int sqlite3RunParser(Parse *pParse, const char *zSql, char **pzEr assert( pParse->pNewTrigger==0 ); assert( pParse->nVar==0 ); assert( pParse->pVList==0 ); + pParse->pParentParse = db->pParse; + db->pParse = pParse; while( 1 ){ n = sqlite3GetToken((u8*)zSql, &tokenType); mxSqlLen -= n; @@ -153101,7 +155636,8 @@ SQLITE_PRIVATE int sqlite3RunParser(Parse *pParse, const char *zSql, char **pzEr sqlite3Parser(pEngine, tokenType, pParse->sLastToken); lastTokenParsed = tokenType; zSql += n; - if( pParse->rc!=SQLITE_OK || db->mallocFailed ) break; + assert( db->mallocFailed==0 || pParse->rc!=SQLITE_OK || startedWithOom ); + if( pParse->rc!=SQLITE_OK ) break; } assert( nErr==0 ); #ifdef YYTRACKMAXSTACKDEPTH @@ -153169,6 +155705,8 @@ SQLITE_PRIVATE int sqlite3RunParser(Parse *pParse, const char *zSql, char **pzEr pParse->pZombieTab = p->pNextZombie; sqlite3DeleteTable(db, p); } + db->pParse = pParse->pParentParse; + pParse->pParentParse = 0; assert( nErr==0 || pParse->rc!=SQLITE_OK ); return nErr; } @@ -154405,7 +156943,7 @@ static int setupLookaside(sqlite3 *db, void *pBuf, int sz, int cnt){ pStart = 0; }else if( pBuf==0 ){ sqlite3BeginBenignMalloc(); - pStart = sqlite3Malloc( sz*cnt ); /* IMP: R-61949-35727 */ + pStart = sqlite3Malloc( sz*(sqlite3_int64)cnt ); /* IMP: R-61949-35727 */ sqlite3EndBenignMalloc(); if( pStart ) cnt = sqlite3MallocSize(pStart)/sz; }else{ @@ -154536,6 +157074,7 @@ SQLITE_API int sqlite3_db_config(sqlite3 *db, int op, ...){ } aFlagOp[] = { { SQLITE_DBCONFIG_ENABLE_FKEY, SQLITE_ForeignKeys }, { SQLITE_DBCONFIG_ENABLE_TRIGGER, SQLITE_EnableTrigger }, + { SQLITE_DBCONFIG_ENABLE_VIEW, SQLITE_EnableView }, { SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER, SQLITE_Fts3Tokenizer }, { SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION, SQLITE_LoadExtension }, { SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE, SQLITE_NoCkptOnClose }, @@ -154543,6 +157082,11 @@ SQLITE_API int sqlite3_db_config(sqlite3 *db, int op, ...){ { SQLITE_DBCONFIG_TRIGGER_EQP, SQLITE_TriggerEQP }, { SQLITE_DBCONFIG_RESET_DATABASE, SQLITE_ResetDatabase }, { SQLITE_DBCONFIG_DEFENSIVE, SQLITE_Defensive }, + { SQLITE_DBCONFIG_WRITABLE_SCHEMA, SQLITE_WriteSchema| + SQLITE_NoSchemaError }, + { SQLITE_DBCONFIG_LEGACY_ALTER_TABLE, SQLITE_LegacyAlter }, + { SQLITE_DBCONFIG_DQS_DDL, SQLITE_DqsDDL }, + { SQLITE_DBCONFIG_DQS_DML, SQLITE_DqsDML }, }; unsigned int i; rc = SQLITE_ERROR; /* IMP: R-42790-23372 */ @@ -154573,28 +157117,17 @@ SQLITE_API int sqlite3_db_config(sqlite3 *db, int op, ...){ return rc; } - -/* -** Return true if the buffer z[0..n-1] contains all spaces. -*/ -static int allSpaces(const char *z, int n){ - while( n>0 && z[n-1]==' ' ){ n--; } - return n==0; -} - /* ** This is the default collating function named "BINARY" which is always ** available. -** -** If the padFlag argument is not NULL then space padding at the end -** of strings is ignored. This implements the RTRIM collation. */ static int binCollFunc( - void *padFlag, + void *NotUsed, int nKey1, const void *pKey1, int nKey2, const void *pKey2 ){ int rc, n; + UNUSED_PARAMETER(NotUsed); n = nKey1xCmp!=binCollFunc || p->pUser!=0 - || strcmp(p->zName,"BINARY")==0 ); - return p==0 || (p->xCmp==binCollFunc && p->pUser==0); + assert( p==0 || p->xCmp!=binCollFunc || strcmp(p->zName,"BINARY")==0 ); + return p==0 || p->xCmp==binCollFunc; } /* @@ -154937,11 +157474,8 @@ SQLITE_PRIVATE void sqlite3LeaveMutexAndCloseZombie(sqlite3 *db){ #ifndef SQLITE_OMIT_VIRTUALTABLE for(i=sqliteHashFirst(&db->aModule); i; i=sqliteHashNext(i)){ Module *pMod = (Module *)sqliteHashData(i); - if( pMod->xDestroy ){ - pMod->xDestroy(pMod->pAux); - } sqlite3VtabEponymousTableClear(db, pMod); - sqlite3DbFree(db, pMod); + sqlite3VtabModuleUnref(db, pMod); } sqlite3HashClear(&db->aModule); #endif @@ -155422,7 +157956,8 @@ SQLITE_PRIVATE int sqlite3CreateFunc( } assert( SQLITE_FUNC_CONSTANT==SQLITE_DETERMINISTIC ); - extraFlags = enc & SQLITE_DETERMINISTIC; + assert( SQLITE_FUNC_DIRECT==SQLITE_DIRECTONLY ); + extraFlags = enc & (SQLITE_DETERMINISTIC|SQLITE_DIRECTONLY|SQLITE_SUBTYPE); enc &= (SQLITE_FUNC_ENCMASK|SQLITE_ANY); #ifndef SQLITE_OMIT_UTF16 @@ -155485,6 +158020,7 @@ SQLITE_PRIVATE int sqlite3CreateFunc( p->u.pDestructor = pDestructor; p->funcFlags = (p->funcFlags & SQLITE_FUNC_ENCMASK) | extraFlags; testcase( p->funcFlags & SQLITE_DETERMINISTIC ); + testcase( p->funcFlags & SQLITE_DIRECTONLY ); p->xSFunc = xSFunc ? xSFunc : xStep; p->xFinalize = xFinal; p->xValue = xValue; @@ -156775,7 +159311,36 @@ static int openDatabase( db->szMmap = sqlite3GlobalConfig.szMmap; db->nextPagesize = 0; db->nMaxSorterMmap = 0x7FFFFFFF; - db->flags |= SQLITE_ShortColNames | SQLITE_EnableTrigger | SQLITE_CacheSpill + db->flags |= SQLITE_ShortColNames + | SQLITE_EnableTrigger + | SQLITE_EnableView + | SQLITE_CacheSpill + +/* The SQLITE_DQS compile-time option determines the default settings +** for SQLITE_DBCONFIG_DQS_DDL and SQLITE_DBCONFIG_DQS_DML. +** +** SQLITE_DQS SQLITE_DBCONFIG_DQS_DDL SQLITE_DBCONFIG_DQS_DML +** ---------- ----------------------- ----------------------- +** undefined on on +** 3 on on +** 2 on off +** 1 off on +** 0 off off +** +** Legacy behavior is 3 (double-quoted string literals are allowed anywhere) +** and so that is the default. But developers are encouranged to use +** -DSQLITE_DQS=0 (best) or -DSQLITE_DQS=1 (second choice) if possible. +*/ +#if !defined(SQLITE_DQS) +# define SQLITE_DQS 3 +#endif +#if (SQLITE_DQS&1)==1 + | SQLITE_DqsDML +#endif +#if (SQLITE_DQS&2)==2 + | SQLITE_DqsDDL +#endif + #if !defined(SQLITE_DEFAULT_AUTOMATIC_INDEX) || SQLITE_DEFAULT_AUTOMATIC_INDEX | SQLITE_AutoIndex #endif @@ -156826,7 +159391,7 @@ static int openDatabase( createCollation(db, sqlite3StrBINARY, SQLITE_UTF16BE, 0, binCollFunc, 0); createCollation(db, sqlite3StrBINARY, SQLITE_UTF16LE, 0, binCollFunc, 0); createCollation(db, "NOCASE", SQLITE_UTF8, 0, nocaseCollatingFunc, 0); - createCollation(db, "RTRIM", SQLITE_UTF8, (void*)1, binCollFunc, 0); + createCollation(db, "RTRIM", SQLITE_UTF8, 0, rtrimCollFunc, 0); if( db->mallocFailed ){ goto opendb_out; } @@ -157498,12 +160063,33 @@ SQLITE_API int sqlite3_test_control(int op, ...){ break; } - /* - ** Reset the PRNG back to its uninitialized state. The next call - ** to sqlite3_randomness() will reseed the PRNG using a single call - ** to the xRandomness method of the default VFS. + /* sqlite3_test_control(SQLITE_TESTCTRL_PRNG_SEED, int x, sqlite3 *db); + ** + ** Control the seed for the pseudo-random number generator (PRNG) that + ** is built into SQLite. Cases: + ** + ** x!=0 && db!=0 Seed the PRNG to the current value of the + ** schema cookie in the main database for db, or + ** x if the schema cookie is zero. This case + ** is convenient to use with database fuzzers + ** as it allows the fuzzer some control over the + ** the PRNG seed. + ** + ** x!=0 && db==0 Seed the PRNG to the value of x. + ** + ** x==0 && db==0 Revert to default behavior of using the + ** xRandomness method on the primary VFS. + ** + ** This test-control also resets the PRNG so that the new seed will + ** be used for the next call to sqlite3_randomness(). */ - case SQLITE_TESTCTRL_PRNG_RESET: { + case SQLITE_TESTCTRL_PRNG_SEED: { + int x = va_arg(ap, int); + int y; + sqlite3 *db = va_arg(ap, sqlite3*); + assert( db==0 || db->aDb[0].pSchema!=0 ); + if( db && (y = db->aDb[0].pSchema->schema_cookie)!=0 ){ x = y; } + sqlite3Config.iPrngSeed = x; sqlite3_randomness(0,0); break; } @@ -157716,6 +160302,17 @@ SQLITE_API int sqlite3_test_control(int op, ...){ break; } + /* sqlite3_test_control(SQLITE_TESTCTRL_EXTRA_SCHEMA_CHECKS, int); + ** + ** Set or clear a flag that causes SQLite to verify that type, name, + ** and tbl_name fields of the sqlite_master table. This is normally + ** on, but it is sometimes useful to turn it off for testing. + */ + case SQLITE_TESTCTRL_EXTRA_SCHEMA_CHECKS: { + sqlite3GlobalConfig.bExtraSchemaChecks = va_arg(ap, int); + break; + } + /* Set the threshold at which OP_Once counters reset back to zero. ** By default this is 0x7ffffffe (over 2 billion), but that value is ** too big to test in a reasonable amount of time, so this control is @@ -157802,6 +160399,22 @@ SQLITE_API int sqlite3_test_control(int op, ...){ break; } #endif /* defined(YYCOVERAGE) */ + + /* sqlite3_test_control(SQLITE_TESTCTRL_RESULT_INTREAL, sqlite3_context*); + ** + ** This test-control causes the most recent sqlite3_result_int64() value + ** to be interpreted as a MEM_IntReal instead of as an MEM_Int. Normally, + ** MEM_IntReal values only arise during an INSERT operation of integer + ** values into a REAL column, so they can be challenging to test. This + ** test-control enables us to write an intreal() SQL function that can + ** inject an intreal() value at arbitrary places in an SQL statement, + ** for testing purposes. + */ + case SQLITE_TESTCTRL_RESULT_INTREAL: { + sqlite3_context *pCtx = va_arg(ap, sqlite3_context*); + sqlite3ResultIntReal(pCtx); + break; + } } va_end(ap); #endif /* SQLITE_UNTESTABLE */ @@ -159646,6 +162259,18 @@ SQLITE_PRIVATE int sqlite3FtsUnicodeIsdiacritic(int); SQLITE_EXTENSION_INIT1 #endif +/* +** The following are copied from sqliteInt.h. +** +** Constants for the largest and smallest possible 64-bit signed integers. +** These macros are designed to work correctly on both 32-bit and 64-bit +** compilers. +*/ +#ifndef SQLITE_AMALGAMATION +# define LARGEST_INT64 (0xffffffff|(((sqlite3_int64)0x7fffffff)<<32)) +# define SMALLEST_INT64 (((sqlite3_int64)-1) - LARGEST_INT64) +#endif + static int fts3EvalNext(Fts3Cursor *pCsr); static int fts3EvalStart(Fts3Cursor *pCsr); static int fts3TermSegReaderCursor( @@ -161211,7 +163836,7 @@ static int fts3ScanInteriorNode( zCsr += fts3GetVarint32(zCsr, &nSuffix); assert( nPrefix>=0 && nSuffix>=0 ); - if( nPrefix>zCsr-zNode || nSuffix>zEnd-zCsr ){ + if( nPrefix>zCsr-zNode || nSuffix>zEnd-zCsr || nSuffix==0 ){ rc = FTS_CORRUPT_VTAB; goto finish_scan; } @@ -161424,10 +164049,11 @@ static void fts3ColumnlistCopy(char **pp, char **ppPoslist){ } /* -** Value used to signify the end of an position-list. This is safe because -** it is not possible to have a document with 2^31 terms. +** Value used to signify the end of an position-list. This must be +** as large or larger than any value that might appear on the +** position-list, even a position list that has been corrupted. */ -#define POSITION_LIST_END 0x7fffffff +#define POSITION_LIST_END LARGEST_INT64 /* ** This function is used to help parse position-lists. When this function is @@ -161503,14 +164129,14 @@ static int fts3PoslistMerge( fts3GetVarint32(&p1[1], &iCol1); if( iCol1==0 ) return FTS_CORRUPT_VTAB; } - else if( *p1==POS_END ) iCol1 = POSITION_LIST_END; + else if( *p1==POS_END ) iCol1 = 0x7fffffff; else iCol1 = 0; if( *p2==POS_COLUMN ){ fts3GetVarint32(&p2[1], &iCol2); if( iCol2==0 ) return FTS_CORRUPT_VTAB; } - else if( *p2==POS_END ) iCol2 = POSITION_LIST_END; + else if( *p2==POS_END ) iCol2 = 0x7fffffff; else iCol2 = 0; if( iCol1==iCol2 ){ @@ -161812,7 +164438,8 @@ static void fts3PutDeltaVarint3( iWrite = *piPrev - iVal; } assert( *pbFirst || *piPrev==0 ); - assert( *pbFirst==0 || iWrite>0 ); + assert_fts3_nc( *pbFirst==0 || iWrite>0 ); + assert( *pbFirst==0 || iWrite>=0 ); *pp += sqlite3Fts3PutVarint(*pp, iWrite); *piPrev = iVal; *pbFirst = 1; @@ -161918,6 +164545,8 @@ static int fts3DoclistOrMerge( fts3PoslistCopy(&p, &p2); fts3GetDeltaVarint3(&p2, pEnd2, bDescDoclist, &i2); } + + assert( (p-aOut)<=((p1?(p1-a1):n1)+(p2?(p2-a2):n2)+FTS3_VARINT_MAX-1) ); } if( rc!=SQLITE_OK ){ @@ -162517,18 +165146,6 @@ static int fts3NextMethod(sqlite3_vtab_cursor *pCursor){ return rc; } -/* -** The following are copied from sqliteInt.h. -** -** Constants for the largest and smallest possible 64-bit signed integers. -** These macros are designed to work correctly on both 32-bit and 64-bit -** compilers. -*/ -#ifndef SQLITE_AMALGAMATION -# define LARGEST_INT64 (0xffffffff|(((sqlite3_int64)0x7fffffff)<<32)) -# define SMALLEST_INT64 (((sqlite3_int64)-1) - LARGEST_INT64) -#endif - /* ** If the numeric type of argument pVal is "integer", then return it ** converted to a 64-bit signed integer. Otherwise, return a copy of @@ -168324,7 +170941,7 @@ static void fts3TokenizerFunc( nName = sqlite3_value_bytes(argv[0])+1; if( argc==2 ){ - if( fts3TokenizerEnabled(context) ){ + if( fts3TokenizerEnabled(context) || sqlite3_value_frombind(argv[1]) ){ void *pOld; int n = sqlite3_value_bytes(argv[1]); if( zName==0 || n!=sizeof(pPtr) ){ @@ -168351,7 +170968,9 @@ static void fts3TokenizerFunc( return; } } - sqlite3_result_blob(context, (void *)&pPtr, sizeof(pPtr), SQLITE_TRANSIENT); + if( fts3TokenizerEnabled(context) || sqlite3_value_frombind(argv[0]) ){ + sqlite3_result_blob(context, (void *)&pPtr, sizeof(pPtr), SQLITE_TRANSIENT); + } } SQLITE_PRIVATE int sqlite3Fts3IsIdChar(char c){ @@ -168439,8 +171058,8 @@ SQLITE_PRIVATE int sqlite3Fts3InitTokenizer( int iArg = 0; z = &z[n+1]; while( zzInput = sqlite3_malloc(nByte+1); + pCsr->zInput = sqlite3_malloc64(nByte+1); if( pCsr->zInput==0 ){ rc = SQLITE_NOMEM; }else{ @@ -170807,7 +173426,9 @@ static int fts3SegReaderNext( /* If iCurrentBlock>=iLeafEndBlock, this is an EOF condition. All leaf ** blocks have already been traversed. */ - assert( pReader->iCurrentBlock<=pReader->iLeafEndBlock ); +#ifdef CORRUPT_DB + assert( pReader->iCurrentBlock<=pReader->iLeafEndBlock || CORRUPT_DB ); +#endif if( pReader->iCurrentBlock>=pReader->iLeafEndBlock ){ return SQLITE_OK; } @@ -171209,8 +173830,9 @@ SQLITE_PRIVATE int sqlite3Fts3SegReaderPending( } if( nElem>0 ){ - int nByte = sizeof(Fts3SegReader) + (nElem+1)*sizeof(Fts3HashElem *); - pReader = (Fts3SegReader *)sqlite3_malloc(nByte); + sqlite3_int64 nByte; + nByte = sizeof(Fts3SegReader) + (nElem+1)*sizeof(Fts3HashElem *); + pReader = (Fts3SegReader *)sqlite3_malloc64(nByte); if( !pReader ){ rc = SQLITE_NOMEM; }else{ @@ -172075,14 +174697,14 @@ static void fts3ColumnFilter( nList -= (int)(p - pList); pList = p; - if( nList==0 ){ + if( nList<=0 ){ break; } p = &pList[1]; p += fts3GetVarint32(p, &iCurrent); } - if( bZero && &pList[nList]!=pEnd ){ + if( bZero && (pEnd - &pList[nList])>0){ memset(&pList[nList], 0, pEnd - &pList[nList]); } *ppList = pList; @@ -172694,8 +175316,10 @@ static int fts3SegmentMerge( if( rc!=SQLITE_OK ) goto finished; assert( csr.nSegment>0 ); - assert( iNewLevel>=getAbsoluteLevel(p, iLangid, iIndex, 0) ); - assert( iNewLevel=getAbsoluteLevel(p, iLangid, iIndex, 0) ); + assert_fts3_nc( + iNewLevelnColumn ); + pBlob = sqlite3_malloc64( 10*(sqlite3_int64)p->nColumn ); if( pBlob==0 ){ *pRC = SQLITE_NOMEM; return; @@ -172872,7 +175496,7 @@ static void fts3UpdateDocTotals( const int nStat = p->nColumn+2; if( *pRC ) return; - a = sqlite3_malloc( (sizeof(u32)+10)*nStat ); + a = sqlite3_malloc64( (sizeof(u32)+10)*(sqlite3_int64)nStat ); if( a==0 ){ *pRC = SQLITE_NOMEM; return; @@ -172993,8 +175617,8 @@ static int fts3DoRebuild(Fts3Table *p){ } if( rc==SQLITE_OK ){ - int nByte = sizeof(u32) * (p->nColumn+1)*3; - aSz = (u32 *)sqlite3_malloc(nByte); + sqlite3_int64 nByte = sizeof(u32) * ((sqlite3_int64)p->nColumn+1)*3; + aSz = (u32 *)sqlite3_malloc64(nByte); if( aSz==0 ){ rc = SQLITE_NOMEM; }else{ @@ -173060,12 +175684,12 @@ static int fts3IncrmergeCsr( ){ int rc; /* Return Code */ sqlite3_stmt *pStmt = 0; /* Statement used to read %_segdir entry */ - int nByte; /* Bytes allocated at pCsr->apSegment[] */ + sqlite3_int64 nByte; /* Bytes allocated at pCsr->apSegment[] */ /* Allocate space for the Fts3MultiSegReader.aCsr[] array */ memset(pCsr, 0, sizeof(*pCsr)); nByte = sizeof(Fts3SegReader *) * nSeg; - pCsr->apSegment = (Fts3SegReader **)sqlite3_malloc(nByte); + pCsr->apSegment = (Fts3SegReader **)sqlite3_malloc64(nByte); if( pCsr->apSegment==0 ){ rc = SQLITE_NOMEM; @@ -173208,7 +175832,7 @@ static int nodeReaderNext(NodeReader *p){ } p->iOff += fts3GetVarint32(&p->aNode[p->iOff], &nSuffix); - if( nPrefix>p->iOff || nSuffix>p->nNode-p->iOff ){ + if( nPrefix>p->term.n || nSuffix>p->nNode-p->iOff || nSuffix==0 ){ return FTS_CORRUPT_VTAB; } blobGrowBuffer(&p->term, nPrefix+nSuffix, &rc); @@ -173227,7 +175851,7 @@ static int nodeReaderNext(NodeReader *p){ } } - assert( p->iOff<=p->nNode ); + assert_fts3_nc( p->iOff<=p->nNode ); return rc; } @@ -173251,14 +175875,14 @@ static int nodeReaderInit(NodeReader *p, const char *aNode, int nNode){ p->nNode = nNode; /* Figure out if this is a leaf or an internal node. */ - if( p->aNode[0] ){ + if( aNode && aNode[0] ){ /* An internal node. */ p->iOff = 1 + sqlite3Fts3GetVarint(&p->aNode[1], &p->iChild); }else{ p->iOff = 1; } - return nodeReaderNext(p); + return aNode ? nodeReaderNext(p) : SQLITE_OK; } /* @@ -173388,13 +176012,14 @@ static int fts3AppendToNode( /* Node must have already been started. There must be a doclist for a ** leaf node, and there must not be a doclist for an internal node. */ assert( pNode->n>0 ); - assert( (pNode->a[0]=='\0')==(aDoclist!=0) ); + assert_fts3_nc( (pNode->a[0]=='\0')==(aDoclist!=0) ); blobGrowBuffer(pPrev, nTerm, &rc); if( rc!=SQLITE_OK ) return rc; nPrefix = fts3PrefixCompress(pPrev->a, pPrev->n, zTerm, nTerm); nSuffix = nTerm - nPrefix; + if( nSuffix<=0 ) return FTS_CORRUPT_VTAB; memcpy(pPrev->a, zTerm, nTerm); pPrev->n = nTerm; @@ -173604,7 +176229,7 @@ static int fts3TermCmp( int nCmp = MIN(nLhs, nRhs); int res; - res = memcmp(zLhs, zRhs, nCmp); + res = (nCmp ? memcmp(zLhs, zRhs, nCmp) : 0); if( res==0 ) res = nLhs - nRhs; return res; @@ -173736,34 +176361,42 @@ static int fts3IncrmergeLoad( pNode = &pWriter->aNodeWriter[nHeight]; pNode->iBlock = pWriter->iStart + pWriter->nLeafEst*nHeight; - blobGrowBuffer(&pNode->block, MAX(nRoot, p->nNodeSize), &rc); + blobGrowBuffer(&pNode->block, + MAX(nRoot, p->nNodeSize)+FTS3_NODE_PADDING, &rc + ); if( rc==SQLITE_OK ){ memcpy(pNode->block.a, aRoot, nRoot); pNode->block.n = nRoot; + memset(&pNode->block.a[nRoot], 0, FTS3_NODE_PADDING); } for(i=nHeight; i>=0 && rc==SQLITE_OK; i--){ NodeReader reader; pNode = &pWriter->aNodeWriter[i]; - rc = nodeReaderInit(&reader, pNode->block.a, pNode->block.n); - while( reader.aNode && rc==SQLITE_OK ) rc = nodeReaderNext(&reader); - blobGrowBuffer(&pNode->key, reader.term.n, &rc); - if( rc==SQLITE_OK ){ - memcpy(pNode->key.a, reader.term.a, reader.term.n); - pNode->key.n = reader.term.n; - if( i>0 ){ - char *aBlock = 0; - int nBlock = 0; - pNode = &pWriter->aNodeWriter[i-1]; - pNode->iBlock = reader.iChild; - rc = sqlite3Fts3ReadBlock(p, reader.iChild, &aBlock, &nBlock, 0); - blobGrowBuffer(&pNode->block, MAX(nBlock, p->nNodeSize), &rc); - if( rc==SQLITE_OK ){ - memcpy(pNode->block.a, aBlock, nBlock); - pNode->block.n = nBlock; + if( pNode->block.a){ + rc = nodeReaderInit(&reader, pNode->block.a, pNode->block.n); + while( reader.aNode && rc==SQLITE_OK ) rc = nodeReaderNext(&reader); + blobGrowBuffer(&pNode->key, reader.term.n, &rc); + if( rc==SQLITE_OK ){ + memcpy(pNode->key.a, reader.term.a, reader.term.n); + pNode->key.n = reader.term.n; + if( i>0 ){ + char *aBlock = 0; + int nBlock = 0; + pNode = &pWriter->aNodeWriter[i-1]; + pNode->iBlock = reader.iChild; + rc = sqlite3Fts3ReadBlock(p, reader.iChild, &aBlock, &nBlock, 0); + blobGrowBuffer(&pNode->block, + MAX(nBlock, p->nNodeSize)+FTS3_NODE_PADDING, &rc + ); + if( rc==SQLITE_OK ){ + memcpy(pNode->block.a, aBlock, nBlock); + pNode->block.n = nBlock; + memset(&pNode->block.a[nBlock], 0, FTS3_NODE_PADDING); + } + sqlite3_free(aBlock); } - sqlite3_free(aBlock); } } nodeReaderRelease(&reader); @@ -174006,7 +176639,10 @@ static int fts3TruncateNode( NodeReader reader; /* Reader object */ Blob prev = {0, 0, 0}; /* Previous term written to new node */ int rc = SQLITE_OK; /* Return code */ - int bLeaf = aNode[0]=='\0'; /* True for a leaf node */ + int bLeaf; /* True for a leaf node */ + + if( nNode<1 ) return FTS_CORRUPT_VTAB; + bLeaf = aNode[0]=='\0'; /* Allocate required output space */ blobGrowBuffer(pNew, nNode, &rc); @@ -175045,7 +177681,7 @@ SQLITE_PRIVATE int sqlite3Fts3UpdateMethod( } /* Allocate space to hold the change in document sizes */ - aSzDel = sqlite3_malloc( sizeof(aSzDel[0])*(p->nColumn+1)*2 ); + aSzDel = sqlite3_malloc64(sizeof(aSzDel[0])*((sqlite3_int64)p->nColumn+1)*2); if( aSzDel==0 ){ rc = SQLITE_NOMEM; goto update_out; @@ -175299,17 +177935,19 @@ struct StrBuffer { /* ** Allocate a two-slot MatchinfoBuffer object. */ -static MatchinfoBuffer *fts3MIBufferNew(int nElem, const char *zMatchinfo){ +static MatchinfoBuffer *fts3MIBufferNew(size_t nElem, const char *zMatchinfo){ MatchinfoBuffer *pRet; - int nByte = sizeof(u32) * (2*nElem + 1) + sizeof(MatchinfoBuffer); - int nStr = (int)strlen(zMatchinfo); + sqlite3_int64 nByte = sizeof(u32) * (2*(sqlite3_int64)nElem + 1) + + sizeof(MatchinfoBuffer); + sqlite3_int64 nStr = strlen(zMatchinfo); - pRet = sqlite3_malloc(nByte + nStr+1); + pRet = sqlite3_malloc64(nByte + nStr+1); if( pRet ){ memset(pRet, 0, nByte); pRet->aMatchinfo[0] = (u8*)(&pRet->aMatchinfo[1]) - (u8*)pRet; - pRet->aMatchinfo[1+nElem] = pRet->aMatchinfo[0] + sizeof(u32)*(nElem+1); - pRet->nElem = nElem; + pRet->aMatchinfo[1+nElem] = pRet->aMatchinfo[0] + + sizeof(u32)*((int)nElem+1); + pRet->nElem = (int)nElem; pRet->zMatchinfo = ((char*)pRet) + nByte; memcpy(pRet->zMatchinfo, zMatchinfo, nStr+1); pRet->aRef[0] = 1; @@ -175600,12 +178238,12 @@ static void fts3SnippetDetails( char *pCsr = pPhrase->pTail; int iCsr = pPhrase->iTail; - while( iCsr<(iStart+pIter->nSnippet) ){ + while( iCsr<(iStart+pIter->nSnippet) && iCsr>=iStart ){ int j; - u64 mPhrase = (u64)1 << i; + u64 mPhrase = (u64)1 << (i%64); u64 mPos = (u64)1 << (iCsr - iStart); assert( iCsr>=iStart && (iCsr - iStart)<=64 ); - assert( i>=0 && i<=64 ); + assert( i>=0 ); if( (mCover|mCovered)&mPhrase ){ iScore++; }else{ @@ -176170,8 +178808,8 @@ static int fts3MatchinfoCheck( return SQLITE_ERROR; } -static int fts3MatchinfoSize(MatchInfo *pInfo, char cArg){ - int nVal; /* Number of integers output by cArg */ +static size_t fts3MatchinfoSize(MatchInfo *pInfo, char cArg){ + size_t nVal; /* Number of integers output by cArg */ switch( cArg ){ case FTS3_MATCHINFO_NDOC: @@ -176455,7 +179093,7 @@ static int fts3MatchinfoValues( case FTS3_MATCHINFO_LHITS_BM: case FTS3_MATCHINFO_LHITS: { - int nZero = fts3MatchinfoSize(pInfo, zArg[i]) * sizeof(u32); + size_t nZero = fts3MatchinfoSize(pInfo, zArg[i]) * sizeof(u32); memset(pInfo->aMatchinfo, 0, nZero); rc = fts3ExprLHitGather(pCsr->pExpr, pInfo); break; @@ -176524,7 +179162,7 @@ static void fts3GetMatchinfo( ** initialize those elements that are constant for every row. */ if( pCsr->pMIBuffer==0 ){ - int nMatchinfo = 0; /* Number of u32 elements in match-info */ + size_t nMatchinfo = 0; /* Number of u32 elements in match-info */ int i; /* Used to iterate through zArg */ /* Determine the number of phrases in the query */ @@ -176714,7 +179352,7 @@ static int fts3ExprTermOffsetInit(Fts3Expr *pExpr, int iPhrase, void *ctx){ nTerm = pExpr->pPhrase->nToken; if( pList ){ fts3GetDeltaPosition(&pList, &iPos); - assert( iPos>=0 ); + assert_fts3_nc( iPos>=0 ); } for(iTerm=0; iTermpList) ){ pTerm->pList = 0; }else{ @@ -178766,6 +181404,7 @@ static JsonNode *jsonLookupStep( const char *zKey; JsonNode *pRoot = &pParse->aNode[iRoot]; if( zPath[0]==0 ) return pRoot; + if( pRoot->jnFlags & JNODE_REPLACE ) return 0; if( zPath[0]=='.' ){ if( pRoot->eType!=JSON_OBJECT ) return 0; zPath++; @@ -178806,7 +181445,7 @@ static JsonNode *jsonLookupStep( u32 iStart, iLabel; JsonNode *pNode; iStart = jsonParseAddNode(pParse, JSON_OBJECT, 2, 0); - iLabel = jsonParseAddNode(pParse, JSON_STRING, i, zPath); + iLabel = jsonParseAddNode(pParse, JSON_STRING, nKey, zKey); zPath += i; pNode = jsonLookupAppend(pParse, zPath, pApnd, pzErr); if( pParse->oom ) return 0; @@ -179502,7 +182141,7 @@ static void jsonArrayStep( if( pStr->zBuf==0 ){ jsonInit(pStr, ctx); jsonAppendChar(pStr, '['); - }else{ + }else if( pStr->nUsed>1 ){ jsonAppendChar(pStr, ','); pStr->pCtx = ctx; } @@ -179550,9 +182189,11 @@ static void jsonGroupInverse( int argc, sqlite3_value **argv ){ - int i; + unsigned int i; int inStr = 0; + int nNest = 0; char *z; + char c; JsonString *pStr; UNUSED_PARAM(argc); UNUSED_PARAM(argv); @@ -179563,12 +182204,18 @@ static void jsonGroupInverse( if( NEVER(!pStr) ) return; #endif z = pStr->zBuf; - for(i=1; z[i]!=',' || inStr; i++){ - assert( inUsed ); - if( z[i]=='"' ){ + for(i=1; (c = z[i])!=',' || inStr || nNest; i++){ + if( i>=pStr->nUsed ){ + pStr->nUsed = 1; + return; + } + if( c=='"' ){ inStr = !inStr; - }else if( z[i]=='\\' ){ + }else if( c=='\\' ){ i++; + }else if( !inStr ){ + if( c=='{' || c=='[' ) nNest++; + if( c=='}' || c==']' ) nNest--; } } pStr->nUsed -= i; @@ -179598,7 +182245,7 @@ static void jsonObjectStep( if( pStr->zBuf==0 ){ jsonInit(pStr, ctx); jsonAppendChar(pStr, '{'); - }else{ + }else if( pStr->nUsed>1 ){ jsonAppendChar(pStr, ','); pStr->pCtx = ctx; } @@ -180186,14 +182833,14 @@ SQLITE_PRIVATE int sqlite3Json1Init(sqlite3 *db){ #endif for(i=0; i */ -/* #include */ -/* #include */ - #ifndef SQLITE_AMALGAMATION #include "sqlite3rtree.h" typedef sqlite3_int64 i64; @@ -180301,7 +182944,17 @@ typedef sqlite3_uint64 u64; typedef unsigned char u8; typedef unsigned short u16; typedef unsigned int u32; +#if !defined(NDEBUG) && !defined(SQLITE_DEBUG) +# define NDEBUG 1 #endif +#if defined(NDEBUG) && defined(SQLITE_DEBUG) +# undef NDEBUG +#endif +#endif + +/* #include */ +/* #include */ +/* #include */ /* The following macro is used to suppress compiler warnings. */ @@ -180890,7 +183543,6 @@ static int nodeAcquire( ** increase its reference count and return it. */ if( (pNode = nodeHashLookup(pRtree, iNode))!=0 ){ - assert( !pParent || !pNode->pParent || pNode->pParent==pParent ); if( pParent && !pNode->pParent ){ if( nodeInParentChain(pNode, pParent) ){ RTREE_IS_CORRUPT(pRtree); @@ -180898,6 +183550,9 @@ static int nodeAcquire( } pParent->nRef++; pNode->pParent = pParent; + }else if( pParent && pNode->pParent && pParent!=pNode->pParent ){ + RTREE_IS_CORRUPT(pRtree); + return SQLITE_CORRUPT_VTAB; } pNode->nRef++; *ppNode = pNode; @@ -181785,13 +184440,14 @@ static int rtreeStepToLeaf(RtreeCursor *pCur){ eInt = pRtree->eCoordType==RTREE_COORD_INT32; while( (p = rtreeSearchPointFirst(pCur))!=0 && p->iLevel>0 ){ + u8 *pCellData; pNode = rtreeNodeOfFirstSearchPoint(pCur, &rc); if( rc ) return rc; nCell = NCELL(pNode); assert( nCell<200 ); + pCellData = pNode->zData + (4+pRtree->nBytesPerCell*p->iCell); while( p->iCellzData + (4+pRtree->nBytesPerCell*p->iCell); eWithin = FULLY_WITHIN; for(ii=0; iiaConstraint + ii; @@ -181804,13 +184460,23 @@ static int rtreeStepToLeaf(RtreeCursor *pCur){ }else{ rtreeNonleafConstraint(pConstraint, eInt, pCellData, &eWithin); } - if( eWithin==NOT_WITHIN ) break; + if( eWithin==NOT_WITHIN ){ + p->iCell++; + pCellData += pRtree->nBytesPerCell; + break; + } } - p->iCell++; if( eWithin==NOT_WITHIN ) continue; + p->iCell++; x.iLevel = p->iLevel - 1; if( x.iLevel ){ x.id = readInt64(pCellData); + for(ii=0; iinPoint; ii++){ + if( pCur->aPoint[ii].id==x.id ){ + RTREE_IS_CORRUPT(pRtree); + return SQLITE_CORRUPT_VTAB; + } + } x.iCell = 0; }else{ x.id = p->id; @@ -183982,49 +186648,45 @@ static int rtreeInit( ** *2 coordinates. */ static void rtreenode(sqlite3_context *ctx, int nArg, sqlite3_value **apArg){ - char *zText = 0; RtreeNode node; Rtree tree; int ii; + int nData; + int errCode; + sqlite3_str *pOut; UNUSED_PARAMETER(nArg); memset(&node, 0, sizeof(RtreeNode)); memset(&tree, 0, sizeof(Rtree)); tree.nDim = (u8)sqlite3_value_int(apArg[0]); + if( tree.nDim<1 || tree.nDim>5 ) return; tree.nDim2 = tree.nDim*2; tree.nBytesPerCell = 8 + 8 * tree.nDim; node.zData = (u8 *)sqlite3_value_blob(apArg[1]); + nData = sqlite3_value_bytes(apArg[1]); + if( nData<4 ) return; + if( nData0 ) sqlite3_str_append(pOut, " ", 1); + sqlite3_str_appendf(pOut, "{%lld", cell.iRowid); for(jj=0; jjnVertex = s.nVertex; @@ -185175,7 +187837,7 @@ static GeoPoly *geopolyBBox( if( pRc ) *pRc = SQLITE_OK; if( aCoord==0 ){ geopolyBboxFill: - pOut = sqlite3_realloc(p, GEOPOLY_SZ(4)); + pOut = sqlite3_realloc64(p, GEOPOLY_SZ(4)); if( pOut==0 ){ sqlite3_free(p); if( context ) sqlite3_result_error_nomem(context); @@ -185571,9 +188233,9 @@ static GeoSegment *geopolySortSegmentsByYAndC(GeoSegment *pList){ ** Determine the overlap between two polygons */ static int geopolyOverlap(GeoPoly *p1, GeoPoly *p2){ - int nVertex = p1->nVertex + p2->nVertex + 2; + sqlite3_int64 nVertex = p1->nVertex + p2->nVertex + 2; GeoOverlap *p; - int nByte; + sqlite3_int64 nByte; GeoEvent *pThisEvent; double rX; int rc = 0; @@ -185585,7 +188247,7 @@ static int geopolyOverlap(GeoPoly *p1, GeoPoly *p2){ nByte = sizeof(GeoEvent)*nVertex*2 + sizeof(GeoSegment)*nVertex + sizeof(GeoOverlap); - p = sqlite3_malloc( nByte ); + p = sqlite3_malloc64( nByte ); if( p==0 ) return -1; p->aEvent = (GeoEvent*)&p[1]; p->aSegment = (GeoSegment*)&p->aEvent[nVertex*2]; @@ -185744,8 +188406,8 @@ static int geopolyInit( ){ int rc = SQLITE_OK; Rtree *pRtree; - int nDb; /* Length of string argv[1] */ - int nName; /* Length of string argv[2] */ + sqlite3_int64 nDb; /* Length of string argv[1] */ + sqlite3_int64 nName; /* Length of string argv[2] */ sqlite3_str *pSql; char *zSql; int ii; @@ -185753,9 +188415,9 @@ static int geopolyInit( sqlite3_vtab_config(db, SQLITE_VTAB_CONSTRAINT_SUPPORT, 1); /* Allocate the sqlite3_vtab structure */ - nDb = (int)strlen(argv[1]); - nName = (int)strlen(argv[2]); - pRtree = (Rtree *)sqlite3_malloc(sizeof(Rtree)+nDb+nName+2); + nDb = strlen(argv[1]); + nName = strlen(argv[2]); + pRtree = (Rtree *)sqlite3_malloc64(sizeof(Rtree)+nDb+nName+2); if( !pRtree ){ return SQLITE_NOMEM; } @@ -188122,6 +190784,7 @@ SQLITE_API void sqlite3rbu_destroy_vfs(const char *zName); typedef struct RbuFrame RbuFrame; typedef struct RbuObjIter RbuObjIter; typedef struct RbuState RbuState; +typedef struct RbuSpan RbuSpan; typedef struct rbu_vfs rbu_vfs; typedef struct rbu_file rbu_file; typedef struct RbuUpdateStmt RbuUpdateStmt; @@ -188166,6 +190829,11 @@ struct RbuUpdateStmt { RbuUpdateStmt *pNext; }; +struct RbuSpan { + const char *zSpan; + int nSpan; +}; + /* ** An iterator of this type is used to iterate through all objects in ** the target database that require updating. For each such table, the @@ -188180,6 +190848,11 @@ struct RbuUpdateStmt { ** it points to an array of flags nTblCol elements in size. The flag is ** set for each column that is either a part of the PK or a part of an ** index. Or clear otherwise. +** +** If there are one or more partial indexes on the table, all fields of +** this array set set to 1. This is because in that case, the module has +** no way to tell which fields will be required to add and remove entries +** from the partial indexes. ** */ struct RbuObjIter { @@ -188210,6 +190883,9 @@ struct RbuObjIter { sqlite3_stmt *pInsert; /* Statement for INSERT operations */ sqlite3_stmt *pDelete; /* Statement for DELETE ops */ sqlite3_stmt *pTmpInsert; /* Insert into rbu_tmp_$zDataTbl */ + int nIdxCol; + RbuSpan *aIdxCol; + char *zIdxSql; /* Last UPDATE used (for PK b-tree updates only), or NULL. */ RbuUpdateStmt *pRbuUpdate; @@ -188624,6 +191300,7 @@ static void rbuFossilDeltaFunc( }else{ nOut2 = rbuDeltaApply(aOrig, nOrig, aDelta, nDelta, aOut); if( nOut2!=nOut ){ + sqlite3_free(aOut); sqlite3_result_error(context, "corrupt fossil delta", -1); }else{ sqlite3_result_blob(context, aOut, nOut, sqlite3_free); @@ -188743,6 +191420,8 @@ static void rbuObjIterClearStatements(RbuObjIter *pIter){ sqlite3_free(pUp); pUp = pTmp; } + sqlite3_free(pIter->aIdxCol); + sqlite3_free(pIter->zIdxSql); pIter->pSelect = 0; pIter->pInsert = 0; @@ -188750,6 +191429,9 @@ static void rbuObjIterClearStatements(RbuObjIter *pIter){ pIter->pRbuUpdate = 0; pIter->pTmpInsert = 0; pIter->nCol = 0; + pIter->nIdxCol = 0; + pIter->aIdxCol = 0; + pIter->zIdxSql = 0; } /* @@ -188864,6 +191546,7 @@ static void rbuTargetNameFunc( zIn = (const char*)sqlite3_value_text(argv[0]); if( zIn ){ if( rbuIsVacuum(p) ){ + assert( argc==2 || argc==1 ); if( argc==1 || 0==sqlite3_value_int(argv[1]) ){ sqlite3_result_text(pCtx, zIn, -1, SQLITE_STATIC); } @@ -188974,7 +191657,7 @@ static int rbuMPrintfExec(sqlite3rbu *p, sqlite3 *db, const char *zFmt, ...){ ** immediately without attempting the allocation or modifying the stored ** error code. */ -static void *rbuMalloc(sqlite3rbu *p, int nByte){ +static void *rbuMalloc(sqlite3rbu *p, sqlite3_int64 nByte){ void *pRet = 0; if( p->rc==SQLITE_OK ){ assert( nByte>0 ); @@ -188995,7 +191678,7 @@ static void *rbuMalloc(sqlite3rbu *p, int nByte){ ** error code in the RBU handle passed as the first argument. */ static void rbuAllocateIterArrays(sqlite3rbu *p, RbuObjIter *pIter, int nCol){ - int nByte = (2*sizeof(char*) + sizeof(int) + 3*sizeof(u8)) * nCol; + sqlite3_int64 nByte = (2*sizeof(char*) + sizeof(int) + 3*sizeof(u8)) * nCol; char **azNew; azNew = (char**)rbuMalloc(p, nByte); @@ -189022,14 +191705,15 @@ static void rbuAllocateIterArrays(sqlite3rbu *p, RbuObjIter *pIter, int nCol){ static char *rbuStrndup(const char *zStr, int *pRc){ char *zRet = 0; - assert( *pRc==SQLITE_OK ); - if( zStr ){ - size_t nCopy = strlen(zStr) + 1; - zRet = (char*)sqlite3_malloc64(nCopy); - if( zRet ){ - memcpy(zRet, zStr, nCopy); - }else{ - *pRc = SQLITE_NOMEM; + if( *pRc==SQLITE_OK ){ + if( zStr ){ + size_t nCopy = strlen(zStr) + 1; + zRet = (char*)sqlite3_malloc64(nCopy); + if( zRet ){ + memcpy(zRet, zStr, nCopy); + }else{ + *pRc = SQLITE_NOMEM; + } } } @@ -189189,14 +191873,21 @@ static void rbuObjIterCacheIndexedCols(sqlite3rbu *p, RbuObjIter *pIter){ pIter->nIndex = 0; while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pList) ){ const char *zIdx = (const char*)sqlite3_column_text(pList, 1); + int bPartial = sqlite3_column_int(pList, 4); sqlite3_stmt *pXInfo = 0; if( zIdx==0 ) break; + if( bPartial ){ + memset(pIter->abIndexed, 0x01, sizeof(u8)*pIter->nTblCol); + } p->rc = prepareFreeAndCollectError(p->dbMain, &pXInfo, &p->zErrmsg, sqlite3_mprintf("PRAGMA main.index_xinfo = %Q", zIdx) ); while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pXInfo) ){ int iCid = sqlite3_column_int(pXInfo, 1); if( iCid>=0 ) pIter->abIndexed[iCid] = 1; + if( iCid==-2 ){ + memset(pIter->abIndexed, 0x01, sizeof(u8)*pIter->nTblCol); + } } rbuFinalize(p, pXInfo); bIndex = 1; @@ -189311,7 +192002,8 @@ static int rbuObjIterCacheTableInfo(sqlite3rbu *p, RbuObjIter *pIter){ } pIter->azTblType[iOrder] = rbuStrndup(zType, &p->rc); - pIter->abTblPk[iOrder] = (iPk!=0); + assert( iPk>=0 ); + pIter->abTblPk[iOrder] = (u8)iPk; pIter->abNotNull[iOrder] = (u8)bNotNull || (iPk!=0); iOrder++; } @@ -189346,6 +192038,213 @@ static char *rbuObjIterGetCollist( return zList; } +/* +** Return a comma separated list of the quoted PRIMARY KEY column names, +** in order, for the current table. Before each column name, add the text +** zPre. After each column name, add the zPost text. Use zSeparator as +** the separator text (usually ", "). +*/ +static char *rbuObjIterGetPkList( + sqlite3rbu *p, /* RBU object */ + RbuObjIter *pIter, /* Object iterator for column names */ + const char *zPre, /* Before each quoted column name */ + const char *zSeparator, /* Separator to use between columns */ + const char *zPost /* After each quoted column name */ +){ + int iPk = 1; + char *zRet = 0; + const char *zSep = ""; + while( 1 ){ + int i; + for(i=0; inTblCol; i++){ + if( (int)pIter->abTblPk[i]==iPk ){ + const char *zCol = pIter->azTblCol[i]; + zRet = rbuMPrintf(p, "%z%s%s\"%w\"%s", zRet, zSep, zPre, zCol, zPost); + zSep = zSeparator; + break; + } + } + if( i==pIter->nTblCol ) break; + iPk++; + } + return zRet; +} + +/* +** This function is called as part of restarting an RBU vacuum within +** stage 1 of the process (while the *-oal file is being built) while +** updating a table (not an index). The table may be a rowid table or +** a WITHOUT ROWID table. It queries the target database to find the +** largest key that has already been written to the target table and +** constructs a WHERE clause that can be used to extract the remaining +** rows from the source table. For a rowid table, the WHERE clause +** is of the form: +** +** "WHERE _rowid_ > ?" +** +** and for WITHOUT ROWID tables: +** +** "WHERE (key1, key2) > (?, ?)" +** +** Instead of "?" placeholders, the actual WHERE clauses created by +** this function contain literal SQL values. +*/ +static char *rbuVacuumTableStart( + sqlite3rbu *p, /* RBU handle */ + RbuObjIter *pIter, /* RBU iterator object */ + int bRowid, /* True for a rowid table */ + const char *zWrite /* Target table name prefix */ +){ + sqlite3_stmt *pMax = 0; + char *zRet = 0; + if( bRowid ){ + p->rc = prepareFreeAndCollectError(p->dbMain, &pMax, &p->zErrmsg, + sqlite3_mprintf( + "SELECT max(_rowid_) FROM \"%s%w\"", zWrite, pIter->zTbl + ) + ); + if( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pMax) ){ + sqlite3_int64 iMax = sqlite3_column_int64(pMax, 0); + zRet = rbuMPrintf(p, " WHERE _rowid_ > %lld ", iMax); + } + rbuFinalize(p, pMax); + }else{ + char *zOrder = rbuObjIterGetPkList(p, pIter, "", ", ", " DESC"); + char *zSelect = rbuObjIterGetPkList(p, pIter, "quote(", "||','||", ")"); + char *zList = rbuObjIterGetPkList(p, pIter, "", ", ", ""); + + if( p->rc==SQLITE_OK ){ + p->rc = prepareFreeAndCollectError(p->dbMain, &pMax, &p->zErrmsg, + sqlite3_mprintf( + "SELECT %s FROM \"%s%w\" ORDER BY %s LIMIT 1", + zSelect, zWrite, pIter->zTbl, zOrder + ) + ); + if( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pMax) ){ + const char *zVal = (const char*)sqlite3_column_text(pMax, 0); + zRet = rbuMPrintf(p, " WHERE (%s) > (%s) ", zList, zVal); + } + rbuFinalize(p, pMax); + } + + sqlite3_free(zOrder); + sqlite3_free(zSelect); + sqlite3_free(zList); + } + return zRet; +} + +/* +** This function is called as part of restating an RBU vacuum when the +** current operation is writing content to an index. If possible, it +** queries the target index b-tree for the largest key already written to +** it, then composes and returns an expression that can be used in a WHERE +** clause to select the remaining required rows from the source table. +** It is only possible to return such an expression if: +** +** * The index contains no DESC columns, and +** * The last key written to the index before the operation was +** suspended does not contain any NULL values. +** +** The expression is of the form: +** +** (index-field1, index-field2, ...) > (?, ?, ...) +** +** except that the "?" placeholders are replaced with literal values. +** +** If the expression cannot be created, NULL is returned. In this case, +** the caller has to use an OFFSET clause to extract only the required +** rows from the sourct table, just as it does for an RBU update operation. +*/ +char *rbuVacuumIndexStart( + sqlite3rbu *p, /* RBU handle */ + RbuObjIter *pIter /* RBU iterator object */ +){ + char *zOrder = 0; + char *zLhs = 0; + char *zSelect = 0; + char *zVector = 0; + char *zRet = 0; + int bFailed = 0; + const char *zSep = ""; + int iCol = 0; + sqlite3_stmt *pXInfo = 0; + + p->rc = prepareFreeAndCollectError(p->dbMain, &pXInfo, &p->zErrmsg, + sqlite3_mprintf("PRAGMA main.index_xinfo = %Q", pIter->zIdx) + ); + while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pXInfo) ){ + int iCid = sqlite3_column_int(pXInfo, 1); + const char *zCollate = (const char*)sqlite3_column_text(pXInfo, 4); + const char *zCol; + if( sqlite3_column_int(pXInfo, 3) ){ + bFailed = 1; + break; + } + + if( iCid<0 ){ + if( pIter->eType==RBU_PK_IPK ){ + int i; + for(i=0; pIter->abTblPk[i]==0; i++); + assert( inTblCol ); + zCol = pIter->azTblCol[i]; + }else{ + zCol = "_rowid_"; + } + }else{ + zCol = pIter->azTblCol[iCid]; + } + + zLhs = rbuMPrintf(p, "%z%s \"%w\" COLLATE %Q", + zLhs, zSep, zCol, zCollate + ); + zOrder = rbuMPrintf(p, "%z%s \"rbu_imp_%d%w\" COLLATE %Q DESC", + zOrder, zSep, iCol, zCol, zCollate + ); + zSelect = rbuMPrintf(p, "%z%s quote(\"rbu_imp_%d%w\")", + zSelect, zSep, iCol, zCol + ); + zSep = ", "; + iCol++; + } + rbuFinalize(p, pXInfo); + if( bFailed ) goto index_start_out; + + if( p->rc==SQLITE_OK ){ + sqlite3_stmt *pSel = 0; + + p->rc = prepareFreeAndCollectError(p->dbMain, &pSel, &p->zErrmsg, + sqlite3_mprintf("SELECT %s FROM \"rbu_imp_%w\" ORDER BY %s LIMIT 1", + zSelect, pIter->zTbl, zOrder + ) + ); + if( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pSel) ){ + zSep = ""; + for(iCol=0; iColnCol; iCol++){ + const char *zQuoted = (const char*)sqlite3_column_text(pSel, iCol); + if( zQuoted[0]=='N' ){ + bFailed = 1; + break; + } + zVector = rbuMPrintf(p, "%z%s%s", zVector, zSep, zQuoted); + zSep = ", "; + } + + if( !bFailed ){ + zRet = rbuMPrintf(p, "(%s) > (%s)", zLhs, zVector); + } + } + rbuFinalize(p, pSel); + } + + index_start_out: + sqlite3_free(zOrder); + sqlite3_free(zSelect); + sqlite3_free(zVector); + sqlite3_free(zLhs); + return zRet; +} + /* ** This function is used to create a SELECT list (the list of SQL ** expressions that follows a SELECT keyword) for a SELECT statement @@ -189400,29 +192299,37 @@ static char *rbuObjIterGetIndexCols( int iCid = sqlite3_column_int(pXInfo, 1); int bDesc = sqlite3_column_int(pXInfo, 3); const char *zCollate = (const char*)sqlite3_column_text(pXInfo, 4); - const char *zCol; + const char *zCol = 0; const char *zType; - if( iCid<0 ){ - /* An integer primary key. If the table has an explicit IPK, use - ** its name. Otherwise, use "rbu_rowid". */ - if( pIter->eType==RBU_PK_IPK ){ - int i; - for(i=0; pIter->abTblPk[i]==0; i++); - assert( inTblCol ); - zCol = pIter->azTblCol[i]; - }else if( rbuIsVacuum(p) ){ - zCol = "_rowid_"; + if( iCid==-2 ){ + int iSeq = sqlite3_column_int(pXInfo, 0); + zRet = sqlite3_mprintf("%z%s(%.*s) COLLATE %Q", zRet, zCom, + pIter->aIdxCol[iSeq].nSpan, pIter->aIdxCol[iSeq].zSpan, zCollate + ); + zType = ""; + }else { + if( iCid<0 ){ + /* An integer primary key. If the table has an explicit IPK, use + ** its name. Otherwise, use "rbu_rowid". */ + if( pIter->eType==RBU_PK_IPK ){ + int i; + for(i=0; pIter->abTblPk[i]==0; i++); + assert( inTblCol ); + zCol = pIter->azTblCol[i]; + }else if( rbuIsVacuum(p) ){ + zCol = "_rowid_"; + }else{ + zCol = "rbu_rowid"; + } + zType = "INTEGER"; }else{ - zCol = "rbu_rowid"; + zCol = pIter->azTblCol[iCid]; + zType = pIter->azTblType[iCid]; } - zType = "INTEGER"; - }else{ - zCol = pIter->azTblCol[iCid]; - zType = pIter->azTblType[iCid]; + zRet = sqlite3_mprintf("%z%s\"%w\" COLLATE %Q", zRet, zCom,zCol,zCollate); } - zRet = sqlite3_mprintf("%z%s\"%w\" COLLATE %Q", zRet, zCom, zCol, zCollate); if( pIter->bUnique==0 || sqlite3_column_int(pXInfo, 5) ){ const char *zOrder = (bDesc ? " DESC" : ""); zImpPK = sqlite3_mprintf("%z%s\"rbu_imp_%d%w\"%s", @@ -189635,7 +192542,7 @@ static char *rbuObjIterGetSetlist( */ static char *rbuObjIterGetBindlist(sqlite3rbu *p, int nBind){ char *zRet = 0; - int nByte = nBind*2 + 1; + sqlite3_int64 nByte = 2*(sqlite3_int64)nBind + 1; zRet = (char*)rbuMalloc(p, nByte); if( zRet ){ @@ -189897,6 +192804,101 @@ static void rbuTmpInsertFunc( } } +static char *rbuObjIterGetIndexWhere(sqlite3rbu *p, RbuObjIter *pIter){ + sqlite3_stmt *pStmt = 0; + int rc = p->rc; + char *zRet = 0; + + assert( pIter->zIdxSql==0 && pIter->nIdxCol==0 && pIter->aIdxCol==0 ); + + if( rc==SQLITE_OK ){ + rc = prepareAndCollectError(p->dbMain, &pStmt, &p->zErrmsg, + "SELECT trim(sql) FROM sqlite_master WHERE type='index' AND name=?" + ); + } + if( rc==SQLITE_OK ){ + int rc2; + rc = sqlite3_bind_text(pStmt, 1, pIter->zIdx, -1, SQLITE_STATIC); + if( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ + char *zSql = (char*)sqlite3_column_text(pStmt, 0); + if( zSql ){ + pIter->zIdxSql = zSql = rbuStrndup(zSql, &rc); + } + if( zSql ){ + int nParen = 0; /* Number of open parenthesis */ + int i; + int iIdxCol = 0; + int nIdxAlloc = 0; + for(i=0; zSql[i]; i++){ + char c = zSql[i]; + + /* If necessary, grow the pIter->aIdxCol[] array */ + if( iIdxCol==nIdxAlloc ){ + RbuSpan *aIdxCol = (RbuSpan*)sqlite3_realloc( + pIter->aIdxCol, (nIdxAlloc+16)*sizeof(RbuSpan) + ); + if( aIdxCol==0 ){ + rc = SQLITE_NOMEM; + break; + } + pIter->aIdxCol = aIdxCol; + nIdxAlloc += 16; + } + + if( c=='(' ){ + if( nParen==0 ){ + assert( iIdxCol==0 ); + pIter->aIdxCol[0].zSpan = &zSql[i+1]; + } + nParen++; + } + else if( c==')' ){ + nParen--; + if( nParen==0 ){ + int nSpan = &zSql[i] - pIter->aIdxCol[iIdxCol].zSpan; + pIter->aIdxCol[iIdxCol++].nSpan = nSpan; + i++; + break; + } + }else if( c==',' && nParen==1 ){ + int nSpan = &zSql[i] - pIter->aIdxCol[iIdxCol].zSpan; + pIter->aIdxCol[iIdxCol++].nSpan = nSpan; + pIter->aIdxCol[iIdxCol].zSpan = &zSql[i+1]; + }else if( c=='"' || c=='\'' || c=='`' ){ + for(i++; 1; i++){ + if( zSql[i]==c ){ + if( zSql[i+1]!=c ) break; + i++; + } + } + }else if( c=='[' ){ + for(i++; 1; i++){ + if( zSql[i]==']' ) break; + } + }else if( c=='-' && zSql[i+1]=='-' ){ + for(i=i+2; zSql[i] && zSql[i]!='\n'; i++); + if( zSql[i]=='\0' ) break; + }else if( c=='/' && zSql[i+1]=='*' ){ + for(i=i+2; zSql[i] && (zSql[i]!='*' || zSql[i+1]!='/'); i++); + if( zSql[i]=='\0' ) break; + i++; + } + } + if( zSql[i] ){ + zRet = rbuStrndup(&zSql[i], &rc); + } + pIter->nIdxCol = iIdxCol; + } + } + + rc2 = sqlite3_finalize(pStmt); + if( rc==SQLITE_OK ) rc = rc2; + } + + p->rc = rc; + return zRet; +} + /* ** Ensure that the SQLite statement handles required to update the ** target database object currently indicated by the iterator passed @@ -189926,9 +192928,11 @@ static int rbuObjIterPrepareAll( char *zImposterPK = 0; /* Primary key declaration for imposter */ char *zWhere = 0; /* WHERE clause on PK columns */ char *zBind = 0; + char *zPart = 0; int nBind = 0; assert( pIter->eType!=RBU_PK_VTAB ); + zPart = rbuObjIterGetIndexWhere(p, pIter); zCollist = rbuObjIterGetIndexCols( p, pIter, &zImposterCols, &zImposterPK, &zWhere, &nBind ); @@ -189964,39 +192968,58 @@ static int rbuObjIterPrepareAll( if( p->rc==SQLITE_OK ){ char *zSql; if( rbuIsVacuum(p) ){ + char *zStart = 0; + if( nOffset ){ + zStart = rbuVacuumIndexStart(p, pIter); + if( zStart ){ + sqlite3_free(zLimit); + zLimit = 0; + } + } + zSql = sqlite3_mprintf( - "SELECT %s, 0 AS rbu_control FROM '%q' ORDER BY %s%s", + "SELECT %s, 0 AS rbu_control FROM '%q' %s %s %s ORDER BY %s%s", zCollist, pIter->zDataTbl, + zPart, + (zStart ? (zPart ? "AND" : "WHERE") : ""), zStart, zCollist, zLimit ); + sqlite3_free(zStart); }else if( pIter->eType==RBU_PK_EXTERNAL || pIter->eType==RBU_PK_NONE ){ zSql = sqlite3_mprintf( - "SELECT %s, rbu_control FROM %s.'rbu_tmp_%q' ORDER BY %s%s", + "SELECT %s, rbu_control FROM %s.'rbu_tmp_%q' %s ORDER BY %s%s", zCollist, p->zStateDb, pIter->zDataTbl, - zCollist, zLimit + zPart, zCollist, zLimit ); }else{ zSql = sqlite3_mprintf( - "SELECT %s, rbu_control FROM %s.'rbu_tmp_%q' " + "SELECT %s, rbu_control FROM %s.'rbu_tmp_%q' %s " "UNION ALL " "SELECT %s, rbu_control FROM '%q' " - "WHERE typeof(rbu_control)='integer' AND rbu_control!=1 " + "%s %s typeof(rbu_control)='integer' AND rbu_control!=1 " "ORDER BY %s%s", - zCollist, p->zStateDb, pIter->zDataTbl, + zCollist, p->zStateDb, pIter->zDataTbl, zPart, zCollist, pIter->zDataTbl, + zPart, + (zPart ? "AND" : "WHERE"), zCollist, zLimit ); } - p->rc = prepareFreeAndCollectError(p->dbRbu, &pIter->pSelect, pz, zSql); + if( p->rc==SQLITE_OK ){ + p->rc = prepareFreeAndCollectError(p->dbRbu,&pIter->pSelect,pz,zSql); + }else{ + sqlite3_free(zSql); + } } sqlite3_free(zImposterCols); sqlite3_free(zImposterPK); sqlite3_free(zWhere); sqlite3_free(zBind); + sqlite3_free(zPart); }else{ int bRbuRowid = (pIter->eType==RBU_PK_VTAB) ||(pIter->eType==RBU_PK_NONE) @@ -190089,18 +193112,42 @@ static int rbuObjIterPrepareAll( /* Create the SELECT statement to read keys from data_xxx */ if( p->rc==SQLITE_OK ){ const char *zRbuRowid = ""; + char *zStart = 0; + char *zOrder = 0; if( bRbuRowid ){ zRbuRowid = rbuIsVacuum(p) ? ",_rowid_ " : ",rbu_rowid"; } - p->rc = prepareFreeAndCollectError(p->dbRbu, &pIter->pSelect, pz, - sqlite3_mprintf( - "SELECT %s,%s rbu_control%s FROM '%q'%s", - zCollist, - (rbuIsVacuum(p) ? "0 AS " : ""), - zRbuRowid, - pIter->zDataTbl, zLimit - ) - ); + + if( rbuIsVacuum(p) ){ + if( nOffset ){ + zStart = rbuVacuumTableStart(p, pIter, bRbuRowid, zWrite); + if( zStart ){ + sqlite3_free(zLimit); + zLimit = 0; + } + } + if( bRbuRowid ){ + zOrder = rbuMPrintf(p, "_rowid_"); + }else{ + zOrder = rbuObjIterGetPkList(p, pIter, "", ", ", ""); + } + } + + if( p->rc==SQLITE_OK ){ + p->rc = prepareFreeAndCollectError(p->dbRbu, &pIter->pSelect, pz, + sqlite3_mprintf( + "SELECT %s,%s rbu_control%s FROM '%q'%s %s %s %s", + zCollist, + (rbuIsVacuum(p) ? "0 AS " : ""), + zRbuRowid, + pIter->zDataTbl, (zStart ? zStart : ""), + (zOrder ? "ORDER BY" : ""), zOrder, + zLimit + ) + ); + } + sqlite3_free(zStart); + sqlite3_free(zOrder); } sqlite3_free(zWhere); @@ -191415,10 +194462,11 @@ static void rbuIndexCntFunc( sqlite3_stmt *pStmt = 0; char *zErrmsg = 0; int rc; + sqlite3 *db = (rbuIsVacuum(p) ? p->dbRbu : p->dbMain); assert( nVal==1 ); - rc = prepareFreeAndCollectError(p->dbMain, &pStmt, &zErrmsg, + rc = prepareFreeAndCollectError(db, &pStmt, &zErrmsg, sqlite3_mprintf("SELECT count(*) FROM sqlite_master " "WHERE type='index' AND tbl_name = %Q", sqlite3_value_text(apVal[0])) ); @@ -191433,7 +194481,7 @@ static void rbuIndexCntFunc( if( rc==SQLITE_OK ){ sqlite3_result_int(pCtx, nIndex); }else{ - sqlite3_result_error(pCtx, sqlite3_errmsg(p->dbMain), -1); + sqlite3_result_error(pCtx, sqlite3_errmsg(db), -1); } } @@ -192327,9 +195375,7 @@ static int rbuVfsFileControl(sqlite3_file *pFile, int op, void *pArg){ }else if( rc==SQLITE_NOTFOUND ){ pRbu->pTargetFd = p; p->pRbu = pRbu; - if( p->openFlags & SQLITE_OPEN_MAIN_DB ){ - rbuMainlistAdd(p); - } + rbuMainlistAdd(p); if( p->pWalFd ) p->pWalFd->pRbu = pRbu; rc = SQLITE_OK; } @@ -192392,10 +195438,7 @@ static int rbuVfsShmLock(sqlite3_file *pFile, int ofst, int n, int flags){ if( ofst==WAL_LOCK_CKPT && n==1 ) rc = SQLITE_BUSY; }else{ int bCapture = 0; - if( n==1 && (flags & SQLITE_SHM_EXCLUSIVE) - && pRbu && pRbu->eStage==RBU_STAGE_CAPTURE - && (ofst==WAL_LOCK_WRITE || ofst==WAL_LOCK_CKPT || ofst==WAL_LOCK_READ0) - ){ + if( pRbu && pRbu->eStage==RBU_STAGE_CAPTURE ){ bCapture = 1; } @@ -192428,20 +195471,24 @@ static int rbuVfsShmMap( ** rbu is in the RBU_STAGE_OAL state, use heap memory for *-shm space ** instead of a file on disk. */ assert( p->openFlags & (SQLITE_OPEN_MAIN_DB|SQLITE_OPEN_TEMP_DB) ); - if( eStage==RBU_STAGE_OAL || eStage==RBU_STAGE_MOVE ){ - if( iRegion<=p->nShm ){ - int nByte = (iRegion+1) * sizeof(char*); - char **apNew = (char**)sqlite3_realloc64(p->apShm, nByte); - if( apNew==0 ){ - rc = SQLITE_NOMEM; - }else{ - memset(&apNew[p->nShm], 0, sizeof(char*) * (1 + iRegion - p->nShm)); - p->apShm = apNew; - p->nShm = iRegion+1; - } + if( eStage==RBU_STAGE_OAL ){ + sqlite3_int64 nByte = (iRegion+1) * sizeof(char*); + char **apNew = (char**)sqlite3_realloc64(p->apShm, nByte); + + /* This is an RBU connection that uses its own heap memory for the + ** pages of the *-shm file. Since no other process can have run + ** recovery, the connection must request *-shm pages in order + ** from start to finish. */ + assert( iRegion==p->nShm ); + if( apNew==0 ){ + rc = SQLITE_NOMEM; + }else{ + memset(&apNew[p->nShm], 0, sizeof(char*) * (1 + iRegion - p->nShm)); + p->apShm = apNew; + p->nShm = iRegion+1; } - if( rc==SQLITE_OK && p->apShm[iRegion]==0 ){ + if( rc==SQLITE_OK ){ char *pNew = (char*)sqlite3_malloc64(szRegion); if( pNew==0 ){ rc = SQLITE_NOMEM; @@ -192670,7 +195717,8 @@ static int rbuVfsAccess( */ if( rc==SQLITE_OK && flags==SQLITE_ACCESS_EXISTS ){ rbu_file *pDb = rbuFindMaindb(pRbuVfs, zPath, 1); - if( pDb && pDb->pRbu && pDb->pRbu->eStage==RBU_STAGE_OAL ){ + if( pDb && pDb->pRbu->eStage==RBU_STAGE_OAL ){ + assert( pDb->pRbu ); if( *pResOut ){ rc = SQLITE_CANTOPEN; }else{ @@ -194941,7 +197989,7 @@ static int sessionGrowHash(int bPatchset, SessionTable *pTab){ if( pTab->nChange==0 || pTab->nEntry>=(pTab->nChange/2) ){ int i; SessionChange **apNew; - int nNew = (pTab->nChange ? pTab->nChange : 128) * 2; + sqlite3_int64 nNew = 2*(sqlite3_int64)(pTab->nChange ? pTab->nChange : 128); apNew = (SessionChange **)sqlite3_malloc64(sizeof(SessionChange *) * nNew); if( apNew==0 ){ @@ -195663,7 +198711,9 @@ SQLITE_API int sqlite3session_diff( } sqlite3_free((char*)azCol); if( bMismatch ){ - *pzErrMsg = sqlite3_mprintf("table schemas do not match"); + if( pzErrMsg ){ + *pzErrMsg = sqlite3_mprintf("table schemas do not match"); + } rc = SQLITE_SCHEMA; } if( bHasPk==0 ){ @@ -195868,13 +198918,13 @@ SQLITE_API int sqlite3session_attach( ** If successful, return zero. Otherwise, if an OOM condition is encountered, ** set *pRc to SQLITE_NOMEM and return non-zero. */ -static int sessionBufferGrow(SessionBuffer *p, int nByte, int *pRc){ - if( *pRc==SQLITE_OK && p->nAlloc-p->nBufnAlloc-p->nBuf)nAlloc ? p->nAlloc : 128; do { nNew = nNew*2; - }while( (nNew-p->nBuf)nBuf)aBuf, nNew); if( 0==aNew ){ @@ -196986,7 +200036,7 @@ static int sessionChangesetReadTblhdr(sqlite3_changeset_iter *p){ } if( rc==SQLITE_OK ){ - int iPK = sizeof(sqlite3_value*)*p->nCol*2; + size_t iPK = sizeof(sqlite3_value*)*p->nCol*2; memset(p->tblhdr.aBuf, 0, iPK); memcpy(&p->tblhdr.aBuf[iPK], &p->in.aData[p->in.iNext], nCopy); p->in.iNext += nCopy; @@ -197901,7 +200951,7 @@ static int sessionSeekToRow( } /* -** This function is called from within sqlite3changset_apply_v2() when +** This function is called from within sqlite3changeset_apply_v2() when ** a conflict is encountered and resolved using conflict resolution ** mode eType (either SQLITE_CHANGESET_OMIT or SQLITE_CHANGESET_REPLACE).. ** It adds a conflict resolution record to the buffer in @@ -198290,7 +201340,7 @@ static int sessionRetryConstraints( rc = sessionChangesetStart(&pIter2, 0, 0, cons.nBuf, cons.aBuf, 0); if( rc==SQLITE_OK ){ - int nByte = 2*pApply->nCol*sizeof(sqlite3_value*); + size_t nByte = 2*pApply->nCol*sizeof(sqlite3_value*); int rc2; pIter2->bPatchset = bPatchset; pIter2->zTab = (char*)zTab; @@ -199683,7 +202733,7 @@ struct Fts5PhraseIter { ** Save the pointer passed as the second argument as the extension functions ** "auxiliary data". The pointer may then be retrieved by the current or any ** future invocation of the same fts5 extension function made as part of -** of the same MATCH query using the xGetAuxdata() API. +** the same MATCH query using the xGetAuxdata() API. ** ** Each extension function is allocated a single auxiliary data slot for ** each FTS query (MATCH expression). If the extension function is invoked @@ -199698,7 +202748,7 @@ struct Fts5PhraseIter { ** The xDelete callback, if one is specified, is also invoked on the ** auxiliary data pointer after the FTS5 query has finished. ** -** If an error (e.g. an OOM condition) occurs within this function, an +** If an error (e.g. an OOM condition) occurs within this function, ** the auxiliary data is set to NULL and an error code returned. If the ** xDelete parameter was not NULL, it is invoked on the auxiliary data ** pointer before returning. @@ -200275,6 +203325,7 @@ struct Fts5Config { char *zContentExprlist; Fts5Tokenizer *pTok; fts5_tokenizer *pTokApi; + int bLock; /* True when table is preparing statement */ /* Values loaded from the %_config table */ int iCookie; /* Incremented when %_config is modified */ @@ -200680,8 +203731,9 @@ static void sqlite3Fts5HashClear(Fts5Hash*); static int sqlite3Fts5HashQuery( Fts5Hash*, /* Hash table to query */ + int nPre, const char *pTerm, int nTerm, /* Query term */ - const u8 **ppDoclist, /* OUT: Pointer to doclist for pTerm */ + void **ppObj, /* OUT: Pointer to doclist for pTerm */ int *pnDoclist /* OUT: Size of doclist in bytes */ ); @@ -200790,6 +203842,7 @@ static int sqlite3Fts5ExprEof(Fts5Expr*); static i64 sqlite3Fts5ExprRowid(Fts5Expr*); static void sqlite3Fts5ExprFree(Fts5Expr*); +static int sqlite3Fts5ExprAnd(Fts5Expr **pp1, Fts5Expr *p2); /* Called during startup to register a UDF with SQLite */ static int sqlite3Fts5ExprInit(Fts5Global*, sqlite3*); @@ -201641,15 +204694,18 @@ static fts5YYACTIONTYPE fts5yy_find_shift_action( do{ i = fts5yy_shift_ofst[stateno]; assert( i>=0 ); - /* assert( i+fts5YYNFTS5TOKEN<=(int)fts5YY_NLOOKAHEAD ); */ + assert( i<=fts5YY_ACTTAB_COUNT ); + assert( i+fts5YYNFTS5TOKEN<=(int)fts5YY_NLOOKAHEAD ); assert( iLookAhead!=fts5YYNOCODE ); assert( iLookAhead < fts5YYNFTS5TOKEN ); i += iLookAhead; - if( i>=fts5YY_NLOOKAHEAD || fts5yy_lookahead[i]!=iLookAhead ){ + assert( i<(int)fts5YY_NLOOKAHEAD ); + if( fts5yy_lookahead[i]!=iLookAhead ){ #ifdef fts5YYFALLBACK fts5YYCODETYPE iFallback; /* Fallback token */ - if( iLookAhead %s\n", @@ -201664,16 +204720,8 @@ static fts5YYACTIONTYPE fts5yy_find_shift_action( #ifdef fts5YYWILDCARD { int j = i - iLookAhead + fts5YYWILDCARD; - if( -#if fts5YY_SHIFT_MIN+fts5YYWILDCARD<0 - j>=0 && -#endif -#if fts5YY_SHIFT_MAX+fts5YYWILDCARD>=fts5YY_ACTTAB_COUNT - j0 - ){ + assert( j<(int)(sizeof(fts5yy_lookahead)/sizeof(fts5yy_lookahead[0])) ); + if( fts5yy_lookahead[j]==fts5YYWILDCARD && iLookAhead>0 ){ #ifndef NDEBUG if( fts5yyTraceFILE ){ fprintf(fts5yyTraceFILE, "%sWILDCARD %s => %s\n", @@ -201687,6 +204735,7 @@ static fts5YYACTIONTYPE fts5yy_find_shift_action( #endif /* fts5YYWILDCARD */ return fts5yy_default[stateno]; }else{ + assert( i>=0 && inDocsize ) iAdj = nDocsize - nToken; if( iAdj<0 ) iAdj = 0; - *piPos = iAdj; + *piPos = (int)iAdj; } return rc; @@ -202979,7 +206027,7 @@ static int fts5Bm25GetData( if( p==0 ){ rc = SQLITE_NOMEM; }else{ - memset(p, 0, nByte); + memset(p, 0, (size_t)nByte); p->nPhrase = nPhrase; p->aIDF = (double*)&p[1]; p->aFreq = &p->aIDF[nPhrase]; @@ -203142,7 +206190,7 @@ static int sqlite3Fts5BufferSize(int *pRc, Fts5Buffer *pBuf, u32 nByte){ *pRc = SQLITE_NOMEM; return 1; }else{ - pBuf->nSpace = nNew; + pBuf->nSpace = (int)nNew; pBuf->p = pNew; } } @@ -203293,10 +206341,19 @@ static int sqlite3Fts5PoslistNext64( i64 iOff = *piOff; int iVal; fts5FastGetVarint32(a, i, iVal); - if( iVal==1 ){ + if( iVal<=1 ){ + if( iVal==0 ){ + *pi = i; + return 0; + } fts5FastGetVarint32(a, i, iVal); iOff = ((i64)iVal) << 32; fts5FastGetVarint32(a, i, iVal); + if( iVal<2 ){ + /* This is a corrupt record. So stop parsing it here. */ + *piOff = -1; + return 1; + } } *piOff = iOff + ((iVal-2) & 0x7FFFFFFF); *pi = i; @@ -203366,7 +206423,7 @@ static void *sqlite3Fts5MallocZero(int *pRc, sqlite3_int64 nByte){ if( pRet==0 ){ if( nByte>0 ) *pRc = SQLITE_NOMEM; }else{ - memset(pRet, 0, nByte); + memset(pRet, 0, (size_t)nByte); } } return pRet; @@ -203835,7 +206892,7 @@ static int fts5ConfigParseSpecial( rc = SQLITE_ERROR; }else{ rc = sqlite3Fts5GetTokenizer(pGlobal, - (const char**)azArg, nArg, &pConfig->pTok, &pConfig->pTokApi, + (const char**)azArg, (int)nArg, &pConfig->pTok, &pConfig->pTokApi, pzErr ); } @@ -203945,7 +207002,7 @@ static const char *fts5ConfigGobbleWord( if( zOut==0 ){ *pRc = SQLITE_NOMEM; }else{ - memcpy(zOut, zIn, nIn+1); + memcpy(zOut, zIn, (size_t)(nIn+1)); if( fts5_isopenquote(zOut[0]) ){ int ii = fts5Dequote(zOut); zRet = &zIn[ii]; @@ -204193,7 +207250,7 @@ static int sqlite3Fts5ConfigDeclareVtab(Fts5Config *pConfig){ rc = sqlite3_declare_vtab(pConfig->db, zSql); sqlite3_free(zSql); } - + return rc; } @@ -204781,6 +207838,42 @@ static void sqlite3Fts5ExprFree(Fts5Expr *p){ } } +static int sqlite3Fts5ExprAnd(Fts5Expr **pp1, Fts5Expr *p2){ + Fts5Parse sParse; + memset(&sParse, 0, sizeof(sParse)); + + if( *pp1 ){ + Fts5Expr *p1 = *pp1; + int nPhrase = p1->nPhrase + p2->nPhrase; + + p1->pRoot = sqlite3Fts5ParseNode(&sParse, FTS5_AND, p1->pRoot, p2->pRoot,0); + p2->pRoot = 0; + + if( sParse.rc==SQLITE_OK ){ + Fts5ExprPhrase **ap = (Fts5ExprPhrase**)sqlite3_realloc( + p1->apExprPhrase, nPhrase * sizeof(Fts5ExprPhrase*) + ); + if( ap==0 ){ + sParse.rc = SQLITE_NOMEM; + }else{ + int i; + memmove(&ap[p2->nPhrase], ap, p1->nPhrase*sizeof(Fts5ExprPhrase*)); + for(i=0; inPhrase; i++){ + ap[i] = p2->apExprPhrase[i]; + } + p1->nPhrase = nPhrase; + p1->apExprPhrase = ap; + } + } + sqlite3_free(p2->apExprPhrase); + sqlite3_free(p2); + }else{ + *pp1 = p2; + } + + return sParse.rc; +} + /* ** Argument pTerm must be a synonym iterator. Return the current rowid ** that it points to. @@ -205959,7 +209052,7 @@ static Fts5ExprNearset *sqlite3Fts5ParseNearset( if( pRet==0 ){ pParse->rc = SQLITE_NOMEM; }else{ - memset(pRet, 0, nByte); + memset(pRet, 0, (size_t)nByte); } }else if( (pNear->nPhrase % SZALLOC)==0 ){ int nNew = pNear->nPhrase + SZALLOC; @@ -206035,7 +209128,7 @@ static int fts5ParseTokenize( if( pSyn==0 ){ rc = SQLITE_NOMEM; }else{ - memset(pSyn, 0, nByte); + memset(pSyn, 0, (size_t)nByte); pSyn->zTerm = ((char*)pSyn) + sizeof(Fts5ExprTerm) + sizeof(Fts5Buffer); memcpy(pSyn->zTerm, pToken, nToken); pSyn->pSynonym = pPhrase->aTerm[pPhrase->nTerm-1].pSynonym; @@ -206195,7 +209288,7 @@ static int sqlite3Fts5ExprClonePhrase( nByte = sizeof(Fts5Colset) + (pColsetOrig->nCol-1) * sizeof(int); pColset = (Fts5Colset*)sqlite3Fts5MallocZero(&rc, nByte); if( pColset ){ - memcpy(pColset, pColsetOrig, nByte); + memcpy(pColset, pColsetOrig, (size_t)nByte); } pNew->pRoot->pNear->pColset = pColset; } @@ -206412,7 +209505,7 @@ static Fts5Colset *fts5CloneColset(int *pRc, Fts5Colset *pOrig){ sqlite3_int64 nByte = sizeof(Fts5Colset) + (pOrig->nCol-1) * sizeof(int); pRet = (Fts5Colset*)sqlite3Fts5MallocZero(pRc, nByte); if( pRet ){ - memcpy(pRet, pOrig, nByte); + memcpy(pRet, pOrig, (size_t)nByte); } }else{ pRet = 0; @@ -207429,7 +210522,7 @@ static int sqlite3Fts5HashNew(Fts5Config *pConfig, Fts5Hash **ppNew, int *pnByte *ppNew = 0; rc = SQLITE_NOMEM; }else{ - memset(pNew->aSlot, 0, nByte); + memset(pNew->aSlot, 0, (size_t)nByte); } } return rc; @@ -207513,19 +210606,25 @@ static int fts5HashResize(Fts5Hash *pHash){ return SQLITE_OK; } -static void fts5HashAddPoslistSize(Fts5Hash *pHash, Fts5HashEntry *p){ +static int fts5HashAddPoslistSize( + Fts5Hash *pHash, + Fts5HashEntry *p, + Fts5HashEntry *p2 +){ + int nRet = 0; if( p->iSzPoslist ){ - u8 *pPtr = (u8*)p; + u8 *pPtr = p2 ? (u8*)p2 : (u8*)p; + int nData = p->nData; if( pHash->eDetail==FTS5_DETAIL_NONE ){ - assert( p->nData==p->iSzPoslist ); + assert( nData==p->iSzPoslist ); if( p->bDel ){ - pPtr[p->nData++] = 0x00; + pPtr[nData++] = 0x00; if( p->bContent ){ - pPtr[p->nData++] = 0x00; + pPtr[nData++] = 0x00; } } }else{ - int nSz = (p->nData - p->iSzPoslist - 1); /* Size in bytes */ + int nSz = (nData - p->iSzPoslist - 1); /* Size in bytes */ int nPos = nSz*2 + p->bDel; /* Value of nPos field */ assert( p->bDel==0 || p->bDel==1 ); @@ -207535,14 +210634,19 @@ static void fts5HashAddPoslistSize(Fts5Hash *pHash, Fts5HashEntry *p){ int nByte = sqlite3Fts5GetVarintLen((u32)nPos); memmove(&pPtr[p->iSzPoslist + nByte], &pPtr[p->iSzPoslist + 1], nSz); sqlite3Fts5PutVarint(&pPtr[p->iSzPoslist], nPos); - p->nData += (nByte-1); + nData += (nByte-1); } } - p->iSzPoslist = 0; - p->bDel = 0; - p->bContent = 0; + nRet = nData - p->nData; + if( p2==0 ){ + p->iSzPoslist = 0; + p->bDel = 0; + p->bContent = 0; + p->nData = nData; + } } + return nRet; } /* @@ -207599,7 +210703,7 @@ static int sqlite3Fts5HashWrite( p = (Fts5HashEntry*)sqlite3_malloc64(nByte); if( !p ) return SQLITE_NOMEM; memset(p, 0, sizeof(Fts5HashEntry)); - p->nAlloc = nByte; + p->nAlloc = (int)nByte; zKey = fts5EntryKey(p); zKey[0] = bByte; memcpy(&zKey[1], pToken, nToken); @@ -207654,7 +210758,7 @@ static int sqlite3Fts5HashWrite( /* If this is a new rowid, append the 4-byte size field for the previous ** entry, and the new rowid for this entry. */ if( iRowid!=p->iRowid ){ - fts5HashAddPoslistSize(pHash, p); + fts5HashAddPoslistSize(pHash, p, 0); p->nData += sqlite3Fts5PutVarint(&pPtr[p->nData], iRowid - p->iRowid); p->iRowid = iRowid; bNew = 1; @@ -207771,7 +210875,9 @@ static int fts5HashEntrySort( for(iSlot=0; iSlotnSlot; iSlot++){ Fts5HashEntry *pIter; for(pIter=pHash->aSlot[iSlot]; pIter; pIter=pIter->pHashNext){ - if( pTerm==0 || 0==memcmp(fts5EntryKey(pIter), pTerm, nTerm) ){ + if( pTerm==0 + || (pIter->nKey+1>=nTerm && 0==memcmp(fts5EntryKey(pIter), pTerm, nTerm)) + ){ Fts5HashEntry *pEntry = pIter; pEntry->pScanNext = 0; for(i=0; ap[i]; i++){ @@ -207799,8 +210905,9 @@ static int fts5HashEntrySort( */ static int sqlite3Fts5HashQuery( Fts5Hash *pHash, /* Hash table to query */ + int nPre, const char *pTerm, int nTerm, /* Query term */ - const u8 **ppDoclist, /* OUT: Pointer to doclist for pTerm */ + void **ppOut, /* OUT: Pointer to new object */ int *pnDoclist /* OUT: Size of doclist in bytes */ ){ unsigned int iHash = fts5HashKey(pHash->nSlot, (const u8*)pTerm, nTerm); @@ -207814,11 +210921,20 @@ static int sqlite3Fts5HashQuery( } if( p ){ - fts5HashAddPoslistSize(pHash, p); - *ppDoclist = (const u8*)&zKey[nTerm+1]; - *pnDoclist = p->nData - (sizeof(Fts5HashEntry) + nTerm + 1); + int nHashPre = sizeof(Fts5HashEntry) + nTerm + 1; + int nList = p->nData - nHashPre; + u8 *pRet = (u8*)(*ppOut = sqlite3_malloc64(nPre + nList + 10)); + if( pRet ){ + Fts5HashEntry *pFaux = (Fts5HashEntry*)&pRet[nPre-nHashPre]; + memcpy(&pRet[nPre], &((u8*)p)[nHashPre], nList); + nList += fts5HashAddPoslistSize(pHash, p, pFaux); + *pnDoclist = nList; + }else{ + *pnDoclist = 0; + return SQLITE_NOMEM; + } }else{ - *ppDoclist = 0; + *ppOut = 0; *pnDoclist = 0; } @@ -207851,7 +210967,7 @@ static void sqlite3Fts5HashScanEntry( if( (p = pHash->pScan) ){ char *zKey = fts5EntryKey(p); int nTerm = (int)strlen(zKey); - fts5HashAddPoslistSize(pHash, p); + fts5HashAddPoslistSize(pHash, p, 0); *pzTerm = zKey; *ppDoclist = (const u8*)&zKey[nTerm+1]; *pnDoclist = p->nData - (sizeof(Fts5HashEntry) + nTerm + 1); @@ -208554,6 +211670,7 @@ static Fts5Data *fts5DataRead(Fts5Index *p, i64 iRowid){ }else{ /* TODO1: Fix this */ pRet->p[nByte] = 0x00; + pRet->p[nByte+1] = 0x00; pRet->szLeaf = fts5GetU16(&pRet->p[2]); } } @@ -208576,7 +211693,7 @@ static void fts5DataRelease(Fts5Data *pData){ static Fts5Data *fts5LeafRead(Fts5Index *p, i64 iRowid){ Fts5Data *pRet = fts5DataRead(p, iRowid); if( pRet ){ - if( pRet->szLeaf>pRet->nn ){ + if( pRet->nn<4 || pRet->szLeaf>pRet->nn ){ p->rc = FTS5_CORRUPT; fts5DataRelease(pRet); pRet = 0; @@ -208856,7 +211973,7 @@ static Fts5Structure *fts5StructureReadUncached(Fts5Index *p){ /* TODO: Do we need this if the leaf-index is appended? Probably... */ memset(&pData->p[pData->nn], 0, FTS5_DATA_PADDING); p->rc = fts5StructureDecode(pData->p, pData->nn, &iCookie, &pRet); - if( p->rc==SQLITE_OK && pConfig->iCookie!=iCookie ){ + if( p->rc==SQLITE_OK && (pConfig->pgsz==0 || pConfig->iCookie!=iCookie) ){ p->rc = sqlite3Fts5ConfigLoad(pConfig, iCookie); } fts5DataRelease(pData); @@ -210321,31 +213438,40 @@ static void fts5SegIterHashInit( int flags, /* Mask of FTS5INDEX_XXX flags */ Fts5SegIter *pIter /* Object to populate */ ){ - const u8 *pList = 0; int nList = 0; const u8 *z = 0; int n = 0; + Fts5Data *pLeaf = 0; assert( p->pHash ); assert( p->rc==SQLITE_OK ); if( pTerm==0 || (flags & FTS5INDEX_QUERY_SCAN) ){ + const u8 *pList = 0; + p->rc = sqlite3Fts5HashScanInit(p->pHash, (const char*)pTerm, nTerm); sqlite3Fts5HashScanEntry(p->pHash, (const char**)&z, &pList, &nList); n = (z ? (int)strlen((const char*)z) : 0); + if( pList ){ + pLeaf = fts5IdxMalloc(p, sizeof(Fts5Data)); + if( pLeaf ){ + pLeaf->p = (u8*)pList; + } + } }else{ - pIter->flags |= FTS5_SEGITER_ONETERM; - sqlite3Fts5HashQuery(p->pHash, (const char*)pTerm, nTerm, &pList, &nList); + p->rc = sqlite3Fts5HashQuery(p->pHash, sizeof(Fts5Data), + (const char*)pTerm, nTerm, (void**)&pLeaf, &nList + ); + if( pLeaf ){ + pLeaf->p = (u8*)&pLeaf[1]; + } z = pTerm; n = nTerm; + pIter->flags |= FTS5_SEGITER_ONETERM; } - if( pList ){ - Fts5Data *pLeaf; + if( pLeaf ){ sqlite3Fts5BufferSet(&p->rc, &pIter->term, n, z); - pLeaf = fts5IdxMalloc(p, sizeof(Fts5Data)); - if( pLeaf==0 ) return; - pLeaf->p = (u8*)pList; pLeaf->nn = pLeaf->szLeaf = nList; pIter->pLeaf = pLeaf; pIter->iLeafOffset = fts5GetVarint(pLeaf->p, (u64*)&pIter->iRowid); @@ -210498,8 +213624,8 @@ static int fts5MultiIterDoCompare(Fts5Iter *pIter, int iOut){ }else{ int res = fts5BufferCompare(&p1->term, &p2->term); if( res==0 ){ - assert( i2>i1 ); - assert( i2!=0 ); + assert_nc( i2>i1 ); + assert_nc( i2!=0 ); pRes->bTermEq = 1; if( p1->iRowid==p2->iRowid ){ p1->bDel = p2->bDel; @@ -211546,7 +214672,7 @@ static int fts5WriteDlidxGrow( if( aDlidx==0 ){ p->rc = SQLITE_NOMEM; }else{ - int nByte = sizeof(Fts5DlidxWriter) * (nLvl - pWriter->nDlidx); + size_t nByte = sizeof(Fts5DlidxWriter) * (nLvl - pWriter->nDlidx); memset(&aDlidx[pWriter->nDlidx], 0, nByte); pWriter->aDlidx = aDlidx; pWriter->nDlidx = nLvl; @@ -212033,13 +215159,14 @@ static void fts5TrimSegments(Fts5Index *p, Fts5Iter *pIter){ /* Set up the new page-index array */ fts5BufferAppendVarint(&p->rc, &buf, 4); if( pSeg->iLeafPgno==pSeg->iTermLeafPgno - && pSeg->iEndofDoclistszLeaf - ){ + && pSeg->iEndofDoclistszLeaf + && pSeg->iPgidxOff<=pData->nn + ){ int nDiff = pData->szLeaf - pSeg->iEndofDoclist; fts5BufferAppendVarint(&p->rc, &buf, buf.n - 1 - nDiff - 4); fts5BufferAppendBlob(&p->rc, &buf, pData->nn - pSeg->iPgidxOff, &pData->p[pSeg->iPgidxOff] - ); + ); } pSeg->pSeg->pgnoFirst = pSeg->iTermLeafPgno; @@ -212807,8 +215934,14 @@ static void fts5MergePrefixLists( ** first rowid in one input is a large negative number, and the first in ** the other a non-negative number, the delta for the non-negative ** number will be larger on disk than the literal integer value - ** was. */ - if( sqlite3Fts5BufferSize(&p->rc, &out, p1->n + p2->n + 9) ) return; + ** was. + ** + ** Or, if the input position-lists are corrupt, then the output might + ** include up to 2 extra 10-byte positions created by interpreting -1 + ** (the value PoslistNext64() uses for EOF) as a position and appending + ** it to the output. This can happen at most once for each input + ** position-list, hence two 10 byte paddings. */ + if( sqlite3Fts5BufferSize(&p->rc, &out, p1->n + p2->n + 9+10+10) ) return; fts5DoclistIterInit(p1, &i1); fts5DoclistIterInit(p2, &i2); @@ -212819,6 +215952,7 @@ static void fts5MergePrefixLists( fts5BufferSafeAppendBlob(&out, i1.aPoslist, i1.nPoslist+i1.nSize); fts5DoclistIterNext(&i1); if( i1.aPoslist==0 ) break; + assert( out.n<=((i1.aPoslist-p1->p) + (i2.aPoslist-p2->p)+9+10+10) ); } else if( i2.iRowid!=i1.iRowid ){ /* Copy entry from i2 */ @@ -212826,6 +215960,7 @@ static void fts5MergePrefixLists( fts5BufferSafeAppendBlob(&out, i2.aPoslist, i2.nPoslist+i2.nSize); fts5DoclistIterNext(&i2); if( i2.aPoslist==0 ) break; + assert( out.n<=((i1.aPoslist-p1->p) + (i2.aPoslist-p2->p)+9+10+10) ); } else{ /* Merge the two position lists. */ @@ -212842,14 +215977,17 @@ static void fts5MergePrefixLists( Fts5PoslistWriter writer; memset(&writer, 0, sizeof(writer)); + /* See the earlier comment in this function for an explanation of why + ** corrupt input position lists might cause the output to consume + ** at most 20 bytes of unexpected space. */ fts5MergeAppendDocid(&out, iLastRowid, i2.iRowid); fts5BufferZero(&tmp); - sqlite3Fts5BufferSize(&p->rc, &tmp, i1.nPoslist + i2.nPoslist); + sqlite3Fts5BufferSize(&p->rc, &tmp, i1.nPoslist + i2.nPoslist + 10 + 10); if( p->rc ) break; sqlite3Fts5PoslistNext64(a1, i1.nPoslist, &iOff1, &iPos1); sqlite3Fts5PoslistNext64(a2, i2.nPoslist, &iOff2, &iPos2); - assert( iPos1>=0 && iPos2>=0 ); + assert_nc( iPos1>=0 && iPos2>=0 ); if( iPos1=0 && iPos2>=0 ){ while( 1 ){ if( iPos1=0 && iPos2!=iPrev ); + assert_nc( iPos2>=0 && iPos2!=iPrev ); sqlite3Fts5PoslistSafeAppend(&tmp, &iPrev, iPos2); aCopy = &a2[iOff2]; nCopy = i2.nPoslist - iOff2; @@ -212893,12 +216030,19 @@ static void fts5MergePrefixLists( } /* WRITEPOSLISTSIZE */ + assert_nc( tmp.n<=i1.nPoslist+i2.nPoslist ); + assert( tmp.n<=i1.nPoslist+i2.nPoslist+10+10 ); + if( tmp.n>i1.nPoslist+i2.nPoslist ){ + if( p->rc==SQLITE_OK ) p->rc = FTS5_CORRUPT; + break; + } fts5BufferSafeAppendVarint(&out, tmp.n * 2); fts5BufferSafeAppendBlob(&out, tmp.p, tmp.n); fts5DoclistIterNext(&i1); fts5DoclistIterNext(&i2); - assert( out.n<=(p1->n+p2->n+9) ); + assert_nc( out.n<=(p1->n+p2->n+9) ); if( i1.aPoslist==0 || i2.aPoslist==0 ) break; + assert( out.n<=((i1.aPoslist-p1->p) + (i2.aPoslist-p2->p)+9+10+10) ); } } @@ -212910,7 +216054,7 @@ static void fts5MergePrefixLists( fts5MergeAppendDocid(&out, iLastRowid, i2.iRowid); fts5BufferSafeAppendBlob(&out, i2.aPoslist, i2.aEof - i2.aPoslist); } - assert( out.n<=(p1->n+p2->n+9) ); + assert_nc( out.n<=(p1->n+p2->n+9) ); fts5BufferSet(&p->rc, p1, out.n, out.p); fts5BufferFree(&tmp); @@ -214893,17 +218037,39 @@ static void fts5SetUniqueFlag(sqlite3_index_info *pIdxInfo){ ** Implementation of the xBestIndex method for FTS5 tables. Within the ** WHERE constraint, it searches for the following: ** -** 1. A MATCH constraint against the special column. +** 1. A MATCH constraint against the table column. ** 2. A MATCH constraint against the "rank" column. -** 3. An == constraint against the rowid column. -** 4. A < or <= constraint against the rowid column. -** 5. A > or >= constraint against the rowid column. +** 3. A MATCH constraint against some other column. +** 4. An == constraint against the rowid column. +** 5. A < or <= constraint against the rowid column. +** 6. A > or >= constraint against the rowid column. ** -** Within the ORDER BY, either: +** Within the ORDER BY, the following are supported: ** ** 5. ORDER BY rank [ASC|DESC] ** 6. ORDER BY rowid [ASC|DESC] ** +** Information for the xFilter call is passed via both the idxNum and +** idxStr variables. Specifically, idxNum is a bitmask of the following +** flags used to encode the ORDER BY clause: +** +** FTS5_BI_ORDER_RANK +** FTS5_BI_ORDER_ROWID +** FTS5_BI_ORDER_DESC +** +** idxStr is used to encode data from the WHERE clause. For each argument +** passed to the xFilter method, the following is appended to idxStr: +** +** Match against table column: "m" +** Match against rank column: "r" +** Match against other column: "" +** Equality constraint against the rowid: "=" +** A < or <= against the rowid: "<" +** A > or >= against the rowid: ">" +** +** This function ensures that there is at most one "r" or "=". And that if +** there exists an "=" then there is no "<" or ">". +** ** Costs are assigned as follows: ** ** a) If an unusable MATCH operator is present in the WHERE clause, the @@ -214931,32 +218097,18 @@ static int fts5BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){ Fts5Config *pConfig = pTab->pConfig; const int nCol = pConfig->nCol; int idxFlags = 0; /* Parameter passed through to xFilter() */ - int bHasMatch; - int iNext; int i; - struct Constraint { - int op; /* Mask against sqlite3_index_constraint.op */ - int fts5op; /* FTS5 mask for idxFlags */ - int iCol; /* 0==rowid, 1==tbl, 2==rank */ - int omit; /* True to omit this if found */ - int iConsIndex; /* Index in pInfo->aConstraint[] */ - } aConstraint[] = { - {SQLITE_INDEX_CONSTRAINT_MATCH|SQLITE_INDEX_CONSTRAINT_EQ, - FTS5_BI_MATCH, 1, 1, -1}, - {SQLITE_INDEX_CONSTRAINT_MATCH|SQLITE_INDEX_CONSTRAINT_EQ, - FTS5_BI_RANK, 2, 1, -1}, - {SQLITE_INDEX_CONSTRAINT_EQ, FTS5_BI_ROWID_EQ, 0, 0, -1}, - {SQLITE_INDEX_CONSTRAINT_LT|SQLITE_INDEX_CONSTRAINT_LE, - FTS5_BI_ROWID_LE, 0, 0, -1}, - {SQLITE_INDEX_CONSTRAINT_GT|SQLITE_INDEX_CONSTRAINT_GE, - FTS5_BI_ROWID_GE, 0, 0, -1}, - }; + char *idxStr; + int iIdxStr = 0; + int iCons = 0; + + int bSeenEq = 0; + int bSeenGt = 0; + int bSeenLt = 0; + int bSeenMatch = 0; + int bSeenRank = 0; - int aColMap[3]; - aColMap[0] = -1; - aColMap[1] = nCol; - aColMap[2] = nCol+1; assert( SQLITE_INDEX_CONSTRAINT_EQbLock ){ + pTab->base.zErrMsg = sqlite3_mprintf( + "recursively defined fts5 content table" + ); + return SQLITE_ERROR; + } + + idxStr = (char*)sqlite3_malloc(pInfo->nConstraint * 6 + 1); + if( idxStr==0 ) return SQLITE_NOMEM; + pInfo->idxStr = idxStr; + pInfo->needToFreeIdxStr = 1; + for(i=0; inConstraint; i++){ struct sqlite3_index_constraint *p = &pInfo->aConstraint[i]; int iCol = p->iColumn; - - if( (p->op==SQLITE_INDEX_CONSTRAINT_MATCH && iCol>=0 && iCol<=nCol) - || (p->op==SQLITE_INDEX_CONSTRAINT_EQ && iCol==nCol) + if( p->op==SQLITE_INDEX_CONSTRAINT_MATCH + || (p->op==SQLITE_INDEX_CONSTRAINT_EQ && iCol>=nCol) ){ /* A MATCH operator or equivalent */ - if( p->usable ){ - idxFlags = (idxFlags & 0xFFFF) | FTS5_BI_MATCH | (iCol << 16); - aConstraint[0].iConsIndex = i; - }else{ + if( p->usable==0 || iCol<0 ){ /* As there exists an unusable MATCH constraint this is an ** unusable plan. Set a prohibitively high cost. */ pInfo->estimatedCost = 1e50; + assert( iIdxStr < pInfo->nConstraint*6 + 1 ); + idxStr[iIdxStr] = 0; return SQLITE_OK; + }else{ + if( iCol==nCol+1 ){ + if( bSeenRank ) continue; + idxStr[iIdxStr++] = 'r'; + bSeenRank = 1; + }else{ + bSeenMatch = 1; + idxStr[iIdxStr++] = 'm'; + if( iColaConstraintUsage[i].argvIndex = ++iCons; + pInfo->aConstraintUsage[i].omit = 1; } - }else if( p->op<=SQLITE_INDEX_CONSTRAINT_MATCH ){ - int j; - for(j=1; jiCol] && (p->op & pC->op) && p->usable ){ - pC->iConsIndex = i; - idxFlags |= pC->fts5op; + } + else if( p->usable && bSeenEq==0 + && p->op==SQLITE_INDEX_CONSTRAINT_EQ && iCol<0 + ){ + idxStr[iIdxStr++] = '='; + bSeenEq = 1; + pInfo->aConstraintUsage[i].argvIndex = ++iCons; + } + } + + if( bSeenEq==0 ){ + for(i=0; inConstraint; i++){ + struct sqlite3_index_constraint *p = &pInfo->aConstraint[i]; + if( p->iColumn<0 && p->usable ){ + int op = p->op; + if( op==SQLITE_INDEX_CONSTRAINT_LT || op==SQLITE_INDEX_CONSTRAINT_LE ){ + if( bSeenLt ) continue; + idxStr[iIdxStr++] = '<'; + pInfo->aConstraintUsage[i].argvIndex = ++iCons; + bSeenLt = 1; + }else + if( op==SQLITE_INDEX_CONSTRAINT_GT || op==SQLITE_INDEX_CONSTRAINT_GE ){ + if( bSeenGt ) continue; + idxStr[iIdxStr++] = '>'; + pInfo->aConstraintUsage[i].argvIndex = ++iCons; + bSeenGt = 1; } } } } + idxStr[iIdxStr] = '\0'; /* Set idxFlags flags for the ORDER BY clause */ if( pInfo->nOrderBy==1 ){ int iSort = pInfo->aOrderBy[0].iColumn; - if( iSort==(pConfig->nCol+1) && BitFlagTest(idxFlags, FTS5_BI_MATCH) ){ + if( iSort==(pConfig->nCol+1) && bSeenMatch ){ idxFlags |= FTS5_BI_ORDER_RANK; }else if( iSort==-1 ){ idxFlags |= FTS5_BI_ORDER_ROWID; @@ -215011,26 +218208,15 @@ static int fts5BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){ } /* Calculate the estimated cost based on the flags set in idxFlags. */ - bHasMatch = BitFlagTest(idxFlags, FTS5_BI_MATCH); - if( BitFlagTest(idxFlags, FTS5_BI_ROWID_EQ) ){ - pInfo->estimatedCost = bHasMatch ? 100.0 : 10.0; - if( bHasMatch==0 ) fts5SetUniqueFlag(pInfo); - }else if( BitFlagAllTest(idxFlags, FTS5_BI_ROWID_LE|FTS5_BI_ROWID_GE) ){ - pInfo->estimatedCost = bHasMatch ? 500.0 : 250000.0; - }else if( BitFlagTest(idxFlags, FTS5_BI_ROWID_LE|FTS5_BI_ROWID_GE) ){ - pInfo->estimatedCost = bHasMatch ? 750.0 : 750000.0; + if( bSeenEq ){ + pInfo->estimatedCost = bSeenMatch ? 100.0 : 10.0; + if( bSeenMatch==0 ) fts5SetUniqueFlag(pInfo); + }else if( bSeenLt && bSeenGt ){ + pInfo->estimatedCost = bSeenMatch ? 500.0 : 250000.0; + }else if( bSeenLt || bSeenGt ){ + pInfo->estimatedCost = bSeenMatch ? 750.0 : 750000.0; }else{ - pInfo->estimatedCost = bHasMatch ? 1000.0 : 1000000.0; - } - - /* Assign argvIndex values to each constraint in use. */ - iNext = 1; - for(i=0; iiConsIndex>=0 ){ - pInfo->aConstraintUsage[pC->iConsIndex].argvIndex = iNext++; - pInfo->aConstraintUsage[pC->iConsIndex].omit = (unsigned char)pC->omit; - } + pInfo->estimatedCost = bSeenMatch ? 1000.0 : 1000000.0; } pInfo->idxNum = idxFlags; @@ -215061,7 +218247,7 @@ static int fts5OpenMethod(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr){ pCsr = (Fts5Cursor*)sqlite3_malloc64(nByte); if( pCsr ){ Fts5Global *pGlobal = pTab->pGlobal; - memset(pCsr, 0, nByte); + memset(pCsr, 0, (size_t)nByte); pCsr->aColumnSize = (int*)&pCsr[1]; pCsr->pNext = pGlobal->pCsr; pGlobal->pCsr = pCsr; @@ -215342,7 +218528,7 @@ static int fts5CursorFirstSorted( nByte = sizeof(Fts5Sorter) + sizeof(int) * (nPhrase-1); pSorter = (Fts5Sorter*)sqlite3_malloc64(nByte); if( pSorter==0 ) return SQLITE_NOMEM; - memset(pSorter, 0, nByte); + memset(pSorter, 0, (size_t)nByte); pSorter->nIdx = nPhrase; /* TODO: It would be better to have some system for reusing statement @@ -215353,7 +218539,7 @@ static int fts5CursorFirstSorted( ** ** If SQLite a built-in statement cache, this wouldn't be a problem. */ rc = fts5PrepareStatement(&pSorter->pStmt, pConfig, - "SELECT rowid, rank FROM %Q.%Q ORDER BY %s(%s%s%s) %s", + "SELECT rowid, rank FROM %Q.%Q ORDER BY %s(\"%w\"%s%s) %s", pConfig->zDb, pConfig->zName, zRank, pConfig->zName, (zRankArgs ? ", " : ""), (zRankArgs ? zRankArgs : ""), @@ -215409,10 +218595,10 @@ static int fts5SpecialMatch( assert( pTab->p.base.zErrMsg==0 ); pCsr->ePlan = FTS5_PLAN_SPECIAL; - if( 0==sqlite3_strnicmp("reads", z, n) ){ + if( n==5 && 0==sqlite3_strnicmp("reads", z, n) ){ pCsr->iSpecial = sqlite3Fts5IndexReads(pTab->p.pIndex); } - else if( 0==sqlite3_strnicmp("id", z, n) ){ + else if( n==2 && 0==sqlite3_strnicmp("id", z, n) ){ pCsr->iSpecial = pCsr->iCsrId; } else{ @@ -215553,7 +218739,7 @@ static i64 fts5GetRowidLimit(sqlite3_value *pVal, i64 iDefault){ static int fts5FilterMethod( sqlite3_vtab_cursor *pCursor, /* The cursor used for this query */ int idxNum, /* Strategy index */ - const char *zUnused, /* Unused */ + const char *idxStr, /* Unused */ int nVal, /* Number of elements in apVal */ sqlite3_value **apVal /* Arguments for the indexing scheme */ ){ @@ -215561,19 +218747,17 @@ static int fts5FilterMethod( Fts5Config *pConfig = pTab->p.pConfig; Fts5Cursor *pCsr = (Fts5Cursor*)pCursor; int rc = SQLITE_OK; /* Error code */ - int iVal = 0; /* Counter for apVal[] */ int bDesc; /* True if ORDER BY [rank|rowid] DESC */ int bOrderByRank; /* True if ORDER BY rank */ - sqlite3_value *pMatch = 0; /* MATCH ? expression (or NULL) */ sqlite3_value *pRank = 0; /* rank MATCH ? expression (or NULL) */ sqlite3_value *pRowidEq = 0; /* rowid = ? expression (or NULL) */ sqlite3_value *pRowidLe = 0; /* rowid <= ? expression (or NULL) */ sqlite3_value *pRowidGe = 0; /* rowid >= ? expression (or NULL) */ int iCol; /* Column on LHS of MATCH operator */ char **pzErrmsg = pConfig->pzErrmsg; - - UNUSED_PARAM(zUnused); - UNUSED_PARAM(nVal); + int i; + int iIdxStr = 0; + Fts5Expr *pExpr = 0; if( pCsr->ePlan ){ fts5FreeCursorComponents(pCsr); @@ -215586,23 +218770,60 @@ static int fts5FilterMethod( assert( pCsr->pRank==0 ); assert( pCsr->zRank==0 ); assert( pCsr->zRankArgs==0 ); + assert( pTab->pSortCsr==0 || nVal==0 ); assert( pzErrmsg==0 || pzErrmsg==&pTab->p.base.zErrMsg ); pConfig->pzErrmsg = &pTab->p.base.zErrMsg; - /* Decode the arguments passed through to this function. - ** - ** Note: The following set of if(...) statements must be in the same - ** order as the corresponding entries in the struct at the top of - ** fts5BestIndexMethod(). */ - if( BitFlagTest(idxNum, FTS5_BI_MATCH) ) pMatch = apVal[iVal++]; - if( BitFlagTest(idxNum, FTS5_BI_RANK) ) pRank = apVal[iVal++]; - if( BitFlagTest(idxNum, FTS5_BI_ROWID_EQ) ) pRowidEq = apVal[iVal++]; - if( BitFlagTest(idxNum, FTS5_BI_ROWID_LE) ) pRowidLe = apVal[iVal++]; - if( BitFlagTest(idxNum, FTS5_BI_ROWID_GE) ) pRowidGe = apVal[iVal++]; - iCol = (idxNum>>16); - assert( iCol>=0 && iCol<=pConfig->nCol ); - assert( iVal==nVal ); + /* Decode the arguments passed through to this function. */ + for(i=0; i='0' && idxStr[iIdxStr]<='9' ){ + iCol = 0; + do{ + iCol = iCol*10 + (idxStr[iIdxStr]-'0'); + iIdxStr++; + }while( idxStr[iIdxStr]>='0' && idxStr[iIdxStr]<='9' ); + }else{ + iCol = pConfig->nCol; + } + + if( zText[0]=='*' ){ + /* The user has issued a query of the form "MATCH '*...'". This + ** indicates that the MATCH expression is not a full text query, + ** but a request for an internal parameter. */ + rc = fts5SpecialMatch(pTab, pCsr, &zText[1]); + goto filter_out; + }else{ + char **pzErr = &pTab->p.base.zErrMsg; + rc = sqlite3Fts5ExprNew(pConfig, iCol, zText, &pExpr, pzErr); + if( rc==SQLITE_OK ){ + rc = sqlite3Fts5ExprAnd(&pCsr->pExpr, pExpr); + pExpr = 0; + } + if( rc!=SQLITE_OK ) goto filter_out; + } + + break; + } + case '=': + pRowidEq = apVal[i]; + break; + case '<': + pRowidLe = apVal[i]; + break; + default: assert( idxStr[iIdxStr-1]=='>' ); + pRowidGe = apVal[i]; + break; + } + } bOrderByRank = ((idxNum & FTS5_BI_ORDER_RANK) ? 1 : 0); pCsr->bDesc = bDesc = ((idxNum & FTS5_BI_ORDER_DESC) ? 1 : 0); @@ -215629,7 +218850,7 @@ static int fts5FilterMethod( ** (pCursor) is used to execute the query issued by function ** fts5CursorFirstSorted() above. */ assert( pRowidEq==0 && pRowidLe==0 && pRowidGe==0 && pRank==0 ); - assert( nVal==0 && pMatch==0 && bOrderByRank==0 && bDesc==0 ); + assert( nVal==0 && bOrderByRank==0 && bDesc==0 ); assert( pCsr->iLastRowid==LARGEST_INT64 ); assert( pCsr->iFirstRowid==SMALLEST_INT64 ); if( pTab->pSortCsr->bDesc ){ @@ -215642,29 +218863,15 @@ static int fts5FilterMethod( pCsr->ePlan = FTS5_PLAN_SOURCE; pCsr->pExpr = pTab->pSortCsr->pExpr; rc = fts5CursorFirst(pTab, pCsr, bDesc); - }else if( pMatch ){ - const char *zExpr = (const char*)sqlite3_value_text(apVal[0]); - if( zExpr==0 ) zExpr = ""; - + }else if( pCsr->pExpr ){ rc = fts5CursorParseRank(pConfig, pCsr, pRank); if( rc==SQLITE_OK ){ - if( zExpr[0]=='*' ){ - /* The user has issued a query of the form "MATCH '*...'". This - ** indicates that the MATCH expression is not a full text query, - ** but a request for an internal parameter. */ - rc = fts5SpecialMatch(pTab, pCsr, &zExpr[1]); + if( bOrderByRank ){ + pCsr->ePlan = FTS5_PLAN_SORTED_MATCH; + rc = fts5CursorFirstSorted(pTab, pCsr, bDesc); }else{ - char **pzErr = &pTab->p.base.zErrMsg; - rc = sqlite3Fts5ExprNew(pConfig, iCol, zExpr, &pCsr->pExpr, pzErr); - if( rc==SQLITE_OK ){ - if( bOrderByRank ){ - pCsr->ePlan = FTS5_PLAN_SORTED_MATCH; - rc = fts5CursorFirstSorted(pTab, pCsr, bDesc); - }else{ - pCsr->ePlan = FTS5_PLAN_MATCH; - rc = fts5CursorFirst(pTab, pCsr, bDesc); - } - } + pCsr->ePlan = FTS5_PLAN_MATCH; + rc = fts5CursorFirst(pTab, pCsr, bDesc); } } }else if( pConfig->zContent==0 ){ @@ -215681,7 +218888,7 @@ static int fts5FilterMethod( ); if( rc==SQLITE_OK ){ if( pCsr->ePlan==FTS5_PLAN_ROWID ){ - sqlite3_bind_value(pCsr->pStmt, 1, apVal[0]); + sqlite3_bind_value(pCsr->pStmt, 1, pRowidEq); }else{ sqlite3_bind_int64(pCsr->pStmt, 1, pCsr->iFirstRowid); sqlite3_bind_int64(pCsr->pStmt, 2, pCsr->iLastRowid); @@ -215690,6 +218897,8 @@ static int fts5FilterMethod( } } + filter_out: + sqlite3Fts5ExprFree(pExpr); pConfig->pzErrmsg = pzErrmsg; return rc; } @@ -216660,7 +219869,7 @@ static void fts5ApiCallback( iCsrId = sqlite3_value_int64(argv[0]); pCsr = fts5CursorFromCsrid(pAux->pGlobal, iCsrId); - if( pCsr==0 ){ + if( pCsr==0 || pCsr->ePlan==0 ){ char *zErr = sqlite3_mprintf("no such cursor: %lld", iCsrId); sqlite3_result_error(context, zErr, -1); sqlite3_free(zErr); @@ -216896,14 +220105,14 @@ static int fts5CreateAux( int rc = sqlite3_overload_function(pGlobal->db, zName, -1); if( rc==SQLITE_OK ){ Fts5Auxiliary *pAux; - int nName; /* Size of zName in bytes, including \0 */ - int nByte; /* Bytes of space to allocate */ + sqlite3_int64 nName; /* Size of zName in bytes, including \0 */ + sqlite3_int64 nByte; /* Bytes of space to allocate */ - nName = (int)strlen(zName) + 1; + nName = strlen(zName) + 1; nByte = sizeof(Fts5Auxiliary) + nName; - pAux = (Fts5Auxiliary*)sqlite3_malloc(nByte); + pAux = (Fts5Auxiliary*)sqlite3_malloc64(nByte); if( pAux ){ - memset(pAux, 0, nByte); + memset(pAux, 0, (size_t)nByte); pAux->zFunc = (char*)&pAux[1]; memcpy(pAux->zFunc, zName, nName); pAux->pGlobal = pGlobal; @@ -216933,15 +220142,15 @@ static int fts5CreateTokenizer( ){ Fts5Global *pGlobal = (Fts5Global*)pApi; Fts5TokenizerModule *pNew; - int nName; /* Size of zName and its \0 terminator */ - int nByte; /* Bytes of space to allocate */ + sqlite3_int64 nName; /* Size of zName and its \0 terminator */ + sqlite3_int64 nByte; /* Bytes of space to allocate */ int rc = SQLITE_OK; - nName = (int)strlen(zName) + 1; + nName = strlen(zName) + 1; nByte = sizeof(Fts5TokenizerModule) + nName; - pNew = (Fts5TokenizerModule*)sqlite3_malloc(nByte); + pNew = (Fts5TokenizerModule*)sqlite3_malloc64(nByte); if( pNew ){ - memset(pNew, 0, nByte); + memset(pNew, 0, (size_t)nByte); pNew->zName = (char*)&pNew[1]; memcpy(pNew->zName, zName, nName); pNew->pUserData = pUserData; @@ -217076,7 +220285,7 @@ static void fts5SourceIdFunc( ){ assert( nArg==0 ); UNUSED_PARAM2(nArg, apUnused); - sqlite3_result_text(pCtx, "fts5: 2019-02-08 13:17:39 0eca3dd3d38b31c92b49ca2d311128b74584714d9e7de895b1a6286ef959a1dd", -1, SQLITE_TRANSIENT); + sqlite3_result_text(pCtx, "fts5: 2019-10-10 20:19:45 18db032d058f1436ce3dea84081f4ee5a0f2259ad97301d43c426bc7f3df1b0b", -1, SQLITE_TRANSIENT); } /* @@ -217348,7 +220557,9 @@ static int fts5StorageGetStmt( }else{ int f = SQLITE_PREPARE_PERSISTENT; if( eStmt>FTS5_STMT_LOOKUP ) f |= SQLITE_PREPARE_NO_VTAB; + p->pConfig->bLock++; rc = sqlite3_prepare_v3(pC->db, zSql, -1, f, &p->aStmt[eStmt], 0); + p->pConfig->bLock--; sqlite3_free(zSql); if( rc!=SQLITE_OK && pzErrMsg ){ *pzErrMsg = sqlite3_mprintf("%s", sqlite3_errmsg(pC->db)); @@ -217499,7 +220710,7 @@ static int sqlite3Fts5StorageOpen( *pp = p = (Fts5Storage*)sqlite3_malloc64(nByte); if( !p ) return SQLITE_NOMEM; - memset(p, 0, nByte); + memset(p, 0, (size_t)nByte); p->aTotalSize = (i64*)&p[1]; p->pConfig = pConfig; p->pIndex = pIndex; @@ -218721,7 +221932,7 @@ static int fts5UnicodeCreate( p->eRemoveDiacritic = FTS5_REMOVE_DIACRITICS_SIMPLE; p->nFold = 64; - p->aFold = sqlite3_malloc(p->nFold * sizeof(char)); + p->aFold = sqlite3_malloc64(p->nFold * sizeof(char)); if( p->aFold==0 ){ rc = SQLITE_NOMEM; } @@ -220409,7 +223620,7 @@ static void sqlite3Fts5UnicodeAscii(u8 *aArray, u8 *aAscii){ int bToken = aArray[ aFts5UnicodeData[iTbl] & 0x1F ]; int n = (aFts5UnicodeData[iTbl] >> 5) + i; for(; i<128 && ibEof==0 && pTab->eType==FTS5_VOCAB_COL ){ - while( pCsr->aDoc[pCsr->iCol]==0 ) pCsr->iCol++; - assert( pCsr->iColpFts5->pConfig->nCol ); + for(/* noop */; pCsr->iColaDoc[pCsr->iCol]==0; pCsr->iCol++); + if( pCsr->iCol==nCol ){ + rc = FTS5_CORRUPT; + } } return rc; } @@ -221840,9 +225053,9 @@ SQLITE_API int sqlite3_stmt_init( #endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_STMTVTAB) */ /************** End of stmt.c ************************************************/ -#if __LINE__!=221843 +#if __LINE__!=225056 #undef SQLITE_SOURCE_ID -#define SQLITE_SOURCE_ID "2019-02-08 13:17:39 0eca3dd3d38b31c92b49ca2d311128b74584714d9e7de895b1a6286ef959alt2" +#define SQLITE_SOURCE_ID "2019-10-10 20:19:45 18db032d058f1436ce3dea84081f4ee5a0f2259ad97301d43c426bc7f3dfalt2" #endif /* Return the source-id for this library */ SQLITE_API const char *sqlite3_sourceid(void){ return SQLITE_SOURCE_ID; } diff --git a/sqlite3.h b/src/database/sqlite3.h similarity index 98% rename from sqlite3.h rename to src/database/sqlite3.h index 686aa8b73..37bfac528 100644 --- a/sqlite3.h +++ b/src/database/sqlite3.h @@ -123,9 +123,9 @@ extern "C" { ** [sqlite3_libversion_number()], [sqlite3_sourceid()], ** [sqlite_version()] and [sqlite_source_id()]. */ -#define SQLITE_VERSION "3.27.1" -#define SQLITE_VERSION_NUMBER 3027001 -#define SQLITE_SOURCE_ID "2019-02-08 13:17:39 0eca3dd3d38b31c92b49ca2d311128b74584714d9e7de895b1a6286ef959a1dd" +#define SQLITE_VERSION "3.30.1" +#define SQLITE_VERSION_NUMBER 3030001 +#define SQLITE_SOURCE_ID "2019-10-10 20:19:45 18db032d058f1436ce3dea84081f4ee5a0f2259ad97301d43c426bc7f3df1b0b" /* ** CAPI3REF: Run-Time Library Version Numbers @@ -189,6 +189,9 @@ SQLITE_API int sqlite3_libversion_number(void); #ifndef SQLITE_OMIT_COMPILEOPTION_DIAGS SQLITE_API int sqlite3_compileoption_used(const char *zOptName); SQLITE_API const char *sqlite3_compileoption_get(int N); +#else +# define sqlite3_compileoption_used(X) 0 +# define sqlite3_compileoption_get(X) ((void*)0) #endif /* @@ -1293,8 +1296,14 @@ typedef struct sqlite3_api_routines sqlite3_api_routines; ** ^The flags argument to xAccess() may be [SQLITE_ACCESS_EXISTS] ** to test for the existence of a file, or [SQLITE_ACCESS_READWRITE] to ** test whether a file is readable and writable, or [SQLITE_ACCESS_READ] -** to test whether a file is at least readable. The file can be a -** directory. +** to test whether a file is at least readable. The SQLITE_ACCESS_READ +** flag is never actually used and is not implemented in the built-in +** VFSes of SQLite. The file is named by the second argument and can be a +** directory. The xAccess method returns [SQLITE_OK] on success or some +** non-zero error code if there is an I/O error or if the name of +** the file given in the second argument is illegal. If SQLITE_OK +** is returned, then non-zero or zero is written into *pResOut to indicate +** whether or not the file is accessible. ** ** ^SQLite will always allocate at least mxPathname+1 bytes for the ** output buffer xFullPathname. The exact size of the output buffer @@ -2084,10 +2093,21 @@ struct sqlite3_mem_methods { ** following this call. The second parameter may be a NULL pointer, in ** which case the trigger setting is not reported back. ** +** [[SQLITE_DBCONFIG_ENABLE_VIEW]] +**
SQLITE_DBCONFIG_ENABLE_VIEW
+**
^This option is used to enable or disable [CREATE VIEW | views]. +** There should be two additional arguments. +** The first argument is an integer which is 0 to disable views, +** positive to enable views or negative to leave the setting unchanged. +** The second parameter is a pointer to an integer into which +** is written 0 or 1 to indicate whether views are disabled or enabled +** following this call. The second parameter may be a NULL pointer, in +** which case the view setting is not reported back.
+** ** [[SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER]] **
SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER
-**
^This option is used to enable or disable the two-argument -** version of the [fts3_tokenizer()] function which is part of the +**
^This option is used to enable or disable the +** [fts3_tokenizer()] function which is part of the ** [FTS3] full-text search engine extension. ** There should be two additional arguments. ** The first argument is an integer which is 0 to disable fts3_tokenizer() or @@ -2195,10 +2215,50 @@ struct sqlite3_mem_methods { ** features include but are not limited to the following: **
    **
  • The [PRAGMA writable_schema=ON] statement. +**
  • The [PRAGMA journal_mode=OFF] statement. **
  • Writes to the [sqlite_dbpage] virtual table. **
  • Direct writes to [shadow tables]. **
**
+** +** [[SQLITE_DBCONFIG_WRITABLE_SCHEMA]]
SQLITE_DBCONFIG_WRITABLE_SCHEMA
+**
The SQLITE_DBCONFIG_WRITABLE_SCHEMA option activates or deactivates the +** "writable_schema" flag. This has the same effect and is logically equivalent +** to setting [PRAGMA writable_schema=ON] or [PRAGMA writable_schema=OFF]. +** The first argument to this setting is an integer which is 0 to disable +** the writable_schema, positive to enable writable_schema, or negative to +** leave the setting unchanged. The second parameter is a pointer to an +** integer into which is written 0 or 1 to indicate whether the writable_schema +** is enabled or disabled following this call. +**
+** +** [[SQLITE_DBCONFIG_LEGACY_ALTER_TABLE]] +**
SQLITE_DBCONFIG_LEGACY_ALTER_TABLE
+**
The SQLITE_DBCONFIG_LEGACY_ALTER_TABLE option activates or deactivates +** the legacy behavior of the [ALTER TABLE RENAME] command such it +** behaves as it did prior to [version 3.24.0] (2018-06-04). See the +** "Compatibility Notice" on the [ALTER TABLE RENAME documentation] for +** additional information. This feature can also be turned on and off +** using the [PRAGMA legacy_alter_table] statement. +**
+** +** [[SQLITE_DBCONFIG_DQS_DML]] +**
SQLITE_DBCONFIG_DQS_DML +**
The SQLITE_DBCONFIG_DQS_DML option activates or deactivates +** the legacy [double-quoted string literal] misfeature for DML statement +** only, that is DELETE, INSERT, SELECT, and UPDATE statements. The +** default value of this setting is determined by the [-DSQLITE_DQS] +** compile-time option. +**
+** +** [[SQLITE_DBCONFIG_DQS_DDL]] +**
SQLITE_DBCONFIG_DQS_DDL +**
The SQLITE_DBCONFIG_DQS option activates or deactivates +** the legacy [double-quoted string literal] misfeature for DDL statements, +** such as CREATE TABLE and CREATE INDEX. The +** default value of this setting is determined by the [-DSQLITE_DQS] +** compile-time option. +**
** */ #define SQLITE_DBCONFIG_MAINDBNAME 1000 /* const char* */ @@ -2212,7 +2272,12 @@ struct sqlite3_mem_methods { #define SQLITE_DBCONFIG_TRIGGER_EQP 1008 /* int int* */ #define SQLITE_DBCONFIG_RESET_DATABASE 1009 /* int int* */ #define SQLITE_DBCONFIG_DEFENSIVE 1010 /* int int* */ -#define SQLITE_DBCONFIG_MAX 1010 /* Largest DBCONFIG */ +#define SQLITE_DBCONFIG_WRITABLE_SCHEMA 1011 /* int int* */ +#define SQLITE_DBCONFIG_LEGACY_ALTER_TABLE 1012 /* int int* */ +#define SQLITE_DBCONFIG_DQS_DML 1013 /* int int* */ +#define SQLITE_DBCONFIG_DQS_DDL 1014 /* int int* */ +#define SQLITE_DBCONFIG_ENABLE_VIEW 1015 /* int int* */ +#define SQLITE_DBCONFIG_MAX 1015 /* Largest DBCONFIG */ /* ** CAPI3REF: Enable Or Disable Extended Result Codes @@ -2369,7 +2434,7 @@ SQLITE_API int sqlite3_changes(sqlite3*); ** not. ^Changes to a view that are intercepted by INSTEAD OF triggers ** are not counted. ** -** This the [sqlite3_total_changes(D)] interface only reports the number +** The [sqlite3_total_changes(D)] interface only reports the number ** of rows that changed due to SQL statement run against database ** connection D. Any changes by other database connections are ignored. ** To detect changes against a database file from other database @@ -3761,7 +3826,7 @@ SQLITE_API int sqlite3_limit(sqlite3*, int id, int newVal); ** ^The specific value of WHERE-clause [parameter] might influence the ** choice of query plan if the parameter is the left-hand side of a [LIKE] ** or [GLOB] operator or if the parameter is compared to an indexed column -** and the [SQLITE_ENABLE_STAT3] compile-time option is enabled. +** and the [SQLITE_ENABLE_STAT4] compile-time option is enabled. ** ** ** @@ -3894,6 +3959,18 @@ SQLITE_API const char *sqlite3_normalized_sql(sqlite3_stmt *pStmt); */ SQLITE_API int sqlite3_stmt_readonly(sqlite3_stmt *pStmt); +/* +** CAPI3REF: Query The EXPLAIN Setting For A Prepared Statement +** METHOD: sqlite3_stmt +** +** ^The sqlite3_stmt_isexplain(S) interface returns 1 if the +** prepared statement S is an EXPLAIN statement, or 2 if the +** statement S is an EXPLAIN QUERY PLAN. +** ^The sqlite3_stmt_isexplain(S) interface returns 0 if S is +** an ordinary statement or a NULL pointer. +*/ +SQLITE_API int sqlite3_stmt_isexplain(sqlite3_stmt *pStmt); + /* ** CAPI3REF: Determine If A Prepared Statement Has Been Reset ** METHOD: sqlite3_stmt @@ -4033,7 +4110,9 @@ typedef struct sqlite3_context sqlite3_context; ** ^The fifth argument to the BLOB and string binding interfaces ** is a destructor used to dispose of the BLOB or ** string after SQLite has finished with it. ^The destructor is called -** to dispose of the BLOB or string even if the call to bind API fails. +** to dispose of the BLOB or string even if the call to the bind API fails, +** except the destructor is not called if the third parameter is a NULL +** pointer or the fourth parameter is negative. ** ^If the fifth argument is ** the special value [SQLITE_STATIC], then SQLite assumes that the ** information is in static, unmanaged space and does not need to be freed. @@ -4782,6 +4861,12 @@ SQLITE_API int sqlite3_reset(sqlite3_stmt *pStmt); ** perform additional optimizations on deterministic functions, so use ** of the [SQLITE_DETERMINISTIC] flag is recommended where possible. ** +** ^The fourth parameter may also optionally include the [SQLITE_DIRECTONLY] +** flag, which if present prevents the function from being invoked from +** within VIEWs or TRIGGERs. For security reasons, the [SQLITE_DIRECTONLY] +** flag is recommended for any application-defined SQL function that has +** side-effects. +** ** ^(The fifth parameter is an arbitrary pointer. The implementation of the ** function can gain access to this pointer using [sqlite3_user_data()].)^ ** @@ -4898,8 +4983,30 @@ SQLITE_API int sqlite3_create_window_function( ** [SQLITE_UTF8 | preferred text encoding] as the fourth argument ** to [sqlite3_create_function()], [sqlite3_create_function16()], or ** [sqlite3_create_function_v2()]. +** +** The SQLITE_DETERMINISTIC flag means that the new function will always +** maps the same inputs into the same output. The abs() function is +** deterministic, for example, but randomblob() is not. +** +** The SQLITE_DIRECTONLY flag means that the function may only be invoked +** from top-level SQL, and cannot be used in VIEWs or TRIGGERs. This is +** a security feature which is recommended for all +** [application-defined SQL functions] that have side-effects. This flag +** prevents an attacker from adding triggers and views to a schema then +** tricking a high-privilege application into causing unintended side-effects +** while performing ordinary queries. +** +** The SQLITE_SUBTYPE flag indicates to SQLite that a function may call +** [sqlite3_value_subtype()] to inspect the sub-types of its arguments. +** Specifying this flag makes no difference for scalar or aggregate user +** functions. However, if it is not specified for a user-defined window +** function, then any sub-types belonging to arguments passed to the window +** function may be discarded before the window function is called (i.e. +** sqlite3_value_subtype() will always return 0). */ -#define SQLITE_DETERMINISTIC 0x800 +#define SQLITE_DETERMINISTIC 0x000000800 +#define SQLITE_DIRECTONLY 0x000080000 +#define SQLITE_SUBTYPE 0x000100000 /* ** CAPI3REF: Deprecated Functions @@ -4950,6 +5057,8 @@ SQLITE_API SQLITE_DEPRECATED int sqlite3_memory_alarm(void(*)(void*,sqlite3_int6 ** sqlite3_value_nochange   ** →  True if the column is unchanged in an UPDATE ** against a virtual table. +** sqlite3_value_frombind   +** →  True if value originated from a [bound parameter] ** ** ** Details: @@ -5011,6 +5120,11 @@ SQLITE_API SQLITE_DEPRECATED int sqlite3_memory_alarm(void(*)(void*,sqlite3_int6 ** than within an [xUpdate] method call for an UPDATE statement, then ** the return value is arbitrary and meaningless. ** +** ^The sqlite3_value_frombind(X) interface returns non-zero if the +** value X originated from one of the [sqlite3_bind_int|sqlite3_bind()] +** interfaces. ^If X comes from an SQL literal value, or a table column, +** and expression, then sqlite3_value_frombind(X) returns zero. +** ** Please pay particular attention to the fact that the pointer returned ** from [sqlite3_value_blob()], [sqlite3_value_text()], or ** [sqlite3_value_text16()] can be invalidated by a subsequent call to @@ -5056,6 +5170,7 @@ SQLITE_API int sqlite3_value_bytes16(sqlite3_value*); SQLITE_API int sqlite3_value_type(sqlite3_value*); SQLITE_API int sqlite3_value_numeric_type(sqlite3_value*); SQLITE_API int sqlite3_value_nochange(sqlite3_value*); +SQLITE_API int sqlite3_value_frombind(sqlite3_value*); /* ** CAPI3REF: Finding The Subtype Of SQL Values @@ -5791,7 +5906,7 @@ SQLITE_API sqlite3 *sqlite3_db_handle(sqlite3_stmt*); ** associated with database N of connection D. ^The main database file ** has the name "main". If there is no attached database N on the database ** connection D, or if database N is a temporary or in-memory database, then -** a NULL pointer is returned. +** this function will return either a NULL pointer or an empty string. ** ** ^The filename returned by this function is the output of the ** xFullPathname method of the [VFS]. ^In other words, the filename @@ -6537,6 +6652,12 @@ struct sqlite3_index_info { ** ^The sqlite3_create_module() ** interface is equivalent to sqlite3_create_module_v2() with a NULL ** destructor. +** +** ^If the third parameter (the pointer to the sqlite3_module object) is +** NULL then no new module is create and any existing modules with the +** same name are dropped. +** +** See also: [sqlite3_drop_modules()] */ SQLITE_API int sqlite3_create_module( sqlite3 *db, /* SQLite connection to register module with */ @@ -6552,6 +6673,23 @@ SQLITE_API int sqlite3_create_module_v2( void(*xDestroy)(void*) /* Module destructor function */ ); +/* +** CAPI3REF: Remove Unnecessary Virtual Table Implementations +** METHOD: sqlite3 +** +** ^The sqlite3_drop_modules(D,L) interface removes all virtual +** table modules from database connection D except those named on list L. +** The L parameter must be either NULL or a pointer to an array of pointers +** to strings where the array is terminated by a single NULL pointer. +** ^If the L parameter is NULL, then all virtual table modules are removed. +** +** See also: [sqlite3_create_module()] +*/ +SQLITE_API int sqlite3_drop_modules( + sqlite3 *db, /* Remove modules from this connection */ + const char **azKeep /* Except, do not remove the ones named here */ +); + /* ** CAPI3REF: Virtual Table Instance Object ** KEYWORDS: sqlite3_vtab @@ -7260,7 +7398,7 @@ SQLITE_API int sqlite3_test_control(int op, ...); #define SQLITE_TESTCTRL_FIRST 5 #define SQLITE_TESTCTRL_PRNG_SAVE 5 #define SQLITE_TESTCTRL_PRNG_RESTORE 6 -#define SQLITE_TESTCTRL_PRNG_RESET 7 +#define SQLITE_TESTCTRL_PRNG_RESET 7 /* NOT USED */ #define SQLITE_TESTCTRL_BITVEC_TEST 8 #define SQLITE_TESTCTRL_FAULT_INSTALL 9 #define SQLITE_TESTCTRL_BENIGN_MALLOC_HOOKS 10 @@ -7282,7 +7420,10 @@ SQLITE_API int sqlite3_test_control(int op, ...); #define SQLITE_TESTCTRL_SORTER_MMAP 24 #define SQLITE_TESTCTRL_IMPOSTER 25 #define SQLITE_TESTCTRL_PARSER_COVERAGE 26 -#define SQLITE_TESTCTRL_LAST 26 /* Largest TESTCTRL */ +#define SQLITE_TESTCTRL_RESULT_INTREAL 27 +#define SQLITE_TESTCTRL_PRNG_SEED 28 +#define SQLITE_TESTCTRL_EXTRA_SCHEMA_CHECKS 29 +#define SQLITE_TESTCTRL_LAST 29 /* Largest TESTCTRL */ /* ** CAPI3REF: SQL Keyword Checking @@ -10892,7 +11033,7 @@ SQLITE_API int sqlite3rebaser_configure( ** in size. This function allocates and populates a buffer with a copy ** of the changeset rebased rebased according to the configuration of the ** rebaser object passed as the first argument. If successful, (*ppOut) -** is set to point to the new buffer containing the rebased changset and +** is set to point to the new buffer containing the rebased changeset and ** (*pnOut) to its size in bytes and SQLITE_OK returned. It is the ** responsibility of the caller to eventually free the new buffer using ** sqlite3_free(). Otherwise, if an error occurs, (*ppOut) and (*pnOut) @@ -11301,7 +11442,7 @@ struct Fts5PhraseIter { ** Save the pointer passed as the second argument as the extension functions ** "auxiliary data". The pointer may then be retrieved by the current or any ** future invocation of the same fts5 extension function made as part of -** of the same MATCH query using the xGetAuxdata() API. +** the same MATCH query using the xGetAuxdata() API. ** ** Each extension function is allocated a single auxiliary data slot for ** each FTS query (MATCH expression). If the extension function is invoked @@ -11316,7 +11457,7 @@ struct Fts5PhraseIter { ** The xDelete callback, if one is specified, is also invoked on the ** auxiliary data pointer after the FTS5 query has finished. ** -** If an error (e.g. an OOM condition) occurs within this function, an +** If an error (e.g. an OOM condition) occurs within this function, ** the auxiliary data is set to NULL and an error code returned. If the ** xDelete parameter was not NULL, it is invoked on the auxiliary data ** pointer before returning. diff --git a/src/datastructure.c b/src/datastructure.c new file mode 100644 index 000000000..a2ff98b1a --- /dev/null +++ b/src/datastructure.c @@ -0,0 +1,386 @@ +/* Pi-hole: A black hole for Internet advertisements +* (c) 2017 Pi-hole, LLC (https://pi-hole.net) +* Network-wide ad blocking via your own hardware. +* +* FTL Engine +* Query processing routines +* +* This file is copyright under the latest version of the EUPL. +* Please see LICENSE file for your rights under this license. */ + +#include "FTL.h" +#include "datastructure.h" +#include "memory.h" +#include "shmem.h" +#include "log.h" +// enum REGEX +#include "regex_r.h" +#include "database/gravity-db.h" + +// converts upper to lower case, and leaves other characters unchanged +void strtolower(char *str) +{ + int i = 0; + while(str[i]){ str[i] = tolower(str[i]); i++; } +} + +int findUpstreamID(const char * upstreamString, const bool count) +{ + // Go through already knows upstream servers and see if we used one of those + for(int upstreamID=0; upstreamID < counters->upstreams; upstreamID++) + { + // Get upstream pointer + upstreamsData* upstream = getUpstream(upstreamID, true); + + // Check if the returned pointer is valid before trying to access it + if(upstream == NULL) + continue; + + if(strcmp(getstr(upstream->ippos), upstreamString) == 0) + { + if(count) upstream->count++; + return upstreamID; + } + } + // This upstream server is not known + // Store ID + const int upstreamID = counters->upstreams; + logg("New upstream server: %s (%i/%u)", upstreamString, upstreamID, counters->upstreams_MAX); + + // Check struct size + memory_check(UPSTREAMS); + + // Get upstream pointer + upstreamsData* upstream = getUpstream(upstreamID, false); + if(upstream == NULL) + { + logg("ERROR: Encountered serious memory error in findupstreamID()"); + return -1; + } + + // Set magic byte + upstream->magic = MAGICBYTE; + // Initialize its counter + if(count) + upstream->count = 1; + else + upstream->count = 0; + // Save upstream destination IP address + upstream->ippos = addstr(upstreamString); + upstream->failed = 0; + // Initialize upstream hostname + // Due to the nature of us being the resolver, + // the actual resolving of the host name has + // to be done separately to be non-blocking + upstream->new = true; + upstream->namepos = 0; // 0 -> string with length zero + // Increase counter by one + counters->upstreams++; + + return upstreamID; +} + +int findDomainID(const char *domainString, const bool count) +{ + for(int domainID = 0; domainID < counters->domains; domainID++) + { + // Get domain pointer + domainsData* domain = getDomain(domainID, true); + + // Check if the returned pointer is valid before trying to access it + if(domain == NULL) + continue; + + // Quick test: Does the domain start with the same character? + if(getstr(domain->domainpos)[0] != domainString[0]) + continue; + + // If so, compare the full domain using strcmp + if(strcmp(getstr(domain->domainpos), domainString) == 0) + { + if(count) + domain->count++; + return domainID; + } + } + + // If we did not return until here, then this domain is not known + // Store ID + const int domainID = counters->domains; + + // Check struct size + memory_check(DOMAINS); + + // Get domain pointer + domainsData* domain = getDomain(domainID, false); + if(domain == NULL) + { + logg("ERROR: Encountered serious memory error in findDomainID()"); + return -1; + } + + // Set magic byte + domain->magic = MAGICBYTE; + // Set its counter to 1 only if this domain is to be counted + // Domains only encountered during CNAME inspection are NOT counted here + domain->count = count ? 1 : 0; + // Set blocked counter to zero + domain->blockedcount = 0; + // Store domain name - no need to check for NULL here as it doesn't harm + domain->domainpos = addstr(domainString); + // Increase counter by one + counters->domains++; + + return domainID; +} + +int findClientID(const char *clientIP, const bool count) +{ + // Compare content of client against known client IP addresses + for(int clientID=0; clientID < counters->clients; clientID++) + { + // Get client pointer + clientsData* client = getClient(clientID, true); + + // Check if the returned pointer is valid before trying to access it + if(client == NULL) + continue; + + // Quick test: Does the clients IP start with the same character? + if(getstr(client->ippos)[0] != clientIP[0]) + continue; + + // If so, compare the full IP using strcmp + if(strcmp(getstr(client->ippos), clientIP) == 0) + { + // Add one if count == true (do not add one, e.g., during ARP table processing) + if(count) client->count++; + return clientID; + } + } + + // Return -1 (= not found) if count is false because we do not want to create a new client here + if(!count) + return -1; + + // If we did not return until here, then this client is definitely new + // Store ID + const int clientID = counters->clients; + + // Check struct size + memory_check(CLIENTS); + + // Get client pointer + clientsData* client = getClient(clientID, false); + if(client == NULL) + { + logg("ERROR: Encountered serious memory error in findClientID()"); + return -1; + } + + // Set magic byte + client->magic = MAGICBYTE; + // Set its counter to 1 + client->count = 1; + // Initialize blocked count to zero + client->blockedcount = 0; + // Store client IP - no need to check for NULL here as it doesn't harm + client->ippos = addstr(clientIP); + // Initialize client hostname + // Due to the nature of us being the resolver, + // the actual resolving of the host name has + // to be done separately to be non-blocking + client->new = true; + client->namepos = 0; + // No query seen so far + client->lastQuery = 0; + client->numQueriesARP = client->count; + + // Initialize client-specific overTime data + for(int i = 0; i < OVERTIME_SLOTS; i++) + client->overTime[i] = 0; + + // Increase counter by one + counters->clients++; + + // Allocate regex substructure + allocate_regex_client_enabled(client, clientID); + + return clientID; +} + +int findCacheID(int domainID, int clientID) +{ + // Compare content of client against known client IP addresses + for(int cacheID = 0; cacheID < counters->dns_cache_size; cacheID++) + { + // Get cache pointer + DNSCacheData* dns_cache = getDNSCache(cacheID, true); + + // Check if the returned pointer is valid before trying to access it + if(dns_cache == NULL) + continue; + + if(dns_cache->domainID == domainID && + dns_cache->clientID == clientID) + { + return cacheID; + } + } + + // Get ID of new cache entry + const int cacheID = counters->dns_cache_size; + + // Check struct size + memory_check(DNS_CACHE); + + // Get client pointer + DNSCacheData* dns_cache = getDNSCache(cacheID, false); + + if(dns_cache == NULL) + { + logg("ERROR: Encountered serious memory error in findCacheID()"); + return -1; + } + + // Initialize cache entry + dns_cache->magic = MAGICBYTE; + dns_cache->blocking_status = UNKNOWN_BLOCKED; + dns_cache->domainID = domainID; + dns_cache->clientID = clientID; + dns_cache->force_reply = 0u; + + // Increase counter by one + counters->dns_cache_size++; + + return cacheID; +} + +bool isValidIPv4(const char *addr) +{ + struct sockaddr_in sa; + return inet_pton(AF_INET, addr, &(sa.sin_addr)) != 0; +} + +bool isValidIPv6(const char *addr) +{ + struct sockaddr_in6 sa; + return inet_pton(AF_INET6, addr, &(sa.sin6_addr)) != 0; +} + +// Privacy-level sensitive subroutine that returns the domain name +// only when appropriate for the requested query +const char *getDomainString(const queriesData* query) +{ + // Check if the returned pointer is valid before trying to access it + if(query == NULL) + return ""; + + if(query->privacylevel < PRIVACY_HIDE_DOMAINS) + { + // Get domain pointer + const domainsData* domain = getDomain(query->domainID, true); + + // Return string + return getstr(domain->domainpos); + } + else + return HIDDEN_DOMAIN; +} + +// Privacy-level sensitive subroutine that returns the domain name +// only when appropriate for the requested query +const char *getCNAMEDomainString(const queriesData* query) +{ + // Check if the returned pointer is valid before trying to access it + if(query == NULL) + return ""; + + if(query->privacylevel < PRIVACY_HIDE_DOMAINS) + { + // Get domain pointer + const domainsData* domain = getDomain(query->CNAME_domainID, true); + + // Return string + return getstr(domain->domainpos); + } + else + return HIDDEN_DOMAIN; +} + +// Privacy-level sensitive subroutine that returns the client IP +// only when appropriate for the requested query +const char *getClientIPString(const queriesData* query) +{ + // Check if the returned pointer is valid before trying to access it + if(query == NULL) + return ""; + + if(query->privacylevel < PRIVACY_HIDE_DOMAINS_CLIENTS) + { + // Get client pointer + const clientsData* client = getClient(query->clientID, false); + + // Return string + return getstr(client->ippos); + } + else + return HIDDEN_CLIENT; +} + +// Privacy-level sensitive subroutine that returns the client host name +// only when appropriate for the requested query +const char *getClientNameString(const queriesData* query) +{ + // Check if the returned pointer is valid before trying to access it + if(query == NULL) + return ""; + + if(query->privacylevel < PRIVACY_HIDE_DOMAINS_CLIENTS) + { + // Get client pointer + const clientsData* client = getClient(query->clientID, true); + + // Return string + return getstr(client->namepos); + } + else + return HIDDEN_CLIENT; +} + +void FTL_reset_per_client_domain_data(void) +{ + for(int domainID = 0; domainID < counters->domains; domainID++) + { + domainsData *domain = getDomain(domainID, true); + if(domain == NULL) + continue; + + for(int cacheID = 0; cacheID < counters->dns_cache_size; cacheID++) + { + // Reset all blocking yes/no fields for all domains and clients + // This forces a reprocessing of all available filters for any + // given domain and client the next time they are seen + DNSCacheData *dns_cache = getDNSCache(cacheID, true); + dns_cache->blocking_status = UNKNOWN_BLOCKED; + } + } +} + +void FTL_reload_all_domainlists(void) +{ + // (Re-)open gravity database connection + gravityDB_close(); + gravityDB_open(); + + // Reset number of blocked domains + counters->gravity = gravityDB_count(GRAVITY_TABLE); + + // Read and compile possible regex filters + // only after having called gravityDB_open() + read_regex_from_database(); + + // Reset FTL's internal DNS cache storing whether a specific domain + // has already been validated for a specific user + FTL_reset_per_client_domain_data(); +} diff --git a/src/datastructure.h b/src/datastructure.h new file mode 100644 index 000000000..74c78b8a0 --- /dev/null +++ b/src/datastructure.h @@ -0,0 +1,101 @@ +/* Pi-hole: A black hole for Internet advertisements +* (c) 2019 Pi-hole, LLC (https://pi-hole.net) +* Network-wide ad blocking via your own hardware. +* +* FTL Engine +* Datastructure prototypes +* +* This file is copyright under the latest version of the EUPL. +* Please see LICENSE file for your rights under this license. */ +#ifndef DATASTRUCTURE_H +#define DATASTRUCTURE_H + +// Definition of sqlite3_stmt +#include "database/sqlite3.h" + +void strtolower(char *str); +int findUpstreamID(const char * upstream, const bool count); +int findDomainID(const char *domain, const bool count); +int findClientID(const char *client, const bool count); +int findCacheID(int domainID, int clientID); +bool isValidIPv4(const char *addr); +bool isValidIPv6(const char *addr); + +void FTL_reload_all_domainlists(void); +void FTL_reset_per_client_domain_data(void); + +typedef struct { + unsigned char magic; + unsigned char type; + unsigned char status; + unsigned char privacylevel; + unsigned char reply; + unsigned char dnssec; + time_t timestamp; + int domainID; + int clientID; + int upstreamID; + int id; // the ID is a (signed) int in dnsmasq, so no need for a long int here + int CNAME_domainID; // only valid if query has a CNAME blocking status + unsigned long response; // saved in units of 1/10 milliseconds (1 = 0.1ms, 2 = 0.2ms, 2500 = 250.0ms, etc.) + int64_t db; + unsigned int timeidx; + bool whitelisted; + bool complete; +} queriesData; + +typedef struct { + unsigned char magic; + bool new; + int count; + int failed; + size_t ippos; + size_t namepos; +} upstreamsData; + +typedef struct { + unsigned char magic; + bool new; + int count; + int blockedcount; + int overTime[OVERTIME_SLOTS]; + unsigned int numQueriesARP; + size_t ippos; + size_t namepos; + time_t lastQuery; +} clientsData; + +typedef struct { + unsigned char magic; + size_t domainpos; + int count; + int blockedcount; +} domainsData; + +typedef struct { + unsigned char magic; + unsigned char blocking_status; + unsigned char force_reply; + int domainID; + int clientID; + int black_regex_idx; +} DNSCacheData; + +const char *getDomainString(const queriesData* query); +const char *getCNAMEDomainString(const queriesData* query); +const char *getClientIPString(const queriesData* query); +const char *getClientNameString(const queriesData* query); + +// Pointer getter functions +#define getQuery(queryID, checkMagic) _getQuery(queryID, checkMagic, __LINE__, __FUNCTION__, __FILE__) +queriesData* _getQuery(int queryID, bool checkMagic, int line, const char * function, const char * file); +#define getClient(clientID, checkMagic) _getClient(clientID, checkMagic, __LINE__, __FUNCTION__, __FILE__) +clientsData* _getClient(int clientID, bool checkMagic, int line, const char * function, const char * file); +#define getDomain(domainID, checkMagic) _getDomain(domainID, checkMagic, __LINE__, __FUNCTION__, __FILE__) +domainsData* _getDomain(int domainID, bool checkMagic, int line, const char * function, const char * file); +#define getUpstream(upstreamID, checkMagic) _getUpstream(upstreamID, checkMagic, __LINE__, __FUNCTION__, __FILE__) +upstreamsData* _getUpstream(int upstreamID, bool checkMagic, int line, const char * function, const char * file); +#define getDNSCache(cacheID, checkMagic) _getDNSCache(cacheID, checkMagic, __LINE__, __FUNCTION__, __FILE__) +DNSCacheData* _getDNSCache(int cacheID, bool checkMagic, int line, const char * function, const char * file); + +#endif //DATASTRUCTURE_H diff --git a/dnsmasq/COPYING-v3 b/src/dnsmasq/COPYING-v3 similarity index 100% rename from dnsmasq/COPYING-v3 rename to src/dnsmasq/COPYING-v3 diff --git a/dnsmasq/arp.c b/src/dnsmasq/arp.c similarity index 86% rename from dnsmasq/arp.c rename to src/dnsmasq/arp.c index 8beaed4aa..3329d6d64 100644 --- a/dnsmasq/arp.c +++ b/src/dnsmasq/arp.c @@ -1,4 +1,4 @@ -/* dnsmasq is Copyright (c) 2000-2018 Simon Kelley +/* dnsmasq is Copyright (c) 2000-2020 Simon Kelley This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -28,7 +28,7 @@ struct arp_record { unsigned short hwlen, status; int family; unsigned char hwaddr[DHCP_CHADDR_MAX]; - struct all_addr addr; + union all_addr addr; struct arp_record *next; }; @@ -44,11 +44,6 @@ static int filter_mac(int family, char *addrp, char *mac, size_t maclen, void *p if (maclen > DHCP_CHADDR_MAX) return 1; -#ifndef HAVE_IPV6 - if (family != AF_INET) - return 1; -#endif - /* Look for existing entry */ for (arp = arps; arp; arp = arp->next) { @@ -57,16 +52,14 @@ static int filter_mac(int family, char *addrp, char *mac, size_t maclen, void *p if (family == AF_INET) { - if (arp->addr.addr.addr4.s_addr != ((struct in_addr *)addrp)->s_addr) + if (arp->addr.addr4.s_addr != ((struct in_addr *)addrp)->s_addr) continue; } -#ifdef HAVE_IPV6 else { - if (!IN6_ARE_ADDR_EQUAL(&arp->addr.addr.addr6, (struct in6_addr *)addrp)) + if (!IN6_ARE_ADDR_EQUAL(&arp->addr.addr6, (struct in6_addr *)addrp)) continue; } -#endif if (arp->status == ARP_EMPTY) { @@ -102,11 +95,9 @@ static int filter_mac(int family, char *addrp, char *mac, size_t maclen, void *p arp->family = family; memcpy(arp->hwaddr, mac, maclen); if (family == AF_INET) - arp->addr.addr.addr4.s_addr = ((struct in_addr *)addrp)->s_addr; -#ifdef HAVE_IPV6 + arp->addr.addr4.s_addr = ((struct in_addr *)addrp)->s_addr; else - memcpy(&arp->addr.addr.addr6, addrp, IN6ADDRSZ); -#endif + memcpy(&arp->addr.addr6, addrp, IN6ADDRSZ); } return 1; @@ -133,14 +124,12 @@ int find_mac(union mysockaddr *addr, unsigned char *mac, int lazy, time_t now) continue; if (arp->family == AF_INET && - arp->addr.addr.addr4.s_addr != addr->in.sin_addr.s_addr) + arp->addr.addr4.s_addr != addr->in.sin_addr.s_addr) continue; -#ifdef HAVE_IPV6 if (arp->family == AF_INET6 && - !IN6_ARE_ADDR_EQUAL(&arp->addr.addr.addr6, &addr->in6.sin6_addr)) + !IN6_ARE_ADDR_EQUAL(&arp->addr.addr6, &addr->in6.sin6_addr)) continue; -#endif /* Only accept positive entries unless in lazy mode. */ if (arp->status != ARP_EMPTY || lazy || updated) @@ -202,11 +191,9 @@ int find_mac(union mysockaddr *addr, unsigned char *mac, int lazy, time_t now) arp->hwlen = 0; if (addr->sa.sa_family == AF_INET) - arp->addr.addr.addr4.s_addr = addr->in.sin_addr.s_addr; -#ifdef HAVE_IPV6 + arp->addr.addr4.s_addr = addr->in.sin_addr.s_addr; else - memcpy(&arp->addr.addr.addr6, &addr->in6.sin6_addr, IN6ADDRSZ); -#endif + memcpy(&arp->addr.addr6, &addr->in6.sin6_addr, IN6ADDRSZ); } return 0; diff --git a/dnsmasq/auth.c b/src/dnsmasq/auth.c similarity index 83% rename from dnsmasq/auth.c rename to src/dnsmasq/auth.c index 2e43c494f..b2fcd4b6f 100644 --- a/dnsmasq/auth.c +++ b/src/dnsmasq/auth.c @@ -1,15 +1,15 @@ -/* dnsmasq is Copyright (c) 2000-2018 Simon Kelley +/* dnsmasq is Copyright (c) 2000-2020 Simon Kelley This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; version 2 dated June, 1991, or (at your option) version 3 dated 29 June, 2007. - + This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. - + You should have received a copy of the GNU General Public License along with this program. If not, see . */ @@ -18,48 +18,46 @@ #ifdef HAVE_AUTH -static struct addrlist *find_addrlist(struct addrlist *list, int flag, struct all_addr *addr_u) +static struct addrlist *find_addrlist(struct addrlist *list, int flag, union all_addr *addr_u) { do { if (!(list->flags & ADDRLIST_IPV6)) { - struct in_addr netmask, addr = addr_u->addr.addr4; - + struct in_addr netmask, addr = addr_u->addr4; + if (!(flag & F_IPV4)) continue; - + netmask.s_addr = htonl(~(in_addr_t)0 << (32 - list->prefixlen)); - - if (is_same_net(addr, list->addr.addr.addr4, netmask)) + + if (is_same_net(addr, list->addr.addr4, netmask)) return list; } -#ifdef HAVE_IPV6 - else if (is_same_net6(&(addr_u->addr.addr6), &list->addr.addr.addr6, list->prefixlen)) + else if (is_same_net6(&(addr_u->addr6), &list->addr.addr6, list->prefixlen)) return list; -#endif - + } while ((list = list->next)); - + return NULL; } -static struct addrlist *find_subnet(struct auth_zone *zone, int flag, struct all_addr *addr_u) +static struct addrlist *find_subnet(struct auth_zone *zone, int flag, union all_addr *addr_u) { if (!zone->subnet) return NULL; - + return find_addrlist(zone->subnet, flag, addr_u); } -static struct addrlist *find_exclude(struct auth_zone *zone, int flag, struct all_addr *addr_u) +static struct addrlist *find_exclude(struct auth_zone *zone, int flag, union all_addr *addr_u) { if (!zone->exclude) return NULL; - + return find_addrlist(zone->exclude, flag, addr_u); } -static int filter_zone(struct auth_zone *zone, int flag, struct all_addr *addr_u) +static int filter_zone(struct auth_zone *zone, int flag, union all_addr *addr_u) { if (find_exclude(zone, flag, addr_u)) return 0; @@ -67,7 +65,7 @@ static int filter_zone(struct auth_zone *zone, int flag, struct all_addr *addr_u /* No subnets specified, no filter */ if (!zone->subnet) return 1; - + return find_subnet(zone, flag, addr_u) != NULL; } @@ -78,18 +76,18 @@ int in_zone(struct auth_zone *zone, char *name, char **cut) if (cut) *cut = NULL; - - if (namelen >= domainlen && + + if (namelen >= domainlen && hostname_isequal(zone->domain, &name[namelen - domainlen])) { - + if (namelen == domainlen) return 1; - + if (name[namelen - domainlen - 1] == '.') { if (cut) - *cut = &name[namelen - domainlen - 1]; + *cut = &name[namelen - domainlen - 1]; return 1; } } @@ -98,8 +96,8 @@ int in_zone(struct auth_zone *zone, char *name, char **cut) } -size_t answer_auth(struct dns_header *header, char *limit, size_t qlen, time_t now, union mysockaddr *peer_addr, - int local_query, int do_bit, int have_pseudoheader) +size_t answer_auth(struct dns_header *header, char *limit, size_t qlen, time_t now, union mysockaddr *peer_addr, + int local_query, int do_bit, int have_pseudoheader) { char *name = daemon->namebuff; unsigned char *p, *ansp; @@ -115,36 +113,36 @@ size_t answer_auth(struct dns_header *header, char *limit, size_t qlen, time_t n struct txt_record *txt; struct interface_name *intr; struct naptr *na; - struct all_addr addr; + union all_addr addr; struct cname *a, *candidate; unsigned int wclen; - + if (ntohs(header->qdcount) == 0 || OPCODE(header) != QUERY ) return 0; /* determine end of question section (we put answers there) */ if (!(ansp = skip_questions(header, qlen))) return 0; /* bad packet */ - + /* now process each question, answers go in RRs after the question */ p = (unsigned char *)(header+1); for (q = ntohs(header->qdcount); q != 0; q--) { - unsigned short flag = 0; + unsigned int flag = 0; int found = 0; int cname_wildcard = 0; - + /* save pointer to name for copying into answers */ nameoffset = p - (unsigned char *)header; /* now extract name as .-concatenated string into name */ if (!extract_name(header, qlen, &p, name, 1, 4)) return 0; /* bad packet */ - - GETSHORT(qtype, p); + + GETSHORT(qtype, p); GETSHORT(qclass, p); - + if (qclass != C_IN) { auth = 0; @@ -158,7 +156,7 @@ size_t answer_auth(struct dns_header *header, char *limit, size_t qlen, time_t n for (zone = daemon->auth_zones; zone; zone = zone->next) if ((subnet = find_subnet(zone, flag, &addr))) break; - + if (!zone) { auth = 0; @@ -178,58 +176,56 @@ size_t answer_auth(struct dns_header *header, char *limit, size_t qlen, time_t n for (intr = daemon->int_names; intr; intr = intr->next) { struct addrlist *addrlist; - + for (addrlist = intr->addr; addrlist; addrlist = addrlist->next) - if (!(addrlist->flags & ADDRLIST_IPV6) && addr.addr.addr4.s_addr == addrlist->addr.addr.addr4.s_addr) + if (!(addrlist->flags & ADDRLIST_IPV6) && addr.addr4.s_addr == addrlist->addr.addr4.s_addr) break; - + if (addrlist) break; else while (intr->next && strcmp(intr->intr, intr->next->intr) == 0) intr = intr->next; } -#ifdef HAVE_IPV6 else if (flag == F_IPV6) for (intr = daemon->int_names; intr; intr = intr->next) { struct addrlist *addrlist; - + for (addrlist = intr->addr; addrlist; addrlist = addrlist->next) - if ((addrlist->flags & ADDRLIST_IPV6) && IN6_ARE_ADDR_EQUAL(&addr.addr.addr6, &addrlist->addr.addr.addr6)) + if ((addrlist->flags & ADDRLIST_IPV6) && IN6_ARE_ADDR_EQUAL(&addr.addr6, &addrlist->addr.addr6)) break; - + if (addrlist) break; else while (intr->next && strcmp(intr->intr, intr->next->intr) == 0) intr = intr->next; } -#endif - + if (intr) { if (local_query || in_zone(zone, intr->name, NULL)) - { + { found = 1; log_query(flag | F_REVERSE | F_CONFIG, intr->name, &addr, NULL); - if (add_resource_record(header, limit, &trunc, nameoffset, &ansp, + if (add_resource_record(header, limit, &trunc, nameoffset, &ansp, daemon->auth_ttl, NULL, T_PTR, C_IN, "d", intr->name)) anscount++; } } - + if ((crecp = cache_find_by_addr(NULL, &addr, now, flag))) - do { + do { strcpy(name, cache_get_name(crecp)); - + if (crecp->flags & F_DHCP && !option_bool(OPT_DHCP_FQDN)) { char *p = strchr(name, '.'); if (p) *p = 0; /* must be bare name */ - + /* add external domain */ if (zone) { @@ -238,7 +234,7 @@ size_t answer_auth(struct dns_header *header, char *limit, size_t qlen, time_t n } log_query(flag | F_DHCP | F_REVERSE, name, &addr, record_source(crecp->uid)); found = 1; - if (add_resource_record(header, limit, &trunc, nameoffset, &ansp, + if (add_resource_record(header, limit, &trunc, nameoffset, &ansp, daemon->auth_ttl, NULL, T_PTR, C_IN, "d", name)) anscount++; @@ -247,14 +243,14 @@ size_t answer_auth(struct dns_header *header, char *limit, size_t qlen, time_t n { log_query(crecp->flags & ~F_FORWARD, name, &addr, record_source(crecp->uid)); found = 1; - if (add_resource_record(header, limit, &trunc, nameoffset, &ansp, + if (add_resource_record(header, limit, &trunc, nameoffset, &ansp, daemon->auth_ttl, NULL, T_PTR, C_IN, "d", name)) anscount++; } else continue; - + } while ((crecp = cache_find_by_addr(crecp, &addr, now, flag))); if (found) @@ -264,7 +260,7 @@ size_t answer_auth(struct dns_header *header, char *limit, size_t qlen, time_t n continue; } - + cname_restart: if (found) /* NS and SOA .arpa requests have set found above. */ @@ -274,7 +270,7 @@ size_t answer_auth(struct dns_header *header, char *limit, size_t qlen, time_t n for (zone = daemon->auth_zones; zone; zone = zone->next) if (in_zone(zone, name, &cut)) break; - + if (!zone) { auth = 0; @@ -286,33 +282,33 @@ size_t answer_auth(struct dns_header *header, char *limit, size_t qlen, time_t n if (!rec->issrv && (rc = hostname_issubdomain(name, rec->name))) { nxdomain = 0; - + if (rc == 2 && qtype == T_MX) { found = 1; - log_query(F_CONFIG | F_RRNAME, name, NULL, ""); + log_query(F_CONFIG | F_RRNAME, name, NULL, ""); if (add_resource_record(header, limit, &trunc, nameoffset, &ansp, daemon->auth_ttl, NULL, T_MX, C_IN, "sd", rec->weight, rec->target)) anscount++; } } - + for (move = NULL, up = &daemon->mxnames, rec = daemon->mxnames; rec; rec = rec->next) if (rec->issrv && (rc = hostname_issubdomain(name, rec->name))) { nxdomain = 0; - + if (rc == 2 && qtype == T_SRV) { found = 1; - log_query(F_CONFIG | F_RRNAME, name, NULL, ""); + log_query(F_CONFIG | F_RRNAME, name, NULL, ""); if (add_resource_record(header, limit, &trunc, nameoffset, &ansp, daemon->auth_ttl, - NULL, T_SRV, C_IN, "sssd", + NULL, T_SRV, C_IN, "sssd", rec->priority, rec->weight, rec->srvport, rec->target)) anscount++; - } - + } + /* unlink first SRV record found */ if (!move) { @@ -320,11 +316,11 @@ size_t answer_auth(struct dns_header *header, char *limit, size_t qlen, time_t n *up = rec->next; } else - up = &rec->next; + up = &rec->next; } else up = &rec->next; - + /* put first SRV record back at the end. */ if (move) { @@ -339,13 +335,13 @@ size_t answer_auth(struct dns_header *header, char *limit, size_t qlen, time_t n if (rc == 2 && txt->class == qtype) { found = 1; - log_query(F_CONFIG | F_RRNAME, name, NULL, querystr(NULL, txt->class)); + log_query(F_CONFIG | F_RRNAME, name, NULL, querystr(NULL, txt->class)); if (add_resource_record(header, limit, &trunc, nameoffset, &ansp, daemon->auth_ttl, NULL, txt->class, C_IN, "t", txt->len, txt->txt)) anscount++; } } - + for (txt = daemon->txt; txt; txt = txt->next) if (txt->class == C_IN && (rc = hostname_issubdomain(name, txt->name))) { @@ -353,7 +349,7 @@ size_t answer_auth(struct dns_header *header, char *limit, size_t qlen, time_t n if (rc == 2 && qtype == T_TXT) { found = 1; - log_query(F_CONFIG | F_RRNAME, name, NULL, ""); + log_query(F_CONFIG | F_RRNAME, name, NULL, ""); if (add_resource_record(header, limit, &trunc, nameoffset, &ansp, daemon->auth_ttl, NULL, T_TXT, C_IN, "t", txt->len, txt->txt)) anscount++; @@ -368,50 +364,47 @@ size_t answer_auth(struct dns_header *header, char *limit, size_t qlen, time_t n { found = 1; log_query(F_CONFIG | F_RRNAME, name, NULL, ""); - if (add_resource_record(header, limit, &trunc, nameoffset, &ansp, daemon->auth_ttl, - NULL, T_NAPTR, C_IN, "sszzzd", + if (add_resource_record(header, limit, &trunc, nameoffset, &ansp, daemon->auth_ttl, + NULL, T_NAPTR, C_IN, "sszzzd", na->order, na->pref, na->flags, na->services, na->regexp, na->replace)) anscount++; } } - + if (qtype == T_A) flag = F_IPV4; - -#ifdef HAVE_IPV6 + if (qtype == T_AAAA) flag = F_IPV6; -#endif - + for (intr = daemon->int_names; intr; intr = intr->next) if ((rc = hostname_issubdomain(name, intr->name))) { struct addrlist *addrlist; - + nxdomain = 0; - + if (rc == 2 && flag) - for (addrlist = intr->addr; addrlist; addrlist = addrlist->next) + for (addrlist = intr->addr; addrlist; addrlist = addrlist->next) if (((addrlist->flags & ADDRLIST_IPV6) ? T_AAAA : T_A) == qtype && (local_query || filter_zone(zone, flag, &addrlist->addr))) { -#ifdef HAVE_IPV6 if (addrlist->flags & ADDRLIST_REVONLY) continue; -#endif + found = 1; log_query(F_FORWARD | F_CONFIG | flag, name, &addrlist->addr, NULL); - if (add_resource_record(header, limit, &trunc, nameoffset, &ansp, - daemon->auth_ttl, NULL, qtype, C_IN, + if (add_resource_record(header, limit, &trunc, nameoffset, &ansp, + daemon->auth_ttl, NULL, qtype, C_IN, qtype == T_A ? "4" : "6", &addrlist->addr)) anscount++; } } - + if (!cut) { nxdomain = 0; - + if (qtype == T_SOA) { auth = soa = 1; /* inhibits auth section */ @@ -421,36 +414,32 @@ size_t answer_auth(struct dns_header *header, char *limit, size_t qlen, time_t n else if (qtype == T_AXFR) { struct iname *peers; - + if (peer_addr->sa.sa_family == AF_INET) peer_addr->in.sin_port = 0; -#ifdef HAVE_IPV6 else { - peer_addr->in6.sin6_port = 0; + peer_addr->in6.sin6_port = 0; peer_addr->in6.sin6_scope_id = 0; } -#endif - + for (peers = daemon->auth_peers; peers; peers = peers->next) if (sockaddr_isequal(peer_addr, &peers->addr)) break; - + /* Refuse all AXFR unless --auth-sec-servers or auth-peers is set */ if ((!daemon->secondary_forward_server && !daemon->auth_peers) || - (daemon->auth_peers && !peers)) + (daemon->auth_peers && !peers)) { if (peer_addr->sa.sa_family == AF_INET) inet_ntop(AF_INET, &peer_addr->in.sin_addr, daemon->addrbuff, ADDRSTRLEN); -#ifdef HAVE_IPV6 else - inet_ntop(AF_INET6, &peer_addr->in6.sin6_addr, daemon->addrbuff, ADDRSTRLEN); -#endif - + inet_ntop(AF_INET6, &peer_addr->in6.sin6_addr, daemon->addrbuff, ADDRSTRLEN); + my_syslog(LOG_WARNING, _("ignoring zone transfer request from %s"), daemon->addrbuff); return 0; } - + auth = 1; soa = 1; /* inhibits auth section */ ns = 1; /* ensure we include NS records! */ @@ -464,70 +453,70 @@ size_t answer_auth(struct dns_header *header, char *limit, size_t qlen, time_t n auth = 1; ns = 1; /* inhibits auth section */ found = 1; - log_query(F_RRNAME | F_AUTH, zone->domain, NULL, ""); + log_query(F_RRNAME | F_AUTH, zone->domain, NULL, ""); } } - + if (!option_bool(OPT_DHCP_FQDN) && cut) - { + { *cut = 0; /* remove domain part */ - + if (!strchr(name, '.') && (crecp = cache_find_by_name(NULL, name, now, F_IPV4 | F_IPV6))) { if (crecp->flags & F_DHCP) do - { + { nxdomain = 0; - if ((crecp->flags & flag) && - (local_query || filter_zone(zone, flag, &(crecp->addr.addr)))) + if ((crecp->flags & flag) && + (local_query || filter_zone(zone, flag, &(crecp->addr)))) { *cut = '.'; /* restore domain part */ - log_query(crecp->flags, name, &crecp->addr.addr, record_source(crecp->uid)); + log_query(crecp->flags, name, &crecp->addr, record_source(crecp->uid)); *cut = 0; /* remove domain part */ found = 1; - if (add_resource_record(header, limit, &trunc, nameoffset, &ansp, - daemon->auth_ttl, NULL, qtype, C_IN, + if (add_resource_record(header, limit, &trunc, nameoffset, &ansp, + daemon->auth_ttl, NULL, qtype, C_IN, qtype == T_A ? "4" : "6", &crecp->addr)) anscount++; } } while ((crecp = cache_find_by_name(crecp, name, now, F_IPV4 | F_IPV6))); } - - *cut = '.'; /* restore domain part */ + + *cut = '.'; /* restore domain part */ } - + if ((crecp = cache_find_by_name(NULL, name, now, F_IPV4 | F_IPV6))) { if ((crecp->flags & F_HOSTS) || (((crecp->flags & F_DHCP) && option_bool(OPT_DHCP_FQDN)))) do - { + { nxdomain = 0; - if ((crecp->flags & flag) && (local_query || filter_zone(zone, flag, &(crecp->addr.addr)))) + if ((crecp->flags & flag) && (local_query || filter_zone(zone, flag, &(crecp->addr)))) { - log_query(crecp->flags, name, &crecp->addr.addr, record_source(crecp->uid)); + log_query(crecp->flags, name, &crecp->addr, record_source(crecp->uid)); found = 1; - if (add_resource_record(header, limit, &trunc, nameoffset, &ansp, - daemon->auth_ttl, NULL, qtype, C_IN, + if (add_resource_record(header, limit, &trunc, nameoffset, &ansp, + daemon->auth_ttl, NULL, qtype, C_IN, qtype == T_A ? "4" : "6", &crecp->addr)) anscount++; } } while ((crecp = cache_find_by_name(crecp, name, now, F_IPV4 | F_IPV6))); } - + /* Only supply CNAME if no record for any type is known. */ if (nxdomain) { - /* Check for possible wildcard match against *.domain + /* Check for possible wildcard match against *.domain return length of match, to get longest. Note that if return length of wildcard section, so we match b.simon to _both_ *.simon and b.simon but return a longer (better) match to b.simon. - */ + */ for (wclen = 0, candidate = NULL, a = daemon->cnames; a; a = a->next) if (a->alias[0] == '*') { char *test = name; - + while ((test = strchr(test+1, '.'))) { if (hostname_isequal(test, &(a->alias[1]))) @@ -541,7 +530,7 @@ size_t answer_auth(struct dns_header *header, char *limit, size_t qlen, time_t n break; } } - + } else if (hostname_isequal(a->alias, name) && strlen(a->alias) > wclen) { @@ -549,7 +538,7 @@ size_t answer_auth(struct dns_header *header, char *limit, size_t qlen, time_t n wclen = strlen(a->alias); candidate = a; } - + if (candidate) { log_query(F_CONFIG | F_CNAME, name, NULL, NULL); @@ -560,11 +549,11 @@ size_t answer_auth(struct dns_header *header, char *limit, size_t qlen, time_t n strcat(name, zone->domain); } found = 1; - if (add_resource_record(header, limit, &trunc, nameoffset, &ansp, + if (add_resource_record(header, limit, &trunc, nameoffset, &ansp, daemon->auth_ttl, &nameoffset, T_CNAME, C_IN, "d", name)) anscount++; - + goto cname_restart; } else if (cache_find_non_terminal(name, now)) @@ -572,9 +561,9 @@ size_t answer_auth(struct dns_header *header, char *limit, size_t qlen, time_t n log_query(flag | F_NEG | (nxdomain ? F_NXDOMAIN : 0) | F_FORWARD | F_AUTH, name, NULL, NULL); } - + } - + /* Add auth section */ if (auth && zone) { @@ -586,14 +575,14 @@ size_t answer_auth(struct dns_header *header, char *limit, size_t qlen, time_t n else { /* handle NS and SOA for PTR records */ - + authname = name; if (!(subnet->flags & ADDRLIST_IPV6)) { - in_addr_t a = ntohl(subnet->addr.addr.addr4.s_addr) >> 8; + in_addr_t a = ntohl(subnet->addr.addr4.s_addr) >> 8; char *p = name; - + if (subnet->prefixlen >= 24) p += sprintf(p, "%u.", a & 0xff); a = a >> 8; @@ -601,33 +590,31 @@ size_t answer_auth(struct dns_header *header, char *limit, size_t qlen, time_t n p += sprintf(p, "%u.", a & 0xff); a = a >> 8; p += sprintf(p, "%u.in-addr.arpa", a & 0xff); - + } -#ifdef HAVE_IPV6 else { char *p = name; int i; - + for (i = subnet->prefixlen-1; i >= 0; i -= 4) - { - int dig = ((unsigned char *)&subnet->addr.addr.addr6)[i>>3]; + { + int dig = ((unsigned char *)&subnet->addr.addr6)[i>>3]; p += sprintf(p, "%.1x.", (i>>2) & 1 ? dig & 15 : dig >> 4); } p += sprintf(p, "ip6.arpa"); - + } -#endif } - + /* handle NS and SOA in auth section or for explicit queries */ newoffset = ansp - (unsigned char *)header; if (((anscount == 0 && !ns) || soa) && - add_resource_record(header, limit, &trunc, 0, &ansp, + add_resource_record(header, limit, &trunc, 0, &ansp, daemon->auth_ttl, NULL, T_SOA, C_IN, "ddlllll", authname, daemon->authserver, daemon->hostmaster, - daemon->soa_sn, daemon->soa_refresh, - daemon->soa_retry, daemon->soa_expiry, + daemon->soa_sn, daemon->soa_refresh, + daemon->soa_retry, daemon->soa_expiry, daemon->auth_ttl)) { offset = newoffset; @@ -636,35 +623,39 @@ size_t answer_auth(struct dns_header *header, char *limit, size_t qlen, time_t n else authcount++; } - + if (anscount != 0 || ns) { struct name_list *secondary; - - newoffset = ansp - (unsigned char *)header; - if (add_resource_record(header, limit, &trunc, -offset, &ansp, - daemon->auth_ttl, NULL, T_NS, C_IN, "d", offset == 0 ? authname : NULL, daemon->authserver)) + + /* Only include the machine running dnsmasq if it's acting as an auth server */ + if (daemon->authinterface) { - if (offset == 0) - offset = newoffset; - if (ns) - anscount++; - else - authcount++; + newoffset = ansp - (unsigned char *)header; + if (add_resource_record(header, limit, &trunc, -offset, &ansp, + daemon->auth_ttl, NULL, T_NS, C_IN, "d", offset == 0 ? authname : NULL, daemon->authserver)) + { + if (offset == 0) + offset = newoffset; + if (ns) + anscount++; + else + authcount++; + } } if (!subnet) for (secondary = daemon->secondary_forward_server; secondary; secondary = secondary->next) - if (add_resource_record(header, limit, &trunc, offset, &ansp, + if (add_resource_record(header, limit, &trunc, offset, &ansp, daemon->auth_ttl, NULL, T_NS, C_IN, "d", secondary->name)) { - if (ns) + if (ns) anscount++; else authcount++; } } - + if (axfr) { for (rec = daemon->mxnames; rec; rec = rec->next) @@ -678,7 +669,7 @@ size_t answer_auth(struct dns_header *header, char *limit, size_t qlen, time_t n if (add_resource_record(header, limit, &trunc, -axfroffset, &ansp, daemon->auth_ttl, NULL, T_SRV, C_IN, "sssd", cut ? rec->name : NULL, rec->priority, rec->weight, rec->srvport, rec->target)) - + anscount++; } else @@ -687,87 +678,85 @@ size_t answer_auth(struct dns_header *header, char *limit, size_t qlen, time_t n NULL, T_MX, C_IN, "sd", cut ? rec->name : NULL, rec->weight, rec->target)) anscount++; } - + /* restore config data */ if (cut) *cut = '.'; } - + for (txt = daemon->rr; txt; txt = txt->next) if (in_zone(zone, txt->name, &cut)) { if (cut) *cut = 0; - + if (add_resource_record(header, limit, &trunc, -axfroffset, &ansp, daemon->auth_ttl, NULL, txt->class, C_IN, "t", cut ? txt->name : NULL, txt->len, txt->txt)) anscount++; - + /* restore config data */ if (cut) *cut = '.'; } - + for (txt = daemon->txt; txt; txt = txt->next) if (txt->class == C_IN && in_zone(zone, txt->name, &cut)) { if (cut) *cut = 0; - + if (add_resource_record(header, limit, &trunc, -axfroffset, &ansp, daemon->auth_ttl, NULL, T_TXT, C_IN, "t", cut ? txt->name : NULL, txt->len, txt->txt)) anscount++; - + /* restore config data */ if (cut) *cut = '.'; } - + for (na = daemon->naptr; na; na = na->next) if (in_zone(zone, na->name, &cut)) { if (cut) *cut = 0; - - if (add_resource_record(header, limit, &trunc, -axfroffset, &ansp, daemon->auth_ttl, + + if (add_resource_record(header, limit, &trunc, -axfroffset, &ansp, daemon->auth_ttl, NULL, T_NAPTR, C_IN, "sszzzd", cut ? na->name : NULL, na->order, na->pref, na->flags, na->services, na->regexp, na->replace)) anscount++; - + /* restore config data */ if (cut) - *cut = '.'; + *cut = '.'; } - + for (intr = daemon->int_names; intr; intr = intr->next) if (in_zone(zone, intr->name, &cut)) { struct addrlist *addrlist; - + if (cut) *cut = 0; - - for (addrlist = intr->addr; addrlist; addrlist = addrlist->next) + + for (addrlist = intr->addr; addrlist; addrlist = addrlist->next) if (!(addrlist->flags & ADDRLIST_IPV6) && - (local_query || filter_zone(zone, F_IPV4, &addrlist->addr)) && - add_resource_record(header, limit, &trunc, -axfroffset, &ansp, + (local_query || filter_zone(zone, F_IPV4, &addrlist->addr)) && + add_resource_record(header, limit, &trunc, -axfroffset, &ansp, daemon->auth_ttl, NULL, T_A, C_IN, "4", cut ? intr->name : NULL, &addrlist->addr)) anscount++; - -#ifdef HAVE_IPV6 - for (addrlist = intr->addr; addrlist; addrlist = addrlist->next) - if ((addrlist->flags & ADDRLIST_IPV6) && + + for (addrlist = intr->addr; addrlist; addrlist = addrlist->next) + if ((addrlist->flags & ADDRLIST_IPV6) && (local_query || filter_zone(zone, F_IPV6, &addrlist->addr)) && - add_resource_record(header, limit, &trunc, -axfroffset, &ansp, + add_resource_record(header, limit, &trunc, -axfroffset, &ansp, daemon->auth_ttl, NULL, T_AAAA, C_IN, "6", cut ? intr->name : NULL, &addrlist->addr)) anscount++; -#endif - + /* restore config data */ if (cut) - *cut = '.'; + *cut = '.'; } - + for (a = daemon->cnames; a; a = a->next) if (in_zone(zone, a->alias, &cut)) { @@ -777,16 +766,16 @@ size_t answer_auth(struct dns_header *header, char *limit, size_t qlen, time_t n strcat(name, "."); strcat(name, zone->domain); } - + if (cut) *cut = 0; - - if (add_resource_record(header, limit, &trunc, -axfroffset, &ansp, + + if (add_resource_record(header, limit, &trunc, -axfroffset, &ansp, daemon->auth_ttl, NULL, T_CNAME, C_IN, "d", cut ? a->alias : NULL, name)) anscount++; } - + cache_enumerate(1); while ((crecp = cache_enumerate(0))) { @@ -797,57 +786,45 @@ size_t answer_auth(struct dns_header *header, char *limit, size_t qlen, time_t n if ((crecp->flags & F_DHCP) && !option_bool(OPT_DHCP_FQDN)) { char *cache_name = cache_get_name(crecp); - if (!strchr(cache_name, '.') && - (local_query || filter_zone(zone, (crecp->flags & (F_IPV6 | F_IPV4)), &(crecp->addr.addr)))) - { - qtype = T_A; -#ifdef HAVE_IPV6 - if (crecp->flags & F_IPV6) - qtype = T_AAAA; -#endif - if (add_resource_record(header, limit, &trunc, -axfroffset, &ansp, - daemon->auth_ttl, NULL, qtype, C_IN, - (crecp->flags & F_IPV4) ? "4" : "6", cache_name, &crecp->addr)) - anscount++; - } + if (!strchr(cache_name, '.') && + (local_query || filter_zone(zone, (crecp->flags & (F_IPV6 | F_IPV4)), &(crecp->addr))) && + add_resource_record(header, limit, &trunc, -axfroffset, &ansp, + daemon->auth_ttl, NULL, (crecp->flags & F_IPV6) ? T_AAAA : T_A, C_IN, + (crecp->flags & F_IPV4) ? "4" : "6", cache_name, &crecp->addr)) + anscount++; } - + if ((crecp->flags & F_HOSTS) || (((crecp->flags & F_DHCP) && option_bool(OPT_DHCP_FQDN)))) { strcpy(name, cache_get_name(crecp)); - if (in_zone(zone, name, &cut) && - (local_query || filter_zone(zone, (crecp->flags & (F_IPV6 | F_IPV4)), &(crecp->addr.addr)))) + if (in_zone(zone, name, &cut) && + (local_query || filter_zone(zone, (crecp->flags & (F_IPV6 | F_IPV4)), &(crecp->addr)))) { - qtype = T_A; -#ifdef HAVE_IPV6 - if (crecp->flags & F_IPV6) - qtype = T_AAAA; -#endif - if (cut) - *cut = 0; - - if (add_resource_record(header, limit, &trunc, -axfroffset, &ansp, - daemon->auth_ttl, NULL, qtype, C_IN, - (crecp->flags & F_IPV4) ? "4" : "6", cut ? name : NULL, &crecp->addr)) - anscount++; + if (cut) + *cut = 0; + + if (add_resource_record(header, limit, &trunc, -axfroffset, &ansp, + daemon->auth_ttl, NULL, (crecp->flags & F_IPV6) ? T_AAAA : T_A, C_IN, + (crecp->flags & F_IPV4) ? "4" : "6", cut ? name : NULL, &crecp->addr)) + anscount++; } } } } - + /* repeat SOA as last record */ - if (add_resource_record(header, limit, &trunc, axfroffset, &ansp, + if (add_resource_record(header, limit, &trunc, axfroffset, &ansp, daemon->auth_ttl, NULL, T_SOA, C_IN, "ddlllll", daemon->authserver, daemon->hostmaster, - daemon->soa_sn, daemon->soa_refresh, - daemon->soa_retry, daemon->soa_expiry, + daemon->soa_sn, daemon->soa_refresh, + daemon->soa_retry, daemon->soa_expiry, daemon->auth_ttl)) anscount++; - + } - + } - + /* done all questions, set up header and return length of result */ /* clear authoritative and truncated flags, set QR flag */ header->hb3 = (header->hb3 & ~(HB3_AA | HB3_TC)) | HB3_QR; @@ -863,14 +840,17 @@ size_t answer_auth(struct dns_header *header, char *limit, size_t qlen, time_t n header->hb4 &= ~HB4_RA; } + /* data is never DNSSEC signed. */ + header->hb4 &= ~HB4_AD; + /* authoritative */ if (auth) header->hb3 |= HB3_AA; - + /* truncation */ if (trunc) header->hb3 |= HB3_TC; - + if ((auth || local_query) && nxdomain) SET_RCODE(header, NXDOMAIN); else @@ -885,8 +865,8 @@ size_t answer_auth(struct dns_header *header, char *limit, size_t qlen, time_t n return ansp - (unsigned char *)header; } - -#endif - + +#endif + diff --git a/dnsmasq/blockdata.c b/src/dnsmasq/blockdata.c similarity index 75% rename from dnsmasq/blockdata.c rename to src/dnsmasq/blockdata.c index 11d490aa8..f33b28e84 100644 --- a/dnsmasq/blockdata.c +++ b/src/dnsmasq/blockdata.c @@ -1,4 +1,4 @@ -/* dnsmasq is Copyright (c) 2000-2018 Simon Kelley +/* dnsmasq is Copyright (c) 2000-2020 Simon Kelley This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -16,19 +16,17 @@ #include "dnsmasq.h" -#ifdef HAVE_DNSSEC - static struct blockdata *keyblock_free; static unsigned int blockdata_count, blockdata_hwm, blockdata_alloced; static void blockdata_expand(int n) { struct blockdata *new = whine_malloc(n * sizeof(struct blockdata)); - + if (new) { int i; - + new[n-1].next = keyblock_free; keyblock_free = new; @@ -47,21 +45,20 @@ void blockdata_init(void) blockdata_count = 0; blockdata_hwm = 0; - /* Note that daemon->cachesize is enforced to have non-zero size if OPT_DNSSEC_VALID is set */ + /* Note that daemon->cachesize is enforced to have non-zero size if OPT_DNSSEC_VALID is set */ if (option_bool(OPT_DNSSEC_VALID)) blockdata_expand(daemon->cachesize); } void blockdata_report(void) { - if (option_bool(OPT_DNSSEC_VALID)) - my_syslog(LOG_INFO, _("DNSSEC memory in use %u, max %u, allocated %u"), - blockdata_count * sizeof(struct blockdata), - blockdata_hwm * sizeof(struct blockdata), - blockdata_alloced * sizeof(struct blockdata)); + my_syslog(LOG_INFO, _("pool memory in use %u, max %u, allocated %u"), + blockdata_count * sizeof(struct blockdata), + blockdata_hwm * sizeof(struct blockdata), + blockdata_alloced * sizeof(struct blockdata)); } -struct blockdata *blockdata_alloc(char *data, size_t len) +static struct blockdata *blockdata_alloc_real(int fd, char *data, size_t len) { struct blockdata *block, *ret = NULL; struct blockdata **prev = &ret; @@ -89,8 +86,17 @@ struct blockdata *blockdata_alloc(char *data, size_t len) blockdata_hwm = blockdata_count; blen = len > KEYBLOCK_LEN ? KEYBLOCK_LEN : len; - memcpy(block->key, data, blen); - data += blen; + if (data) + { + memcpy(block->key, data, blen); + data += blen; + } + else if (!read_write(fd, block->key, blen, 1)) + { + /* failed read free partial chain */ + blockdata_free(ret); + return NULL; + } len -= blen; *prev = block; prev = &block->next; @@ -100,6 +106,10 @@ struct blockdata *blockdata_alloc(char *data, size_t len) return ret; } +struct blockdata *blockdata_alloc(char *data, size_t len) +{ + return blockdata_alloc_real(0, data, len); +} void blockdata_free(struct blockdata *blocks) { @@ -148,5 +158,20 @@ void *blockdata_retrieve(struct blockdata *block, size_t len, void *data) return data; } - -#endif + + +void blockdata_write(struct blockdata *block, size_t len, int fd) +{ + for (; len > 0 && block; block = block->next) + { + size_t blen = len > KEYBLOCK_LEN ? KEYBLOCK_LEN : len; + read_write(fd, block->key, blen, 0); + len -= blen; + } +} + +struct blockdata *blockdata_read(int fd, size_t len) +{ + return blockdata_alloc_real(fd, NULL, len); +} + diff --git a/dnsmasq/bpf.c b/src/dnsmasq/bpf.c similarity index 89% rename from dnsmasq/bpf.c rename to src/dnsmasq/bpf.c index 6e68394d4..0ac1c8fd3 100644 --- a/dnsmasq/bpf.c +++ b/src/dnsmasq/bpf.c @@ -1,15 +1,15 @@ -/* dnsmasq is Copyright (c) 2000-2018 Simon Kelley +/* dnsmasq is Copyright (c) 2000-2020 Simon Kelley This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; version 2 dated June, 1991, or (at your option) version 3 dated 29 June, 2007. - + This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. - + You should have received a copy of the GNU General Public License along with this program. If not, see . */ @@ -28,12 +28,10 @@ #include #include #if defined(__FreeBSD__) -# include +# include #endif #include -#ifdef HAVE_IPV6 -# include -#endif +#include #ifndef SA_SIZE #define SA_SIZE(sa) \ @@ -44,7 +42,7 @@ #ifdef HAVE_BSD_NETWORK static int del_family = 0; -static struct all_addr del_addr; +static union all_addr del_addr; #endif #if defined(HAVE_BSD_NETWORK) && !defined(__APPLE__) @@ -72,11 +70,11 @@ int arp_enumerate(void *parm, int (*callback)()) mib[5] = RTF_LLINFO; #else mib[5] = 0; -#endif +#endif if (sysctl(mib, 6, NULL, &needed, NULL, 0) == -1 || needed == 0) return 0; - while (1) + while (1) { if (!expand_buf(&buff, needed)) return 0; @@ -87,7 +85,7 @@ int arp_enumerate(void *parm, int (*callback)()) } if (rc == -1) return 0; - + for (next = buff.iov_base ; next < (char *)buff.iov_base + needed; next += rtm->rtm_msglen) { rtm = (struct rt_msghdr *)next; @@ -121,18 +119,18 @@ int iface_enumerate(int family, void *parm, int (*callback)()) if (getifaddrs(&head) == -1) return 0; -#if defined(HAVE_BSD_NETWORK) && defined(HAVE_IPV6) +#if defined(HAVE_BSD_NETWORK) if (family == AF_INET6) fd = socket(PF_INET6, SOCK_DGRAM, 0); #endif - + for (addrs = head; addrs; addrs = addrs->ifa_next) { if (addrs->ifa_addr->sa_family == family) { int iface_index = if_nametoindex(addrs->ifa_name); - if (iface_index == 0 || !addrs->ifa_addr || + if (iface_index == 0 || !addrs->ifa_addr || (!addrs->ifa_netmask && family != AF_LINK)) continue; @@ -141,18 +139,17 @@ int iface_enumerate(int family, void *parm, int (*callback)()) struct in_addr addr, netmask, broadcast; addr = ((struct sockaddr_in *) addrs->ifa_addr)->sin_addr; #ifdef HAVE_BSD_NETWORK - if (del_family == AF_INET && del_addr.addr.addr4.s_addr == addr.s_addr) + if (del_family == AF_INET && del_addr.addr4.s_addr == addr.s_addr) continue; #endif netmask = ((struct sockaddr_in *) addrs->ifa_netmask)->sin_addr; if (addrs->ifa_broadaddr) - broadcast = ((struct sockaddr_in *) addrs->ifa_broadaddr)->sin_addr; - else - broadcast.s_addr = 0; + broadcast = ((struct sockaddr_in *) addrs->ifa_broadaddr)->sin_addr; + else + broadcast.s_addr = 0; if (!((*callback)(addr, iface_index, NULL, netmask, broadcast, parm))) goto err; } -#ifdef HAVE_IPV6 else if (family == AF_INET6) { struct in6_addr *addr = &((struct sockaddr_in6 *) addrs->ifa_addr)->sin6_addr; @@ -162,7 +159,7 @@ int iface_enumerate(int family, void *parm, int (*callback)()) u32 valid = 0xffffffff, preferred = 0xffffffff; int flags = 0; #ifdef HAVE_BSD_NETWORK - if (del_family == AF_INET6 && IN6_ARE_ADDR_EQUAL(&del_addr.addr.addr6, addr)) + if (del_family == AF_INET6 && IN6_ARE_ADDR_EQUAL(&del_addr.addr6, addr)) continue; #endif #if defined(HAVE_BSD_NETWORK) && !defined(__APPLE__) @@ -170,13 +167,13 @@ int iface_enumerate(int family, void *parm, int (*callback)()) memset(&ifr6, 0, sizeof(ifr6)); safe_strncpy(ifr6.ifr_name, addrs->ifa_name, sizeof(ifr6.ifr_name)); - + ifr6.ifr_addr = *((struct sockaddr_in6 *) addrs->ifa_addr); if (fd != -1 && ioctl(fd, SIOCGIFAFLAG_IN6, &ifr6) != -1) { if (ifr6.ifr_ifru.ifru_flags6 & IN6_IFF_TENTATIVE) flags |= IFACE_TENTATIVE; - + if (ifr6.ifr_ifru.ifru_flags6 & IN6_IFF_DEPRECATED) flags |= IFACE_DEPRECATED; @@ -190,7 +187,7 @@ int iface_enumerate(int family, void *parm, int (*callback)()) flags |= IFACE_PERMANENT; #endif } - + ifr6.ifr_addr = *((struct sockaddr_in6 *) addrs->ifa_addr); if (fd != -1 && ioctl(fd, SIOCGIFALIFETIME_IN6, &ifr6) != -1) { @@ -198,47 +195,46 @@ int iface_enumerate(int family, void *parm, int (*callback)()) preferred = ifr6.ifr_ifru.ifru_lifetime.ia6t_pltime; } #endif - - for (i = 0; i < IN6ADDRSZ; i++, prefix += 8) + + for (i = 0; i < IN6ADDRSZ; i++, prefix += 8) if (netmask[i] != 0xff) break; - - if (i != IN6ADDRSZ && netmask[i]) - for (j = 7; j > 0; j--, prefix++) + + if (i != IN6ADDRSZ && netmask[i]) + for (j = 7; j > 0; j--, prefix++) if ((netmask[i] & (1 << j)) == 0) break; - + /* voodoo to clear interface field in address */ if (!option_bool(OPT_NOWILD) && IN6_IS_ADDR_LINKLOCAL(addr)) { addr->s6_addr[2] = 0; addr->s6_addr[3] = 0; - } - + } + if (!((*callback)(addr, prefix, scope_id, iface_index, flags, (int) preferred, (int)valid, parm))) - goto err; + goto err; } -#endif /* HAVE_IPV6 */ -#ifdef HAVE_DHCP6 +#ifdef HAVE_DHCP6 else if (family == AF_LINK) - { + { /* Assume ethernet again here */ struct sockaddr_dl *sdl = (struct sockaddr_dl *) addrs->ifa_addr; - if (sdl->sdl_alen != 0 && + if (sdl->sdl_alen != 0 && !((*callback)(iface_index, ARPHRD_ETHER, LLADDR(sdl), sdl->sdl_alen, parm))) goto err; } -#endif +#endif } } - + ret = 1; err: errsave = errno; - freeifaddrs(head); + freeifaddrs(head); if (fd != -1) close(fd); errno = errsave; @@ -255,7 +251,7 @@ void init_bpf(void) { int i = 0; - while (1) + while (1) { sprintf(daemon->dhcp_buff, "/dev/bpf%d", i++); if ((daemon->dhcp_raw_fd = open(daemon->dhcp_buff, O_RDWR, 0)) != -1) @@ -263,7 +259,7 @@ void init_bpf(void) if (errno != EBUSY) die(_("cannot create DHCP BPF socket: %s"), NULL, EC_BADNET); - } + } } void send_via_bpf(struct dhcp_packet *mess, size_t len, @@ -271,11 +267,11 @@ void send_via_bpf(struct dhcp_packet *mess, size_t len, { /* Hairy stuff, packet either has to go to the net broadcast or the destination can't reply to ARP yet, - but we do know the physical address. + but we do know the physical address. Build the packet by steam, and send directly, bypassing the kernel IP stack */ - - struct ether_header ether; + + struct ether_header ether; struct ip ip; struct udphdr { u16 uh_sport; /* source port */ @@ -283,25 +279,25 @@ void send_via_bpf(struct dhcp_packet *mess, size_t len, u16 uh_ulen; /* udp length */ u16 uh_sum; /* udp checksum */ } udp; - + u32 i, sum; struct iovec iov[4]; /* Only know how to do ethernet on *BSD */ if (mess->htype != ARPHRD_ETHER || mess->hlen != ETHER_ADDR_LEN) { - my_syslog(MS_DHCP | LOG_WARNING, _("DHCP request for unsupported hardware type (%d) received on %s"), + my_syslog(MS_DHCP | LOG_WARNING, _("DHCP request for unsupported hardware type (%d) received on %s"), mess->htype, ifr->ifr_name); return; } - + ifr->ifr_addr.sa_family = AF_LINK; if (ioctl(daemon->dhcpfd, SIOCGIFADDR, ifr) < 0) return; - + memcpy(ether.ether_shost, LLADDR((struct sockaddr_dl *)&ifr->ifr_addr), ETHER_ADDR_LEN); ether.ether_type = htons(ETHERTYPE_IP); - + if (ntohs(mess->flags) & 0x8000) { memset(ether.ether_dhost, 255, ETHER_ADDR_LEN); @@ -309,13 +305,13 @@ void send_via_bpf(struct dhcp_packet *mess, size_t len, } else { - memcpy(ether.ether_dhost, mess->chaddr, ETHER_ADDR_LEN); + memcpy(ether.ether_dhost, mess->chaddr, ETHER_ADDR_LEN); ip.ip_dst.s_addr = mess->yiaddr.s_addr; } - + ip.ip_p = IPPROTO_UDP; ip.ip_src.s_addr = iface_addr.s_addr; - ip.ip_len = htons(sizeof(struct ip) + + ip.ip_len = htons(sizeof(struct ip) + sizeof(struct udphdr) + len) ; ip.ip_hl = sizeof(struct ip) / 4; @@ -328,9 +324,9 @@ void send_via_bpf(struct dhcp_packet *mess, size_t len, for (sum = 0, i = 0; i < sizeof(struct ip) / 2; i++) sum += ((u16 *)&ip)[i]; while (sum>>16) - sum = (sum & 0xffff) + (sum >> 16); + sum = (sum & 0xffff) + (sum >> 16); ip.ip_sum = (sum == 0xffff) ? sum : ~sum; - + udp.uh_sport = htons(daemon->dhcp_server_port); udp.uh_dport = htons(daemon->dhcp_client_port); if (len & 1) @@ -349,9 +345,9 @@ void send_via_bpf(struct dhcp_packet *mess, size_t len, while (sum>>16) sum = (sum & 0xffff) + (sum >> 16); udp.uh_sum = (sum == 0xffff) ? sum : ~sum; - + ioctl(daemon->dhcp_raw_fd, BIOCSETIF, ifr); - + iov[0].iov_base = ðer; iov[0].iov_len = sizeof(ether); iov[1].iov_base = &ip; @@ -365,7 +361,7 @@ void send_via_bpf(struct dhcp_packet *mess, size_t len, } #endif /* defined(HAVE_BSD_NETWORK) && defined(HAVE_DHCP) */ - + #ifdef HAVE_BSD_NETWORK @@ -373,7 +369,7 @@ void route_init(void) { /* AF_UNSPEC: all addr families */ daemon->routefd = socket(PF_ROUTE, SOCK_RAW, AF_UNSPEC); - + if (daemon->routefd == -1 || !fix_fd(daemon->routefd)) die(_("cannot create PF_ROUTE socket: %s"), NULL, EC_BADNET); } @@ -387,7 +383,7 @@ void route_sock(void) return; msg = (struct if_msghdr *)daemon->packet; - + if (rc < msg->ifm_msglen) return; @@ -415,32 +411,30 @@ void route_sock(void) RTA_IFP, RTA_IFA, RTA_AUTHOR, RTA_BRD }; int of; unsigned int i; - - for (i = 0, of = sizeof(struct ifa_msghdr); of < rc && i < sizeof(maskvec)/sizeof(maskvec[0]); i++) - if (mask & maskvec[i]) + + for (i = 0, of = sizeof(struct ifa_msghdr); of < rc && i < sizeof(maskvec)/sizeof(maskvec[0]); i++) + if (mask & maskvec[i]) { struct sockaddr *sa = (struct sockaddr *)((char *)msg + of); size_t diff = (sa->sa_len != 0) ? sa->sa_len : sizeof(long); - + if (maskvec[i] == RTA_IFA) { del_family = sa->sa_family; if (del_family == AF_INET) - del_addr.addr.addr4 = ((struct sockaddr_in *)sa)->sin_addr; -#ifdef HAVE_IPV6 + del_addr.addr4 = ((struct sockaddr_in *)sa)->sin_addr; else if (del_family == AF_INET6) - del_addr.addr.addr6 = ((struct sockaddr_in6 *)sa)->sin6_addr; -#endif + del_addr.addr6 = ((struct sockaddr_in6 *)sa)->sin6_addr; else del_family = 0; } - + of += diff; /* round up as needed */ - if (diff & (sizeof(long) - 1)) + if (diff & (sizeof(long) - 1)) of += sizeof(long) - (diff & (sizeof(long) - 1)); } - + queue_event(EVENT_NEWADDR); } } diff --git a/dnsmasq/cache.c b/src/dnsmasq/cache.c similarity index 75% rename from dnsmasq/cache.c rename to src/dnsmasq/cache.c index 77afd3919..19a80677b 100644 --- a/dnsmasq/cache.c +++ b/src/dnsmasq/cache.c @@ -1,15 +1,15 @@ -/* dnsmasq is Copyright (c) 2000-2018 Simon Kelley +/* dnsmasq is Copyright (c) 2000-2020 Simon Kelley This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; version 2 dated June, 1991, or (at your option) version 3 dated 29 June, 2007. - + This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. - + You should have received a copy of the GNU General Public License along with this program. If not, see . */ @@ -27,6 +27,8 @@ static union bigname *big_free = NULL; static int bignames_left, hash_size; static void make_non_terminals(struct crec *source); +static struct crec *really_insert(char *name, union all_addr *addr, unsigned short class, + time_t now, unsigned long ttl, unsigned int flags); /* type->string mapping: this is also used by the name-hash function as a mixing table. */ static const struct { @@ -40,7 +42,7 @@ static const struct { { 10, "NULL" }, { 11, "WKS" }, { 12, "PTR" }, - { 13, "HINFO" }, + { 13, "HINFO" }, { 15, "MX" }, { 16, "TXT" }, { 22, "NSAP" }, @@ -88,11 +90,11 @@ void next_uid(struct crec *crecp) if (crecp->uid == UID_NONE) { uid++; - + /* uid == 0 used to indicate CNAME to interface name. */ if (uid == UID_NONE) uid++; - + crecp->uid = uid; } } @@ -101,13 +103,13 @@ void cache_init(void) { struct crec *crecp; int i; - + bignames_left = daemon->cachesize/10; - + if (daemon->cachesize > 0) { crecp = safe_malloc(daemon->cachesize*sizeof(struct crec)); - + for (i=0; i < daemon->cachesize; i++, crecp++) { cache_link(crecp); @@ -115,7 +117,7 @@ void cache_init(void) crecp->uid = UID_NONE; } } - + /* create initial hash table*/ rehash(daemon->cachesize); } @@ -131,7 +133,7 @@ void rehash(int size) /* hash_size is a power of two. */ for (new_size = 64; new_size < size/10; new_size = new_size << 1); - + /* must succeed in getting first instance, failure later is non-fatal */ if (!hash_table) new = safe_malloc(new_size * sizeof(struct crec *)); @@ -145,7 +147,7 @@ void rehash(int size) old_size = hash_size; hash_table = new; hash_size = new_size; - + if (old) { for (i = 0; i < old_size; i++) @@ -157,11 +159,11 @@ void rehash(int size) free(old); } } - + static struct crec **hash_bucket(char *name) { unsigned int c, val = 017465; /* Barker code - minimum self-correlation in cyclic shift */ - const unsigned char *mix_tab = (const unsigned char*)typestr; + const unsigned char *mix_tab = (const unsigned char*)typestr; while((c = (unsigned char) *name++)) { @@ -169,8 +171,8 @@ static struct crec **hash_bucket(char *name) if (c >= 'A' && c <= 'Z') c += 'a' - 'A'; val = ((val << 7) | (val >> (32 - 7))) + (mix_tab[(val + c) & 0x3F] ^ c); - } - + } + /* hash_size is a power of two */ return hash_table + ((val ^ (val >> 16)) & (hash_size - 1)); } @@ -187,8 +189,8 @@ static void cache_hash(struct crec *crecp) if (!(crecp->flags & F_REVERSE)) { while (*up && ((*up)->flags & F_REVERSE)) - up = &((*up)->hash_next); - + up = &((*up)->hash_next); + if (crecp->flags & F_IMMORTAL) while (*up && !((*up)->flags & F_IMMORTAL)) up = &((*up)->hash_next); @@ -197,15 +199,20 @@ static void cache_hash(struct crec *crecp) *up = crecp; } -#ifdef HAVE_DNSSEC static void cache_blockdata_free(struct crec *crecp) { - if (crecp->flags & F_DNSKEY) - blockdata_free(crecp->addr.key.keydata); - else if ((crecp->flags & F_DS) && !(crecp->flags & F_NEG)) - blockdata_free(crecp->addr.ds.keydata); -} + if (!(crecp->flags & F_NEG)) + { + if (crecp->flags & F_SRV) + blockdata_free(crecp->addr.srv.target); +#ifdef HAVE_DNSSEC + else if (crecp->flags & F_DNSKEY) + blockdata_free(crecp->addr.key.keydata); + else if (crecp->flags & F_DS) + blockdata_free(crecp->addr.ds.keydata); #endif + } +} static void cache_free(struct crec *crecp) { @@ -220,7 +227,7 @@ static void cache_free(struct crec *crecp) crecp->prev = cache_tail; crecp->next = NULL; cache_tail = crecp; - + /* retrieve big name for further use. */ if (crecp->flags & F_BIGNAME) { @@ -229,10 +236,8 @@ static void cache_free(struct crec *crecp) crecp->flags &= ~F_BIGNAME; } -#ifdef HAVE_DNSSEC cache_blockdata_free(crecp); -#endif -} +} /* insert a new cache entry at the head of the list (youngest entry) */ static void cache_link(struct crec *crecp) @@ -246,7 +251,7 @@ static void cache_link(struct crec *crecp) cache_tail = crecp; } -/* remove an arbitrary cache entry for promotion */ +/* remove an arbitrary cache entry for promotion */ static void cache_unlink (struct crec *crecp) { if (crecp->prev) @@ -264,18 +269,18 @@ char *cache_get_name(struct crec *crecp) { if (crecp->flags & F_BIGNAME) return crecp->name.bname->name; - else if (crecp->flags & F_NAMEP) + else if (crecp->flags & F_NAMEP) return crecp->name.namep; - + return crecp->name.sname; } char *cache_get_cname_target(struct crec *crecp) { - if (crecp->addr.cname.uid != SRC_INTERFACE) + if (crecp->addr.cname.is_name_ptr) + return crecp->addr.cname.target.name; + else return cache_get_name(crecp->addr.cname.target.cache); - - return crecp->addr.cname.target.int_name->name; } @@ -294,27 +299,27 @@ struct crec *cache_enumerate(int init) cache = cache->hash_next; else { - cache = NULL; + cache = NULL; while (bucket < hash_size) if ((cache = hash_table[bucket++])) break; } - + return cache; } static int is_outdated_cname_pointer(struct crec *crecp) { - if (!(crecp->flags & F_CNAME) || crecp->addr.cname.uid == SRC_INTERFACE) + if (!(crecp->flags & F_CNAME) || crecp->addr.cname.is_name_ptr) return 0; - - /* NB. record may be reused as DS or DNSKEY, where uid is + + /* NB. record may be reused as DS or DNSKEY, where uid is overloaded for something completely different */ - if (crecp->addr.cname.target.cache && - (crecp->addr.cname.target.cache->flags & (F_IPV4 | F_IPV6 | F_CNAME)) && + if (crecp->addr.cname.target.cache && + !(crecp->addr.cname.target.cache->flags & (F_DNSKEY | F_DS)) && crecp->addr.cname.uid == crecp->addr.cname.target.cache->uid) return 0; - + return 1; } @@ -325,31 +330,33 @@ static int is_expired(time_t now, struct crec *crecp) if (difftime(now, crecp->ttd) < 0) return 0; - + return 1; } -static struct crec *cache_scan_free(char *name, struct all_addr *addr, time_t now, unsigned short flags, - struct crec **target_crec, unsigned int *target_uid) +static struct crec *cache_scan_free(char *name, union all_addr *addr, unsigned short class, time_t now, + unsigned int flags, struct crec **target_crec, unsigned int *target_uid) { /* Scan and remove old entries. If (flags & F_FORWARD) then remove any forward entries for name and any expired entries but only in the same hash bucket as name. If (flags & F_REVERSE) then remove any reverse entries for addr and any expired entries in the whole cache. - If (flags == 0) remove any expired entries in the whole cache. + If (flags == 0) remove any expired entries in the whole cache. In the flags & F_FORWARD case, the return code is valid, and returns a non-NULL pointer to a cache entry if the name exists in the cache as a HOSTS or DHCP entry (these are never deleted) We take advantage of the fact that hash chains have stuff in the order ,, - so that when we hit an entry which isn't reverse and is immortal, we're done. + so that when we hit an entry which isn't reverse and is immortal, we're done. If we free a crec which is a CNAME target, return the entry and uid in target_crec and target_uid. This entry will get re-used with the same name, to preserve CNAMEs. */ - + struct crec *crecp, **up; + (void)class; + if (flags & F_FORWARD) { for (up = hash_bucket(name), crecp = *up; crecp; crecp = crecp->hash_next) @@ -357,7 +364,7 @@ static struct crec *cache_scan_free(char *name, struct all_addr *addr, time_t no if ((crecp->flags & F_FORWARD) && hostname_isequal(cache_get_name(crecp), name)) { /* Don't delete DNSSEC in favour of a CNAME, they can co-exist */ - if ((flags & crecp->flags & (F_IPV4 | F_IPV6)) || + if ((flags & crecp->flags & (F_IPV4 | F_IPV6 | F_SRV)) || (((crecp->flags | flags) & F_CNAME) && !(crecp->flags & (F_DNSKEY | F_DS)))) { if (crecp->flags & (F_HOSTS | F_DHCP | F_CONFIG)) @@ -377,10 +384,10 @@ static struct crec *cache_scan_free(char *name, struct all_addr *addr, time_t no cache_free(crecp); continue; } - + #ifdef HAVE_DNSSEC /* Deletion has to be class-sensitive for DS and DNSKEY */ - if ((flags & crecp->flags & (F_DNSKEY | F_DS)) && crecp->uid == addr->addr.dnssec.class) + if ((flags & crecp->flags & (F_DNSKEY | F_DS)) && crecp->uid == class) { if (crecp->flags & F_CONFIG) return crecp; @@ -393,7 +400,7 @@ static struct crec *cache_scan_free(char *name, struct all_addr *addr, time_t no } if (is_expired(now, crecp) || is_outdated_cname_pointer(crecp)) - { + { *up = crecp->hash_next; if (!(crecp->flags & (F_HOSTS | F_DHCP | F_CONFIG))) { @@ -401,36 +408,33 @@ static struct crec *cache_scan_free(char *name, struct all_addr *addr, time_t no cache_free(crecp); } continue; - } - + } + up = &crecp->hash_next; } } else { int i; -#ifdef HAVE_IPV6 int addrlen = (flags & F_IPV6) ? IN6ADDRSZ : INADDRSZ; -#else - int addrlen = INADDRSZ; -#endif + for (i = 0; i < hash_size; i++) - for (crecp = hash_table[i], up = &hash_table[i]; + for (crecp = hash_table[i], up = &hash_table[i]; crecp && ((crecp->flags & F_REVERSE) || !(crecp->flags & F_IMMORTAL)); crecp = crecp->hash_next) if (is_expired(now, crecp)) { *up = crecp->hash_next; if (!(crecp->flags & (F_HOSTS | F_DHCP | F_CONFIG))) - { + { cache_unlink(crecp); cache_free(crecp); } } else if (!(crecp->flags & (F_HOSTS | F_DHCP | F_CONFIG)) && - (flags & crecp->flags & F_REVERSE) && + (flags & crecp->flags & F_REVERSE) && (flags & crecp->flags & (F_IPV4 | F_IPV6)) && - memcmp(&crecp->addr.addr, addr, addrlen) == 0) + memcmp(&crecp->addr, addr, addrlen) == 0) { *up = crecp->hash_next; cache_unlink(crecp); @@ -439,7 +443,7 @@ static struct crec *cache_scan_free(char *name, struct all_addr *addr, time_t no else up = &crecp->hash_next; } - + return NULL; } @@ -448,7 +452,7 @@ static struct crec *cache_scan_free(char *name, struct all_addr *addr, time_t no cache_insert * n cache_end_insert - but an abort can cause the cache_end_insert to be missed + but an abort can cause the cache_end_insert to be missed in which can the next cache_start_insert cleans things up. */ void cache_start_insert(void) @@ -466,17 +470,11 @@ void cache_start_insert(void) insert_error = 0; } -struct crec *cache_insert(char *name, struct all_addr *addr, - time_t now, unsigned long ttl, unsigned short flags) +struct crec *cache_insert(char *name, union all_addr *addr, unsigned short class, + time_t now, unsigned long ttl, unsigned int flags) { - struct crec *new, *target_crec = NULL; - union bigname *big_name = NULL; - int freed_all = flags & F_REVERSE; - int free_avail = 0; - unsigned int target_uid; - /* Don't log DNSSEC records here, done elsewhere */ - if (flags & (F_IPV4 | F_IPV6 | F_CNAME)) + if (flags & (F_IPV4 | F_IPV6 | F_CNAME | F_SRV)) { log_query(flags | F_UPSTREAM, name, addr, NULL); /* Don't mess with TTL for DNSSEC records. */ @@ -486,36 +484,54 @@ struct crec *cache_insert(char *name, struct all_addr *addr, ttl = daemon->min_cache_ttl; FTL_reply(flags, name, addr, daemon->log_display_id); } + + return really_insert(name, addr, class, now, ttl, flags); +} + +static struct crec *really_insert(char *name, union all_addr *addr, unsigned short class, + time_t now, unsigned long ttl, unsigned int flags) +{ + struct crec *new, *target_crec = NULL; + union bigname *big_name = NULL; + int freed_all = flags & F_REVERSE; + int free_avail = 0; + unsigned int target_uid; + /* if previous insertion failed give up now. */ if (insert_error) return NULL; + /* we don't cache zero-TTL records. */ + if (ttl == 0) + { + insert_error = 1; + return NULL; + } + /* First remove any expired entries and entries for the name/address we are currently inserting. */ - if ((new = cache_scan_free(name, addr, now, flags, &target_crec, &target_uid))) + if ((new = cache_scan_free(name, addr, class, now, flags, &target_crec, &target_uid))) { - /* We're trying to insert a record over one from - /etc/hosts or DHCP, or other config. If the - existing record is for an A or AAAA and - the record we're trying to insert is the same, + /* We're trying to insert a record over one from + /etc/hosts or DHCP, or other config. If the + existing record is for an A or AAAA or CNAME and + the record we're trying to insert is the same, just drop the insert, but don't error the whole process. */ if ((flags & (F_IPV4 | F_IPV6)) && (flags & F_FORWARD) && addr) { if ((flags & F_IPV4) && (new->flags & F_IPV4) && - new->addr.addr.addr.addr4.s_addr == addr->addr.addr4.s_addr) + new->addr.addr4.s_addr == addr->addr4.s_addr) return new; -#ifdef HAVE_IPV6 else if ((flags & F_IPV6) && (new->flags & F_IPV6) && - IN6_ARE_ADDR_EQUAL(&new->addr.addr.addr.addr6, &addr->addr.addr6)) + IN6_ARE_ADDR_EQUAL(&new->addr.addr6, &addr->addr6)) return new; -#endif } insert_error = 1; return NULL; } - + /* Now get a cache entry from the end of the LRU list */ if (!target_crec) while (1) { @@ -524,7 +540,7 @@ struct crec *cache_insert(char *name, struct all_addr *addr, insert_error = 1; return NULL; } - + /* Free entry at end of LRU list, use it. */ if (!(new->flags & (F_FORWARD | F_REVERSE))) break; @@ -532,7 +548,7 @@ struct crec *cache_insert(char *name, struct all_addr *addr, /* End of LRU list is still in use: if we didn't scan all the hash chains for expired entries do that now. If we already tried that then it's time to start spilling things. */ - + /* If free_avail set, we believe that an entry has been freed. Bugs have been known to make this not true, resulting in a tight loop here. If that happens, abandon the @@ -548,34 +564,27 @@ struct crec *cache_insert(char *name, struct all_addr *addr, insert_error = 1; return NULL; } - + if (freed_all) { - struct all_addr free_addr = new->addr.addr;; - -#ifdef HAVE_DNSSEC - /* For DNSSEC records, addr holds class. */ - if (new->flags & (F_DS | F_DNSKEY)) - free_addr.addr.dnssec.class = new->uid; -#endif - + /* For DNSSEC records, uid holds class. */ free_avail = 1; /* Must be free space now. */ - cache_scan_free(cache_get_name(new), &free_addr, now, new->flags, NULL, NULL); + cache_scan_free(cache_get_name(new), &new->addr, new->uid, now, new->flags, NULL, NULL); daemon->metrics[METRIC_DNS_CACHE_LIVE_FREED]++; } else { - cache_scan_free(NULL, NULL, now, 0, NULL, NULL); + cache_scan_free(NULL, NULL, class, now, 0, NULL, NULL); freed_all = 1; } } - + /* Check if we need to and can allocate extra memory for a long name. If that fails, give up now, always succeed for DNSSEC records. */ if (name && (strlen(name) > SMALLDNAME-1)) { if (big_free) - { + { big_name = big_free; big_free = big_free->next; } @@ -587,7 +596,7 @@ struct crec *cache_insert(char *name, struct all_addr *addr, } else if (bignames_left != 0) bignames_left--; - + } /* If we freed a cache entry for our name which was a CNAME target, use that. @@ -597,10 +606,10 @@ struct crec *cache_insert(char *name, struct all_addr *addr, new = target_crec; new->uid = target_uid; } - + /* Got the rest: finally grab entry. */ cache_unlink(new); - + new->flags = flags; if (big_name) { @@ -613,15 +622,13 @@ struct crec *cache_insert(char *name, struct all_addr *addr, else *cache_get_name(new) = 0; - if (addr) - { #ifdef HAVE_DNSSEC - if (flags & (F_DS | F_DNSKEY)) - new->uid = addr->addr.dnssec.class; - else + if (flags & (F_DS | F_DNSKEY)) + new->uid = class; #endif - new->addr.addr = *addr; - } + + if (addr) + new->addr = *addr; new->ttd = now + (time_t)ttl; new->next = new_chain; @@ -635,9 +642,9 @@ void cache_end_insert(void) { if (insert_error) return; - + while (new_chain) - { + { struct crec *tmp = new_chain->next; /* drop CNAMEs which didn't find a target. */ if (is_outdated_cname_pointer(new_chain)) @@ -647,12 +654,145 @@ void cache_end_insert(void) cache_hash(new_chain); cache_link(new_chain); daemon->metrics[METRIC_DNS_CACHE_INSERTED]++; + + /* If we're a child process, send this cache entry up the pipe to the master. + The marshalling process is rather nasty. */ + if (daemon->pipe_to_parent != -1) + { + char *name = cache_get_name(new_chain); + ssize_t m = strlen(name); + unsigned int flags = new_chain->flags; +#ifdef HAVE_DNSSEC + u16 class = new_chain->uid; +#endif + + read_write(daemon->pipe_to_parent, (unsigned char *)&m, sizeof(m), 0); + read_write(daemon->pipe_to_parent, (unsigned char *)name, m, 0); + read_write(daemon->pipe_to_parent, (unsigned char *)&new_chain->ttd, sizeof(new_chain->ttd), 0); + read_write(daemon->pipe_to_parent, (unsigned char *)&flags, sizeof(flags), 0); + + if (flags & (F_IPV4 | F_IPV6 | F_DNSKEY | F_DS | F_SRV)) + read_write(daemon->pipe_to_parent, (unsigned char *)&new_chain->addr, sizeof(new_chain->addr), 0); + if (flags & F_SRV) + { + /* A negative SRV entry is possible and has no data, obviously. */ + if (!(flags & F_NEG)) + blockdata_write(new_chain->addr.srv.target, new_chain->addr.srv.targetlen, daemon->pipe_to_parent); + } +#ifdef HAVE_DNSSEC + if (flags & F_DNSKEY) + { + read_write(daemon->pipe_to_parent, (unsigned char *)&class, sizeof(class), 0); + blockdata_write(new_chain->addr.key.keydata, new_chain->addr.key.keylen, daemon->pipe_to_parent); + } + else if (flags & F_DS) + { + read_write(daemon->pipe_to_parent, (unsigned char *)&class, sizeof(class), 0); + /* A negative DS entry is possible and has no data, obviously. */ + if (!(flags & F_NEG)) + blockdata_write(new_chain->addr.ds.keydata, new_chain->addr.ds.keylen, daemon->pipe_to_parent); + } +#endif + } } + new_chain = tmp; } + + /* signal end of cache insert in master process */ + if (daemon->pipe_to_parent != -1) + { + ssize_t m = -1; + read_write(daemon->pipe_to_parent, (unsigned char *)&m, sizeof(m), 0); + } + new_chain = NULL; } + +/* A marshalled cache entry arrives on fd, read, unmarshall and insert into cache of master process. */ +int cache_recv_insert(time_t now, int fd) +{ + ssize_t m; + union all_addr addr; + unsigned long ttl; + time_t ttd; + unsigned int flags; + struct crec *crecp = NULL; + + cache_start_insert(); + + while(1) + { + + if (!read_write(fd, (unsigned char *)&m, sizeof(m), 1)) + return 0; + + if (m == -1) + { + cache_end_insert(); + return 1; + } + + if (!read_write(fd, (unsigned char *)daemon->namebuff, m, 1) || + !read_write(fd, (unsigned char *)&ttd, sizeof(ttd), 1) || + !read_write(fd, (unsigned char *)&flags, sizeof(flags), 1)) + return 0; + + daemon->namebuff[m] = 0; + + ttl = difftime(ttd, now); + + if (flags & (F_IPV4 | F_IPV6 | F_DNSKEY | F_DS | F_SRV)) + { + unsigned short class = C_IN; + + if (!read_write(fd, (unsigned char *)&addr, sizeof(addr), 1)) + return 0; + + if ((flags & F_SRV) && !(flags & F_NEG) && !(addr.srv.target = blockdata_read(fd, addr.srv.targetlen))) + return 0; + +#ifdef HAVE_DNSSEC + if (flags & F_DNSKEY) + { + if (!read_write(fd, (unsigned char *)&class, sizeof(class), 1) || + !(addr.key.keydata = blockdata_read(fd, addr.key.keylen))) + return 0; + } + else if (flags & F_DS) + { + if (!read_write(fd, (unsigned char *)&class, sizeof(class), 1) || + (!(flags & F_NEG) && !(addr.key.keydata = blockdata_read(fd, addr.key.keylen)))) + return 0; + } +#endif + + crecp = really_insert(daemon->namebuff, &addr, class, now, ttl, flags); + } + else if (flags & F_CNAME) + { + struct crec *newc = really_insert(daemon->namebuff, NULL, C_IN, now, ttl, flags); + /* This relies on the fact that the target of a CNAME immediately precedes + it because of the order of extraction in extract_addresses, and + the order reversal on the new_chain. */ + if (newc) + { + newc->addr.cname.is_name_ptr = 0; + + if (!crecp) + newc->addr.cname.target.cache = NULL; + else + { + next_uid(crecp); + newc->addr.cname.target.cache = crecp; + newc->addr.cname.uid = crecp->uid; + } + } + } + } +} + int cache_find_non_terminal(char *name, time_t now) { struct crec *crecp; @@ -661,6 +801,7 @@ int cache_find_non_terminal(char *name, time_t now) if (!is_outdated_cname_pointer(crecp) && !is_expired(now, crecp) && (crecp->flags & F_FORWARD) && + !(crecp->flags & F_NXDOMAIN) && hostname_isequal(name, cache_get_name(crecp))) return 1; @@ -673,7 +814,7 @@ struct crec *cache_find_by_name(struct crec *crecp, char *name, time_t now, unsi int no_rr = prot & F_NO_RR; prot &= ~F_NO_RR; - + if (crecp) /* iterating */ ans = crecp->next; else @@ -681,15 +822,15 @@ struct crec *cache_find_by_name(struct crec *crecp, char *name, time_t now, unsi /* first search, look for relevant entries and push to top of list also free anything which has expired */ struct crec *next, **up, **insert = NULL, **chainp = &ans; - unsigned short ins_flags = 0; - + unsigned int ins_flags = 0; + for (up = hash_bucket(name), crecp = *up; crecp; crecp = next) { next = crecp->hash_next; - + if (!is_expired(now, crecp) && !is_outdated_cname_pointer(crecp)) { - if ((crecp->flags & F_FORWARD) && + if ((crecp->flags & F_FORWARD) && (crecp->flags & prot) && hostname_isequal(cache_get_name(crecp), name)) { @@ -703,11 +844,11 @@ struct crec *cache_find_by_name(struct crec *crecp, char *name, time_t now, unsi cache_unlink(crecp); cache_link(crecp); } - + /* Move all but the first entry up the hash chain - this implements round-robin. + this implements round-robin. Make sure that re-ordering doesn't break the hash-chain - order invariants. + order invariants. */ if (insert && (crecp->flags & (F_REVERSE | F_IMMORTAL)) == ins_flags) { @@ -723,67 +864,63 @@ struct crec *cache_find_by_name(struct crec *crecp, char *name, time_t now, unsi insert = up; ins_flags = crecp->flags & (F_REVERSE | F_IMMORTAL); } - up = &crecp->hash_next; + up = &crecp->hash_next; } } else /* case : not expired, incorrect entry. */ - up = &crecp->hash_next; + up = &crecp->hash_next; } else { /* expired entry, free it */ *up = crecp->hash_next; if (!(crecp->flags & (F_HOSTS | F_DHCP | F_CONFIG))) - { + { cache_unlink(crecp); cache_free(crecp); } } } - + *chainp = cache_head; } - if (ans && + if (ans && (ans->flags & F_FORWARD) && - (ans->flags & prot) && + (ans->flags & prot) && hostname_isequal(cache_get_name(ans), name)) return ans; - + return NULL; } -struct crec *cache_find_by_addr(struct crec *crecp, struct all_addr *addr, +struct crec *cache_find_by_addr(struct crec *crecp, union all_addr *addr, time_t now, unsigned int prot) { struct crec *ans; -#ifdef HAVE_IPV6 int addrlen = (prot == F_IPV6) ? IN6ADDRSZ : INADDRSZ; -#else - int addrlen = INADDRSZ; -#endif - + if (crecp) /* iterating */ ans = crecp->next; else - { + { /* first search, look for relevant entries and push to top of list also free anything which has expired. All the reverse entries are at the - start of the hash chain, so we can give up when we find the first + start of the hash chain, so we can give up when we find the first non-REVERSE one. */ int i; struct crec **up, **chainp = &ans; - + for (i=0; iflags & F_REVERSE); crecp = crecp->hash_next) if (!is_expired(now, crecp)) - { + { if ((crecp->flags & prot) && - memcmp(&crecp->addr.addr, addr, addrlen) == 0) - { + memcmp(&crecp->addr, addr, addrlen) == 0) + { if (crecp->flags & (F_HOSTS | F_DHCP | F_CONFIG)) { *chainp = crecp; @@ -806,92 +943,64 @@ struct crec *cache_find_by_addr(struct crec *crecp, struct all_addr *addr, cache_free(crecp); } } - + *chainp = cache_head; } - - if (ans && + + if (ans && (ans->flags & F_REVERSE) && (ans->flags & prot) && - memcmp(&ans->addr.addr, addr, addrlen) == 0) + memcmp(&ans->addr, addr, addrlen) == 0) return ans; - + return NULL; } -static void add_hosts_cname(struct crec *target) -{ - struct crec *crec; - struct cname *a; - - for (a = daemon->cnames; a; a = a->next) - if (a->alias[1] != '*' && - hostname_isequal(cache_get_name(target), a->target) && - (crec = whine_malloc(SIZEOF_POINTER_CREC))) - { - crec->flags = F_FORWARD | F_IMMORTAL | F_NAMEP | F_CONFIG | F_CNAME; - crec->ttd = a->ttl; - crec->name.namep = a->alias; - crec->addr.cname.target.cache = target; - next_uid(target); - crec->addr.cname.uid = target->uid; - crec->uid = UID_NONE; - cache_hash(crec); - make_non_terminals(crec); - - add_hosts_cname(crec); /* handle chains */ - } -} - -void add_hosts_entry(struct crec *cache, struct all_addr *addr, int addrlen, - unsigned int index, struct crec **rhash, int hashsz) +void add_hosts_entry(struct crec *cache, union all_addr *addr, int addrlen, + unsigned int index, struct crec **rhash, int hashsz) { struct crec *lookup = cache_find_by_name(NULL, cache_get_name(cache), 0, cache->flags & (F_IPV4 | F_IPV6)); - int i, nameexists = 0; - unsigned int j; + int i; + unsigned int j; /* Remove duplicates in hosts files. */ - if (lookup && (lookup->flags & F_HOSTS)) + if (lookup && (lookup->flags & F_HOSTS) && memcmp(&lookup->addr, addr, addrlen) == 0) { - nameexists = 1; - if (memcmp(&lookup->addr.addr, addr, addrlen) == 0) - { - free(cache); - return; - } + free(cache); + return; } - - /* Ensure there is only one address -> name mapping (first one trumps) + + /* Ensure there is only one address -> name mapping (first one trumps) We do this by steam here, The entries are kept in hash chains, linked by ->next (which is unused at this point) held in hash buckets in the array rhash, hashed on address. Note that rhash and the values in ->next are only valid whilst reading hosts files: the buckets are - then freed, and the ->next pointer used for other things. + then freed, and the ->next pointer used for other things. Only insert each unique address once into this hashing structure. This complexity avoids O(n^2) divergent CPU use whilst reading - large (10000 entry) hosts files. + large (10000 entry) hosts files. - Note that we only do this process when bulk-reading hosts files, + Note that we only do this process when bulk-reading hosts files, for incremental reads, rhash is NULL, and we use cache lookups instead. */ - + if (rhash) { /* hash address */ for (j = 0, i = 0; i < addrlen; i++) j = (j*2 +((unsigned char *)addr)[i]) % hashsz; - + for (lookup = rhash[j]; lookup; lookup = lookup->next) if ((lookup->flags & cache->flags & (F_IPV4 | F_IPV6)) && - memcmp(&lookup->addr.addr, addr, addrlen) == 0) + memcmp(&lookup->addr, addr, addrlen) == 0) { cache->flags &= ~F_REVERSE; break; } - + /* maintain address hash chain, insert new unique address */ if (!lookup) { @@ -908,13 +1017,9 @@ void add_hosts_entry(struct crec *cache, struct all_addr *addr, int addrlen, } cache->uid = index; - memcpy(&cache->addr.addr, addr, addrlen); + memcpy(&cache->addr, addr, addrlen); cache_hash(cache); make_non_terminals(cache); - - /* don't need to do alias stuff for second and subsequent addresses. */ - if (!nameexists) - add_hosts_cname(cache); } static int eatspace(FILE *f) @@ -926,7 +1031,7 @@ static int eatspace(FILE *f) if ((c = getc(f)) == '#') while (c != '\n' && c != EOF) c = getc(f); - + if (c == EOF) return 1; @@ -937,25 +1042,25 @@ static int eatspace(FILE *f) } if (c == '\n') - nl = 1; + nl++; } } - + static int gettok(FILE *f, char *token) { int c, count = 0; - + while (1) { if ((c = getc(f)) == EOF) - return (count == 0) ? EOF : 1; + return (count == 0) ? -1 : 1; if (isspace(c) || c == '#') { ungetc(c, f); return eatspace(f); } - + if (count < (MAXDNAME - 1)) { token[count++] = c; @@ -965,12 +1070,12 @@ static int gettok(FILE *f, char *token) } int read_hostsfile(char *filename, unsigned int index, int cache_size, struct crec **rhash, int hashsz) -{ +{ FILE *f = fopen(filename, "r"); char *token = daemon->namebuff, *domain_suffix = NULL; - int addr_count = 0, name_count = cache_size, lineno = 0; - unsigned short flags = 0; - struct all_addr addr; + int addr_count = 0, name_count = cache_size, lineno = 1; + unsigned int flags = 0; + union all_addr addr; int atnl, addrlen = 0; if (!f) @@ -978,54 +1083,48 @@ int read_hostsfile(char *filename, unsigned int index, int cache_size, struct cr my_syslog(LOG_ERR, _("failed to load names from %s: %s"), filename, strerror(errno)); return cache_size; } - - eatspace(f); - - name_count = FTL_listsfile(filename, index, f, cache_size, rhash, hashsz); - addr_count = name_count - cache_size; - - while ((atnl = gettok(f, token)) != EOF) + + lineno += eatspace(f); + + while ((atnl = gettok(f, token)) != -1) { - lineno++; - if (inet_pton(AF_INET, token, &addr) > 0) { flags = F_HOSTS | F_IMMORTAL | F_FORWARD | F_REVERSE | F_IPV4; addrlen = INADDRSZ; - domain_suffix = get_domain(addr.addr.addr4); + domain_suffix = get_domain(addr.addr4); } -#ifdef HAVE_IPV6 else if (inet_pton(AF_INET6, token, &addr) > 0) { flags = F_HOSTS | F_IMMORTAL | F_FORWARD | F_REVERSE | F_IPV6; addrlen = IN6ADDRSZ; - domain_suffix = get_domain6(&addr.addr.addr6); + domain_suffix = get_domain6(&addr.addr6); } -#endif else { - my_syslog(LOG_ERR, _("bad address at %s line %d"), filename, lineno); + my_syslog(LOG_ERR, _("bad address at %s line %d"), filename, lineno); while (atnl == 0) atnl = gettok(f, token); + lineno += atnl; continue; } - + addr_count++; - + /* rehash every 1000 names. */ if (rhash && ((name_count - cache_size) > 1000)) { rehash(name_count); cache_size = name_count; - } - + } + while (atnl == 0) { struct crec *cache; int fqdn, nomem; char *canon; - - if ((atnl = gettok(f, token)) == EOF) + + if ((atnl = gettok(f, token)) == -1) break; fqdn = !!strchr(token, '.'); @@ -1033,7 +1132,7 @@ int read_hostsfile(char *filename, unsigned int index, int cache_size, struct cr if ((canon = canonicalise(token, &nomem))) { /* If set, add a version of the name with a default domain appended */ - if (option_bool(OPT_EXPAND) && domain_suffix && !fqdn && + if (option_bool(OPT_EXPAND) && domain_suffix && !fqdn && (cache = whine_malloc(SIZEOF_BARE_CREC + strlen(canon) + 2 + strlen(domain_suffix)))) { strcpy(cache->name.sname, canon); @@ -1053,23 +1152,25 @@ int read_hostsfile(char *filename, unsigned int index, int cache_size, struct cr name_count++; } free(canon); - + } else if (!nomem) - my_syslog(LOG_ERR, _("bad name at %s line %d"), filename, lineno); + my_syslog(LOG_ERR, _("bad name at %s line %d"), filename, lineno); } - } - fclose(f); + lineno += atnl; + } + fclose(f); + if (rhash) - rehash(name_count); - + rehash(name_count); + my_syslog(LOG_INFO, _("read %s - %d addresses"), filename, addr_count); - + return name_count; } - + void cache_reload(void) { struct crec *cache, **up, *tmp; @@ -1078,20 +1179,24 @@ void cache_reload(void) struct host_record *hr; struct name_list *nl; struct cname *a; + struct crec lrec; + struct mx_srv_record *mx; + struct txt_record *txt; struct interface_name *intr; + struct ptr_record *ptr; + struct naptr *naptr; #ifdef HAVE_DNSSEC struct ds_config *ds; #endif daemon->metrics[METRIC_DNS_CACHE_INSERTED] = 0; daemon->metrics[METRIC_DNS_CACHE_LIVE_FREED] = 0; - + for (i=0; ihash_next; if (cache->flags & (F_HOSTS | F_CONFIG)) { @@ -1111,25 +1216,22 @@ void cache_reload(void) else up = &cache->hash_next; } - - /* Add CNAMEs to interface_names to the cache */ + + /* Add locally-configured CNAMEs to the cache */ for (a = daemon->cnames; a; a = a->next) - for (intr = daemon->int_names; intr; intr = intr->next) - if (a->alias[1] != '*' && - hostname_isequal(a->target, intr->name) && - ((cache = whine_malloc(SIZEOF_POINTER_CREC)))) - { - cache->flags = F_FORWARD | F_NAMEP | F_CNAME | F_IMMORTAL | F_CONFIG; - cache->ttd = a->ttl; - cache->name.namep = a->alias; - cache->addr.cname.target.int_name = intr; - cache->addr.cname.uid = SRC_INTERFACE; - cache->uid = UID_NONE; - cache_hash(cache); - make_non_terminals(cache); - add_hosts_cname(cache); /* handle chains */ - } - + if (a->alias[1] != '*' && + ((cache = whine_malloc(SIZEOF_POINTER_CREC)))) + { + cache->flags = F_FORWARD | F_NAMEP | F_CNAME | F_IMMORTAL | F_CONFIG; + cache->ttd = a->ttl; + cache->name.namep = a->alias; + cache->addr.cname.target.name = a->target; + cache->addr.cname.is_name_ptr = 1; + cache->uid = UID_NONE; + cache_hash(cache); + make_non_terminals(cache); + } + #ifdef HAVE_DNSSEC for (ds = daemon->ds; ds; ds = ds->next) if ((cache = whine_malloc(SIZEOF_POINTER_CREC)) && @@ -1147,7 +1249,7 @@ void cache_reload(void) make_non_terminals(cache); } #endif - + /* borrow the packet buffer for a temporary by-address hash */ memset(daemon->packet, 0, daemon->packet_buff_sz); revhashsz = daemon->packet_buff_sz / sizeof(struct crec *); @@ -1158,26 +1260,25 @@ void cache_reload(void) for (hr = daemon->host_records; hr; hr = hr->next) for (nl = hr->names; nl; nl = nl->next) { - if (hr->addr.s_addr != 0 && + if ((hr->flags & HR_4) && (cache = whine_malloc(SIZEOF_POINTER_CREC))) { cache->name.namep = nl->name; cache->ttd = hr->ttl; cache->flags = F_HOSTS | F_IMMORTAL | F_FORWARD | F_REVERSE | F_IPV4 | F_NAMEP | F_CONFIG; - add_hosts_entry(cache, (struct all_addr *)&hr->addr, INADDRSZ, SRC_CONFIG, (struct crec **)daemon->packet, revhashsz); + add_hosts_entry(cache, (union all_addr *)&hr->addr, INADDRSZ, SRC_CONFIG, (struct crec **)daemon->packet, revhashsz); } -#ifdef HAVE_IPV6 - if (!IN6_IS_ADDR_UNSPECIFIED(&hr->addr6) && + + if ((hr->flags & HR_6) && (cache = whine_malloc(SIZEOF_POINTER_CREC))) { cache->name.namep = nl->name; cache->ttd = hr->ttl; cache->flags = F_HOSTS | F_IMMORTAL | F_FORWARD | F_REVERSE | F_IPV6 | F_NAMEP | F_CONFIG; - add_hosts_entry(cache, (struct all_addr *)&hr->addr6, IN6ADDRSZ, SRC_CONFIG, (struct crec **)daemon->packet, revhashsz); + add_hosts_entry(cache, (union all_addr *)&hr->addr6, IN6ADDRSZ, SRC_CONFIG, (struct crec **)daemon->packet, revhashsz); } -#endif } - + if (option_bool(OPT_NO_HOSTS) && !daemon->addn_hosts) { if (daemon->cachesize > 0) @@ -1187,31 +1288,64 @@ void cache_reload(void) { if (!option_bool(OPT_NO_HOSTS)) total_size = read_hostsfile(HOSTSFILE, SRC_HOSTS, total_size, (struct crec **)daemon->packet, revhashsz); - + daemon->addn_hosts = expand_filelist(daemon->addn_hosts); for (ah = daemon->addn_hosts; ah; ah = ah->next) if (!(ah->flags & AH_INACTIVE)) total_size = read_hostsfile(ah->fname, ah->index, total_size, (struct crec **)daemon->packet, revhashsz); } + + /* Make non-terminal records for all locally-define RRs */ + lrec.flags = F_FORWARD | F_CONFIG | F_NAMEP | F_IMMORTAL; + + for (txt = daemon->txt; txt; txt = txt->next) + { + lrec.name.namep = txt->name; + make_non_terminals(&lrec); + } + for (naptr = daemon->naptr; naptr; naptr = naptr->next) + { + lrec.name.namep = naptr->name; + make_non_terminals(&lrec); + } + + for (mx = daemon->mxnames; mx; mx = mx->next) + { + lrec.name.namep = mx->name; + make_non_terminals(&lrec); + } + + for (intr = daemon->int_names; intr; intr = intr->next) + { + lrec.name.namep = intr->name; + make_non_terminals(&lrec); + } + + for (ptr = daemon->ptr; ptr; ptr = ptr->next) + { + lrec.name.namep = ptr->name; + make_non_terminals(&lrec); + } + #ifdef HAVE_INOTIFY set_dynamic_inotify(AH_HOSTS, total_size, (struct crec **)daemon->packet, revhashsz); #endif - -} + +} #ifdef HAVE_DHCP struct in_addr a_record_from_hosts(char *name, time_t now) { struct crec *crecp = NULL; struct in_addr ret; - + while ((crecp = cache_find_by_name(crecp, name, now, F_IPV4))) if (crecp->flags & F_HOSTS) - return *(struct in_addr *)&crecp->addr; + return crecp->addr.addr4; my_syslog(MS_DHCP | LOG_WARNING, _("No IPv4 address found for %s"), name); - + ret.s_addr = 0; return ret; } @@ -1233,111 +1367,76 @@ void cache_unhash_dhcp(void) up = &cache->hash_next; } -static void add_dhcp_cname(struct crec *target, time_t ttd) -{ - struct crec *aliasc; - struct cname *a; - - for (a = daemon->cnames; a; a = a->next) - if (a->alias[1] != '*' && - hostname_isequal(cache_get_name(target), a->target)) - { - if ((aliasc = dhcp_spare)) - dhcp_spare = dhcp_spare->next; - else /* need new one */ - aliasc = whine_malloc(SIZEOF_POINTER_CREC); - - if (aliasc) - { - aliasc->flags = F_FORWARD | F_NAMEP | F_DHCP | F_CNAME | F_CONFIG; - if (ttd == 0) - aliasc->flags |= F_IMMORTAL; - else - aliasc->ttd = ttd; - aliasc->name.namep = a->alias; - aliasc->addr.cname.target.cache = target; - next_uid(target); - aliasc->addr.cname.uid = target->uid; - aliasc->uid = UID_NONE; - cache_hash(aliasc); - make_non_terminals(aliasc); - add_dhcp_cname(aliasc, ttd); - } - } -} - void cache_add_dhcp_entry(char *host_name, int prot, - struct all_addr *host_address, time_t ttd) + union all_addr *host_address, time_t ttd) { struct crec *crec = NULL, *fail_crec = NULL; - unsigned short flags = F_IPV4; + unsigned int flags = F_IPV4; int in_hosts = 0; size_t addrlen = sizeof(struct in_addr); -#ifdef HAVE_IPV6 if (prot == AF_INET6) { flags = F_IPV6; addrlen = sizeof(struct in6_addr); } -#endif - + inet_ntop(prot, host_address, daemon->addrbuff, ADDRSTRLEN); - + while ((crec = cache_find_by_name(crec, host_name, 0, flags | F_CNAME))) { /* check all addresses associated with name */ if (crec->flags & (F_HOSTS | F_CONFIG)) { if (crec->flags & F_CNAME) - my_syslog(MS_DHCP | LOG_WARNING, + my_syslog(MS_DHCP | LOG_WARNING, _("%s is a CNAME, not giving it to the DHCP lease of %s"), host_name, daemon->addrbuff); - else if (memcmp(&crec->addr.addr, host_address, addrlen) == 0) + else if (memcmp(&crec->addr, host_address, addrlen) == 0) in_hosts = 1; else fail_crec = crec; } else if (!(crec->flags & F_DHCP)) { - cache_scan_free(host_name, NULL, 0, crec->flags & (flags | F_CNAME | F_FORWARD), NULL, NULL); + cache_scan_free(host_name, NULL, C_IN, 0, crec->flags & (flags | F_CNAME | F_FORWARD), NULL, NULL); /* scan_free deletes all addresses associated with name */ break; } } - + /* if in hosts, don't need DHCP record */ if (in_hosts) return; - + /* Name in hosts, address doesn't match */ if (fail_crec) { - inet_ntop(prot, &fail_crec->addr.addr, daemon->namebuff, MAXDNAME); - my_syslog(MS_DHCP | LOG_WARNING, + inet_ntop(prot, &fail_crec->addr, daemon->namebuff, MAXDNAME); + my_syslog(MS_DHCP | LOG_WARNING, _("not giving name %s to the DHCP lease of %s because " - "the name exists in %s with address %s"), + "the name exists in %s with address %s"), host_name, daemon->addrbuff, record_source(fail_crec->uid), daemon->namebuff); return; - } - - if ((crec = cache_find_by_addr(NULL, (struct all_addr *)host_address, 0, flags))) + } + + if ((crec = cache_find_by_addr(NULL, (union all_addr *)host_address, 0, flags))) { if (crec->flags & F_NEG) { flags |= F_REVERSE; - cache_scan_free(NULL, (struct all_addr *)host_address, 0, flags, NULL, NULL); + cache_scan_free(NULL, (union all_addr *)host_address, C_IN, 0, flags, NULL, NULL); } } else flags |= F_REVERSE; - + if ((crec = dhcp_spare)) dhcp_spare = dhcp_spare->next; else /* need new one */ crec = whine_malloc(SIZEOF_POINTER_CREC); - + if (crec) /* malloc may fail */ { crec->flags = flags | F_NAMEP | F_DHCP | F_FORWARD; @@ -1345,13 +1444,11 @@ void cache_add_dhcp_entry(char *host_name, int prot, crec->flags |= F_IMMORTAL; else crec->ttd = ttd; - crec->addr.addr = *host_address; + crec->addr = *host_address; crec->name.namep = host_name; crec->uid = UID_NONE; cache_hash(crec); make_non_terminals(crec); - - add_dhcp_cname(crec, ttd); } } #endif @@ -1370,11 +1467,11 @@ static void make_non_terminals(struct crec *source) if (source->flags & F_DHCP) type = F_DHCP; #endif - + /* First delete any empty entries for our new real name. Note that we only delete empty entries deriving from DHCP for a new DHCP-derived - entry and vice-versa for HOSTS and CONFIG. This ensures that - non-terminals from DHCP go when we reload DHCP and + entry and vice-versa for HOSTS and CONFIG. This ensures that + non-terminals from DHCP go when we reload DHCP and for HOSTS/CONFIG when we re-read. */ for (up = hash_bucket(name), crecp = *up; crecp; crecp = tmp) { @@ -1383,7 +1480,7 @@ static void make_non_terminals(struct crec *source) if (!is_outdated_cname_pointer(crecp) && (crecp->flags & F_FORWARD) && (crecp->flags & type) && - !(crecp->flags & (F_IPV4 | F_IPV6 | F_CNAME | F_DNSKEY | F_DS)) && + !(crecp->flags & (F_IPV4 | F_IPV6 | F_CNAME | F_SRV | F_DNSKEY | F_DS)) && hostname_isequal(name, cache_get_name(crecp))) { *up = crecp->hash_next; @@ -1401,7 +1498,7 @@ static void make_non_terminals(struct crec *source) else up = &crecp->hash_next; } - + while ((name = strchr(name, '.'))) { name++; @@ -1413,7 +1510,7 @@ static void make_non_terminals(struct crec *source) (crecp->flags & type) && hostname_isequal(name, cache_get_name(crecp))) break; - + if (crecp) { /* If the new name expires later, transfer that time to @@ -1427,7 +1524,7 @@ static void make_non_terminals(struct crec *source) } continue; } - + #ifdef HAVE_DHCP if ((source->flags & F_DHCP) && dhcp_spare) { @@ -1440,10 +1537,10 @@ static void make_non_terminals(struct crec *source) if (crecp) { - crecp->flags = (source->flags | F_NAMEP) & ~(F_IPV4 | F_IPV6 | F_CNAME | F_DNSKEY | F_DS | F_REVERSE); + crecp->flags = (source->flags | F_NAMEP) & ~(F_IPV4 | F_IPV6 | F_CNAME | F_SRV | F_DNSKEY | F_DS | F_REVERSE); crecp->ttd = source->ttd; crecp->name.namep = name; - + cache_hash(crecp); } } @@ -1451,7 +1548,7 @@ static void make_non_terminals(struct crec *source) #ifndef NO_ID int cache_make_stat(struct txt_record *t) -{ +{ static char *buff = NULL; static int bufflen = 60; int len; @@ -1462,7 +1559,7 @@ int cache_make_stat(struct txt_record *t) return 0; p = buff; - + switch (t->stat) { case TXT_STAT_CACHESIZE: @@ -1501,17 +1598,17 @@ int cache_make_stat(struct txt_record *t) /* sum counts from different records for same server */ for (serv = daemon->servers; serv; serv = serv->next) serv->flags &= ~SERV_COUNTED; - + for (serv = daemon->servers; serv; serv = serv->next) - if (!(serv->flags & + if (!(serv->flags & (SERV_NO_ADDR | SERV_LITERAL_ADDRESS | SERV_COUNTED | SERV_USE_RESOLV | SERV_NO_REBIND))) { char *new, *lenp; int port, newlen, bytes_avail, bytes_needed; unsigned int queries = 0, failed_queries = 0; for (serv1 = serv; serv1; serv1 = serv1->next) - if (!(serv1->flags & - (SERV_NO_ADDR | SERV_LITERAL_ADDRESS | SERV_COUNTED | SERV_USE_RESOLV | SERV_NO_REBIND)) && + if (!(serv1->flags & + (SERV_NO_ADDR | SERV_LITERAL_ADDRESS | SERV_COUNTED | SERV_USE_RESOLV | SERV_NO_REBIND)) && sockaddr_isequal(&serv->addr, &serv1->addr)) { serv1->flags |= SERV_COUNTED; @@ -1544,7 +1641,7 @@ int cache_make_stat(struct txt_record *t) t->len = p - buff; return 1; } - + len = strlen(buff+1); t->txt = (unsigned char *)buff; t->len = len + 1; @@ -1553,7 +1650,7 @@ int cache_make_stat(struct txt_record *t) } #endif -/* There can be names in the cache containing control chars, don't +/* There can be names in the cache containing control chars, don't mess up logging or open security holes. */ static char *sanitise(char *name) { @@ -1572,30 +1669,29 @@ void dump_cache(time_t now) struct server *serv, *serv1; my_syslog(LOG_INFO, _("time %lu"), (unsigned long)now); - my_syslog(LOG_INFO, _("cache size %d, %d/%d cache insertions re-used unexpired cache entries."), + my_syslog(LOG_INFO, _("cache size %d, %d/%d cache insertions re-used unexpired cache entries."), daemon->cachesize, daemon->metrics[METRIC_DNS_CACHE_LIVE_FREED], daemon->metrics[METRIC_DNS_CACHE_INSERTED]); - my_syslog(LOG_INFO, _("queries forwarded %u, queries answered locally %u"), + my_syslog(LOG_INFO, _("queries forwarded %u, queries answered locally %u"), daemon->metrics[METRIC_DNS_QUERIES_FORWARDED], daemon->metrics[METRIC_DNS_LOCAL_ANSWERED]); #ifdef HAVE_AUTH my_syslog(LOG_INFO, _("queries for authoritative zones %u"), daemon->metrics[METRIC_DNS_AUTH_ANSWERED]); #endif -#ifdef HAVE_DNSSEC + blockdata_report(); -#endif /* sum counts from different records for same server */ for (serv = daemon->servers; serv; serv = serv->next) serv->flags &= ~SERV_COUNTED; - + for (serv = daemon->servers; serv; serv = serv->next) - if (!(serv->flags & + if (!(serv->flags & (SERV_NO_ADDR | SERV_LITERAL_ADDRESS | SERV_COUNTED | SERV_USE_RESOLV | SERV_NO_REBIND))) { int port; unsigned int queries = 0, failed_queries = 0; for (serv1 = serv; serv1; serv1 = serv1->next) - if (!(serv1->flags & - (SERV_NO_ADDR | SERV_LITERAL_ADDRESS | SERV_COUNTED | SERV_USE_RESOLV | SERV_NO_REBIND)) && + if (!(serv1->flags & + (SERV_NO_ADDR | SERV_LITERAL_ADDRESS | SERV_COUNTED | SERV_USE_RESOLV | SERV_NO_REBIND)) && sockaddr_isequal(&serv->addr, &serv1->addr)) { serv1->flags |= SERV_COUNTED; @@ -1605,13 +1701,13 @@ void dump_cache(time_t now) port = prettyprint_addr(&serv->addr, daemon->addrbuff); my_syslog(LOG_INFO, _("server %s#%d: queries sent %u, retried or failed %u"), daemon->addrbuff, port, queries, failed_queries); } - + if (option_bool(OPT_DEBUG) || option_bool(OPT_LOG)) { struct crec *cache ; int i; my_syslog(LOG_INFO, "Host Address Flags Expires"); - + for (i=0; ihash_next) { @@ -1623,6 +1719,17 @@ void dump_cache(time_t now) p += sprintf(p, "%-30.30s ", sanitise(n)); if ((cache->flags & F_CNAME) && !is_outdated_cname_pointer(cache)) a = sanitise(cache_get_cname_target(cache)); + else if ((cache->flags & F_SRV) && !(cache->flags & F_NEG)) + { + int targetlen = cache->addr.srv.targetlen; + ssize_t len = sprintf(a, "%u %u %u ", cache->addr.srv.priority, + cache->addr.srv.weight, cache->addr.srv.srvport); + + if (targetlen > (40 - len)) + targetlen = 40 - len; + blockdata_retrieve(cache->addr.srv.target, targetlen, a + len); + a[len + targetlen] = 0; + } #ifdef HAVE_DNSSEC else if (cache->flags & F_DS) { @@ -1635,14 +1742,12 @@ void dump_cache(time_t now) cache->addr.key.algo, cache->addr.key.flags); #endif else if (!(cache->flags & F_NEG) || !(cache->flags & F_FORWARD)) - { + { a = daemon->addrbuff; if (cache->flags & F_IPV4) - inet_ntop(AF_INET, &cache->addr.addr, a, ADDRSTRLEN); -#ifdef HAVE_IPV6 + inet_ntop(AF_INET, &cache->addr, a, ADDRSTRLEN); else if (cache->flags & F_IPV6) - inet_ntop(AF_INET6, &cache->addr.addr, a, ADDRSTRLEN); -#endif + inet_ntop(AF_INET6, &cache->addr, a, ADDRSTRLEN); } if (cache->flags & F_IPV4) @@ -1651,6 +1756,8 @@ void dump_cache(time_t now) t = "6"; else if (cache->flags & F_CNAME) t = "C"; + else if (cache->flags & F_SRV) + t = "V"; #ifdef HAVE_DNSSEC else if (cache->flags & F_DS) t = "S"; @@ -1686,10 +1793,6 @@ char *record_source(unsigned int index) return "config"; else if (index == SRC_HOSTS) return HOSTSFILE; - /*----- Pi-hole modification -----*/ - else if (index == SRC_REGEX) - return (char*)regexlistname; - /*--------------------------------*/ for (ah = daemon->addn_hosts; ah; ah = ah->next) if (ah->index == index) @@ -1726,14 +1829,14 @@ char *querystr(char *desc, unsigned short type) len += strlen(desc); } len++; /* terminator */ - + if (!buff || bufflen < len) { if (buff) free(buff); else if (len < 20) len = 20; - + buff = whine_malloc(len); bufflen = len; } @@ -1755,15 +1858,16 @@ char *querystr(char *desc, unsigned short type) sprintf(buff, "type=%d", type); } } - + return buff ? buff : ""; } -void log_query(unsigned int flags, char *name, struct all_addr *addr, char *arg) +// Modified by Pi-hole +void _log_query(unsigned int flags, char *name, union all_addr *addr, char *arg, const char* file, const int line) { char *source, *dest = daemon->addrbuff; char *verb = "is"; - + if (!option_bool(OPT_LOG)) return; @@ -1772,10 +1876,10 @@ void log_query(unsigned int flags, char *name, struct all_addr *addr, char *arg) if (addr) { if (flags & F_KEYTAG) - sprintf(daemon->addrbuff, arg, addr->addr.log.keytag, addr->addr.log.algo, addr->addr.log.digest); + sprintf(daemon->addrbuff, arg, addr->log.keytag, addr->log.algo, addr->log.digest); else if (flags & F_RCODE) { - unsigned int rcode = addr->addr.rcode.rcode; + unsigned int rcode = addr->log.rcode; if (rcode == SERVFAIL) dest = "SERVFAIL"; @@ -1787,14 +1891,9 @@ void log_query(unsigned int flags, char *name, struct all_addr *addr, char *arg) sprintf(daemon->addrbuff, "%u", rcode); } else - { -#ifdef HAVE_IPV6 - inet_ntop(flags & F_IPV4 ? AF_INET : AF_INET6, - addr, daemon->addrbuff, ADDRSTRLEN); -#else - strncpy(daemon->addrbuff, inet_ntoa(addr->addr.addr4), ADDRSTRLEN); -#endif - } + inet_ntop(flags & F_IPV4 ? AF_INET : AF_INET6, + addr, daemon->addrbuff, ADDRSTRLEN); + } else dest = arg; @@ -1804,13 +1903,13 @@ void log_query(unsigned int flags, char *name, struct all_addr *addr, char *arg) dest = name; name = daemon->addrbuff; } - + if (flags & F_NEG) { if (flags & F_NXDOMAIN) dest = "NXDOMAIN"; else - { + { if (flags & F_IPV4) dest = "NODATA-IPv4"; else if (flags & F_IPV6) @@ -1821,9 +1920,11 @@ void log_query(unsigned int flags, char *name, struct all_addr *addr, char *arg) } else if (flags & F_CNAME) dest = ""; + else if (flags & F_SRV) + dest = ""; else if (flags & F_RRNAME) dest = arg; - + if (flags & F_CONFIG) source = "config"; else if (flags & F_DHCP) @@ -1860,10 +1961,13 @@ void log_query(unsigned int flags, char *name, struct all_addr *addr, char *arg) } else source = "cached"; - + if (strlen(name) == 0) name = "."; - +/************************************************************** Pi-hole modification **************************************************************/ +if(debug_dnsmasq_lines == 0) +{ +/***************************************************************************************************************************************************/ if (option_bool(OPT_EXTRALOG)) { int port = prettyprint_addr(daemon->log_source_addr, daemon->addrbuff2); @@ -1874,6 +1978,22 @@ void log_query(unsigned int flags, char *name, struct all_addr *addr, char *arg) } else my_syslog(LOG_INFO, "%s %s %s %s", source, name, verb, dest); +/************************************************************** Pi-hole modification **************************************************************/ +} +else +{ + if (option_bool(OPT_EXTRALOG)) + { + int port = prettyprint_addr(daemon->log_source_addr, daemon->addrbuff2); + if (flags & F_NOEXTRA) + my_syslog(LOG_INFO, "* %s/%u %s %s %s %s (%s:%d)", daemon->addrbuff2, port, source, name, verb, dest, file, line); + else + my_syslog(LOG_INFO, "%u %s/%u %s %s %s %s (%s:%d)", daemon->log_display_id, daemon->addrbuff2, port, source, name, verb, dest, file, line); + } + else + my_syslog(LOG_INFO, "%s %s %s %s (%s:%d)", source, name, verb, dest, file, line); +} +/***************************************************************************************************************************************************/ } - + diff --git a/dnsmasq/config.h b/src/dnsmasq/config.h similarity index 85% rename from dnsmasq/config.h rename to src/dnsmasq/config.h index 2192a11bb..e5367bc78 100644 --- a/dnsmasq/config.h +++ b/src/dnsmasq/config.h @@ -1,15 +1,15 @@ -/* dnsmasq is Copyright (c) 2000-2018 Simon Kelley +/* dnsmasq is Copyright (c) 2000-2020 Simon Kelley This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; version 2 dated June, 1991, or (at your option) version 3 dated 29 June, 2007. - + This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. - + You should have received a copy of the GNU General Public License along with this program. If not, see . */ @@ -52,13 +52,14 @@ #define RANDFILE "/dev/urandom" #define DNSMASQ_SERVICE "uk.org.thekelleys.dnsmasq" /* Default - may be overridden by config */ #define DNSMASQ_PATH "/uk/org/thekelleys/dnsmasq" +#define DNSMASQ_UBUS_NAME "dnsmasq" /* Default - may be overridden by config */ #define AUTH_TTL 600 /* default TTL for auth DNS */ #define SOA_REFRESH 1200 /* SOA refresh default */ #define SOA_RETRY 180 /* SOA retry default */ #define SOA_EXPIRY 1209600 /* SOA expiry default */ #define LOOP_TEST_DOMAIN "test" /* domain for loop testing, "test" is reserved by RFC 2606 and won't therefore clash */ #define LOOP_TEST_TYPE T_TXT - + /* compile-time options: uncomment below to enable or do eg. make COPTS=-DHAVE_BROKEN_RTC @@ -68,9 +69,9 @@ HAVE_BROKEN_RTC for timing, and keep lease lengths rather than expiry times in its leases file. This also make dnsmasq "flash disk friendly". Normally, dnsmasq tries very hard to keep the on-disk leases file - up-to-date: rewriting it after every renewal. When HAVE_BROKEN_RTC - is in effect, the lease file is only written when a new lease is - created, or an old one destroyed. (Because those are the only times + up-to-date: rewriting it after every renewal. When HAVE_BROKEN_RTC + is in effect, the lease file is only written when a new lease is + created, or an old one destroyed. (Because those are the only times it changes.) This vastly reduces the number of file writes, and makes it viable to keep the lease file on a flash filesystem. NOTE: when enabling or disabling this, be sure to delete any old @@ -89,11 +90,11 @@ HAVE_SCRIPT define this to get the ability to call scripts on lease-change. HAVE_LUASCRIPT - define this to get the ability to call Lua script on lease-change. (implies HAVE_SCRIPT) + define this to get the ability to call Lua script on lease-change. (implies HAVE_SCRIPT) HAVE_DBUS define this if you want to link against libdbus, and have dnsmasq - support some methods to allow (re)configuration of the upstream DNS + support some methods to allow (re)configuration of the upstream DNS servers via DBus. HAVE_UBUS @@ -101,7 +102,7 @@ HAVE_UBUS HAVE_IDN define this if you want international domain name 2003 support. - + HAVE_LIBIDN2 define this if you want international domain name 2008 support. @@ -133,7 +134,6 @@ HAVE_INOTIFY NO_ID Don't report *.bind CHAOS info to clients, forward such requests upstream instead. -NO_IPV6 NO_TFTP NO_DHCP NO_DHCP6 @@ -142,9 +142,9 @@ NO_LARGEFILE NO_AUTH NO_DUMPFILE NO_INOTIFY - these are available to explicitly disable compile time options which would - otherwise be enabled automatically (HAVE_IPV6, >2Gb file sizes) or - which are enabled by default in the distributed source tree. Building dnsmasq + these are available to explicitly disable compile time options which would + otherwise be enabled automatically or which are enabled by default + in the distributed source tree. Building dnsmasq with something like "make COPTS=-DNO_SCRIPT" will do the trick. NO_GMP Don't use and link against libgmp, Useful if nettle is built with --enable-mini-gmp. @@ -152,13 +152,13 @@ NO_GMP LEASEFILE CONFFILE RESOLVFILE - the default locations of these files are determined below, but may be overridden + the default locations of these files are determined below, but may be overridden in a build command line using COPTS. */ -/* Defining this builds a binary which handles time differently and works better on a system without a - stable RTC (it uses uptime, not epoch time) and writes the DHCP leases file less often to avoid flash wear. +/* Defining this builds a binary which handles time differently and works better on a system without a + stable RTC (it uses uptime, not epoch time) and writes the DHCP leases file less often to avoid flash wear. */ /* #define HAVE_BROKEN_RTC */ @@ -167,16 +167,16 @@ RESOLVFILE has no library dependencies other than libc */ #define HAVE_DHCP -#define HAVE_DHCP6 +#define HAVE_DHCP6 #define HAVE_TFTP #define HAVE_SCRIPT #define HAVE_AUTH -#define HAVE_IPSET +#define HAVE_IPSET #define HAVE_LOOP #define HAVE_DUMPFILE /* Build options which require external libraries. - + Defining HAVE__STATIC as _well_ as HAVE_ will link the library statically. You can use "make COPTS=-DHAVE_" instead of editing these. @@ -236,33 +236,19 @@ HAVE_SOLARIS_NETWORK define exactly one of these to alter interaction with kernel networking. HAVE_GETOPT_LONG - defined when GNU-style getopt_long available. + defined when GNU-style getopt_long available. HAVE_SOCKADDR_SA_LEN - defined if struct sockaddr has sa_len field (*BSD) + defined if struct sockaddr has sa_len field (*BSD) */ -/* Must precede __linux__ since uClinux defines __linux__ too. */ -#if defined(__uClinux__) -#define HAVE_LINUX_NETWORK -#define HAVE_GETOPT_LONG -#undef HAVE_SOCKADDR_SA_LEN -/* Never use fork() on uClinux. Note that this is subtly different from the - --keep-in-foreground option, since it also suppresses forking new - processes for TCP connections and disables the call-a-script on leasechange - system. It's intended for use on MMU-less kernels. */ -#define NO_FORK - -#elif defined(__UCLIBC__) +#if defined(__UCLIBC__) #define HAVE_LINUX_NETWORK #if defined(__UCLIBC_HAS_GNU_GETOPT__) || \ ((__UCLIBC_MAJOR__==0) && (__UCLIBC_MINOR__==9) && (__UCLIBC_SUBLEVEL__<21)) # define HAVE_GETOPT_LONG #endif #undef HAVE_SOCKADDR_SA_LEN -#if !defined(__ARCH_HAS_MMU__) && !defined(__UCLIBC_HAS_MMU__) -# define NO_FORK -#endif #if defined(__UCLIBC_HAS_IPV6__) # ifndef IPV6_V6ONLY # define IPV6_V6ONLY 26 @@ -290,11 +276,16 @@ HAVE_SOCKADDR_SA_LEN #define HAVE_BSD_NETWORK #define HAVE_GETOPT_LONG #define HAVE_SOCKADDR_SA_LEN +#define NO_IPSET /* Define before sys/socket.h is included so we get socklen_t */ #define _BSD_SOCKLEN_T_ -/* Select the RFC_3542 version of the IPv6 socket API. +/* Select the RFC_3542 version of the IPv6 socket API. Define before netinet6/in6.h is included. */ #define __APPLE_USE_RFC_3542 +/* Required for Mojave. */ +#ifndef SOL_TCP +# define SOL_TCP IPPROTO_TCP +#endif #define NO_IPSET #elif defined(__NetBSD__) @@ -306,33 +297,13 @@ HAVE_SOCKADDR_SA_LEN #define HAVE_SOLARIS_NETWORK #define HAVE_GETOPT_LONG #undef HAVE_SOCKADDR_SA_LEN -#define ETHER_ADDR_LEN 6 - +#define ETHER_ADDR_LEN 6 + #endif -/* Decide if we're going to support IPv6 */ -/* We assume that systems which don't have IPv6 - headers don't have ntop and pton either */ - -#if defined(INET6_ADDRSTRLEN) && defined(IPV6_V6ONLY) -# define HAVE_IPV6 -# define ADDRSTRLEN INET6_ADDRSTRLEN -#else -# if !defined(INET_ADDRSTRLEN) -# define INET_ADDRSTRLEN 16 /* 4*3 + 3 dots + NULL */ -# endif -# undef HAVE_IPV6 -# define ADDRSTRLEN INET_ADDRSTRLEN -#endif - - -/* rules to implement compile-time option dependencies and +/* rules to implement compile-time option dependencies and the NO_XXX flags */ -#ifdef NO_IPV6 -#undef HAVE_IPV6 -#endif - #ifdef NO_TFTP #undef HAVE_TFTP #endif @@ -342,7 +313,7 @@ HAVE_SOCKADDR_SA_LEN #undef HAVE_DHCP6 #endif -#if defined(NO_DHCP6) || !defined(HAVE_IPV6) +#if defined(NO_DHCP6) #undef HAVE_DHCP6 #endif @@ -351,7 +322,7 @@ HAVE_SOCKADDR_SA_LEN #define HAVE_DHCP #endif -#if defined(NO_SCRIPT) || defined(NO_FORK) +#if defined(NO_SCRIPT) #undef HAVE_SCRIPT #undef HAVE_LUASCRIPT #endif @@ -386,10 +357,7 @@ HAVE_SOCKADDR_SA_LEN #ifdef DNSMASQ_COMPILE_OPTS -static char *compile_opts = -#ifndef HAVE_IPV6 -"no-" -#endif +static char *compile_opts = "IPv6 " #ifndef HAVE_GETOPT_LONG "no-" @@ -398,13 +366,14 @@ static char *compile_opts = #ifdef HAVE_BROKEN_RTC "no-RTC " #endif -#ifdef NO_FORK -"no-MMU " -#endif #ifndef HAVE_DBUS "no-" #endif "DBus " +#ifndef HAVE_UBUS +"no-" +#endif +"UBus " #ifndef LOCALEDIR "no-" #endif @@ -414,8 +383,8 @@ static char *compile_opts = #else #if !defined(HAVE_IDN) "no-" - #endif -"IDN " + #endif +"IDN " #endif #ifndef HAVE_DHCP "no-" @@ -424,7 +393,7 @@ static char *compile_opts = #if defined(HAVE_DHCP) # if !defined (HAVE_DHCP6) "no-" -# endif +# endif "DHCPv6 " #endif #if !defined(HAVE_SCRIPT) diff --git a/dnsmasq/conntrack.c b/src/dnsmasq/conntrack.c similarity index 87% rename from dnsmasq/conntrack.c rename to src/dnsmasq/conntrack.c index 2929f8ca9..33f5ceb52 100644 --- a/dnsmasq/conntrack.c +++ b/src/dnsmasq/conntrack.c @@ -1,4 +1,4 @@ -/* dnsmasq is Copyright (c) 2000-2018 Simon Kelley +/* dnsmasq is Copyright (c) 2000-2020 Simon Kelley This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -24,7 +24,7 @@ static int gotit = 0; /* yuck */ static int callback(enum nf_conntrack_msg_type type, struct nf_conntrack *ct, void *data); -int get_incoming_mark(union mysockaddr *peer_addr, struct all_addr *local_addr, int istcp, unsigned int *markp) +int get_incoming_mark(union mysockaddr *peer_addr, union all_addr *local_addr, int istcp, unsigned int *markp) { struct nf_conntrack *ct; struct nfct_handle *h; @@ -36,21 +36,19 @@ int get_incoming_mark(union mysockaddr *peer_addr, struct all_addr *local_addr, nfct_set_attr_u8(ct, ATTR_L4PROTO, istcp ? IPPROTO_TCP : IPPROTO_UDP); nfct_set_attr_u16(ct, ATTR_PORT_DST, htons(daemon->port)); -#ifdef HAVE_IPV6 if (peer_addr->sa.sa_family == AF_INET6) { nfct_set_attr_u8(ct, ATTR_L3PROTO, AF_INET6); nfct_set_attr(ct, ATTR_IPV6_SRC, peer_addr->in6.sin6_addr.s6_addr); nfct_set_attr_u16(ct, ATTR_PORT_SRC, peer_addr->in6.sin6_port); - nfct_set_attr(ct, ATTR_IPV6_DST, local_addr->addr.addr6.s6_addr); + nfct_set_attr(ct, ATTR_IPV6_DST, local_addr->addr6.s6_addr); } else -#endif { nfct_set_attr_u8(ct, ATTR_L3PROTO, AF_INET); nfct_set_attr_u32(ct, ATTR_IPV4_SRC, peer_addr->in.sin_addr.s_addr); nfct_set_attr_u16(ct, ATTR_PORT_SRC, peer_addr->in.sin_port); - nfct_set_attr_u32(ct, ATTR_IPV4_DST, local_addr->addr.addr4.s_addr); + nfct_set_attr_u32(ct, ATTR_IPV4_DST, local_addr->addr4.s_addr); } diff --git a/dnsmasq/crypto.c b/src/dnsmasq/crypto.c similarity index 79% rename from dnsmasq/crypto.c rename to src/dnsmasq/crypto.c index 3b98974f4..ca6311122 100644 --- a/dnsmasq/crypto.c +++ b/src/dnsmasq/crypto.c @@ -1,15 +1,15 @@ -/* dnsmasq is Copyright (c) 2000-2018 Simon Kelley +/* dnsmasq is Copyright (c) 2000-2020 Simon Kelley This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; version 2 dated June, 1991, or (at your option) version 3 dated 29 June, 2007. - + This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. - + You should have received a copy of the GNU General Public License along with this program. If not, see . */ @@ -19,17 +19,19 @@ #ifdef HAVE_DNSSEC #include -#include #include #include #include +#if NETTLE_VERSION_MAJOR == 3 && NETTLE_VERSION_MINOR >= 6 +# include +#endif #include #include /* Implement a "hash-function" to the nettle API, which simply returns the input data, concatenated into a single, statically maintained, buffer. - Used for the EdDSA sigs, which operate on the whole message, rather + Used for the EdDSA sigs, which operate on the whole message, rather than a digest. */ struct null_hash_digest @@ -56,11 +58,11 @@ static void null_hash_update(void *ctxv, size_t length, const uint8_t *src) { struct null_hash_ctx *ctx = ctxv; size_t new_len = ctx->len + length; - + if (new_len > null_hash_buff_sz) { uint8_t *new; - + if (!(new = whine_malloc(new_len + BUFF_INCR))) return; @@ -70,7 +72,7 @@ static void null_hash_update(void *ctxv, size_t length, const uint8_t *src) memcpy(new, null_hash_buff, ctx->len); free(null_hash_buff); } - + null_hash_buff_sz = new_len + BUFF_INCR; null_hash_buff = new; } @@ -78,12 +80,12 @@ static void null_hash_update(void *ctxv, size_t length, const uint8_t *src) memcpy(null_hash_buff + ctx->len, src, length); ctx->len += length; } - + static void null_hash_digest(void *ctx, size_t length, uint8_t *dst) { (void)length; - + ((struct null_hash_digest *)dst)->buff = null_hash_buff; ((struct null_hash_digest *)dst)->len = ((struct null_hash_ctx *)ctx)->len; } @@ -103,7 +105,7 @@ const struct nettle_hash *hash_find(char *name) { if (!name) return NULL; - + /* We provide a "null" hash which returns the input data as digest. */ if (strcmp(null_hash.name, name) == 0) return &null_hash; @@ -112,7 +114,7 @@ const struct nettle_hash *hash_find(char *name) incompatibilities if sizeof(nettle_hashes) changes between library versions. It also #defines nettle_hashes, so use that to tell if we have the new facilities. */ - + #ifdef nettle_hashes return nettle_lookup_hash(name); #else @@ -123,7 +125,7 @@ const struct nettle_hash *hash_find(char *name) if (strcmp(nettle_hashes[i]->name, name) == 0) return nettle_hashes[i]; } - + return NULL; #endif } @@ -147,7 +149,7 @@ int hash_init(const struct nettle_hash *hash, void **ctxp, unsigned char **diges ctx = new; ctx_sz = hash->context_size; } - + if (digest_sz < hash->digest_size) { if (!(new = whine_malloc(hash->digest_size))) @@ -165,50 +167,48 @@ int hash_init(const struct nettle_hash *hash, void **ctxp, unsigned char **diges return 1; } - + static int dnsmasq_rsa_verify(struct blockdata *key_data, unsigned int key_len, unsigned char *sig, size_t sig_len, unsigned char *digest, size_t digest_len, int algo) { unsigned char *p; size_t exp_len; - + static struct rsa_public_key *key = NULL; static mpz_t sig_mpz; (void)digest_len; - + if (key == NULL) { if (!(key = whine_malloc(sizeof(struct rsa_public_key)))) return 0; - + nettle_rsa_public_key_init(key); mpz_init(sig_mpz); } - + if ((key_len < 3) || !(p = blockdata_retrieve(key_data, key_len, NULL))) return 0; - + key_len--; if ((exp_len = *p++) == 0) { GETSHORT(exp_len, p); key_len -= 2; } - + if (exp_len >= key_len) return 0; - + key->size = key_len - exp_len; mpz_import(key->e, exp_len, 1, 1, 0, 0, p); mpz_import(key->n, key->size, 1, 1, 0, 0, p + exp_len); mpz_import(sig_mpz, sig_len, 1, 1, 0, 0, sig); - + switch (algo) { - case 1: - return nettle_rsa_md5_verify_digest(key, digest, sig_mpz); case 5: case 7: return nettle_rsa_sha1_verify_digest(key, digest, sig_mpz); case 8: @@ -218,53 +218,9 @@ static int dnsmasq_rsa_verify(struct blockdata *key_data, unsigned int key_len, } return 0; -} - -static int dnsmasq_dsa_verify(struct blockdata *key_data, unsigned int key_len, unsigned char *sig, size_t sig_len, - unsigned char *digest, size_t digest_len, int algo) -{ - unsigned char *p; - unsigned int t; +} - static mpz_t y; - static struct dsa_params *params = NULL; - static struct dsa_signature *sig_struct; - - (void)digest_len; - - if (params == NULL) - { - if (!(sig_struct = whine_malloc(sizeof(struct dsa_signature))) || - !(params = whine_malloc(sizeof(struct dsa_params)))) - return 0; - - mpz_init(y); - nettle_dsa_params_init(params); - nettle_dsa_signature_init(sig_struct); - } - - if ((sig_len < 41) || !(p = blockdata_retrieve(key_data, key_len, NULL))) - return 0; - - t = *p++; - - if (key_len < (213 + (t * 24))) - return 0; - - mpz_import(params->q, 20, 1, 1, 0, 0, p); p += 20; - mpz_import(params->p, 64 + (t*8), 1, 1, 0, 0, p); p += 64 + (t*8); - mpz_import(params->g, 64 + (t*8), 1, 1, 0, 0, p); p += 64 + (t*8); - mpz_import(y, 64 + (t*8), 1, 1, 0, 0, p); p += 64 + (t*8); - - mpz_import(sig_struct->r, 20, 1, 1, 0, 0, sig+1); - mpz_import(sig_struct->s, 20, 1, 1, 0, 0, sig+21); - - (void)algo; - - return nettle_dsa_verify(params, y, digest_len, digest, sig_struct); -} - -static int dnsmasq_ecdsa_verify(struct blockdata *key_data, unsigned int key_len, +static int dnsmasq_ecdsa_verify(struct blockdata *key_data, unsigned int key_len, unsigned char *sig, size_t sig_len, unsigned char *digest, size_t digest_len, int algo) { @@ -275,17 +231,21 @@ static int dnsmasq_ecdsa_verify(struct blockdata *key_data, unsigned int key_len static struct ecc_point *key_256 = NULL, *key_384 = NULL; static mpz_t x, y; static struct dsa_signature *sig_struct; - +#if NETTLE_VERSION_MAJOR == 3 && NETTLE_VERSION_MINOR < 4 +#define nettle_get_secp_256r1() (&nettle_secp_256r1) +#define nettle_get_secp_384r1() (&nettle_secp_384r1) +#endif + if (!sig_struct) { if (!(sig_struct = whine_malloc(sizeof(struct dsa_signature)))) return 0; - + nettle_dsa_signature_init(sig_struct); mpz_init(x); mpz_init(y); } - + switch (algo) { case 13: @@ -293,73 +253,126 @@ static int dnsmasq_ecdsa_verify(struct blockdata *key_data, unsigned int key_len { if (!(key_256 = whine_malloc(sizeof(struct ecc_point)))) return 0; - - nettle_ecc_point_init(key_256, &nettle_secp_256r1); + + nettle_ecc_point_init(key_256, nettle_get_secp_256r1()); } - + key = key_256; t = 32; break; - + case 14: if (!key_384) { if (!(key_384 = whine_malloc(sizeof(struct ecc_point)))) return 0; - - nettle_ecc_point_init(key_384, &nettle_secp_384r1); + + nettle_ecc_point_init(key_384, nettle_get_secp_384r1()); } - + key = key_384; t = 48; break; - + default: return 0; } - + if (sig_len != 2*t || key_len != 2*t || !(p = blockdata_retrieve(key_data, key_len, NULL))) return 0; - + mpz_import(x, t , 1, 1, 0, 0, p); mpz_import(y, t , 1, 1, 0, 0, p + t); if (!ecc_point_set(key, x, y)) return 0; - + mpz_import(sig_struct->r, t, 1, 1, 0, 0, sig); mpz_import(sig_struct->s, t, 1, 1, 0, 0, sig + t); - + return nettle_ecdsa_verify(key, digest_len, digest, sig_struct); } -static int dnsmasq_eddsa_verify(struct blockdata *key_data, unsigned int key_len, - unsigned char *sig, size_t sig_len, - unsigned char *digest, size_t digest_len, int algo) +#if NETTLE_VERSION_MAJOR == 3 && NETTLE_VERSION_MINOR >= 6 +static int dnsmasq_gostdsa_verify(struct blockdata *key_data, unsigned int key_len, + unsigned char *sig, size_t sig_len, + unsigned char *digest, size_t digest_len, int algo) { unsigned char *p; + + static struct ecc_point *gost_key = NULL; + static mpz_t x, y; + static struct dsa_signature *sig_struct; - if (key_len != ED25519_KEY_SIZE || - sig_len != ED25519_SIGNATURE_SIZE || - digest_len != sizeof(struct null_hash_digest) || + if (algo != 12 || + sig_len != 64 || key_len != 64 || !(p = blockdata_retrieve(key_data, key_len, NULL))) return 0; + + if (!sig_struct) + { + if (!(sig_struct = whine_malloc(sizeof(struct dsa_signature))) || + !(gost_key = whine_malloc(sizeof(struct ecc_point)))) + return 0; + + nettle_dsa_signature_init(sig_struct); + nettle_ecc_point_init(gost_key, nettle_get_gost_gc256b()); + mpz_init(x); + mpz_init(y); + } + + mpz_import(x, 32 , 1, 1, 0, 0, p); + mpz_import(y, 32 , 1, 1, 0, 0, p + 32); + + if (!ecc_point_set(gost_key, x, y)) + return 0; + + mpz_import(sig_struct->r, 32, 1, 1, 0, 0, sig); + mpz_import(sig_struct->s, 32, 1, 1, 0, 0, sig + 32); + + return nettle_gostdsa_verify(gost_key, digest_len, digest, sig_struct); +} +#endif +static int dnsmasq_eddsa_verify(struct blockdata *key_data, unsigned int key_len, + unsigned char *sig, size_t sig_len, + unsigned char *digest, size_t digest_len, int algo) +{ + unsigned char *p; + + if (digest_len != sizeof(struct null_hash_digest) || + !(p = blockdata_retrieve(key_data, key_len, NULL))) + return 0; + /* The "digest" returned by the null_hash function is simply a struct null_hash_digest which has a pointer to the actual data and a length, because the buffer may need to be extended during "hashing". */ - + switch (algo) { case 15: + if (key_len != ED25519_KEY_SIZE || + sig_len != ED25519_SIGNATURE_SIZE) + return 0; + return ed25519_sha512_verify(p, ((struct null_hash_digest *)digest)->len, ((struct null_hash_digest *)digest)->buff, sig); + +#if NETTLE_VERSION_MAJOR == 3 && NETTLE_VERSION_MINOR >= 6 case 16: - /* Ed448 when available */ - return 0; + if (key_len != ED448_KEY_SIZE || + sig_len != ED448_SIGNATURE_SIZE) + return 0; + + return ed448_shake256_verify(p, + ((struct null_hash_digest *)digest)->len, + ((struct null_hash_digest *)digest)->buff, + sig); +#endif + } return 0; @@ -368,27 +381,29 @@ static int dnsmasq_eddsa_verify(struct blockdata *key_data, unsigned int key_len static int (*verify_func(int algo))(struct blockdata *key_data, unsigned int key_len, unsigned char *sig, size_t sig_len, unsigned char *digest, size_t digest_len, int algo) { - - /* Enure at runtime that we have support for this digest */ + + /* Ensure at runtime that we have support for this digest */ if (!hash_find(algo_digest_name(algo))) return NULL; - + /* This switch defines which sig algorithms we support, can't introspect Nettle for that. */ switch (algo) { - case 1: case 5: case 7: case 8: case 10: + case 5: case 7: case 8: case 10: return dnsmasq_rsa_verify; - case 3: case 6: - return dnsmasq_dsa_verify; - +#if NETTLE_VERSION_MAJOR == 3 && NETTLE_VERSION_MINOR >= 6 + case 12: + return dnsmasq_gostdsa_verify; +#endif + case 13: case 14: return dnsmasq_ecdsa_verify; case 15: case 16: return dnsmasq_eddsa_verify; } - + return NULL; } @@ -398,9 +413,9 @@ int verify(struct blockdata *key_data, unsigned int key_len, unsigned char *sig, int (*func)(struct blockdata *key_data, unsigned int key_len, unsigned char *sig, size_t sig_len, unsigned char *digest, size_t digest_len, int algo); - + func = verify_func(algo); - + if (!func) return 0; @@ -409,7 +424,7 @@ int verify(struct blockdata *key_data, unsigned int key_len, unsigned char *sig, /* Note the ds_digest_name(), algo_digest_name() and nsec3_digest_name() define which algo numbers we support. If algo_digest_name() returns - non-NULL for an algorithm number, we assume that algorithm is + non-NULL for an algorithm number, we assume that algorithm is supported by verify(). */ /* http://www.iana.org/assignments/ds-rr-types/ds-rr-types.xhtml */ @@ -424,7 +439,7 @@ char *ds_digest_name(int digest) default: return NULL; } } - + /* http://www.iana.org/assignments/dns-sec-alg-numbers/dns-sec-alg-numbers.xhtml */ char *algo_digest_name(int algo) { @@ -432,21 +447,21 @@ char *algo_digest_name(int algo) { case 1: return NULL; /* RSA/MD5 - Must Not Implement. RFC 6944 para 2.3. */ case 2: return NULL; /* Diffie-Hellman */ - case 3: return "sha1"; /* DSA/SHA1 */ + case 3: return NULL; ; /* DSA/SHA1 - Must Not Implement. RFC 8624 section 3.1 */ case 5: return "sha1"; /* RSA/SHA1 */ - case 6: return "sha1"; /* DSA-NSEC3-SHA1 */ + case 6: return NULL; /* DSA-NSEC3-SHA1 - Must Not Implement. RFC 8624 section 3.1 */ case 7: return "sha1"; /* RSASHA1-NSEC3-SHA1 */ case 8: return "sha256"; /* RSA/SHA-256 */ case 10: return "sha512"; /* RSA/SHA-512 */ - case 12: return NULL; /* ECC-GOST */ + case 12: return "gosthash94"; /* ECC-GOST */ case 13: return "sha256"; /* ECDSAP256SHA256 */ - case 14: return "sha384"; /* ECDSAP384SHA384 */ + case 14: return "sha384"; /* ECDSAP384SHA384 */ case 15: return "null_hash"; /* ED25519 */ - case 16: return NULL; /* ED448 */ + case 16: return "null_hash"; /* ED448 */ default: return NULL; } } - + /* http://www.iana.org/assignments/dnssec-nsec3-parameters/dnssec-nsec3-parameters.xhtml */ char *nsec3_digest_name(int digest) { diff --git a/dnsmasq/dbus.c b/src/dnsmasq/dbus.c similarity index 97% rename from dnsmasq/dbus.c rename to src/dnsmasq/dbus.c index b8d5bece0..412339108 100644 --- a/dnsmasq/dbus.c +++ b/src/dnsmasq/dbus.c @@ -1,4 +1,4 @@ -/* dnsmasq is Copyright (c) 2000-2018 Simon Kelley +/* dnsmasq is Copyright (c) 2000-2020 Simon Kelley This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -185,9 +185,6 @@ static void dbus_read_servers(DBusMessage *message) } } -#ifndef HAVE_IPV6 - my_syslog(LOG_WARNING, _("attempt to set an IPv6 server address via DBus - no IPv6 support")); -#else if (i == sizeof(struct in6_addr)) { memcpy(&addr.in6.sin6_addr, p, sizeof(struct in6_addr)); @@ -202,7 +199,6 @@ static void dbus_read_servers(DBusMessage *message) source_addr.in6.sin6_port = htons(daemon->query_port); skip = 0; } -#endif } else /* At the end */ @@ -460,7 +456,7 @@ static DBusMessage *dbus_add_lease(DBusMessage* message) int clid_len, hostname_len, hw_len, hw_type; dbus_uint32_t expires, ia_id; dbus_bool_t is_temporary; - struct all_addr addr; + union all_addr addr; time_t now = dnsmasq_time(); unsigned char dhcp_chaddr[DHCP_CHADDR_MAX]; @@ -530,20 +526,20 @@ static DBusMessage *dbus_add_lease(DBusMessage* message) dbus_message_iter_get_basic(&iter, &is_temporary); - if (inet_pton(AF_INET, ipaddr, &addr.addr.addr4)) + if (inet_pton(AF_INET, ipaddr, &addr.addr4)) { if (ia_id != 0 || is_temporary) return dbus_message_new_error(message, DBUS_ERROR_INVALID_ARGS, "ia_id and is_temporary must be zero for IPv4 lease"); - if (!(lease = lease_find_by_addr(addr.addr.addr4))) - lease = lease4_allocate(addr.addr.addr4); + if (!(lease = lease_find_by_addr(addr.addr4))) + lease = lease4_allocate(addr.addr4); } #ifdef HAVE_DHCP6 - else if (inet_pton(AF_INET6, ipaddr, &addr.addr.addr6)) + else if (inet_pton(AF_INET6, ipaddr, &addr.addr6)) { - if (!(lease = lease6_find_by_addr(&addr.addr.addr6, 128, 0))) - lease = lease6_allocate(&addr.addr.addr6, + if (!(lease = lease6_find_by_addr(&addr.addr6, 128, 0))) + lease = lease6_allocate(&addr.addr6, is_temporary ? LEASE_TA : LEASE_NA); lease_set_iaid(lease, ia_id); } @@ -574,7 +570,7 @@ static DBusMessage *dbus_del_lease(DBusMessage* message) DBusMessageIter iter; const char *ipaddr; DBusMessage *reply; - struct all_addr addr; + union all_addr addr; dbus_bool_t ret = 1; time_t now = dnsmasq_time(); @@ -588,11 +584,11 @@ static DBusMessage *dbus_del_lease(DBusMessage* message) dbus_message_iter_get_basic(&iter, &ipaddr); - if (inet_pton(AF_INET, ipaddr, &addr.addr.addr4)) - lease = lease_find_by_addr(addr.addr.addr4); + if (inet_pton(AF_INET, ipaddr, &addr.addr4)) + lease = lease_find_by_addr(addr.addr4); #ifdef HAVE_DHCP6 - else if (inet_pton(AF_INET6, ipaddr, &addr.addr.addr6)) - lease = lease6_find_by_addr(&addr.addr.addr6, 128, 0); + else if (inet_pton(AF_INET6, ipaddr, &addr.addr6)) + lease = lease6_find_by_addr(&addr.addr6, 128, 0); #endif else return dbus_message_new_error_printf(message, DBUS_ERROR_INVALID_ARGS, diff --git a/dnsmasq/dhcp-common.c b/src/dnsmasq/dhcp-common.c similarity index 88% rename from dnsmasq/dhcp-common.c rename to src/dnsmasq/dhcp-common.c index 9d3b8b551..ea449cbc3 100644 --- a/dnsmasq/dhcp-common.c +++ b/src/dnsmasq/dhcp-common.c @@ -1,4 +1,4 @@ -/* dnsmasq is Copyright (c) 2000-2018 Simon Kelley +/* dnsmasq is Copyright (c) 2000-2020 Simon Kelley This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -271,35 +271,45 @@ static int is_config_in_context(struct dhcp_context *context, struct dhcp_config { if (!context) /* called via find_config() from lease_update_from_configs() */ return 1; - - if (!(config->flags & (CONFIG_ADDR | CONFIG_ADDR6))) - return 1; #ifdef HAVE_DHCP6 - if ((context->flags & CONTEXT_V6) && (config->flags & CONFIG_WILDCARD)) - return 1; -#endif + if (context->flags & CONTEXT_V6) + { + struct addrlist *addr_list; - for (; context; context = context->current) -#ifdef HAVE_DHCP6 - if (context->flags & CONTEXT_V6) - { - if ((config->flags & CONFIG_ADDR6) && is_same_net6(&config->addr6, &context->start6, context->prefix)) - return 1; - } - else + if (!(config->flags & CONFIG_ADDR6)) + return 1; + + for (; context; context = context->current) + for (addr_list = config->addr6; addr_list; addr_list = addr_list->next) + { + if ((addr_list->flags & ADDRLIST_WILDCARD) && context->prefix == 64) + return 1; + + if (is_same_net6(&addr_list->addr.addr6, &context->start6, context->prefix)) + return 1; + } + } + else #endif - if ((config->flags & CONFIG_ADDR) && is_same_net(config->addr, context->start, context->netmask)) + { + if (!(config->flags & CONFIG_ADDR)) return 1; + + for (; context; context = context->current) + if ((config->flags & CONFIG_ADDR) && is_same_net(config->addr, context->start, context->netmask)) + return 1; + } return 0; } -struct dhcp_config *find_config(struct dhcp_config *configs, - struct dhcp_context *context, - unsigned char *clid, int clid_len, - unsigned char *hwaddr, int hw_len, - int hw_type, char *hostname) +static struct dhcp_config *find_config_match(struct dhcp_config *configs, + struct dhcp_context *context, + unsigned char *clid, int clid_len, + unsigned char *hwaddr, int hw_len, + int hw_type, char *hostname, + struct dhcp_netid *tags, int tag_not_needed) { int count, new; struct dhcp_config *config, *candidate; @@ -311,7 +321,9 @@ struct dhcp_config *find_config(struct dhcp_config *configs, { if (config->clid_len == clid_len && memcmp(config->clid, clid, clid_len) == 0 && - is_config_in_context(context, config)) + is_config_in_context(context, config) && + match_netid(config->filter, tags, tag_not_needed)) + return config; /* dhcpcd prefixes ASCII client IDs by zero which is wrong, but we try and @@ -319,7 +331,8 @@ struct dhcp_config *find_config(struct dhcp_config *configs, see lease_update_from_configs() */ if ((!context || !(context->flags & CONTEXT_V6)) && *clid == 0 && config->clid_len == clid_len-1 && memcmp(config->clid, clid+1, clid_len-1) == 0 && - is_config_in_context(context, config)) + is_config_in_context(context, config) && + match_netid(config->filter, tags, tag_not_needed)) return config; } @@ -327,14 +340,16 @@ struct dhcp_config *find_config(struct dhcp_config *configs, if (hwaddr) for (config = configs; config; config = config->next) if (config_has_mac(config, hwaddr, hw_len, hw_type) && - is_config_in_context(context, config)) + is_config_in_context(context, config) && + match_netid(config->filter, tags, tag_not_needed)) return config; if (hostname && context) for (config = configs; config; config = config->next) if ((config->flags & CONFIG_NAME) && hostname_isequal(config->hostname, hostname) && - is_config_in_context(context, config)) + is_config_in_context(context, config) && + match_netid(config->filter, tags, tag_not_needed)) return config; @@ -343,7 +358,8 @@ struct dhcp_config *find_config(struct dhcp_config *configs, /* use match with fewest wildcard octets */ for (candidate = NULL, count = 0, config = configs; config; config = config->next) - if (is_config_in_context(context, config)) + if (is_config_in_context(context, config) && + match_netid(config->filter, tags, tag_not_needed)) for (conf_addr = config->hwaddr; conf_addr; conf_addr = conf_addr->next) if (conf_addr->wildcard_mask != 0 && conf_addr->hwaddr_len == hw_len && @@ -357,6 +373,21 @@ struct dhcp_config *find_config(struct dhcp_config *configs, return candidate; } +/* Find tagged configs first. */ +struct dhcp_config *find_config(struct dhcp_config *configs, + struct dhcp_context *context, + unsigned char *clid, int clid_len, + unsigned char *hwaddr, int hw_len, + int hw_type, char *hostname, struct dhcp_netid *tags) +{ + struct dhcp_config *ret = find_config_match(configs, context, clid, clid_len, hwaddr, hw_len, hw_type, hostname, tags, 0); + + if (!ret) + ret = find_config_match(configs, context, clid, clid_len, hwaddr, hw_len, hw_type, hostname, tags, 1); + + return ret; +} + void dhcp_update_configs(struct dhcp_config *configs) { /* Some people like to keep all static IP addresses in /etc/hosts. @@ -371,8 +402,14 @@ void dhcp_update_configs(struct dhcp_config *configs) int prot = AF_INET; for (config = configs; config; config = config->next) + { if (config->flags & CONFIG_ADDR_HOSTS) - config->flags &= ~(CONFIG_ADDR | CONFIG_ADDR6 | CONFIG_ADDR_HOSTS); + config->flags &= ~(CONFIG_ADDR | CONFIG_ADDR_HOSTS); +#ifdef HAVE_DHCP6 + if (config->flags & CONFIG_ADDR6_HOSTS) + config->flags &= ~(CONFIG_ADDR6 | CONFIG_ADDR6_HOSTS); +#endif + } #ifdef HAVE_DHCP6 again: @@ -403,30 +440,41 @@ void dhcp_update_configs(struct dhcp_config *configs) crec = cache_find_by_name(crec, config->hostname, 0, cacheflags); if (!crec) continue; /* should be never */ - inet_ntop(prot, &crec->addr.addr, daemon->addrbuff, ADDRSTRLEN); + inet_ntop(prot, &crec->addr, daemon->addrbuff, ADDRSTRLEN); my_syslog(MS_DHCP | LOG_WARNING, _("%s has more than one address in hostsfile, using %s for DHCP"), config->hostname, daemon->addrbuff); } if (prot == AF_INET && - (!(conf_tmp = config_find_by_address(configs, crec->addr.addr.addr.addr4)) || conf_tmp == config)) + (!(conf_tmp = config_find_by_address(configs, crec->addr.addr4)) || conf_tmp == config)) { - config->addr = crec->addr.addr.addr.addr4; + config->addr = crec->addr.addr4; config->flags |= CONFIG_ADDR | CONFIG_ADDR_HOSTS; continue; } #ifdef HAVE_DHCP6 if (prot == AF_INET6 && - (!(conf_tmp = config_find_by_address6(configs, &crec->addr.addr.addr.addr6, 128, 0)) || conf_tmp == config)) + (!(conf_tmp = config_find_by_address6(configs, NULL, 0, &crec->addr.addr6)) || conf_tmp == config)) { - memcpy(&config->addr6, &crec->addr.addr.addr.addr6, IN6ADDRSZ); - config->flags |= CONFIG_ADDR6 | CONFIG_ADDR_HOSTS; + /* host must have exactly one address if comming from /etc/hosts. */ + if (!config->addr6 && (config->addr6 = whine_malloc(sizeof(struct addrlist)))) + { + config->addr6->next = NULL; + config->addr6->flags = 0; + } + + if (config->addr6 && !config->addr6->next && !(config->addr6->flags & (ADDRLIST_WILDCARD|ADDRLIST_PREFIX))) + { + memcpy(&config->addr6->addr.addr6, &crec->addr.addr6, IN6ADDRSZ); + config->flags |= CONFIG_ADDR6 | CONFIG_ADDR6_HOSTS; + } + continue; } #endif - inet_ntop(prot, &crec->addr.addr, daemon->addrbuff, ADDRSTRLEN); + inet_ntop(prot, &crec->addr, daemon->addrbuff, ADDRSTRLEN); my_syslog(MS_DHCP | LOG_WARNING, _("duplicate IP address %s (%s) in dhcp-config directive"), daemon->addrbuff, config->hostname); @@ -482,7 +530,7 @@ char *whichdevice(void) return NULL; } - + void bindtodevice(char *device, int fd) { size_t len = strlen(device)+1; @@ -570,6 +618,7 @@ static const struct opttab_t { { "sip-server", 120, 0 }, { "classless-static-route", 121, 0 }, { "vendor-id-encap", 125, 0 }, + { "tftp-server-address", 150, OT_ADDR_LIST }, { "server-ip-address", 255, OT_ADDR_LIST }, /* special, internal only, sets siaddr */ { NULL, 0, 0 } }; @@ -600,7 +649,7 @@ static const struct opttab_t opttab6[] = { { "sntp-server", 31, OT_ADDR_LIST }, { "information-refresh-time", 32, OT_TIME }, { "FQDN", 39, OT_INTERNAL | OT_RFC1035_NAME }, - { "ntp-server", 56, 0 }, + { "ntp-server", 56, 0 /* OT_ADDR_LIST | OT_RFC1035_NAME */ }, { "bootfile-url", 59, OT_NAME }, { "bootfile-param", 60, OT_CSTRING }, { NULL, 0, 0 } @@ -693,7 +742,7 @@ char *option_string(int prot, unsigned int opt, unsigned char *val, int opt_len, if (ot[o].size & OT_ADDR_LIST) { - struct all_addr addr; + union all_addr addr; int addr_len = INADDRSZ; #ifdef HAVE_DHCP6 diff --git a/dnsmasq/dhcp-protocol.h b/src/dnsmasq/dhcp-protocol.h similarity index 98% rename from dnsmasq/dhcp-protocol.h rename to src/dnsmasq/dhcp-protocol.h index 389c85e02..e9007c878 100644 --- a/dnsmasq/dhcp-protocol.h +++ b/src/dnsmasq/dhcp-protocol.h @@ -1,4 +1,4 @@ -/* dnsmasq is Copyright (c) 2000-2018 Simon Kelley +/* dnsmasq is Copyright (c) 2000-2020 Simon Kelley This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by diff --git a/dnsmasq/dhcp.c b/src/dnsmasq/dhcp.c similarity index 85% rename from dnsmasq/dhcp.c rename to src/dnsmasq/dhcp.c index 02b9b13ae..bea468819 100644 --- a/dnsmasq/dhcp.c +++ b/src/dnsmasq/dhcp.c @@ -1,15 +1,15 @@ -/* dnsmasq is Copyright (c) 2000-2018 Simon Kelley +/* dnsmasq is Copyright (c) 2000-2020 Simon Kelley This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; version 2 dated June, 1991, or (at your option) version 3 dated 29 June, 2007. - + This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. - + You should have received a copy of the GNU General Public License along with this program. If not, see . */ @@ -51,7 +51,7 @@ static int make_fd(int port) if (fd == -1) die (_("cannot create DHCP socket: %s"), NULL, EC_BADNET); - + if (!fix_fd(fd) || #if defined(IP_MTU_DISCOVER) && defined(IP_PMTUDISC_DONT) setsockopt(fd, IPPROTO_IP, IP_MTU_DISCOVER, &mtu, sizeof(mtu)) == -1 || @@ -64,31 +64,31 @@ static int make_fd(int port) #else setsockopt(fd, IPPROTO_IP, IP_RECVIF, &oneopt, sizeof(oneopt)) == -1 || #endif - setsockopt(fd, SOL_SOCKET, SO_BROADCAST, &oneopt, sizeof(oneopt)) == -1) + setsockopt(fd, SOL_SOCKET, SO_BROADCAST, &oneopt, sizeof(oneopt)) == -1) die(_("failed to set options on DHCP socket: %s"), NULL, EC_BADNET); - + /* When bind-interfaces is set, there might be more than one dnsmasq instance binding port 67. That's OK if they serve different networks. Need to set REUSEADDR|REUSEPORT to make this possible. - Handle the case that REUSEPORT is defined, but the kernel doesn't + Handle the case that REUSEPORT is defined, but the kernel doesn't support it. This handles the introduction of REUSEPORT on Linux. */ if (option_bool(OPT_NOWILD) || option_bool(OPT_CLEVERBIND)) { int rc = 0; #ifdef SO_REUSEPORT - if ((rc = setsockopt(fd, SOL_SOCKET, SO_REUSEPORT, &oneopt, sizeof(oneopt))) == -1 && + if ((rc = setsockopt(fd, SOL_SOCKET, SO_REUSEPORT, &oneopt, sizeof(oneopt))) == -1 && errno == ENOPROTOOPT) rc = 0; #endif - + if (rc != -1) rc = setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &oneopt, sizeof(oneopt)); - + if (rc == -1) die(_("failed to set SO_REUSE{ADDR|PORT} on DHCP socket: %s"), NULL, EC_BADNET); } - + memset(&saddr, 0, sizeof(saddr)); saddr.sin_family = AF_INET; saddr.sin_port = htons(port); @@ -119,16 +119,16 @@ void dhcp_init(void) /* When we're not using capabilities, we need to do this here before we drop root. Also, set buffer size small, to avoid wasting kernel buffers */ - + if (option_bool(OPT_NO_PING)) daemon->dhcp_icmp_fd = -1; else if ((daemon->dhcp_icmp_fd = make_icmp_sock()) == -1 || setsockopt(daemon->dhcp_icmp_fd, SOL_SOCKET, SO_RCVBUF, &oneopt, sizeof(oneopt)) == -1 ) die(_("cannot create ICMP raw socket: %s."), NULL, EC_BADNET); - + /* Make BPF raw send socket */ init_bpf(); -#endif +#endif } void dhcp_packet(time_t now, int pxe_fd) @@ -144,7 +144,7 @@ void dhcp_packet(time_t now, int pxe_fd) struct sockaddr_in dest; struct cmsghdr *cmptr; struct iovec iov; - ssize_t sz; + ssize_t sz; int iface_index = 0, unicast_dest = 0, is_inform = 0, loopback = 0; int rcvd_iface_index; struct in_addr iface_addr; @@ -154,14 +154,14 @@ void dhcp_packet(time_t now, int pxe_fd) struct arpreq arp_req; struct timeval tv; #endif - + union { struct cmsghdr align; /* this ensures alignment */ #if defined(HAVE_LINUX_NETWORK) char control[CMSG_SPACE(sizeof(struct in_pktinfo))]; #elif defined(HAVE_SOLARIS_NETWORK) char control[CMSG_SPACE(sizeof(unsigned int))]; -#elif defined(HAVE_BSD_NETWORK) +#elif defined(HAVE_BSD_NETWORK) char control[CMSG_SPACE(sizeof(struct sockaddr_dl))]; #endif } control_u; @@ -173,11 +173,11 @@ void dhcp_packet(time_t now, int pxe_fd) msg.msg_namelen = sizeof(dest); msg.msg_iov = &daemon->dhcp_packet; msg.msg_iovlen = 1; - - if ((sz = recv_dhcp_packet(fd, &msg)) == -1 || - (sz < (ssize_t)(sizeof(*mess) - sizeof(mess->options)))) + + if ((sz = recv_dhcp_packet(fd, &msg)) == -1 || + (sz < (ssize_t)(sizeof(*mess) - sizeof(mess->options)))) return; - + #if defined (HAVE_LINUX_NETWORK) if (ioctl(fd, SIOCGSTAMP, &tv) == 0) recvtime = tv.tv_sec; @@ -196,7 +196,7 @@ void dhcp_packet(time_t now, int pxe_fd) unicast_dest = 1; } -#elif defined(HAVE_BSD_NETWORK) +#elif defined(HAVE_BSD_NETWORK) if (msg.msg_controllen >= sizeof(struct cmsghdr)) for (cmptr = CMSG_FIRSTHDR(&msg); cmptr; cmptr = CMSG_NXTHDR(&msg, cmptr)) if (cmptr->cmsg_level == IPPROTO_IP && cmptr->cmsg_type == IP_RECVIF) @@ -208,8 +208,8 @@ void dhcp_packet(time_t now, int pxe_fd) p.c = CMSG_DATA(cmptr); iface_index = p.s->sdl_index; } - -#elif defined(HAVE_SOLARIS_NETWORK) + +#elif defined(HAVE_SOLARIS_NETWORK) if (msg.msg_controllen >= sizeof(struct cmsghdr)) for (cmptr = CMSG_FIRSTHDR(&msg); cmptr; cmptr = CMSG_NXTHDR(&msg, cmptr)) if (cmptr->cmsg_level == IPPROTO_IP && cmptr->cmsg_type == IP_RECVIF) @@ -222,18 +222,18 @@ void dhcp_packet(time_t now, int pxe_fd) iface_index = *(p.i); } #endif - + if (!indextoname(daemon->dhcpfd, iface_index, ifr.ifr_name) || ioctl(daemon->dhcpfd, SIOCGIFFLAGS, &ifr) != 0) return; - + mess = (struct dhcp_packet *)daemon->dhcp_packet.iov_base; loopback = !mess->giaddr.s_addr && (ifr.ifr_flags & IFF_LOOPBACK); - + #ifdef HAVE_LINUX_NETWORK /* ARP fiddling uses original interface even if we pretend to use a different one. */ safe_strncpy(arp_req.arp_dev, ifr.ifr_name, sizeof(arp_req.arp_dev)); -#endif +#endif /* If the interface on which the DHCP request was received is an alias of some other interface (as specified by the @@ -253,13 +253,13 @@ void dhcp_packet(time_t now, int pxe_fd) bridge->iface); return; } - else + else { safe_strncpy(ifr.ifr_name, bridge->iface, sizeof(ifr.ifr_name)); break; } } - + if (alias) break; } @@ -269,18 +269,18 @@ void dhcp_packet(time_t now, int pxe_fd) if (!(msg.msg_flags & MSG_BCAST)) unicast_dest = 1; #endif - + if ((relay = relay_reply4((struct dhcp_packet *)daemon->dhcp_packet.iov_base, ifr.ifr_name))) { /* Reply from server, using us as relay. */ rcvd_iface_index = relay->iface_index; if (!indextoname(daemon->dhcpfd, rcvd_iface_index, ifr.ifr_name)) return; - is_relay_reply = 1; + is_relay_reply = 1; iov.iov_len = sz; #ifdef HAVE_LINUX_NETWORK safe_strncpy(arp_req.arp_dev, ifr.ifr_name, sizeof(arp_req.arp_dev)); -#endif +#endif } else { @@ -293,43 +293,43 @@ void dhcp_packet(time_t now, int pxe_fd) my_syslog(MS_DHCP | LOG_WARNING, _("DHCP packet received on %s which has no address"), ifr.ifr_name); return; } - + for (tmp = daemon->dhcp_except; tmp; tmp = tmp->next) if (tmp->name && wildcard_match(tmp->name, ifr.ifr_name)) return; - + /* unlinked contexts/relays are marked by context->current == context */ for (context = daemon->dhcp; context; context = context->next) context->current = context; - + for (relay = daemon->relay4; relay; relay = relay->next) relay->current = relay; - + parm.current = NULL; parm.relay = NULL; parm.relay_local.s_addr = 0; parm.ind = iface_index; - - if (!iface_check(AF_INET, (struct all_addr *)&iface_addr, ifr.ifr_name, NULL)) + + if (!iface_check(AF_INET, (union all_addr *)&iface_addr, ifr.ifr_name, NULL)) { /* If we failed to match the primary address of the interface, see if we've got a --listen-address for a secondary */ struct match_param match; - + match.matched = 0; match.ind = iface_index; - + if (!daemon->if_addrs || !iface_enumerate(AF_INET, &match, check_listen_addrs) || !match.matched) return; - + iface_addr = match.addr; /* make sure secondary address gets priority in case there is more than one address on the interface in the same subnet */ complete_context(match.addr, iface_index, NULL, match.netmask, match.broadcast, &parm); - } - + } + if (!iface_enumerate(AF_INET, &parm, complete_context)) return; @@ -343,11 +343,11 @@ void dhcp_packet(time_t now, int pxe_fd) return; lease_prune(NULL, now); /* lose any expired leases */ - iov.iov_len = dhcp_reply(parm.current, ifr.ifr_name, iface_index, (size_t)sz, + iov.iov_len = dhcp_reply(parm.current, ifr.ifr_name, iface_index, (size_t)sz, now, unicast_dest, loopback, &is_inform, pxe_fd, iface_addr, recvtime); lease_update_file(now); lease_update_dns(0); - + if (iov.iov_len == 0) return; } @@ -358,16 +358,16 @@ void dhcp_packet(time_t now, int pxe_fd) msg.msg_controllen = 0; msg.msg_iov = &iov; iov.iov_base = daemon->dhcp_packet.iov_base; - + /* packet buffer may have moved */ mess = (struct dhcp_packet *)daemon->dhcp_packet.iov_base; - + #ifdef HAVE_SOCKADDR_SA_LEN dest.sin_len = sizeof(struct sockaddr_in); #endif - + if (pxe_fd) - { + { if (mess->ciaddr.s_addr != 0) dest.sin_addr = mess->ciaddr; } @@ -375,21 +375,21 @@ void dhcp_packet(time_t now, int pxe_fd) { /* Send to BOOTP relay */ dest.sin_port = htons(daemon->dhcp_server_port); - dest.sin_addr = mess->giaddr; + dest.sin_addr = mess->giaddr; } else if (mess->ciaddr.s_addr) { /* If the client's idea of its own address tallys with the source address in the request packet, we believe the - source port too, and send back to that. If we're replying + source port too, and send back to that. If we're replying to a DHCPINFORM, trust the source address always. */ if ((!is_inform && dest.sin_addr.s_addr != mess->ciaddr.s_addr) || dest.sin_port == 0 || dest.sin_addr.s_addr == 0 || is_relay_reply) { - dest.sin_port = htons(daemon->dhcp_client_port); + dest.sin_port = htons(daemon->dhcp_client_port); dest.sin_addr = mess->ciaddr; } - } + } #if defined(HAVE_LINUX_NETWORK) else { @@ -401,7 +401,8 @@ void dhcp_packet(time_t now, int pxe_fd) pkt = (struct in_pktinfo *)CMSG_DATA(cmptr); pkt->ipi_ifindex = rcvd_iface_index; pkt->ipi_spec_dst.s_addr = 0; - msg.msg_controllen = cmptr->cmsg_len = CMSG_LEN(sizeof(struct in_pktinfo)); + msg.msg_controllen = CMSG_SPACE(sizeof(struct in_pktinfo)); + cmptr->cmsg_len = CMSG_LEN(sizeof(struct in_pktinfo)); cmptr->cmsg_level = IPPROTO_IP; cmptr->cmsg_type = IP_PKTINFO; @@ -438,9 +439,9 @@ void dhcp_packet(time_t now, int pxe_fd) } else { - /* unicast to unconfigured client. Inject mac address direct into ARP cache. + /* unicast to unconfigured client. Inject mac address direct into ARP cache. Note that this only works for ethernet on solaris, because we use SIOCSARP - and not SIOCSXARP, which would be perfect, except that it returns ENXIO + and not SIOCSXARP, which would be perfect, except that it returns ENXIO mysteriously. Bah. Fall back to broadcast for other net types. */ struct arpreq req; dest.sin_addr = mess->yiaddr; @@ -452,17 +453,17 @@ void dhcp_packet(time_t now, int pxe_fd) ioctl(daemon->dhcpfd, SIOCSARP, &req); } #elif defined(HAVE_BSD_NETWORK) - else + else { send_via_bpf(mess, iov.iov_len, iface_addr, &ifr); return; } #endif - + #ifdef HAVE_SOLARIS_NETWORK setsockopt(fd, IPPROTO_IP, IP_BOUND_IF, &iface_index, sizeof(iface_index)); #endif - + while(retry_send(sendmsg(fd, &msg, 0))); /* This can fail when, eg, iptables DROPS destination 255.255.255.255 */ @@ -493,11 +494,11 @@ static int check_listen_addrs(struct in_addr local, int if_index, char *label, break; } } - + return 1; } -/* This is a complex routine: it gets called with each (address,netmask,broadcast) triple +/* This is a complex routine: it gets called with each (address,netmask,broadcast) triple of each interface (and any relay address) and does the following things: 1) Discards stuff for interfaces other than the one on which a DHCP packet just arrived. @@ -507,33 +508,83 @@ static int check_listen_addrs(struct in_addr local, int if_index, char *label, Note that the current chain may be superseded later for configured hosts or those coming via gateways. */ -static int complete_context(struct in_addr local, int if_index, char *label, - struct in_addr netmask, struct in_addr broadcast, void *vparam) +static void guess_range_netmask(struct in_addr addr, struct in_addr netmask) { struct dhcp_context *context; - struct dhcp_relay *relay; - struct iface_param *param = vparam; - - (void)label; for (context = daemon->dhcp; context; context = context->next) - { - if (!(context->flags & CONTEXT_NETMASK) && - (is_same_net(local, context->start, netmask) || - is_same_net(local, context->end, netmask))) - { + if (!(context->flags & CONTEXT_NETMASK) && + (is_same_net(addr, context->start, netmask) || + is_same_net(addr, context->end, netmask))) + { if (context->netmask.s_addr != netmask.s_addr && - !(is_same_net(local, context->start, netmask) && - is_same_net(local, context->end, netmask))) + !(is_same_net(addr, context->start, netmask) && + is_same_net(addr, context->end, netmask))) { strcpy(daemon->dhcp_buff, inet_ntoa(context->start)); strcpy(daemon->dhcp_buff2, inet_ntoa(context->end)); my_syslog(MS_DHCP | LOG_WARNING, _("DHCP range %s -- %s is not consistent with netmask %s"), daemon->dhcp_buff, daemon->dhcp_buff2, inet_ntoa(netmask)); - } - context->netmask = netmask; + } + context->netmask = netmask; } +} + +static int complete_context(struct in_addr local, int if_index, char *label, + struct in_addr netmask, struct in_addr broadcast, void *vparam) +{ + struct dhcp_context *context; + struct dhcp_relay *relay; + struct iface_param *param = vparam; + struct shared_network *share; + + (void)label; + + for (share = daemon->shared_networks; share; share = share->next) + { + +#ifdef HAVE_DHCP6 + if (share->shared_addr.s_addr == 0) + continue; +#endif + + if (share->if_index != 0) + { + if (share->if_index != if_index) + continue; + } + else + { + if (share->match_addr.s_addr != local.s_addr) + continue; + } + + for (context = daemon->dhcp; context; context = context->next) + { + if (context->netmask.s_addr != 0 && + is_same_net(share->shared_addr, context->start, context->netmask) && + is_same_net(share->shared_addr, context->end, context->netmask)) + { + /* link it onto the current chain if we've not seen it before */ + if (context->current == context) + { + /* For a shared network, we have no way to guess what the default route should be. */ + context->router.s_addr = 0; + context->local = local; /* Use configured address for Server Identifier */ + context->current = param->current; + param->current = context; + } + + if (!(context->flags & CONTEXT_BRDCAST)) + context->broadcast.s_addr = context->start.s_addr | ~context->netmask.s_addr; + } + } + } + guess_range_netmask(local, netmask); + + for (context = daemon->dhcp; context; context = context->next) + { if (context->netmask.s_addr != 0 && is_same_net(local, context->start, context->netmask) && is_same_net(local, context->end, context->netmask)) @@ -546,44 +597,44 @@ static int complete_context(struct in_addr local, int if_index, char *label, context->current = param->current; param->current = context; } - + if (!(context->flags & CONTEXT_BRDCAST)) { if (is_same_net(broadcast, context->start, context->netmask)) context->broadcast = broadcast; - else + else context->broadcast.s_addr = context->start.s_addr | ~context->netmask.s_addr; } - } + } } for (relay = daemon->relay4; relay; relay = relay->next) - if (if_index == param->ind && relay->local.addr.addr4.s_addr == local.s_addr && relay->current == relay && + if (if_index == param->ind && relay->local.addr4.s_addr == local.s_addr && relay->current == relay && (param->relay_local.s_addr == 0 || param->relay_local.s_addr == local.s_addr)) { relay->current = param->relay; param->relay = relay; - param->relay_local = local; + param->relay_local = local; } return 1; } - -struct dhcp_context *address_available(struct dhcp_context *context, + +struct dhcp_context *address_available(struct dhcp_context *context, struct in_addr taddr, struct dhcp_netid *netids) { /* Check is an address is OK for this network, check all possible ranges. Make sure that the address isn't in use by the server itself. */ - + unsigned int start, end, addr = ntohl(taddr.s_addr); struct dhcp_context *tmp; for (tmp = context; tmp; tmp = tmp->current) if (taddr.s_addr == context->router.s_addr) return NULL; - + for (tmp = context; tmp; tmp = tmp->current) { start = ntohl(tmp->start.s_addr); @@ -599,7 +650,7 @@ struct dhcp_context *address_available(struct dhcp_context *context, return NULL; } -struct dhcp_context *narrow_context(struct dhcp_context *context, +struct dhcp_context *narrow_context(struct dhcp_context *context, struct in_addr taddr, struct dhcp_netid *netids) { @@ -608,19 +659,19 @@ struct dhcp_context *narrow_context(struct dhcp_context *context, Here we have an address, and return the actual context corresponding to that address. Note that none may fit, if the address came a dhcp-host and is outside any dhcp-range. In that case we return a static range if possible, or failing that, - any context on the correct subnet. (If there's more than one, this is a dodgy + any context on the correct subnet. (If there's more than one, this is a dodgy configuration: maybe there should be a warning.) */ - + struct dhcp_context *tmp; if (!(tmp = address_available(context, taddr, netids))) { for (tmp = context; tmp; tmp = tmp->current) if (match_netid(tmp->filter, netids, 1) && - is_same_net(taddr, tmp->start, tmp->netmask) && + is_same_net(taddr, tmp->start, tmp->netmask) && (tmp->flags & CONTEXT_STATIC)) break; - + if (!tmp) for (tmp = context; tmp; tmp = tmp->current) if (match_netid(tmp->filter, netids, 1) && @@ -628,18 +679,18 @@ struct dhcp_context *narrow_context(struct dhcp_context *context, !(tmp->flags & CONTEXT_PROXY)) break; } - + /* Only one context allowed now */ if (tmp) tmp->current = NULL; - + return tmp; } struct dhcp_config *config_find_by_address(struct dhcp_config *configs, struct in_addr addr) { struct dhcp_config *config; - + for (config = configs; config; config = config->next) if ((config->flags & CONFIG_ADDR) && config->addr.s_addr == addr.s_addr) return config; @@ -662,18 +713,18 @@ struct ping_result *do_icmp_ping(time_t now, struct in_addr addr, unsigned int h PING_CACHE_TIME seconds. If so, assume the same situation still exists. This avoids problems when a stupid client bangs on us repeatedly. As a final check, if we did more - than 60% of the possible ping checks in the last + than 60% of the possible ping checks in the last PING_CACHE_TIME, we are in high-load mode, so don't do any more. */ for (count = 0, r = daemon->ping_results; r; r = r->next) if (difftime(now, r->time) > (float)PING_CACHE_TIME) victim = r; /* old record */ - else + else { count++; if (r->addr.s_addr == addr.s_addr) return r; } - + /* didn't find cached entry */ if ((count >= max) || option_bool(OPT_NO_PING) || loopback) { @@ -694,8 +745,8 @@ struct ping_result *do_icmp_ping(time_t now, struct in_addr addr, unsigned int h daemon->ping_results = victim; } } - - /* record that this address is OK for 30s + + /* record that this address is OK for 30s without more ping checks */ if (victim) { @@ -708,8 +759,8 @@ struct ping_result *do_icmp_ping(time_t now, struct in_addr addr, unsigned int h } int address_allocate(struct dhcp_context *context, - struct in_addr *addrp, unsigned char *hwaddr, int hw_len, - struct dhcp_netid *netids, time_t now, int loopback) + struct in_addr *addrp, unsigned char *hwaddr, int hw_len, + struct dhcp_netid *netids, time_t now, int loopback) { /* Find a free address: exclude anything in use and anything allocated to a particular hwaddr/clientid/hostname in our configuration. @@ -718,17 +769,17 @@ int address_allocate(struct dhcp_context *context, struct in_addr start, addr; struct dhcp_context *c, *d; int i, pass; - unsigned int j; + unsigned int j; /* hash hwaddr: use the SDBM hashing algorithm. Seems to give good - dispersal even with similarly-valued "strings". */ + dispersal even with similarly-valued "strings". */ for (j = 0, i = 0; i < hw_len; i++) j = hwaddr[i] + (j << 6) + (j << 16) - j; /* j == 0 is marker */ if (j == 0) j = 1; - + for (pass = 0; pass <= 1; pass++) for (c = context; c; c = c->current) if (c->flags & (CONTEXT_STATIC | CONTEXT_PROXY)) @@ -742,55 +793,64 @@ int address_allocate(struct dhcp_context *context, start = lease_find_max_addr(c); else /* pick a seed based on hwaddr */ - start.s_addr = htonl(ntohl(c->start.s_addr) + + start.s_addr = htonl(ntohl(c->start.s_addr) + ((j + c->addr_epoch) % (1 + ntohl(c->end.s_addr) - ntohl(c->start.s_addr)))); /* iterate until we find a free address. */ addr = start; - + do { /* eliminate addresses in use by the server. */ for (d = context; d; d = d->current) if (addr.s_addr == d->router.s_addr) break; - /* Addresses which end in .255 and .0 are broken in Windows even when using + /* Addresses which end in .255 and .0 are broken in Windows even when using supernetting. ie dhcp-range=192.168.0.1,192.168.1.254,255,255,254.0 then 192.168.0.255 is a valid IP address, but not for Windows as it's - in the class C range. See KB281579. We therefore don't allocate these - addresses to avoid hard-to-diagnose problems. Thanks Bill. */ + in the class C range. See KB281579. We therefore don't allocate these + addresses to avoid hard-to-diagnose problems. Thanks Bill. */ if (!d && - !lease_find_by_addr(addr) && + !lease_find_by_addr(addr) && !config_find_by_address(daemon->dhcp_conf, addr) && - (!IN_CLASSC(ntohl(addr.s_addr)) || + (!IN_CLASSC(ntohl(addr.s_addr)) || ((ntohl(addr.s_addr) & 0xff) != 0xff && ((ntohl(addr.s_addr) & 0xff) != 0x0)))) { - struct ping_result *r; - - if ((r = do_icmp_ping(now, addr, j, loopback))) - { - /* consec-ip mode: we offered this address for another client - (different hash) recently, don't offer it to this one. */ - if (!option_bool(OPT_CONSEC_ADDR) || r->hash == j) - { - *addrp = addr; - return 1; - } - } + /* in consec-ip mode, skip addresses equal to + the number of addresses rejected by clients. This + should avoid the same client being offered the same + address after it has rjected it. */ + if (option_bool(OPT_CONSEC_ADDR) && c->addr_epoch) + c->addr_epoch--; else { - /* address in use: perturb address selection so that we are - less likely to try this address again. */ - if (!option_bool(OPT_CONSEC_ADDR)) - c->addr_epoch++; + struct ping_result *r; + + if ((r = do_icmp_ping(now, addr, j, loopback))) + { + /* consec-ip mode: we offered this address for another client + (different hash) recently, don't offer it to this one. */ + if (!option_bool(OPT_CONSEC_ADDR) || r->hash == j) + { + *addrp = addr; + return 1; + } + } + else + { + /* address in use: perturb address selection so that we are + less likely to try this address again. */ + if (!option_bool(OPT_CONSEC_ADDR)) + c->addr_epoch++; + } } } - + addr.s_addr = htonl(ntohl(addr.s_addr) + 1); - + if (addr.s_addr == htonl(ntohl(c->end.s_addr) + 1)) addr = c->start; - + } while (addr.s_addr != start.s_addr); } @@ -810,7 +870,7 @@ void dhcp_read_ethers(void) int count = 0, lineno = 0; addr.s_addr = 0; /* eliminate warning */ - + if (!f) { my_syslog(MS_DHCP | LOG_ERR, _("failed to read %s: %s"), ETHERSFILE, strerror(errno)); @@ -837,54 +897,54 @@ void dhcp_read_ethers(void) while (fgets(buff, MAXDNAME, f)) { char *host = NULL; - + lineno++; - + while (strlen(buff) > 0 && isspace((int)buff[strlen(buff)-1])) buff[strlen(buff)-1] = 0; - + if ((*buff == '#') || (*buff == '+') || (*buff == 0)) continue; - + for (ip = buff; *ip && !isspace((int)*ip); ip++); for(; *ip && isspace((int)*ip); ip++) *ip = 0; if (!*ip || parse_hex(buff, hwaddr, ETHER_ADDR_LEN, NULL, NULL) != ETHER_ADDR_LEN) { - my_syslog(MS_DHCP | LOG_ERR, _("bad line at %s line %d"), ETHERSFILE, lineno); + my_syslog(MS_DHCP | LOG_ERR, _("bad line at %s line %d"), ETHERSFILE, lineno); continue; } - + /* check for name or dotted-quad */ for (cp = ip; *cp; cp++) if (!(*cp == '.' || (*cp >='0' && *cp <= '9'))) break; - + if (!*cp) { if ((addr.s_addr = inet_addr(ip)) == (in_addr_t)-1) { - my_syslog(MS_DHCP | LOG_ERR, _("bad address at %s line %d"), ETHERSFILE, lineno); + my_syslog(MS_DHCP | LOG_ERR, _("bad address at %s line %d"), ETHERSFILE, lineno); continue; } flags = CONFIG_ADDR; - + for (config = daemon->dhcp_conf; config; config = config->next) if ((config->flags & CONFIG_ADDR) && config->addr.s_addr == addr.s_addr) break; } - else + else { int nomem; if (!(host = canonicalise(ip, &nomem)) || !legal_hostname(host)) { if (!nomem) - my_syslog(MS_DHCP | LOG_ERR, _("bad name at %s line %d"), ETHERSFILE, lineno); + my_syslog(MS_DHCP | LOG_ERR, _("bad name at %s line %d"), ETHERSFILE, lineno); free(host); continue; } - + flags = CONFIG_NAME; for (config = daemon->dhcp_conf; config; config = config->next) @@ -894,24 +954,24 @@ void dhcp_read_ethers(void) if (config && (config->flags & CONFIG_FROM_ETHERS)) { - my_syslog(MS_DHCP | LOG_ERR, _("ignoring %s line %d, duplicate name or IP address"), ETHERSFILE, lineno); + my_syslog(MS_DHCP | LOG_ERR, _("ignoring %s line %d, duplicate name or IP address"), ETHERSFILE, lineno); continue; } - + if (!config) - { + { for (config = daemon->dhcp_conf; config; config = config->next) { struct hwaddr_config *conf_addr = config->hwaddr; - if (conf_addr && - conf_addr->next == NULL && + if (conf_addr && + conf_addr->next == NULL && conf_addr->wildcard_mask == 0 && conf_addr->hwaddr_len == ETHER_ADDR_LEN && (conf_addr->hwaddr_type == ARPHRD_ETHER || conf_addr->hwaddr_type == 0) && memcmp(conf_addr->hwaddr, hwaddr, ETHER_ADDR_LEN) == 0) break; } - + if (!config) { if (!(config = whine_malloc(sizeof(struct dhcp_config)))) @@ -923,19 +983,19 @@ void dhcp_read_ethers(void) config->next = daemon->dhcp_conf; daemon->dhcp_conf = config; } - + config->flags |= flags; - + if (flags & CONFIG_NAME) { config->hostname = host; host = NULL; } - + if (flags & CONFIG_ADDR) config->addr = addr; } - + config->flags |= CONFIG_NOCLID; if (!config->hwaddr) config->hwaddr = whine_malloc(sizeof(struct hwaddr_config)); @@ -948,11 +1008,11 @@ void dhcp_read_ethers(void) config->hwaddr->next = NULL; } count++; - + free(host); } - + fclose(f); my_syslog(MS_DHCP | LOG_INFO, _("read %s - %d addresses"), ETHERSFILE, count); @@ -962,7 +1022,7 @@ void dhcp_read_ethers(void) /* If we've not found a hostname any other way, try and see if there's one in /etc/hosts for this address. If it has a domain part, that must match the set domain and it gets stripped. The set of legal domain names is bigger than the set of legal hostnames - so check here that the domain name is legal as a hostname. + so check here that the domain name is legal as a hostname. NOTE: we're only allowed to overwrite daemon->dhcp_buff if we succeed. */ char *host_from_dns(struct in_addr addr) { @@ -970,14 +1030,14 @@ char *host_from_dns(struct in_addr addr) if (daemon->port == 0) return NULL; /* DNS disabled. */ - - lookup = cache_find_by_addr(NULL, (struct all_addr *)&addr, 0, F_IPV4); + + lookup = cache_find_by_addr(NULL, (union all_addr *)&addr, 0, F_IPV4); if (lookup && (lookup->flags & F_HOSTS)) { char *dot, *hostname = cache_get_name(lookup); dot = strchr(hostname, '.'); - + if (dot && strlen(dot+1) != 0) { char *d2 = get_domain(addr); @@ -987,38 +1047,38 @@ char *host_from_dns(struct in_addr addr) if (!legal_hostname(hostname)) return NULL; - + safe_strncpy(daemon->dhcp_buff, hostname, 256); strip_hostname(daemon->dhcp_buff); return daemon->dhcp_buff; } - + return NULL; } static int relay_upstream4(struct dhcp_relay *relay, struct dhcp_packet *mess, size_t sz, int iface_index) { /* ->local is same value for all relays on ->current chain */ - struct all_addr from; - + union all_addr from; + if (mess->op != BOOTREQUEST) return 0; /* source address == relay address */ - from.addr.addr4 = relay->local.addr.addr4; - + from.addr4 = relay->local.addr4; + /* already gatewayed ? */ if (mess->giaddr.s_addr) { /* if so check if by us, to stomp on loops. */ - if (mess->giaddr.s_addr == relay->local.addr.addr4.s_addr) + if (mess->giaddr.s_addr == relay->local.addr4.s_addr) return 1; } else { /* plug in our address */ - mess->giaddr.s_addr = relay->local.addr.addr4.s_addr; + mess->giaddr.s_addr = relay->local.addr4.s_addr; } if ((mess->hops++) > 20) @@ -1027,23 +1087,23 @@ static int relay_upstream4(struct dhcp_relay *relay, struct dhcp_packet *mess, for (; relay; relay = relay->current) { union mysockaddr to; - + to.sa.sa_family = AF_INET; - to.in.sin_addr = relay->server.addr.addr4; + to.in.sin_addr = relay->server.addr4; to.in.sin_port = htons(daemon->dhcp_server_port); - + send_from(daemon->dhcpfd, 0, (char *)mess, sz, &to, &from, 0); - + if (option_bool(OPT_LOG_OPTS)) { inet_ntop(AF_INET, &relay->local, daemon->addrbuff, ADDRSTRLEN); - my_syslog(MS_DHCP | LOG_INFO, _("DHCP relay %s -> %s"), daemon->addrbuff, inet_ntoa(relay->server.addr.addr4)); + my_syslog(MS_DHCP | LOG_INFO, _("DHCP relay %s -> %s"), daemon->addrbuff, inet_ntoa(relay->server.addr4)); } - + /* Save this for replies */ relay->iface_index = iface_index; } - + return 1; } @@ -1057,14 +1117,14 @@ static struct dhcp_relay *relay_reply4(struct dhcp_packet *mess, char *arrival_i for (relay = daemon->relay4; relay; relay = relay->next) { - if (mess->giaddr.s_addr == relay->local.addr.addr4.s_addr) + if (mess->giaddr.s_addr == relay->local.addr4.s_addr) { if (!relay->interface || wildcard_match(relay->interface, arrival_interface)) return relay->iface_index != 0 ? relay : NULL; } } - - return NULL; -} + + return NULL; +} #endif diff --git a/dnsmasq/dhcp6-protocol.h b/src/dnsmasq/dhcp6-protocol.h similarity index 91% rename from dnsmasq/dhcp6-protocol.h rename to src/dnsmasq/dhcp6-protocol.h index fee5d282a..48b027b01 100644 --- a/dnsmasq/dhcp6-protocol.h +++ b/src/dnsmasq/dhcp6-protocol.h @@ -1,4 +1,4 @@ -/* dnsmasq is Copyright (c) 2000-2018 Simon Kelley +/* dnsmasq is Copyright (c) 2000-2020 Simon Kelley This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -59,12 +59,12 @@ #define OPTION6_REMOTE_ID 37 #define OPTION6_SUBSCRIBER_ID 38 #define OPTION6_FQDN 39 +#define OPTION6_NTP_SERVER 56 #define OPTION6_CLIENT_MAC 79 -/* replace this with the real number when allocated. - defining this also enables the relevant code. */ -/* #define OPTION6_PREFIX_CLASS 99 */ - +#define NTP_SUBOPTION_SRV_ADDR 1 +#define NTP_SUBOPTION_MC_ADDR 2 +#define NTP_SUBOPTION_SRV_FQDN 3 #define DHCP6SUCCESS 0 #define DHCP6UNSPEC 1 diff --git a/dnsmasq/dhcp6.c b/src/dnsmasq/dhcp6.c similarity index 74% rename from dnsmasq/dhcp6.c rename to src/dnsmasq/dhcp6.c index 911704301..096e2e171 100644 --- a/dnsmasq/dhcp6.c +++ b/src/dnsmasq/dhcp6.c @@ -1,15 +1,15 @@ -/* dnsmasq is Copyright (c) 2000-2018 Simon Kelley +/* dnsmasq is Copyright (c) 2000-2020 Simon Kelley This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; version 2 dated June, 1991, or (at your option) version 3 dated 29 June, 2007. - + This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. - + You should have received a copy of the GNU General Public License along with this program. If not, see . */ @@ -29,9 +29,9 @@ struct iface_param { static int complete_context6(struct in6_addr *local, int prefix, - int scope, int if_index, int flags, + int scope, int if_index, int flags, unsigned int preferred, unsigned int valid, void *vparam); -static int make_duid1(int index, unsigned int type, char *mac, size_t maclen, void *parm); +static int make_duid1(int index, unsigned int type, char *mac, size_t maclen, void *parm); void dhcp6_init(void) { @@ -50,11 +50,11 @@ void dhcp6_init(void) !fix_fd(fd) || !set_ipv6pktinfo(fd)) die (_("cannot create DHCPv6 socket: %s"), NULL, EC_BADNET); - + /* When bind-interfaces is set, there might be more than one dnsmasq instance binding port 547. That's OK if they serve different networks. Need to set REUSEADDR|REUSEPORT to make this possible. - Handle the case that REUSEPORT is defined, but the kernel doesn't + Handle the case that REUSEPORT is defined, but the kernel doesn't support it. This handles the introduction of REUSEPORT on Linux. */ if (option_bool(OPT_NOWILD) || option_bool(OPT_CLEVERBIND)) { @@ -65,14 +65,14 @@ void dhcp6_init(void) errno == ENOPROTOOPT) rc = 0; #endif - + if (rc != -1) rc = setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &oneopt, sizeof(oneopt)); - + if (rc == -1) die(_("failed to set SO_REUSE{ADDR|PORT} on DHCPv6 socket: %s"), NULL, EC_BADNET); } - + memset(&saddr, 0, sizeof(saddr)); #ifdef HAVE_SOCKADDR_SA_LEN saddr.sin6_len = sizeof(struct sockaddr_in6); @@ -80,10 +80,10 @@ void dhcp6_init(void) saddr.sin6_family = AF_INET6; saddr.sin6_addr = in6addr_any; saddr.sin6_port = htons(DHCPV6_SERVER_PORT); - + if (bind(fd, (struct sockaddr *)&saddr, sizeof(struct sockaddr_in6))) die(_("failed to bind DHCPv6 server socket: %s"), NULL, EC_BADNET); - + daemon->dhcp6fd = fd; } @@ -100,7 +100,7 @@ void dhcp6_packet(time_t now) char control6[CMSG_SPACE(sizeof(struct in6_pktinfo))]; } control_u; struct sockaddr_in6 from; - ssize_t sz; + ssize_t sz; struct ifreq ifr; struct iname *tmp; unsigned short port; @@ -115,10 +115,10 @@ void dhcp6_packet(time_t now) msg.msg_namelen = sizeof(from); msg.msg_iov = &daemon->dhcp_packet; msg.msg_iovlen = 1; - + if ((sz = recv_dhcp_packet(daemon->dhcp6fd, &msg)) == -1) return; - + for (cmptr = CMSG_FIRSTHDR(&msg); cmptr; cmptr = CMSG_NXTHDR(&msg, cmptr)) if (cmptr->cmsg_level == IPPROTO_IPV6 && cmptr->cmsg_type == daemon->v6pktinfo) { @@ -127,7 +127,7 @@ void dhcp6_packet(time_t now) struct in6_pktinfo *p; } p; p.c = CMSG_DATA(cmptr); - + if_index = p.p->ipi6_ifindex; dst_addr = p.p->ipi6_addr; } @@ -135,18 +135,25 @@ void dhcp6_packet(time_t now) if (!indextoname(daemon->dhcp6fd, if_index, ifr.ifr_name)) return; - if ((port = relay_reply6(&from, sz, ifr.ifr_name)) == 0) + if ((port = relay_reply6(&from, sz, ifr.ifr_name)) != 0) + { + from.sin6_port = htons(port); + while (retry_send(sendto(daemon->dhcp6fd, daemon->outpacket.iov_base, + save_counter(-1), 0, (struct sockaddr *)&from, + sizeof(from)))); + } + else { struct dhcp_bridge *bridge, *alias; for (tmp = daemon->if_except; tmp; tmp = tmp->next) if (tmp->name && wildcard_match(tmp->name, ifr.ifr_name)) return; - + for (tmp = daemon->dhcp_except; tmp; tmp = tmp->next) if (tmp->name && wildcard_match(tmp->name, ifr.ifr_name)) return; - + parm.current = NULL; parm.relay = NULL; memset(&parm.relay_local, 0, IN6ADDRSZ); @@ -179,7 +186,7 @@ void dhcp6_packet(time_t now) if (alias) break; } - + for (context = daemon->dhcp6; context; context = context->next) if (IN6_IS_ADDR_UNSPECIFIED(&context->start6) && context->prefix == 0) { @@ -196,66 +203,68 @@ void dhcp6_packet(time_t now) for (relay = daemon->relay6; relay; relay = relay->next) relay->current = relay; - + if (!iface_enumerate(AF_INET6, &parm, complete_context6)) return; if (daemon->if_names || daemon->if_addrs) { - + for (tmp = daemon->if_names; tmp; tmp = tmp->next) if (tmp->name && wildcard_match(tmp->name, ifr.ifr_name)) break; - + if (!tmp && !parm.addr_match) return; } - + if (parm.relay) { /* Ignore requests sent to the ALL_SERVERS multicast address for relay when we're listening there for DHCPv6 server reasons. */ struct in6_addr all_servers; - + inet_pton(AF_INET6, ALL_SERVERS, &all_servers); - + if (!IN6_ARE_ADDR_EQUAL(&dst_addr, &all_servers)) relay_upstream6(parm.relay, sz, &from.sin6_addr, from.sin6_scope_id, now); return; } - + /* May have configured relay, but not DHCP server */ if (!daemon->doing_dhcp6) return; lease_prune(NULL, now); /* lose any expired leases */ - - port = dhcp6_reply(parm.current, if_index, ifr.ifr_name, &parm.fallback, + + port = dhcp6_reply(parm.current, if_index, ifr.ifr_name, &parm.fallback, &parm.ll_addr, &parm.ula_addr, sz, &from.sin6_addr, now); + + /* The port in the source address of the original request should + be correct, but at least once client sends from the server port, + so we explicitly send to the client port to a client, and the + server port to a relay. */ + if (port != 0) + { + from.sin6_port = htons(port); + while (retry_send(sendto(daemon->dhcp6fd, daemon->outpacket.iov_base, + save_counter(-1), 0, (struct sockaddr *)&from, + sizeof(from)))); + } + /* These need to be called _after_ we send DHCPv6 packet, since lease_update_file() + may trigger sending an RA packet, which overwrites our buffer. */ lease_update_file(now); lease_update_dns(0); } - - /* The port in the source address of the original request should - be correct, but at least once client sends from the server port, - so we explicitly send to the client port to a client, and the - server port to a relay. */ - if (port != 0) - { - from.sin6_port = htons(port); - while (retry_send(sendto(daemon->dhcp6fd, daemon->outpacket.iov_base, - save_counter(0), 0, (struct sockaddr *)&from, - sizeof(from)))); - } } void get_client_mac(struct in6_addr *client, int iface, unsigned char *mac, unsigned int *maclenp, unsigned int *mactypep, time_t now) { /* Receiving a packet from a host does not populate the neighbour - cache, so we send a neighbour discovery request if we can't + cache, so we send a neighbour discovery request if we can't find the sender. Repeat a few times in case of packet loss. */ - + struct neigh_packet neigh; union mysockaddr addr; int i, maclen; @@ -266,7 +275,7 @@ void get_client_mac(struct in6_addr *client, int iface, unsigned char *mac, unsi neigh.target = *client; /* RFC4443 section-2.3: checksum has to be zero to be calculated */ neigh.checksum = 0; - + memset(&addr, 0, sizeof(addr)); #ifdef HAVE_SOCKADDR_SA_LEN addr.in6.sin6_len = sizeof(struct sockaddr_in6); @@ -275,16 +284,16 @@ void get_client_mac(struct in6_addr *client, int iface, unsigned char *mac, unsi addr.in6.sin6_port = htons(IPPROTO_ICMPV6); addr.in6.sin6_addr = *client; addr.in6.sin6_scope_id = iface; - + for (i = 0; i < 5; i++) { struct timespec ts; - + if ((maclen = find_mac(&addr, mac, 0, now)) != 0) break; - + sendto(daemon->icmp6fd, &neigh, sizeof(neigh), 0, &addr.sa, sizeof(addr)); - + ts.tv_sec = 0; ts.tv_nsec = 100000000; /* 100ms */ nanosleep(&ts, NULL); @@ -293,117 +302,147 @@ void get_client_mac(struct in6_addr *client, int iface, unsigned char *mac, unsi *maclenp = maclen; *mactypep = ARPHRD_ETHER; } - + static int complete_context6(struct in6_addr *local, int prefix, - int scope, int if_index, int flags, unsigned int preferred, + int scope, int if_index, int flags, unsigned int preferred, unsigned int valid, void *vparam) { struct dhcp_context *context; + struct shared_network *share; struct dhcp_relay *relay; struct iface_param *param = vparam; struct iname *tmp; - + (void)scope; /* warning */ - - if (if_index == param->ind) - { - if (IN6_IS_ADDR_LINKLOCAL(local)) - param->ll_addr = *local; - else if (IN6_IS_ADDR_ULA(local)) - param->ula_addr = *local; - - if (!IN6_IS_ADDR_LOOPBACK(local) && - !IN6_IS_ADDR_LINKLOCAL(local) && - !IN6_IS_ADDR_MULTICAST(local)) - { - /* if we have --listen-address config, see if the - arrival interface has a matching address. */ - for (tmp = daemon->if_addrs; tmp; tmp = tmp->next) - if (tmp->addr.sa.sa_family == AF_INET6 && - IN6_ARE_ADDR_EQUAL(&tmp->addr.in6.sin6_addr, local)) - param->addr_match = 1; - - /* Determine a globally address on the arrival interface, even - if we have no matching dhcp-context, because we're only - allocating on remote subnets via relays. This - is used as a default for the DNS server option. */ - param->fallback = *local; - - for (context = daemon->dhcp6; context; context = context->next) - { - if ((context->flags & CONTEXT_DHCP) && - !(context->flags & (CONTEXT_TEMPLATE | CONTEXT_OLD)) && - prefix <= context->prefix && - is_same_net6(local, &context->start6, context->prefix) && - is_same_net6(local, &context->end6, context->prefix)) - { - - - /* link it onto the current chain if we've not seen it before */ - if (context->current == context) - { - struct dhcp_context *tmp, **up; - - /* use interface values only for constructed contexts */ - if (!(context->flags & CONTEXT_CONSTRUCTED)) - preferred = valid = 0xffffffff; - else if (flags & IFACE_DEPRECATED) - preferred = 0; - - if (context->flags & CONTEXT_DEPRECATE) - preferred = 0; - - /* order chain, longest preferred time first */ - for (up = ¶m->current, tmp = param->current; tmp; tmp = tmp->current) - if (tmp->preferred <= preferred) - break; - else - up = &tmp->current; - - context->current = *up; - *up = context; - context->local6 = *local; - context->preferred = preferred; - context->valid = valid; - } - } - } - } - - for (relay = daemon->relay6; relay; relay = relay->next) - if (IN6_ARE_ADDR_EQUAL(local, &relay->local.addr.addr6) && relay->current == relay && - (IN6_IS_ADDR_UNSPECIFIED(¶m->relay_local) || IN6_ARE_ADDR_EQUAL(local, ¶m->relay_local))) + + if (if_index != param->ind) + return 1; + + if (IN6_IS_ADDR_LINKLOCAL(local)) + param->ll_addr = *local; + else if (IN6_IS_ADDR_ULA(local)) + param->ula_addr = *local; + + if (IN6_IS_ADDR_LOOPBACK(local) || + IN6_IS_ADDR_LINKLOCAL(local) || + IN6_IS_ADDR_MULTICAST(local)) + return 1; + + /* if we have --listen-address config, see if the + arrival interface has a matching address. */ + for (tmp = daemon->if_addrs; tmp; tmp = tmp->next) + if (tmp->addr.sa.sa_family == AF_INET6 && + IN6_ARE_ADDR_EQUAL(&tmp->addr.in6.sin6_addr, local)) + param->addr_match = 1; + + /* Determine a globally address on the arrival interface, even + if we have no matching dhcp-context, because we're only + allocating on remote subnets via relays. This + is used as a default for the DNS server option. */ + param->fallback = *local; + + for (context = daemon->dhcp6; context; context = context->next) + if ((context->flags & CONTEXT_DHCP) && + !(context->flags & (CONTEXT_TEMPLATE | CONTEXT_OLD)) && + prefix <= context->prefix && + context->current == context) + { + if (is_same_net6(local, &context->start6, context->prefix) && + is_same_net6(local, &context->end6, context->prefix)) { - relay->current = param->relay; - param->relay = relay; - param->relay_local = *local; + struct dhcp_context *tmp, **up; + + /* use interface values only for constructed contexts */ + if (!(context->flags & CONTEXT_CONSTRUCTED)) + preferred = valid = 0xffffffff; + else if (flags & IFACE_DEPRECATED) + preferred = 0; + + if (context->flags & CONTEXT_DEPRECATE) + preferred = 0; + + /* order chain, longest preferred time first */ + for (up = ¶m->current, tmp = param->current; tmp; tmp = tmp->current) + if (tmp->preferred <= preferred) + break; + else + up = &tmp->current; + + context->current = *up; + *up = context; + context->local6 = *local; + context->preferred = preferred; + context->valid = valid; } + else + { + for (share = daemon->shared_networks; share; share = share->next) + { + /* IPv4 shared_address - ignore */ + if (share->shared_addr.s_addr != 0) + continue; + + if (share->if_index != 0) + { + if (share->if_index != if_index) + continue; + } + else + { + if (!IN6_ARE_ADDR_EQUAL(&share->match_addr6, local)) + continue; + } + + if (is_same_net6(&share->shared_addr6, &context->start6, context->prefix) && + is_same_net6(&share->shared_addr6, &context->end6, context->prefix)) + { + context->current = param->current; + param->current = context; + context->local6 = *local; + context->preferred = context->flags & CONTEXT_DEPRECATE ? 0 :0xffffffff; + context->valid = 0xffffffff; + } + } + } + } - } - - return 1; + for (relay = daemon->relay6; relay; relay = relay->next) + if (IN6_ARE_ADDR_EQUAL(local, &relay->local.addr6) && relay->current == relay && + (IN6_IS_ADDR_UNSPECIFIED(¶m->relay_local) || IN6_ARE_ADDR_EQUAL(local, ¶m->relay_local))) + { + relay->current = param->relay; + param->relay = relay; + param->relay_local = *local; + } + + return 1; } -struct dhcp_config *config_find_by_address6(struct dhcp_config *configs, struct in6_addr *net, int prefix, u64 addr) +struct dhcp_config *config_find_by_address6(struct dhcp_config *configs, struct in6_addr *net, int prefix, struct in6_addr *addr) { struct dhcp_config *config; - + for (config = configs; config; config = config->next) - if ((config->flags & CONFIG_ADDR6) && - is_same_net6(&config->addr6, net, prefix) && - (prefix == 128 || addr6part(&config->addr6) == addr)) - return config; - + if (config->flags & CONFIG_ADDR6) + { + struct addrlist *addr_list; + + for (addr_list = config->addr6; addr_list; addr_list = addr_list->next) + if ((!net || is_same_net6(&addr_list->addr.addr6, net, prefix) || ((addr_list->flags & ADDRLIST_WILDCARD) && prefix == 64)) && + is_same_net6(&addr_list->addr.addr6, addr, (addr_list->flags & ADDRLIST_PREFIX) ? addr_list->prefixlen : 128)) + return config; + } + return NULL; } struct dhcp_context *address6_allocate(struct dhcp_context *context, unsigned char *clid, int clid_len, int temp_addr, - int iaid, int serial, struct dhcp_netid *netids, int plain_range, struct in6_addr *ans) + unsigned int iaid, int serial, struct dhcp_netid *netids, int plain_range, struct in6_addr *ans) { /* Find a free address: exclude anything in use and anything allocated to a particular hwaddr/clientid/hostname in our configuration. - Try to return from contexts which match netids first. - + Try to return from contexts which match netids first. + Note that we assume the address prefix lengths are 64 or greater, so we can get by with 64 bit arithmetic. */ @@ -411,17 +450,17 @@ struct dhcp_context *address6_allocate(struct dhcp_context *context, unsigned c u64 start, addr; struct dhcp_context *c, *d; int i, pass; - u64 j; + u64 j; /* hash hwaddr: use the SDBM hashing algorithm. This works - for MAC addresses, let's see how it manages with client-ids! + for MAC addresses, let's see how it manages with client-ids! For temporary addresses, we generate a new random one each time. */ if (temp_addr) j = rand64(); else for (j = iaid, i = 0; i < clid_len; i++) j = clid[i] + (j << 6) + (j << 16) - j; - + for (pass = 0; pass <= plain_range ? 1 : 0; pass++) for (c = context; c; c = c->current) if (c->flags & (CONTEXT_DEPRECATE | CONTEXT_STATIC | CONTEXT_RA_STATELESS | CONTEXT_USED)) @@ -429,10 +468,17 @@ struct dhcp_context *address6_allocate(struct dhcp_context *context, unsigned c else if (!match_netid(c->filter, netids, pass)) continue; else - { + { if (!temp_addr && option_bool(OPT_CONSEC_ADDR)) - /* seed is largest extant lease addr in this context */ - start = lease_find_max_addr6(c) + serial; + { + /* seed is largest extant lease addr in this context, + skip addresses equal to the number of addresses rejected + by clients. This should avoid the same client being offered the same + address after it has rjected it. */ + start = lease_find_max_addr6(c) + 1 + serial + c->addr_epoch; + if (c->addr_epoch) + c->addr_epoch--; + } else { u64 range = 1 + addr6part(&c->end6) - addr6part(&c->start6); @@ -447,42 +493,41 @@ struct dhcp_context *address6_allocate(struct dhcp_context *context, unsigned c /* iterate until we find a free address. */ addr = start; - + do { /* eliminate addresses in use by the server. */ for (d = context; d; d = d->current) if (addr == addr6part(&d->local6)) break; + + *ans = c->start6; + setaddr6part (ans, addr); if (!d && - !lease6_find_by_addr(&c->start6, c->prefix, addr) && - !config_find_by_address6(daemon->dhcp_conf, &c->start6, c->prefix, addr)) - { - *ans = c->start6; - setaddr6part (ans, addr); - return c; - } - + !lease6_find_by_addr(&c->start6, c->prefix, addr) && + !config_find_by_address6(daemon->dhcp_conf, &c->start6, c->prefix, ans)) + return c; + addr++; - + if (addr == addr6part(&c->end6) + 1) addr = addr6part(&c->start6); - + } while (addr != start); } - + return NULL; } /* can dynamically allocate addr */ -struct dhcp_context *address6_available(struct dhcp_context *context, +struct dhcp_context *address6_available(struct dhcp_context *context, struct in6_addr *taddr, struct dhcp_netid *netids, int plain_range) { u64 start, end, addr = addr6part(taddr); struct dhcp_context *tmp; - + for (tmp = context; tmp; tmp = tmp->current) { start = addr6part(&tmp->start6); @@ -501,13 +546,13 @@ struct dhcp_context *address6_available(struct dhcp_context *context, } /* address OK if configured */ -struct dhcp_context *address6_valid(struct dhcp_context *context, +struct dhcp_context *address6_valid(struct dhcp_context *context, struct in6_addr *taddr, struct dhcp_netid *netids, int plain_range) { struct dhcp_context *tmp; - + for (tmp = context; tmp; tmp = tmp->current) if (is_same_net6(&tmp->start6, taddr, tmp->prefix) && match_netid(tmp->filter, netids, plain_range)) @@ -516,27 +561,6 @@ struct dhcp_context *address6_valid(struct dhcp_context *context, return NULL; } -int config_valid(struct dhcp_config *config, struct dhcp_context *context, struct in6_addr *addr) -{ - if (!config || !(config->flags & CONFIG_ADDR6)) - return 0; - - if ((config->flags & CONFIG_WILDCARD) && context->prefix == 64) - { - *addr = context->start6; - setaddr6part(addr, addr6part(&config->addr6)); - return 1; - } - - if (is_same_net6(&context->start6, &config->addr6, context->prefix)) - { - *addr = config->addr6; - return 1; - } - - return 0; -} - void make_duid(time_t now) { (void)now; @@ -544,7 +568,7 @@ void make_duid(time_t now) if (daemon->duid_config) { unsigned char *p; - + daemon->duid = p = safe_malloc(daemon->duid_config_len + 6); daemon->duid_len = daemon->duid_config_len + 6; PUTSHORT(2, p); /* DUID_EN */ @@ -554,16 +578,16 @@ void make_duid(time_t now) else { time_t newnow = 0; - + /* If we have no persistent lease database, or a non-stable RTC, use DUID_LL (newnow == 0) */ #ifndef HAVE_BROKEN_RTC /* rebase epoch to 1/1/2000 */ if (!option_bool(OPT_LEASE_RO) || daemon->lease_change_command) newnow = now - 946684800; -#endif - +#endif + iface_enumerate(AF_LOCAL, &newnow, make_duid1); - + if(!daemon->duid) die("Cannot create DHCPv6 server DUID: %s", NULL, EC_MISC); } @@ -573,14 +597,14 @@ static int make_duid1(int index, unsigned int type, char *mac, size_t maclen, vo { /* create DUID as specified in RFC3315. We use the MAC of the first interface we find that isn't loopback or P-to-P and - has address-type < 256. Address types above 256 are things like + has address-type < 256. Address types above 256 are things like tunnels which don't have usable MAC addresses. */ - + unsigned char *p; (void)index; (void)parm; time_t newnow = *((time_t *)parm); - + if (type >= 256) return 1; @@ -599,7 +623,7 @@ static int make_duid1(int index, unsigned int type, char *mac, size_t maclen, vo PUTSHORT(type, p); /* address type */ PUTLONG(*((time_t *)parm), p); /* time */ } - + memcpy(p, mac, maclen); return 0; @@ -610,14 +634,15 @@ struct cparam { int newone, newname; }; -static int construct_worker(struct in6_addr *local, int prefix, - int scope, int if_index, int flags, +static int construct_worker(struct in6_addr *local, int prefix, + int scope, int if_index, int flags, int preferred, int valid, void *vparam) { char ifrn_name[IFNAMSIZ]; struct in6_addr start6, end6; struct dhcp_context *template, *context; - + struct iname *tmp; + (void)scope; (void)flags; (void)valid; @@ -636,8 +661,14 @@ static int construct_worker(struct in6_addr *local, int prefix, if (flags & IFACE_DEPRECATED) return 1; - if (!indextoname(daemon->icmp6fd, if_index, ifrn_name)) - return 0; + /* Ignore interfaces where we're not doing RA/DHCP6 */ + if (!indextoname(daemon->icmp6fd, if_index, ifrn_name) || + !iface_check(AF_LOCAL, NULL, ifrn_name, NULL)) + return 1; + + for (tmp = daemon->dhcp_except; tmp; tmp = tmp->next) + if (tmp->name && wildcard_match(tmp->name, ifrn_name)) + return 1; for (template = daemon->dhcp6; template; template = template->next) if (!(template->flags & (CONTEXT_TEMPLATE | CONTEXT_CONSTRUCTED))) @@ -648,16 +679,16 @@ static int construct_worker(struct in6_addr *local, int prefix, is_same_net6(local, &template->end6, template->prefix)) { /* First time found, do fast RA. */ - if (template->if_index != if_index || !IN6_ARE_ADDR_EQUAL(&template->local6, local)) + if (template->if_index == 0) { ra_start_unsolicited(param->now, template); param->newone = 1; } - + template->if_index = if_index; template->local6 = *local; } - + } else if (wildcard_match(template->template_interface, ifrn_name) && template->prefix >= prefix) @@ -666,7 +697,7 @@ static int construct_worker(struct in6_addr *local, int prefix, setaddr6part(&start6, addr6part(&template->start6)); end6 = *local; setaddr6part(&end6, addr6part(&template->end6)); - + for (context = daemon->dhcp6; context; context = context->next) if (!(context->flags & CONTEXT_TEMPLATE) && IN6_ARE_ADDR_EQUAL(&start6, &context->start6) && @@ -676,7 +707,7 @@ static int construct_worker(struct in6_addr *local, int prefix, then don't construct one as well. */ if (!(context->flags & CONTEXT_CONSTRUCTED)) break; - + if (context->if_index == if_index) { int cflags = context->flags; @@ -684,19 +715,19 @@ static int construct_worker(struct in6_addr *local, int prefix, if (cflags & CONTEXT_OLD) { /* address went, now it's back, and on the same interface */ - log_context(AF_INET6, context); + log_context(AF_INET6, context); /* fast RAs for a while */ ra_start_unsolicited(param->now, context); - param->newone = 1; + param->newone = 1; /* Add address to name again */ if (context->flags & CONTEXT_RA_NAME) param->newname = 1; - + } break; } } - + if (!context && (context = whine_malloc(sizeof (struct dhcp_context)))) { *context = *template; @@ -707,28 +738,28 @@ static int construct_worker(struct in6_addr *local, int prefix, context->if_index = if_index; context->local6 = *local; context->saved_valid = 0; - + context->next = daemon->dhcp6; daemon->dhcp6 = context; ra_start_unsolicited(param->now, context); /* we created a new one, need to call lease_update_file to get periodic functions called */ - param->newone = 1; + param->newone = 1; /* Will need to add new putative SLAAC addresses to existing leases */ if (context->flags & CONTEXT_RA_NAME) param->newname = 1; - + log_context(AF_INET6, context); - } + } } - + return 1; } void dhcp_construct_contexts(time_t now) -{ +{ struct dhcp_context *context, *tmp, **up; struct cparam param; param.newone = 0; @@ -738,14 +769,14 @@ void dhcp_construct_contexts(time_t now) for (context = daemon->dhcp6; context; context = context->next) if (context->flags & CONTEXT_CONSTRUCTED) context->flags |= CONTEXT_GC; - + iface_enumerate(AF_INET6, ¶m, construct_worker); for (up = &daemon->dhcp6, context = daemon->dhcp6; context; context = tmp) { - - tmp = context->next; - + + tmp = context->next; + if (context->flags & CONTEXT_GC && !(context->flags & CONTEXT_OLD)) { if ((context->flags & CONTEXT_RA) || option_bool(OPT_RA)) @@ -760,13 +791,13 @@ void dhcp_construct_contexts(time_t now) if (context->saved_valid > 7200) /* 2 hours */ context->saved_valid = 7200; ra_start_unsolicited(now, context); - param.newone = 1; /* include deletion */ - + param.newone = 1; /* include deletion */ + if (context->flags & CONTEXT_RA_NAME) - param.newname = 1; - + param.newname = 1; + log_context(AF_INET6, context); - + up = &context->next; } else @@ -779,7 +810,7 @@ void dhcp_construct_contexts(time_t now) else up = &context->next; } - + if (param.newone) { if (daemon->dhcp || daemon->doing_dhcp6) @@ -788,7 +819,7 @@ void dhcp_construct_contexts(time_t now) lease_update_slaac(now); lease_update_file(now); } - else + else /* Not doing DHCP, so no lease system, manage alarms for ra only */ send_alarm(periodic_ra(now), now); } diff --git a/dnsmasq/dns-protocol.h b/src/dnsmasq/dns-protocol.h similarity index 98% rename from dnsmasq/dns-protocol.h rename to src/dnsmasq/dns-protocol.h index 523763c11..8edb9f3f9 100644 --- a/dnsmasq/dns-protocol.h +++ b/src/dnsmasq/dns-protocol.h @@ -1,4 +1,4 @@ -/* dnsmasq is Copyright (c) 2000-2018 Simon Kelley +/* dnsmasq is Copyright (c) 2000-2020 Simon Kelley This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by diff --git a/dnsmasq/dnsmasq.c b/src/dnsmasq/dnsmasq.c similarity index 80% rename from dnsmasq/dnsmasq.c rename to src/dnsmasq/dnsmasq.c index dca13c433..91aa8362c 100644 --- a/dnsmasq/dnsmasq.c +++ b/src/dnsmasq/dnsmasq.c @@ -1,15 +1,15 @@ -/* dnsmasq is Copyright (c) 2000-2018 Simon Kelley +/* dnsmasq is Copyright (c) 2000-2020 Simon Kelley This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; version 2 dated June, 1991, or (at your option) version 3 dated 29 June, 2007. - + This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. - + You should have received a copy of the GNU General Public License along with this program. If not, see . */ @@ -54,9 +54,14 @@ int main_dnsmasq (int argc, char **argv) #if defined(HAVE_LINUX_NETWORK) cap_user_header_t hdr = NULL; cap_user_data_t data = NULL; + int need_cap_net_admin = 0; + int need_cap_net_raw = 0; + int need_cap_net_bind_service = 0; char *bound_device = NULL; int did_bind = 0; -#endif + struct server *serv; + char *netlink_warn; +#endif #if defined(HAVE_DHCP) || defined(HAVE_DHCP6) struct dhcp_context *context; struct dhcp_relay *relay; @@ -67,7 +72,7 @@ int main_dnsmasq (int argc, char **argv) #ifdef LOCALEDIR setlocale(LC_ALL, ""); - bindtextdomain("dnsmasq", LOCALEDIR); + bindtextdomain("dnsmasq", LOCALEDIR); textdomain("dnsmasq"); #endif @@ -81,7 +86,7 @@ int main_dnsmasq (int argc, char **argv) sigaction(SIGALRM, &sigact, NULL); sigaction(SIGCHLD, &sigact, NULL); sigaction(SIGINT, &sigact, NULL); - + /* ignore SIGPIPE */ sigact.sa_handler = SIG_IGN; sigaction(SIGPIPE, &sigact, NULL); @@ -89,31 +94,35 @@ int main_dnsmasq (int argc, char **argv) umask(022); /* known umask, create leases and pid files as 0644 */ rand_init(); /* Must precede read_opts() */ - + read_opts(argc, argv, compile_opts); + +#ifdef HAVE_LINUX_NETWORK + daemon->kernel_version = kernel_version(); +#endif if (daemon->edns_pktsz < PACKETSZ) daemon->edns_pktsz = PACKETSZ; - /* Min buffer size: we check after adding each record, so there must be + /* Min buffer size: we check after adding each record, so there must be memory for the largest packet, and the largest record so the min for DNS is PACKETSZ+MAXDNAME+RRFIXEDSZ which is < 1000. - This might be increased is EDNS packet size if greater than the minimum. */ + This might be increased is EDNS packet size if greater than the minimum. */ daemon->packet_buff_sz = daemon->edns_pktsz + MAXDNAME + RRFIXEDSZ; daemon->packet = safe_malloc(daemon->packet_buff_sz); - + daemon->addrbuff = safe_malloc(ADDRSTRLEN); if (option_bool(OPT_EXTRALOG)) daemon->addrbuff2 = safe_malloc(ADDRSTRLEN); - + #ifdef HAVE_DNSSEC if (option_bool(OPT_DNSSEC_VALID)) { /* Note that both /000 and '.' are allowed within labels. These get represented in presentation format using NAME_ESCAPE as an escape - character when in DNSSEC mode. + character when in DNSSEC mode. In theory, if all the characters in a name were /000 or - '.' or NAME_ESCAPE then all would have to be escaped, so the + '.' or NAME_ESCAPE then all would have to be escaped, so the presentation format would be twice as long as the spec. daemon->namebuff was previously allocated by the option-reading @@ -124,7 +133,7 @@ int main_dnsmasq (int argc, char **argv) daemon->workspacename = safe_malloc(MAXDNAME * 2); /* one char flag per possible RR in answer section (may get extended). */ daemon->rr_status_sz = 64; - daemon->rr_status = safe_malloc(daemon->rr_status_sz); + daemon->rr_status = safe_malloc(sizeof(*daemon->rr_status) * daemon->rr_status_sz); } #endif @@ -135,21 +144,19 @@ int main_dnsmasq (int argc, char **argv) daemon->lease_file = LEASEFILE; } #endif - - /* Close any file descriptors we inherited apart from std{in|out|err} - - Ensure that at least stdin, stdout and stderr (fd 0, 1, 2) exist, - otherwise file descriptors we create can end up being 0, 1, or 2 - and then get accidentally closed later when we make 0, 1, and 2 - open to /dev/null. Normally we'll be started with 0, 1 and 2 open, - but it's not guaranteed. By opening /dev/null three times, we + + /* Ensure that at least stdin, stdout and stderr (fd 0, 1, 2) exist, + otherwise file descriptors we create can end up being 0, 1, or 2 + and then get accidentally closed later when we make 0, 1, and 2 + open to /dev/null. Normally we'll be started with 0, 1 and 2 open, + but it's not guaranteed. By opening /dev/null three times, we ensure that we're not using those fds for real stuff. */ - for (i = 0; i < max_fd; i++) - if (i != STDOUT_FILENO && i != STDERR_FILENO && i != STDIN_FILENO) - close(i); - else - open("/dev/null", O_RDWR); - + for (i = 0; i < 3; i++) + open("/dev/null", O_RDWR); + + /* Close any file descriptors we inherited apart from std{in|out|err} */ + close_fds(max_fd, -1, -1, -1); + #ifndef HAVE_LINUX_NETWORK # if !(defined(IP_RECVDSTADDR) && defined(IP_RECVIF) && defined(IP_SENDSRCADDR)) if (!option_bool(OPT_NOWILD)) @@ -158,7 +165,7 @@ int main_dnsmasq (int argc, char **argv) set_option_bool(OPT_NOWILD); } # endif - + /* -- bind-dynamic not supported on !Linux, fall back to --bind-interfaces */ if (option_bool(OPT_CLEVERBIND)) { @@ -172,7 +179,7 @@ int main_dnsmasq (int argc, char **argv) if (daemon->dynamic_dirs) die(_("dhcp-hostsdir, dhcp-optsdir and hostsdir are not supported on this platform"), NULL, EC_BADCONF); #endif - + if (option_bool(OPT_DNSSEC_VALID)) { #ifdef HAVE_DNSSEC @@ -186,10 +193,10 @@ int main_dnsmasq (int argc, char **argv) if (!ds) die(_("no root trust anchor provided for DNSSEC"), NULL, EC_BADCONF); - + if (daemon->cachesize < CACHESIZ) die(_("cannot reduce cache size from default when DNSSEC enabled"), NULL, EC_BADCONF); -#else +#else die(_("DNSSEC not available: set HAVE_DNSSEC in src/config.h"), NULL, EC_BADCONF); #endif } @@ -201,7 +208,7 @@ int main_dnsmasq (int argc, char **argv) #ifdef HAVE_CONNTRACK if (option_bool(OPT_CONNTRACK) && (daemon->query_port != 0 || daemon->osport)) - die (_("cannot use --conntrack AND --query-port"), NULL, EC_BADCONF); + die (_("cannot use --conntrack AND --query-port"), NULL, EC_BADCONF); #else if (option_bool(OPT_CONNTRACK)) die(_("conntrack support not available: set HAVE_CONNTRACK in src/config.h"), NULL, EC_BADCONF); @@ -211,7 +218,7 @@ int main_dnsmasq (int argc, char **argv) if (daemon->max_logs != 0) die(_("asynchronous logging is not available under Solaris"), NULL, EC_BADCONF); #endif - + #ifdef __ANDROID__ if (daemon->max_logs != 0) die(_("asynchronous logging is not available under Android"), NULL, EC_BADCONF); @@ -231,7 +238,7 @@ int main_dnsmasq (int argc, char **argv) if (option_bool(OPT_UBUS)) die(_("Ubus not available: set HAVE_UBUS in src/config.h"), NULL, EC_BADCONF); #endif - + if (daemon->max_port < daemon->min_port) die(_("max_port cannot be smaller than min_port"), NULL, EC_BADCONF); @@ -251,12 +258,12 @@ int main_dnsmasq (int argc, char **argv) daemon->soa_sn = now; #endif } - + #ifdef HAVE_DHCP6 if (daemon->dhcp6) { daemon->doing_ra = option_bool(OPT_RA); - + for (context = daemon->dhcp6; context; context = context->next) { if (context->flags & CONTEXT_DHCP) @@ -266,33 +273,46 @@ int main_dnsmasq (int argc, char **argv) #if !defined(HAVE_LINUX_NETWORK) && !defined(HAVE_BSD_NETWORK) if (context->flags & CONTEXT_TEMPLATE) die (_("dhcp-range constructor not available on this platform"), NULL, EC_BADCONF); -#endif +#endif } } #endif - + #ifdef HAVE_DHCP /* Note that order matters here, we must call lease_init before creating any file descriptors which shouldn't be leaked to the lease-script init process. We need to call common_init before lease_init to allocate buffers it uses. The script subsystem relies on DHCP buffers, hence the last two - conditions below. */ - if (daemon->dhcp || daemon->doing_dhcp6 || daemon->relay4 || + conditions below. */ + if (daemon->dhcp || daemon->doing_dhcp6 || daemon->relay4 || daemon->relay6 || option_bool(OPT_TFTP) || option_bool(OPT_SCRIPT_ARP)) { dhcp_common_init(); if (daemon->dhcp || daemon->doing_dhcp6) lease_init(now); } - + if (daemon->dhcp || daemon->relay4) - dhcp_init(); - + { + dhcp_init(); +# ifdef HAVE_LINUX_NETWORK + if (!option_bool(OPT_NO_PING)) + need_cap_net_raw = 1; + need_cap_net_admin = 1; +# endif + } + # ifdef HAVE_DHCP6 if (daemon->doing_ra || daemon->doing_dhcp6 || daemon->relay6) - ra_init(now); - + { + ra_init(now); +# ifdef HAVE_LINUX_NETWORK + need_cap_net_raw = 1; + need_cap_net_admin = 1; +# endif + } + if (daemon->doing_dhcp6 || daemon->relay6) dhcp6_init(); # endif @@ -301,25 +321,30 @@ int main_dnsmasq (int argc, char **argv) #ifdef HAVE_IPSET if (daemon->ipsets) - ipset_init(); + { + ipset_init(); +# ifdef HAVE_LINUX_NETWORK + need_cap_net_admin = 1; +# endif + } #endif #if defined(HAVE_LINUX_NETWORK) - netlink_init(); + netlink_warn = netlink_init(); #elif defined(HAVE_BSD_NETWORK) route_init(); #endif if (option_bool(OPT_NOWILD) && option_bool(OPT_CLEVERBIND)) die(_("cannot set --bind-interfaces and --bind-dynamic"), NULL, EC_BADCONF); - + if (!enumerate_interfaces(1) || !enumerate_interfaces(0)) die(_("failed to find list of interfaces: %s"), NULL, EC_MISC); - - if (option_bool(OPT_NOWILD) || option_bool(OPT_CLEVERBIND)) + + if (option_bool(OPT_NOWILD) || option_bool(OPT_CLEVERBIND)) { create_bound_listeners(1); - + if (!option_bool(OPT_CLEVERBIND)) for (if_tmp = daemon->if_names; if_tmp; if_tmp = if_tmp->next) if (if_tmp->name && !if_tmp->used) @@ -328,7 +353,7 @@ int main_dnsmasq (int argc, char **argv) #if defined(HAVE_LINUX_NETWORK) && defined(HAVE_DHCP) /* after enumerate_interfaces() */ bound_device = whichdevice(); - + if (daemon->dhcp) { if (!daemon->relay4 && bound_device) @@ -352,9 +377,9 @@ int main_dnsmasq (int argc, char **argv) } #endif } - else + else create_wildcard_listeners(); - + #ifdef HAVE_DHCP6 /* after enumerate_interfaces() */ if (daemon->doing_dhcp6 || daemon->relay6 || daemon->doing_ra) @@ -363,14 +388,12 @@ int main_dnsmasq (int argc, char **argv) /* After netlink_init() and before create_helper() */ lease_make_duid(now); #endif - + if (daemon->port != 0) { cache_init(); -#ifdef HAVE_DNSSEC blockdata_init(); -#endif } #ifdef HAVE_INOTIFY @@ -384,12 +407,12 @@ int main_dnsmasq (int argc, char **argv) if (daemon->dump_file) #ifdef HAVE_DUMPFILE dump_init(); - else + else daemon->dumpfd = -1; #else die(_("Packet dumps not available: set HAVE_DUMP in src/config.h"), NULL, EC_BADCONF); #endif - + if (option_bool(OPT_DBUS)) #ifdef HAVE_DBUS { @@ -403,17 +426,27 @@ int main_dnsmasq (int argc, char **argv) die(_("DBus not available: set HAVE_DBUS in src/config.h"), NULL, EC_BADCONF); #endif + if (option_bool(OPT_UBUS)) +#ifdef HAVE_UBUS + { + daemon->ubus = NULL; + ubus_init(); + } +#else + die(_("UBus not available: set HAVE_UBUS in src/dnsmasq/config.h"), NULL, EC_BADCONF); +#endif + if (daemon->port != 0) pre_allocate_sfds(); #if defined(HAVE_SCRIPT) /* Note getpwnam returns static storage */ - if ((daemon->dhcp || daemon->dhcp6) && - daemon->scriptuser && + if ((daemon->dhcp || daemon->dhcp6) && + daemon->scriptuser && (daemon->lease_change_command || daemon->luascript)) { struct passwd *scr_pw; - + if ((scr_pw = getpwnam(daemon->scriptuser))) { script_uid = scr_pw->pw_uid; @@ -423,7 +456,7 @@ int main_dnsmasq (int argc, char **argv) baduser = daemon->scriptuser; } #endif - + if (daemon->username && !(ent_pw = getpwnam(daemon->username))) baduser = daemon->username; else if (daemon->groupname && !(gp = getgrnam(daemon->groupname))) @@ -437,98 +470,149 @@ int main_dnsmasq (int argc, char **argv) { if (!(gp = getgrnam(CHGRP)) && ent_pw) gp = getgrgid(ent_pw->pw_gid); - + /* for error message */ if (gp) - daemon->groupname = gp->gr_name; + daemon->groupname = gp->gr_name; } #if defined(HAVE_LINUX_NETWORK) - /* determine capability API version here, while we can still - call safe_malloc */ - if (ent_pw && ent_pw->pw_uid != 0) + /* We keep CAP_NETADMIN (for ARP-injection) and + CAP_NET_RAW (for icmp) if we're doing dhcp, + if we have yet to bind ports because of DAD, + or we're doing it dynamically, we need CAP_NET_BIND_SERVICE. */ + if ((is_dad_listeners() || option_bool(OPT_CLEVERBIND)) && + (option_bool(OPT_TFTP) || (daemon->port != 0 && daemon->port <= 1024))) + need_cap_net_bind_service = 1; + + /* usptream servers which bind to an interface call SO_BINDTODEVICE + for each TCP connection, so need CAP_NET_RAW */ + for (serv = daemon->servers; serv; serv = serv->next) + if (serv->interface[0] != 0) + need_cap_net_raw = 1; + + /* If we're doing Dbus or UBus, the above can be set dynamically, + (as can ports) so always (potentially) needed. */ +#ifdef HAVE_DBUS + if (option_bool(OPT_DBUS)) { - int capsize = 1; /* for header version 1 */ - hdr = safe_malloc(sizeof(*hdr)); - - /* find version supported by kernel */ - memset(hdr, 0, sizeof(*hdr)); - capget(hdr, NULL); - - if (hdr->version != LINUX_CAPABILITY_VERSION_1) - { - /* if unknown version, use largest supported version (3) */ - if (hdr->version != LINUX_CAPABILITY_VERSION_2) - hdr->version = LINUX_CAPABILITY_VERSION_3; - capsize = 2; - } - - data = safe_malloc(sizeof(*data) * capsize); - memset(data, 0, sizeof(*data) * capsize); + need_cap_net_bind_service = 1; + need_cap_net_raw = 1; } #endif - /* Use a pipe to carry signals and other events back to the event loop +#ifdef HAVE_UBUS + if (option_bool(OPT_UBUS)) + { + need_cap_net_bind_service = 1; + need_cap_net_raw = 1; + } +#endif + + /* determine capability API version here, while we can still + call safe_malloc */ + int capsize = 1; /* for header version 1 */ + char *fail = NULL; + + hdr = safe_malloc(sizeof(*hdr)); + + /* find version supported by kernel */ + memset(hdr, 0, sizeof(*hdr)); + capget(hdr, NULL); + + if (hdr->version != LINUX_CAPABILITY_VERSION_1) + { + /* if unknown version, use largest supported version (3) */ + if (hdr->version != LINUX_CAPABILITY_VERSION_2) + hdr->version = LINUX_CAPABILITY_VERSION_3; + capsize = 2; + } + + data = safe_malloc(sizeof(*data) * capsize); + capget(hdr, data); /* Get current values, for verification */ + + if (need_cap_net_admin && !(data->permitted & (1 << CAP_NET_ADMIN))) + fail = "NET_ADMIN"; + else if (need_cap_net_raw && !(data->permitted & (1 << CAP_NET_RAW))) + fail = "NET_RAW"; + else if (need_cap_net_bind_service && !(data->permitted & (1 << CAP_NET_BIND_SERVICE))) + fail = "NET_BIND_SERVICE"; + + if (fail) + die(_("process is missing required capability %s"), fail, EC_MISC); + + /* Now set bitmaps to set caps after daemonising */ + memset(data, 0, sizeof(*data) * capsize); + + if (need_cap_net_admin) + data->effective |= (1 << CAP_NET_ADMIN); + if (need_cap_net_raw) + data->effective |= (1 << CAP_NET_RAW); + if (need_cap_net_bind_service) + data->effective |= (1 << CAP_NET_BIND_SERVICE); + + data->permitted = data->effective; +#endif + + /* Use a pipe to carry signals and other events back to the event loop in a race-free manner and another to carry errors to daemon-invoking process */ safe_pipe(pipefd, 1); - + piperead = pipefd[0]; pipewrite = pipefd[1]; /* prime the pipe to load stuff first time. */ - send_event(pipewrite, EVENT_INIT, 0, NULL); + send_event(pipewrite, EVENT_INIT, 0, NULL); err_pipe[1] = -1; - - if (!option_bool(OPT_DEBUG)) + + if (!option_bool(OPT_DEBUG)) { - /* The following code "daemonizes" the process. + /* The following code "daemonizes" the process. See Stevens section 12.4 */ - + if (chdir("/") != 0) - die(_("cannot chdir to filesystem root: %s"), NULL, EC_MISC); + die(_("cannot chdir to filesystem root: %s"), NULL, EC_MISC); -#ifndef NO_FORK if (!option_bool(OPT_NO_FORK)) { pid_t pid; - + /* pipe to carry errors back to original process. When startup is complete we close this and the process terminates. */ safe_pipe(err_pipe, 0); - + if ((pid = fork()) == -1) /* fd == -1 since we've not forked, never returns. */ send_event(-1, EVENT_FORK_ERR, errno, NULL); - + if (pid != 0) { struct event_desc ev; char *msg; /* close our copy of write-end */ - while (retry_send(close(err_pipe[1]))); - + close(err_pipe[1]); + /* check for errors after the fork */ if (read_event(err_pipe[0], &ev, &msg)) fatal_event(&ev, msg); - + _exit(EC_GOOD); - } - - while (retry_send(close(err_pipe[0]))); + } + + close(err_pipe[0]); /* NO calls to die() from here on. */ - + setsid(); - + if ((pid = fork()) == -1) send_event(err_pipe[1], EVENT_FORK_ERR, errno, NULL); - + if (pid != 0) _exit(0); } -#endif - + /* write pidfile _after_ forking ! */ if (daemon->runfile) { @@ -538,25 +622,25 @@ int main_dnsmasq (int argc, char **argv) /* Explanation: Some installations of dnsmasq (eg Debian/Ubuntu) locate the pid-file in a directory which is writable by the non-privileged user that dnsmasq runs as. This - allows the daemon to delete the file as part of its shutdown. This is a security hole to the - extent that an attacker running as the unprivileged user could replace the pidfile with a - symlink, and have the target of that symlink overwritten as root next time dnsmasq starts. + allows the daemon to delete the file as part of its shutdown. This is a security hole to the + extent that an attacker running as the unprivileged user could replace the pidfile with a + symlink, and have the target of that symlink overwritten as root next time dnsmasq starts. The following code first deletes any existing file, and then opens it with the O_EXCL flag, - ensuring that the open() fails should there be any existing file (because the unlink() failed, + ensuring that the open() fails should there be any existing file (because the unlink() failed, or an attacker exploited the race between unlink() and open()). This ensures that no symlink - attack can succeed. + attack can succeed. Any compromise of the non-privileged user still theoretically allows the pid-file to be - replaced whilst dnsmasq is running. The worst that could allow is that the usual + replaced whilst dnsmasq is running. The worst that could allow is that the usual "shutdown dnsmasq" shell command could be tricked into stopping any other process. - Note that if dnsmasq is started as non-root (eg for testing) it silently ignores + Note that if dnsmasq is started as non-root (eg for testing) it silently ignores failure to write the pid-file. */ - unlink(daemon->runfile); - + unlink(daemon->runfile); + if ((fd = open(daemon->runfile, O_WRONLY|O_CREAT|O_TRUNC|O_EXCL, S_IWUSR|S_IRUSR|S_IRGRP|S_IROTH)) == -1) { /* only complain if started as root */ @@ -567,7 +651,7 @@ int main_dnsmasq (int argc, char **argv) { /* We're still running as root here. Change the ownership of the PID file to the user we will be running as. Note that this is not to allow - us to delete the file, since that depends on the permissions + us to delete the file, since that depends on the permissions of the directory containing the file. That directory will need to by owned by the dnsmasq user, and the ownership of the file has to match, to keep systemd >273 happy. */ @@ -578,8 +662,7 @@ int main_dnsmasq (int argc, char **argv) err = 1; else { - while (retry_send(close(fd))); - if (errno != 0) + if (close(fd) == -1) err = 1; } } @@ -593,11 +676,11 @@ int main_dnsmasq (int argc, char **argv) } FTL_fork_and_bind_sockets(ent_pw); - + log_err = log_start(ent_pw, err_pipe[1]); - if (!option_bool(OPT_DEBUG)) - { + if (!option_bool(OPT_DEBUG)) + { /* open stdout etc to /dev/null */ int nullfd = open("/dev/null", O_RDWR); if (nullfd != -1) @@ -608,52 +691,43 @@ int main_dnsmasq (int argc, char **argv) close(nullfd); } } - + /* if we are to run scripts, we need to fork a helper before dropping root. */ daemon->helperfd = -1; -#ifdef HAVE_SCRIPT - if ((daemon->dhcp || daemon->dhcp6 || option_bool(OPT_TFTP) || option_bool(OPT_SCRIPT_ARP)) && +#ifdef HAVE_SCRIPT + if ((daemon->dhcp || daemon->dhcp6 || option_bool(OPT_TFTP) || option_bool(OPT_SCRIPT_ARP)) && (daemon->lease_change_command || daemon->luascript)) daemon->helperfd = create_helper(pipewrite, err_pipe[1], script_uid, script_gid, max_fd); #endif - if (!option_bool(OPT_DEBUG) && getuid() == 0) + if (!option_bool(OPT_DEBUG) && getuid() == 0) { int bad_capabilities = 0; gid_t dummy; - + /* remove all supplementary groups */ - if (gp && + if (gp && (setgroups(0, &dummy) == -1 || setgid(gp->gr_gid) == -1)) { send_event(err_pipe[1], EVENT_GROUP_ERR, errno, daemon->groupname); _exit(0); } - + if (ent_pw && ent_pw->pw_uid != 0) - { -#if defined(HAVE_LINUX_NETWORK) - /* On linux, we keep CAP_NETADMIN (for ARP-injection) and - CAP_NET_RAW (for icmp) if we're doing dhcp. If we have yet to bind - ports because of DAD, or we're doing it dynamically, - we need CAP_NET_BIND_SERVICE too. */ - if (is_dad_listeners() || option_bool(OPT_CLEVERBIND)) - data->effective = data->permitted = data->inheritable = - (1 << CAP_NET_ADMIN) | (1 << CAP_NET_RAW) | - (1 << CAP_SETUID) | (1 << CAP_NET_BIND_SERVICE); - else - data->effective = data->permitted = data->inheritable = - (1 << CAP_NET_ADMIN) | (1 << CAP_NET_RAW) | (1 << CAP_SETUID); - + { +#if defined(HAVE_LINUX_NETWORK) + /* Need to be able to drop root. */ + data->effective |= (1 << CAP_SETUID); + data->permitted |= (1 << CAP_SETUID); /* Tell kernel to not clear capabilities when dropping root */ if (capset(hdr, data) == -1 || prctl(PR_SET_KEEPCAPS, 1, 0, 0, 0) == -1) bad_capabilities = errno; - + #elif defined(HAVE_SOLARIS_NETWORK) /* http://developers.sun.com/solaris/articles/program_privileges.html */ priv_set_t *priv_set; - + if (!(priv_set = priv_str_to_set("basic", ",", NULL)) || priv_addset(priv_set, PRIV_NET_ICMPACCESS) == -1 || priv_addset(priv_set, PRIV_SYS_NET_CONFIG) == -1) @@ -662,7 +736,7 @@ int main_dnsmasq (int argc, char **argv) if (priv_set && bad_capabilities == 0) { priv_inverse(priv_set); - + if (setppriv(PRIV_OFF, PRIV_LIMIT, priv_set) == -1) bad_capabilities = errno; } @@ -670,45 +744,40 @@ int main_dnsmasq (int argc, char **argv) if (priv_set) priv_freeset(priv_set); -#endif +#endif if (bad_capabilities != 0) { send_event(err_pipe[1], EVENT_CAP_ERR, bad_capabilities, NULL); _exit(0); } - + /* finally drop root */ if (setuid(ent_pw->pw_uid) == -1) { send_event(err_pipe[1], EVENT_USER_ERR, errno, daemon->username); _exit(0); - } + } #ifdef HAVE_LINUX_NETWORK - if (is_dad_listeners() || option_bool(OPT_CLEVERBIND)) - data->effective = data->permitted = - (1 << CAP_NET_ADMIN) | (1 << CAP_NET_RAW) | (1 << CAP_NET_BIND_SERVICE); - else - data->effective = data->permitted = - (1 << CAP_NET_ADMIN) | (1 << CAP_NET_RAW); - data->inheritable = 0; - - /* lose the setuid and setgid capabilities */ + data->effective &= ~(1 << CAP_SETUID); + data->permitted &= ~(1 << CAP_SETUID); + + /* lose the setuid capability */ if (capset(hdr, data) == -1) { send_event(err_pipe[1], EVENT_CAP_ERR, errno, NULL); _exit(0); } #endif - + } } - + #ifdef HAVE_LINUX_NETWORK free(hdr); free(data); - if (option_bool(OPT_DEBUG)) + if (option_bool(OPT_DEBUG)) prctl(PR_SET_DUMPABLE, 1, 0, 0, 0); #endif @@ -717,7 +786,7 @@ int main_dnsmasq (int argc, char **argv) { DIR *dir; struct tftp_prefix *p; - + if (daemon->tftp_prefix) { if (!((dir = opendir(daemon->tftp_prefix)))) @@ -753,7 +822,7 @@ int main_dnsmasq (int argc, char **argv) if (daemon->port == 0) my_syslog(LOG_INFO, _("started, version %s DNS disabled"), VERSION); - else + else { if (daemon->cachesize != 0) { @@ -767,12 +836,12 @@ int main_dnsmasq (int argc, char **argv) if (option_bool(OPT_LOCAL_SERVICE)) my_syslog(LOG_INFO, _("DNS service limited to local subnets")); } - + my_syslog(LOG_INFO, _("compile time options: %s"), compile_opts); if (chown_warn != 0) my_syslog(LOG_WARNING, "chown of PID file %s failed: %s", daemon->runfile, strerror(chown_warn)); - + #ifdef HAVE_DBUS if (option_bool(OPT_DBUS)) { @@ -783,30 +852,40 @@ int main_dnsmasq (int argc, char **argv) } #endif +#ifdef HAVE_UBUS + if (option_bool(OPT_UBUS)) + { + if (daemon->ubus) + my_syslog(LOG_INFO, _("UBus support enabled: connected to system bus")); + else + my_syslog(LOG_INFO, _("UBus support enabled: bus connection pending")); + } +#endif + #ifdef HAVE_DNSSEC if (option_bool(OPT_DNSSEC_VALID)) { int rc; struct ds_config *ds; - + /* Delay creating the timestamp file until here, after we've changed user, so that - it has the correct owner to allow updating the mtime later. + it has the correct owner to allow updating the mtime later. This means we have to report fatal errors via the pipe. */ if ((rc = setup_timestamp()) == -1) { send_event(err_pipe[1], EVENT_TIME_ERR, errno, daemon->timestamp_file); _exit(0); } - + if (option_bool(OPT_DNSSEC_IGN_NS)) my_syslog(LOG_INFO, _("DNSSEC validation enabled but all unsigned answers are trusted")); else my_syslog(LOG_INFO, _("DNSSEC validation enabled")); - + daemon->dnssec_no_time_check = option_bool(OPT_DNSSEC_TIME); if (option_bool(OPT_DNSSEC_TIME) && !daemon->back_to_the_future) my_syslog(LOG_INFO, _("DNSSEC signature timestamps not checked until receipt of SIGINT")); - + if (rc == 1) my_syslog(LOG_INFO, _("DNSSEC signature timestamps not checked until system time valid")); @@ -817,9 +896,9 @@ int main_dnsmasq (int argc, char **argv) #endif if (log_err != 0) - my_syslog(LOG_WARNING, _("warning: failed to change owner of %s: %s"), + my_syslog(LOG_WARNING, _("warning: failed to change owner of %s: %s"), daemon->log_file, strerror(log_err)); - + if (bind_fallback) my_syslog(LOG_WARNING, _("setting --bind-interfaces option because of OS limitations")); @@ -829,12 +908,12 @@ int main_dnsmasq (int argc, char **argv) warn_wild_labels(); warn_int_names(); - - if (!option_bool(OPT_NOWILD)) + + if (!option_bool(OPT_NOWILD)) for (if_tmp = daemon->if_names; if_tmp; if_tmp = if_tmp->next) if (if_tmp->name && !if_tmp->used) my_syslog(LOG_WARNING, _("warning: interface %s does not currently exist"), if_tmp->name); - + if (daemon->port != 0 && option_bool(OPT_NO_RESOLV)) { if (daemon->resolv_files && !daemon->resolv_files->is_default) @@ -842,11 +921,11 @@ int main_dnsmasq (int argc, char **argv) daemon->resolv_files = NULL; if (!daemon->servers) my_syslog(LOG_WARNING, _("warning: no upstream servers configured")); - } + } if (daemon->max_logs != 0) my_syslog(LOG_INFO, _("asynchronous logging enabled, queue limit is %d messages"), daemon->max_logs); - + #ifdef HAVE_DHCP for (context = daemon->dhcp; context; context = context->next) @@ -861,10 +940,10 @@ int main_dnsmasq (int argc, char **argv) for (relay = daemon->relay6; relay; relay = relay->next) log_relay(AF_INET6, relay); - + if (daemon->doing_dhcp6 || daemon->doing_ra) dhcp_construct_contexts(now); - + if (option_bool(OPT_RA)) my_syslog(MS_DHCP | LOG_INFO, _("IPv6 router advertisement enabled")); # endif @@ -872,6 +951,9 @@ int main_dnsmasq (int argc, char **argv) # ifdef HAVE_LINUX_NETWORK if (did_bind) my_syslog(MS_DHCP | LOG_INFO, _("DHCP, sockets bound exclusively to interface %s"), bound_device); + + if (netlink_warn) + my_syslog(LOG_WARNING, netlink_warn); # endif /* after dhcp_construct_contexts */ @@ -884,10 +966,11 @@ int main_dnsmasq (int argc, char **argv) { struct tftp_prefix *p; - my_syslog(MS_TFTP | LOG_INFO, "TFTP %s%s %s", + my_syslog(MS_TFTP | LOG_INFO, "TFTP %s%s %s %s", daemon->tftp_prefix ? _("root is ") : _("enabled"), - daemon->tftp_prefix ? daemon->tftp_prefix: "", - option_bool(OPT_TFTP_SECURE) ? _("secure mode") : ""); + daemon->tftp_prefix ? daemon->tftp_prefix : "", + option_bool(OPT_TFTP_SECURE) ? _("secure mode") : "", + option_bool(OPT_SINGLE_PORT) ? _("single port mode") : ""); if (tftp_prefix_missing) my_syslog(MS_TFTP | LOG_WARNING, _("warning: %s inaccessible"), daemon->tftp_prefix); @@ -896,21 +979,21 @@ int main_dnsmasq (int argc, char **argv) if (p->missing) my_syslog(MS_TFTP | LOG_WARNING, _("warning: TFTP directory %s inaccessible"), p->prefix); - /* This is a guess, it assumes that for small limits, - disjoint files might be served, but for large limits, + /* This is a guess, it assumes that for small limits, + disjoint files might be served, but for large limits, a single file will be sent to may clients (the file only needs one fd). */ max_fd -= 30; /* use other than TFTP */ - + if (max_fd < 0) max_fd = 5; - else if (max_fd < 100) + else if (max_fd < 100 && !option_bool(OPT_SINGLE_PORT)) max_fd = max_fd/2; else max_fd = max_fd - 20; - - /* if we have to use a limited range of ports, + + /* if we have to use a limited range of ports, that will limit the number of transfers */ if (daemon->start_tftp_port != 0 && daemon->end_tftp_port - daemon->start_tftp_port + 1 < max_fd) @@ -919,8 +1002,8 @@ int main_dnsmasq (int argc, char **argv) if (daemon->tftp_max > max_fd) { daemon->tftp_max = max_fd; - my_syslog(MS_TFTP | LOG_WARNING, - _("restricting maximum simultaneous TFTP transfers to %d"), + my_syslog(MS_TFTP | LOG_WARNING, + _("restricting maximum simultaneous TFTP transfers to %d"), daemon->tftp_max); } } @@ -928,24 +1011,28 @@ int main_dnsmasq (int argc, char **argv) /* finished start-up - release original process */ if (err_pipe[1] != -1) - while (retry_send(close(err_pipe[1]))); - + close(err_pipe[1]); + if (daemon->port != 0) check_servers(); - + pid = getpid(); + daemon->pipe_to_parent = -1; + for (i = 0; i < MAX_PROCS; i++) + daemon->tcp_pipes[i] = -1; + #ifdef HAVE_INOTIFY /* Using inotify, have to select a resolv file at startup */ poll_resolv(1, 0, now); #endif - + while (!terminate) { int t, timeout = -1; - + poll_reset(); - + /* if we are out of resources, find how long we have to wait for some to come free, we'll loop around then and restart listening for queries */ @@ -967,9 +1054,9 @@ int main_dnsmasq (int argc, char **argv) #ifdef HAVE_UBUS if (option_bool(OPT_UBUS)) - set_ubus_listeners(); + set_ubus_listeners(); #endif - + #ifdef HAVE_DHCP if (daemon->dhcp || daemon->relay4) { @@ -982,11 +1069,11 @@ int main_dnsmasq (int argc, char **argv) #ifdef HAVE_DHCP6 if (daemon->doing_dhcp6 || daemon->relay6) poll_listen(daemon->dhcp6fd, POLLIN); - + if (daemon->doing_ra) - poll_listen(daemon->icmp6fd, POLLIN); + poll_listen(daemon->icmp6fd, POLLIN); #endif - + #ifdef HAVE_INOTIFY if (daemon->inotifyfd != -1) poll_listen(daemon->inotifyfd, POLLIN); @@ -997,12 +1084,12 @@ int main_dnsmasq (int argc, char **argv) #elif defined(HAVE_BSD_NETWORK) poll_listen(daemon->routefd, POLLIN); #endif - + poll_listen(piperead, POLLIN); #ifdef HAVE_SCRIPT # ifdef HAVE_DHCP - while (helper_buf_empty() && do_script_run(now)); + while (helper_buf_empty() && do_script_run(now)); # endif /* Refresh cache */ @@ -1024,20 +1111,20 @@ int main_dnsmasq (int argc, char **argv) while (do_arp_script_run()); -# ifdef HAVE_TFTP +# ifdef HAVE_TFTP while (do_tftp_script_run()); # endif #endif - - /* must do this just before select(), when we know no + + /* must do this just before do_poll(), when we know no more calls to my_syslog() can occur */ set_log_writer(); - + if (do_poll(timeout) < 0) continue; - + now = dnsmasq_time(); check_log_writer(0); @@ -1068,27 +1155,27 @@ int main_dnsmasq (int argc, char **argv) { if (daemon->port != 0 && !option_bool(OPT_NO_POLL)) poll_resolv(1, 1, now); - } + } #else /* Check for changes to resolv files once per second max. */ /* Don't go silent for long periods if the clock goes backwards. */ - if (daemon->last_resolv == 0 || - difftime(now, daemon->last_resolv) > 1.0 || + if (daemon->last_resolv == 0 || + difftime(now, daemon->last_resolv) > 1.0 || difftime(now, daemon->last_resolv) < -1.0) { - /* poll_resolv doesn't need to reload first time through, since + /* poll_resolv doesn't need to reload first time through, since that's queued anyway. */ - poll_resolv(0, daemon->last_resolv != 0, now); + poll_resolv(0, daemon->last_resolv != 0, now); daemon->last_resolv = now; } #endif if (poll_check(piperead, POLLIN)) async_event(piperead, now); - + #ifdef HAVE_DBUS - /* if we didn't create a DBus connection, retry now. */ + /* if we didn't create a DBus connection, retry now. */ if (option_bool(OPT_DBUS) && !daemon->dbus) { char *err; @@ -1102,14 +1189,22 @@ int main_dnsmasq (int argc, char **argv) #ifdef HAVE_UBUS if (option_bool(OPT_UBUS)) - check_ubus_listeners(); + { + /* if we didn't create a UBus connection, retry now. */ + if (!daemon->ubus) + { + ubus_init(); + } + + check_ubus_listeners(); + } #endif check_dns_listeners(now); #ifdef HAVE_TFTP check_tftp_listeners(now); -#endif +#endif #ifdef HAVE_DHCP if (daemon->dhcp || daemon->relay4) @@ -1151,13 +1246,19 @@ static void sig_handler(int sig) { /* alarm is used to kill TCP children after a fixed time. */ if (sig == SIGALRM) - _exit(0); + { + /*** Pi-hole modification ***/ + // TCP workers ignore all signals except SIGALRM + FTL_TCP_worker_terminating(); + /*** Pi-hole modification ***/ + _exit(0); + } } else { /* master process */ int event, errsave = errno; - + if (sig == SIGHUP) event = EVENT_RELOAD; else if (sig == SIGCHLD) @@ -1182,7 +1283,7 @@ static void sig_handler(int sig) else return; - send_event(pipewrite, event, 0, NULL); + send_event(pipewrite, event, 0, NULL); errno = errsave; } } @@ -1195,8 +1296,8 @@ void send_alarm(time_t event, time_t now) /* alarm(0) or alarm(-ve) doesn't do what we want.... */ if ((now == 0 || difftime(event, now) <= 0.0)) send_event(pipewrite, EVENT_ALARM, 0, NULL); - else - alarm((unsigned)difftime(event, now)); + else + alarm((unsigned)difftime(event, now)); } } @@ -1213,12 +1314,12 @@ void send_event(int fd, int event, int data, char *msg) ev.event = event; ev.data = data; ev.msg_sz = msg ? strlen(msg) : 0; - + iov[0].iov_base = &ev; iov[0].iov_len = sizeof(ev); iov[1].iov_base = msg; iov[1].iov_len = ev.msg_sz; - + /* error pipe, debug mode. */ if (fd == -1) fatal_event(&ev, msg); @@ -1236,10 +1337,10 @@ static int read_event(int fd, struct event_desc *evp, char **msg) if (!read_write(fd, (unsigned char *)evp, sizeof(struct event_desc), 1)) return 0; - + *msg = NULL; - - if (evp->msg_sz != 0 && + + if (evp->msg_sz != 0 && (buf = malloc(evp->msg_sz + 1)) && read_write(fd, (unsigned char *)buf, evp->msg_sz, 1)) { @@ -1249,11 +1350,11 @@ static int read_event(int fd, struct event_desc *evp, char **msg) return 1; } - + static void fatal_event(struct event_desc *ev, char *msg) { errno = ev->data; - + switch (ev->event) { case EVENT_DIE: @@ -1298,29 +1399,29 @@ static void fatal_event(struct event_desc *ev, char *msg) case EVENT_TIME_ERR: die(_("cannot create timestamp file %s: %s" ), msg, EC_BADCONF); } -} - +} + static void async_event(int pipe, time_t now) { pid_t p; struct event_desc ev; int i, check = 0; char *msg; - + /* NOTE: the memory used to return msg is leaked: use msgs in events only to describe fatal errors. */ - + if (read_event(pipe, &ev, &msg)) switch (ev.event) { case EVENT_RELOAD: daemon->soa_sn++; /* Bump zone serial, as it may have changed. */ - + /* fall through */ - + case EVENT_INIT: clear_cache_and_reload(now); - + if (daemon->port != 0) { if (daemon->resolv_files && option_bool(OPT_NO_POLL)) @@ -1343,12 +1444,12 @@ static void async_event(int pipe, time_t now) rerun_scripts(); #endif break; - + case EVENT_DUMP: if (daemon->port != 0) dump_cache(now); break; - + case EVENT_ALARM: #ifdef HAVE_DHCP if (daemon->dhcp || daemon->doing_dhcp6) @@ -1363,7 +1464,7 @@ static void async_event(int pipe, time_t now) #endif #endif break; - + case EVENT_CHILD: /* See Stevens 5.10 */ while ((p = waitpid(-1, NULL, WNOHANG)) != 0) @@ -1371,14 +1472,14 @@ static void async_event(int pipe, time_t now) { if (errno != EINTR) break; - } - else + } + else for (i = 0 ; i < MAX_PROCS; i++) if (daemon->tcp_pids[i] == p) daemon->tcp_pids[i] = 0; break; - -#if defined(HAVE_SCRIPT) + +#if defined(HAVE_SCRIPT) case EVENT_KILLED: my_syslog(LOG_WARNING, _("script process killed by signal %d"), ev.data); break; @@ -1388,7 +1489,7 @@ static void async_event(int pipe, time_t now) break; case EVENT_EXEC_ERR: - my_syslog(LOG_ERR, _("failed to execute %s: %s"), + my_syslog(LOG_ERR, _("failed to execute %s: %s"), daemon->lease_change_command, strerror(ev.data)); break; @@ -1412,6 +1513,7 @@ static void async_event(int pipe, time_t now) we leave them logging to the old file. */ if (daemon->log_file != NULL) log_reopen(daemon->log_file); + break; case EVENT_NEWADDR: @@ -1434,27 +1536,27 @@ static void async_event(int pipe, time_t now) } #endif break; - + case EVENT_TERM: /* Knock all our children on the head. */ for (i = 0; i < MAX_PROCS; i++) if (daemon->tcp_pids[i] != 0) kill(daemon->tcp_pids[i], SIGALRM); - + #if defined(HAVE_SCRIPT) && defined(HAVE_DHCP) /* handle pending lease transitions */ if (daemon->helperfd != -1) { /* block in writes until all done */ if ((i = fcntl(daemon->helperfd, F_GETFL)) != -1) - fcntl(daemon->helperfd, F_SETFL, i & ~O_NONBLOCK); + fcntl(daemon->helperfd, F_SETFL, i & ~O_NONBLOCK); do { helper_write(); } while (!helper_buf_empty() || do_script_run(now)); - while (retry_send(close(daemon->helperfd))); + close(daemon->helperfd); } #endif - + if (daemon->lease_stream) fclose(daemon->lease_stream); @@ -1474,11 +1576,13 @@ static void async_event(int pipe, time_t now) if (daemon->dumpfd != -1) close(daemon->dumpfd); #endif - + my_syslog(LOG_INFO, _("exiting on receipt of SIGTERM")); flush_log(); + /*** Pi-hole modification ***/ // exit(EC_GOOD); terminate = 1; + /*** Pi-hole modification ***/ } } @@ -1487,28 +1591,28 @@ static void poll_resolv(int force, int do_reload, time_t now) struct resolvc *res, *latest; struct stat statbuf; time_t last_change = 0; - /* There may be more than one possible file. + /* There may be more than one possible file. Go through and find the one which changed _last_. Warn of any which can't be read. */ if (daemon->port == 0 || option_bool(OPT_NO_POLL)) return; - + for (latest = NULL, res = daemon->resolv_files; res; res = res->next) if (stat(res->name, &statbuf) == -1) { if (force) { - res->mtime = 0; + res->mtime = 0; continue; } if (!res->logged) my_syslog(LOG_WARNING, _("failed to access %s: %s"), res->name, strerror(errno)); res->logged = 1; - + if (res->mtime != 0) - { + { /* existing file evaporated, force selection of the latest file even if its mtime hasn't changed since we last looked */ poll_resolv(1, do_reload, now); @@ -1528,7 +1632,7 @@ static void poll_resolv(int force, int do_reload, time_t now) } } } - + if (latest) { static int warned = 0; @@ -1540,7 +1644,7 @@ static void poll_resolv(int force, int do_reload, time_t now) if (option_bool(OPT_RELOAD) && do_reload) clear_cache_and_reload(now); } - else + else { latest->mtime = 0; if (!warned) @@ -1550,7 +1654,7 @@ static void poll_resolv(int force, int do_reload, time_t now) } } } -} +} void clear_cache_and_reload(time_t now) { @@ -1560,7 +1664,7 @@ void clear_cache_and_reload(time_t now) if (daemon->port != 0) cache_reload(); - + #ifdef HAVE_DHCP if (daemon->dhcp || daemon->doing_dhcp6) { @@ -1568,13 +1672,13 @@ void clear_cache_and_reload(time_t now) dhcp_read_ethers(); reread_dhcp(); dhcp_update_configs(daemon->dhcp_conf); - lease_update_from_configs(); - lease_update_file(now); + lease_update_from_configs(); + lease_update_file(now); lease_update_dns(1); } #ifdef HAVE_DHCP6 else if (daemon->doing_ra) - /* Not doing DHCP, so no lease system, manage + /* Not doing DHCP, so no lease system, manage alarms for ra only */ send_alarm(periodic_ra(now), now); #endif @@ -1586,52 +1690,59 @@ static int set_dns_listeners(time_t now) struct serverfd *serverfdp; struct listener *listener; int wait = 0, i; - + #ifdef HAVE_TFTP int tftp = 0; struct tftp_transfer *transfer; - for (transfer = daemon->tftp_trans; transfer; transfer = transfer->next) - { - tftp++; - poll_listen(transfer->sockfd, POLLIN); - } + if (!option_bool(OPT_SINGLE_PORT)) + for (transfer = daemon->tftp_trans; transfer; transfer = transfer->next) + { + tftp++; + poll_listen(transfer->sockfd, POLLIN); + } #endif - + /* will we be able to get memory? */ if (daemon->port != 0) - get_new_frec(now, &wait, 0); - + get_new_frec(now, &wait, NULL); + for (serverfdp = daemon->sfds; serverfdp; serverfdp = serverfdp->next) poll_listen(serverfdp->fd, POLLIN); - + if (daemon->port != 0 && !daemon->osport) for (i = 0; i < RANDOM_SOCKS; i++) if (daemon->randomsocks[i].refcount != 0) poll_listen(daemon->randomsocks[i].fd, POLLIN); - + for (listener = daemon->listeners; listener; listener = listener->next) { /* only listen for queries if we have resources */ if (listener->fd != -1 && wait == 0) poll_listen(listener->fd, POLLIN); - + /* death of a child goes through the select loop, so we don't need to explicitly arrange to wake up here */ if (listener->tcpfd != -1) for (i = 0; i < MAX_PROCS; i++) - if (daemon->tcp_pids[i] == 0) + if (daemon->tcp_pids[i] == 0 && daemon->tcp_pipes[i] == -1) { poll_listen(listener->tcpfd, POLLIN); break; } #ifdef HAVE_TFTP + /* tftp == 0 in single-port mode. */ if (tftp <= daemon->tftp_max && listener->tftpfd != -1) poll_listen(listener->tftpfd, POLLIN); #endif } - + + if (!option_bool(OPT_DEBUG)) + for (i = 0; i < MAX_PROCS; i++) + if (daemon->tcp_pipes[i] != -1) + poll_listen(daemon->tcp_pipes[i], POLLIN); + return wait; } @@ -1640,23 +1751,41 @@ static void check_dns_listeners(time_t now) struct serverfd *serverfdp; struct listener *listener; int i; - + int pipefd[2]; + for (serverfdp = daemon->sfds; serverfdp; serverfdp = serverfdp->next) if (poll_check(serverfdp->fd, POLLIN)) reply_query(serverfdp->fd, serverfdp->source_addr.sa.sa_family, now); - + if (daemon->port != 0 && !daemon->osport) for (i = 0; i < RANDOM_SOCKS; i++) - if (daemon->randomsocks[i].refcount != 0 && + if (daemon->randomsocks[i].refcount != 0 && poll_check(daemon->randomsocks[i].fd, POLLIN)) reply_query(daemon->randomsocks[i].fd, daemon->randomsocks[i].family, now); + /* Races. The child process can die before we read all of the data from the + pipe, or vice versa. Therefore send tcp_pids to zero when we wait() the + process, and tcp_pipes to -1 and close the FD when we read the last + of the data - indicated by cache_recv_insert returning zero. + The order of these events is indeterminate, and both are needed + to free the process slot. Once the child process has gone, poll() + returns POLLHUP, not POLLIN, so have to check for both here. */ + if (!option_bool(OPT_DEBUG)) + for (i = 0; i < MAX_PROCS; i++) + if (daemon->tcp_pipes[i] != -1 && + poll_check(daemon->tcp_pipes[i], POLLIN | POLLHUP) && + !cache_recv_insert(now, daemon->tcp_pipes[i])) + { + close(daemon->tcp_pipes[i]); + daemon->tcp_pipes[i] = -1; + } + for (listener = daemon->listeners; listener; listener = listener->next) { if (listener->fd != -1 && poll_check(listener->fd, POLLIN)) - receive_query(listener, now); - -#ifdef HAVE_TFTP + receive_query(listener, now); + +#ifdef HAVE_TFTP if (listener->tftpfd != -1 && poll_check(listener->tftpfd, POLLIN)) tftp_request(listener, now); #endif @@ -1670,18 +1799,18 @@ static void check_dns_listeners(time_t now) socklen_t tcp_len = sizeof(union mysockaddr); while ((confd = accept(listener->tcpfd, NULL, NULL)) == -1 && errno == EINTR); - + if (confd == -1) continue; - + if (getsockname(confd, (struct sockaddr *)&tcp_addr, &tcp_len) == -1) { - while (retry_send(close(confd))); + close(confd); continue; } - + /* Make sure that the interface list is up-to-date. - + We do this here as we may need the results below, and the DNS code needs them for --interface-name stuff. @@ -1690,86 +1819,108 @@ static void check_dns_listeners(time_t now) have no effect. This avoids two processes reading from the same netlink fd and screwing the pooch entirely. */ - + enumerate_interfaces(0); - + if (option_bool(OPT_NOWILD)) iface = listener->iface; /* May be NULL */ - else + else { int if_index; char intr_name[IF_NAMESIZE]; - + /* if we can find the arrival interface, check it's one that's allowed */ if ((if_index = tcp_interface(confd, tcp_addr.sa.sa_family)) != 0 && indextoname(listener->tcpfd, if_index, intr_name)) { - struct all_addr addr; - addr.addr.addr4 = tcp_addr.in.sin_addr; -#ifdef HAVE_IPV6 + union all_addr addr; + if (tcp_addr.sa.sa_family == AF_INET6) - addr.addr.addr6 = tcp_addr.in6.sin6_addr; -#endif - + addr.addr6 = tcp_addr.in6.sin6_addr; + else + addr.addr4 = tcp_addr.in.sin_addr; + for (iface = daemon->interfaces; iface; iface = iface->next) if (iface->index == if_index) break; - + if (!iface && !loopback_exception(listener->tcpfd, tcp_addr.sa.sa_family, &addr, intr_name)) client_ok = 0; } - + if (option_bool(OPT_CLEVERBIND)) iface = listener->iface; /* May be NULL */ else { /* Check for allowed interfaces when binding the wildcard address: - we do this by looking for an interface with the same address as + we do this by looking for an interface with the same address as the local address of the TCP connection, then looking to see if that's an allowed interface. As a side effect, we get the netmask of the interface too, for localisation. */ - + for (iface = daemon->interfaces; iface; iface = iface->next) if (sockaddr_isequal(&iface->addr, &tcp_addr)) break; - + if (!iface) client_ok = 0; } } - + if (!client_ok) { shutdown(confd, SHUT_RDWR); - while (retry_send(close(confd))); + close(confd); } -#ifndef NO_FORK - else if (!option_bool(OPT_DEBUG) && (p = fork()) != 0) + else if (!option_bool(OPT_DEBUG) && pipe(pipefd) == 0 && (p = fork()) != 0) { - if (p != -1) + close(pipefd[1]); /* parent needs read pipe end. */ + if (p == -1) + close(pipefd[0]); + else { int i; + for (i = 0; i < MAX_PROCS; i++) - if (daemon->tcp_pids[i] == 0) + if (daemon->tcp_pids[i] == 0 && daemon->tcp_pipes[i] == -1) { + char a; + (void)a; /* suppress potential unused warning */ + daemon->tcp_pids[i] = p; + daemon->tcp_pipes[i] = pipefd[0]; +#ifdef HAVE_LINUX_NETWORK + /* The child process inherits the netlink socket, + which it never uses, but when the parent (us) + uses it in the future, the answer may go to the + child, resulting in the parent blocking + forever awaiting the result. To avoid this + the child closes the netlink socket, but there's + a nasty race, since the parent may use netlink + before the child has done the close. + + To avoid this, the parent blocks here until a + single byte comes back up the pipe, which + is sent by the child after it has closed the + netlink socket. */ + retry_send(read(pipefd[0], &a, 1)); +#endif break; } } - while (retry_send(close(confd))); + close(confd); /* The child can use up to TCP_MAX_QUERIES ids, so skip that many. */ daemon->log_id += TCP_MAX_QUERIES; } -#endif else { unsigned char *buff; - struct server *s; + struct server *s; int flags; struct in_addr netmask; int auth_dns; - + if (iface) { netmask = iface->netmask; @@ -1781,44 +1932,55 @@ static void check_dns_listeners(time_t now) auth_dns = 0; } -#ifndef NO_FORK /* Arrange for SIGALRM after CHILD_LIFETIME seconds to terminate the process. */ if (!option_bool(OPT_DEBUG)) - alarm(CHILD_LIFETIME); + { + char a = 0; + (void)a; /* suppress potential unused warning */ + alarm(CHILD_LIFETIME); + close(pipefd[0]); /* close read end in child. */ + daemon->pipe_to_parent = pipefd[1]; +#ifdef HAVE_LINUX_NETWORK + /* See comment above re netlink socket. */ + close(daemon->netlinkfd); + retry_send(write(pipefd[1], &a, 1)); #endif + } /* start with no upstream connections. */ for (s = daemon->servers; s; s = s->next) - s->tcpfd = -1; - + s->tcpfd = -1; + /* The connected socket inherits non-blocking - attribute from the listening socket. + attribute from the listening socket. Reset that here. */ if ((flags = fcntl(confd, F_GETFL, 0)) != -1) fcntl(confd, F_SETFL, flags & ~O_NONBLOCK); - + buff = tcp_request(confd, now, &tcp_addr, netmask, auth_dns); - + shutdown(confd, SHUT_RDWR); - while (retry_send(close(confd))); - + close(confd); + if (buff) free(buff); - + for (s = daemon->servers; s; s = s->next) if (s->tcpfd != -1) { shutdown(s->tcpfd, SHUT_RDWR); - while (retry_send(close(s->tcpfd))); + close(s->tcpfd); } -#ifndef NO_FORK if (!option_bool(OPT_DEBUG)) { + /*** Pi-hole modification ***/ + // TCP workers ignore all signals except SIGALRM + FTL_TCP_worker_terminating(); + /*** Pi-hole modification ***/ flush_log(); _exit(0); } -#endif } } } @@ -1849,7 +2011,7 @@ int icmp_ping(struct in_addr addr) int fd; struct sockaddr_in saddr; - struct { + struct { struct ip ip; struct icmp icmp; } packet; @@ -1872,23 +2034,23 @@ int icmp_ping(struct in_addr addr) #ifdef HAVE_SOCKADDR_SA_LEN saddr.sin_len = sizeof(struct sockaddr_in); #endif - + memset(&packet.icmp, 0, sizeof(packet.icmp)); packet.icmp.icmp_type = ICMP_ECHO; packet.icmp.icmp_id = id; for (j = 0, i = 0; i < sizeof(struct icmp) / 2; i++) j += ((u16 *)&packet.icmp)[i]; while (j>>16) - j = (j & 0xffff) + (j >> 16); + j = (j & 0xffff) + (j >> 16); packet.icmp.icmp_cksum = (j == 0xffff) ? j : ~j; - - while (retry_send(sendto(fd, (char *)&packet.icmp, sizeof(struct icmp), 0, + + while (retry_send(sendto(fd, (char *)&packet.icmp, sizeof(struct icmp), 0, (struct sockaddr *)&saddr, sizeof(saddr)))); - + gotreply = delay_dhcp(dnsmasq_time(), PING_WAIT, fd, addr.s_addr, id); #if defined(HAVE_LINUX_NETWORK) || defined(HAVE_SOLARIS_NETWORK) - while (retry_send(close(fd))); + close(fd); #else opt = 1; setsockopt(fd, SOL_SOCKET, SO_RCVBUF, &opt, sizeof(opt)); @@ -1927,29 +2089,29 @@ int delay_dhcp(time_t start, int sec, int fd, uint32_t addr, unsigned short id) poll_listen(fd, POLLIN); set_dns_listeners(now); set_log_writer(); - + #ifdef HAVE_DHCP6 if (daemon->doing_ra) - poll_listen(daemon->icmp6fd, POLLIN); + poll_listen(daemon->icmp6fd, POLLIN); #endif - + rc = do_poll(250); - + if (rc < 0) continue; else if (rc == 0) timeout_count++; now = dnsmasq_time(); - + check_log_writer(0); check_dns_listeners(now); - + #ifdef HAVE_DHCP6 if (daemon->doing_ra && poll_check(daemon->icmp6fd, POLLIN)) icmp6_packet(now); #endif - + #ifdef HAVE_TFTP check_tftp_listeners(now); #endif @@ -1962,7 +2124,7 @@ int delay_dhcp(time_t start, int sec, int fd, uint32_t addr, unsigned short id) } packet; struct sockaddr_in faddr; socklen_t len = sizeof(faddr); - + if (poll_check(fd, POLLIN) && recvfrom(fd, &packet, sizeof(packet), 0, (struct sockaddr *)&faddr, &len) == sizeof(packet) && addr == faddr.sin_addr.s_addr && @@ -1975,6 +2137,4 @@ int delay_dhcp(time_t start, int sec, int fd, uint32_t addr, unsigned short id) return 0; } -#endif - - +#endif /* HAVE_DHCP */ diff --git a/dnsmasq/dnsmasq.h b/src/dnsmasq/dnsmasq.h similarity index 86% rename from dnsmasq/dnsmasq.h rename to src/dnsmasq/dnsmasq.h index 4eb64557c..baa7cded9 100644 --- a/dnsmasq/dnsmasq.h +++ b/src/dnsmasq/dnsmasq.h @@ -1,26 +1,26 @@ -/* dnsmasq is Copyright (c) 2000-2018 Simon Kelley - +/* dnsmasq is Copyright (c) 2000-2020 Simon Kelley + This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; version 2 dated June, 1991, or (at your option) version 3 dated 29 June, 2007. - + This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. - + You should have received a copy of the GNU General Public License along with this program. If not, see . */ -#define COPYRIGHT "Copyright (c) 2000-2018 Simon Kelley" +#define COPYRIGHT "Copyright (c) 2000-2020 Simon Kelley" /* We do defines that influence behavior of stdio.h, so complain if included too early. */ #ifdef _STDIO_H # error "Header file stdio.h included too early!" -#endif +#endif #ifndef NO_LARGEFILE /* Ensure we can use files >2GB (log files may grow this big) */ @@ -33,7 +33,7 @@ # ifndef __ANDROID__ # define _GNU_SOURCE # endif -# include +# include #endif /* Need these defined early */ @@ -49,7 +49,7 @@ #endif /* get these before config.h for IPv6 stuff... */ -#include +#include #include #ifdef __APPLE__ @@ -85,7 +85,7 @@ typedef unsigned long long u64; # define _(S) (S) #else # include -# include +# include # define _(S) gettext(S) #endif @@ -95,7 +95,11 @@ typedef unsigned long long u64; #if defined(HAVE_SOLARIS_NETWORK) # include #endif -#include +#if defined(HAVE_POLL_H) +# include +#else +# include +#endif #include #include #include @@ -126,10 +130,9 @@ typedef unsigned long long u64; #include #include #include -#ifdef HAVE_IPV6 #include -#endif #include +#include #include #include #include @@ -138,8 +141,10 @@ typedef unsigned long long u64; #endif #if defined(HAVE_LINUX_NETWORK) +#include +#include #include -/* There doesn't seem to be a universally-available +/* There doesn't seem to be a universally-available userspace header for these. */ extern int capset(cap_user_header_t header, cap_user_data_t data); extern int capget(cap_user_header_t header, cap_user_data_t data); @@ -159,6 +164,8 @@ extern int capget(cap_user_header_t header, cap_user_data_t data); /* daemon is function in the C library.... */ #define daemon dnsmasq_daemon +#define ADDRSTRLEN INET6_ADDRSTRLEN + /* Async event queue */ struct event_desc { int event, data, msg_sz; @@ -200,9 +207,6 @@ struct event_desc { #define EC_MISC 5 #define EC_INIT_OFFSET 10 -/* Trust the compiler dead-code eliminator.... */ -#define option_bool(x) (((x) < 32) ? daemon->options & (1u << (x)) : daemon->options2 & (1u << ((x) - 32))) - #define OPT_BOGUSPRIV 0 #define OPT_FILTER 1 #define OPT_LOG 2 @@ -228,7 +232,7 @@ struct event_desc { #define OPT_LEASE_RO 22 #define OPT_ALL_SERVERS 23 #define OPT_RELOAD 24 -#define OPT_LOCAL_REBIND 25 +#define OPT_LOCAL_REBIND 25 #define OPT_TFTP_SECURE 26 #define OPT_TFTP_NOBLOCK 27 #define OPT_LOG_OPTS 28 @@ -251,7 +255,7 @@ struct event_desc { #define OPT_DNSSEC_VALID 45 #define OPT_DNSSEC_TIME 46 #define OPT_DNSSEC_DEBUG 47 -#define OPT_DNSSEC_IGN_NS 48 +#define OPT_DNSSEC_IGN_NS 48 #define OPT_LOCAL_SERVICE 49 #define OPT_LOOP_DETECT 50 #define OPT_EXTRALOG 51 @@ -262,35 +266,61 @@ struct event_desc { #define OPT_TFTP_APREF_MAC 56 #define OPT_RAPID_COMMIT 57 #define OPT_UBUS 58 -#define OPT_LAST 59 - -/* extra flags for my_syslog, we use a couple of facilities since they are known +#define OPT_IGNORE_CLID 59 +#define OPT_SINGLE_PORT 60 +#define OPT_LEASE_RENEW 61 +#define OPT_LAST 62 + +#define OPTION_BITS (sizeof(unsigned int)*8) +#define OPTION_SIZE ( (OPT_LAST/OPTION_BITS)+((OPT_LAST%OPTION_BITS)!=0) ) +#define option_var(x) (daemon->options[(x) / OPTION_BITS]) +#define option_val(x) ((1u) << ((x) % OPTION_BITS)) +#define option_bool(x) (option_var(x) & option_val(x)) + +/* extra flags for my_syslog, we use a couple of facilities since they are known not to occupy the same bits as priorities, no matter how syslog.h is set up. */ #define MS_TFTP LOG_USER #define MS_DHCP LOG_DAEMON #define MS_SCRIPT LOG_MAIL -struct all_addr { - union { - struct in_addr addr4; -#ifdef HAVE_IPV6 - struct in6_addr addr6; -#endif - /* for log_query */ - struct { - unsigned short keytag, algo, digest; - } log; - /* for log_query */ - struct { - unsigned int rcode; - } rcode; - /* for cache_insert of DNSKEY, DS */ - struct { - unsigned short class, type; - } dnssec; - } addr; +/* Note that this is used widely as a container for IPv4/IPv6 addresses, + so for that reason, was well as to avoid wasting memory in almost every + cache entry, the other variants should not be larger than + sizeof(struct in6_addr) - 16 bytes. +*/ +union all_addr { + struct in_addr addr4; + struct in6_addr addr6; + struct { + union { + struct crec *cache; + char *name; + } target; + unsigned int uid; + int is_name_ptr; /* disciminates target union */ + } cname; + struct { + struct blockdata *keydata; + unsigned short keylen, flags, keytag; + unsigned char algo; + } key; + struct { + struct blockdata *keydata; + unsigned short keylen, keytag; + unsigned char algo; + unsigned char digest; + } ds; + struct { + struct blockdata *target; + unsigned short targetlen, srvport, priority, weight; + } srv; + /* for log_query */ + struct { + unsigned short keytag, algo, digest, rcode; + } log; }; + struct bogus_addr { struct in_addr addr; struct bogus_addr *next; @@ -345,7 +375,7 @@ struct cname { int ttl, flag; char *alias, *target; struct cname *next, *targetp; -}; +}; struct ds_config { char *name, *digest; @@ -353,13 +383,17 @@ struct ds_config { struct ds_config *next; }; -#define ADDRLIST_LITERAL 1 -#define ADDRLIST_IPV6 2 -#define ADDRLIST_REVONLY 4 +#define ADDRLIST_LITERAL 1 +#define ADDRLIST_IPV6 2 +#define ADDRLIST_REVONLY 4 +#define ADDRLIST_PREFIX 8 +#define ADDRLIST_WILDCARD 16 +#define ADDRLIST_DECLINED 32 struct addrlist { - struct all_addr addr; + union all_addr addr; int flags, prefixlen; + time_t decline_time; struct addrlist *next; }; @@ -378,17 +412,17 @@ struct auth_zone { struct auth_zone *next; }; +#define HR_6 1 +#define HR_4 2 struct host_record { - int ttl; + int ttl, flags; struct name_list { char *name; struct name_list *next; } *names; struct in_addr addr; -#ifdef HAVE_IPV6 struct in6_addr addr6; -#endif struct host_record *next; }; @@ -410,34 +444,13 @@ struct blockdata { unsigned char key[KEYBLOCK_LEN]; }; -struct crec { +struct crec { struct crec *next, *prev, *hash_next; - /* union is 16 bytes when doing IPv6, 8 bytes on 32 bit machines without IPv6 */ - union { - struct all_addr addr; - struct { - union { - struct crec *cache; - struct interface_name *int_name; - } target; - unsigned int uid; /* 0 if union is interface-name */ - } cname; - struct { - struct blockdata *keydata; - unsigned short keylen, flags, keytag; - unsigned char algo; - } key; - struct { - struct blockdata *keydata; - unsigned short keylen, keytag; - unsigned char algo; - unsigned char digest; - } ds; - } addr; + union all_addr addr; time_t ttd; /* time to die */ /* used as class if DNSKEY/DS, index to source for F_HOSTS */ - unsigned int uid; - unsigned short flags; + unsigned int uid; + unsigned int flags; union { char sname[SMALLDNAME]; union bigname *bname; @@ -453,7 +466,7 @@ struct crec { #define F_REVERSE (1u<<2) #define F_FORWARD (1u<<3) #define F_DHCP (1u<<4) -#define F_NEG (1u<<5) +#define F_NEG (1u<<5) #define F_HOSTS (1u<<6) #define F_IPV4 (1u<<7) #define F_IPV6 (1u<<8) @@ -464,9 +477,6 @@ struct crec { #define F_CONFIG (1u<<13) #define F_DS (1u<<14) #define F_DNSSECOK (1u<<15) - -/* below here are only valid as args to log_query: cache - entries are limited to 16 bits */ #define F_UPSTREAM (1u<<16) #define F_RRNAME (1u<<17) #define F_SERVER (1u<<18) @@ -479,24 +489,15 @@ struct crec { #define F_NO_RR (1u<<25) #define F_IPSET (1u<<26) #define F_NOEXTRA (1u<<27) -#define F_SERVFAIL (1u<<28) +#define F_SERVFAIL (1u<<28) /* currently unused. */ #define F_RCODE (1u<<29) +#define F_SRV (1u<<30) #define UID_NONE 0 /* Values of uid in crecs with F_CONFIG bit set. */ -/* cname to uid SRC_INTERFACE are to interface names, - so use UID_NONE for that to eliminate clashes with - any other uid */ -#define SRC_INTERFACE UID_NONE #define SRC_CONFIG 1 #define SRC_HOSTS 2 -/*----- Pi-hole modification -----*/ -// ID 3 will be used for the regex list file name -// ID 4 will be used as starting index for any Additional Hosts (AH) files -#define SRC_REGEX 3 -#define SRC_AH 4 -const char *regexlistname; -/*--------------------------------*/ +#define SRC_AH 3 /* struct sockaddr is not large enough to hold any address, and specifically not big enough to hold an IPv6 address. @@ -504,9 +505,7 @@ const char *regexlistname; union mysockaddr { struct sockaddr sa; struct sockaddr_in in; -#if defined(HAVE_IPV6) struct sockaddr_in6 in6; -#endif }; /* bits in flag param to IPv6 callbacks from iface_enumerate() */ @@ -517,7 +516,7 @@ union mysockaddr { #define SERV_FROM_RESOLV 1 /* 1 for servers from resolv, 0 for command line. */ #define SERV_NO_ADDR 2 /* no server, this domain is local only */ -#define SERV_LITERAL_ADDRESS 4 /* addr is the answer, not the server */ +#define SERV_LITERAL_ADDRESS 4 /* addr is the answer, not the server */ #define SERV_HAS_DOMAIN 8 /* server for one domain only */ #define SERV_HAS_SOURCE 16 /* source address defined */ #define SERV_FOR_NODOTS 32 /* server for names with no domain part only */ @@ -545,19 +544,19 @@ struct randfd { int fd; unsigned short refcount, family; }; - + struct server { union mysockaddr addr, source_addr; char interface[IF_NAMESIZE+1]; - struct serverfd *sfd; - char *domain; /* set if this server only handles a domain. */ + struct serverfd *sfd; + char *domain; /* set if this server only handles a domain. */ int flags, tcpfd, edns_pktsz; time_t pktsz_reduced; unsigned int queries, failed_queries; #ifdef HAVE_LOOP u32 uid; #endif - struct server *next; + struct server *next; }; struct ipsets { @@ -570,7 +569,7 @@ struct irec { union mysockaddr addr; struct in_addr netmask; /* only valid for IPv4 */ int tftp_ok, dhcp_ok, mtu, done, warned, dad, dns_auth, index, multicast_done, found, label; - char *name; + char *name; struct irec *next; }; @@ -655,7 +654,7 @@ struct hostsfile { #define FREC_DO_QUESTION 64 #define FREC_ADDED_PHEADER 128 #define FREC_TEST_PKTSZ 256 -#define FREC_HAS_EXTRADATA 512 +#define FREC_HAS_EXTRADATA 512 #ifdef HAVE_DNSSEC #define HASH_SIZE 20 /* SHA-1 digest size */ @@ -665,18 +664,16 @@ struct hostsfile { struct frec { union mysockaddr source; - struct all_addr dest; + union all_addr dest; struct server *sentto; /* NULL means free */ struct randfd *rfd4; -#ifdef HAVE_IPV6 struct randfd *rfd6; -#endif unsigned int iface; unsigned short orig_id, new_id; int log_id, fd, forwardall, flags; time_t time; unsigned char *hash[HASH_SIZE]; -#ifdef HAVE_DNSSEC +#ifdef HAVE_DNSSEC int class, work_counter; struct blockdata *stash; /* Saved reply, whilst we validate */ size_t stash_len; @@ -692,7 +689,7 @@ struct frec { #define OT_INTERNAL 0x2000 #define OT_NAME 0x1000 #define OT_CSTRING 0x0800 -#define OT_DEC 0x0400 +#define OT_DEC 0x0400 #define OT_TIME 0x0200 /* actions in the daemon->helper RPC */ @@ -712,6 +709,7 @@ struct frec { #define LEASE_NA 32 /* IPv6 no-temporary lease */ #define LEASE_TA 64 /* IPv6 temporary lease */ #define LEASE_HAVE_HWADDR 128 /* Have set hwaddress */ +#define LEASE_EXP_CHANGED 256 /* Lease expiry time changed */ struct dhcp_lease { int clid_len; /* length of client identifier */ @@ -724,7 +722,7 @@ struct dhcp_lease { unsigned int length; #endif int hwaddr_len, hwaddr_type; - unsigned char hwaddr[DHCP_CHADDR_MAX]; + unsigned char hwaddr[DHCP_CHADDR_MAX]; struct in_addr addr, override, giaddr; unsigned char *extradata; unsigned int extradata_len, extradata_size; @@ -733,7 +731,7 @@ struct dhcp_lease { int new_prefixlen; /* and its prefix length */ #ifdef HAVE_DHCP6 struct in6_addr addr6; - int iaid; + unsigned int iaid; struct slaac_address { struct in6_addr addr; time_t ping_time; @@ -780,8 +778,9 @@ struct dhcp_config { unsigned char *clid; /* clientid */ char *hostname, *domain; struct dhcp_netid_list *netid; + struct dhcp_netid *filter; #ifdef HAVE_DHCP6 - struct in6_addr addr6; + struct addrlist *addr6; #endif struct in_addr addr; time_t decline_time; @@ -790,7 +789,7 @@ struct dhcp_config { struct dhcp_config *next; }; -#define have_config(config, mask) ((config) && ((config)->flags & (mask))) +#define have_config(config, mask) ((config) && ((config)->flags & (mask))) #define CONFIG_DISABLE 1 #define CONFIG_CLID 2 @@ -803,7 +802,7 @@ struct dhcp_config { #define CONFIG_DECLINED 1024 /* address declined by client */ #define CONFIG_BANK 2048 /* from dhcp hosts file */ #define CONFIG_ADDR6 4096 -#define CONFIG_WILDCARD 8192 +#define CONFIG_ADDR6_HOSTS 16384 /* address added by from /etc/hosts */ struct dhcp_opt { int opt, len, flags; @@ -847,7 +846,7 @@ struct dhcp_match_name { }; struct pxe_service { - unsigned short CSA, type; + unsigned short CSA, type; char *menu, *basename, *sname; struct in_addr server; struct dhcp_netid *netid; @@ -885,20 +884,10 @@ struct dhcp_bridge { struct cond_domain { char *domain, *prefix; struct in_addr start, end; -#ifdef HAVE_IPV6 struct in6_addr start6, end6; -#endif int is6, indexed; struct cond_domain *next; -}; - -#ifdef OPTION6_PREFIX_CLASS -struct prefix_class { - int class; - struct dhcp_netid tag; - struct prefix_class *next; -}; -#endif +}; struct ra_interface { char *name; @@ -925,6 +914,16 @@ struct dhcp_context { struct dhcp_context *next, *current; }; +struct shared_network { + int if_index; + struct in_addr match_addr, shared_addr; +#ifdef HAVE_DHCP6 + /* shared_addr == 0 for IP6 entries. */ + struct in6_addr match_addr6, shared_addr6; +#endif + struct shared_network *next; +}; + #define CONTEXT_STATIC (1u<<0) #define CONTEXT_NETMASK (1u<<1) #define CONTEXT_BRDCAST (1u<<2) @@ -967,6 +966,8 @@ struct tftp_transfer { unsigned int block, blocksize, expansion; off_t offset; union mysockaddr peer; + union all_addr source; + int if_index; char opt_blocksize, opt_transize, netascii, carrylf; struct tftp_file *file; struct tftp_transfer *next; @@ -985,18 +986,18 @@ struct tftp_prefix { }; struct dhcp_relay { - struct all_addr local, server; + union all_addr local, server; char *interface; /* Allowable interface for replies from server, and dest for IPv6 multicast */ int iface_index; /* working - interface in which requests arrived, for return */ struct dhcp_relay *current, *next; }; extern struct daemon { - /* datastuctures representing the command-line and + /* datastuctures representing the command-line and config file arguments. All set (including defaults) in option.c */ - unsigned int options, options2; + unsigned int options[OPTION_SIZE]; struct resolvc default_resolv, *resolv_files; time_t last_resolv; char *servers_file; @@ -1020,7 +1021,7 @@ extern struct daemon { int group_set, osport; char *domain_suffix; struct cond_domain *cond_domain, *synth_domains; - char *runfile; + char *runfile; char *lease_change_command; struct iname *if_names, *if_addrs, *if_except, *dhcp_except, *auth_peers, *tftp_interfaces; struct bogus_addr *bogus_addr, *ignore_addr; @@ -1043,34 +1044,32 @@ extern struct daemon { struct dhcp_mac *dhcp_macs; struct dhcp_boot *boot_config; struct pxe_service *pxe_services; - struct tag_if *tag_if; + struct tag_if *tag_if; struct addr_list *override_relays; struct dhcp_relay *relay4, *relay6; struct delay_config *delay_conf; int override; int enable_pxe; int doing_ra, doing_dhcp6; - struct dhcp_netid_list *dhcp_ignore, *dhcp_ignore_names, *dhcp_gen_names; + struct dhcp_netid_list *dhcp_ignore, *dhcp_ignore_names, *dhcp_gen_names; struct dhcp_netid_list *force_broadcast, *bootp_dynamic; struct hostsfile *dhcp_hosts_file, *dhcp_opts_file, *dynamic_dirs; int dhcp_max, tftp_max, tftp_mtu; int dhcp_server_port, dhcp_client_port; - int start_tftp_port, end_tftp_port; + int start_tftp_port, end_tftp_port; unsigned int min_leasetime; struct doctor *doctors; unsigned short edns_pktsz; - char *tftp_prefix; + char *tftp_prefix; struct tftp_prefix *if_prefix; /* per-interface TFTP prefixes */ unsigned int duid_enterprise, duid_config_len; unsigned char *duid_config; char *dbus_name; + char *ubus_name; char *dump_file; int dump_mask; unsigned long soa_sn, soa_refresh, soa_retry, soa_expiry; u32 metrics[__METRIC_MAX]; -#ifdef OPTION6_PREFIX_CLASS - struct prefix_class *prefix_classes; -#endif #ifdef HAVE_DNSSEC struct ds_config *ds; char *timestamp_file; @@ -1083,7 +1082,7 @@ extern struct daemon { #ifdef HAVE_DNSSEC char *keyname; /* MAXDNAME size buffer */ char *workspacename; /* ditto */ - char *rr_status; /* flags for individual RRs */ + unsigned long *rr_status; /* ceiling in TTL from DNSSEC or zero for insecure */ int rr_status_sz; int dnssec_no_time_check; int back_to_the_future; @@ -1099,19 +1098,21 @@ extern struct daemon { size_t packet_len; /* " " */ struct randfd *rfd_save; /* " " */ pid_t tcp_pids[MAX_PROCS]; + int tcp_pipes[MAX_PROCS]; + int pipe_to_parent; struct randfd randomsocks[RANDOM_SOCKS]; - int v6pktinfo; + int v6pktinfo; struct addrlist *interface_addrs; /* list of all addresses/prefix lengths associated with all local interfaces */ int log_id, log_display_id; /* ids of transactions for logging */ union mysockaddr *log_source_addr; /* DHCP state */ - int dhcpfd, helperfd, pxefd; + int dhcpfd, helperfd, pxefd; #ifdef HAVE_INOTIFY int inotifyfd; #endif #if defined(HAVE_LINUX_NETWORK) - int netlinkfd; + int netlinkfd, kernel_version; #elif defined(HAVE_BSD_NETWORK) int dhcp_raw_fd, dhcp_icmp_fd, routefd; #endif @@ -1120,6 +1121,7 @@ extern struct daemon { struct ping_result *ping_results; FILE *lease_stream; struct dhcp_bridge *bridges; + struct shared_network *shared_networks; #ifdef HAVE_DHCP6 int duid_len; unsigned char *duid; @@ -1132,6 +1134,11 @@ extern struct daemon { #ifdef HAVE_DBUS struct watch *watches; #endif + /* UBus stuff */ +#ifdef HAVE_UBUS + /* void * here to avoid depending on ubus headers outside ubus.c */ + void *ubus; +#endif /* TFTP stuff */ struct tftp_transfer *tftp_trans, *tftp_done_trans; @@ -1149,21 +1156,25 @@ extern struct daemon { /* cache.c */ void cache_init(void); void next_uid(struct crec *crecp); -void log_query(unsigned int flags, char *name, struct all_addr *addr, char *arg); +/********************************************* Pi-hole modification ***********************************************/ +#define log_query(flags,name,addr,arg) _log_query(flags, name, addr, arg, __FILE__, __LINE__) +void _log_query(unsigned int flags, char *name, union all_addr *addr, char *arg, const char* file, const int line); +/******************************************************************************************************************/ char *record_source(unsigned int index); char *querystr(char *desc, unsigned short type); int cache_find_non_terminal(char *name, time_t now); struct crec *cache_find_by_addr(struct crec *crecp, - struct all_addr *addr, time_t now, + union all_addr *addr, time_t now, unsigned int prot); -struct crec *cache_find_by_name(struct crec *crecp, +struct crec *cache_find_by_name(struct crec *crecp, char *name, time_t now, unsigned int prot); void cache_end_insert(void); void cache_start_insert(void); -struct crec *cache_insert(char *name, struct all_addr *addr, - time_t now, unsigned long ttl, unsigned short flags); +int cache_recv_insert(time_t now, int fd); +struct crec *cache_insert(char *name, union all_addr *addr, unsigned short class, + time_t now, unsigned long ttl, unsigned int flags); void cache_reload(void); -void cache_add_dhcp_entry(char *host_name, int prot, struct all_addr *host_address, time_t ttd); +void cache_add_dhcp_entry(char *host_name, int prot, union all_addr *host_address, time_t ttd); struct in_addr a_record_from_hosts(char *name, time_t now); void cache_unhash_dhcp(void); void dump_cache(time_t now); @@ -1173,59 +1184,57 @@ int cache_make_stat(struct txt_record *t); char *cache_get_name(struct crec *crecp); char *cache_get_cname_target(struct crec *crecp); struct crec *cache_enumerate(int init); -int read_hostsfile(char *filename, unsigned int index, int cache_size, +int read_hostsfile(char *filename, unsigned int index, int cache_size, struct crec **rhash, int hashsz); /* blockdata.c */ -#ifdef HAVE_DNSSEC void blockdata_init(void); void blockdata_report(void); struct blockdata *blockdata_alloc(char *data, size_t len); void *blockdata_retrieve(struct blockdata *block, size_t len, void *data); +struct blockdata *blockdata_read(int fd, size_t len); +void blockdata_write(struct blockdata *block, size_t len, int fd); void blockdata_free(struct blockdata *blocks); -#endif /* domain.c */ char *get_domain(struct in_addr addr); -#ifdef HAVE_IPV6 char *get_domain6(struct in6_addr *addr); -#endif -int is_name_synthetic(int flags, char *name, struct all_addr *addr); -int is_rev_synth(int flag, struct all_addr *addr, char *name); +int is_name_synthetic(int flags, char *name, union all_addr *addr); +int is_rev_synth(int flag, union all_addr *addr, char *name); /* rfc1035.c */ -int extract_name(struct dns_header *header, size_t plen, unsigned char **pp, +int extract_name(struct dns_header *header, size_t plen, unsigned char **pp, char *name, int isExtract, int extrabytes); unsigned char *skip_name(unsigned char *ansp, struct dns_header *header, size_t plen, int extrabytes); unsigned char *skip_questions(struct dns_header *header, size_t plen); unsigned char *skip_section(unsigned char *ansp, int count, struct dns_header *header, size_t plen); -unsigned int extract_request(struct dns_header *header, size_t qlen, +unsigned int extract_request(struct dns_header *header, size_t qlen, char *name, unsigned short *typep); size_t setup_reply(struct dns_header *header, size_t qlen, - struct all_addr *addrp, unsigned int flags, + union all_addr *addrp, unsigned int flags, unsigned long ttl); int extract_addresses(struct dns_header *header, size_t qlen, char *name, time_t now, char **ipsets, int is_sign, int check_rebind, int no_cache_dnssec, int secure, int *doctored); -size_t answer_request(struct dns_header *header, char *limit, size_t qlen, - struct in_addr local_addr, struct in_addr local_netmask, +size_t answer_request(struct dns_header *header, char *limit, size_t qlen, + struct in_addr local_addr, struct in_addr local_netmask, time_t now, int ad_reqd, int do_bit, int have_pseudoheader); -int check_for_bogus_wildcard(struct dns_header *header, size_t qlen, char *name, +int check_for_bogus_wildcard(struct dns_header *header, size_t qlen, char *name, struct bogus_addr *baddr, time_t now); int check_for_ignored_address(struct dns_header *header, size_t qlen, struct bogus_addr *baddr); int check_for_local_domain(char *name, time_t now); unsigned int questions_crc(struct dns_header *header, size_t plen, char *name); -size_t resize_packet(struct dns_header *header, size_t plen, +size_t resize_packet(struct dns_header *header, size_t plen, unsigned char *pheader, size_t hlen); int add_resource_record(struct dns_header *header, char *limit, int *truncp, - int nameoffset, unsigned char **pp, unsigned long ttl, + int nameoffset, unsigned char **pp, unsigned long ttl, int *offset, unsigned short type, unsigned short class, char *format, ...); -int in_arpa_name_2_addr(char *namein, struct all_addr *addrp); +int in_arpa_name_2_addr(char *namein, union all_addr *addrp); int private_net(struct in_addr addr, int ban_localhost); /* auth.c */ #ifdef HAVE_AUTH -size_t answer_auth(struct dns_header *header, char *limit, size_t qlen, +size_t answer_auth(struct dns_header *header, char *limit, size_t qlen, time_t now, union mysockaddr *peer_addr, int local_query, int do_bit, int have_pseudoheader); int in_zone(struct auth_zone *zone, char *name, char **cut); @@ -1236,7 +1245,7 @@ size_t dnssec_generate_query(struct dns_header *header, unsigned char *end, char int dnssec_validate_by_ds(time_t now, struct dns_header *header, size_t plen, char *name, char *keyname, int class); int dnssec_validate_ds(time_t now, struct dns_header *header, size_t plen, char *name, char *keyname, int class); int dnssec_validate_reply(time_t now, struct dns_header *header, size_t plen, char *name, char *keyname, int *class, - int check_unsigned, int *neganswer, int *nons); + int check_unsigned, int *neganswer, int *nons, int *nsec_ttl); int dnskey_keytag(int alg, int flags, unsigned char *key, int keylen); size_t filter_rrsigs(struct dns_header *header, size_t plen); unsigned char* hash_questions(struct dns_header *header, size_t plen, char *name); @@ -1270,24 +1279,25 @@ int hostname_issubdomain(char *a, char *b); time_t dnsmasq_time(void); int netmask_length(struct in_addr mask); int is_same_net(struct in_addr a, struct in_addr b, struct in_addr mask); -#ifdef HAVE_IPV6 int is_same_net6(struct in6_addr *a, struct in6_addr *b, int prefixlen); u64 addr6part(struct in6_addr *addr); void setaddr6part(struct in6_addr *addr, u64 host); -#endif int retry_send(ssize_t rc); void prettyprint_time(char *buf, unsigned int t); int prettyprint_addr(union mysockaddr *addr, char *buf); -int parse_hex(char *in, unsigned char *out, int maxlen, +int parse_hex(char *in, unsigned char *out, int maxlen, unsigned int *wildcard_mask, int *mac_type); -int memcmp_masked(unsigned char *a, unsigned char *b, int len, +int memcmp_masked(unsigned char *a, unsigned char *b, int len, unsigned int mask); int expand_buf(struct iovec *iov, size_t size); char *print_mac(char *buff, unsigned char *mac, int len); int read_write(int fd, unsigned char *packet, int size, int rw); - +void close_fds(long max_fd, int spare1, int spare2, int spare3); int wildcard_match(const char* wildcard, const char* match); int wildcard_matchn(const char* wildcard, const char* match, int num); +#ifdef HAVE_LINUX_NETWORK +int kernel_version(void); +#endif /* log.c */ void die(char *message, char *arg1, int exit_code) ATTRIBUTE_NORETURN; @@ -1302,14 +1312,14 @@ void flush_log(void); /* option.c */ void read_opts (int argc, char **argv, char *compile_opts); -char *option_string(int prot, unsigned int opt, unsigned char *val, +char *option_string(int prot, unsigned int opt, unsigned char *val, int opt_len, char *buf, int buf_len); void reread_dhcp(void); void read_servers_file(void); void set_option_bool(unsigned int opt); void reset_option_bool(unsigned int opt); struct hostsfile *expand_filelist(struct hostsfile *list); -char *parse_server(char *arg, union mysockaddr *addr, +char *parse_server(char *arg, union mysockaddr *addr, union mysockaddr *source_addr, char *interface, int *flags); int option_read_dynfile(char *file, int flags); @@ -1319,9 +1329,9 @@ void receive_query(struct listener *listen, time_t now); unsigned char *tcp_request(int confd, time_t now, union mysockaddr *local_addr, struct in_addr netmask, int auth_dns); void server_gone(struct server *server); -struct frec *get_new_frec(time_t now, int *wait, int force); -int send_from(int fd, int nowild, char *packet, size_t len, - union mysockaddr *to, struct all_addr *source, +struct frec *get_new_frec(time_t now, int *wait, struct frec *force); +int send_from(int fd, int nowild, char *packet, size_t len, + union mysockaddr *to, union all_addr *source, unsigned int iface); void resend_query(void); struct randfd *allocate_rfd(int family); @@ -1348,14 +1358,12 @@ void warn_bound_listeners(void); void warn_wild_labels(void); void warn_int_names(void); int is_dad_listeners(void); -int iface_check(int family, struct all_addr *addr, char *name, int *auth); -int loopback_exception(int fd, int family, struct all_addr *addr, char *name); -int label_exception(int index, int family, struct all_addr *addr); +int iface_check(int family, union all_addr *addr, char *name, int *auth); +int loopback_exception(int fd, int family, union all_addr *addr, char *name); +int label_exception(int index, int family, union all_addr *addr); int fix_fd(int fd); int tcp_interface(int fd, int af); -#ifdef HAVE_IPV6 int set_ipv6pktinfo(int fd); -#endif #ifdef HAVE_DHCP6 void join_multicast(int dienow); #endif @@ -1368,10 +1376,10 @@ void newaddress(time_t now); #ifdef HAVE_DHCP void dhcp_init(void); void dhcp_packet(time_t now, int pxe_fd); -struct dhcp_context *address_available(struct dhcp_context *context, +struct dhcp_context *address_available(struct dhcp_context *context, struct in_addr taddr, struct dhcp_netid *netids); -struct dhcp_context *narrow_context(struct dhcp_context *context, +struct dhcp_context *narrow_context(struct dhcp_context *context, struct in_addr taddr, struct dhcp_netid *netids); struct ping_result *do_icmp_ping(time_t now, struct in_addr addr, @@ -1392,15 +1400,16 @@ void lease_init(time_t now); struct dhcp_lease *lease4_allocate(struct in_addr addr); #ifdef HAVE_DHCP6 struct dhcp_lease *lease6_allocate(struct in6_addr *addrp, int lease_type); -struct dhcp_lease *lease6_find(unsigned char *clid, int clid_len, - int lease_type, int iaid, struct in6_addr *addr); +struct dhcp_lease *lease6_find(unsigned char *clid, int clid_len, + int lease_type, unsigned int iaid, struct in6_addr *addr); void lease6_reset(void); -struct dhcp_lease *lease6_find_by_client(struct dhcp_lease *first, int lease_type, unsigned char *clid, int clid_len, int iaid); +struct dhcp_lease *lease6_find_by_client(struct dhcp_lease *first, int lease_type, + unsigned char *clid, int clid_len, unsigned int iaid); struct dhcp_lease *lease6_find_by_addr(struct in6_addr *net, int prefix, u64 addr); u64 lease_find_max_addr6(struct dhcp_context *context); void lease_ping_reply(struct in6_addr *sender, unsigned char *packet, char *interface); void lease_update_slaac(time_t now); -void lease_set_iaid(struct dhcp_lease *lease, int iaid); +void lease_set_iaid(struct dhcp_lease *lease, unsigned int iaid); void lease_make_duid(time_t now); #endif void lease_set_hwaddr(struct dhcp_lease *lease, const unsigned char *hwaddr, @@ -1409,7 +1418,7 @@ void lease_set_hwaddr(struct dhcp_lease *lease, const unsigned char *hwaddr, void lease_set_hostname(struct dhcp_lease *lease, const char *name, int auth, char *domain, char *config_domain); void lease_set_expires(struct dhcp_lease *lease, unsigned int len, time_t now); void lease_set_interface(struct dhcp_lease *lease, int interface, time_t now); -struct dhcp_lease *lease_find_by_client(unsigned char *hwaddr, int hw_len, int hw_type, +struct dhcp_lease *lease_find_by_client(unsigned char *hwaddr, int hw_len, int hw_type, unsigned char *clid, int clid_len); struct dhcp_lease *lease_find_by_addr(struct in_addr addr); struct in_addr lease_find_max_addr(struct dhcp_context *context); @@ -1419,7 +1428,7 @@ int do_script_run(time_t now); void rerun_scripts(void); void lease_find_interfaces(time_t now); #ifdef HAVE_SCRIPT -void lease_add_extradata(struct dhcp_lease *lease, unsigned char *data, +void lease_add_extradata(struct dhcp_lease *lease, unsigned char *data, unsigned int len, int delim); #endif #endif @@ -1429,7 +1438,7 @@ void lease_add_extradata(struct dhcp_lease *lease, unsigned char *data, size_t dhcp_reply(struct dhcp_context *context, char *iface_name, int int_index, size_t sz, time_t now, int unicast_dest, int loopback, int *is_inform, int pxe, struct in_addr fallback, time_t recvtime); -unsigned char *extended_hwaddr(int hwtype, int hwlen, unsigned char *hwaddr, +unsigned char *extended_hwaddr(int hwtype, int hwlen, unsigned char *hwaddr, int clid_len, unsigned char *clid, int *len_out); #endif @@ -1446,7 +1455,7 @@ void clear_cache_and_reload(time_t now); /* netlink.c */ #ifdef HAVE_LINUX_NETWORK -void netlink_init(void); +char *netlink_init(void); void netlink_multicast(void); #endif @@ -1485,6 +1494,7 @@ void emit_dbus_signal(int action, struct dhcp_lease *lease, char *hostname); /* ubus.c */ #ifdef HAVE_UBUS +void ubus_init(void); void set_ubus_listeners(void); void check_ubus_listeners(void); void ubus_event_bcast(const char *type, const char *mac, const char *ip, const char *name, const char *interface); @@ -1493,20 +1503,20 @@ void ubus_event_bcast(const char *type, const char *mac, const char *ip, const c /* ipset.c */ #ifdef HAVE_IPSET void ipset_init(void); -int add_to_ipset(const char *setname, const struct all_addr *ipaddr, int flags, int remove); +int add_to_ipset(const char *setname, const union all_addr *ipaddr, int flags, int remove); #endif /* helper.c */ #if defined(HAVE_SCRIPT) int create_helper(int event_fd, int err_fd, uid_t uid, gid_t gid, long max_fd); void helper_write(void); -void queue_script(int action, struct dhcp_lease *lease, +void queue_script(int action, struct dhcp_lease *lease, char *hostname, time_t now); #ifdef HAVE_TFTP void queue_tftp(off_t file_len, char *filename, union mysockaddr *peer); #endif void queue_arp(int action, unsigned char *mac, int maclen, - int family, struct all_addr *addr); + int family, union all_addr *addr); int helper_buf_empty(void); #endif @@ -1519,7 +1529,7 @@ int do_tftp_script_run(void); /* conntrack.c */ #ifdef HAVE_CONNTRACK -int get_incoming_mark(union mysockaddr *peer_addr, struct all_addr *local_addr, +int get_incoming_mark(union mysockaddr *peer_addr, union all_addr *local_addr, int istcp, unsigned int *markp); #endif @@ -1528,30 +1538,29 @@ int get_incoming_mark(union mysockaddr *peer_addr, struct all_addr *local_addr, void dhcp6_init(void); void dhcp6_packet(time_t now); struct dhcp_context *address6_allocate(struct dhcp_context *context, unsigned char *clid, int clid_len, int temp_addr, - int iaid, int serial, struct dhcp_netid *netids, int plain_range, struct in6_addr *ans); -int config_valid(struct dhcp_config *config, struct dhcp_context *context, struct in6_addr *addr); -struct dhcp_context *address6_available(struct dhcp_context *context, + unsigned int iaid, int serial, struct dhcp_netid *netids, int plain_range, struct in6_addr *ans); +struct dhcp_context *address6_available(struct dhcp_context *context, struct in6_addr *taddr, struct dhcp_netid *netids, int plain_range); -struct dhcp_context *address6_valid(struct dhcp_context *context, +struct dhcp_context *address6_valid(struct dhcp_context *context, struct in6_addr *taddr, struct dhcp_netid *netids, int plain_range); -struct dhcp_config *config_find_by_address6(struct dhcp_config *configs, struct in6_addr *net, - int prefix, u64 addr); +struct dhcp_config *config_find_by_address6(struct dhcp_config *configs, struct in6_addr *net, + int prefix, struct in6_addr *addr); void make_duid(time_t now); void dhcp_construct_contexts(time_t now); -void get_client_mac(struct in6_addr *client, int iface, unsigned char *mac, +void get_client_mac(struct in6_addr *client, int iface, unsigned char *mac, unsigned int *maclenp, unsigned int *mactypep, time_t now); #endif - + /* rfc3315.c */ #ifdef HAVE_DHCP6 -unsigned short dhcp6_reply(struct dhcp_context *context, int interface, char *iface_name, +unsigned short dhcp6_reply(struct dhcp_context *context, int interface, char *iface_name, struct in6_addr *fallback, struct in6_addr *ll_addr, struct in6_addr *ula_addr, size_t sz, struct in6_addr *client_addr, time_t now); -void relay_upstream6(struct dhcp_relay *relay, ssize_t sz, struct in6_addr *peer_address, +void relay_upstream6(struct dhcp_relay *relay, ssize_t sz, struct in6_addr *peer_address, u32 scope_id, time_t now); unsigned short relay_reply6( struct sockaddr_in6 *peer, ssize_t sz, char *arrival_interface); @@ -1575,8 +1584,9 @@ int lookup_dhcp_len(int prot, int val); struct dhcp_config *find_config(struct dhcp_config *configs, struct dhcp_context *context, unsigned char *clid, int clid_len, - unsigned char *hwaddr, int hw_len, - int hw_type, char *hostname); + unsigned char *hwaddr, int hw_len, + int hw_type, char *hostname, + struct dhcp_netid *filter); int config_has_mac(struct dhcp_config *config, unsigned char *hwaddr, int len, int type); #ifdef HAVE_LINUX_NETWORK char *whichdevice(void); @@ -1611,7 +1621,7 @@ time_t periodic_ra(time_t now); void ra_start_unsolicited(time_t now, struct dhcp_context *context); #endif -/* slaac.c */ +/* slaac.c */ #ifdef HAVE_DHCP6 void slaac_add_addrs(struct dhcp_lease *lease, time_t now, int force); time_t periodic_slaac(time_t now, struct dhcp_lease *leases); @@ -1645,10 +1655,10 @@ int expand_workspace(unsigned char ***wkspc, int *szp, int new); /* edns0.c */ unsigned char *find_pseudoheader(struct dns_header *header, size_t plen, size_t *len, unsigned char **p, int *is_sign, int *is_last); -size_t add_pseudoheader(struct dns_header *header, size_t plen, unsigned char *limit, +size_t add_pseudoheader(struct dns_header *header, size_t plen, unsigned char *limit, unsigned short udp_sz, int optno, unsigned char *opt, size_t optlen, int set_do, int replace); size_t add_do_bit(struct dns_header *header, size_t plen, unsigned char *limit); -size_t add_edns0_config(struct dns_header *header, size_t plen, unsigned char *limit, +size_t add_edns0_config(struct dns_header *header, size_t plen, unsigned char *limit, union mysockaddr *source, time_t now, int *check_subnet); int check_source(struct dns_header *header, size_t plen, unsigned char *pseudoheader, union mysockaddr *peer); diff --git a/dnsmasq/dnssec.c b/src/dnsmasq/dnssec.c similarity index 86% rename from dnsmasq/dnssec.c rename to src/dnsmasq/dnssec.c index 6a76d983f..db5c2d11c 100644 --- a/dnsmasq/dnssec.c +++ b/src/dnsmasq/dnssec.c @@ -1,5 +1,5 @@ /* dnssec.c is Copyright (c) 2012 Giovanni Bajo - and Copyright (c) 2012-2018 Simon Kelley + and Copyright (c) 2012-2020 Simon Kelley This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -30,14 +30,14 @@ then calling to_wire() removes compression and maps case, thus generating names in canonical form. Calling to_wire followed by from_wire is almost an identity, - except that the UC remains mapped to LC. + except that the UC remains mapped to LC. Note that both /000 and '.' are allowed within labels. These get represented in presentation format using NAME_ESCAPE as an escape character. In theory, if all the characters in a name were /000 or - '.' or NAME_ESCAPE then all would have to be escaped, so the - presentation format would be twice as long as the spec (1024). - The buffers are all declared as 2049 (allowing for the trailing zero) + '.' or NAME_ESCAPE then all would have to be escaped, so the + presentation format would be twice as long as the spec (1024). + The buffers are all declared as 2049 (allowing for the trailing zero) for this reason. */ static int to_wire(char *name) @@ -57,17 +57,17 @@ static int to_wire(char *name) (*p)--; } term = *p; - + if ((len = p - l) != 0) memmove(l+1, l, len); *l = len; - + p++; - + if (term == 0) *p = 0; } - + return l + 1 - (unsigned char *)name; } @@ -76,9 +76,9 @@ static void from_wire(char *name) { unsigned char *l, *p, *last; int len; - + for (last = (unsigned char *)name; *last != 0; last += *last+1); - + for (l = (unsigned char *)name; *l != 0; l += len+1) { len = *l; @@ -88,10 +88,10 @@ static void from_wire(char *name) { memmove(p+1, p, 1 + last - p); len++; - *p++ = NAME_ESCAPE; + *p++ = NAME_ESCAPE; (*p)++; } - + l[len] = '.'; } @@ -104,7 +104,7 @@ static int count_labels(char *name) { int i; char *p; - + if (*name == 0) return 0; @@ -143,12 +143,12 @@ static time_t timestamp_time; int setup_timestamp(void) { struct stat statbuf; - + daemon->back_to_the_future = 0; - + if (!daemon->timestamp_file) return 0; - + if (stat(daemon->timestamp_file, &statbuf) != -1) { timestamp_time = statbuf.st_mtime; @@ -163,17 +163,17 @@ int setup_timestamp(void) } return 1; } - + if (errno == ENOENT) { - /* NB. for explanation of O_EXCL flag, see comment on pidfile in dnsmasq.c */ + /* NB. for explanation of O_EXCL flag, see comment on pidfile in dnsmasq.c */ int fd = open(daemon->timestamp_file, O_WRONLY | O_CREAT | O_NONBLOCK | O_EXCL, 0666); if (fd != -1) { struct timeval tv[2]; close(fd); - + timestamp_time = 1420070400; /* 1-1-2015 */ tv[0].tv_sec = tv[1].tv_sec = timestamp_time; tv[0].tv_usec = tv[1].tv_usec = 0; @@ -186,12 +186,10 @@ int setup_timestamp(void) } /* Check whether today/now is between date_start and date_end */ -static int check_date_range(u32 date_start, u32 date_end) +static int is_check_date(unsigned long curtime) { - unsigned long curtime = time(0); - /* Checking timestamps may be temporarily disabled */ - + /* If the current time if _before_ the timestamp on our persistent timestamp file, then assume the time if not yet correct, and don't check the @@ -204,31 +202,34 @@ static int check_date_range(u32 date_start, u32 date_end) { if (utimes(daemon->timestamp_file, NULL) != 0) my_syslog(LOG_ERR, _("failed to update mtime on %s: %s"), daemon->timestamp_file, strerror(errno)); - + my_syslog(LOG_INFO, _("system time considered valid, now checking DNSSEC signature timestamps.")); daemon->back_to_the_future = 1; daemon->dnssec_no_time_check = 0; queue_event(EVENT_RELOAD); /* purge cache */ - } + } - if (daemon->back_to_the_future == 0) - return 1; + return daemon->back_to_the_future; } - else if (daemon->dnssec_no_time_check) - return 1; + else + return !daemon->dnssec_no_time_check; +} +/* Check whether today/now is between date_start and date_end */ +static int check_date_range(unsigned long curtime, u32 date_start, u32 date_end) +{ /* We must explicitly check against wanted values, because of SERIAL_UNDEF */ return serial_compare_32(curtime, date_start) == SERIAL_GT && serial_compare_32(curtime, date_end) == SERIAL_LT; } -/* Return bytes of canonicalised rdata, when the return value is zero, the remaining +/* Return bytes of canonicalised rdata, when the return value is zero, the remaining data, pointed to by *p, should be used raw. */ static int get_rdata(struct dns_header *header, size_t plen, unsigned char *end, char *buff, int bufflen, unsigned char **p, u16 **desc) { int d = **desc; - + /* No more data needs mangling */ if (d == (u16)-1) { @@ -240,33 +241,33 @@ static int get_rdata(struct dns_header *header, size_t plen, unsigned char *end, *p += bufflen; return bufflen; } - + return 0; } - + (*desc)++; - + if (d == 0 && extract_name(header, plen, p, buff, 1, 0)) /* domain-name, canonicalise */ return to_wire(buff); else - { + { /* plain data preceding a domain-name, don't run off the end of the data */ if ((end - *p) < d) d = end - *p; - + if (d != 0) { memcpy(buff, *p, d); *p += d; } - + return d; } } -/* Bubble sort the RRset into the canonical order. - Note that the byte-streams from two RRs may get unsynced: consider +/* Bubble sort the RRset into the canonical order. + Note that the byte-streams from two RRs may get unsynced: consider RRs which have two domain-names at the start and then other data. The domain-names may have different lengths in each RR, but sort equal @@ -279,11 +280,11 @@ static int get_rdata(struct dns_header *header, size_t plen, unsigned char *end, leaving the following bytes as deciding the order. Hence the nasty left1 and left2 variables. */ -static int sort_rrset(struct dns_header *header, size_t plen, u16 *rr_desc, int rrsetidx, +static int sort_rrset(struct dns_header *header, size_t plen, u16 *rr_desc, int rrsetidx, unsigned char **rrset, char *buff1, char *buff2) { int swap, quit, i, j; - + do { for (swap = 0, i = 0; i < rrsetidx-1; i++) @@ -295,22 +296,22 @@ static int sort_rrset(struct dns_header *header, size_t plen, u16 *rr_desc, int so we don't need to check for NULL return here. */ unsigned char *p1 = skip_name(rrset[i], header, plen, 10); unsigned char *p2 = skip_name(rrset[i+1], header, plen, 10); - + p1 += 8; /* skip class, type, ttl */ GETSHORT(rdlen1, p1); end1 = p1 + rdlen1; - + p2 += 8; /* skip class, type, ttl */ GETSHORT(rdlen2, p2); - end2 = p2 + rdlen2; - + end2 = p2 + rdlen2; + dp1 = dp2 = rr_desc; - + for (quit = 0, left1 = 0, left2 = 0, len1 = 0, len2 = 0; !quit;) { if (left1 != 0) memmove(buff1, buff1 + len1 - left1, left1); - + if ((len1 = get_rdata(header, plen, end1, buff1 + left1, (MAXDNAME * 2) - left1, &p1, &dp1)) == 0) { quit = 1; @@ -318,10 +319,10 @@ static int sort_rrset(struct dns_header *header, size_t plen, u16 *rr_desc, int memcpy(buff1 + left1, p1, len1); } len1 += left1; - + if (left2 != 0) memmove(buff2, buff2 + len2 - left2, left2); - + if ((len2 = get_rdata(header, plen, end2, buff2 + left2, (MAXDNAME *2) - left2, &p2, &dp2)) == 0) { quit = 1; @@ -329,14 +330,14 @@ static int sort_rrset(struct dns_header *header, size_t plen, u16 *rr_desc, int memcpy(buff2 + left2, p2, len2); } len2 += left2; - + if (len1 > len2) left1 = len1 - len2, left2 = 0, len = len2; else left2 = len2 - len1, left1 = 0, len = len1; - + rc = (len == 0) ? 0 : memcmp(buff1, buff2, len); - + if (rc > 0 || (rc == 0 && quit && len1 > len2)) { unsigned char *tmp = rrset[i+1]; @@ -365,58 +366,58 @@ static unsigned char **rrset = NULL, **sigs = NULL; /* Get pointers to RRset members and signature(s) for same. Check signatures, and return keyname associated in keyname. */ -static int explore_rrset(struct dns_header *header, size_t plen, int class, int type, +static int explore_rrset(struct dns_header *header, size_t plen, int class, int type, char *name, char *keyname, int *sigcnt, int *rrcnt) { - static int rrset_sz = 0, sig_sz = 0; + static int rrset_sz = 0, sig_sz = 0; unsigned char *p; int rrsetidx, sigidx, j, rdlen, res; int gotkey = 0; if (!(p = skip_questions(header, plen))) - return STAT_BOGUS; + return 0; /* look for RRSIGs for this RRset and get pointers to each RR in the set. */ - for (rrsetidx = 0, sigidx = 0, j = ntohs(header->ancount) + ntohs(header->nscount); - j != 0; j--) + for (rrsetidx = 0, sigidx = 0, j = ntohs(header->ancount) + ntohs(header->nscount); + j != 0; j--) { unsigned char *pstart, *pdata; int stype, sclass, type_covered; pstart = p; - + if (!(res = extract_name(header, plen, &p, name, 0, 10))) - return STAT_BOGUS; /* bad packet */ - + return 0; /* bad packet */ + GETSHORT(stype, p); GETSHORT(sclass, p); - p += 4; /* TTL */ - + pdata = p; + p += 4; /* TTL */ GETSHORT(rdlen, p); - + if (!CHECK_LEN(header, p, plen, rdlen)) - return 0; - + return 0; + if (res == 1 && sclass == class) { if (stype == type) { if (!expand_workspace(&rrset, &rrset_sz, rrsetidx)) - return 0; - + return 0; + rrset[rrsetidx++] = pstart; } - + if (stype == T_RRSIG) { if (rdlen < 18) - return 0; /* bad packet */ - + return 0; /* bad packet */ + GETSHORT(type_covered, p); p += 16; /* algo, labels, orig_ttl, sig_expiration, sig_inception, key_tag */ - + if (gotkey) { /* If there's more than one SIG, ensure they all have same keyname */ @@ -426,15 +427,15 @@ static int explore_rrset(struct dns_header *header, size_t plen, int class, int else { gotkey = 1; - + if (!extract_name(header, plen, &p, keyname, 1, 0)) return 0; - + /* RFC 4035 5.3.1 says that the Signer's Name field MUST equal the name of the zone containing the RRset. We can't tell that for certain, but we can check that the RRset name is equal to - or encloses the signers name, which should be enough to stop - an attacker using signatures made with the key of an unrelated + or encloses the signers name, which should be enough to stop + an attacker using signatures made with the key of an unrelated zone he controls. Note that the root key is always allowed. */ if (*keyname != 0) { @@ -446,35 +447,35 @@ static int explore_rrset(struct dns_header *header, size_t plen, int class, int return 0; } } - - + + if (type_covered == type) { if (!expand_workspace(&sigs, &sig_sz, sigidx)) - return 0; - + return 0; + sigs[sigidx++] = pdata; - } - - p = pdata + 2; /* restore for ADD_RDLEN */ + } + + p = pdata + 6; /* restore for ADD_RDLEN */ } } - + if (!ADD_RDLEN(header, p, plen, rdlen)) return 0; } - + *sigcnt = sigidx; *rrcnt = rrsetidx; return 1; } -/* Validate a single RRset (class, type, name) in the supplied DNS reply +/* Validate a single RRset (class, type, name) in the supplied DNS reply Return code: STAT_SECURE if it validates. STAT_SECURE_WILDCARD if it validates and is the result of wildcard expansion. - (In this case *wildcard_out points to the "body" of the wildcard within name.) + (In this case *wildcard_out points to the "body" of the wildcard within name.) STAT_BOGUS signature is wrong, bad packet. STAT_NEED_KEY need DNSKEY to complete validation (name is returned in keyname) STAT_NEED_DS need DS to complete validation (name is returned in keyname) @@ -485,26 +486,32 @@ static int explore_rrset(struct dns_header *header, size_t plen, int class, int Name is unchanged on exit. keyname is used as workspace and trashed. Call explore_rrset first to find and count RRs and sigs. + + ttl_out is the floor on TTL, based on TTL and orig_ttl and expiration of sig used to validate. */ -static int validate_rrset(time_t now, struct dns_header *header, size_t plen, int class, int type, int sigidx, int rrsetidx, - char *name, char *keyname, char **wildcard_out, struct blockdata *key, int keylen, int algo_in, int keytag_in) +static int validate_rrset(time_t now, struct dns_header *header, size_t plen, int class, int type, int sigidx, int rrsetidx, + char *name, char *keyname, char **wildcard_out, struct blockdata *key, int keylen, + int algo_in, int keytag_in, unsigned long *ttl_out) { unsigned char *p; - int rdlen, j, name_labels, algo, labels, orig_ttl, key_tag; + int rdlen, j, name_labels, algo, labels, key_tag; struct crec *crecp = NULL; u16 *rr_desc = rrfilter_desc(type); - u32 sig_expiration, sig_inception -; + u32 sig_expiration, sig_inception; + + unsigned long curtime = time(0); + int time_check = is_check_date(curtime); + if (wildcard_out) *wildcard_out = NULL; - + name_labels = count_labels(name); /* For 4035 5.3.2 check */ - /* Sort RRset records into canonical order. + /* Sort RRset records into canonical order. Note that at this point keyname and daemon->workspacename buffs are unused, and used as workspace by the sort. */ rrsetidx = sort_rrset(header, plen, rr_desc, rrsetidx, rrset, daemon->workspacename, keyname); - + /* Now try all the sigs to try and find one which validates */ for (j = 0; j = 18 checked previously */ psav = p; - + p += 2; /* type_covered - already checked */ algo = *p++; labels = *p++; @@ -526,11 +534,11 @@ static int validate_rrset(time_t now, struct dns_header *header, size_t plen, in GETLONG(sig_expiration, p); GETLONG(sig_inception, p); GETSHORT(key_tag, p); - + if (!extract_name(header, plen, &p, keyname, 1, 0)) return STAT_BOGUS; - if (!check_date_range(sig_inception, sig_expiration) || + if ((time_check && !check_date_range(curtime, sig_inception, sig_expiration)) || labels > name_labels || !(hash = hash_find(algo_digest_name(algo))) || !hash_init(hash, &ctx, &digest)) @@ -540,29 +548,41 @@ static int validate_rrset(time_t now, struct dns_header *header, size_t plen, in if (!key && !(crecp = cache_find_by_name(NULL, keyname, now, F_DNSKEY))) return STAT_NEED_KEY; + if (ttl_out) + { + /* 4035 5.3.3 rules on TTLs */ + if (orig_ttl < ttl) + ttl = orig_ttl; + + if (time_check && difftime(sig_expiration, curtime) < ttl) + ttl = difftime(sig_expiration, curtime); + + *ttl_out = ttl; + } + sig = p; sig_len = rdlen - (p - psav); - + nsigttl = htonl(orig_ttl); - + hash->update(ctx, 18, psav); wire_len = to_wire(keyname); hash->update(ctx, (unsigned int)wire_len, (unsigned char*)keyname); from_wire(keyname); - + for (i = 0; i < rrsetidx; ++i) { int seg; unsigned char *end, *cp; u16 len, *dp; - + p = rrset[i]; - - if (!extract_name(header, plen, &p, name, 1, 10)) + + if (!extract_name(header, plen, &p, name, 1, 10)) return STAT_BOGUS; name_start = name; - + /* if more labels than in RRsig name, hash *. 4035 5.3.2 */ if (labels < name_labels) { @@ -574,26 +594,26 @@ static int validate_rrset(time_t now, struct dns_header *header, size_t plen, in if (k != 1 && *name_start == '.') name_start++; } - + if (wildcard_out) *wildcard_out = name_start+1; name_start--; *name_start = '*'; } - + wire_len = to_wire(name_start); hash->update(ctx, (unsigned int)wire_len, (unsigned char *)name_start); hash->update(ctx, 4, p); /* class and type */ hash->update(ctx, 4, (unsigned char *)&nsigttl); - + p += 8; /* skip class, type, ttl */ GETSHORT(rdlen, p); if (!CHECK_LEN(header, p, plen, rdlen)) - return STAT_BOGUS; - + return STAT_BOGUS; + end = p + rdlen; - + /* canonicalise rdata and calculate length of same, use name buffer as workspace. Note that name buffer is twice MAXDNAME long in DNSSEC mode. */ cp = p; @@ -601,8 +621,8 @@ static int validate_rrset(time_t now, struct dns_header *header, size_t plen, in for (len = 0; (seg = get_rdata(header, plen, end, name, MAXDNAME * 2, &cp, &dp)) != 0; len += seg); len += end - cp; len = htons(len); - hash->update(ctx, 2, (unsigned char *)&len); - + hash->update(ctx, 2, (unsigned char *)&len); + /* Now canonicalise again and digest. */ cp = p; dp = rr_desc; @@ -611,9 +631,9 @@ static int validate_rrset(time_t now, struct dns_header *header, size_t plen, in if (cp != end) hash->update(ctx, end - cp, cp); } - + hash->digest(ctx, hash->digest_size, digest); - + /* namebuff used for workspace above, restore to leave unchanged on exit */ p = (unsigned char*)(rrset[0]); extract_name(header, plen, &p, name, 1, 0); @@ -628,7 +648,7 @@ static int validate_rrset(time_t now, struct dns_header *header, size_t plen, in { /* iterate through all possible keys 4035 5.3.1 */ for (; crecp; crecp = cache_find_by_name(crecp, keyname, now, F_DNSKEY)) - if (crecp->addr.key.algo == algo && + if (crecp->addr.key.algo == algo && crecp->addr.key.keytag == key_tag && crecp->uid == (unsigned int)class && verify(crecp->addr.key.keydata, crecp->addr.key.keylen, sig, sig_len, digest, hash->digest_size, algo)) @@ -638,7 +658,7 @@ static int validate_rrset(time_t now, struct dns_header *header, size_t plen, in return STAT_BOGUS; } - + /* The DNS packet is expected to contain the answer to a DNSKEY query. Put all DNSKEYs in the answer which are valid into the cache. @@ -646,24 +666,26 @@ static int validate_rrset(time_t now, struct dns_header *header, size_t plen, in STAT_OK Done, key(s) in cache. STAT_BOGUS No DNSKEYs found, which can be validated with DS, or self-sign for DNSKEY RRset is not valid, bad packet. - STAT_NEED_DS DS records to validate a key not found, name in keyname - STAT_NEED_KEY DNSKEY records to validate a key not found, name in keyname + STAT_NEED_DS DS records to validate a key not found, name in keyname + STAT_NEED_KEY DNSKEY records to validate a key not found, name in keyname */ int dnssec_validate_by_ds(time_t now, struct dns_header *header, size_t plen, char *name, char *keyname, int class) { unsigned char *psave, *p = (unsigned char *)(header+1); struct crec *crecp, *recp1; - int rc, j, qtype, qclass, ttl, rdlen, flags, algo, valid, keytag; + int rc, j, qtype, qclass, rdlen, flags, algo, valid, keytag; + unsigned long ttl, sig_ttl; struct blockdata *key; - struct all_addr a; + union all_addr a; if (ntohs(header->qdcount) != 1 || + RCODE(header) == SERVFAIL || RCODE(header) == REFUSED || !extract_name(header, plen, &p, name, 1, 4)) return STAT_BOGUS; GETSHORT(qtype, p); GETSHORT(qclass, p); - + if (qtype != T_DNSKEY || qclass != class || ntohs(header->ancount) == 0) return STAT_BOGUS; @@ -673,43 +695,43 @@ int dnssec_validate_by_ds(time_t now, struct dns_header *header, size_t plen, ch strcpy(keyname, name); return STAT_NEED_DS; } - + /* NOTE, we need to find ONE DNSKEY which matches the DS */ - for (valid = 0, j = ntohs(header->ancount); j != 0 && !valid; j--) + for (valid = 0, j = ntohs(header->ancount); j != 0 && !valid; j--) { /* Ensure we have type, class TTL and length */ if (!(rc = extract_name(header, plen, &p, name, 0, 10))) return STAT_BOGUS; /* bad packet */ - - GETSHORT(qtype, p); + + GETSHORT(qtype, p); GETSHORT(qclass, p); GETLONG(ttl, p); GETSHORT(rdlen, p); - + if (!CHECK_LEN(header, p, plen, rdlen) || rdlen < 4) return STAT_BOGUS; /* bad packet */ - + if (qclass != class || qtype != T_DNSKEY || rc == 2) { p += rdlen; continue; } - + psave = p; - + GETSHORT(flags, p); if (*p++ != 3) return STAT_BOGUS; algo = *p++; keytag = dnskey_keytag(algo, flags, p, rdlen - 4); key = NULL; - + /* key must have zone key flag set */ if (flags & 0x100) key = blockdata_alloc((char*)p, rdlen - 4); - + p = psave; - + if (!ADD_RDLEN(header, p, plen, rdlen)) { if (key) @@ -720,7 +742,7 @@ int dnssec_validate_by_ds(time_t now, struct dns_header *header, size_t plen, ch /* No zone key flag or malloc failure */ if (!key) continue; - + for (recp1 = crecp; recp1; recp1 = cache_find_by_name(recp1, name, now, F_DS)) { void *ctx; @@ -728,31 +750,31 @@ int dnssec_validate_by_ds(time_t now, struct dns_header *header, size_t plen, ch const struct nettle_hash *hash; int sigcnt, rrcnt; - if (recp1->addr.ds.algo == algo && + if (recp1->addr.ds.algo == algo && recp1->addr.ds.keytag == keytag && recp1->uid == (unsigned int)class && (hash = hash_find(ds_digest_name(recp1->addr.ds.digest))) && hash_init(hash, &ctx, &digest)) - + { int wire_len = to_wire(name); - - /* Note that digest may be different between DSs, so + + /* Note that digest may be different between DSs, so we can't move this outside the loop. */ hash->update(ctx, (unsigned int)wire_len, (unsigned char *)name); hash->update(ctx, (unsigned int)rdlen, psave); hash->digest(ctx, hash->digest_size, digest); - + from_wire(name); - + if (!(recp1->flags & F_NEG) && recp1->addr.ds.keylen == (int)hash->digest_size && - (ds_digest = blockdata_retrieve(recp1->addr.key.keydata, recp1->addr.ds.keylen, NULL)) && + (ds_digest = blockdata_retrieve(recp1->addr.ds.keydata, recp1->addr.ds.keylen, NULL)) && memcmp(ds_digest, digest, recp1->addr.ds.keylen) == 0 && explore_rrset(header, plen, class, T_DNSKEY, name, keyname, &sigcnt, &rrcnt) && sigcnt != 0 && rrcnt != 0 && - validate_rrset(now, header, plen, class, T_DNSKEY, sigcnt, rrcnt, name, keyname, - NULL, key, rdlen - 4, algo, keytag) == STAT_SECURE) + validate_rrset(now, header, plen, class, T_DNSKEY, sigcnt, rrcnt, name, keyname, + NULL, key, rdlen - 4, algo, keytag, &sig_ttl) == STAT_SECURE) { valid = 1; break; @@ -766,73 +788,74 @@ int dnssec_validate_by_ds(time_t now, struct dns_header *header, size_t plen, ch { /* DNSKEY RRset determined to be OK, now cache it. */ cache_start_insert(); - + p = skip_questions(header, plen); - for (j = ntohs(header->ancount); j != 0; j--) + for (j = ntohs(header->ancount); j != 0; j--) { /* Ensure we have type, class TTL and length */ if (!(rc = extract_name(header, plen, &p, name, 0, 10))) return STAT_BOGUS; /* bad packet */ - - GETSHORT(qtype, p); + + GETSHORT(qtype, p); GETSHORT(qclass, p); GETLONG(ttl, p); GETSHORT(rdlen, p); + /* TTL may be limited by sig. */ + if (sig_ttl < ttl) + ttl = sig_ttl; + if (!CHECK_LEN(header, p, plen, rdlen)) return STAT_BOGUS; /* bad packet */ - + if (qclass == class && rc == 1) { psave = p; - + if (qtype == T_DNSKEY) { if (rdlen < 4) return STAT_BOGUS; /* bad packet */ - + GETSHORT(flags, p); if (*p++ != 3) return STAT_BOGUS; algo = *p++; keytag = dnskey_keytag(algo, flags, p, rdlen - 4); - - /* Cache needs to known class for DNSSEC stuff */ - a.addr.dnssec.class = class; - + if ((key = blockdata_alloc((char*)p, rdlen - 4))) { - if (!(recp1 = cache_insert(name, &a, now, ttl, F_FORWARD | F_DNSKEY | F_DNSSECOK))) + a.key.keylen = rdlen - 4; + a.key.keydata = key; + a.key.algo = algo; + a.key.keytag = keytag; + a.key.flags = flags; + + if (!cache_insert(name, &a, class, now, ttl, F_FORWARD | F_DNSKEY | F_DNSSECOK)) { blockdata_free(key); return STAT_BOGUS; } else { - a.addr.log.keytag = keytag; - a.addr.log.algo = algo; + a.log.keytag = keytag; + a.log.algo = algo; if (algo_digest_name(algo)) log_query(F_NOEXTRA | F_KEYTAG | F_UPSTREAM, name, &a, "DNSKEY keytag %hu, algo %hu"); else log_query(F_NOEXTRA | F_KEYTAG | F_UPSTREAM, name, &a, "DNSKEY keytag %hu, algo %hu (not supported)"); - - recp1->addr.key.keylen = rdlen - 4; - recp1->addr.key.keydata = key; - recp1->addr.key.algo = algo; - recp1->addr.key.keytag = keytag; - recp1->addr.key.flags = flags; } } } - + p = psave; } if (!ADD_RDLEN(header, p, plen, rdlen)) return STAT_BOGUS; /* bad packet */ } - + /* commit cache insert. */ cache_end_insert(); return STAT_OK; @@ -844,7 +867,7 @@ int dnssec_validate_by_ds(time_t now, struct dns_header *header, size_t plen, ch /* The DNS packet is expected to contain the answer to a DS query Put all DSs in the answer which are valid into the cache. - Also handles replies which prove that there's no DS at this location, + Also handles replies which prove that there's no DS at this location, either because the zone is unsigned or this isn't a zone cut. These are cached too. return codes: @@ -857,33 +880,33 @@ int dnssec_validate_by_ds(time_t now, struct dns_header *header, size_t plen, ch int dnssec_validate_ds(time_t now, struct dns_header *header, size_t plen, char *name, char *keyname, int class) { unsigned char *p = (unsigned char *)(header+1); - int qtype, qclass, rc, i, neganswer, nons; + int qtype, qclass, rc, i, neganswer, nons, neg_ttl = 0; int aclass, atype, rdlen; unsigned long ttl; - struct all_addr a; + union all_addr a; if (ntohs(header->qdcount) != 1 || !(p = skip_name(p, header, plen, 4))) return STAT_BOGUS; - + GETSHORT(qtype, p); GETSHORT(qclass, p); if (qtype != T_DS || qclass != class) rc = STAT_BOGUS; else - rc = dnssec_validate_reply(now, header, plen, name, keyname, NULL, 0, &neganswer, &nons); - + rc = dnssec_validate_reply(now, header, plen, name, keyname, NULL, 0, &neganswer, &nons, &neg_ttl); + if (rc == STAT_INSECURE) { - my_syslog(LOG_WARNING, _("Insecure DS reply received, do upstream DNS servers support DNSSEC?")); + my_syslog(LOG_WARNING, _("Insecure DS reply received for %s, check domain configuration and upstream DNS server DNSSEC support"), name); rc = STAT_BOGUS; } - + p = (unsigned char *)(header+1); extract_name(header, plen, &p, name, 1, 4); p += 4; /* qtype, qclass */ - + /* If the key needed to validate the DS is on the same domain as the DS, we'll loop getting nowhere. Stop that now. This can happen of the DS answer comes from the DS's zone, and not the parent zone. */ @@ -892,69 +915,65 @@ int dnssec_validate_ds(time_t now, struct dns_header *header, size_t plen, char log_query(F_NOEXTRA | F_UPSTREAM, name, NULL, "BOGUS DS"); return STAT_BOGUS; } - + if (rc != STAT_SECURE) return rc; - + if (!neganswer) { cache_start_insert(); - + for (i = 0; i < ntohs(header->ancount); i++) { if (!(rc = extract_name(header, plen, &p, name, 0, 10))) return STAT_BOGUS; /* bad packet */ - + GETSHORT(atype, p); GETSHORT(aclass, p); GETLONG(ttl, p); GETSHORT(rdlen, p); - + if (!CHECK_LEN(header, p, plen, rdlen)) return STAT_BOGUS; /* bad packet */ - + if (aclass == class && atype == T_DS && rc == 1) - { + { int algo, digest, keytag; unsigned char *psave = p; struct blockdata *key; - struct crec *crecp; - + if (rdlen < 4) return STAT_BOGUS; /* bad packet */ - + GETSHORT(keytag, p); algo = *p++; digest = *p++; - - /* Cache needs to known class for DNSSEC stuff */ - a.addr.dnssec.class = class; - + if ((key = blockdata_alloc((char*)p, rdlen - 4))) { - if (!(crecp = cache_insert(name, &a, now, ttl, F_FORWARD | F_DS | F_DNSSECOK))) + a.ds.digest = digest; + a.ds.keydata = key; + a.ds.algo = algo; + a.ds.keytag = keytag; + a.ds.keylen = rdlen - 4; + + if (!cache_insert(name, &a, class, now, ttl, F_FORWARD | F_DS | F_DNSSECOK)) { blockdata_free(key); return STAT_BOGUS; } else { - a.addr.log.keytag = keytag; - a.addr.log.algo = algo; - a.addr.log.digest = digest; + a.log.keytag = keytag; + a.log.algo = algo; + a.log.digest = digest; if (ds_digest_name(digest) && algo_digest_name(algo)) log_query(F_NOEXTRA | F_KEYTAG | F_UPSTREAM, name, &a, "DS keytag %hu, algo %hu, digest %hu"); else log_query(F_NOEXTRA | F_KEYTAG | F_UPSTREAM, name, &a, "DS keytag %hu, algo %hu, digest %hu (not supported)"); - - crecp->addr.ds.digest = digest; - crecp->addr.ds.keydata = key; - crecp->addr.ds.algo = algo; - crecp->addr.ds.keytag = keytag; - crecp->addr.ds.keylen = rdlen - 4; - } + } } - + p = psave; } if (!ADD_RDLEN(header, p, plen, rdlen)) @@ -967,70 +986,26 @@ int dnssec_validate_ds(time_t now, struct dns_header *header, size_t plen, char else { int flags = F_FORWARD | F_DS | F_NEG | F_DNSSECOK; - unsigned long minttl = ULONG_MAX; - - if (!(p = skip_section(p, ntohs(header->ancount), header, plen))) - return STAT_BOGUS; - + if (RCODE(header) == NXDOMAIN) flags |= F_NXDOMAIN; - - /* We only cache validated DS records, DNSSECOK flag hijacked + + /* We only cache validated DS records, DNSSECOK flag hijacked to store presence/absence of NS. */ if (nons) flags &= ~F_DNSSECOK; - - for (i = ntohs(header->nscount); i != 0; i--) - { - if (!(p = skip_name(p, header, plen, 0))) - return STAT_BOGUS; - - GETSHORT(atype, p); - GETSHORT(aclass, p); - GETLONG(ttl, p); - GETSHORT(rdlen, p); - - if (!CHECK_LEN(header, p, plen, rdlen)) - return STAT_BOGUS; /* bad packet */ - - if (aclass != class || atype != T_SOA) - { - p += rdlen; - continue; - } - - if (ttl < minttl) - minttl = ttl; - - /* MNAME */ - if (!(p = skip_name(p, header, plen, 0))) - return STAT_BOGUS; - /* RNAME */ - if (!(p = skip_name(p, header, plen, 20))) - return STAT_BOGUS; - p += 16; /* SERIAL REFRESH RETRY EXPIRE */ - - GETLONG(ttl, p); /* minTTL */ - if (ttl < minttl) - minttl = ttl; - - break; - } - - if (i != 0) - { - cache_start_insert(); - - a.addr.dnssec.class = class; - if (!cache_insert(name, &a, now, ttl, flags)) - return STAT_BOGUS; - - cache_end_insert(); - - log_query(F_NOEXTRA | F_UPSTREAM, name, NULL, "no DS"); - } + + cache_start_insert(); + + /* Use TTL from NSEC for negative cache entries */ + if (!cache_insert(name, NULL, class, now, neg_ttl, flags)) + return STAT_BOGUS; + + cache_end_insert(); + + log_query(F_NOEXTRA | F_UPSTREAM, name, NULL, nons ? "no DS/cut" : "no DS"); } - + return STAT_OK; } @@ -1040,60 +1015,60 @@ static int hostname_cmp(const char *a, const char *b) { char *sa, *ea, *ca, *sb, *eb, *cb; unsigned char ac, bc; - + sa = ea = (char *)a + strlen(a); sb = eb = (char *)b + strlen(b); - + while (1) { while (sa != a && *(sa-1) != '.') sa--; - + while (sb != b && *(sb-1) != '.') sb--; ca = sa; cb = sb; - while (1) + while (1) { if (ca == ea) { if (cb == eb) break; - + return -1; } - + if (cb == eb) return 1; - + ac = (unsigned char) *ca++; bc = (unsigned char) *cb++; - + if (ac >= 'A' && ac <= 'Z') ac += 'a' - 'A'; if (bc >= 'A' && bc <= 'Z') bc += 'a' - 'A'; - + if (ac < bc) return -1; else if (ac != bc) return 1; } - + if (sa == a) { if (sb == b) return 0; - + return -1; } - + if (sb == b) return 1; - + ea = --sa; eb = --sb; } @@ -1109,7 +1084,7 @@ static int prove_non_existence_nsec(struct dns_header *header, size_t plen, unsi if (nons) *nons = 1; - + /* Find NSEC record that proves name doesn't exist */ for (i = 0; i < nsec_count; i++) { @@ -1140,13 +1115,13 @@ static int prove_non_existence_nsec(struct dns_header *header, size_t plen, unsi if (k != 1 && *workspace1 == '.') workspace1++; } - + workspace1--; *workspace1 = '*'; } - + rc = hostname_cmp(workspace1, name); - + if (rc == 0) { /* 4035 para 5.4. Last sentence */ @@ -1157,20 +1132,20 @@ static int prove_non_existence_nsec(struct dns_header *header, size_t plen, unsi that the type in question doesn't appear in the type map */ rdlen -= p - psave; /* rdlen is now length of type map, and p points to it */ - + /* If we can prove that there's no NS record, return that information. */ if (nons && rdlen >= 2 && p[0] == 0 && (p[2] & (0x80 >> T_NS)) != 0) *nons = 0; - + if (rdlen >= 2 && p[0] == 0) { - /* A CNAME answer would also be valid, so if there's a CNAME is should + /* A CNAME answer would also be valid, so if there's a CNAME is should have been returned. */ if ((p[2] & (0x80 >> T_CNAME)) != 0) return 0; - + /* If the SOA bit is set for a DS record, then we have the - DS from the wrong side of the delegation. For the root DS, + DS from the wrong side of the delegation. For the root DS, this is expected. */ if (name_labels != 0 && type == T_DS && (p[2] & (0x80 >> T_SOA)) != 0) return 0; @@ -1180,20 +1155,20 @@ static int prove_non_existence_nsec(struct dns_header *header, size_t plen, unsi { if (!CHECK_LEN(header, p, plen, rdlen)) return 0; - + if (p[0] == type >> 8) { /* Does the NSEC say our type exists? */ if (offset < p[1] && (p[offset+2] & mask) != 0) return 0; - + break; /* finished checking */ } - + rdlen -= p[1]; p += p[1]; } - + return 1; } else if (rc == -1) @@ -1203,19 +1178,19 @@ static int prove_non_existence_nsec(struct dns_header *header, size_t plen, unsi if (hostname_cmp(workspace2, name) >= 0 || hostname_cmp(workspace1, workspace2) >= 0) return 1; } - else + else { /* wrap around case, name falls between start and next domain name */ if (hostname_cmp(workspace1, workspace2) >= 0 && hostname_cmp(workspace2, name) >=0 ) return 1; } } - + return 0; } /* return digest length, or zero on error */ -static int hash_name(char *in, unsigned char **out, struct nettle_hash const *hash, +static int hash_name(char *in, unsigned char **out, struct nettle_hash const *hash, unsigned char *salt, int salt_len, int iterations) { void *ctx; @@ -1224,7 +1199,7 @@ static int hash_name(char *in, unsigned char **out, struct nettle_hash const *ha if (!hash_init(hash, &ctx, &digest)) return 0; - + hash->update(ctx, to_wire(in), (unsigned char *)in); hash->update(ctx, salt_len, salt); hash->digest(ctx, hash->digest_size, digest); @@ -1235,7 +1210,7 @@ static int hash_name(char *in, unsigned char **out, struct nettle_hash const *ha hash->update(ctx, salt_len, salt); hash->digest(ctx, hash->digest_size, digest); } - + from_wire(in); *out = digest; @@ -1247,8 +1222,8 @@ static int base32_decode(char *in, unsigned char *out) { int oc, on, c, mask, i; unsigned char *p = out; - - for (c = *in, oc = 0, on = 0; c != 0 && c != '.'; c = *++in) + + for (c = *in, oc = 0, on = 0; c != 0 && c != '.'; c = *++in) { if (c >= '0' && c <= '9') c -= '0'; @@ -1258,7 +1233,7 @@ static int base32_decode(char *in, unsigned char *out) c -= 'A', c += 10; else return 0; - + for (mask = 0x10, i = 0; i < 5; i++) { if (c & mask) @@ -1269,7 +1244,7 @@ static int base32_decode(char *in, unsigned char *out) oc = oc << 1; } } - + if ((on & 7) != 0) return 0; @@ -1288,7 +1263,7 @@ static int check_nsec3_coverage(struct dns_header *header, size_t plen, int dige if (!extract_name(header, plen, &p, workspace1, 1, 0) || !(base32_len = base32_decode(workspace1, (unsigned char *)workspace2))) return 0; - + p += 8; /* class, type, TTL */ GETSHORT(rdlen, p); psave = p; @@ -1298,10 +1273,10 @@ static int check_nsec3_coverage(struct dns_header *header, size_t plen, int dige salt_len = *p++; /* salt_len */ p += salt_len; /* salt */ hash_len = *p++; /* p now points to next hashed name */ - + if (!CHECK_LEN(header, p, plen, hash_len)) return 0; - + if (digest_len == base32_len && hash_len == base32_len) { int rc = memcmp(workspace2, digest, digest_len); @@ -1310,29 +1285,29 @@ static int check_nsec3_coverage(struct dns_header *header, size_t plen, int dige { /* We found an NSEC3 whose hashed name exactly matches the query, so we just need to check the type map. p points to the RR data for the record. */ - + int offset = (type & 0xff) >> 3; int mask = 0x80 >> (type & 0x07); - + p += hash_len; /* skip next-domain hash */ rdlen -= p - psave; if (!CHECK_LEN(header, p, plen, rdlen)) return 0; - + if (rdlen >= 2 && p[0] == 0) { /* If we can prove that there's no NS record, return that information. */ if (nons && (p[2] & (0x80 >> T_NS)) != 0) *nons = 0; - - /* A CNAME answer would also be valid, so if there's a CNAME is should + + /* A CNAME answer would also be valid, so if there's a CNAME is should have been returned. */ if ((p[2] & (0x80 >> T_CNAME)) != 0) return 0; - + /* If the SOA bit is set for a DS record, then we have the - DS from the wrong side of the delegation. For the root DS, + DS from the wrong side of the delegation. For the root DS, this is expected. */ if (name_labels != 0 && type == T_DS && (p[2] & (0x80 >> T_SOA)) != 0) return 0; @@ -1345,14 +1320,14 @@ static int check_nsec3_coverage(struct dns_header *header, size_t plen, int dige /* Does the NSEC3 say our type exists? */ if (offset < p[1] && (p[offset+2] & mask) != 0) return 0; - + break; /* finished checking */ } - + rdlen -= p[1]; p += p[1]; } - + return 1; } else if (rc < 0) @@ -1367,7 +1342,7 @@ static int check_nsec3_coverage(struct dns_header *header, size_t plen, int dige return 1; } } - else + else { /* wrap around case, name falls between start and next domain name */ if (memcmp(workspace2, p, digest_len) >= 0 && memcmp(p, digest, digest_len) >= 0) @@ -1391,25 +1366,25 @@ static int prove_non_existence_nsec3(struct dns_header *header, size_t plen, uns int digest_len, i, iterations, salt_len, base32_len, algo = 0; struct nettle_hash const *hash; char *closest_encloser, *next_closest, *wildcard; - + if (nons) *nons = 1; - - /* Look though the NSEC3 records to find the first one with + + /* Look though the NSEC3 records to find the first one with an algorithm we support. Take the algo, iterations, and salt of that record - as the ones we're going to use, and prune any + as the ones we're going to use, and prune any that don't match. */ - + for (i = 0; i < nsec_count; i++) { if (!(p = skip_name(nsecs[i], header, plen, 15))) return 0; /* bad packet */ - + p += 10; /* type, class, TTL, rdlen */ algo = *p++; - + if ((hash = hash_find(nsec3_digest_name(algo)))) break; /* known algo */ } @@ -1427,12 +1402,12 @@ static int prove_non_existence_nsec3(struct dns_header *header, size_t plen, uns to the largest bound, for 4096-bit keys. RFC 5155 10.3 */ if (iterations > 2500) return 0; - + salt_len = *p++; salt = p; if (!CHECK_LEN(header, salt, plen, salt_len)) return 0; /* bad packet */ - + /* Now prune so we only have NSEC3 records with same iterations, salt and algo */ for (i = 0; i < nsec_count; i++) { @@ -1440,17 +1415,17 @@ static int prove_non_existence_nsec3(struct dns_header *header, size_t plen, uns int this_iter, flags; nsecs[i] = NULL; /* Speculative, will be restored if OK. */ - + if (!(p = skip_name(nsec3p, header, plen, 15))) return 0; /* bad packet */ - + p += 10; /* type, class, TTL, rdlen */ - + if (*p++ != algo) continue; - + flags = *p++; /* flags */ - + /* 5155 8.2 */ if (flags != 0 && flags != 1) continue; @@ -1461,7 +1436,7 @@ static int prove_non_existence_nsec3(struct dns_header *header, size_t plen, uns if (salt_len != *p++) continue; - + if (!CHECK_LEN(header, p, plen, salt_len)) return 0; /* bad packet */ @@ -1474,11 +1449,11 @@ static int prove_non_existence_nsec3(struct dns_header *header, size_t plen, uns if ((digest_len = hash_name(name, &digest, hash, salt, salt_len, iterations)) == 0) return 0; - + if (check_nsec3_coverage(header, plen, digest_len, digest, type, workspace1, workspace2, nsecs, nsec_count, nons, count_labels(name))) return 1; - /* Can't find an NSEC3 which covers the name directly, we need the "closest encloser NSEC3" + /* Can't find an NSEC3 which covers the name directly, we need the "closest encloser NSEC3" or an answer inferred from a wildcard record. */ closest_encloser = name; next_closest = NULL; @@ -1493,84 +1468,93 @@ static int prove_non_existence_nsec3(struct dns_header *header, size_t plen, uns if ((digest_len = hash_name(closest_encloser, &digest, hash, salt, salt_len, iterations)) == 0) return 0; - + for (i = 0; i < nsec_count; i++) if ((p = nsecs[i])) { if (!extract_name(header, plen, &p, workspace1, 1, 0) || !(base32_len = base32_decode(workspace1, (unsigned char *)workspace2))) return 0; - + if (digest_len == base32_len && memcmp(digest, workspace2, digest_len) == 0) break; /* Gotit */ } - + if (i != nsec_count) break; - + next_closest = closest_encloser; } while ((closest_encloser = strchr(closest_encloser, '.'))); - + if (!closest_encloser || !next_closest) return 0; - + /* Look for NSEC3 that proves the non-existence of the next-closest encloser */ if ((digest_len = hash_name(next_closest, &digest, hash, salt, salt_len, iterations)) == 0) return 0; if (!check_nsec3_coverage(header, plen, digest_len, digest, type, workspace1, workspace2, nsecs, nsec_count, NULL, 1)) return 0; - + /* Finally, check that there's no seat of wildcard synthesis */ if (!wildname) { if (!(wildcard = strchr(next_closest, '.')) || wildcard == next_closest) return 0; - + wildcard--; *wildcard = '*'; - + if ((digest_len = hash_name(wildcard, &digest, hash, salt, salt_len, iterations)) == 0) return 0; - + if (!check_nsec3_coverage(header, plen, digest_len, digest, type, workspace1, workspace2, nsecs, nsec_count, NULL, 1)) return 0; } - + return 1; } -static int prove_non_existence(struct dns_header *header, size_t plen, char *keyname, char *name, int qtype, int qclass, char *wildname, int *nons) +static int prove_non_existence(struct dns_header *header, size_t plen, char *keyname, char *name, int qtype, int qclass, char *wildname, int *nons, int *nsec_ttl) { static unsigned char **nsecset = NULL, **rrsig_labels = NULL; static int nsecset_sz = 0, rrsig_labels_sz = 0; - + int type_found = 0; unsigned char *auth_start, *p = skip_questions(header, plen); int type, class, rdlen, i, nsecs_found; - + unsigned long ttl; + /* Move to NS section */ if (!p || !(p = skip_section(p, ntohs(header->ancount), header, plen))) return 0; auth_start = p; - - for (nsecs_found = 0, i = ntohs(header->nscount); i != 0; i--) + + for (nsecs_found = 0, i = 0; i < ntohs(header->nscount); i++) { unsigned char *pstart = p; - + if (!extract_name(header, plen, &p, daemon->workspacename, 1, 10)) return 0; - - GETSHORT(type, p); + + GETSHORT(type, p); GETSHORT(class, p); - p += 4; /* TTL */ + GETLONG(ttl, p); GETSHORT(rdlen, p); if (class == qclass && (type == T_NSEC || type == T_NSEC3)) { + if (nsec_ttl) + { + /* Limit TTL with sig TTL */ + if (daemon->rr_status[ntohs(header->ancount) + i] < ttl) + ttl = daemon->rr_status[ntohs(header->ancount) + i]; + *nsec_ttl = ttl; + } + /* No mixed NSECing 'round here, thankyouverymuch */ if (type_found != 0 && type_found != type) return 0; @@ -1578,11 +1562,11 @@ static int prove_non_existence(struct dns_header *header, size_t plen, char *key type_found = type; if (!expand_workspace(&nsecset, &nsecset_sz, nsecs_found)) - return 0; - + return 0; + if (type == T_NSEC) { - /* If we're looking for NSECs, find the corresponding SIGs, to + /* If we're looking for NSECs, find the corresponding SIGs, to extract the labels value, which we need in case the NSECs are the result of wildcard expansion. Note that the NSEC may not have been validated yet @@ -1591,30 +1575,30 @@ static int prove_non_existence(struct dns_header *header, size_t plen, char *key If there are no SIGs, that's an error */ unsigned char *p1 = auth_start; int res, j, rdlen1, type1, class1; - + if (!expand_workspace(&rrsig_labels, &rrsig_labels_sz, nsecs_found)) return 0; - + rrsig_labels[nsecs_found] = NULL; - + for (j = ntohs(header->nscount); j != 0; j--) { if (!(res = extract_name(header, plen, &p1, daemon->workspacename, 0, 10))) return 0; - GETSHORT(type1, p1); + GETSHORT(type1, p1); GETSHORT(class1, p1); p1 += 4; /* TTL */ GETSHORT(rdlen1, p1); if (!CHECK_LEN(header, p1, plen, rdlen1)) return 0; - + if (res == 1 && class1 == qclass && type1 == T_RRSIG) { int type_covered; unsigned char *psav = p1; - + if (rdlen1 < 18) return 0; /* bad packet */ @@ -1623,7 +1607,7 @@ static int prove_non_existence(struct dns_header *header, size_t plen, char *key if (type_covered == T_NSEC) { p1++; /* algo */ - + /* labels field must be the same in every SIG we find. */ if (!rrsig_labels[nsecs_found]) rrsig_labels[nsecs_found] = p1; @@ -1632,7 +1616,7 @@ static int prove_non_existence(struct dns_header *header, size_t plen, char *key } p1 = psav; } - + if (!ADD_RDLEN(header, p1, plen, rdlen1)) return 0; } @@ -1642,13 +1626,13 @@ static int prove_non_existence(struct dns_header *header, size_t plen, char *key return 0; } - nsecset[nsecs_found++] = pstart; + nsecset[nsecs_found++] = pstart; } - + if (!ADD_RDLEN(header, p, plen, rdlen)) return 0; } - + if (type_found == T_NSEC) return prove_non_existence_nsec(header, plen, nsecset, rrsig_labels, nsecs_found, daemon->workspacename, keyname, name, qtype, nons); else if (type_found == T_NSEC3) @@ -1691,10 +1675,10 @@ static int zone_status(char *name, int class, char *keyname, time_t now) while (1) { strcpy(keyname, &name[name_start]); - + if (!(crecp = cache_find_by_name(NULL, keyname, now, F_DS))) return STAT_NEED_DS; - + /* F_DNSSECOK misused in DS cache records to non-existence of NS record. F_NEG && !F_DNSSECOK implies that we've proved there's no DS record here, but that's because there's no NS record either, ie this isn't the start @@ -1715,7 +1699,7 @@ static int zone_status(char *name, int class, char *keyname, time_t now) to assume we can validate the zone and if we can't later, because an RRSIG is missing we return BOGUS. */ - do + do { if (crecp->uid == (unsigned int)class && ds_digest_name(crecp->addr.ds.digest) && @@ -1732,17 +1716,17 @@ static int zone_status(char *name, int class, char *keyname, time_t now) break; for (p = &name[name_start-2]; (*p != '.') && (p != name); p--); - + if (p != name) p++; - + name_start = p - name; - } + } return STAT_SECURE; } - -/* Validate all the RRsets in the answer and authority sections of the reply (4035:3.2.3) + +/* Validate all the RRsets in the answer and authority sections of the reply (4035:3.2.3) Return code: STAT_SECURE if it validates. STAT_INSECURE at least one RRset not validated, because in unsigned zone. @@ -1750,11 +1734,15 @@ static int zone_status(char *name, int class, char *keyname, time_t now) STAT_NEED_KEY need DNSKEY to complete validation (name is returned in keyname, class in *class) STAT_NEED_DS need DS to complete validation (name is returned in keyname) - daemon->rr_status points to a char array which corressponds to the RRs in the - answer section (only). This is set to 1 for each RR which is validated, and 0 for any which aren't. + daemon->rr_status points to a char array which corressponds to the RRs in the + answer and auth sections. This is set to 1 for each RR which is validated, and 0 for any which aren't. + + When validating replies to DS records, we're only interested in the NSEC{3} RRs in the auth section. + Other RRs in that section missing sigs will not cause am INSECURE reply. We determine this mode + is the nons argument is non-NULL. */ -int dnssec_validate_reply(time_t now, struct dns_header *header, size_t plen, char *name, char *keyname, - int *class, int check_unsigned, int *neganswer, int *nons) +int dnssec_validate_reply(time_t now, struct dns_header *header, size_t plen, char *name, char *keyname, + int *class, int check_unsigned, int *neganswer, int *nons, int *nsec_ttl) { static unsigned char **targets = NULL; static int target_sz = 0; @@ -1765,140 +1753,149 @@ int dnssec_validate_reply(time_t now, struct dns_header *header, size_t plen, ch int secure = STAT_SECURE; /* extend rr_status if necessary */ - if (daemon->rr_status_sz < ntohs(header->ancount)) + if (daemon->rr_status_sz < ntohs(header->ancount) + ntohs(header->nscount)) { - char *new = whine_malloc(ntohs(header->ancount) + 64); + unsigned long *new = whine_malloc(sizeof(*daemon->rr_status) * (ntohs(header->ancount) + ntohs(header->nscount) + 64)); if (!new) return STAT_BOGUS; free(daemon->rr_status); daemon->rr_status = new; - daemon->rr_status_sz = ntohs(header->ancount) + 64; + daemon->rr_status_sz = ntohs(header->ancount) + ntohs(header->nscount) + 64; } - - memset(daemon->rr_status, 0, ntohs(header->ancount)); - + + memset(daemon->rr_status, 0, sizeof(*daemon->rr_status) * daemon->rr_status_sz); + if (neganswer) *neganswer = 0; - + if (RCODE(header) == SERVFAIL || ntohs(header->qdcount) != 1) return STAT_BOGUS; - + if (RCODE(header) != NXDOMAIN && RCODE(header) != NOERROR) return STAT_INSECURE; p1 = (unsigned char *)(header+1); - + /* Find all the targets we're looking for answers to. The zeroth array element is for the query, subsequent ones for CNAME targets, unless the query is for a CNAME. */ if (!expand_workspace(&targets, &target_sz, 0)) return STAT_BOGUS; - + targets[0] = p1; targetidx = 1; - + if (!extract_name(header, plen, &p1, name, 1, 4)) return STAT_BOGUS; - + GETSHORT(qtype, p1); GETSHORT(qclass, p1); ans_start = p1; - + /* Can't validate an RRSIG query */ if (qtype == T_RRSIG) return STAT_INSECURE; - + if (qtype != T_CNAME) - for (j = ntohs(header->ancount); j != 0; j--) + for (j = ntohs(header->ancount); j != 0; j--) { if (!(p1 = skip_name(p1, header, plen, 10))) return STAT_BOGUS; /* bad packet */ - - GETSHORT(type2, p1); + + GETSHORT(type2, p1); p1 += 6; /* class, TTL */ - GETSHORT(rdlen2, p1); - + GETSHORT(rdlen2, p1); + if (type2 == T_CNAME) { if (!expand_workspace(&targets, &target_sz, targetidx)) return STAT_BOGUS; - + targets[targetidx++] = p1; /* pointer to target name */ } - + if (!ADD_RDLEN(header, p1, plen, rdlen2)) return STAT_BOGUS; } - + for (p1 = ans_start, i = 0; i < ntohs(header->ancount) + ntohs(header->nscount); i++) { if (i != 0 && !ADD_RDLEN(header, p1, plen, rdlen1)) return STAT_BOGUS; - + if (!extract_name(header, plen, &p1, name, 1, 10)) return STAT_BOGUS; /* bad packet */ - + GETSHORT(type1, p1); GETSHORT(class1, p1); p1 += 4; /* TTL */ GETSHORT(rdlen1, p1); - + /* Don't try and validate RRSIGs! */ if (type1 == T_RRSIG) continue; - + /* Check if we've done this RRset already */ for (p2 = ans_start, j = 0; j < i; j++) { if (!(rc = extract_name(header, plen, &p2, name, 0, 10))) return STAT_BOGUS; /* bad packet */ - + GETSHORT(type2, p2); GETSHORT(class2, p2); p2 += 4; /* TTL */ GETSHORT(rdlen2, p2); - + if (type2 == type1 && class2 == class1 && rc == 1) break; /* Done it before: name, type, class all match. */ - + if (!ADD_RDLEN(header, p2, plen, rdlen2)) return STAT_BOGUS; } - + + /* Done already: copy the validation status */ if (j != i) - { - /* Done already: copy the validation status */ - if (i < ntohs(header->ancount)) - daemon->rr_status[i] = daemon->rr_status[j]; - } + daemon->rr_status[i] = daemon->rr_status[j]; else { /* Not done, validate now */ int sigcnt, rrcnt; char *wildname; - + if (!explore_rrset(header, plen, class1, type1, name, keyname, &sigcnt, &rrcnt)) return STAT_BOGUS; - + /* No signatures for RRset. We can be configured to assume this is OK and return an INSECURE result. */ if (sigcnt == 0) { - if (check_unsigned) + /* NSEC and NSEC3 records must be signed. We make this assumption elsewhere. */ + if (type1 == T_NSEC || type1 == T_NSEC3) + rc = STAT_INSECURE; + else if (nons && i >= ntohs(header->ancount)) + /* If we're validating a DS reply, rather than looking for the value of AD bit, + we only care that NSEC and NSEC3 RRs in the auth section are signed. + Return SECURE even if others (SOA....) are not. */ + rc = STAT_SECURE; + else { - rc = zone_status(name, class1, keyname, now); - if (rc == STAT_SECURE) - rc = STAT_BOGUS; - if (class) - *class = class1; /* Class for NEED_DS or NEED_KEY */ + /* unsigned RRsets in auth section are not BOGUS, but do make reply insecure. */ + if (check_unsigned && i < ntohs(header->ancount)) + { + rc = zone_status(name, class1, keyname, now); + if (rc == STAT_SECURE) + rc = STAT_BOGUS; + if (class) + *class = class1; /* Class for NEED_DS or NEED_KEY */ + } + else + rc = STAT_INSECURE; + + if (rc != STAT_INSECURE) + return rc; } - else - rc = STAT_INSECURE; - - if (rc != STAT_INSECURE) - return rc; } else { @@ -1906,33 +1903,33 @@ int dnssec_validate_reply(time_t now, struct dns_header *header, size_t plen, ch Can't overwrite name here. */ strcpy(daemon->workspacename, keyname); rc = zone_status(daemon->workspacename, class1, keyname, now); - + if (rc == STAT_BOGUS || rc == STAT_NEED_KEY || rc == STAT_NEED_DS) { if (class) *class = class1; /* Class for NEED_DS or NEED_KEY */ return rc; } - + /* Zone is insecure, don't need to validate RRset */ if (rc == STAT_SECURE) { + unsigned long sig_ttl; rc = validate_rrset(now, header, plen, class1, type1, sigcnt, - rrcnt, name, keyname, &wildname, NULL, 0, 0, 0); - + rrcnt, name, keyname, &wildname, NULL, 0, 0, 0, &sig_ttl); + if (rc == STAT_BOGUS || rc == STAT_NEED_KEY || rc == STAT_NEED_DS) { if (class) *class = class1; /* Class for DS or DNSKEY */ return rc; - } - + } + /* rc is now STAT_SECURE or STAT_SECURE_WILDCARD */ - + /* Note that RR is validated */ - if (i < ntohs(header->ancount)) - daemon->rr_status[i] = 1; - + daemon->rr_status[i] = sig_ttl; + /* Note if we've validated either the answer to the question or the target of a CNAME. Any not noted will need NSEC or to be in unsigned space. */ @@ -1942,20 +1939,20 @@ int dnssec_validate_reply(time_t now, struct dns_header *header, size_t plen, ch int rc1; if (!(rc1 = extract_name(header, plen, &p2, name, 0, 10))) return STAT_BOGUS; /* bad packet */ - + if (class1 == qclass && rc1 == 1 && (type1 == T_CNAME || type1 == qtype || qtype == T_ANY )) targets[j] = NULL; } - + /* An attacker replay a wildcard answer with a different answer and overlay a genuine RR. To prove this hasn't happened, the answer must prove that - the genuine record doesn't exist. Check that here. - Note that we may not yet have validated the NSEC/NSEC3 RRsets. + the genuine record doesn't exist. Check that here. + Note that we may not yet have validated the NSEC/NSEC3 RRsets. That's not a problem since if the RRsets later fail we'll return BOGUS then. */ if (rc == STAT_SECURE_WILDCARD && - !prove_non_existence(header, plen, keyname, name, type1, class1, wildname, NULL)) + !prove_non_existence(header, plen, keyname, name, type1, class1, wildname, NULL, NULL)) return STAT_BOGUS; rc = STAT_SECURE; @@ -1974,31 +1971,31 @@ int dnssec_validate_reply(time_t now, struct dns_header *header, size_t plen, ch { if (neganswer) *neganswer = 1; - + if (!extract_name(header, plen, &p2, name, 1, 10)) return STAT_BOGUS; /* bad packet */ - + /* NXDOMAIN or NODATA reply, unanswered question is (name, qclass, qtype) */ - + /* For anything other than a DS record, this situation is OK if either the answer is in an unsigned zone, or there's a NSEC records. */ - if (!prove_non_existence(header, plen, keyname, name, qtype, qclass, NULL, nons)) + if (!prove_non_existence(header, plen, keyname, name, qtype, qclass, NULL, nons, nsec_ttl)) { /* Empty DS without NSECS */ if (qtype == T_DS) return STAT_BOGUS; - + if ((rc = zone_status(name, qclass, keyname, now)) != STAT_SECURE) { if (class) *class = qclass; /* Class for NEED_DS or NEED_KEY */ return rc; - } - + } + return STAT_BOGUS; /* signed zone, no NSECs */ } } - + return secure; } @@ -2025,7 +2022,7 @@ int dnskey_keytag(int alg, int flags, unsigned char *key, int keylen) } } -size_t dnssec_generate_query(struct dns_header *header, unsigned char *end, char *name, int class, +size_t dnssec_generate_query(struct dns_header *header, unsigned char *end, char *name, int class, int type, int edns_pktsz) { unsigned char *p; @@ -2036,7 +2033,7 @@ size_t dnssec_generate_query(struct dns_header *header, unsigned char *end, char header->nscount = htons(0); header->arcount = htons(0); - header->hb3 = HB3_RD; + header->hb3 = HB3_RD; SET_OPCODE(header, QUERY); /* For debugging, set Checking Disabled, otherwise, have the upstream check too, this allows it to select auth servers when one is returning bad data. */ @@ -2045,7 +2042,7 @@ size_t dnssec_generate_query(struct dns_header *header, unsigned char *end, char /* ID filled in later */ p = (unsigned char *)(header+1); - + p = do_rfc1035_name(p, name, NULL); *p++ = 0; PUTSHORT(type, p); @@ -2067,15 +2064,15 @@ unsigned char* hash_questions(struct dns_header *header, size_t plen, char *name const struct nettle_hash *hash; void *ctx; unsigned char *digest; - + if (!(hash = hash_find("sha1")) || !hash_init(hash, &ctx, &digest)) return NULL; - - for (q = ntohs(header->qdcount); q != 0; q--) + + for (q = ntohs(header->qdcount); q != 0; q--) { if (!extract_name(header, plen, &p, name, 1, 4)) break; /* bad packet */ - + len = to_wire(name); hash->update(ctx, len, (unsigned char *)name); /* CRC the class and type as well */ @@ -2085,7 +2082,7 @@ unsigned char* hash_questions(struct dns_header *header, size_t plen, char *name if (!CHECK_LEN(header, p, plen, 0)) break; /* bad packet */ } - + hash->digest(ctx, hash->digest_size, digest); return digest; } diff --git a/dnsmasq/domain.c b/src/dnsmasq/domain.c similarity index 78% rename from dnsmasq/domain.c rename to src/dnsmasq/domain.c index 6a4382b20..3dc96abc0 100644 --- a/dnsmasq/domain.c +++ b/src/dnsmasq/domain.c @@ -1,4 +1,4 @@ -/* dnsmasq is Copyright (c) 2000-2018 Simon Kelley +/* dnsmasq is Copyright (c) 2000-2020 Simon Kelley This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -18,21 +18,14 @@ static struct cond_domain *search_domain(struct in_addr addr, struct cond_domain *c); -#ifdef HAVE_IPV6 static struct cond_domain *search_domain6(struct in6_addr *addr, struct cond_domain *c); -#endif -int is_name_synthetic(int flags, char *name, struct all_addr *addr) +int is_name_synthetic(int flags, char *name, union all_addr *addr) { char *p; struct cond_domain *c = NULL; - int prot = AF_INET; - -#ifdef HAVE_IPV6 - if (flags & F_IPV6) - prot = AF_INET6; -#endif + int prot = (flags & F_IPV6) ? AF_INET6 : AF_INET; for (c = daemon->synth_domains; c; c = c->next) { @@ -52,7 +45,7 @@ int is_name_synthetic(int flags, char *name, struct all_addr *addr) if (c1 != c2) break; } - + if (pref && *pref != 0) continue; /* prefix match fail */ @@ -61,16 +54,16 @@ int is_name_synthetic(int flags, char *name, struct all_addr *addr) for (p = tail; *p; p++) { char c = *p; - + if (c < '0' || c > '9') break; } - + if (*p != '.') continue; - + *p = 0; - + if (hostname_isequal(c->domain, p+1)) { if (prot == AF_INET) @@ -80,25 +73,23 @@ int is_name_synthetic(int flags, char *name, struct all_addr *addr) if (!c->is6 && index <= ntohl(c->end.s_addr) - ntohl(c->start.s_addr)) { - addr->addr.addr4.s_addr = htonl(ntohl(c->start.s_addr) + index); + addr->addr4.s_addr = htonl(ntohl(c->start.s_addr) + index); found = 1; } - } -#ifdef HAVE_IPV6 + } else { u64 index = atoll(tail); - + if (c->is6 && index <= addr6part(&c->end6) - addr6part(&c->start6)) { u64 start = addr6part(&c->start6); - addr->addr.addr6 = c->start6; - setaddr6part(&addr->addr.addr6, start + index); + addr->addr6 = c->start6; + setaddr6part(&addr->addr6, start + index); found = 1; } } -#endif } } else @@ -107,24 +98,21 @@ int is_name_synthetic(int flags, char *name, struct all_addr *addr) for (p = tail; *p; p++) { char c = *p; - + if ((c >='0' && c <= '9') || c == '-') continue; - -#ifdef HAVE_IPV6 - if (prot == AF_INET6 && ((c >='A' && c <= 'F') || (c >='a' && c <= 'f'))) + + if (prot == AF_INET6 && ((c >='A' && c <= 'F') || (c >='a' && c <= 'f'))) continue; -#endif - + break; } - + if (*p != '.') continue; - - *p = 0; - -#ifdef HAVE_IPV6 + + *p = 0; + if (prot == AF_INET6 && strstr(tail, "--ffff-") == tail) { /* special hack for v4-mapped. */ @@ -134,7 +122,6 @@ int is_name_synthetic(int flags, char *name, struct all_addr *addr) *p = '.'; } else -#endif { /* swap . or : for - */ for (p = tail; *p; p++) @@ -142,34 +129,30 @@ int is_name_synthetic(int flags, char *name, struct all_addr *addr) { if (prot == AF_INET) *p = '.'; -#ifdef HAVE_IPV6 else *p = ':'; -#endif } } - + if (hostname_isequal(c->domain, p+1) && inet_pton(prot, tail, addr)) { if (prot == AF_INET) { if (!c->is6 && - ntohl(addr->addr.addr4.s_addr) >= ntohl(c->start.s_addr) && - ntohl(addr->addr.addr4.s_addr) <= ntohl(c->end.s_addr)) + ntohl(addr->addr4.s_addr) >= ntohl(c->start.s_addr) && + ntohl(addr->addr4.s_addr) <= ntohl(c->end.s_addr)) found = 1; } -#ifdef HAVE_IPV6 else { - u64 addrpart = addr6part(&addr->addr.addr6); - + u64 addrpart = addr6part(&addr->addr6); + if (c->is6 && - is_same_net6(&addr->addr.addr6, &c->start6, 64) && + is_same_net6(&addr->addr6, &c->start6, 64) && addrpart >= addr6part(&c->start6) && addrpart <= addr6part(&c->end6)) found = 1; } -#endif } } @@ -178,10 +161,10 @@ int is_name_synthetic(int flags, char *name, struct all_addr *addr) for (p = tail; *p; p++) if (*p == '.' || *p == ':') *p = '-'; - + *p = '.'; - - + + if (found) return 1; } @@ -190,68 +173,67 @@ int is_name_synthetic(int flags, char *name, struct all_addr *addr) } -int is_rev_synth(int flag, struct all_addr *addr, char *name) +int is_rev_synth(int flag, union all_addr *addr, char *name) { struct cond_domain *c; - if (flag & F_IPV4 && (c = search_domain(addr->addr.addr4, daemon->synth_domains))) + if (flag & F_IPV4 && (c = search_domain(addr->addr4, daemon->synth_domains))) { char *p; - + *name = 0; if (c->indexed) { - unsigned int index = ntohl(addr->addr.addr4.s_addr) - ntohl(c->start.s_addr); + unsigned int index = ntohl(addr->addr4.s_addr) - ntohl(c->start.s_addr); snprintf(name, MAXDNAME, "%s%u", c->prefix ? c->prefix : "", index); } else { if (c->prefix) strncpy(name, c->prefix, MAXDNAME - ADDRSTRLEN); - - inet_ntop(AF_INET, &addr->addr.addr4, name + strlen(name), ADDRSTRLEN); + + inet_ntop(AF_INET, &addr->addr4, name + strlen(name), ADDRSTRLEN); for (p = name; *p; p++) if (*p == '.') *p = '-'; } - + strncat(name, ".", MAXDNAME); strncat(name, c->domain, MAXDNAME); return 1; } -#ifdef HAVE_IPV6 - if (flag & F_IPV6 && (c = search_domain6(&addr->addr.addr6, daemon->synth_domains))) + if ((flag & F_IPV6) && (c = search_domain6(&addr->addr6, daemon->synth_domains))) { char *p; - + *name = 0; if (c->indexed) { - u64 index = addr6part(&addr->addr.addr6) - addr6part(&c->start6); + u64 index = addr6part(&addr->addr6) - addr6part(&c->start6); snprintf(name, MAXDNAME, "%s%llu", c->prefix ? c->prefix : "", index); } else { if (c->prefix) strncpy(name, c->prefix, MAXDNAME - ADDRSTRLEN); - - inet_ntop(AF_INET6, &addr->addr.addr6, name + strlen(name), ADDRSTRLEN); + + inet_ntop(AF_INET6, &addr->addr6, name + strlen(name), ADDRSTRLEN); /* IPv6 presentation address can start with ":", but valid domain names cannot start with "-" so prepend a zero in that case. */ if (!c->prefix && *name == ':') { *name = '0'; - inet_ntop(AF_INET6, &addr->addr.addr6, name+1, ADDRSTRLEN); + inet_ntop(AF_INET6, &addr->addr6, name+1, ADDRSTRLEN); } - + /* V4-mapped have periods.... */ for (p = name; *p; p++) if (*p == ':' || *p == '.') *p = '-'; - + } strncat(name, ".", MAXDNAME); @@ -259,7 +241,6 @@ int is_rev_synth(int flag, struct all_addr *addr, char *name) return 1; } -#endif return 0; } @@ -286,7 +267,7 @@ char *get_domain(struct in_addr addr) return daemon->domain_suffix; } -#ifdef HAVE_IPV6 + static struct cond_domain *search_domain6(struct in6_addr *addr, struct cond_domain *c) { u64 addrpart = addr6part(addr); @@ -310,4 +291,3 @@ char *get_domain6(struct in6_addr *addr) return daemon->domain_suffix; } -#endif diff --git a/dnsmasq/dump.c b/src/dnsmasq/dump.c similarity index 94% rename from dnsmasq/dump.c rename to src/dnsmasq/dump.c index 4279815e2..9bd3a5f9b 100644 --- a/dnsmasq/dump.c +++ b/src/dnsmasq/dump.c @@ -1,15 +1,15 @@ -/* dnsmasq is Copyright (c) 2000-2018 Simon Kelley +/* dnsmasq is Copyright (c) 2000-2020 Simon Kelley This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; version 2 dated June, 1991, or (at your option) version 3 dated 29 June, 2007. - + This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. - + You should have received a copy of the GNU General Public License along with this program. If not, see . */ @@ -46,7 +46,7 @@ void dump_init(void) struct pcaprec_hdr_s pcap_header; packet_count = 0; - + if (stat(daemon->dump_file, &buf) == -1) { /* doesn't exist, create and add header */ @@ -82,10 +82,8 @@ void dump_init(void) void dump_packet(int mask, void *packet, size_t len, union mysockaddr *src, union mysockaddr *dst) { struct ip ip; -#ifdef HAVE_IPV6 struct ip6_hdr ip6; int family; -#endif struct udphdr { u16 uh_sport; /* source port */ u16 uh_dport; /* destination port */ @@ -98,14 +96,13 @@ void dump_packet(int mask, void *packet, size_t len, union mysockaddr *src, unio void *iphdr; size_t ipsz; int rc; - + if (daemon->dumpfd == -1 || !(mask & daemon->dump_mask)) return; - + /* So wireshark can Id the packet. */ udp.uh_sport = udp.uh_dport = htons(NAMESERVER_PORT); -#ifdef HAVE_IPV6 if (src) family = src->sa.sa_family; else @@ -116,7 +113,7 @@ void dump_packet(int mask, void *packet, size_t len, union mysockaddr *src, unio iphdr = &ip6; ipsz = sizeof(ip6); memset(&ip6, 0, sizeof(ip6)); - + ip6.ip6_vfc = 6 << 4; ip6.ip6_plen = htons(sizeof(struct udphdr) + len); ip6.ip6_nxt = IPPROTO_UDP; @@ -127,30 +124,33 @@ void dump_packet(int mask, void *packet, size_t len, union mysockaddr *src, unio memcpy(&ip6.ip6_src, &src->in6.sin6_addr, IN6ADDRSZ); udp.uh_sport = src->in6.sin6_port; } - + if (dst) { memcpy(&ip6.ip6_dst, &dst->in6.sin6_addr, IN6ADDRSZ); udp.uh_dport = dst->in6.sin6_port; } - + /* start UDP checksum */ - for (sum = 0, i = 0; i < IN6ADDRSZ; i++) - sum += ((u16 *)&ip6.ip6_src)[i]; + for (sum = 0, i = 0; i < IN6ADDRSZ; i+=2) + { + sum += ip6.ip6_src.s6_addr[i] + (ip6.ip6_src.s6_addr[i+1] << 8) ; + sum += ip6.ip6_dst.s6_addr[i] + (ip6.ip6_dst.s6_addr[i+1] << 8) ; + + } } else -#endif { iphdr = &ip; ipsz = sizeof(ip); memset(&ip, 0, sizeof(ip)); - + ip.ip_v = IPVERSION; ip.ip_hl = sizeof(struct ip) / 4; - ip.ip_len = htons(sizeof(struct ip) + sizeof(struct udphdr) + len); + ip.ip_len = htons(sizeof(struct ip) + sizeof(struct udphdr) + len); ip.ip_ttl = IPDEFTTL; ip.ip_p = IPPROTO_UDP; - + if (src) { ip.ip_src = src->in.sin_addr; @@ -162,21 +162,21 @@ void dump_packet(int mask, void *packet, size_t len, union mysockaddr *src, unio ip.ip_dst = dst->in.sin_addr; udp.uh_dport = dst->in.sin_port; } - + ip.ip_sum = 0; for (sum = 0, i = 0; i < sizeof(struct ip) / 2; i++) sum += ((u16 *)&ip)[i]; while (sum >> 16) - sum = (sum & 0xffff) + (sum >> 16); + sum = (sum & 0xffff) + (sum >> 16); ip.ip_sum = (sum == 0xffff) ? sum : ~sum; - + /* start UDP checksum */ sum = ip.ip_src.s_addr & 0xffff; sum += (ip.ip_src.s_addr >> 16) & 0xffff; sum += ip.ip_dst.s_addr & 0xffff; sum += (ip.ip_dst.s_addr >> 16) & 0xffff; } - + if (len & 1) ((unsigned char *)packet)[len] = 0; /* for checksum, in case length is odd. */ @@ -196,7 +196,7 @@ void dump_packet(int mask, void *packet, size_t len, union mysockaddr *src, unio pcap_header.ts_sec = time.tv_sec; pcap_header.ts_usec = time.tv_usec; pcap_header.incl_len = pcap_header.orig_len = ipsz + sizeof(udp) + len; - + if (rc == -1 || !read_write(daemon->dumpfd, (void *)&pcap_header, sizeof(pcap_header), 0) || !read_write(daemon->dumpfd, iphdr, ipsz, 0) || diff --git a/dnsmasq/edns0.c b/src/dnsmasq/edns0.c similarity index 97% rename from dnsmasq/edns0.c rename to src/dnsmasq/edns0.c index af3387761..d75d3cc1f 100644 --- a/dnsmasq/edns0.c +++ b/src/dnsmasq/edns0.c @@ -1,4 +1,4 @@ -/* dnsmasq is Copyright (c) 2000-2018 Simon Kelley +/* dnsmasq is Copyright (c) 2000-2020 Simon Kelley This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -301,20 +301,14 @@ static size_t add_mac(struct dns_header *header, size_t plen, unsigned char *lim struct subnet_opt { u16 family; - u8 source_netmask, scope_netmask; -#ifdef HAVE_IPV6 + u8 source_netmask, scope_netmask; u8 addr[IN6ADDRSZ]; -#else - u8 addr[INADDRSZ]; -#endif }; static void *get_addrp(union mysockaddr *addr, const short family) { -#ifdef HAVE_IPV6 if (family == AF_INET6) return &addr->in6.sin6_addr; -#endif return &addr->in.sin_addr; } @@ -330,7 +324,6 @@ static size_t calc_subnet_opt(struct subnet_opt *opt, union mysockaddr *source) opt->source_netmask = 0; opt->scope_netmask = 0; -#ifdef HAVE_IPV6 if (source->sa.sa_family == AF_INET6 && daemon->add_subnet6) { opt->source_netmask = daemon->add_subnet6->mask; @@ -342,7 +335,6 @@ static size_t calc_subnet_opt(struct subnet_opt *opt, union mysockaddr *source) else addrp = &source->in6.sin6_addr; } -#endif if (source->sa.sa_family == AF_INET && daemon->add_subnet4) { @@ -356,11 +348,7 @@ static size_t calc_subnet_opt(struct subnet_opt *opt, union mysockaddr *source) addrp = &source->in.sin_addr; } -#ifdef HAVE_IPV6 opt->family = htons(sa_family == AF_INET6 ? 2 : 1); -#else - opt->family = htons(1); -#endif len = 0; diff --git a/dnsmasq/forward.c b/src/dnsmasq/forward.c similarity index 82% rename from dnsmasq/forward.c rename to src/dnsmasq/forward.c index dd0ac07cf..597fa3e82 100644 --- a/dnsmasq/forward.c +++ b/src/dnsmasq/forward.c @@ -1,15 +1,15 @@ -/* dnsmasq is Copyright (c) 2000-2018 Simon Kelley +/* dnsmasq is Copyright (c) 2000-2020 Simon Kelley This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; version 2 dated June, 1991, or (at your option) version 3 dated 29 June, 2007. - + This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. - + You should have received a copy of the GNU General Public License along with this program. If not, see . */ @@ -24,14 +24,14 @@ static struct frec *lookup_frec_by_sender(unsigned short id, static unsigned short get_id(void); static void free_frec(struct frec *f); -/* Send a UDP packet with its source address set as "source" +/* Send a UDP packet with its source address set as "source" unless nowild is true, when we just send it with the kernel default */ -int send_from(int fd, int nowild, char *packet, size_t len, - union mysockaddr *to, struct all_addr *source, +int send_from(int fd, int nowild, char *packet, size_t len, + union mysockaddr *to, union all_addr *source, unsigned int iface) { struct msghdr msg; - struct iovec iov[1]; + struct iovec iov[1]; union { struct cmsghdr align; /* this ensures alignment */ #if defined(HAVE_LINUX_NETWORK) @@ -39,11 +39,9 @@ int send_from(int fd, int nowild, char *packet, size_t len, #elif defined(IP_SENDSRCADDR) char control[CMSG_SPACE(sizeof(struct in_addr))]; #endif -#ifdef HAVE_IPV6 char control6[CMSG_SPACE(sizeof(struct in6_pktinfo))]; -#endif } control_u; - + iov[0].iov_base = packet; iov[0].iov_len = len; @@ -54,7 +52,7 @@ int send_from(int fd, int nowild, char *packet, size_t len, msg.msg_namelen = sa_len(to); msg.msg_iov = iov; msg.msg_iovlen = 1; - + if (!nowild) { struct cmsghdr *cmptr; @@ -67,72 +65,76 @@ int send_from(int fd, int nowild, char *packet, size_t len, #if defined(HAVE_LINUX_NETWORK) struct in_pktinfo p; p.ipi_ifindex = 0; - p.ipi_spec_dst = source->addr.addr4; + p.ipi_spec_dst = source->addr4; + msg.msg_controllen = CMSG_SPACE(sizeof(struct in_pktinfo)); memcpy(CMSG_DATA(cmptr), &p, sizeof(p)); - msg.msg_controllen = cmptr->cmsg_len = CMSG_LEN(sizeof(struct in_pktinfo)); + cmptr->cmsg_len = CMSG_LEN(sizeof(struct in_pktinfo)); cmptr->cmsg_level = IPPROTO_IP; cmptr->cmsg_type = IP_PKTINFO; #elif defined(IP_SENDSRCADDR) - memcpy(CMSG_DATA(cmptr), &(source->addr.addr4), sizeof(source->addr.addr4)); - msg.msg_controllen = cmptr->cmsg_len = CMSG_LEN(sizeof(struct in_addr)); + msg.msg_controllen = CMSG_SPACE(sizeof(struct in_addr)); + memcpy(CMSG_DATA(cmptr), &(source->addr4), sizeof(source->addr4)); + cmptr->cmsg_len = CMSG_LEN(sizeof(struct in_addr)); cmptr->cmsg_level = IPPROTO_IP; cmptr->cmsg_type = IP_SENDSRCADDR; #endif } else -#ifdef HAVE_IPV6 { struct in6_pktinfo p; p.ipi6_ifindex = iface; /* Need iface for IPv6 to handle link-local addrs */ - p.ipi6_addr = source->addr.addr6; + p.ipi6_addr = source->addr6; + msg.msg_controllen = CMSG_SPACE(sizeof(struct in6_pktinfo)); memcpy(CMSG_DATA(cmptr), &p, sizeof(p)); - msg.msg_controllen = cmptr->cmsg_len = CMSG_LEN(sizeof(struct in6_pktinfo)); + cmptr->cmsg_len = CMSG_LEN(sizeof(struct in6_pktinfo)); cmptr->cmsg_type = daemon->v6pktinfo; cmptr->cmsg_level = IPPROTO_IPV6; } -#else - (void)iface; /* eliminate warning */ -#endif } - + while (retry_send(sendmsg(fd, &msg, 0))); - /* If interface is still in DAD, EINVAL results - ignore that. */ - if (errno != 0 && errno != EINVAL) + if (errno != 0) { - my_syslog(LOG_ERR, _("failed to send packet: %s"), strerror(errno)); +#ifdef HAVE_LINUX_NETWORK + /* If interface is still in DAD, EINVAL results - ignore that. */ + if (errno != EINVAL) + my_syslog(LOG_ERR, _("failed to send packet: %s"), strerror(errno)); +#endif return 0; } - + return 1; } - -static unsigned int search_servers(time_t now, struct all_addr **addrpp, unsigned int qtype, + +static unsigned int search_servers(time_t now, union all_addr **addrpp, unsigned int qtype, char *qdomain, int *type, char **domain, int *norebind) - + { /* If the query ends in the domain in one of our servers, set domain to point to that name. We find the largest match to allow both domain.org and sub.domain.org to exist. */ - + unsigned int namelen = strlen(qdomain); unsigned int matchlen = 0; struct server *serv; unsigned int flags = 0; - static struct all_addr zero; - + static union all_addr zero; + for (serv = daemon->servers; serv; serv=serv->next) if (qtype == F_DNSSECOK && !(serv->flags & SERV_DO_DNSSEC)) continue; /* domain matches take priority over NODOTS matches */ else if ((serv->flags & SERV_FOR_NODOTS) && *type != SERV_HAS_DOMAIN && !strchr(qdomain, '.') && namelen != 0) { - unsigned int sflag = serv->addr.sa.sa_family == AF_INET ? F_IPV4 : F_IPV6; + unsigned int sflag = serv->addr.sa.sa_family == AF_INET ? F_IPV4 : F_IPV6; *type = SERV_FOR_NODOTS; - if (serv->flags & SERV_NO_ADDR) + if ((serv->flags & SERV_NO_REBIND) && norebind) + *norebind = 1; + else if (serv->flags & SERV_NO_ADDR) flags = F_NXDOMAIN; else if (serv->flags & SERV_LITERAL_ADDRESS) - { + { /* literal address = '#' -> return all-zero address for IPv4 and IPv6 */ if ((serv->flags & SERV_USE_RESOLV) && (qtype & (F_IPV6 | F_IPV4))) { @@ -143,16 +145,14 @@ static unsigned int search_servers(time_t now, struct all_addr **addrpp, unsigne else if (sflag & qtype) { flags = sflag; - if (serv->addr.sa.sa_family == AF_INET) - *addrpp = (struct all_addr *)&serv->addr.in.sin_addr; -#ifdef HAVE_IPV6 + if (serv->addr.sa.sa_family == AF_INET) + *addrpp = (union all_addr *)&serv->addr.in.sin_addr; else - *addrpp = (struct all_addr *)&serv->addr.in6.sin6_addr; -#endif + *addrpp = (union all_addr *)&serv->addr.in6.sin6_addr; } else if (!flags || (flags & F_NXDOMAIN)) flags = F_NOERR; - } + } } else if (serv->flags & SERV_HAS_DOMAIN) { @@ -162,7 +162,7 @@ static unsigned int search_servers(time_t now, struct all_addr **addrpp, unsigne hostname_isequal(matchstart, serv->domain) && (domainlen == 0 || namelen == domainlen || *(matchstart-1) == '.' )) { - if ((serv->flags & SERV_NO_REBIND) && norebind) + if ((serv->flags & SERV_NO_REBIND) && norebind) *norebind = 1; else { @@ -183,7 +183,7 @@ static unsigned int search_servers(time_t now, struct all_addr **addrpp, unsigne continue; } } - + if (domainlen >= matchlen) { *type = serv->flags & (SERV_HAS_DOMAIN | SERV_USE_RESOLV | SERV_NO_REBIND | SERV_DO_DNSSEC); @@ -195,7 +195,7 @@ static unsigned int search_servers(time_t now, struct all_addr **addrpp, unsigne { /* literal address = '#' -> return all-zero address for IPv4 and IPv6 */ if ((serv->flags & SERV_USE_RESOLV) && (qtype & (F_IPV6 | F_IPV4))) - { + { memset(&zero, 0, sizeof(zero)); flags = qtype; *addrpp = &zero; @@ -203,28 +203,26 @@ static unsigned int search_servers(time_t now, struct all_addr **addrpp, unsigne else if (sflag & qtype) { flags = sflag; - if (serv->addr.sa.sa_family == AF_INET) - *addrpp = (struct all_addr *)&serv->addr.in.sin_addr; -#ifdef HAVE_IPV6 + if (serv->addr.sa.sa_family == AF_INET) + *addrpp = (union all_addr *)&serv->addr.in.sin_addr; else - *addrpp = (struct all_addr *)&serv->addr.in6.sin6_addr; -#endif + *addrpp = (union all_addr *)&serv->addr.in6.sin6_addr; } else if (!flags || (flags & F_NXDOMAIN)) flags = F_NOERR; } else flags = 0; - } + } } } } - - if (flags == 0 && !(qtype & (F_QUERY | F_DNSSECOK)) && + + if (flags == 0 && !(qtype & (F_QUERY | F_DNSSECOK)) && option_bool(OPT_NODOTS_LOCAL) && !strchr(qdomain, '.') && namelen != 0) /* don't forward A or AAAA queries for simple names, except the empty name */ flags = F_NOERR; - + if (flags == F_NXDOMAIN && check_for_local_domain(qdomain, now)) flags = F_NOERR; @@ -243,13 +241,11 @@ static unsigned int search_servers(time_t now, struct all_addr **addrpp, unsigne log_query((flags | F_CONFIG | F_FORWARD) & ~F_IPV6, qdomain, *addrpp, NULL); FTL_reply((flags | F_CONFIG | F_FORWARD) & ~F_IPV6, qdomain, *addrpp, daemon->log_display_id); } -#ifdef HAVE_IPV6 if (flags & F_IPV6) { log_query((flags | F_CONFIG | F_FORWARD) & ~F_IPV4, qdomain, *addrpp, NULL); FTL_reply((flags | F_CONFIG | F_FORWARD) & ~F_IPV4, qdomain, *addrpp, daemon->log_display_id); - } -#endif + } } } else if ((*type) & SERV_USE_RESOLV) @@ -261,13 +257,13 @@ static unsigned int search_servers(time_t now, struct all_addr **addrpp, unsigne } static int forward_query(int udpfd, union mysockaddr *udpaddr, - struct all_addr *dst_addr, unsigned int dst_iface, - struct dns_header *header, size_t plen, time_t now, + union all_addr *dst_addr, unsigned int dst_iface, + struct dns_header *header, size_t plen, time_t now, struct frec *forward, int ad_reqd, int do_bit) { char *domain = NULL; int type = SERV_DO_DNSSEC, norebind = 0; - struct all_addr *addrp = NULL; + union all_addr *addrp = NULL; unsigned int flags = 0; struct server *start = NULL; #ifdef HAVE_DNSSEC @@ -289,7 +285,7 @@ static int forward_query(int udpfd, union mysockaddr *udpaddr, If that generates an answer, it will become the new default for this server */ forward->flags |= FREC_TEST_PKTSZ; - + #ifdef HAVE_DNSSEC /* If we've already got an answer to this query, but we're awaiting keys for validation, there's no point retrying the query, retry the key query instead...... */ @@ -297,42 +293,39 @@ static int forward_query(int udpfd, union mysockaddr *udpaddr, { int fd, is_sign; unsigned char *pheader; - + forward->flags &= ~FREC_TEST_PKTSZ; - + while (forward->blocking_query) forward = forward->blocking_query; - + blockdata_retrieve(forward->stash, forward->stash_len, (void *)header); plen = forward->stash_len; - + forward->flags |= FREC_TEST_PKTSZ; if (find_pseudoheader(header, plen, NULL, &pheader, &is_sign, NULL) && !is_sign) PUTSHORT(SAFE_PKTSZ, pheader); - - if (forward->sentto->addr.sa.sa_family == AF_INET) - log_query(F_NOEXTRA | F_DNSSEC | F_IPV4, "retry", (struct all_addr *)&forward->sentto->addr.in.sin_addr, "dnssec"); -#ifdef HAVE_IPV6 + + if (forward->sentto->addr.sa.sa_family == AF_INET) + log_query(F_NOEXTRA | F_DNSSEC | F_IPV4, "retry", (union all_addr *)&forward->sentto->addr.in.sin_addr, "dnssec"); else - log_query(F_NOEXTRA | F_DNSSEC | F_IPV6, "retry", (struct all_addr *)&forward->sentto->addr.in6.sin6_addr, "dnssec"); -#endif + log_query(F_NOEXTRA | F_DNSSEC | F_IPV6, "retry", (union all_addr *)&forward->sentto->addr.in6.sin6_addr, "dnssec"); + if (forward->sentto->sfd) fd = forward->sentto->sfd->fd; else { -#ifdef HAVE_IPV6 if (forward->sentto->addr.sa.sa_family == AF_INET6) fd = forward->rfd6->fd; else -#endif fd = forward->rfd4->fd; } - + while (retry_send(sendto(fd, (char *)header, plen, 0, &forward->sentto->addr.sa, sa_len(&forward->sentto->addr)))); - + return 1; } #endif @@ -356,20 +349,20 @@ static int forward_query(int udpfd, union mysockaddr *udpaddr, FTL_forwarding_failed(forward->sentto); } - else + else { if (gotname) flags = search_servers(now, &addrp, gotname, daemon->namebuff, &type, &domain, &norebind); - + #ifdef HAVE_DNSSEC do_dnssec = type & SERV_DO_DNSSEC; #endif - type &= ~SERV_DO_DNSSEC; + type &= ~SERV_DO_DNSSEC; if (daemon->servers && !flags) - forward = get_new_frec(now, NULL, 0); + forward = get_new_frec(now, NULL, NULL); /* table full - flags == 0, return REFUSED */ - + if (forward) { forward->source = *udpaddr; @@ -392,14 +385,14 @@ static int forward_query(int udpfd, union mysockaddr *udpaddr, if (do_bit) forward->flags |= FREC_DO_QUESTION; #endif - + header->id = htons(forward->new_id); - - /* In strict_order mode, always try servers in the order - specified in resolv.conf, if a domain is given + + /* In strict_order mode, always try servers in the order + specified in resolv.conf, if a domain is given always try all the available servers, otherwise, use the one last known to work. */ - + if (type == 0) { if (option_bool(OPT_ORDER)) @@ -423,30 +416,30 @@ static int forward_query(int udpfd, union mysockaddr *udpaddr, } } - /* check for send errors here (no route to host) + /* check for send errors here (no route to host) if we fail to send to all nameservers, send back an error packet straight away (helps modem users when offline) */ - + if (!flags && forward) { struct server *firstsentto = start; int subnet, forwarded = 0; size_t edns0_len; unsigned char *pheader; - + /* If a query is retried, use the log_id for the retry when logging the answer. */ forward->log_id = daemon->log_id; - + plen = add_edns0_config(header, plen, ((unsigned char *)header) + PACKETSZ, &forward->source, now, &subnet); - + if (subnet) forward->flags |= FREC_HAS_SUBNET; - + #ifdef HAVE_DNSSEC if (option_bool(OPT_DNSSEC_VALID) && do_dnssec) { plen = add_do_bit(header, plen, ((unsigned char *) header) + PACKETSZ); - + /* For debugging, set Checking Disabled, otherwise, have the upstream check too, this allows it to select auth servers when one is returning bad data. */ if (option_bool(OPT_DNSSEC_DEBUG)) @@ -469,13 +462,13 @@ static int forward_query(int udpfd, union mysockaddr *udpaddr, if (forward->flags & FREC_TEST_PKTSZ) PUTSHORT(SAFE_PKTSZ, pheader); } - + while (1) - { + { /* only send to servers dealing with our domain. - domain may be NULL, in which case server->domain + domain may be NULL, in which case server->domain must be NULL also. */ - + if (type == (start->flags & SERV_TYPE) && (type != SERV_HAS_DOMAIN || hostname_isequal(domain, start->domain)) && !(start->flags & (SERV_LITERAL_ADDRESS | SERV_LOOP))) @@ -485,9 +478,8 @@ static int forward_query(int udpfd, union mysockaddr *udpaddr, /* find server socket to use, may need to get random one. */ if (start->sfd) fd = start->sfd->fd; - else + else { -#ifdef HAVE_IPV6 if (start->addr.sa.sa_family == AF_INET6) { if (!forward->rfd6 && @@ -497,7 +489,6 @@ static int forward_query(int udpfd, union mysockaddr *udpaddr, fd = forward->rfd6->fd; } else -#endif { if (!forward->rfd4 && !(forward->rfd4 = allocate_rfd(AF_INET))) @@ -516,7 +507,7 @@ static int forward_query(int udpfd, union mysockaddr *udpaddr, } #endif } - + #ifdef HAVE_DNSSEC if (option_bool(OPT_DNSSEC_VALID) && (forward->flags & FREC_ADDED_PHEADER)) { @@ -525,7 +516,7 @@ static int forward_query(int udpfd, union mysockaddr *udpaddr, The RRSIGS will be stripped out before the answer goes back, so the packet should shrink again. So, if we added a do-bit, bump the udp packet size to the value known to be OK for this server. We check returned size after stripping and set - the truncated bit if it's still too big. */ + the truncated bit if it's still too big. */ unsigned char *pheader; int is_sign; if (find_pseudoheader(header, plen, NULL, &pheader, &is_sign, NULL) && !is_sign) @@ -537,59 +528,57 @@ static int forward_query(int udpfd, union mysockaddr *udpaddr, &start->addr.sa, sa_len(&start->addr)))) continue; - + if (errno == 0) { #ifdef HAVE_DUMPFILE dump_packet(DUMP_UP_QUERY, (void *)header, plen, NULL, &start->addr); #endif - + /* Keep info in case we want to re-send this packet */ daemon->srv_save = start; daemon->packet_len = plen; - + if (!gotname) strcpy(daemon->namebuff, "query"); if (start->addr.sa.sa_family == AF_INET) { - log_query(F_SERVER | F_IPV4 | F_FORWARD, daemon->namebuff, - (struct all_addr *)&start->addr.in.sin_addr, NULL); + log_query(F_SERVER | F_IPV4 | F_FORWARD, daemon->namebuff, + (union all_addr *)&start->addr.in.sin_addr, NULL); FTL_forwarded(F_SERVER | F_IPV4 | F_FORWARD, daemon->namebuff, - (struct all_addr *)&start->addr.in.sin_addr, daemon->log_display_id); + (union all_addr *)&start->addr.in.sin_addr, daemon->log_display_id); } -#ifdef HAVE_IPV6 else { - log_query(F_SERVER | F_IPV6 | F_FORWARD, daemon->namebuff, - (struct all_addr *)&start->addr.in6.sin6_addr, NULL); + log_query(F_SERVER | F_IPV6 | F_FORWARD, daemon->namebuff, + (union all_addr *)&start->addr.in6.sin6_addr, NULL); FTL_forwarded(F_SERVER | F_IPV6 | F_FORWARD, daemon->namebuff, - (struct all_addr *)&start->addr.in6.sin6_addr, daemon->log_display_id); + (union all_addr *)&start->addr.in6.sin6_addr, daemon->log_display_id); } -#endif start->queries++; forwarded = 1; forward->sentto = start; - if (!forward->forwardall) + if (!forward->forwardall) break; forward->forwardall++; } - } - + } + if (!(start = start->next)) start = daemon->servers; - + if (start == firstsentto) break; } - + if (forwarded) return 1; - - /* could not send on, prepare to return */ + + /* could not send on, prepare to return */ header->id = htons(forward->orig_id); free_frec(forward); /* cancel */ - } - + } + /* could not send on, return empty answer or address if known for whole domain */ if (udpfd != -1) { @@ -602,16 +591,16 @@ static int forward_query(int udpfd, union mysockaddr *udpaddr, return 0; } -static size_t process_reply(struct dns_header *header, time_t now, struct server *server, size_t n, int check_rebind, - int no_cache, int cache_secure, int bogusanswer, int ad_reqd, int do_bit, int added_pheader, +static size_t process_reply(struct dns_header *header, time_t now, struct server *server, size_t n, int check_rebind, + int no_cache, int cache_secure, int bogusanswer, int ad_reqd, int do_bit, int added_pheader, int check_subnet, union mysockaddr *query_source) { unsigned char *pheader, *sizep; char **sets = 0; int munged = 0, is_sign; unsigned int rcode = RCODE(header); - size_t plen; - + size_t plen; + (void)ad_reqd; (void)do_bit; (void)bogusanswer; @@ -623,13 +612,13 @@ static size_t process_reply(struct dns_header *header, time_t now, struct server struct ipsets *ipset_pos; unsigned int namelen = strlen(daemon->namebuff); unsigned int matchlen = 0; - for (ipset_pos = daemon->ipsets; ipset_pos; ipset_pos = ipset_pos->next) + for (ipset_pos = daemon->ipsets; ipset_pos; ipset_pos = ipset_pos->next) { unsigned int domainlen = strlen(ipset_pos->domain); char *matchstart = daemon->namebuff + namelen - domainlen; if (namelen >= domainlen && hostname_isequal(matchstart, ipset_pos->domain) && (domainlen == 0 || namelen == domainlen || *(matchstart - 1) == '.' ) && - domainlen >= matchlen) + domainlen >= matchlen) { matchlen = domainlen; sets = ipset_pos->sets; @@ -637,7 +626,7 @@ static size_t process_reply(struct dns_header *header, time_t now, struct server } } #endif - + if ((pheader = find_pseudoheader(header, n, &plen, &sizep, &is_sign, NULL))) { /* Get extended RCODE. */ @@ -648,7 +637,7 @@ static size_t process_reply(struct dns_header *header, time_t now, struct server my_syslog(LOG_WARNING, _("discarding DNS reply: subnet option mismatch")); return 0; } - + if (!is_sign) { if (added_pheader) @@ -686,26 +675,27 @@ static size_t process_reply(struct dns_header *header, time_t now, struct server } } } - + FTL_header_analysis(header->hb4, rcode, daemon->log_display_id); /* RFC 4035 sect 4.6 para 3 */ if (!is_sign && !option_bool(OPT_DNSSEC_PROXY)) header->hb4 &= ~HB4_AD; - + if (OPCODE(header) != QUERY) return resize_packet(header, n, pheader, plen); if (rcode != NOERROR && rcode != NXDOMAIN) { - struct all_addr a; - a.addr.rcode.rcode = rcode; + union all_addr a; + a.log.rcode = rcode; FTL_upstream_error(rcode, daemon->log_display_id); log_query(F_UPSTREAM | F_RCODE, "error", &a, NULL); - + FTL_upstream_error(rcode, daemon->log_display_id); + return resize_packet(header, n, pheader, plen); } - + /* Complain loudly if the upstream server is non-recursive. */ if (!(header->hb4 & HB4_RA) && rcode == NOERROR && server && !(server->flags & SERV_WARNED_RECURSIVE)) @@ -714,7 +704,7 @@ static size_t process_reply(struct dns_header *header, time_t now, struct server my_syslog(LOG_WARNING, _("nameserver %s refused to do a recursive query"), daemon->namebuff); if (!option_bool(OPT_LOG)) server->flags |= SERV_WARNED_RECURSIVE; - } + } if (daemon->bogus_addr && rcode != NXDOMAIN && check_for_bogus_wildcard(header, n, daemon->namebuff, daemon->bogus_addr, now)) @@ -724,15 +714,15 @@ static size_t process_reply(struct dns_header *header, time_t now, struct server header->hb3 &= ~HB3_AA; cache_secure = 0; } - else + else { int doctored = 0; - - if (rcode == NXDOMAIN && + + if (rcode == NXDOMAIN && extract_request(header, n, daemon->namebuff, NULL) && check_for_local_domain(daemon->namebuff, now)) { - /* if we forwarded a query for a locally known name (because it was for + /* if we forwarded a query for a locally known name (because it was for an unknown type) and the answer is NXDOMAIN, convert that to NODATA, since we know that the domain exists, even if upstream doesn't */ munged = 1; @@ -740,18 +730,25 @@ static size_t process_reply(struct dns_header *header, time_t now, struct server SET_RCODE(header, NOERROR); cache_secure = 0; } - - if (extract_addresses(header, n, daemon->namebuff, now, sets, is_sign, check_rebind, no_cache, cache_secure, &doctored)) + /******************************** Pi-hole modification ********************************/ + int ret = extract_addresses(header, n, daemon->namebuff, now, sets, is_sign, check_rebind, no_cache, cache_secure, &doctored); + if (ret == 2) + { + munged = 1; + cache_secure = 0; + } + else if(ret) { my_syslog(LOG_WARNING, _("possible DNS-rebind attack detected: %s"), daemon->namebuff); munged = 1; cache_secure = 0; } + /**************************************************************************************/ if (doctored) cache_secure = 0; } - + #ifdef HAVE_DNSSEC if (bogusanswer && !(header->hb4 & HB4_CD) && !option_bool(OPT_DNSSEC_DEBUG)) { @@ -763,10 +760,10 @@ static size_t process_reply(struct dns_header *header, time_t now, struct server if (option_bool(OPT_DNSSEC_VALID)) { header->hb4 &= ~HB4_AD; - + if (!(header->hb4 & HB4_CD) && ad_reqd && cache_secure) header->hb4 |= HB4_AD; - + /* If the requestor didn't set the DO bit, don't return DNSSEC info. */ if (!do_bit) n = rrfilter(header, n, 1); @@ -775,7 +772,7 @@ static size_t process_reply(struct dns_header *header, time_t now, struct server /* do this after extract_addresses. Ensure NODATA reply and remove nameserver info. */ - + if (munged) { header->ancount = htons(0); @@ -783,7 +780,7 @@ static size_t process_reply(struct dns_header *header, time_t now, struct server header->arcount = htons(0); header->hb3 &= ~HB3_TC; } - + /* the bogus-nxdomain stuff, doctor and NXDOMAIN->NODATA munging can all elide sections of the packet. Find the new length here and put back pseudoheader if it was removed. */ @@ -809,25 +806,22 @@ void reply_query(int fd, int family, time_t now) /* packet buffer overwritten */ daemon->srv_save = NULL; - + /* Determine the address of the server replying so that we can mark that as good */ - serveraddr.sa.sa_family = family; -#ifdef HAVE_IPV6 - if (serveraddr.sa.sa_family == AF_INET6) + if ((serveraddr.sa.sa_family = family) == AF_INET6) serveraddr.in6.sin6_flowinfo = 0; -#endif - + header = (struct dns_header *)daemon->packet; if (n < (int)sizeof(struct dns_header) || !(header->hb3 & HB3_QR)) return; - + /* spoof check: answer must come from known server, */ for (server = daemon->servers; server; server = server->next) if (!(server->flags & (SERV_LITERAL_ADDRESS | SERV_NO_ADDR)) && sockaddr_isequal(&server->addr, &serveraddr)) break; - + if (!server) return; @@ -841,20 +835,20 @@ void reply_query(int fd, int family, time_t now) hash = &crc; crc = questions_crc(header, n, daemon->namebuff); #endif - + if (!(forward = lookup_frec(ntohs(header->id), hash))) return; - + #ifdef HAVE_DUMPFILE dump_packet((forward->flags & (FREC_DNSKEY_QUERY | FREC_DS_QUERY)) ? DUMP_SEC_REPLY : DUMP_UP_REPLY, (void *)header, n, &serveraddr, NULL); #endif - /* log_query gets called indirectly all over the place, so + /* log_query gets called indirectly all over the place, so pass these in global variables - sorry. */ daemon->log_display_id = forward->log_id; daemon->log_source_addr = &forward->source; - + if (daemon->ignore_addr && RCODE(header) == NOERROR && check_for_ignored_address(header, n, daemon->ignore_addr)) return; @@ -875,7 +869,7 @@ void reply_query(int fd, int family, time_t now) if (forward->flags & (FREC_DNSKEY_QUERY | FREC_DS_QUERY)) { struct server *start; - + blockdata_retrieve(forward->stash, forward->stash_len, (void *)header); plen = forward->stash_len; @@ -890,18 +884,17 @@ void reply_query(int fd, int family, time_t now) start = daemon->servers; if (start == forward->sentto) break; - + if ((start->flags & SERV_TYPE) == 0 && (start->flags & SERV_DO_DNSSEC)) break; } - - + + if (start->sfd) fd = start->sfd->fd; else { -#ifdef HAVE_IPV6 if (start->addr.sa.sa_family == AF_INET6) { /* may have changed family */ @@ -910,7 +903,6 @@ void reply_query(int fd, int family, time_t now) fd = forward->rfd6->fd; } else -#endif { /* may have changed family */ if (!forward->rfd4) @@ -919,25 +911,27 @@ void reply_query(int fd, int family, time_t now) } } +#ifdef HAVE_DUMPFILE + dump_packet(DUMP_SEC_QUERY, (void *)header, (size_t)plen, NULL, &start->addr); +#endif + while (retry_send(sendto(fd, (char *)header, plen, 0, &start->addr.sa, sa_len(&start->addr)))); - - if (start->addr.sa.sa_family == AF_INET) - log_query(F_NOEXTRA | F_DNSSEC | F_IPV4, "retry", (struct all_addr *)&start->addr.in.sin_addr, "dnssec"); -#ifdef HAVE_IPV6 + + if (start->addr.sa.sa_family == AF_INET) + log_query(F_NOEXTRA | F_DNSSEC | F_IPV4, "retry", (union all_addr *)&start->addr.in.sin_addr, "dnssec"); else - log_query(F_NOEXTRA | F_DNSSEC | F_IPV6, "retry", (struct all_addr *)&start->addr.in6.sin6_addr, "dnssec"); -#endif - + log_query(F_NOEXTRA | F_DNSSEC | F_IPV6, "retry", (union all_addr *)&start->addr.in6.sin6_addr, "dnssec"); + return; } #endif - + /* In strict order mode, there must be a server later in the chain left to send to, otherwise without the forwardall mechanism, code further on will cycle around the list forwever if they - all return REFUSED. Note that server is always non-NULL before + all return REFUSED. Note that server is always non-NULL before this executes. */ if (option_bool(OPT_ORDER)) for (server = forward->sentto->next; server; server = server->next) @@ -965,8 +959,8 @@ void reply_query(int fd, int family, time_t now) return; } } - } - + } + server = forward->sentto; if ((forward->sentto->flags & SERV_TYPE) == 0) { @@ -975,8 +969,8 @@ void reply_query(int fd, int family, time_t now) else { struct server *last_server; - - /* find good server by address if possible, otherwise assume the last one we sent to */ + + /* find good server by address if possible, otherwise assume the last one we sent to */ for (last_server = daemon->servers; last_server; last_server = last_server->next) if (!(last_server->flags & (SERV_LITERAL_ADDRESS | SERV_HAS_DOMAIN | SERV_FOR_NODOTS | SERV_NO_ADDR)) && sockaddr_isequal(&last_server->addr, &serveraddr)) @@ -984,43 +978,42 @@ void reply_query(int fd, int family, time_t now) server = last_server; break; } - } + } if (!option_bool(OPT_ALL_SERVERS)) daemon->last_server = server; } - + /* We tried resending to this server with a smaller maximum size and got an answer. Make that permanent. To avoid reduxing the packet size for a single dropped packet, only do this when we get a truncated answer, or one larger than the safe size. */ - if (server && server->edns_pktsz > SAFE_PKTSZ && (forward->flags & FREC_TEST_PKTSZ) && + if (forward->sentto->edns_pktsz > SAFE_PKTSZ && (forward->flags & FREC_TEST_PKTSZ) && ((header->hb3 & HB3_TC) || n >= SAFE_PKTSZ)) { - server->edns_pktsz = SAFE_PKTSZ; - server->pktsz_reduced = now; - prettyprint_addr(&server->addr, daemon->addrbuff); + forward->sentto->edns_pktsz = SAFE_PKTSZ; + forward->sentto->pktsz_reduced = now; + prettyprint_addr(&forward->sentto->addr, daemon->addrbuff); my_syslog(LOG_WARNING, _("reducing DNS packet size for nameserver %s to %d"), daemon->addrbuff, SAFE_PKTSZ); } - + /* If the answer is an error, keep the forward record in place in case we get a good reply from another server. Kill it when we've had replies from all to avoid filling the forwarding table when everything is broken */ - if (forward->forwardall == 0 || --forward->forwardall == 1 || - (RCODE(header) != REFUSED && RCODE(header) != SERVFAIL)) + if (forward->forwardall == 0 || --forward->forwardall == 1 || RCODE(header) != REFUSED) { int check_rebind = 0, no_cache_dnssec = 0, cache_secure = 0, bogusanswer = 0; - + if (option_bool(OPT_NO_REBIND)) check_rebind = !(forward->flags & FREC_NOREBIND); - + /* Don't cache replies where DNSSEC validation was turned off, either the upstream server told us so, or the original query specified it. */ if ((header->hb4 & HB4_CD) || (forward->flags & FREC_CHECKING_DISABLED)) no_cache_dnssec = 1; - + #ifdef HAVE_DNSSEC - if (server && (server->flags & SERV_DO_DNSSEC) && + if ((forward->sentto->flags & SERV_DO_DNSSEC) && option_bool(OPT_DNSSEC_VALID) && !(forward->flags & FREC_CHECKING_DISABLED)) { int status = 0; @@ -1028,7 +1021,7 @@ void reply_query(int fd, int family, time_t now) /* We've had a reply already, which we're validating. Ignore this duplicate */ if (forward->blocking_query) return; - + /* Truncated answer can't be validated. If this is an answer to a DNSSEC-generated query, we still need to get the client to retry over TCP, so return @@ -1036,7 +1029,7 @@ void reply_query(int fd, int family, time_t now) */ if (header->hb3 & HB3_TC) status = STAT_TRUNCATED; - + while (1) { /* As soon as anything returns BOGUS, we stop and unwind, to do otherwise @@ -1049,53 +1042,56 @@ void reply_query(int fd, int family, time_t now) else if (forward->flags & FREC_DS_QUERY) status = dnssec_validate_ds(now, header, n, daemon->namebuff, daemon->keyname, forward->class); else - status = dnssec_validate_reply(now, header, n, daemon->namebuff, daemon->keyname, &forward->class, - !option_bool(OPT_DNSSEC_IGN_NS) && (server->flags & SERV_DO_DNSSEC), - NULL, NULL); + status = dnssec_validate_reply(now, header, n, daemon->namebuff, daemon->keyname, &forward->class, + !option_bool(OPT_DNSSEC_IGN_NS) && (forward->sentto->flags & SERV_DO_DNSSEC), + NULL, NULL, NULL); #ifdef HAVE_DUMPFILE if (status == STAT_BOGUS) dump_packet((forward->flags & (FREC_DNSKEY_QUERY | FREC_DS_QUERY)) ? DUMP_SEC_BOGUS : DUMP_BOGUS, header, (size_t)n, &serveraddr, NULL); #endif } - + /* Can't validate, as we're missing key data. Put this - answer aside, whilst we get that. */ + answer aside, whilst we get that. */ if (status == STAT_NEED_DS || status == STAT_NEED_KEY) { struct frec *new, *orig; - + /* Free any saved query */ if (forward->stash) blockdata_free(forward->stash); - + /* Now save reply pending receipt of key data */ if (!(forward->stash = blockdata_alloc((char *)header, n))) return; forward->stash_len = n; - + /* Find the original query that started it all.... */ for (orig = forward; orig->dependent; orig = orig->dependent); - - if (--orig->work_counter == 0 || !(new = get_new_frec(now, NULL, 1))) + + /* Make sure we don't expire and free the orig frec during the + allocation of a new one. */ + if (--orig->work_counter == 0 || !(new = get_new_frec(now, NULL, orig))) status = STAT_ABANDONED; else { int querytype, fd, type = SERV_DO_DNSSEC; struct frec *next = new->next; char *domain; - + *new = *forward; /* copy everything, then overwrite */ new->next = next; new->blocking_query = NULL; - /* Find server to forward to. This will normally be the + /* Find server to forward to. This will normally be the same as for the original query, but may be another if - servers for domains are involved. */ + servers for domains are involved. */ if (search_servers(now, NULL, F_DNSSECOK, daemon->keyname, &type, &domain, NULL) == 0) { - struct server *start = server, *new_server = NULL; - + struct server *start, *new_server = NULL; + start = server = forward->sentto; + while (1) { if (type == (start->flags & (SERV_TYPE | SERV_DO_DNSSEC)) && @@ -1109,34 +1105,32 @@ void reply_query(int fd, int family, time_t now) break; } } - + if (!(start = start->next)) start = daemon->servers; if (start == server) break; } - + if (new_server) server = new_server; } - + new->sentto = server; new->rfd4 = NULL; -#ifdef HAVE_IPV6 new->rfd6 = NULL; -#endif new->flags &= ~(FREC_DNSKEY_QUERY | FREC_DS_QUERY | FREC_HAS_EXTRADATA); new->forwardall = 0; - + new->dependent = forward; /* to find query awaiting new one. */ forward->blocking_query = new; /* for garbage cleaning */ /* validate routines leave name of required record in daemon->keyname */ if (status == STAT_NEED_KEY) { - new->flags |= FREC_DNSKEY_QUERY; + new->flags |= FREC_DNSKEY_QUERY; querytype = T_DNSKEY; } - else + else { new->flags |= FREC_DS_QUERY; querytype = T_DS; @@ -1145,15 +1139,13 @@ void reply_query(int fd, int family, time_t now) nn = dnssec_generate_query(header,((unsigned char *) header) + server->edns_pktsz, daemon->keyname, forward->class, querytype, server->edns_pktsz); - if (server->addr.sa.sa_family == AF_INET) - log_query(F_NOEXTRA | F_DNSSEC | F_IPV4, daemon->keyname, (struct all_addr *)&(server->addr.in.sin_addr), + if (server->addr.sa.sa_family == AF_INET) + log_query(F_NOEXTRA | F_DNSSEC | F_IPV4, daemon->keyname, (union all_addr *)&(server->addr.in.sin_addr), querystr("dnssec-query", querytype)); -#ifdef HAVE_IPV6 else - log_query(F_NOEXTRA | F_DNSSEC | F_IPV6, daemon->keyname, (struct all_addr *)&(server->addr.in6.sin6_addr), + log_query(F_NOEXTRA | F_DNSSEC | F_IPV6, daemon->keyname, (union all_addr *)&(server->addr.in6.sin6_addr), querystr("dnssec-query", querytype)); -#endif - + if ((hash = hash_questions(header, nn, daemon->namebuff))) memcpy(new->hash, hash, HASH_SIZE); new->new_id = get_id(); @@ -1161,29 +1153,27 @@ void reply_query(int fd, int family, time_t now) /* Save query for retransmission */ new->stash = blockdata_alloc((char *)header, nn); new->stash_len = nn; - + /* Don't resend this. */ daemon->srv_save = NULL; - + if (server->sfd) fd = server->sfd->fd; else { fd = -1; -#ifdef HAVE_IPV6 if (server->addr.sa.sa_family == AF_INET6) { if (new->rfd6 || (new->rfd6 = allocate_rfd(AF_INET6))) fd = new->rfd6->fd; } else -#endif { if (new->rfd4 || (new->rfd4 = allocate_rfd(AF_INET))) fd = new->rfd4->fd; } } - + if (fd != -1) { #ifdef HAVE_CONNTRACK @@ -1195,24 +1185,24 @@ void reply_query(int fd, int family, time_t now) setsockopt(fd, SOL_SOCKET, SO_MARK, &mark, sizeof(unsigned int)); } #endif - + #ifdef HAVE_DUMPFILE dump_packet(DUMP_SEC_QUERY, (void *)header, (size_t)nn, NULL, &server->addr); #endif - - while (retry_send(sendto(fd, (char *)header, nn, 0, - &server->addr.sa, - sa_len(&server->addr)))); + + while (retry_send(sendto(fd, (char *)header, nn, 0, + &server->addr.sa, + sa_len(&server->addr)))); server->queries++; } - } + } return; } - + /* Validated original answer, all done. */ if (!forward->dependent) break; - + /* validated subsidiary query, (and cached result) pop that and return to the previous query we were working on. */ struct frec *prev = forward->dependent; @@ -1222,16 +1212,16 @@ void reply_query(int fd, int family, time_t now) blockdata_retrieve(forward->stash, forward->stash_len, (void *)header); n = forward->stash_len; } - - + + no_cache_dnssec = 0; - + if (status == STAT_TRUNCATED) header->hb3 |= HB3_TC; else { char *result, *domain = "result"; - + if (status == STAT_ABANDONED) { result = "ABANDONED"; @@ -1239,14 +1229,14 @@ void reply_query(int fd, int family, time_t now) } else result = (status == STAT_SECURE ? "SECURE" : (status == STAT_INSECURE ? "INSECURE" : "BOGUS")); - + if (status == STAT_BOGUS && extract_request(header, n, daemon->namebuff, NULL)) domain = daemon->namebuff; - + log_query(F_SECSTAT, domain, NULL, result); FTL_dnssec(status, daemon->log_display_id); } - + if (status == STAT_SECURE) cache_secure = 1; else if (status == STAT_BOGUS) @@ -1255,6 +1245,7 @@ void reply_query(int fd, int family, time_t now) bogusanswer = 1; } } + #endif /* restore CD bit to the value in the query */ @@ -1262,9 +1253,9 @@ void reply_query(int fd, int family, time_t now) header->hb4 |= HB4_CD; else header->hb4 &= ~HB4_CD; - - if ((nn = process_reply(header, now, forward->sentto, (size_t)n, check_rebind, no_cache_dnssec, cache_secure, bogusanswer, - forward->flags & FREC_AD_QUESTION, forward->flags & FREC_DO_QUESTION, + + if ((nn = process_reply(header, now, forward->sentto, (size_t)n, check_rebind, no_cache_dnssec, cache_secure, bogusanswer, + forward->flags & FREC_AD_QUESTION, forward->flags & FREC_DO_QUESTION, forward->flags & FREC_ADDED_PHEADER, forward->flags & FREC_HAS_SUBNET, &forward->source))) { header->id = htons(forward->orig_id); @@ -1286,8 +1277,8 @@ void reply_query(int fd, int family, time_t now) #ifdef HAVE_DUMPFILE dump_packet(DUMP_REPLY, daemon->packet, (size_t)nn, NULL, &forward->source); #endif - - send_from(forward->fd, option_bool(OPT_NOWILD) || option_bool (OPT_CLEVERBIND), daemon->packet, nn, + + send_from(forward->fd, option_bool(OPT_NOWILD) || option_bool (OPT_CLEVERBIND), daemon->packet, nn, &forward->source, &forward->dest, forward->iface); } free_frec(forward); /* cancel */ @@ -1301,7 +1292,7 @@ void receive_query(struct listener *listen, time_t now) union mysockaddr source_addr; unsigned char *pheader; unsigned short type, udp_size = PACKETSZ; /* default if no EDNS0 */ - struct all_addr dst_addr; + union all_addr dst_addr; struct in_addr netmask, dst_addr_4; size_t m; ssize_t n; @@ -1314,9 +1305,7 @@ void receive_query(struct listener *listen, time_t now) struct cmsghdr *cmptr; union { struct cmsghdr align; /* this ensures alignment */ -#ifdef HAVE_IPV6 char control6[CMSG_SPACE(sizeof(struct in6_pktinfo))]; -#endif #if defined(HAVE_LINUX_NETWORK) char control[CMSG_SPACE(sizeof(struct in_pktinfo))]; #elif defined(IP_RECVDSTADDR) && defined(HAVE_SOLARIS_NETWORK) @@ -1327,33 +1316,34 @@ void receive_query(struct listener *listen, time_t now) CMSG_SPACE(sizeof(struct sockaddr_dl))]; #endif } control_u; -#ifdef HAVE_IPV6 /* Can always get recvd interface for IPv6 */ int check_dst = !option_bool(OPT_NOWILD) || listen->family == AF_INET6; -#else - int check_dst = !option_bool(OPT_NOWILD); -#endif + + /************ Pi-hole modification ************/ + bool piholeblocked = false; + const char* blockingreason = NULL; + /**********************************************/ /* packet buffer overwritten */ daemon->srv_save = NULL; - - dst_addr_4.s_addr = dst_addr.addr.addr4.s_addr = 0; + + dst_addr_4.s_addr = dst_addr.addr4.s_addr = 0; netmask.s_addr = 0; - + if (option_bool(OPT_NOWILD) && listen->iface) { auth_dns = listen->iface->dns_auth; - + if (listen->family == AF_INET) { - dst_addr_4 = dst_addr.addr.addr4 = listen->iface->addr.in.sin_addr; + dst_addr_4 = dst_addr.addr4 = listen->iface->addr.in.sin_addr; netmask = listen->iface->netmask; } } - + iov[0].iov_base = daemon->packet; iov[0].iov_len = daemon->edns_pktsz; - + msg.msg_control = control_u.control; msg.msg_controllen = sizeof(control_u); msg.msg_flags = 0; @@ -1361,11 +1351,11 @@ void receive_query(struct listener *listen, time_t now) msg.msg_namelen = sizeof(source_addr); msg.msg_iov = iov; msg.msg_iovlen = 1; - + if ((n = recvmsg(listen->fd, &msg, 0)) == -1) return; - - if (n < (int)sizeof(struct dns_header) || + + if (n < (int)sizeof(struct dns_header) || (msg.msg_flags & MSG_TRUNC) || (header->hb3 & HB3_QR)) return; @@ -1373,17 +1363,16 @@ void receive_query(struct listener *listen, time_t now) /* Clear buffer beyond request to avoid risk of information disclosure. */ memset(daemon->packet + n, 0, daemon->edns_pktsz - n); - + source_addr.sa.sa_family = listen->family; - + if (listen->family == AF_INET) { - /* Source-port == 0 is an error, we can't send back to that. + /* Source-port == 0 is an error, we can't send back to that. http://www.ietf.org/mail-archive/web/dnsop/current/msg11441.html */ if (source_addr.in.sin_port == 0) return; } -#ifdef HAVE_IPV6 else { /* Source-port == 0 is an error, we can't send back to that. */ @@ -1391,29 +1380,27 @@ void receive_query(struct listener *listen, time_t now) return; source_addr.in6.sin6_flowinfo = 0; } -#endif - + /* We can be configured to only accept queries from at-most-one-hop-away addresses. */ if (option_bool(OPT_LOCAL_SERVICE)) { struct addrlist *addr; -#ifdef HAVE_IPV6 - if (listen->family == AF_INET6) + + if (listen->family == AF_INET6) { for (addr = daemon->interface_addrs; addr; addr = addr->next) if ((addr->flags & ADDRLIST_IPV6) && - is_same_net6(&addr->addr.addr.addr6, &source_addr.in6.sin6_addr, addr->prefixlen)) + is_same_net6(&addr->addr.addr6, &source_addr.in6.sin6_addr, addr->prefixlen)) break; } else -#endif { struct in_addr netmask; for (addr = daemon->interface_addrs; addr; addr = addr->next) { netmask.s_addr = htonl(~(in_addr_t)0 << (32 - addr->prefixlen)); if (!(addr->flags & ADDRLIST_IPV6) && - is_same_net(addr->addr.addr.addr4, source_addr.in.sin_addr, netmask)) + is_same_net(addr->addr.addr4, source_addr.in.sin_addr, netmask)) break; } } @@ -1428,7 +1415,7 @@ void receive_query(struct listener *listen, time_t now) return; } } - + if (check_dst) { struct ifreq ifr; @@ -1446,7 +1433,7 @@ void receive_query(struct listener *listen, time_t now) struct in_pktinfo *p; } p; p.c = CMSG_DATA(cmptr); - dst_addr_4 = dst_addr.addr.addr4 = p.p->ipi_spec_dst; + dst_addr_4 = dst_addr.addr4 = p.p->ipi_spec_dst; if_index = p.p->ipi_ifindex; } #elif defined(IP_RECVDSTADDR) && defined(IP_RECVIF) @@ -1464,7 +1451,7 @@ void receive_query(struct listener *listen, time_t now) } p; p.c = CMSG_DATA(cmptr); if (cmptr->cmsg_level == IPPROTO_IP && cmptr->cmsg_type == IP_RECVDSTADDR) - dst_addr_4 = dst_addr.addr.addr4 = *(p.a); + dst_addr_4 = dst_addr.addr4 = *(p.a); else if (cmptr->cmsg_level == IPPROTO_IP && cmptr->cmsg_type == IP_RECVIF) #ifdef HAVE_SOLARIS_NETWORK if_index = *(p.i); @@ -1474,8 +1461,7 @@ void receive_query(struct listener *listen, time_t now) } } #endif - -#ifdef HAVE_IPV6 + if (listen->family == AF_INET6) { for (cmptr = CMSG_FIRSTHDR(&msg); cmptr; cmptr = CMSG_NXTHDR(&msg, cmptr)) @@ -1486,22 +1472,21 @@ void receive_query(struct listener *listen, time_t now) struct in6_pktinfo *p; } p; p.c = CMSG_DATA(cmptr); - - dst_addr.addr.addr6 = p.p->ipi6_addr; + + dst_addr.addr6 = p.p->ipi6_addr; if_index = p.p->ipi6_ifindex; } } -#endif - + /* enforce available interface configuration */ - + if (!indextoname(listen->fd, if_index, ifr.ifr_name)) return; - + if (!iface_check(listen->family, &dst_addr, ifr.ifr_name, &auth_dns)) { if (!option_bool(OPT_CLEVERBIND)) - enumerate_interfaces(0); + enumerate_interfaces(0); if (!loopback_exception(listen->fd, listen->family, &dst_addr, ifr.ifr_name) && !label_exception(if_index, listen->family, &dst_addr)) return; @@ -1510,24 +1495,24 @@ void receive_query(struct listener *listen, time_t now) if (listen->family == AF_INET && option_bool(OPT_LOCALISE)) { struct irec *iface; - + /* get the netmask of the interface which has the address we were sent to. This is no necessarily the interface we arrived on. */ - + for (iface = daemon->interfaces; iface; iface = iface->next) if (iface->addr.sa.sa_family == AF_INET && iface->addr.in.sin_addr.s_addr == dst_addr_4.s_addr) break; - + /* interface may be new */ if (!iface && !option_bool(OPT_CLEVERBIND)) - enumerate_interfaces(0); - + enumerate_interfaces(0); + for (iface = daemon->interfaces; iface; iface = iface->next) if (iface->addr.sa.sa_family == AF_INET && iface->addr.in.sin_addr.s_addr == dst_addr_4.s_addr) break; - + /* If we failed, abandon localisation */ if (iface) netmask = iface->netmask; @@ -1535,8 +1520,8 @@ void receive_query(struct listener *listen, time_t now) dst_addr_4.s_addr = 0; } } - - /* log_query gets called indirectly all over the place, so + + /* log_query gets called indirectly all over the place, so pass these in global variables - sorry. */ daemon->log_display_id = ++daemon->log_id; daemon->log_source_addr = &source_addr; @@ -1544,30 +1529,28 @@ void receive_query(struct listener *listen, time_t now) #ifdef HAVE_DUMPFILE dump_packet(DUMP_QUERY, daemon->packet, (size_t)n, &source_addr, NULL); #endif - + if (extract_request(header, (size_t)n, daemon->namebuff, &type)) { #ifdef HAVE_AUTH struct auth_zone *zone; #endif char *types = querystr(auth_dns ? "auth" : "query", type); - - if (listen->family == AF_INET) + + if (listen->family == AF_INET) { - log_query(F_QUERY | F_IPV4 | F_FORWARD, daemon->namebuff, - (struct all_addr *)&source_addr.in.sin_addr, types); - FTL_new_query(F_QUERY | F_IPV4 | F_FORWARD, daemon->namebuff, - (struct all_addr *)&source_addr.in.sin_addr, types, daemon->log_display_id, UDP); + log_query(F_QUERY | F_IPV4 | F_FORWARD, daemon->namebuff, + (union all_addr *)&source_addr.in.sin_addr, types); + piholeblocked = FTL_new_query(F_QUERY | F_IPV4 | F_FORWARD, daemon->namebuff, &blockingreason, + (union all_addr *)&source_addr.in.sin_addr, types, daemon->log_display_id, UDP); } -#ifdef HAVE_IPV6 else { log_query(F_QUERY | F_IPV6 | F_FORWARD, daemon->namebuff, - (struct all_addr *)&source_addr.in6.sin6_addr, types); - FTL_new_query(F_QUERY | F_IPV6 | F_FORWARD, daemon->namebuff, - (struct all_addr *)&source_addr.in6.sin6_addr, types, daemon->log_display_id, UDP); + (union all_addr *)&source_addr.in6.sin6_addr, types); + piholeblocked = FTL_new_query(F_QUERY | F_IPV6 | F_FORWARD, daemon->namebuff, &blockingreason, + (union all_addr *)&source_addr.in6.sin6_addr, types, daemon->log_display_id, UDP); } -#endif #ifdef HAVE_AUTH /* find queries for zones we're authoritative for, and answer them directly */ @@ -1580,26 +1563,26 @@ void receive_query(struct listener *listen, time_t now) break; } #endif - + #ifdef HAVE_LOOP /* Check for forwarding loop */ if (detect_loop(daemon->namebuff, type)) return; #endif } - + if (find_pseudoheader(header, (size_t)n, NULL, &pheader, NULL, NULL)) - { + { unsigned short flags; - + have_pseudoheader = 1; GETSHORT(udp_size, pheader); pheader += 2; /* ext_rcode */ GETSHORT(flags, pheader); - + if (flags & 0x8000) - do_bit = 1;/* do bit */ - + do_bit = 1;/* do bit */ + /* If the client provides an EDNS0 UDP size, use that to limit our reply. (bounded by the maximum configured). If no EDNS0, then it defaults to 512 */ @@ -1612,7 +1595,7 @@ void receive_query(struct listener *listen, time_t now) #ifdef HAVE_AUTH if (auth_dns) { - m = answer_auth(header, ((char *) header) + udp_size, (size_t)n, now, &source_addr, + m = answer_auth(header, ((char *) header) + udp_size, (size_t)n, now, &source_addr, local_auth, do_bit, have_pseudoheader); if (m >= 1) { @@ -1629,9 +1612,26 @@ void receive_query(struct listener *listen, time_t now) if (header->hb4 & HB4_AD) ad_reqd = 1; - m = answer_request(header, ((char *) header) + udp_size, (size_t)n, - dst_addr_4, netmask, now, ad_reqd, do_bit, have_pseudoheader); + /************ Pi-hole modification ************/ + if(piholeblocked) + { + size_t plen = n; + union all_addr *addrp = NULL; + // DNS resource record type for AAAA is 28 (decimal) following RFC 3596, section 2.1 + unsigned int flags = (type == 28u) ? F_IPV6 : F_IPV4; + FTL_get_blocking_metadata(&addrp, &flags); + log_query(flags, daemon->namebuff, addrp, (char*)blockingreason); + plen = setup_reply(header, n, addrp, flags, daemon->local_ttl); + if (find_pseudoheader(header, plen, NULL, NULL, NULL, NULL)) + plen = add_pseudoheader(header, plen, ((unsigned char *) header) + PACKETSZ, daemon->edns_pktsz, 0, NULL, 0, do_bit, 0); + send_from(listen->fd, option_bool(OPT_NOWILD) || option_bool(OPT_CLEVERBIND), (char *)header, plen, (union mysockaddr*)&source_addr, &dst_addr, if_index); + return; + } + /**********************************************/ + m = answer_request(header, ((char *) header) + udp_size, (size_t)n, + dst_addr_4, netmask, now, ad_reqd, do_bit, have_pseudoheader); + if (m >= 1) { send_from(listen->fd, option_bool(OPT_NOWILD) || option_bool(OPT_CLEVERBIND), @@ -1648,8 +1648,8 @@ void receive_query(struct listener *listen, time_t now) #ifdef HAVE_DNSSEC /* Recurse up the key hierarchy */ -static int tcp_key_recurse(time_t now, int status, struct dns_header *header, size_t n, - int class, char *name, char *keyname, struct server *server, +static int tcp_key_recurse(time_t now, int status, struct dns_header *header, size_t n, + int class, char *name, char *keyname, struct server *server, int have_mark, unsigned int mark, int *keycount) { int new_status; @@ -1657,15 +1657,15 @@ static int tcp_key_recurse(time_t now, int status, struct dns_header *header, si unsigned char *payload = NULL; struct dns_header *new_header = NULL; u16 *length = NULL; - + while (1) { int type = SERV_DO_DNSSEC; char *domain; - size_t m; + size_t m; unsigned char c1, c2; struct server *firstsendto = NULL; - + /* limit the amount of work we do, to avoid cycling forever on loops in the DNS */ if (--(*keycount) == 0) new_status = STAT_ABANDONED; @@ -1673,11 +1673,11 @@ static int tcp_key_recurse(time_t now, int status, struct dns_header *header, si new_status = dnssec_validate_by_ds(now, header, n, name, keyname, class); else if (status == STAT_NEED_DS) new_status = dnssec_validate_ds(now, header, n, name, keyname, class); - else + else new_status = dnssec_validate_reply(now, header, n, name, keyname, &class, !option_bool(OPT_DNSSEC_IGN_NS) && (server->flags & SERV_DO_DNSSEC), - NULL, NULL); - + NULL, NULL, NULL); + if (new_status != STAT_NEED_DS && new_status != STAT_NEED_KEY) break; @@ -1690,29 +1690,31 @@ static int tcp_key_recurse(time_t now, int status, struct dns_header *header, si new_header = (struct dns_header *)payload; length = (u16 *)packet; } - + if (!packet) { new_status = STAT_ABANDONED; break; } - m = dnssec_generate_query(new_header, ((unsigned char *) new_header) + 65536, keyname, class, + m = dnssec_generate_query(new_header, ((unsigned char *) new_header) + 65536, keyname, class, new_status == STAT_NEED_KEY ? T_DNSKEY : T_DS, server->edns_pktsz); - + *length = htons(m); - /* Find server to forward to. This will normally be the + /* Find server to forward to. This will normally be the same as for the original query, but may be another if - servers for domains are involved. */ + servers for domains are involved. */ if (search_servers(now, NULL, F_DNSSECOK, keyname, &type, &domain, NULL) != 0) { new_status = STAT_ABANDONED; break; } - + while (1) { + int data_sent = 0; + if (!firstsendto) firstsendto = server; else @@ -1726,7 +1728,7 @@ static int tcp_key_recurse(time_t now, int status, struct dns_header *header, si break; } } - + if (type != (server->flags & (SERV_TYPE | SERV_DO_DNSSEC)) || (type == SERV_HAS_DOMAIN && !hostname_isequal(domain, server->domain)) || (server->flags & (SERV_LITERAL_ADDRESS | SERV_LOOP))) @@ -1738,25 +1740,39 @@ static int tcp_key_recurse(time_t now, int status, struct dns_header *header, si { if ((server->tcpfd = socket(server->addr.sa.sa_family, SOCK_STREAM, 0)) == -1) continue; /* No good, next server */ - + #ifdef HAVE_CONNTRACK /* Copy connection mark of incoming query to outgoing connection. */ if (have_mark) setsockopt(server->tcpfd, SOL_SOCKET, SO_MARK, &mark, sizeof(unsigned int)); +#endif + + if (!local_bind(server->tcpfd, &server->source_addr, server->interface, 0, 1)) + { + close(server->tcpfd); + server->tcpfd = -1; + continue; /* No good, next server */ + } + +#ifdef MSG_FASTOPEN + while(retry_send(sendto(server->tcpfd, packet, m + sizeof(u16), + MSG_FASTOPEN, &server->addr.sa, sa_len(&server->addr)))); + + if (errno == 0) + data_sent = 1; #endif - - if (!local_bind(server->tcpfd, &server->source_addr, server->interface, 0, 1) || - connect(server->tcpfd, &server->addr.sa, sa_len(&server->addr)) == -1) + + if (!data_sent && connect(server->tcpfd, &server->addr.sa, sa_len(&server->addr)) == -1) { close(server->tcpfd); server->tcpfd = -1; continue; /* No good, next server */ } - + server->flags &= ~SERV_GOT_TCP; } - - if (!read_write(server->tcpfd, packet, m + sizeof(u16), 0) || + + if ((!data_sent && !read_write(server->tcpfd, packet, m + sizeof(u16), 0)) || !read_write(server->tcpfd, &c1, 1, 1) || !read_write(server->tcpfd, &c2, 1, 1) || !read_write(server->tcpfd, payload, (c1 << 8) | c2, 1)) @@ -1773,29 +1789,27 @@ static int tcp_key_recurse(time_t now, int status, struct dns_header *header, si } - if (server->addr.sa.sa_family == AF_INET) - log_query(F_NOEXTRA | F_DNSSEC | F_IPV4, keyname, (struct all_addr *)&(server->addr.in.sin_addr), + if (server->addr.sa.sa_family == AF_INET) + log_query(F_NOEXTRA | F_DNSSEC | F_IPV4, keyname, (union all_addr *)&(server->addr.in.sin_addr), querystr("dnssec-query", new_status == STAT_NEED_KEY ? T_DNSKEY : T_DS)); -#ifdef HAVE_IPV6 else - log_query(F_NOEXTRA | F_DNSSEC | F_IPV6, keyname, (struct all_addr *)&(server->addr.in6.sin6_addr), + log_query(F_NOEXTRA | F_DNSSEC | F_IPV6, keyname, (union all_addr *)&(server->addr.in6.sin6_addr), querystr("dnssec-query", new_status == STAT_NEED_KEY ? T_DNSKEY : T_DS)); -#endif - + server->flags |= SERV_GOT_TCP; - + m = (c1 << 8) | c2; new_status = tcp_key_recurse(now, new_status, new_header, m, class, name, keyname, server, have_mark, mark, keycount); break; } - + if (new_status != STAT_OK) break; } - + if (packet) free(packet); - + return new_status; } #endif @@ -1837,6 +1851,11 @@ unsigned char *tcp_request(int confd, time_t now, (void)mark; (void)have_mark; + /************ Pi-hole modification ************/ + bool piholeblocked = false; + const char* blockingreason = NULL; + /**********************************************/ + if (getpeername(confd, (struct sockaddr *)&peer_addr, &peer_len) == -1) return packet; @@ -1844,39 +1863,37 @@ unsigned char *tcp_request(int confd, time_t now, /* Get connection mark of incoming query to set on outgoing connections. */ if (option_bool(OPT_CONNTRACK)) { - struct all_addr local; -#ifdef HAVE_IPV6 + union all_addr local; + if (local_addr->sa.sa_family == AF_INET6) - local.addr.addr6 = local_addr->in6.sin6_addr; + local.addr6 = local_addr->in6.sin6_addr; else -#endif - local.addr.addr4 = local_addr->in.sin_addr; - + local.addr4 = local_addr->in.sin_addr; + have_mark = get_incoming_mark(&peer_addr, &local, 1, &mark); } -#endif +#endif /* We can be configured to only accept queries from at-most-one-hop-away addresses. */ if (option_bool(OPT_LOCAL_SERVICE)) { struct addrlist *addr; -#ifdef HAVE_IPV6 - if (peer_addr.sa.sa_family == AF_INET6) + + if (peer_addr.sa.sa_family == AF_INET6) { for (addr = daemon->interface_addrs; addr; addr = addr->next) if ((addr->flags & ADDRLIST_IPV6) && - is_same_net6(&addr->addr.addr.addr6, &peer_addr.in6.sin6_addr, addr->prefixlen)) + is_same_net6(&addr->addr.addr6, &peer_addr.in6.sin6_addr, addr->prefixlen)) break; } else -#endif { struct in_addr netmask; for (addr = daemon->interface_addrs; addr; addr = addr->next) { netmask.s_addr = htonl(~(in_addr_t)0 << (32 - addr->prefixlen)); - if (!(addr->flags & ADDRLIST_IPV6) && - is_same_net(addr->addr.addr.addr4, peer_addr.in.sin_addr, netmask)) + if (!(addr->flags & ADDRLIST_IPV6) && + is_same_net(addr->addr.addr4, peer_addr.in.sin_addr, netmask)) break; } } @@ -1894,50 +1911,48 @@ unsigned char *tcp_request(int confd, time_t now, !read_write(confd, &c1, 1, 1) || !read_write(confd, &c2, 1, 1) || !(size = c1 << 8 | c2) || !read_write(confd, payload, size, 1)) - return packet; - + return packet; + if (size < (int)sizeof(struct dns_header)) continue; /* Clear buffer beyond request to avoid risk of information disclosure. */ memset(payload + size, 0, 65536 - size); - + query_count++; - /* log_query gets called indirectly all over the place, so + /* log_query gets called indirectly all over the place, so pass these in global variables - sorry. */ daemon->log_display_id = ++daemon->log_id; daemon->log_source_addr = &peer_addr; - + /* save state of "cd" flag in query */ if ((checking_disabled = header->hb4 & HB4_CD)) no_cache_dnssec = 1; - + if ((gotname = extract_request(header, (unsigned int)size, daemon->namebuff, &qtype))) { #ifdef HAVE_AUTH struct auth_zone *zone; #endif char *types = querystr(auth_dns ? "auth" : "query", qtype); - - if (peer_addr.sa.sa_family == AF_INET) + + if (peer_addr.sa.sa_family == AF_INET) { - log_query(F_QUERY | F_IPV4 | F_FORWARD, daemon->namebuff, - (struct all_addr *)&peer_addr.in.sin_addr, types); - FTL_new_query(F_QUERY | F_IPV4 | F_FORWARD, daemon->namebuff, - (struct all_addr *)&peer_addr.in.sin_addr, types, daemon->log_display_id, TCP); + log_query(F_QUERY | F_IPV4 | F_FORWARD, daemon->namebuff, + (union all_addr *)&peer_addr.in.sin_addr, types); + piholeblocked = FTL_new_query(F_QUERY | F_IPV4 | F_FORWARD, daemon->namebuff, &blockingreason, + (union all_addr *)&peer_addr.in.sin_addr, types, daemon->log_display_id, TCP); } -#ifdef HAVE_IPV6 else { log_query(F_QUERY | F_IPV6 | F_FORWARD, daemon->namebuff, - (struct all_addr *)&peer_addr.in6.sin6_addr, types); - FTL_new_query(F_QUERY | F_IPV6 | F_FORWARD, daemon->namebuff, - (struct all_addr *)&peer_addr.in6.sin6_addr, types, daemon->log_display_id, TCP); + (union all_addr *)&peer_addr.in6.sin6_addr, types); + piholeblocked = FTL_new_query(F_QUERY | F_IPV6 | F_FORWARD, daemon->namebuff, &blockingreason, + (union all_addr *)&peer_addr.in6.sin6_addr, types, daemon->log_display_id, TCP); } -#endif - + #ifdef HAVE_AUTH /* find queries for zones we're authoritative for, and answer them directly */ if (!auth_dns && !option_bool(OPT_LOCALISE)) @@ -1950,29 +1965,29 @@ unsigned char *tcp_request(int confd, time_t now, } #endif } - + if (local_addr->sa.sa_family == AF_INET) dst_addr_4 = local_addr->in.sin_addr; else dst_addr_4.s_addr = 0; - + do_bit = 0; if (find_pseudoheader(header, (size_t)size, NULL, &pheader, NULL, NULL)) - { + { unsigned short flags; - + have_pseudoheader = 1; pheader += 4; /* udp_size, ext_rcode */ GETSHORT(flags, pheader); - + if (flags & 0x8000) - do_bit = 1; /* do bit */ + do_bit = 1; /* do bit */ } #ifdef HAVE_AUTH if (auth_dns) - m = answer_auth(header, ((char *) header) + 65536, (size_t)size, now, &peer_addr, + m = answer_auth(header, ((char *) header) + 65536, (size_t)size, now, &peer_addr, local_auth, do_bit, have_pseudoheader); else #endif @@ -1981,18 +1996,34 @@ unsigned char *tcp_request(int confd, time_t now, /* RFC 6840 5.7 */ if (header->hb4 & HB4_AD) ad_reqd = 1; - + + /* Do this by steam now we're not in the select() loop */ + check_log_writer(1); + + /************ Pi-hole modification ************/ + if(piholeblocked) + { + union all_addr *addrp = NULL; + // DNS resource record type for AAAA is 28 (decimal) following RFC 3596, section 2.1 + unsigned int flags = (qtype == 28u) ? F_IPV6 : F_IPV4; + FTL_get_blocking_metadata(&addrp, &flags); + log_query(flags, daemon->namebuff, addrp, (char*)blockingreason); + m = setup_reply(header, size, addrp, flags, daemon->local_ttl); + if (have_pseudoheader) + m = add_pseudoheader(header, m, ((unsigned char *) header) + 65536, daemon->edns_pktsz, 0, NULL, 0, do_bit, 0); + } + else + { + /**********************************************/ + /* m > 0 if answered from cache */ - m = answer_request(header, ((char *) header) + 65536, (size_t)size, + m = answer_request(header, ((char *) header) + 65536, (size_t)size, dst_addr_4, netmask, now, ad_reqd, do_bit, have_pseudoheader); - - /* Do this by steam now we're not in the select() loop */ - check_log_writer(1); - + if (m == 0) { unsigned int flags = 0; - struct all_addr *addrp = NULL; + union all_addr *addrp = NULL; int type = SERV_DO_DNSSEC; char *domain = NULL; unsigned char *oph = find_pseudoheader(header, size, NULL, NULL, NULL, NULL); @@ -2006,7 +2037,7 @@ unsigned char *tcp_request(int confd, time_t now, if (option_bool(OPT_DNSSEC_VALID) && (type & SERV_DO_DNSSEC)) { size = add_do_bit(header, size, ((unsigned char *) header) + 65536); - + /* For debugging, set Checking Disabled, otherwise, have the upstream check too, this allows it to select auth servers when one is returning bad data. */ if (option_bool(OPT_DNSSEC_DEBUG)) @@ -2020,12 +2051,12 @@ unsigned char *tcp_request(int confd, time_t now, added_pheader = 1; type &= ~SERV_DO_DNSSEC; - + if (type != 0 || option_bool(OPT_ORDER) || !daemon->last_server) last_server = daemon->servers; else last_server = daemon->last_server; - + if (!flags && last_server) { struct server *firstsendto = NULL; @@ -2037,23 +2068,25 @@ unsigned char *tcp_request(int confd, time_t now, memset(hash, 0, HASH_SIZE); #else unsigned int crc = questions_crc(header, (unsigned int)size, daemon->namebuff); -#endif +#endif /* Loop round available servers until we succeed in connecting to one. Note that this code subtly ensures that consecutive queries on this connection which can go to the same server, do so. */ - while (1) + while (1) { + int data_sent = 0; + if (!firstsendto) firstsendto = last_server; else { if (!(last_server = last_server->next)) last_server = daemon->servers; - + if (last_server == firstsendto) break; } - + /* server for wrong domain */ if (type != (last_server->flags & SERV_TYPE) || (type == SERV_HAS_DOMAIN && !hostname_isequal(domain, last_server->domain)) || @@ -2061,35 +2094,49 @@ unsigned char *tcp_request(int confd, time_t now, continue; retry: + *length = htons(size); + if (last_server->tcpfd == -1) { if ((last_server->tcpfd = socket(last_server->addr.sa.sa_family, SOCK_STREAM, 0)) == -1) continue; - + #ifdef HAVE_CONNTRACK /* Copy connection mark of incoming query to outgoing connection. */ if (have_mark) setsockopt(last_server->tcpfd, SOL_SOCKET, SO_MARK, &mark, sizeof(unsigned int)); -#endif - - if ((!local_bind(last_server->tcpfd, &last_server->source_addr, last_server->interface, 0, 1) || - connect(last_server->tcpfd, &last_server->addr.sa, sa_len(&last_server->addr)) == -1)) +#endif + + if ((!local_bind(last_server->tcpfd, &last_server->source_addr, last_server->interface, 0, 1))) { close(last_server->tcpfd); last_server->tcpfd = -1; continue; } - + +#ifdef MSG_FASTOPEN + while(retry_send(sendto(last_server->tcpfd, packet, size + sizeof(u16), + MSG_FASTOPEN, &last_server->addr.sa, sa_len(&last_server->addr)))); + + if (errno == 0) + data_sent = 1; +#endif + + if (!data_sent && connect(last_server->tcpfd, &last_server->addr.sa, sa_len(&last_server->addr)) == -1) + { + close(last_server->tcpfd); + last_server->tcpfd = -1; + continue; + } + last_server->flags &= ~SERV_GOT_TCP; } - - *length = htons(size); - + /* get query name again for logging - may have been overwritten */ if (!(gotname = extract_request(header, (unsigned int)size, daemon->namebuff, &qtype))) strcpy(daemon->namebuff, "query"); - - if (!read_write(last_server->tcpfd, packet, size + sizeof(u16), 0) || + + if ((!data_sent && !read_write(last_server->tcpfd, packet, size + sizeof(u16), 0)) || !read_write(last_server->tcpfd, &c1, 1, 1) || !read_write(last_server->tcpfd, &c2, 1, 1) || !read_write(last_server->tcpfd, payload, (c1 << 8) | c2, 1)) @@ -2104,36 +2151,34 @@ unsigned char *tcp_request(int confd, time_t now, else continue; } - + last_server->flags |= SERV_GOT_TCP; m = (c1 << 8) | c2; - + if (last_server->addr.sa.sa_family == AF_INET) { - log_query(F_SERVER | F_IPV4 | F_FORWARD, daemon->namebuff, - (struct all_addr *)&last_server->addr.in.sin_addr, NULL); + log_query(F_SERVER | F_IPV4 | F_FORWARD, daemon->namebuff, + (union all_addr *)&last_server->addr.in.sin_addr, NULL); FTL_forwarded(F_SERVER | F_IPV4 | F_FORWARD, daemon->namebuff, - (struct all_addr *)&last_server->addr.in.sin_addr, daemon->log_display_id); + (union all_addr *)&last_server->addr.in.sin_addr, daemon->log_display_id); } -#ifdef HAVE_IPV6 else { - log_query(F_SERVER | F_IPV6 | F_FORWARD, daemon->namebuff, - (struct all_addr *)&last_server->addr.in6.sin6_addr, NULL); + log_query(F_SERVER | F_IPV6 | F_FORWARD, daemon->namebuff, + (union all_addr *)&last_server->addr.in6.sin6_addr, NULL); FTL_forwarded(F_SERVER | F_IPV6 | F_FORWARD, daemon->namebuff, - (struct all_addr *)&last_server->addr.in6.sin6_addr, daemon->log_display_id); + (union all_addr *)&last_server->addr.in6.sin6_addr, daemon->log_display_id); } -#endif #ifdef HAVE_DNSSEC if (option_bool(OPT_DNSSEC_VALID) && !checking_disabled && (last_server->flags & SERV_DO_DNSSEC)) { int keycount = DNSSEC_WORK; /* Limit to number of DNSSEC questions, to catch loops and avoid filling cache. */ - int status = tcp_key_recurse(now, STAT_OK, header, m, 0, daemon->namebuff, daemon->keyname, + int status = tcp_key_recurse(now, STAT_OK, header, m, 0, daemon->namebuff, daemon->keyname, last_server, have_mark, mark, &keycount); char *result, *domain = "result"; - + if (status == STAT_ABANDONED) { result = "ABANDONED"; @@ -2141,13 +2186,13 @@ unsigned char *tcp_request(int confd, time_t now, } else result = (status == STAT_SECURE ? "SECURE" : (status == STAT_INSECURE ? "INSECURE" : "BOGUS")); - + if (status == STAT_BOGUS && extract_request(header, m, daemon->namebuff, NULL)) domain = daemon->namebuff; log_query(F_SECSTAT, domain, NULL, result); FTL_dnssec(status, daemon->log_display_id); - + if (status == STAT_BOGUS) { no_cache_dnssec = 1; @@ -2164,21 +2209,21 @@ unsigned char *tcp_request(int confd, time_t now, header->hb4 |= HB4_CD; else header->hb4 &= ~HB4_CD; - + /* There's no point in updating the cache, since this process will exit and - lose the information after a few queries. We make this call for the alias and + lose the information after a few queries. We make this call for the alias and bogus-nxdomain side-effects. */ /* If the crc of the question section doesn't match the crc we sent, then - someone might be attempting to insert bogus values into the cache by + someone might be attempting to insert bogus values into the cache by sending replies containing questions and bogus answers. */ #ifdef HAVE_DNSSEC newhash = hash_questions(header, (unsigned int)m, daemon->namebuff); if (!newhash || memcmp(hash, newhash, HASH_SIZE) != 0) - { + { m = 0; break; } -#else +#else if (crc != questions_crc(header, (unsigned int)m, daemon->namebuff)) { m = 0; @@ -2186,14 +2231,14 @@ unsigned char *tcp_request(int confd, time_t now, } #endif - m = process_reply(header, now, last_server, (unsigned int)m, + m = process_reply(header, now, last_server, (unsigned int)m, option_bool(OPT_NO_REBIND) && !norebind, no_cache_dnssec, cache_secure, bogusanswer, - ad_reqd, do_bit, added_pheader, check_subnet, &peer_addr); - + ad_reqd, do_bit, added_pheader, check_subnet, &peer_addr); + break; } } - + /* In case of local answer or no connections made. */ if (m == 0) { @@ -2202,12 +2247,15 @@ unsigned char *tcp_request(int confd, time_t now, m = add_pseudoheader(header, m, ((unsigned char *) header) + 65536, daemon->edns_pktsz, 0, NULL, 0, do_bit, 0); } } + /************ Pi-hole modification ************/ + } + /**********************************************/ } - + check_log_writer(1); - + *length = htons(m); - + if (m == 0 || !read_write(confd, packet, m + sizeof(u16), 0)) return packet; } @@ -2216,7 +2264,7 @@ unsigned char *tcp_request(int confd, time_t now, static struct frec *allocate_frec(time_t now) { struct frec *f; - + if ((f = (struct frec *)whine_malloc(sizeof(struct frec)))) { f->next = daemon->frec_list; @@ -2224,9 +2272,7 @@ static struct frec *allocate_frec(time_t now) f->sentto = NULL; f->rfd4 = NULL; f->flags = 0; -#ifdef HAVE_IPV6 f->rfd6 = NULL; -#endif #ifdef HAVE_DNSSEC f->dependent = NULL; f->blocking_query = NULL; @@ -2243,7 +2289,7 @@ struct randfd *allocate_rfd(int family) static int finger = 0; int i; - /* limit the number of sockets we have open to avoid starvation of + /* limit the number of sockets we have open to avoid starvation of (eg) TFTP. Once we have a reasonable number, randomness should be OK */ for (i = 0; i < RANDOM_SOCKS; i++) @@ -2251,7 +2297,7 @@ struct randfd *allocate_rfd(int family) { if ((daemon->randomsocks[i].fd = random_sock(family)) == -1) break; - + daemon->randomsocks[i].refcount = 1; daemon->randomsocks[i].family = family; return &daemon->randomsocks[i]; @@ -2262,7 +2308,7 @@ struct randfd *allocate_rfd(int family) { int j = (i+finger) % RANDOM_SOCKS; if (daemon->randomsocks[j].refcount != 0 && - daemon->randomsocks[j].family == family && + daemon->randomsocks[j].family == family && daemon->randomsocks[j].refcount != 0xffff) { finger = j; @@ -2286,11 +2332,8 @@ static void free_frec(struct frec *f) f->rfd4 = NULL; f->sentto = NULL; f->flags = 0; - -#ifdef HAVE_IPV6 free_rfd(f->rfd6); f->rfd6 = NULL; -#endif #ifdef HAVE_DNSSEC if (f->stash) @@ -2313,26 +2356,27 @@ static void free_frec(struct frec *f) else return *wait zero if one available, or *wait is delay to when the oldest in-use record will expire. Impose an absolute limit of 4*TIMEOUT before we wipe things (for random sockets). - If force is set, always return a result, even if we have - to allocate above the limit. */ -struct frec *get_new_frec(time_t now, int *wait, int force) + If force is non-NULL, always return a result, even if we have + to allocate above the limit, and never free the record pointed + to by the force argument. */ +struct frec *get_new_frec(time_t now, int *wait, struct frec *force) { struct frec *f, *oldest, *target; int count; - + if (wait) *wait = 0; for (f = daemon->frec_list, oldest = NULL, target = NULL, count = 0; f; f = f->next, count++) if (!f->sentto) target = f; - else + else { #ifdef HAVE_DNSSEC /* Don't free DNSSEC sub-queries here, as we may end up with - dangling references to them. They'll go when their "real" query + dangling references to them. They'll go when their "real" query is freed. */ - if (!f->dependent) + if (!f->dependent && f != force) #endif { if (difftime(now, f->time) >= 4*TIMEOUT) @@ -2340,8 +2384,8 @@ struct frec *get_new_frec(time_t now, int *wait, int force) free_frec(f); target = f; } - - + + if (!oldest || difftime(f->time, oldest->time) <= 0) oldest = f; } @@ -2352,14 +2396,14 @@ struct frec *get_new_frec(time_t now, int *wait, int force) target->time = now; return target; } - + /* can't find empty one, use oldest if there is one and it's older than timeout */ if (!force && oldest && ((int)difftime(now, oldest->time)) >= TIMEOUT) - { + { /* keep stuff for twice timeout if we can by allocating a new record instead */ - if (difftime(now, oldest->time) < 2*TIMEOUT && + if (difftime(now, oldest->time) < 2*TIMEOUT && count <= daemon->ftabsize && (f = allocate_frec(now))) return f; @@ -2371,15 +2415,15 @@ struct frec *get_new_frec(time_t now, int *wait, int force) } return oldest; } - + /* none available, calculate time 'till oldest record expires */ if (!force && count > daemon->ftabsize) { static time_t last_log = 0; - + if (oldest && wait) *wait = oldest->time + (time_t)TIMEOUT - now; - + if ((int)difftime(now, last_log) > 5) { last_log = now; @@ -2388,7 +2432,7 @@ struct frec *get_new_frec(time_t now, int *wait, int force) return NULL; } - + if (!(f = allocate_frec(now)) && wait) /* wait one second on malloc failure */ *wait = 1; @@ -2402,10 +2446,10 @@ static struct frec *lookup_frec(unsigned short id, void *hash) struct frec *f; for(f = daemon->frec_list; f; f = f->next) - if (f->sentto && f->new_id == id && + if (f->sentto && f->new_id == id && (!hash || memcmp(hash, f->hash, HASH_SIZE) == 0)) return f; - + return NULL; } @@ -2414,34 +2458,34 @@ static struct frec *lookup_frec_by_sender(unsigned short id, void *hash) { struct frec *f; - + for(f = daemon->frec_list; f; f = f->next) if (f->sentto && - f->orig_id == id && + f->orig_id == id && memcmp(hash, f->hash, HASH_SIZE) == 0 && sockaddr_isequal(&f->source, addr)) return f; - + return NULL; } - + /* Send query packet again, if we can. */ void resend_query() { if (daemon->srv_save) { int fd; - + if (daemon->srv_save->sfd) fd = daemon->srv_save->sfd->fd; else if (daemon->rfd_save && daemon->rfd_save->refcount != 0) fd = daemon->rfd_save->fd; else return; - + while(retry_send(sendto(fd, daemon->packet, daemon->packet_len, 0, - &daemon->srv_save->addr.sa, - sa_len(&daemon->srv_save->addr)))); + &daemon->srv_save->addr.sa, + sa_len(&daemon->srv_save->addr)))); } } @@ -2449,11 +2493,11 @@ void resend_query() void server_gone(struct server *server) { struct frec *f; - + for (f = daemon->frec_list; f; f = f->next) if (f->sentto && f->sentto == server) free_frec(f); - + if (daemon->last_server == server) daemon->last_server = NULL; @@ -2465,11 +2509,11 @@ void server_gone(struct server *server) static unsigned short get_id(void) { unsigned short ret = 0; - - do + + do ret = rand16(); while (lookup_frec(ret, NULL)); - + return ret; } diff --git a/dnsmasq/helper.c b/src/dnsmasq/helper.c similarity index 97% rename from dnsmasq/helper.c rename to src/dnsmasq/helper.c index c13407150..7072cf475 100644 --- a/dnsmasq/helper.c +++ b/src/dnsmasq/helper.c @@ -1,4 +1,4 @@ -/* dnsmasq is Copyright (c) 2000-2018 Simon Kelley +/* dnsmasq is Copyright (c) 2000-2020 Simon Kelley This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -64,11 +64,10 @@ struct script_data #ifdef HAVE_TFTP off_t file_len; #endif -#ifdef HAVE_IPV6 struct in6_addr addr6; -#endif #ifdef HAVE_DHCP6 - int iaid, vendorclass_count; + int vendorclass_count; + unsigned int iaid; #endif unsigned char hwaddr[DHCP_CHADDR_MAX]; char interface[IF_NAMESIZE]; @@ -82,7 +81,8 @@ int create_helper(int event_fd, int err_fd, uid_t uid, gid_t gid, long max_fd) pid_t pid; int i, pipefd[2]; struct sigaction sigact; - + unsigned char *alloc_buff = NULL; + /* create the pipe through which the main program sends us commands, then fork our process. */ if (pipe(pipefd) == -1 || !fix_fd(pipefd[1]) || (pid = fork()) == -1) @@ -131,12 +131,8 @@ int create_helper(int event_fd, int err_fd, uid_t uid, gid_t gid, long max_fd) Don't close err_fd, in case the lua-init fails. Note that we have to do this before lua init so we don't close any lua fds. */ - for (max_fd--; max_fd >= 0; max_fd--) - if (max_fd != STDOUT_FILENO && max_fd != STDERR_FILENO && - max_fd != STDIN_FILENO && max_fd != pipefd[0] && - max_fd != event_fd && max_fd != err_fd) - close(max_fd); - + close_fds(max_fd, pipefd[0], event_fd, err_fd); + #ifdef HAVE_LUASCRIPT if (daemon->luascript) { @@ -188,11 +184,16 @@ int create_helper(int event_fd, int err_fd, uid_t uid, gid_t gid, long max_fd) struct script_data data; char *p, *action_str, *hostname = NULL, *domain = NULL; unsigned char *buf = (unsigned char *)daemon->namebuff; - unsigned char *end, *extradata, *alloc_buff = NULL; + unsigned char *end, *extradata; int is6, err = 0; int pipeout[2]; - free(alloc_buff); + /* Free rarely-allocated memory from previous iteration. */ + if (alloc_buff) + { + free(alloc_buff); + alloc_buff = NULL; + } /* we read zero bytes when pipe closed: this is our signal to exit */ if (!read_write(pipefd[0], (unsigned char *)&data, sizeof(data), 1)) @@ -302,10 +303,8 @@ int create_helper(int event_fd, int err_fd, uid_t uid, gid_t gid, long max_fd) if (!is6) inet_ntop(AF_INET, &data.addr, daemon->addrbuff, ADDRSTRLEN); -#ifdef HAVE_IPV6 else inet_ntop(AF_INET6, &data.addr6, daemon->addrbuff, ADDRSTRLEN); -#endif #ifdef HAVE_TFTP /* file length */ @@ -826,10 +825,8 @@ void queue_tftp(off_t file_len, char *filename, union mysockaddr *peer) if ((buf->flags = peer->sa.sa_family) == AF_INET) buf->addr = peer->in.sin_addr; -#ifdef HAVE_IPV6 else buf->addr6 = peer->in6.sin6_addr; -#endif memcpy((unsigned char *)(buf+1), filename, filename_len); @@ -837,7 +834,7 @@ void queue_tftp(off_t file_len, char *filename, union mysockaddr *peer) } #endif -void queue_arp(int action, unsigned char *mac, int maclen, int family, struct all_addr *addr) +void queue_arp(int action, unsigned char *mac, int maclen, int family, union all_addr *addr) { /* no script */ if (daemon->helperfd == -1) @@ -850,11 +847,9 @@ void queue_arp(int action, unsigned char *mac, int maclen, int family, struct al buf->hwaddr_len = maclen; buf->hwaddr_type = ARPHRD_ETHER; if ((buf->flags = family) == AF_INET) - buf->addr = addr->addr.addr4; -#ifdef HAVE_IPV6 + buf->addr = addr->addr4; else - buf->addr6 = addr->addr.addr6; -#endif + buf->addr6 = addr->addr6; memcpy(buf->hwaddr, mac, maclen); diff --git a/dnsmasq/inotify.c b/src/dnsmasq/inotify.c similarity index 93% rename from dnsmasq/inotify.c rename to src/dnsmasq/inotify.c index b76d9fd1b..d5804cbc2 100644 --- a/dnsmasq/inotify.c +++ b/src/dnsmasq/inotify.c @@ -1,15 +1,15 @@ -/* dnsmasq is Copyright (c) 2000-2018 Simon Kelley - +/* dnsmasq is Copyright (c) 2000-2020 Simon Kelley + This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; version 2 dated June, 1991, or (at your option) version 3 dated 29 June, 2007. - + This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. - + You should have received a copy of the GNU General Public License along with this program. If not, see . */ @@ -21,16 +21,16 @@ #include /* For MAXSYMLINKS */ /* the strategy is to set an inotify on the directories containing - resolv files, for any files in the directory which are close-write + resolv files, for any files in the directory which are close-write or moved into the directory. - + When either of those happen, we look to see if the file involved is actually a resolv-file, and if so, call poll-resolv with the "force" argument, to ensure it's read. This adds one new error condition: the directories containing all specified resolv-files must exist at start-up, even if the actual - files don't. + files don't. */ static char *inotify_buffer; @@ -49,7 +49,7 @@ static char *my_readlink(char *path) { buf = safe_malloc(size); rc = readlink(path, buf, (size_t)size); - + if (rc == -1) { /* Not link or doesn't exist. */ @@ -64,7 +64,7 @@ static char *my_readlink(char *path) else if (rc < size-1) { char *d; - + buf[rc] = 0; if (buf[0] != '/' && ((d = strrchr(path, '/')))) { @@ -90,13 +90,13 @@ void inotify_dnsmasq_init() struct resolvc *res; inotify_buffer = safe_malloc(INOTIFY_SZ); daemon->inotifyfd = inotify_init1(IN_NONBLOCK | IN_CLOEXEC); - + if (daemon->inotifyfd == -1) die(_("failed to create inotify: %s"), NULL, EC_MISC); if (option_bool(OPT_NO_RESOLV)) return; - + for (res = daemon->resolv_files; res; res = res->next) { char *d, *new_path, *path = safe_malloc(strlen(res->name) + 1); @@ -122,14 +122,14 @@ void inotify_dnsmasq_init() res->file = d+1; /* pointer to filename */ *d = '/'; - + if (res->wd == -1 && errno == ENOENT) die(_("directory %s for resolv-file is missing, cannot poll"), res->name, EC_MISC); - } - + } + if (res->wd == -1) die(_("failed to create inotify for %s: %s"), res->name, EC_MISC); - + } } @@ -138,23 +138,23 @@ void inotify_dnsmasq_init() void set_dynamic_inotify(int flag, int total_size, struct crec **rhash, int revhashsz) { struct hostsfile *ah; - + for (ah = daemon->dynamic_dirs; ah; ah = ah->next) { DIR *dir_stream = NULL; struct dirent *ent; struct stat buf; - + if (!(ah->flags & flag)) continue; - + if (stat(ah->fname, &buf) == -1 || !(S_ISDIR(buf.st_mode))) { - my_syslog(LOG_ERR, _("bad dynamic directory %s: %s"), + my_syslog(LOG_ERR, _("bad dynamic directory %s: %s"), ah->fname, strerror(errno)); continue; } - + if (!(ah->flags & AH_WD_DONE)) { ah->wd = inotify_add_watch(daemon->inotifyfd, ah->fname, IN_CLOSE_WRITE | IN_MOVED_TO); @@ -175,20 +175,20 @@ void set_dynamic_inotify(int flag, int total_size, struct crec **rhash, int revh size_t lendir = strlen(ah->fname); size_t lenfile = strlen(ent->d_name); char *path; - + /* ignore emacs backups and dotfiles */ - if (lenfile == 0 || + if (lenfile == 0 || ent->d_name[lenfile - 1] == '~' || (ent->d_name[0] == '#' && ent->d_name[lenfile - 1] == '#') || ent->d_name[0] == '.') continue; - + if ((path = whine_malloc(lendir + lenfile + 2))) { strcpy(path, ah->fname); strcat(path, "/"); strcat(path, ent->d_name); - + /* ignore non-regular files */ if (stat(path, &buf) != -1 && S_ISREG(buf.st_mode)) { @@ -197,7 +197,7 @@ void set_dynamic_inotify(int flag, int total_size, struct crec **rhash, int revh #ifdef HAVE_DHCP else if (ah->flags & (AH_DHCP_HST | AH_DHCP_OPT)) option_read_dynfile(path, ah->flags); -#endif +#endif } free(path); @@ -221,16 +221,16 @@ int inotify_check(time_t now) struct inotify_event *in; while ((rc = read(daemon->inotifyfd, inotify_buffer, INOTIFY_SZ)) == -1 && errno == EINTR); - + if (rc <= 0) break; - - for (p = inotify_buffer; rc - (p - inotify_buffer) >= (int)sizeof(struct inotify_event); p += sizeof(struct inotify_event) + in->len) + + for (p = inotify_buffer; rc - (p - inotify_buffer) >= (int)sizeof(struct inotify_event); p += sizeof(struct inotify_event) + in->len) { size_t namelen; in = (struct inotify_event*)p; - + /* ignore emacs backups and dotfiles */ if (in->len == 0 || (namelen = strlen(in->name)) == 0 || in->name[namelen - 1] == '~' || @@ -247,25 +247,25 @@ int inotify_check(time_t now) { size_t lendir = strlen(ah->fname); char *path; - + if ((path = whine_malloc(lendir + in->len + 2))) { strcpy(path, ah->fname); strcat(path, "/"); strcat(path, in->name); - + my_syslog(LOG_INFO, _("inotify, new or changed file %s"), path); if (ah->flags & AH_HOSTS) { read_hostsfile(path, ah->index, 0, NULL, 0); #ifdef HAVE_DHCP - if (daemon->dhcp || daemon->doing_dhcp6) + if (daemon->dhcp || daemon->doing_dhcp6) { /* Propagate the consequences of loading a new dhcp-host */ dhcp_update_configs(daemon->dhcp_conf); - lease_update_from_configs(); - lease_update_file(now); + lease_update_from_configs(); + lease_update_file(now); lease_update_dns(1); } #endif @@ -277,15 +277,15 @@ int inotify_check(time_t now) { /* Propagate the consequences of loading a new dhcp-host */ dhcp_update_configs(daemon->dhcp_conf); - lease_update_from_configs(); - lease_update_file(now); + lease_update_from_configs(); + lease_update_file(now); lease_update_dns(1); } } else if (ah->flags & AH_DHCP_OPT) option_read_dynfile(path, AH_DHCP_OPT); #endif - + free(path); } } @@ -295,4 +295,4 @@ int inotify_check(time_t now) } #endif /* INOTIFY */ - + diff --git a/dnsmasq/ip6addr.h b/src/dnsmasq/ip6addr.h similarity index 96% rename from dnsmasq/ip6addr.h rename to src/dnsmasq/ip6addr.h index 6f6dff7e0..352b33005 100644 --- a/dnsmasq/ip6addr.h +++ b/src/dnsmasq/ip6addr.h @@ -1,4 +1,4 @@ -/* dnsmasq is Copyright (c) 2000-2018 Simon Kelley +/* dnsmasq is Copyright (c) 2000-2020 Simon Kelley This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by diff --git a/dnsmasq/ipset.c b/src/dnsmasq/ipset.c similarity index 85% rename from dnsmasq/ipset.c rename to src/dnsmasq/ipset.c index 3a5ecc581..0c014cb72 100644 --- a/dnsmasq/ipset.c +++ b/src/dnsmasq/ipset.c @@ -22,9 +22,7 @@ #include #include #include -#include #include -#include #include /* We want to be able to compile against old header files @@ -87,20 +85,7 @@ static inline void add_attr(struct nlmsghdr *nlh, uint16_t type, size_t len, con void ipset_init(void) { - struct utsname utsname; - int version; - char *split; - - if (uname(&utsname) < 0) - die(_("failed to find kernel version: %s"), NULL, EC_MISC); - - split = strtok(utsname.release, "."); - version = (split ? atoi(split) : 0); - split = strtok(NULL, "."); - version = version * 256 + (split ? atoi(split) : 0); - split = strtok(NULL, "."); - version = version * 256 + (split ? atoi(split) : 0); - old_kernel = (version < KERNEL_VERSION(2,6,32)); + old_kernel = (daemon->kernel_version < KERNEL_VERSION(2,6,32)); if (old_kernel && (ipset_sock = socket(AF_INET, SOCK_RAW, IPPROTO_RAW)) != -1) return; @@ -114,19 +99,14 @@ void ipset_init(void) die (_("failed to create IPset control socket: %s"), NULL, EC_MISC); } -static int new_add_to_ipset(const char *setname, const struct all_addr *ipaddr, int af, int remove) +static int new_add_to_ipset(const char *setname, const union all_addr *ipaddr, int af, int remove) { struct nlmsghdr *nlh; struct my_nfgenmsg *nfg; struct my_nlattr *nested[2]; uint8_t proto; - int addrsz = INADDRSZ; + int addrsz = (af == AF_INET6) ? IN6ADDRSZ : INADDRSZ; -#ifdef HAVE_IPV6 - if (af == AF_INET6) - addrsz = IN6ADDRSZ; -#endif - if (strlen(setname) >= IPSET_MAXNAMELEN) { errno = ENAMETOOLONG; @@ -157,7 +137,7 @@ static int new_add_to_ipset(const char *setname, const struct all_addr *ipaddr, nested[1]->nla_type = NLA_F_NESTED | IPSET_ATTR_IP; add_attr(nlh, (af == AF_INET ? IPSET_ATTR_IPADDR_IPV4 : IPSET_ATTR_IPADDR_IPV6) | NLA_F_NET_BYTEORDER, - addrsz, &ipaddr->addr); + addrsz, ipaddr); nested[1]->nla_len = (void *)buffer + NL_ALIGN(nlh->nlmsg_len) - (void *)nested[1]; nested[0]->nla_len = (void *)buffer + NL_ALIGN(nlh->nlmsg_len) - (void *)nested[0]; @@ -168,7 +148,7 @@ static int new_add_to_ipset(const char *setname, const struct all_addr *ipaddr, } -static int old_add_to_ipset(const char *setname, const struct all_addr *ipaddr, int remove) +static int old_add_to_ipset(const char *setname, const union all_addr *ipaddr, int remove) { socklen_t size; struct ip_set_req_adt_get { @@ -200,7 +180,7 @@ static int old_add_to_ipset(const char *setname, const struct all_addr *ipaddr, return -1; req_adt.op = remove ? 0x102 : 0x101; req_adt.index = req_adt_get.set.index; - req_adt.ip = ntohl(ipaddr->addr.addr4.s_addr); + req_adt.ip = ntohl(ipaddr->addr4.s_addr); if (setsockopt(ipset_sock, SOL_IP, 83, &req_adt, sizeof(req_adt)) < 0) return -1; @@ -209,11 +189,10 @@ static int old_add_to_ipset(const char *setname, const struct all_addr *ipaddr, -int add_to_ipset(const char *setname, const struct all_addr *ipaddr, int flags, int remove) +int add_to_ipset(const char *setname, const union all_addr *ipaddr, int flags, int remove) { int ret = 0, af = AF_INET; -#ifdef HAVE_IPV6 if (flags & F_IPV6) { af = AF_INET6; @@ -224,7 +203,6 @@ int add_to_ipset(const char *setname, const struct all_addr *ipaddr, int flags, ret = -1; } } -#endif if (ret != -1) ret = old_kernel ? old_add_to_ipset(setname, ipaddr, remove) : new_add_to_ipset(setname, ipaddr, af, remove); diff --git a/dnsmasq/lease.c b/src/dnsmasq/lease.c similarity index 89% rename from dnsmasq/lease.c rename to src/dnsmasq/lease.c index 787fe12ca..23e6fe0a2 100644 --- a/dnsmasq/lease.c +++ b/src/dnsmasq/lease.c @@ -1,15 +1,15 @@ -/* dnsmasq is Copyright (c) 2000-2018 Simon Kelley +/* dnsmasq is Copyright (c) 2000-2020 Simon Kelley This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; version 2 dated June, 1991, or (at your option) version 3 dated 29 June, 2007. - + This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. - + You should have received a copy of the GNU General Public License along with this program. If not, see . */ @@ -24,7 +24,7 @@ static int dns_dirty, file_dirty, leases_left; static int read_leases(time_t now, FILE *leasestream) { unsigned long ei; - struct all_addr addr; + union all_addr addr; struct dhcp_lease *lease; int clid_len, hw_len, hw_type; int items; @@ -45,7 +45,7 @@ static int read_leases(time_t now, FILE *leasestream) { *daemon->namebuff = *daemon->dhcp_buff = *daemon->packet = '\0'; hw_len = hw_type = clid_len = 0; - + #ifdef HAVE_DHCP6 if (strcmp(daemon->dhcp_buff3, "duid") == 0) { @@ -57,23 +57,28 @@ static int read_leases(time_t now, FILE *leasestream) continue; } #endif - + if (fscanf(leasestream, " %64s %255s %764s", daemon->namebuff, daemon->dhcp_buff, daemon->packet) != 3) - return 0; - - if (inet_pton(AF_INET, daemon->namebuff, &addr.addr.addr4)) { - if ((lease = lease4_allocate(addr.addr.addr4))) + my_syslog(MS_DHCP | LOG_WARNING, _("ignoring invalid line in lease database: %s %s %s %s ..."), + daemon->dhcp_buff3, daemon->dhcp_buff2, + daemon->namebuff, daemon->dhcp_buff); + continue; + } + + if (inet_pton(AF_INET, daemon->namebuff, &addr.addr4)) + { + if ((lease = lease4_allocate(addr.addr4))) domain = get_domain(lease->addr); - + hw_len = parse_hex(daemon->dhcp_buff2, (unsigned char *)daemon->dhcp_buff2, DHCP_CHADDR_MAX, NULL, &hw_type); /* For backwards compatibility, no explicit MAC address type means ether. */ if (hw_type == 0 && hw_len != 0) - hw_type = ARPHRD_ETHER; + hw_type = ARPHRD_ETHER; } #ifdef HAVE_DHCP6 - else if (inet_pton(AF_INET6, daemon->namebuff, &addr.addr.addr6)) + else if (inet_pton(AF_INET6, daemon->namebuff, &addr.addr6)) { char *s = daemon->dhcp_buff2; int lease_type = LEASE_NA; @@ -83,8 +88,8 @@ static int read_leases(time_t now, FILE *leasestream) lease_type = LEASE_TA; s++; } - - if ((lease = lease6_allocate(&addr.addr.addr6, lease_type))) + + if ((lease = lease6_allocate(&addr.addr6, lease_type))) { lease_set_iaid(lease, strtoul(s, NULL, 10)); domain = get_domain6(&lease->addr6); @@ -92,17 +97,22 @@ static int read_leases(time_t now, FILE *leasestream) } #endif else - return 0; + { + my_syslog(MS_DHCP | LOG_WARNING, _("ignoring invalid line in lease database, bad address: %s"), + daemon->namebuff); + continue; + } + if (!lease) die (_("too many stored leases"), NULL, EC_MISC); if (strcmp(daemon->packet, "*") != 0) clid_len = parse_hex(daemon->packet, (unsigned char *)daemon->packet, 255, NULL, NULL); - - lease_set_hwaddr(lease, (unsigned char *)daemon->dhcp_buff2, (unsigned char *)daemon->packet, + + lease_set_hwaddr(lease, (unsigned char *)daemon->dhcp_buff2, (unsigned char *)daemon->packet, hw_len, hw_type, clid_len, now, 0); - + if (strcmp(daemon->dhcp_buff, "*") != 0) lease_set_hostname(lease, daemon->dhcp_buff, 0, domain, NULL); @@ -119,14 +129,14 @@ static int read_leases(time_t now, FILE *leasestream) even when sizeof(time_t) == 8 */ lease->expires = (time_t)ei; #endif - + /* set these correctly: the "old" events are generated later from the startup synthesised SIGHUP. */ lease->flags &= ~(LEASE_NEW | LEASE_CHANGED); - + *daemon->dhcp_buff3 = *daemon->dhcp_buff2 = '\0'; } - + return (items == 0 || items == EOF); } @@ -172,14 +182,12 @@ void lease_init(time_t now) if (leasestream) { if (!read_leases(now, leasestream)) - my_syslog(MS_DHCP | LOG_ERR, _("failed to parse lease database, invalid line: %s %s %s %s ..."), - daemon->dhcp_buff3, daemon->dhcp_buff2, - daemon->namebuff, daemon->dhcp_buff); - + my_syslog(MS_DHCP | LOG_ERR, _("failed to parse lease database cleanly")); + if (ferror(leasestream)) die(_("failed to read lease file %s: %s"), daemon->lease_file, EC_FILE); } - + #ifdef HAVE_SCRIPT if (!daemon->lease_stream) { @@ -195,7 +203,7 @@ void lease_init(time_t now) die(_("cannot run lease-init script %s: %s"), daemon->lease_change_command, EC_FILE); } - + if (WEXITSTATUS(rc) != 0) { sprintf(daemon->dhcp_buff, "%d", WEXITSTATUS(rc)); @@ -213,16 +221,16 @@ void lease_init(time_t now) void lease_update_from_configs(void) { /* changes to the config may change current leases. */ - + struct dhcp_lease *lease; struct dhcp_config *config; char *name; - + for (lease = leases; lease; lease = lease->next) if (lease->flags & (LEASE_TA | LEASE_NA)) continue; - else if ((config = find_config(daemon->dhcp_conf, NULL, lease->clid, lease->clid_len, - lease->hwaddr, lease->hwaddr_len, lease->hwaddr_type, NULL)) && + else if ((config = find_config(daemon->dhcp_conf, NULL, lease->clid, lease->clid_len, + lease->hwaddr, lease->hwaddr_len, lease->hwaddr_type, NULL, NULL)) && (config->flags & CONFIG_NAME) && (!(config->flags & CONFIG_ADDR) || config->addr.s_addr == lease->addr.s_addr)) lease_set_hostname(lease, config->hostname, 1, get_domain(lease->addr), NULL); @@ -233,7 +241,7 @@ void lease_update_from_configs(void) static void ourprintf(int *errp, char *format, ...) { va_list ap; - + va_start(ap, format); if (!(*errp) && vfprintf(daemon->lease_stream, format, ap) < 0) *errp = errno; @@ -252,7 +260,7 @@ void lease_update_file(time_t now) rewind(daemon->lease_stream); if (errno != 0 || ftruncate(fileno(daemon->lease_stream), 0) != 0) err = errno; - + for (lease = leases; lease; lease = lease->next) { @@ -267,7 +275,7 @@ void lease_update_file(time_t now) ourprintf(&err, "%lu ", (unsigned long)lease->expires); #endif - if (lease->hwaddr_type != ARPHRD_ETHER || lease->hwaddr_len == 0) + if (lease->hwaddr_type != ARPHRD_ETHER || lease->hwaddr_len == 0) ourprintf(&err, "%.2x-", lease->hwaddr_type); for (i = 0; i < lease->hwaddr_len; i++) { @@ -275,12 +283,12 @@ void lease_update_file(time_t now) if (i != lease->hwaddr_len - 1) ourprintf(&err, ":"); } - - inet_ntop(AF_INET, &lease->addr, daemon->addrbuff, ADDRSTRLEN); + + inet_ntop(AF_INET, &lease->addr, daemon->addrbuff, ADDRSTRLEN); ourprintf(&err, " %s ", daemon->addrbuff); ourprintf(&err, "%s ", lease->hostname ? lease->hostname : "*"); - + if (lease->clid && lease->clid_len != 0) { for (i = 0; i < lease->clid_len - 1; i++) @@ -288,20 +296,20 @@ void lease_update_file(time_t now) ourprintf(&err, "%.2x\n", lease->clid[i]); } else - ourprintf(&err, "*\n"); + ourprintf(&err, "*\n"); } - -#ifdef HAVE_DHCP6 + +#ifdef HAVE_DHCP6 if (daemon->duid) { ourprintf(&err, "duid "); for (i = 0; i < daemon->duid_len - 1; i++) ourprintf(&err, "%.2x:", daemon->duid[i]); ourprintf(&err, "%.2x\n", daemon->duid[i]); - + for (lease = leases; lease; lease = lease->next) { - + if (!(lease->flags & (LEASE_TA | LEASE_NA))) continue; @@ -310,13 +318,13 @@ void lease_update_file(time_t now) #else ourprintf(&err, "%lu ", (unsigned long)lease->expires); #endif - + inet_ntop(AF_INET6, &lease->addr6, daemon->addrbuff, ADDRSTRLEN); - + ourprintf(&err, "%s%u %s ", (lease->flags & LEASE_TA) ? "T" : "", lease->iaid, daemon->addrbuff); ourprintf(&err, "%s ", lease->hostname ? lease->hostname : "*"); - + if (lease->clid && lease->clid_len != 0) { for (i = 0; i < lease->clid_len - 1; i++) @@ -324,19 +332,19 @@ void lease_update_file(time_t now) ourprintf(&err, "%.2x\n", lease->clid[i]); } else - ourprintf(&err, "*\n"); + ourprintf(&err, "*\n"); } } -#endif - +#endif + if (fflush(daemon->lease_stream) != 0 || fsync(fileno(daemon->lease_stream)) < 0) err = errno; - + if (!err) file_dirty = 0; } - + /* Set alarm for when the first lease expires. */ next_event = 0; @@ -345,13 +353,13 @@ void lease_update_file(time_t now) if (daemon->doing_ra) { time_t event; - + if ((event = periodic_slaac(now, leases)) != 0) { if (next_event == 0 || difftime(next_event, event) > 0.0) next_event = event; } - + if ((event = periodic_ra(now)) != 0) { if (next_event == 0 || difftime(next_event, event) > 0.0) @@ -364,13 +372,13 @@ void lease_update_file(time_t now) if (lease->expires != 0 && (next_event == 0 || difftime(next_event, lease->expires) > 0.0)) next_event = lease->expires; - + if (err) { if (next_event == 0 || difftime(next_event, LEASE_RETRY + now) > 0.0) next_event = LEASE_RETRY + now; - - my_syslog(MS_DHCP | LOG_ERR, _("failed to write %s: %s (retry in %us)"), + + my_syslog(MS_DHCP | LOG_ERR, _("failed to write %s: %s (retry in %us)"), daemon->lease_file, strerror(err), (unsigned int)difftime(next_event, now)); } @@ -391,8 +399,8 @@ static int find_interface_v4(struct in_addr local, int if_index, char *label, for (lease = leases; lease; lease = lease->next) if (!(lease->flags & (LEASE_TA | LEASE_NA)) && - is_same_net(local, lease->addr, netmask) && - prefix > lease->new_prefixlen) + is_same_net(local, lease->addr, netmask) && + prefix > lease->new_prefixlen) { lease->new_interface = if_index; lease->new_prefixlen = prefix; @@ -403,7 +411,7 @@ static int find_interface_v4(struct in_addr local, int if_index, char *label, #ifdef HAVE_DHCP6 static int find_interface_v6(struct in6_addr *local, int prefix, - int scope, int if_index, int flags, + int scope, int if_index, int flags, int preferred, int valid, void *vparam) { struct dhcp_lease *lease; @@ -441,7 +449,7 @@ void lease_update_slaac(time_t now) new SLAAC addresses to existing leases. */ struct dhcp_lease *lease; - + if (daemon->dhcp) for (lease = leases; lease; lease = lease->next) slaac_add_addrs(lease, now, 0); @@ -457,7 +465,7 @@ void lease_update_slaac(time_t now) void lease_find_interfaces(time_t now) { struct dhcp_lease *lease; - + for (lease = leases; lease; lease = lease->next) lease->new_prefixlen = lease->new_interface = 0; @@ -467,7 +475,7 @@ void lease_find_interfaces(time_t now) #endif for (lease = leases; lease; lease = lease->next) - if (lease->new_interface != 0) + if (lease->new_interface != 0) lease_set_interface(lease, lease->new_interface, now); } @@ -496,13 +504,13 @@ void lease_update_dns(int force) /* force transfer to authoritative secondaries */ daemon->soa_sn++; #endif - + cache_unhash_dhcp(); for (lease = leases; lease; lease = lease->next) { int prot = AF_INET; - + #ifdef HAVE_DHCP6 if (lease->flags & (LEASE_TA | LEASE_NA)) prot = AF_INET6; @@ -514,31 +522,31 @@ void lease_update_dns(int force) if (slaac->backoff == 0) { if (lease->fqdn) - cache_add_dhcp_entry(lease->fqdn, AF_INET6, (struct all_addr *)&slaac->addr, lease->expires); + cache_add_dhcp_entry(lease->fqdn, AF_INET6, (union all_addr *)&slaac->addr, lease->expires); if (!option_bool(OPT_DHCP_FQDN) && lease->hostname) - cache_add_dhcp_entry(lease->hostname, AF_INET6, (struct all_addr *)&slaac->addr, lease->expires); + cache_add_dhcp_entry(lease->hostname, AF_INET6, (union all_addr *)&slaac->addr, lease->expires); } } - + if (lease->fqdn) - cache_add_dhcp_entry(lease->fqdn, prot, - prot == AF_INET ? (struct all_addr *)&lease->addr : (struct all_addr *)&lease->addr6, + cache_add_dhcp_entry(lease->fqdn, prot, + prot == AF_INET ? (union all_addr *)&lease->addr : (union all_addr *)&lease->addr6, lease->expires); - + if (!option_bool(OPT_DHCP_FQDN) && lease->hostname) - cache_add_dhcp_entry(lease->hostname, prot, - prot == AF_INET ? (struct all_addr *)&lease->addr : (struct all_addr *)&lease->addr6, + cache_add_dhcp_entry(lease->hostname, prot, + prot == AF_INET ? (union all_addr *)&lease->addr : (union all_addr *)&lease->addr6, lease->expires); - + #else if (lease->fqdn) - cache_add_dhcp_entry(lease->fqdn, prot, (struct all_addr *)&lease->addr, lease->expires); - + cache_add_dhcp_entry(lease->fqdn, prot, (union all_addr *)&lease->addr, lease->expires); + if (!option_bool(OPT_DHCP_FQDN) && lease->hostname) - cache_add_dhcp_entry(lease->hostname, prot, (struct all_addr *)&lease->addr, lease->expires); + cache_add_dhcp_entry(lease->hostname, prot, (union all_addr *)&lease->addr, lease->expires); #endif } - + dns_dirty = 0; } } @@ -550,7 +558,7 @@ void lease_prune(struct dhcp_lease *target, time_t now) for (lease = leases, up = &leases; lease; lease = tmp) { tmp = lease->next; - if ((lease->expires != 0 && difftime(now, lease->expires) > 0) || lease == target) + if ((lease->expires != 0 && difftime(now, lease->expires) >= 0) || lease == target) { file_dirty = 1; if (lease->hostname) @@ -559,20 +567,20 @@ void lease_prune(struct dhcp_lease *target, time_t now) daemon->metrics[lease->addr.s_addr ? METRIC_LEASES_PRUNED_4 : METRIC_LEASES_PRUNED_6]++; *up = lease->next; /* unlink */ - + /* Put on old_leases list 'till we can run the script */ lease->next = old_leases; old_leases = lease; - + leases_left++; } else up = &lease->next; } -} - - +} + + struct dhcp_lease *lease_find_by_client(unsigned char *hwaddr, int hw_len, int hw_type, unsigned char *clid, int clid_len) { @@ -589,15 +597,15 @@ struct dhcp_lease *lease_find_by_client(unsigned char *hwaddr, int hw_len, int h memcmp(clid, lease->clid, clid_len) == 0) return lease; } - - for (lease = leases; lease; lease = lease->next) + + for (lease = leases; lease; lease = lease->next) { #ifdef HAVE_DHCP6 if (lease->flags & (LEASE_TA | LEASE_NA)) continue; -#endif - if ((!lease->clid || !clid) && - hw_len != 0 && +#endif + if ((!lease->clid || !clid) && + hw_len != 0 && lease->hwaddr_len == hw_len && lease->hwaddr_type == hw_type && memcmp(hwaddr, lease->hwaddr, hw_len) == 0) @@ -616,7 +624,7 @@ struct dhcp_lease *lease_find_by_addr(struct in_addr addr) #ifdef HAVE_DHCP6 if (lease->flags & (LEASE_TA | LEASE_NA)) continue; -#endif +#endif if (lease->addr.s_addr == addr.s_addr) return lease; } @@ -626,11 +634,12 @@ struct dhcp_lease *lease_find_by_addr(struct in_addr addr) #ifdef HAVE_DHCP6 /* find address for {CLID, IAID, address} */ -struct dhcp_lease *lease6_find(unsigned char *clid, int clid_len, - int lease_type, int iaid, struct in6_addr *addr) +struct dhcp_lease *lease6_find(unsigned char *clid, int clid_len, + int lease_type, unsigned int iaid, + struct in6_addr *addr) { struct dhcp_lease *lease; - + for (lease = leases; lease; lease = lease->next) { if (!(lease->flags & lease_type) || lease->iaid != iaid) @@ -638,14 +647,14 @@ struct dhcp_lease *lease6_find(unsigned char *clid, int clid_len, if (!IN6_ARE_ADDR_EQUAL(&lease->addr6, addr)) continue; - + if ((clid_len != lease->clid_len || memcmp(clid, lease->clid, clid_len) != 0)) continue; - + return lease; } - + return NULL; } @@ -653,13 +662,15 @@ struct dhcp_lease *lease6_find(unsigned char *clid, int clid_len, void lease6_reset(void) { struct dhcp_lease *lease; - + for (lease = leases; lease; lease = lease->next) lease->flags &= ~LEASE_USED; } /* enumerate all leases belonging to {CLID, IAID} */ -struct dhcp_lease *lease6_find_by_client(struct dhcp_lease *first, int lease_type, unsigned char *clid, int clid_len, int iaid) +struct dhcp_lease *lease6_find_by_client(struct dhcp_lease *first, int lease_type, + unsigned char *clid, int clid_len, + unsigned int iaid) { struct dhcp_lease *lease; @@ -675,40 +686,40 @@ struct dhcp_lease *lease6_find_by_client(struct dhcp_lease *first, int lease_typ if (!(lease->flags & lease_type) || lease->iaid != iaid) continue; - + if ((clid_len != lease->clid_len || memcmp(clid, lease->clid, clid_len) != 0)) continue; return lease; } - + return NULL; } struct dhcp_lease *lease6_find_by_addr(struct in6_addr *net, int prefix, u64 addr) { struct dhcp_lease *lease; - + for (lease = leases; lease; lease = lease->next) { if (!(lease->flags & (LEASE_TA | LEASE_NA))) continue; - + if (is_same_net6(&lease->addr6, net, prefix) && (prefix == 128 || addr6part(&lease->addr6) == addr)) return lease; } - + return NULL; -} +} /* Find largest assigned address in context */ u64 lease_find_max_addr6(struct dhcp_context *context) { struct dhcp_lease *lease; u64 addr = addr6part(&context->start6); - + if (!(context->flags & (CONTEXT_STATIC | CONTEXT_PROXY))) for (lease = leases; lease; lease = lease->next) { @@ -721,7 +732,7 @@ u64 lease_find_max_addr6(struct dhcp_context *context) addr6part(&lease->addr6) > addr) addr = addr6part(&lease->addr6); } - + return addr; } @@ -732,7 +743,7 @@ struct in_addr lease_find_max_addr(struct dhcp_context *context) { struct dhcp_lease *lease; struct in_addr addr = context->start; - + if (!(context->flags & (CONTEXT_STATIC | CONTEXT_PROXY))) for (lease = leases; lease; lease = lease->next) { @@ -745,7 +756,7 @@ struct in_addr lease_find_max_addr(struct dhcp_context *context) ((unsigned)ntohl(lease->addr.s_addr)) > ((unsigned)ntohl(addr.s_addr))) addr = lease->addr; } - + return addr; } @@ -764,7 +775,7 @@ static struct dhcp_lease *lease_allocate(void) lease->hwaddr_len = 256; /* illegal value */ lease->next = leases; leases = lease; - + file_dirty = 1; leases_left--; @@ -779,7 +790,7 @@ struct dhcp_lease *lease4_allocate(struct in_addr addr) lease->addr = addr; daemon->metrics[METRIC_LEASES_ALLOCATED_4]++; } - + return lease; } @@ -825,23 +836,23 @@ void lease_set_expires(struct dhcp_lease *lease, unsigned int len, time_t now) dns_dirty = 1; lease->expires = exp; #ifndef HAVE_BROKEN_RTC - lease->flags |= LEASE_AUX_CHANGED; + lease->flags |= LEASE_AUX_CHANGED | LEASE_EXP_CHANGED; file_dirty = 1; #endif } - + #ifdef HAVE_BROKEN_RTC if (len != lease->length) { lease->length = len; lease->flags |= LEASE_AUX_CHANGED; - file_dirty = 1; + file_dirty = 1; } #endif -} +} #ifdef HAVE_DHCP6 -void lease_set_iaid(struct dhcp_lease *lease, int iaid) +void lease_set_iaid(struct dhcp_lease *lease, unsigned int iaid) { if (lease->iaid != iaid) { @@ -864,7 +875,7 @@ void lease_set_hwaddr(struct dhcp_lease *lease, const unsigned char *hwaddr, (void)now; if (hw_len != lease->hwaddr_len || - hw_type != lease->hwaddr_type || + hw_type != lease->hwaddr_type || (hw_len != 0 && memcmp(lease->hwaddr, hwaddr, hw_len) != 0)) { if (hw_len != 0) @@ -892,7 +903,7 @@ void lease_set_hwaddr(struct dhcp_lease *lease, const unsigned char *hwaddr, return; #ifdef HAVE_DHCP6 change = 1; -#endif +#endif } else if (memcmp(lease->clid, clid, clid_len) != 0) { @@ -900,13 +911,13 @@ void lease_set_hwaddr(struct dhcp_lease *lease, const unsigned char *hwaddr, file_dirty = 1; #ifdef HAVE_DHCP6 change = 1; -#endif +#endif } - + lease->clid_len = clid_len; memcpy(lease->clid, clid, clid_len); } - + #ifdef HAVE_DHCP6 if (change) slaac_add_addrs(lease, now, force); @@ -916,11 +927,11 @@ void lease_set_hwaddr(struct dhcp_lease *lease, const unsigned char *hwaddr, static void kill_name(struct dhcp_lease *lease) { /* run script to say we lost our old name */ - + /* this shouldn't happen unless updates are very quick and the script very slow, we just avoid a memory leak if it does. */ free(lease->old_hostname); - + /* If we know the fqdn, pass that. The helper will derive the unqualified name from it, free the unqualified name here. */ @@ -942,25 +953,25 @@ void lease_set_hostname(struct dhcp_lease *lease, const char *name, int auth, ch if (config_domain && (!domain || !hostname_isequal(domain, config_domain))) my_syslog(MS_DHCP | LOG_WARNING, _("Ignoring domain %s for DHCP host name %s"), config_domain, name); - + if (lease->hostname && name && hostname_isequal(lease->hostname, name)) { if (auth) lease->flags |= LEASE_AUTH_NAME; return; } - + if (!name && !lease->hostname) return; /* If a machine turns up on a new net without dropping the old lease, or two machines claim the same name, then we end up with two interfaces with the same name. Check for that here and remove the name from the old lease. - Note that IPv6 leases are different. All the leases to the same DUID are + Note that IPv6 leases are different. All the leases to the same DUID are allowed the same name. Don't allow a name from the client to override a name from dnsmasq config. */ - + if (name) { if ((new_name = whine_malloc(strlen(name) + 1))) @@ -973,7 +984,7 @@ void lease_set_hostname(struct dhcp_lease *lease, const char *name, int auth, ch strcat(new_fqdn, domain); } } - + /* Depending on mode, we check either unqualified name or FQDN. */ for (lease_tmp = leases; lease_tmp; lease_tmp = lease_tmp->next) { @@ -985,7 +996,7 @@ void lease_set_hostname(struct dhcp_lease *lease, const char *name, int auth, ch else { if (!new_name || !lease_tmp->hostname || !hostname_isequal(lease_tmp->hostname, new_name) ) - continue; + continue; } if (lease->flags & (LEASE_TA | LEASE_NA)) @@ -997,18 +1008,18 @@ void lease_set_hostname(struct dhcp_lease *lease, const char *name, int auth, ch if (lease->clid_len == lease_tmp->clid_len && lease->clid && lease_tmp->clid && memcmp(lease->clid, lease_tmp->clid, lease->clid_len) == 0) - continue; + continue; } else if (lease_tmp->flags & (LEASE_TA | LEASE_NA)) continue; - + if ((lease_tmp->flags & LEASE_AUTH_NAME) && !auth) { free(new_name); free(new_fqdn); return; } - + kill_name(lease_tmp); break; } @@ -1019,12 +1030,12 @@ void lease_set_hostname(struct dhcp_lease *lease, const char *name, int auth, ch lease->hostname = new_name; lease->fqdn = new_fqdn; - + if (auth) lease->flags |= LEASE_AUTH_NAME; - + file_dirty = 1; - dns_dirty = 1; + dns_dirty = 1; lease->flags |= LEASE_CHANGED; /* run script on change */ } @@ -1036,7 +1047,7 @@ void lease_set_interface(struct dhcp_lease *lease, int interface, time_t now) return; lease->last_interface = interface; - lease->flags |= LEASE_CHANGED; + lease->flags |= LEASE_CHANGED; #ifdef HAVE_DHCP6 slaac_add_addrs(lease, now, 0); @@ -1046,9 +1057,9 @@ void lease_set_interface(struct dhcp_lease *lease, int interface, time_t now) void rerun_scripts(void) { struct dhcp_lease *lease; - + for (lease = leases; lease; lease = lease->next) - lease->flags |= LEASE_CHANGED; + lease->flags |= LEASE_CHANGED; } /* deleted leases get transferred to the old_leases list. @@ -1072,7 +1083,7 @@ int do_script_run(time_t now) if (old_leases) { lease = old_leases; - + /* If the lease still has an old_hostname, do the "old" action on that first */ if (lease->old_hostname) { @@ -1083,7 +1094,7 @@ int do_script_run(time_t now) lease->old_hostname = NULL; return 1; } - else + else { #ifdef HAVE_DHCP6 struct slaac_address *slaac, *tmp; @@ -1101,20 +1112,20 @@ int do_script_run(time_t now) emit_dbus_signal(ACTION_DEL, lease, lease->old_hostname); #endif old_leases = lease->next; - - free(lease->old_hostname); + + free(lease->old_hostname); free(lease->clid); free(lease->extradata); free(lease); - - return 1; + + return 1; } } - + /* make sure we announce the loss of a hostname before its new location. */ for (lease = leases; lease; lease = lease->next) if (lease->old_hostname) - { + { #ifdef HAVE_SCRIPT queue_script(ACTION_OLD_HOSTNAME, lease, lease->old_hostname, now); #endif @@ -1122,25 +1133,26 @@ int do_script_run(time_t now) lease->old_hostname = NULL; return 1; } - + for (lease = leases; lease; lease = lease->next) - if ((lease->flags & (LEASE_NEW | LEASE_CHANGED)) || - ((lease->flags & LEASE_AUX_CHANGED) && option_bool(OPT_LEASE_RO))) + if ((lease->flags & (LEASE_NEW | LEASE_CHANGED)) || + ((lease->flags & LEASE_AUX_CHANGED) && option_bool(OPT_LEASE_RO)) || + ((lease->flags & LEASE_EXP_CHANGED) && option_bool(OPT_LEASE_RENEW))) { #ifdef HAVE_SCRIPT - queue_script((lease->flags & LEASE_NEW) ? ACTION_ADD : ACTION_OLD, lease, + queue_script((lease->flags & LEASE_NEW) ? ACTION_ADD : ACTION_OLD, lease, lease->fqdn ? lease->fqdn : lease->hostname, now); #endif #ifdef HAVE_DBUS emit_dbus_signal((lease->flags & LEASE_NEW) ? ACTION_ADD : ACTION_OLD, lease, lease->fqdn ? lease->fqdn : lease->hostname); #endif - lease->flags &= ~(LEASE_NEW | LEASE_CHANGED | LEASE_AUX_CHANGED); - + lease->flags &= ~(LEASE_NEW | LEASE_CHANGED | LEASE_AUX_CHANGED | LEASE_EXP_CHANGED); + /* this is used for the "add" call, then junked, since they're not in the database */ free(lease->extradata); lease->extradata = NULL; - + return 1; } @@ -1152,7 +1164,7 @@ int do_script_run(time_t now) void lease_add_extradata(struct dhcp_lease *lease, unsigned char *data, unsigned int len, int delim) { unsigned int i; - + if (delim == -1) delim = 0; else @@ -1163,15 +1175,15 @@ void lease_add_extradata(struct dhcp_lease *lease, unsigned char *data, unsigned len = i; break; } - + if ((lease->extradata_size - lease->extradata_len) < (len + 1)) { size_t newsz = lease->extradata_len + len + 100; unsigned char *new = whine_malloc(newsz); - + if (!new) return; - + if (lease->extradata) { memcpy(new, lease->extradata, lease->extradata_len); @@ -1185,12 +1197,12 @@ void lease_add_extradata(struct dhcp_lease *lease, unsigned char *data, unsigned if (len != 0) memcpy(lease->extradata + lease->extradata_len, data, len); lease->extradata[lease->extradata_len + len] = delim; - lease->extradata_len += len + 1; + lease->extradata_len += len + 1; } #endif #endif + - - + diff --git a/dnsmasq/log.c b/src/dnsmasq/log.c similarity index 92% rename from dnsmasq/log.c rename to src/dnsmasq/log.c index 423164062..dbd6bd41e 100644 --- a/dnsmasq/log.c +++ b/src/dnsmasq/log.c @@ -1,15 +1,15 @@ -/* dnsmasq is Copyright (c) 2000-2018 Simon Kelley +/* dnsmasq is Copyright (c) 2000-2020 Simon Kelley This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; version 2 dated June, 1991, or (at your option) version 3 dated 29 June, 2007. - + This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. - + You should have received a copy of the GNU General Public License along with this program. If not, see . */ @@ -20,10 +20,10 @@ # include #endif -/* Implement logging to /dev/log asynchronously. If syslogd is +/* Implement logging to /dev/log asynchronously. If syslogd is making DNS lookups through dnsmasq, and dnsmasq blocks awaiting syslogd, then the two daemons can deadlock. We get around this - by not blocking when talking to syslog, instead we queue up to + by not blocking when talking to syslog, instead we queue up to MAX_LOGS messages. If more are queued, they will be dropped, and the drop event itself logged. */ @@ -69,7 +69,7 @@ int log_start(struct passwd *ent_pw, int errfd) #endif if (daemon->log_file) - { + { log_to_file = 1; daemon->max_logs = 0; if (strcmp(daemon->log_file, "-") == 0) @@ -79,7 +79,7 @@ int log_start(struct passwd *ent_pw, int errfd) log_fd = dup(STDERR_FILENO); } } - + max_logs = daemon->max_logs; if (!log_reopen(daemon->log_file)) @@ -91,7 +91,7 @@ int log_start(struct passwd *ent_pw, int errfd) /* if queuing is inhibited, make sure we allocate the one required buffer now. */ if (max_logs == 0) - { + { free_entries = safe_malloc(sizeof(struct log_entry)); free_entries->next = NULL; entries_alloced = 1; @@ -101,7 +101,7 @@ int log_start(struct passwd *ent_pw, int errfd) change the ownership here so that the file is always owned by the dnsmasq user. Then logrotate can just copy the owner. Failure of the chown call is OK, (for instance when started as non-root) */ - if (log_to_file && !log_stderr && ent_pw && ent_pw->pw_uid != 0 && + if (log_to_file && !log_stderr && ent_pw && ent_pw->pw_uid != 0 && fchown(log_fd, ent_pw->pw_uid, -1) != 0) ret = errno; @@ -111,14 +111,14 @@ int log_start(struct passwd *ent_pw, int errfd) int log_reopen(char *log_file) { if (!log_stderr) - { + { if (log_fd != -1) close(log_fd); - + /* NOTE: umask is set to 022 by the time this gets called */ - + if (log_file) - log_fd = open(log_file, O_WRONLY|O_CREAT|O_APPEND, S_IRUSR|S_IWUSR|S_IRGRP); + log_fd = open(log_file, O_WRONLY|O_CREAT|O_APPEND, S_IRUSR|S_IWUSR|S_IRGRP); else { #if defined(HAVE_SOLARIS_NETWORK) || defined(__ANDROID__) @@ -129,14 +129,14 @@ int log_reopen(char *log_file) #else int flags; log_fd = socket(AF_UNIX, connection_type, 0); - + /* if max_logs is zero, leave the socket blocking */ if (log_fd != -1 && max_logs != 0 && (flags = fcntl(log_fd, F_GETFL)) != -1) fcntl(log_fd, F_SETFL, flags | O_NONBLOCK); #endif } } - + return log_fd != -1; } @@ -146,19 +146,19 @@ static void free_entry(void) entries = tmp->next; tmp->next = free_entries; free_entries = tmp; -} +} static void log_write(void) { ssize_t rc; - + while (entries) { - /* The data in the payload is written with a terminating zero character - and the length reflects this. For a stream connection we need to - send the zero as a record terminator, but this isn't done for a - datagram connection, so treat the length as one less than reality - to elide the zero. If we're logging to a file, turn the zero into + /* The data in the payload is written with a terminating zero character + and the length reflects this. For a stream connection we need to + send the zero as a record terminator, but this isn't done for a + datagram connection, so treat the length as one less than reality + to elide the zero. If we're logging to a file, turn the zero into a newline, and leave the length alone. */ int len_adjust = 0; @@ -188,24 +188,24 @@ static void log_write(void) int e = entries_lost; entries_lost = 0; /* avoid wild recursion */ my_syslog(LOG_WARNING, _("overflow: %d log entries lost"), e); - } + } } continue; } - + if (errno == EINTR) continue; if (errno == EAGAIN || errno == EWOULDBLOCK) return; /* syslogd busy, go again when select() or poll() says so */ - + if (errno == ENOBUFS) { connection_good = 0; return; } - /* errors handling after this assumes sockets */ + /* errors handling after this assumes sockets */ if (!log_to_file) { /* Once a stream socket hits EPIPE, we have to close and re-open @@ -215,43 +215,43 @@ static void log_write(void) if (log_reopen(NULL)) continue; } - else if (errno == ECONNREFUSED || - errno == ENOTCONN || - errno == EDESTADDRREQ || + else if (errno == ECONNREFUSED || + errno == ENOTCONN || + errno == EDESTADDRREQ || errno == ECONNRESET) { /* socket went (syslogd down?), try and reconnect. If we fail, - stop trying until the next call to my_syslog() + stop trying until the next call to my_syslog() ECONNREFUSED -> connection went down ENOTCONN -> nobody listening (ECONNRESET, EDESTADDRREQ are *BSD equivalents) */ - + struct sockaddr_un logaddr; - + #ifdef HAVE_SOCKADDR_SA_LEN - logaddr.sun_len = sizeof(logaddr) - sizeof(logaddr.sun_path) + strlen(_PATH_LOG) + 1; + logaddr.sun_len = sizeof(logaddr) - sizeof(logaddr.sun_path) + strlen(_PATH_LOG) + 1; #endif logaddr.sun_family = AF_UNIX; safe_strncpy(logaddr.sun_path, _PATH_LOG, sizeof(logaddr.sun_path)); - + /* Got connection back? try again. */ if (connect(log_fd, (struct sockaddr *)&logaddr, sizeof(logaddr)) != -1) continue; - + /* errors from connect which mean we should keep trying */ - if (errno == ENOENT || - errno == EALREADY || + if (errno == ENOENT || + errno == EALREADY || errno == ECONNREFUSED || - errno == EISCONN || + errno == EISCONN || errno == EINTR || - errno == EAGAIN || + errno == EAGAIN || errno == EWOULDBLOCK) { /* try again on next syslog() call */ connection_good = 0; return; } - + /* try the other sort of socket... */ if (errno == EPROTOTYPE) { @@ -290,7 +290,7 @@ void my_syslog(int priority, const char *format, ...) func = "-dhcp"; else if ((LOG_FACMASK & priority) == MS_SCRIPT) func = "-script"; - + #ifdef LOG_PRI priority = LOG_PRI(priority); #else @@ -298,7 +298,7 @@ void my_syslog(int priority, const char *format, ...) priority &= LOG_PRIMASK; #endif - if (echo_stderr) + if (echo_stderr) { fprintf(stderr, "dnsmasq%s: ", func); va_start(ap, format); @@ -310,10 +310,10 @@ void my_syslog(int priority, const char *format, ...) if (log_fd == -1) { #ifdef __ANDROID__ - /* do android-specific logging. + /* do android-specific logging. log_fd is always -1 on Android except when logging to a file. */ int alog_lvl; - + if (priority <= LOG_ERR) alog_lvl = ANDROID_LOG_ERROR; else if (priority == LOG_WARNING) @@ -327,7 +327,7 @@ void my_syslog(int priority, const char *format, ...) __android_log_vprint(alog_lvl, "dnsmasq", format, ap); va_end(ap); #else - /* fall-back to syslog if we die during startup or + /* fall-back to syslog if we die during startup or fail during running (always on Solaris). */ static int isopen = 0; @@ -336,19 +336,19 @@ void my_syslog(int priority, const char *format, ...) openlog("dnsmasq", LOG_PID, log_fac); isopen = 1; } - va_start(ap, format); + va_start(ap, format); vsyslog(priority, format, ap); va_end(ap); #endif return; } - + if ((entry = free_entries)) free_entries = entry->next; else if (entries_alloced < max_logs && (entry = malloc(sizeof(struct log_entry)))) entries_alloced++; - + if (!entry) entries_lost++; else @@ -363,36 +363,36 @@ void my_syslog(int priority, const char *format, ...) for (tmp = entries; tmp->next; tmp = tmp->next); tmp->next = entry; } - + time(&time_now); p = entry->payload; if (!log_to_file) p += sprintf(p, "<%d>", priority | log_fac); /* Omit timestamp for default daemontools situation */ - if (!log_stderr || !option_bool(OPT_NO_FORK)) + if (!log_stderr || !option_bool(OPT_NO_FORK)) p += sprintf(p, "%.15s ", ctime(&time_now) + 4); - + p += sprintf(p, "dnsmasq%s[%d]: ", func, (int)pid); - + len = p - entry->payload; - va_start(ap, format); + va_start(ap, format); len += vsnprintf(p, MAX_MESSAGE - len, format, ap) + 1; /* include zero-terminator */ va_end(ap); entry->length = len > MAX_MESSAGE ? MAX_MESSAGE : len; entry->offset = 0; entry->pid = pid; } - + /* almost always, logging won't block, so try and write this now, to save collecting too many log messages during a select loop. */ log_write(); - + /* Since we're doing things asynchronously, a cache-dump, for instance, can now generate log lines very fast. With a small buffer (desirable), that means it can overflow the log-buffer very quickly, - so that the cache dump becomes mainly a count of how many lines - overflowed. To avoid this, we delay here, the delay is controlled + so that the cache dump becomes mainly a count of how many lines + overflowed. To avoid this, we delay here, the delay is controlled by queue-occupancy, and grows exponentially. The delay is limited to (2^8)ms. The scaling stuff ensures that when the queue is bigger than 8, the delay only occurs for the last 8 entries. Once the queue is full, we stop delaying @@ -402,9 +402,9 @@ void my_syslog(int priority, const char *format, ...) if (entries && max_logs != 0) { int d; - + for (d = 0,entry = entries; entry; entry = entry->next, d++); - + if (d == max_logs) d = 0; else if (max_logs > 8) @@ -416,11 +416,11 @@ void my_syslog(int priority, const char *format, ...) waiter.tv_sec = 0; waiter.tv_nsec = 1000000 << (d - 1); /* 1 ms */ nanosleep(&waiter, NULL); - + /* Have another go now */ log_write(); } - } + } } void set_log_writer(void) @@ -445,7 +445,7 @@ void flush_log(void) log_write(); if (!entries || !connection_good) { - close(log_fd); + close(log_fd); break; } waiter.tv_sec = 0; @@ -457,7 +457,7 @@ void flush_log(void) void die(char *message, char *arg1, int exit_code) { char *errmess = strerror(errno); - + if (!arg1) arg1 = errmess; @@ -470,6 +470,6 @@ void die(char *message, char *arg1, int exit_code) echo_stderr = 0; my_syslog(LOG_CRIT, _("FAILED to start up")); flush_log(); - + exit(exit_code); } diff --git a/dnsmasq/loop.c b/src/dnsmasq/loop.c similarity index 98% rename from dnsmasq/loop.c rename to src/dnsmasq/loop.c index 0b47a2f6f..78e1238d2 100644 --- a/dnsmasq/loop.c +++ b/src/dnsmasq/loop.c @@ -1,4 +1,4 @@ -/* dnsmasq is Copyright (c) 2000-2018 Simon Kelley +/* dnsmasq is Copyright (c) 2000-2020 Simon Kelley This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by diff --git a/dnsmasq/metrics.c b/src/dnsmasq/metrics.c similarity index 95% rename from dnsmasq/metrics.c rename to src/dnsmasq/metrics.c index 86728074f..79f20b0da 100644 --- a/dnsmasq/metrics.c +++ b/src/dnsmasq/metrics.c @@ -1,4 +1,4 @@ -/* dnsmasq is Copyright (c) 2000-2018 Simon Kelley +/* dnsmasq is Copyright (c) 2000-2020 Simon Kelley This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by diff --git a/dnsmasq/metrics.h b/src/dnsmasq/metrics.h similarity index 95% rename from dnsmasq/metrics.h rename to src/dnsmasq/metrics.h index cbe5e3cf7..ccc92a86a 100644 --- a/dnsmasq/metrics.h +++ b/src/dnsmasq/metrics.h @@ -1,15 +1,15 @@ -/* dnsmasq is Copyright (c) 2000-2018 Simon Kelley - +/* dnsmasq is Copyright (c) 2000-2020 Simon Kelley + This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; version 2 dated June, 1991, or (at your option) version 3 dated 29 June, 2007. - + This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. - + You should have received a copy of the GNU General Public License along with this program. If not, see . */ @@ -36,7 +36,7 @@ enum { METRIC_LEASES_PRUNED_4, METRIC_LEASES_ALLOCATED_6, METRIC_LEASES_PRUNED_6, - + __METRIC_MAX, }; diff --git a/dnsmasq/netlink.c b/src/dnsmasq/netlink.c similarity index 87% rename from dnsmasq/netlink.c rename to src/dnsmasq/netlink.c index 7df5dda78..8e8431fab 100644 --- a/dnsmasq/netlink.c +++ b/src/dnsmasq/netlink.c @@ -1,15 +1,15 @@ -/* dnsmasq is Copyright (c) 2000-2018 Simon Kelley +/* dnsmasq is Copyright (c) 2000-2020 Simon Kelley This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; version 2 dated June, 1991, or (at your option) version 3 dated 29 June, 2007. - + This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. - + You should have received a copy of the GNU General Public License along with this program. If not, see . */ @@ -22,7 +22,16 @@ #include #include -/* linux 2.6.19 buggers up the headers, patch it up here. */ +/* Blergh. Radv does this, so that's our excuse. */ +#ifndef SOL_NETLINK +#define SOL_NETLINK 270 +#endif + +#ifndef NETLINK_NO_ENOBUFS +#define NETLINK_NO_ENOBUFS 5 +#endif + +/* linux 2.6.19 buggers up the headers, patch it up here. */ #ifndef IFA_RTA # define IFA_RTA(r) \ ((struct rtattr*)(((char*)(r)) + NLMSG_ALIGN(sizeof(struct ifaddrmsg)))) @@ -31,8 +40,8 @@ #endif #ifndef NDA_RTA -# define NDA_RTA(r) ((struct rtattr*)(((char*)(r)) + NLMSG_ALIGN(sizeof(struct ndmsg)))) -#endif +# define NDA_RTA(r) ((struct rtattr*)(((char*)(r)) + NLMSG_ALIGN(sizeof(struct ndmsg)))) +#endif static struct iovec iov; @@ -40,27 +49,27 @@ static u32 netlink_pid; static void nl_async(struct nlmsghdr *h); -void netlink_init(void) +char *netlink_init(void) { struct sockaddr_nl addr; socklen_t slen = sizeof(addr); + int opt = 1; addr.nl_family = AF_NETLINK; addr.nl_pad = 0; addr.nl_pid = 0; /* autobind */ addr.nl_groups = RTMGRP_IPV4_ROUTE; if (option_bool(OPT_CLEVERBIND)) - addr.nl_groups |= RTMGRP_IPV4_IFADDR; -#ifdef HAVE_IPV6 + addr.nl_groups |= RTMGRP_IPV4_IFADDR; addr.nl_groups |= RTMGRP_IPV6_ROUTE; if (option_bool(OPT_CLEVERBIND)) addr.nl_groups |= RTMGRP_IPV6_IFADDR; -#endif + #ifdef HAVE_DHCP6 if (daemon->doing_ra || daemon->doing_dhcp6) addr.nl_groups |= RTMGRP_IPV6_IFADDR; #endif - + /* May not be able to have permission to set multicast groups don't die in that case */ if ((daemon->netlinkfd = socket(AF_NETLINK, SOCK_RAW, NETLINK_ROUTE)) != -1) { @@ -71,16 +80,23 @@ void netlink_init(void) daemon->netlinkfd = -1; } } - - if (daemon->netlinkfd == -1 || + + if (daemon->netlinkfd == -1 || getsockname(daemon->netlinkfd, (struct sockaddr *)&addr, &slen) == -1) die(_("cannot create netlink socket: %s"), NULL, EC_MISC); - - /* save pid assigned by bind() and retrieved by getsockname() */ + + + /* save pid assigned by bind() and retrieved by getsockname() */ netlink_pid = addr.nl_pid; - + iov.iov_len = 100; iov.iov_base = safe_malloc(iov.iov_len); + + if (daemon->kernel_version >= KERNEL_VERSION(2,6,30) && + setsockopt(daemon->netlinkfd, SOL_NETLINK, NETLINK_NO_ENOBUFS, &opt, sizeof(opt)) == -1) + return _("warning: failed to set NETLINK_NO_ENOBUFS on netlink socket"); + + return NULL; } static ssize_t netlink_recv(void) @@ -98,9 +114,9 @@ static ssize_t netlink_recv(void) msg.msg_iov = &iov; msg.msg_iovlen = 1; msg.msg_flags = 0; - + while ((rc = recvmsg(daemon->netlinkfd, &msg, MSG_PEEK | MSG_TRUNC)) == -1 && errno == EINTR); - + /* make buffer big enough */ if (rc != -1 && (msg.msg_flags & MSG_TRUNC)) { @@ -117,22 +133,22 @@ static ssize_t netlink_recv(void) /* read it for real */ msg.msg_flags = 0; while ((rc = recvmsg(daemon->netlinkfd, &msg, 0)) == -1 && errno == EINTR); - + /* Make sure this is from the kernel */ if (rc == -1 || nladdr.nl_pid == 0) break; } - + /* discard stuff which is truncated at this point (expand_buf() may fail) */ if (msg.msg_flags & MSG_TRUNC) { rc = -1; errno = ENOMEM; } - + return rc; } - + /* family = AF_UNSPEC finds ARP table entries. family = AF_LOCAL finds MAC addresses. */ @@ -146,15 +162,15 @@ int iface_enumerate(int family, void *parm, int (*callback)()) struct { struct nlmsghdr nlh; - struct rtgenmsg g; + struct rtgenmsg g; } req; memset(&req, 0, sizeof(req)); memset(&addr, 0, sizeof(addr)); addr.nl_family = AF_NETLINK; - - again: + + again: if (family == AF_UNSPEC) req.nlh.nlmsg_type = RTM_GETNEIGH; else if (family == AF_LOCAL) @@ -163,18 +179,18 @@ int iface_enumerate(int family, void *parm, int (*callback)()) req.nlh.nlmsg_type = RTM_GETADDR; req.nlh.nlmsg_len = sizeof(req); - req.nlh.nlmsg_flags = NLM_F_ROOT | NLM_F_MATCH | NLM_F_REQUEST | NLM_F_ACK; + req.nlh.nlmsg_flags = NLM_F_ROOT | NLM_F_MATCH | NLM_F_REQUEST | NLM_F_ACK; req.nlh.nlmsg_pid = 0; req.nlh.nlmsg_seq = ++seq; - req.g.rtgen_family = family; + req.g.rtgen_family = family; /* Don't block in recvfrom if send fails */ - while(retry_send(sendto(daemon->netlinkfd, (void *)&req, sizeof(req), 0, + while(retry_send(sendto(daemon->netlinkfd, (void *)&req, sizeof(req), 0, (struct sockaddr *)&addr, sizeof(addr)))); if (errno != 0) return 0; - + while (1) { if ((len = netlink_recv()) == -1) @@ -203,10 +219,10 @@ int iface_enumerate(int family, void *parm, int (*callback)()) return callback_ok; else if (h->nlmsg_type == RTM_NEWADDR && family != AF_UNSPEC && family != AF_LOCAL) { - struct ifaddrmsg *ifa = NLMSG_DATA(h); + struct ifaddrmsg *ifa = NLMSG_DATA(h); struct rtattr *rta = IFA_RTA(ifa); unsigned int len1 = h->nlmsg_len - NLMSG_LENGTH(sizeof(*ifa)); - + if (ifa->ifa_family == family) { if (ifa->ifa_family == AF_INET) @@ -218,7 +234,7 @@ int iface_enumerate(int family, void *parm, int (*callback)()) addr.s_addr = 0; broadcast.s_addr = 0; - + while (RTA_OK(rta, len1)) { if (rta->rta_type == IFA_LOCAL) @@ -227,25 +243,24 @@ int iface_enumerate(int family, void *parm, int (*callback)()) broadcast = *((struct in_addr *)(rta+1)); else if (rta->rta_type == IFA_LABEL) label = RTA_DATA(rta); - + rta = RTA_NEXT(rta, len1); } - + if (addr.s_addr && callback_ok) if (!((*callback)(addr, ifa->ifa_index, label, netmask, broadcast, parm))) callback_ok = 0; } -#ifdef HAVE_IPV6 else if (ifa->ifa_family == AF_INET6) { struct in6_addr *addrp = NULL; u32 valid = 0, preferred = 0; int flags = 0; - + while (RTA_OK(rta, len1)) { if (rta->rta_type == IFA_ADDRESS) - addrp = ((struct in6_addr *)(rta+1)); + addrp = ((struct in6_addr *)(rta+1)); else if (rta->rta_type == IFA_CACHEINFO) { struct ifa_cacheinfo *ifc = (struct ifa_cacheinfo *)(rta+1); @@ -254,33 +269,32 @@ int iface_enumerate(int family, void *parm, int (*callback)()) } rta = RTA_NEXT(rta, len1); } - + if (ifa->ifa_flags & IFA_F_TENTATIVE) flags |= IFACE_TENTATIVE; - + if (ifa->ifa_flags & IFA_F_DEPRECATED) flags |= IFACE_DEPRECATED; - + if (!(ifa->ifa_flags & IFA_F_TEMPORARY)) flags |= IFACE_PERMANENT; - + if (addrp && callback_ok) - if (!((*callback)(addrp, (int)(ifa->ifa_prefixlen), (int)(ifa->ifa_scope), - (int)(ifa->ifa_index), flags, + if (!((*callback)(addrp, (int)(ifa->ifa_prefixlen), (int)(ifa->ifa_scope), + (int)(ifa->ifa_index), flags, (int) preferred, (int)valid, parm))) callback_ok = 0; } -#endif } } else if (h->nlmsg_type == RTM_NEWNEIGH && family == AF_UNSPEC) { - struct ndmsg *neigh = NLMSG_DATA(h); + struct ndmsg *neigh = NLMSG_DATA(h); struct rtattr *rta = NDA_RTA(neigh); unsigned int len1 = h->nlmsg_len - NLMSG_LENGTH(sizeof(*neigh)); size_t maclen = 0; char *inaddr = NULL, *mac = NULL; - + while (RTA_OK(rta, len1)) { if (rta->rta_type == NDA_DST) @@ -290,7 +304,7 @@ int iface_enumerate(int family, void *parm, int (*callback)()) maclen = rta->rta_len - sizeof(struct rtattr); mac = (char *)(rta+1); } - + rta = RTA_NEXT(rta, len1); } @@ -315,11 +329,11 @@ int iface_enumerate(int family, void *parm, int (*callback)()) maclen = rta->rta_len - sizeof(struct rtattr); mac = (char *)(rta+1); } - + rta = RTA_NEXT(rta, len1); } - if (mac && callback_ok && !((link->ifi_flags & (IFF_LOOPBACK | IFF_POINTOPOINT))) && + if (mac && callback_ok && !((link->ifi_flags & (IFF_LOOPBACK | IFF_POINTOPOINT))) && !((*callback)((int)link->ifi_index, (unsigned int)link->ifi_type, mac, maclen, parm))) callback_ok = 0; } @@ -332,16 +346,16 @@ void netlink_multicast(void) ssize_t len; struct nlmsghdr *h; int flags; - + /* don't risk blocking reading netlink messages here. */ if ((flags = fcntl(daemon->netlinkfd, F_GETFL)) == -1 || - fcntl(daemon->netlinkfd, F_SETFL, flags | O_NONBLOCK) == -1) + fcntl(daemon->netlinkfd, F_SETFL, flags | O_NONBLOCK) == -1) return; - + if ((len = netlink_recv()) != -1) for (h = (struct nlmsghdr *)iov.iov_base; NLMSG_OK(h, (size_t)len); h = NLMSG_NEXT(h, len)) nl_async(h); - + /* restore non-blocking status */ fcntl(daemon->netlinkfd, F_SETFL, flags); } @@ -354,21 +368,23 @@ static void nl_async(struct nlmsghdr *h) if (err->error != 0) my_syslog(LOG_ERR, _("netlink returns error: %s"), strerror(-(err->error))); } - else if (h->nlmsg_pid == 0 && h->nlmsg_type == RTM_NEWROUTE) + else if (h->nlmsg_pid == 0 && h->nlmsg_type == RTM_NEWROUTE) { /* We arrange to receive netlink multicast messages whenever the network route is added. If this happens and we still have a DNS packet in the buffer, we re-send it. This helps on DoD links, where frequently the packet which triggers dialling is a DNS query, which then gets lost. By re-sending, we can avoid the lookup - failing. */ + failing. */ struct rtmsg *rtm = NLMSG_DATA(h); - - if (rtm->rtm_type == RTN_UNICAST && rtm->rtm_scope == RT_SCOPE_LINK) + + if (rtm->rtm_type == RTN_UNICAST && rtm->rtm_scope == RT_SCOPE_LINK && + (rtm->rtm_table == RT_TABLE_MAIN || + rtm->rtm_table == RT_TABLE_LOCAL)) queue_event(EVENT_NEWROUTE); } - else if (h->nlmsg_type == RTM_NEWADDR || h->nlmsg_type == RTM_DELADDR) + else if (h->nlmsg_type == RTM_NEWADDR || h->nlmsg_type == RTM_DELADDR) queue_event(EVENT_NEWADDR); } #endif - + diff --git a/dnsmasq/network.c b/src/dnsmasq/network.c similarity index 91% rename from dnsmasq/network.c rename to src/dnsmasq/network.c index f0c53acf3..69d23dbe2 100644 --- a/dnsmasq/network.c +++ b/src/dnsmasq/network.c @@ -1,15 +1,15 @@ -/* dnsmasq is Copyright (c) 2000-2018 Simon Kelley +/* dnsmasq is Copyright (c) 2000-2020 Simon Kelley This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; version 2 dated June, 1991, or (at your option) version 3 dated 29 June, 2007. - + This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. - + You should have received a copy of the GNU General Public License along with this program. If not, see . */ @@ -22,7 +22,7 @@ int indextoname(int fd, int index, char *name) { struct ifreq ifr; - + if (index == 0) return 0; @@ -51,42 +51,42 @@ int indextoname(int fd, int index, char *name) int numifs, bufsize, i; struct lifconf lifc; struct lifreq *lifrp; - + if (index == 0) return 0; - - if (getzoneid() == GLOBAL_ZONEID) + + if (getzoneid() == GLOBAL_ZONEID) { if (!if_indextoname(index, name)) return 0; return 1; } - + lifc_flags = LIFC_NOXMIT | LIFC_TEMPORARY | LIFC_ALLZONES | LIFC_UNDER_IPMP; lifn.lifn_family = AF_UNSPEC; lifn.lifn_flags = lifc_flags; - if (ioctl(fd, SIOCGLIFNUM, &lifn) < 0) + if (ioctl(fd, SIOCGLIFNUM, &lifn) < 0) return 0; - + numifs = lifn.lifn_count; bufsize = numifs * sizeof(struct lifreq); - + lifc.lifc_family = AF_UNSPEC; lifc.lifc_flags = lifc_flags; lifc.lifc_len = bufsize; lifc.lifc_buf = alloca(bufsize); - - if (ioctl(fd, SIOCGLIFCONF, &lifc) < 0) + + if (ioctl(fd, SIOCGLIFCONF, &lifc) < 0) return 0; - + lifrp = lifc.lifc_req; - for (i = lifc.lifc_len / sizeof(struct lifreq); i; i--, lifrp++) + for (i = lifc.lifc_len / sizeof(struct lifreq); i; i--, lifrp++) { struct lifreq lifr; safe_strncpy(lifr.lifr_name, lifrp->lifr_name, IF_NAMESIZE); - if (ioctl(fd, SIOCGLIFINDEX, &lifr) < 0) + if (ioctl(fd, SIOCGLIFINDEX, &lifr) < 0) return 0; - + if (lifr.lifr_index == index) { safe_strncpy(name, lifr.lifr_name, IF_NAMESIZE); return 1; @@ -99,7 +99,7 @@ int indextoname(int fd, int index, char *name) #else int indextoname(int fd, int index, char *name) -{ +{ (void)fd; if (index == 0 || !if_indextoname(index, name)) @@ -110,7 +110,7 @@ int indextoname(int fd, int index, char *name) #endif -int iface_check(int family, struct all_addr *addr, char *name, int *auth) +int iface_check(int family, union all_addr *addr, char *name, int *auth) { struct iname *tmp; int ret = 1, match_addr = 0; @@ -119,10 +119,10 @@ int iface_check(int family, struct all_addr *addr, char *name, int *auth) "used" flags. May be called with family == AF_LOCALto check interface by name only. */ - + if (auth) *auth = 0; - + if (daemon->if_names || daemon->if_addrs) { ret = 0; @@ -130,28 +130,26 @@ int iface_check(int family, struct all_addr *addr, char *name, int *auth) for (tmp = daemon->if_names; tmp; tmp = tmp->next) if (tmp->name && wildcard_match(tmp->name, name)) ret = tmp->used = 1; - + if (addr) for (tmp = daemon->if_addrs; tmp; tmp = tmp->next) if (tmp->addr.sa.sa_family == family) { if (family == AF_INET && - tmp->addr.in.sin_addr.s_addr == addr->addr.addr4.s_addr) + tmp->addr.in.sin_addr.s_addr == addr->addr4.s_addr) ret = match_addr = tmp->used = 1; -#ifdef HAVE_IPV6 else if (family == AF_INET6 && - IN6_ARE_ADDR_EQUAL(&tmp->addr.in6.sin6_addr, - &addr->addr.addr6)) + IN6_ARE_ADDR_EQUAL(&tmp->addr.in6.sin6_addr, + &addr->addr6)) ret = match_addr = tmp->used = 1; -#endif - } + } } - + if (!match_addr) for (tmp = daemon->if_except; tmp; tmp = tmp->next) if (tmp->name && wildcard_match(tmp->name, name)) ret = 0; - + for (tmp = daemon->authinterface; tmp; tmp = tmp->next) if (tmp->name) @@ -161,30 +159,28 @@ int iface_check(int family, struct all_addr *addr, char *name, int *auth) break; } else if (addr && tmp->addr.sa.sa_family == AF_INET && family == AF_INET && - tmp->addr.in.sin_addr.s_addr == addr->addr.addr4.s_addr) + tmp->addr.in.sin_addr.s_addr == addr->addr4.s_addr) break; -#ifdef HAVE_IPV6 else if (addr && tmp->addr.sa.sa_family == AF_INET6 && family == AF_INET6 && - IN6_ARE_ADDR_EQUAL(&tmp->addr.in6.sin6_addr, &addr->addr.addr6)) + IN6_ARE_ADDR_EQUAL(&tmp->addr.in6.sin6_addr, &addr->addr6)) break; -#endif - if (tmp && auth) + if (tmp && auth) { *auth = 1; ret = 1; } - return ret; + return ret; } /* Fix for problem that the kernel sometimes reports the loopback interface as the - arrival interface when a packet originates locally, even when sent to address of - an interface other than the loopback. Accept packet if it arrived via a loopback + arrival interface when a packet originates locally, even when sent to address of + an interface other than the loopback. Accept packet if it arrived via a loopback interface, even when we're not accepting packets that way, as long as the destination address is one we're believing. Interface list must be up-to-date before calling. */ -int loopback_exception(int fd, int family, struct all_addr *addr, char *name) +int loopback_exception(int fd, int family, union all_addr *addr, char *name) { struct ifreq ifr; struct irec *iface; @@ -198,14 +194,11 @@ int loopback_exception(int fd, int family, struct all_addr *addr, char *name) { if (family == AF_INET) { - if (iface->addr.in.sin_addr.s_addr == addr->addr.addr4.s_addr) + if (iface->addr.in.sin_addr.s_addr == addr->addr4.s_addr) return 1; } -#ifdef HAVE_IPV6 - else if (IN6_ARE_ADDR_EQUAL(&iface->addr.in6.sin6_addr, &addr->addr.addr6)) + else if (IN6_ARE_ADDR_EQUAL(&iface->addr.in6.sin6_addr, &addr->addr6)) return 1; -#endif - } } return 0; @@ -213,9 +206,9 @@ int loopback_exception(int fd, int family, struct all_addr *addr, char *name) /* If we're configured with something like --interface=eth0:0 then we'll listen correctly on the relevant address, but the name of the arrival interface, derived from the - index won't match the config. Check that we found an interface address for the arrival + index won't match the config. Check that we found an interface address for the arrival interface: daemon->interfaces must be up-to-date. */ -int label_exception(int index, int family, struct all_addr *addr) +int label_exception(int index, int family, union all_addr *addr) { struct irec *iface; @@ -225,7 +218,7 @@ int label_exception(int index, int family, struct all_addr *addr) for (iface = daemon->interfaces; iface; iface = iface->next) if (iface->index == index && iface->addr.sa.sa_family == AF_INET && - iface->addr.in.sin_addr.s_addr == addr->addr.addr4.s_addr) + iface->addr.in.sin_addr.s_addr == addr->addr4.s_addr) return 1; return 0; @@ -237,7 +230,7 @@ struct iface_param { }; static int iface_allowed(struct iface_param *param, int if_index, char *label, - union mysockaddr *addr, struct in_addr netmask, int prefixlen, int iface_flags) + union mysockaddr *addr, struct in_addr netmask, int prefixlen, int iface_flags) { struct irec *iface; int mtu = 0, loopback; @@ -255,20 +248,20 @@ static int iface_allowed(struct iface_param *param, int if_index, char *label, if (!indextoname(param->fd, if_index, ifr.ifr_name) || ioctl(param->fd, SIOCGIFFLAGS, &ifr) == -1) return 0; - + loopback = ifr.ifr_flags & IFF_LOOPBACK; - + if (loopback) dhcp_ok = 0; - + if (ioctl(param->fd, SIOCGIFMTU, &ifr) != -1) mtu = ifr.ifr_mtu; - + if (!label) label = ifr.ifr_name; else is_label = strcmp(label, ifr.ifr_name); - + /* maintain a list of all addresses on all interfaces for --local-service option */ if (option_bool(OPT_LOCAL_SERVICE)) { @@ -281,31 +274,27 @@ static int iface_allowed(struct iface_param *param, int if_index, char *label, } else al = whine_malloc(sizeof(struct addrlist)); - + if (al) { al->next = daemon->interface_addrs; daemon->interface_addrs = al; al->prefixlen = prefixlen; - + if (addr->sa.sa_family == AF_INET) { - al->addr.addr.addr4 = addr->in.sin_addr; + al->addr.addr4 = addr->in.sin_addr; al->flags = 0; } -#ifdef HAVE_IPV6 else { - al->addr.addr.addr6 = addr->in6.sin6_addr; + al->addr.addr6 = addr->in6.sin6_addr; al->flags = ADDRLIST_IPV6; - } -#endif + } } } - -#ifdef HAVE_IPV6 + if (addr->sa.sa_family != AF_INET6 || !IN6_IS_ADDR_LINKLOCAL(&addr->in6.sin6_addr)) -#endif { struct interface_name *int_name; struct addrlist *al; @@ -327,18 +316,17 @@ static int iface_allowed(struct iface_param *param, int if_index, char *label, } else al = whine_malloc(sizeof(struct addrlist)); - + if (al) { al->next = zone->subnet; zone->subnet = al; al->prefixlen = prefixlen; - al->addr.addr.addr4 = addr->in.sin_addr; + al->addr.addr4 = addr->in.sin_addr; al->flags = 0; } } - -#ifdef HAVE_IPV6 + if (addr->sa.sa_family == AF_INET6 && (name->flags & AUTH6)) { if (param->spare) @@ -348,25 +336,23 @@ static int iface_allowed(struct iface_param *param, int if_index, char *label, } else al = whine_malloc(sizeof(struct addrlist)); - + if (al) { al->next = zone->subnet; zone->subnet = al; al->prefixlen = prefixlen; - al->addr.addr.addr6 = addr->in6.sin6_addr; + al->addr.addr6 = addr->in6.sin6_addr; al->flags = ADDRLIST_IPV6; } - } -#endif - + } } #endif - + /* Update addresses from interface_names. These are a set independent - of the set we're listening on. */ + of the set we're listening on. */ for (int_name = daemon->int_names; int_name; int_name = int_name->next) - if (strncmp(label, int_name->intr, IF_NAMESIZE) == 0 && + if (strncmp(label, int_name->intr, IF_NAMESIZE) == 0 && (addr->sa.sa_family == int_name->family || int_name->family == 0)) { if (param->spare) @@ -376,35 +362,33 @@ static int iface_allowed(struct iface_param *param, int if_index, char *label, } else al = whine_malloc(sizeof(struct addrlist)); - + if (al) { al->next = int_name->addr; int_name->addr = al; - + if (addr->sa.sa_family == AF_INET) { - al->addr.addr.addr4 = addr->in.sin_addr; + al->addr.addr4 = addr->in.sin_addr; al->flags = 0; } -#ifdef HAVE_IPV6 else { - al->addr.addr.addr6 = addr->in6.sin6_addr; + al->addr.addr6 = addr->in6.sin6_addr; al->flags = ADDRLIST_IPV6; /* Privacy addresses and addresses still undergoing DAD and deprecated addresses don't appear in forward queries, but will in reverse ones. */ if (!(iface_flags & IFACE_PERMANENT) || (iface_flags & (IFACE_DEPRECATED | IFACE_TENTATIVE))) al->flags |= ADDRLIST_REVONLY; - } -#endif + } } } } - - /* check whether the interface IP has been added already + + /* check whether the interface IP has been added already we call this routine multiple times. */ - for (iface = daemon->interfaces; iface; iface = iface->next) + for (iface = daemon->interfaces; iface; iface = iface->next) if (sockaddr_isequal(&iface->addr, addr)) { iface->dad = !!(iface_flags & IFACE_TENTATIVE); @@ -420,8 +404,8 @@ static int iface_allowed(struct iface_param *param, int if_index, char *label, for (lo = daemon->if_names; lo; lo = lo->next) if (lo->name && strcmp(lo->name, ifr.ifr_name) == 0) break; - - if (!lo && (lo = whine_malloc(sizeof(struct iname)))) + + if (!lo && (lo = whine_malloc(sizeof(struct iname)))) { if ((lo->name = whine_malloc(strlen(ifr.ifr_name)+1))) { @@ -434,17 +418,15 @@ static int iface_allowed(struct iface_param *param, int if_index, char *label, free(lo); } } - + if (addr->sa.sa_family == AF_INET && - !iface_check(AF_INET, (struct all_addr *)&addr->in.sin_addr, label, &auth_dns)) + !iface_check(AF_INET, (union all_addr *)&addr->in.sin_addr, label, &auth_dns)) return 1; -#ifdef HAVE_IPV6 if (addr->sa.sa_family == AF_INET6 && - !iface_check(AF_INET6, (struct all_addr *)&addr->in6.sin6_addr, label, &auth_dns)) + !iface_check(AF_INET6, (union all_addr *)&addr->in6.sin6_addr, label, &auth_dns)) return 1; -#endif - + #ifdef HAVE_DHCP /* No DHCP where we're doing auth DNS. */ if (auth_dns) @@ -460,8 +442,8 @@ static int iface_allowed(struct iface_param *param, int if_index, char *label, dhcp_ok = 0; } #endif - - + + #ifdef HAVE_TFTP if (daemon->tftp_interfaces) { @@ -472,7 +454,7 @@ static int iface_allowed(struct iface_param *param, int if_index, char *label, tftp_ok = 1; } #endif - + /* add to list */ if ((iface = whine_malloc(sizeof(struct irec)))) { @@ -497,14 +479,13 @@ static int iface_allowed(struct iface_param *param, int if_index, char *label, free(iface); } - - errno = ENOMEM; + + errno = ENOMEM; return 0; } -#ifdef HAVE_IPV6 -static int iface_allowed_v6(struct in6_addr *local, int prefix, - int scope, int if_index, int flags, +static int iface_allowed_v6(struct in6_addr *local, int prefix, + int scope, int if_index, int flags, int preferred, int valid, void *vparam) { union mysockaddr addr; @@ -514,7 +495,7 @@ static int iface_allowed_v6(struct in6_addr *local, int prefix, (void)scope; /* warning */ (void)preferred; (void)valid; - + memset(&addr, 0, sizeof(addr)); #ifdef HAVE_SOCKADDR_SA_LEN addr.in6.sin6_len = sizeof(addr.in6); @@ -527,17 +508,16 @@ static int iface_allowed_v6(struct in6_addr *local, int prefix, addr.in6.sin6_scope_id = if_index; else addr.in6.sin6_scope_id = 0; - + return iface_allowed((struct iface_param *)vparam, if_index, NULL, &addr, netmask, prefix, flags); } -#endif static int iface_allowed_v4(struct in_addr local, int if_index, char *label, struct in_addr netmask, struct in_addr broadcast, void *vparam) { union mysockaddr addr; int prefix, bit; - + (void)broadcast; /* warning */ memset(&addr, 0, sizeof(addr)); @@ -553,7 +533,7 @@ static int iface_allowed_v4(struct in_addr local, int if_index, char *label, return iface_allowed((struct iface_param *)vparam, if_index, label, &addr, netmask, prefix, 0); } - + int enumerate_interfaces(int reset) { static struct addrlist *spare = NULL; @@ -583,9 +563,9 @@ int enumerate_interfaces(int reset) if ((param.fd = socket(PF_INET, SOCK_DGRAM, 0)) == -1) return 0; - + /* Mark interfaces for garbage collection */ - for (iface = daemon->interfaces; iface; iface = iface->next) + for (iface = daemon->interfaces; iface; iface = iface->next) iface->found = 0; /* remove addresses stored against interface_names */ @@ -597,7 +577,7 @@ int enumerate_interfaces(int reset) addr->next = spare; spare = addr; } - + intname->addr = NULL; } @@ -609,9 +589,9 @@ int enumerate_interfaces(int reset) spare = addr; } daemon->interface_addrs = NULL; - + #ifdef HAVE_AUTH - /* remove addresses stored against auth_zone subnets, but not + /* remove addresses stored against auth_zone subnets, but not ones configured as address literals */ for (zone = daemon->auth_zones; zone; zone = zone->next) if (zone->interface_names) @@ -633,55 +613,53 @@ int enumerate_interfaces(int reset) #endif param.spare = spare; - -#ifdef HAVE_IPV6 + ret = iface_enumerate(AF_INET6, ¶m, iface_allowed_v6); -#endif if (ret) - ret = iface_enumerate(AF_INET, ¶m, iface_allowed_v4); - + ret = iface_enumerate(AF_INET, ¶m, iface_allowed_v4); + errsave = errno; close(param.fd); - + if (option_bool(OPT_CLEVERBIND)) - { + { /* Garbage-collect listeners listening on addresses that no longer exist. - Does nothing when not binding interfaces or for listeners on localhost, + Does nothing when not binding interfaces or for listeners on localhost, since the ->iface field is NULL. Note that this needs the protections against reentrancy, hence it's here. It also means there's a possibility, in OPT_CLEVERBIND mode, that at listener will just disappear after a call to enumerate_interfaces, this is checked OK on all calls. */ struct listener *l, *tmp, **up; - + for (up = &daemon->listeners, l = daemon->listeners; l; l = tmp) { tmp = l->next; - + if (!l->iface || l->iface->found) up = &l->next; else { *up = l->next; - + /* In case it ever returns */ l->iface->done = 0; - + if (l->fd != -1) close(l->fd); if (l->tcpfd != -1) close(l->tcpfd); if (l->tftpfd != -1) close(l->tftpfd); - + free(l); } } } - + errno = errsave; spare = param.spare; - + return ret; } @@ -693,7 +671,7 @@ int fix_fd(int fd) if ((flags = fcntl(fd, F_GETFL)) == -1 || fcntl(fd, F_SETFL, flags | O_NONBLOCK) == -1) return 0; - + return 1; } @@ -701,7 +679,7 @@ static int make_sock(union mysockaddr *addr, int type, int dienow) { int family = addr->sa.sa_family; int fd, rc, opt = 1; - + if ((fd = socket(family, type, 0)) == -1) { int port, errsave; @@ -712,17 +690,17 @@ static int make_sock(union mysockaddr *addr, int type, int dienow) errno == EAFNOSUPPORT || errno == EINVAL) return -1; - + err: errsave = errno; port = prettyprint_addr(addr, daemon->addrbuff); if (!option_bool(OPT_NOWILD) && !option_bool(OPT_CLEVERBIND)) sprintf(daemon->addrbuff, "port %d", port); s = _("failed to create listening socket for %s: %s"); - + if (fd != -1) close (fd); - + errno = errsave; if (dienow) @@ -734,23 +712,26 @@ static int make_sock(union mysockaddr *addr, int type, int dienow) } else my_syslog(LOG_WARNING, s, daemon->addrbuff, strerror(errno)); - + return -1; - } - + } + if (setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &opt, sizeof(opt)) == -1 || !fix_fd(fd)) goto err; - -#ifdef HAVE_IPV6 + if (family == AF_INET6 && setsockopt(fd, IPPROTO_IPV6, IPV6_V6ONLY, &opt, sizeof(opt)) == -1) goto err; -#endif - + if ((rc = bind(fd, (struct sockaddr *)addr, sa_len(addr))) == -1) goto err; - + if (type == SOCK_STREAM) { +#ifdef TCP_FASTOPEN + int qlen = 5; + setsockopt(fd, IPPROTO_TCP, TCP_FASTOPEN, &qlen, sizeof(qlen)); +#endif + if (listen(fd, TCP_BACKLOG) == -1) goto err; } @@ -758,7 +739,7 @@ static int make_sock(union mysockaddr *addr, int type, int dienow) { if (!option_bool(OPT_NOWILD)) { -#if defined(HAVE_LINUX_NETWORK) +#if defined(HAVE_LINUX_NETWORK) if (setsockopt(fd, IPPROTO_IP, IP_PKTINFO, &opt, sizeof(opt)) == -1) goto err; #elif defined(IP_RECVDSTADDR) && defined(IP_RECVIF) @@ -768,15 +749,12 @@ static int make_sock(union mysockaddr *addr, int type, int dienow) #endif } } -#ifdef HAVE_IPV6 else if (!set_ipv6pktinfo(fd)) goto err; -#endif - + return fd; } -#ifdef HAVE_IPV6 int set_ipv6pktinfo(int fd) { int opt = 1; @@ -785,7 +763,7 @@ int set_ipv6pktinfo(int fd) handle all combinations of headers and kernel. OpenWrt note that this fixes the problem addressed by your very broken patch. */ daemon->v6pktinfo = IPV6_PKTINFO; - + #ifdef IPV6_RECVPKTINFO if (setsockopt(fd, IPPROTO_IPV6, IPV6_RECVPKTINFO, &opt, sizeof(opt)) != -1) return 1; @@ -795,7 +773,7 @@ int set_ipv6pktinfo(int fd) daemon->v6pktinfo = IPV6_2292PKTINFO; return 1; } -# endif +# endif #else if (setsockopt(fd, IPPROTO_IPV6, IPV6_PKTINFO, &opt, sizeof(opt)) != -1) return 1; @@ -803,12 +781,13 @@ int set_ipv6pktinfo(int fd) return 0; } -#endif /* Find the interface on which a TCP connection arrived, if possible, or zero otherwise. */ int tcp_interface(int fd, int af) -{ +{ + (void)fd; /* suppress potential unused warning */ + (void)af; /* suppress potential unused warning */ int if_index = 0; #ifdef HAVE_LINUX_NETWORK @@ -816,14 +795,14 @@ int tcp_interface(int fd, int af) struct cmsghdr *cmptr; struct msghdr msg; socklen_t len; - + /* use mshdr so that the CMSDG_* macros are available */ msg.msg_control = daemon->packet; msg.msg_controllen = len = daemon->packet_buff_sz; - + /* we overwrote the buffer... */ daemon->srv_save = NULL; - + if (af == AF_INET) { if (setsockopt(fd, IPPROTO_IP, IP_PKTINFO, &opt, sizeof(opt)) != -1 && @@ -837,22 +816,21 @@ int tcp_interface(int fd, int af) unsigned char *c; struct in_pktinfo *p; } p; - + p.c = CMSG_DATA(cmptr); if_index = p.p->ipi_ifindex; } } } -#ifdef HAVE_IPV6 else { /* Only the RFC-2292 API has the ability to find the interface for TCP connections, - it was removed in RFC-3542 !!!! + it was removed in RFC-3542 !!!! Fortunately, Linux kept the 2292 ABI when it moved to 3542. The following code always uses the old ABI, and should work with pre- and post-3542 kernel headers */ -#ifdef IPV6_2292PKTOPTIONS +#ifdef IPV6_2292PKTOPTIONS # define PKTOPTIONS IPV6_2292PKTOPTIONS #else # define PKTOPTIONS IPV6_PKTOPTIONS @@ -870,17 +848,16 @@ int tcp_interface(int fd, int af) struct in6_pktinfo *p; } p; p.c = CMSG_DATA(cmptr); - + if_index = p.p->ipi6_ifindex; } } } -#endif /* IPV6 */ #endif /* Linux */ - + return if_index; } - + static struct listener *create_listeners(union mysockaddr *addr, int do_tftp, int dienow) { struct listener *l = NULL; @@ -893,7 +870,7 @@ static struct listener *create_listeners(union mysockaddr *addr, int do_tftp, in fd = make_sock(addr, SOCK_DGRAM, dienow); tcpfd = make_sock(addr, SOCK_STREAM, dienow); } - + #ifdef HAVE_TFTP if (do_tftp) { @@ -905,15 +882,13 @@ static struct listener *create_listeners(union mysockaddr *addr, int do_tftp, in tftpfd = make_sock(addr, SOCK_DGRAM, dienow); addr->in.sin_port = save; } -# ifdef HAVE_IPV6 else { short save = addr->in6.sin6_port; addr->in6.sin6_port = htons(TFTP_PORT); tftpfd = make_sock(addr, SOCK_DGRAM, dienow); addr->in6.sin6_port = save; - } -# endif + } } #endif @@ -924,7 +899,7 @@ static struct listener *create_listeners(union mysockaddr *addr, int do_tftp, in l->family = addr->sa.sa_family; l->fd = fd; l->tcpfd = tcpfd; - l->tftpfd = tftpfd; + l->tftpfd = tftpfd; l->iface = NULL; } @@ -946,21 +921,19 @@ void create_wildcard_listeners(void) l = create_listeners(&addr, !!option_bool(OPT_TFTP), 1); -#ifdef HAVE_IPV6 memset(&addr, 0, sizeof(addr)); -# ifdef HAVE_SOCKADDR_SA_LEN +#ifdef HAVE_SOCKADDR_SA_LEN addr.in6.sin6_len = sizeof(addr.in6); -# endif +#endif addr.in6.sin6_family = AF_INET6; addr.in6.sin6_addr = in6addr_any; addr.in6.sin6_port = htons(daemon->port); - + l6 = create_listeners(&addr, !!option_bool(OPT_TFTP), 1); - if (l) + if (l) l->next = l6; - else + else l = l6; -#endif daemon->listeners = l; } @@ -985,7 +958,7 @@ void create_bound_listeners(int dienow) no interface with a matching address. These may be valid: eg it's possible to listen on 127.0.1.1 even if the loopback interface is 127.0.0.1 - If the address isn't valid the bind() will fail and we'll die() + If the address isn't valid the bind() will fail and we'll die() (except in bind-dynamic mode, when we'll complain but keep trying.) The resulting listeners have the ->iface field NULL, and this has to be @@ -993,7 +966,7 @@ void create_bound_listeners(int dienow) (no netmask) and some MTU login the tftp code. */ for (if_tmp = daemon->if_addrs; if_tmp; if_tmp = if_tmp->next) - if (!if_tmp->used && + if (!if_tmp->used && (new = create_listeners(&if_tmp->addr, !!option_bool(OPT_TFTP), dienow))) { new->next = daemon->listeners; @@ -1001,10 +974,10 @@ void create_bound_listeners(int dienow) } } -/* In --bind-interfaces, the only access control is the addresses we're listening on. +/* In --bind-interfaces, the only access control is the addresses we're listening on. There's nothing to avoid a query to the address of an internal interface arriving via - an external interface where we don't want to accept queries, except that in the usual - case the addresses of internal interfaces are RFC1918. When bind-interfaces in use, + an external interface where we don't want to accept queries, except that in the usual + case the addresses of internal interfaces are RFC1918. When bind-interfaces in use, and we listen on an address that looks like it's probably globally routeable, shout. The fix is to use --bind-dynamic, which actually checks the arrival interface too. @@ -1016,7 +989,7 @@ void create_bound_listeners(int dienow) void warn_bound_listeners(void) { - struct irec *iface; + struct irec *iface; int advice = 0; for (iface = daemon->interfaces; iface; iface = iface->next) @@ -1028,15 +1001,15 @@ void warn_bound_listeners(void) { inet_ntop(AF_INET, &iface->addr.in.sin_addr, daemon->addrbuff, ADDRSTRLEN); iface->warned = advice = 1; - my_syslog(LOG_WARNING, + my_syslog(LOG_WARNING, _("LOUD WARNING: listening on %s may accept requests via interfaces other than %s"), daemon->addrbuff, iface->name); } } } - + if (advice) - my_syslog(LOG_WARNING, _("LOUD WARNING: use --bind-dynamic rather than --bind-interfaces to avoid DNS amplification attacks via these interface(s)")); + my_syslog(LOG_WARNING, _("LOUD WARNING: use --bind-dynamic rather than --bind-interfaces to avoid DNS amplification attacks via these interface(s)")); } void warn_wild_labels(void) @@ -1051,65 +1024,65 @@ void warn_wild_labels(void) void warn_int_names(void) { struct interface_name *intname; - + for (intname = daemon->int_names; intname; intname = intname->next) if (!intname->addr) my_syslog(LOG_WARNING, _("warning: no addresses found for interface %s"), intname->intr); } - + int is_dad_listeners(void) { struct irec *iface; - + if (option_bool(OPT_NOWILD)) for (iface = daemon->interfaces; iface; iface = iface->next) if (iface->dad && !iface->done) return 1; - + return 0; } #ifdef HAVE_DHCP6 -void join_multicast(int dienow) +void join_multicast(int dienow) { struct irec *iface, *tmp; for (iface = daemon->interfaces; iface; iface = iface->next) if (iface->addr.sa.sa_family == AF_INET6 && iface->dhcp_ok && !iface->multicast_done) { - /* There's an irec per address but we only want to join for multicast + /* There's an irec per address but we only want to join for multicast once per interface. Weed out duplicates. */ for (tmp = daemon->interfaces; tmp; tmp = tmp->next) if (tmp->multicast_done && tmp->index == iface->index) break; - + iface->multicast_done = 1; - + if (!tmp) { struct ipv6_mreq mreq; int err = 0; mreq.ipv6mr_interface = iface->index; - + inet_pton(AF_INET6, ALL_RELAY_AGENTS_AND_SERVERS, &mreq.ipv6mr_multiaddr); - + if ((daemon->doing_dhcp6 || daemon->relay6) && setsockopt(daemon->dhcp6fd, IPPROTO_IPV6, IPV6_JOIN_GROUP, &mreq, sizeof(mreq)) == -1) err = errno; - + inet_pton(AF_INET6, ALL_SERVERS, &mreq.ipv6mr_multiaddr); - - if (daemon->doing_dhcp6 && + + if (daemon->doing_dhcp6 && setsockopt(daemon->dhcp6fd, IPPROTO_IPV6, IPV6_JOIN_GROUP, &mreq, sizeof(mreq)) == -1) err = errno; - + inet_pton(AF_INET6, ALL_ROUTERS, &mreq.ipv6mr_multiaddr); - + if (daemon->doing_ra && setsockopt(daemon->icmp6fd, IPPROTO_IPV6, IPV6_JOIN_GROUP, &mreq, sizeof(mreq)) == -1) err = errno; - + if (err) { char *s = _("interface %s failed to join DHCPv6 multicast group: %s"); @@ -1151,8 +1124,8 @@ int random_sock(int family) while(tries--) { unsigned short port = htons(daemon->min_port + (rand16() % ((unsigned short)ports_avail))); - - if (family == AF_INET) + + if (family == AF_INET) { addr.in.sin_addr.s_addr = INADDR_ANY; addr.in.sin_port = port; @@ -1160,20 +1133,18 @@ int random_sock(int family) addr.in.sin_len = sizeof(struct sockaddr_in); #endif } -#ifdef HAVE_IPV6 else { - addr.in6.sin6_addr = in6addr_any; + addr.in6.sin6_addr = in6addr_any; addr.in6.sin6_port = port; #ifdef HAVE_SOCKADDR_SA_LEN addr.in6.sin6_len = sizeof(struct sockaddr_in6); #endif } -#endif - + if (bind(fd, (struct sockaddr *)&addr, sa_len(&addr)) == 0) return fd; - + if (errno != EADDRINUSE && errno != EACCES) break; } @@ -1181,9 +1152,9 @@ int random_sock(int family) close(fd); } - return -1; + return -1; } - + int local_bind(int fd, union mysockaddr *addr, char *intname, unsigned int ifindex, int is_tcp) { @@ -1194,12 +1165,10 @@ int local_bind(int fd, union mysockaddr *addr, char *intname, unsigned int ifind { if (addr_copy.sa.sa_family == AF_INET) addr_copy.in.sin_port = 0; -#ifdef HAVE_IPV6 else addr_copy.in6.sin6_port = 0; -#endif } - + if (bind(fd, (struct sockaddr *)&addr_copy, sa_len(&addr_copy)) == -1) return 0; @@ -1212,7 +1181,7 @@ int local_bind(int fd, union mysockaddr *addr, char *intname, unsigned int ifind return setsockopt(fd, IPPROTO_IP, IP_UNICAST_IF, &ifindex_opt, sizeof(ifindex_opt)) == 0; } #endif -#if defined(HAVE_IPV6) && defined (IPV6_UNICAST_IF) +#if defined (IPV6_UNICAST_IF) if (addr_copy.sa.sa_family == AF_INET6) { uint32_t ifindex_opt = htonl(ifindex); @@ -1221,6 +1190,7 @@ int local_bind(int fd, union mysockaddr *addr, char *intname, unsigned int ifind #endif } + (void)intname; /* suppress potential unused warning */ #if defined(SO_BINDTODEVICE) if (intname[0] != 0 && setsockopt(fd, SOL_SOCKET, SO_BINDTODEVICE, intname, IF_NAMESIZE) == -1) @@ -1236,41 +1206,39 @@ static struct serverfd *allocate_sfd(union mysockaddr *addr, char *intname) unsigned int ifindex = 0; int errsave; int opt = 1; - + /* when using random ports, servers which would otherwise use the INADDR_ANY/port0 socket have sfd set to NULL */ if (!daemon->osport && intname[0] == 0) { errno = 0; - + if (addr->sa.sa_family == AF_INET && addr->in.sin_addr.s_addr == INADDR_ANY && - addr->in.sin_port == htons(0)) + addr->in.sin_port == htons(0)) return NULL; -#ifdef HAVE_IPV6 if (addr->sa.sa_family == AF_INET6 && memcmp(&addr->in6.sin6_addr, &in6addr_any, sizeof(in6addr_any)) == 0 && - addr->in6.sin6_port == htons(0)) + addr->in6.sin6_port == htons(0)) return NULL; -#endif } if (intname && strlen(intname) != 0) ifindex = if_nametoindex(intname); /* index == 0 when not binding to an interface */ - + /* may have a suitable one already */ for (sfd = daemon->sfds; sfd; sfd = sfd->next ) if (sockaddr_isequal(&sfd->source_addr, addr) && strcmp(intname, sfd->interface) == 0 && - ifindex == sfd->ifindex) + ifindex == sfd->ifindex) return sfd; - + /* need to make a new one. */ errno = ENOMEM; /* in case malloc fails. */ if (!(sfd = whine_malloc(sizeof(struct serverfd)))) return NULL; - + if ((sfd->fd = socket(addr->sa.sa_family, SOCK_DGRAM, 0)) == -1) { free(sfd); @@ -1279,7 +1247,7 @@ static struct serverfd *allocate_sfd(union mysockaddr *addr, char *intname) if ((addr->sa.sa_family == AF_INET6 && setsockopt(sfd->fd, IPPROTO_IPV6, IPV6_V6ONLY, &opt, sizeof(opt)) == -1) || !local_bind(sfd->fd, addr, intname, ifindex, 0) || !fix_fd(sfd->fd)) - { + { errsave = errno; /* save error from bind/setsockopt. */ close(sfd->fd); free(sfd); @@ -1287,14 +1255,14 @@ static struct serverfd *allocate_sfd(union mysockaddr *addr, char *intname) return NULL; } - safe_strncpy(sfd->interface, intname, sizeof(sfd->interface)); + safe_strncpy(sfd->interface, intname, sizeof(sfd->interface)); sfd->source_addr = *addr; sfd->next = daemon->sfds; sfd->ifindex = ifindex; sfd->preallocated = 0; daemon->sfds = sfd; - return sfd; + return sfd; } /* create upstream sockets during startup, before root is dropped which may be needed @@ -1303,7 +1271,7 @@ void pre_allocate_sfds(void) { struct server *srv; struct serverfd *sfd; - + if (daemon->query_port != 0) { union mysockaddr addr; @@ -1316,7 +1284,7 @@ void pre_allocate_sfds(void) #endif if ((sfd = allocate_sfd(&addr, ""))) sfd->preallocated = 1; -#ifdef HAVE_IPV6 + memset(&addr, 0, sizeof(addr)); addr.in6.sin6_family = AF_INET6; addr.in6.sin6_addr = in6addr_any; @@ -1326,9 +1294,8 @@ void pre_allocate_sfds(void) #endif if ((sfd = allocate_sfd(&addr, ""))) sfd->preallocated = 1; -#endif } - + for (srv = daemon->servers; srv; srv = srv->next) if (!(srv->flags & (SERV_LITERAL_ADDRESS | SERV_NO_ADDR | SERV_USE_RESOLV | SERV_NO_REBIND)) && !allocate_sfd(&srv->source_addr, srv->interface) && @@ -1343,7 +1310,7 @@ void pre_allocate_sfds(void) } die(_("failed to bind server socket for %s: %s"), daemon->namebuff, EC_BADNET); - } + } } void mark_servers(int flag) @@ -1367,7 +1334,7 @@ void cleanup_servers(void) struct server *serv, *tmp, **up; /* unlink and free anything still marked. */ - for (serv = daemon->servers, up = &daemon->servers; serv; serv = tmp) + for (serv = daemon->servers, up = &daemon->servers; serv; serv = tmp) { tmp = serv->next; if (serv->flags & SERV_MARK) @@ -1378,7 +1345,7 @@ void cleanup_servers(void) free(serv->domain); free(serv); } - else + else up = &serv->next; } @@ -1396,7 +1363,7 @@ void add_update_server(int flags, { struct server *serv, *next = NULL; char *domain_str = NULL; - + /* See if there is a suitable candidate, and unmark */ for (serv = daemon->servers; serv; serv = serv->next) if (serv->flags & SERV_MARK) @@ -1411,7 +1378,7 @@ void add_update_server(int flags, if (serv->flags & SERV_HAS_DOMAIN) continue; } - + break; } @@ -1443,7 +1410,7 @@ void add_update_server(int flags, strcpy(domain_str, domain); } } - + if (serv) { memset(serv, 0, sizeof(struct server)); @@ -1453,11 +1420,11 @@ void add_update_server(int flags, serv->queries = serv->failed_queries = 0; #ifdef HAVE_LOOP serv->uid = rand32(); -#endif +#endif if (domain) serv->flags |= SERV_HAS_DOMAIN; - + if (interface) safe_strncpy(serv->interface, interface, sizeof(serv->interface)); if (addr) @@ -1490,28 +1457,28 @@ void check_servers(void) /* Init edns_pktsz for newly created server records. */ if (serv->edns_pktsz == 0) serv->edns_pktsz = daemon->edns_pktsz; - + #ifdef HAVE_DNSSEC if (option_bool(OPT_DNSSEC_VALID)) - { + { if (!(serv->flags & SERV_FOR_NODOTS)) serv->flags |= SERV_DO_DNSSEC; - + /* Disable DNSSEC validation when using server=/domain/.... servers unless there's a configured trust anchor. */ if (serv->flags & SERV_HAS_DOMAIN) { struct ds_config *ds; char *domain = serv->domain; - + /* .example.com is valid */ while (*domain == '.') domain++; - + for (ds = daemon->ds; ds; ds = ds->next) if (ds->name[0] != 0 && hostname_isequal(domain, ds->name)) break; - + if (!ds) serv->flags &= ~SERV_DO_DNSSEC; } @@ -1519,7 +1486,7 @@ void check_servers(void) #endif port = prettyprint_addr(&serv->addr, daemon->namebuff); - + /* 0.0.0.0 is nothing, the stack treats it like 127.0.0.1 */ if (serv->addr.sa.sa_family == AF_INET && serv->addr.in.sin_addr.s_addr == 0) @@ -1537,28 +1504,28 @@ void check_servers(void) serv->flags |= SERV_MARK; continue; } - + /* Do we need a socket set? */ - if (!serv->sfd && + if (!serv->sfd && !(serv->sfd = allocate_sfd(&serv->source_addr, serv->interface)) && errno != 0) { - my_syslog(LOG_WARNING, + my_syslog(LOG_WARNING, _("ignoring nameserver %s - cannot make/bind socket: %s"), daemon->namebuff, strerror(errno)); serv->flags |= SERV_MARK; continue; } - + if (serv->sfd) serv->sfd->used = 1; } - + if (!(serv->flags & SERV_NO_REBIND) && !(serv->flags & SERV_LITERAL_ADDRESS)) { if (++count > SERVERS_LOGGED) continue; - + if (serv->flags & (SERV_HAS_DOMAIN | SERV_FOR_NODOTS | SERV_USE_RESOLV)) { char *s1, *s2, *s3 = ""; @@ -1572,29 +1539,29 @@ void check_servers(void) s1 = _("default"), s2 = ""; else s1 = _("domain"), s2 = serv->domain; - + if (serv->flags & SERV_NO_ADDR) { count--; if (++locals <= LOCALS_LOGGED) - my_syslog(LOG_INFO, _("using local addresses only for %s %s"), s1, s2); + my_syslog(LOG_INFO, _("using only locally-known addresses for %s %s"), s1, s2); } else if (serv->flags & SERV_USE_RESOLV) my_syslog(LOG_INFO, _("using standard nameservers for %s %s"), s1, s2); - else + else my_syslog(LOG_INFO, _("using nameserver %s#%d for %s %s %s"), daemon->namebuff, port, s1, s2, s3); } #ifdef HAVE_LOOP else if (serv->flags & SERV_LOOP) - my_syslog(LOG_INFO, _("NOT using nameserver %s#%d - query loop detected"), daemon->namebuff, port); + my_syslog(LOG_INFO, _("NOT using nameserver %s#%d - query loop detected"), daemon->namebuff, port); #endif else if (serv->interface[0] != 0) - my_syslog(LOG_INFO, _("using nameserver %s#%d(via %s)"), daemon->namebuff, port, serv->interface); + my_syslog(LOG_INFO, _("using nameserver %s#%d(via %s)"), daemon->namebuff, port, serv->interface); else - my_syslog(LOG_INFO, _("using nameserver %s#%d"), daemon->namebuff, port); + my_syslog(LOG_INFO, _("using nameserver %s#%d"), daemon->namebuff, port); } } - + if (locals > LOCALS_LOGGED) my_syslog(LOG_INFO, _("using %d more local addresses"), locals - LOCALS_LOGGED); if (count - 1 > SERVERS_LOGGED) @@ -1604,16 +1571,16 @@ void check_servers(void) for (sfd = daemon->sfds, up = &daemon->sfds; sfd; sfd = tmp) { tmp = sfd->next; - if (!sfd->used) + if (!sfd->used) { *up = sfd->next; close(sfd->fd); free(sfd); - } + } else up = &sfd->next; } - + cleanup_servers(); } @@ -1631,24 +1598,24 @@ int reload_servers(char *fname) my_syslog(LOG_ERR, _("failed to read %s: %s"), fname, strerror(errno)); return 0; } - + mark_servers(SERV_FROM_RESOLV); - + while ((line = fgets(daemon->namebuff, MAXDNAME, f))) { union mysockaddr addr, source_addr; char *token = strtok(line, " \t\n\r"); - + if (!token) continue; if (strcmp(token, "nameserver") != 0 && strcmp(token, "server") != 0) continue; if (!(token = strtok(NULL, " \t\n\r"))) continue; - + memset(&addr, 0, sizeof(addr)); memset(&source_addr, 0, sizeof(source_addr)); - + if ((addr.in.sin_addr.s_addr = inet_addr(token)) != (in_addr_t) -1) { #ifdef HAVE_SOCKADDR_SA_LEN @@ -1659,18 +1626,17 @@ int reload_servers(char *fname) source_addr.in.sin_addr.s_addr = INADDR_ANY; source_addr.in.sin_port = htons(daemon->query_port); } -#ifdef HAVE_IPV6 - else - { + else + { int scope_index = 0; char *scope_id = strchr(token, '%'); - + if (scope_id) { *(scope_id++) = 0; scope_index = if_nametoindex(scope_id); } - + if (inet_pton(AF_INET6, token, &addr.in6.sin6_addr) > 0) { #ifdef HAVE_SOCKADDR_SA_LEN @@ -1687,15 +1653,11 @@ int reload_servers(char *fname) else continue; } -#else /* IPV6 */ - else - continue; -#endif add_update_server(SERV_FROM_RESOLV, &addr, &source_addr, NULL, NULL); gotone = 1; } - + fclose(f); cleanup_servers(); @@ -1706,21 +1668,21 @@ int reload_servers(char *fname) void newaddress(time_t now) { (void)now; - + if (option_bool(OPT_CLEVERBIND) || option_bool(OPT_LOCAL_SERVICE) || daemon->doing_dhcp6 || daemon->relay6 || daemon->doing_ra) enumerate_interfaces(0); - + if (option_bool(OPT_CLEVERBIND)) create_bound_listeners(0); - + #ifdef HAVE_DHCP6 if (daemon->doing_dhcp6 || daemon->relay6 || daemon->doing_ra) join_multicast(0); - + if (daemon->doing_dhcp6 || daemon->doing_ra) dhcp_construct_contexts(now); - + if (daemon->doing_dhcp6) lease_find_interfaces(now); #endif diff --git a/dnsmasq/option.c b/src/dnsmasq/option.c similarity index 83% rename from dnsmasq/option.c rename to src/dnsmasq/option.c index b6f4a6367..71c9fddaa 100644 --- a/dnsmasq/option.c +++ b/src/dnsmasq/option.c @@ -1,15 +1,15 @@ -/* dnsmasq is Copyright (c) 2000-2018 Simon Kelley +/* dnsmasq is Copyright (c) 2000-2020 Simon Kelley This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; version 2 dated June, 1991, or (at your option) version 3 dated 29 June, 2007. - + This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. - + You should have received a copy of the GNU General Public License along with this program. If not, see . */ @@ -19,6 +19,10 @@ #include "dnsmasq.h" #include +/* Pi-hole modification */ +extern char *get_FTL_version(void); +/************************/ + static volatile int mem_recover = 0; static jmp_buf mem_jmp; static int one_file(char *file, int hard_opt); @@ -129,9 +133,6 @@ struct myoption { #define LOPT_AUTHPEER 318 #define LOPT_IPSET 319 #define LOPT_SYNTH 320 -#ifdef OPTION6_PREFIX_CLASS -#define LOPT_PREF_CLSS 321 -#endif #define LOPT_RELAY 323 #define LOPT_RA_PARAM 324 #define LOPT_ADD_SBNET 325 @@ -166,13 +167,17 @@ struct myoption { #define LOPT_UBUS 354 #define LOPT_NAME_MATCH 355 #define LOPT_CAA 356 - +#define LOPT_SHARED_NET 357 +#define LOPT_IGNORE_CLID 358 +#define LOPT_SINGLE_PORT 359 +#define LOPT_SCRIPT_TIME 360 + #ifdef HAVE_GETOPT_LONG -static const struct option opts[] = +static const struct option opts[] = #else -static const struct myoption opts[] = +static const struct myoption opts[] = #endif - { + { { "version", 0, 0, 'v' }, { "no-hosts", 0, 0, 'h' }, { "no-poll", 0, 0, 'n' }, @@ -237,7 +242,7 @@ static const struct myoption opts[] = { "caa-record", 1, 0 , LOPT_CAA }, { "dns-rr", 1, 0, LOPT_RR }, { "enable-dbus", 2, 0, '1' }, - { "enable-ubus", 0, 0, LOPT_UBUS }, + { "enable-ubus", 2, 0, LOPT_UBUS }, { "bootp-dynamic", 2, 0, '3' }, { "dhcp-mac", 1, 0, '4' }, { "no-ping", 0, 0, '5' }, @@ -245,6 +250,7 @@ static const struct myoption opts[] = { "conf-dir", 1, 0, '7' }, { "log-facility", 1, 0 ,'8' }, { "leasefile-ro", 0, 0, '9' }, + { "script-on-renewal", 0, 0, LOPT_SCRIPT_TIME}, { "dns-forward-max", 1, 0, '0' }, { "clear-on-reload", 0, 0, LOPT_RELOAD }, { "dhcp-ignore-names", 2, 0, LOPT_NO_NAMES }, @@ -256,9 +262,11 @@ static const struct myoption opts[] = { "tftp-max", 1, 0, LOPT_TFTP_MAX }, { "tftp-mtu", 1, 0, LOPT_TFTP_MTU }, { "tftp-lowercase", 0, 0, LOPT_TFTP_LC }, + { "tftp-single-port", 0, 0, LOPT_SINGLE_PORT }, { "ptr-record", 1, 0, LOPT_PTR }, { "naptr-record", 1, 0, LOPT_NAPTR }, { "bridge-interface", 1, 0 , LOPT_BRIDGE }, + { "shared-network", 1, 0, LOPT_SHARED_NET }, { "dhcp-option-force", 1, 0, LOPT_FORCE }, { "tftp-no-blocksize", 0, 0, LOPT_NOBLOCK }, { "log-dhcp", 0, 0, LOPT_LOG_OPTS }, @@ -275,7 +283,7 @@ static const struct myoption opts[] = { "tftp-port-range", 1, 0, LOPT_TFTPPORTS }, { "stop-dns-rebind", 0, 0, LOPT_REBIND }, { "rebind-domain-ok", 1, 0, LOPT_NO_REBIND }, - { "all-servers", 0, 0, LOPT_NOLAST }, + { "all-servers", 0, 0, LOPT_NOLAST }, { "dhcp-match", 1, 0, LOPT_MATCH }, { "dhcp-name-match", 1, 0, LOPT_NAME_MATCH }, { "dhcp-broadcast", 2, 0, LOPT_BROADCAST }, @@ -313,7 +321,7 @@ static const struct myoption opts[] = { "auth-ttl", 1, 0, LOPT_AUTHTTL }, { "auth-soa", 1, 0, LOPT_AUTHSOA }, { "auth-sec-servers", 1, 0, LOPT_AUTHSFS }, - { "auth-peer", 1, 0, LOPT_AUTHPEER }, + { "auth-peer", 1, 0, LOPT_AUTHPEER }, { "ipset", 1, 0, LOPT_IPSET }, { "synth-domain", 1, 0, LOPT_SYNTH }, { "dnssec", 0, 0, LOPT_SEC_VALID }, @@ -322,9 +330,6 @@ static const struct myoption opts[] = { "dnssec-check-unsigned", 2, 0, LOPT_DNSSEC_CHECK }, { "dnssec-no-timecheck", 0, 0, LOPT_DNSSEC_TIME }, { "dnssec-timestamp", 1, 0, LOPT_DNSSEC_STAMP }, -#ifdef OPTION6_PREFIX_CLASS - { "dhcp-prefix-class", 1, 0, LOPT_PREF_CLSS }, -#endif { "dhcp-relay", 1, 0, LOPT_RELAY }, { "ra-param", 1, 0, LOPT_RA_PARAM }, { "quiet-dhcp", 0, 0, LOPT_QUIET_DHCP }, @@ -337,6 +342,7 @@ static const struct myoption opts[] = { "dhcp-rapid-commit", 0, 0, LOPT_RAPID_COMMIT }, { "dumpfile", 1, 0, LOPT_DUMPFILE }, { "dumpmask", 1, 0, LOPT_DUMPMASK }, + { "dhcp-ignore-clid", 0, 0, LOPT_IGNORE_CLID }, { NULL, 0, 0, 0 } }; @@ -356,11 +362,11 @@ static struct { { 'a', ARG_DUP, "", gettext_noop("Specify local address(es) to listen on."), NULL }, { 'A', ARG_DUP, "//", gettext_noop("Return ipaddr for all hosts in specified domains."), NULL }, { 'b', OPT_BOGUSPRIV, NULL, gettext_noop("Fake reverse lookups for RFC1918 private address ranges."), NULL }, - { 'B', ARG_DUP, "", gettext_noop("Treat ipaddr as NXDOMAIN (defeats Verisign wildcard)."), NULL }, + { 'B', ARG_DUP, "", gettext_noop("Treat ipaddr as NXDOMAIN (defeats Verisign wildcard)."), NULL }, { 'c', ARG_ONE, "", gettext_noop("Specify the size of the cache in entries (defaults to %s)."), "$" }, { 'C', ARG_DUP, "", gettext_noop("Specify configuration file (defaults to %s)."), CONFFILE }, { 'd', OPT_DEBUG, NULL, gettext_noop("Do NOT fork into the background: run in debug mode."), NULL }, - { 'D', OPT_NODOTS_LOCAL, NULL, gettext_noop("Do NOT forward queries with no domain part."), NULL }, + { 'D', OPT_NODOTS_LOCAL, NULL, gettext_noop("Do NOT forward queries with no domain part."), NULL }, { 'e', OPT_SELFMX, NULL, gettext_noop("Return self-pointing MX records for local hosts."), NULL }, { 'E', OPT_EXPAND, NULL, gettext_noop("Expand simple names in /etc/hosts with domain-suffix."), NULL }, { 'f', OPT_FILTER, NULL, gettext_noop("Don't forward spurious DNS requests from Windows hosts."), NULL }, @@ -369,8 +375,8 @@ static struct { { 'G', ARG_DUP, "", gettext_noop("Set address or hostname for a specified machine."), NULL }, { LOPT_DHCP_HOST, ARG_DUP, "", gettext_noop("Read DHCP host specs from file."), NULL }, { LOPT_DHCP_OPTS, ARG_DUP, "", gettext_noop("Read DHCP option specs from file."), NULL }, - { LOPT_DHCP_INOTIFY, ARG_DUP, "", gettext_noop("Read DHCP host specs from a directory."), NULL }, - { LOPT_DHOPT_INOTIFY, ARG_DUP, "", gettext_noop("Read DHCP options from a directory."), NULL }, + { LOPT_DHCP_INOTIFY, ARG_DUP, "", gettext_noop("Read DHCP host specs from a directory."), NULL }, + { LOPT_DHOPT_INOTIFY, ARG_DUP, "", gettext_noop("Read DHCP options from a directory."), NULL }, { LOPT_TAG_IF, ARG_DUP, "tag-expression", gettext_noop("Evaluate conditional tag expression."), NULL }, { 'h', OPT_NO_HOSTS, NULL, gettext_noop("Do NOT load %s file."), HOSTSFILE }, { 'H', ARG_DUP, "", gettext_noop("Specify a hosts file to be read in addition to %s."), HOSTSFILE }, @@ -382,14 +388,14 @@ static struct { { LOPT_REMOTE, ARG_DUP, "set:,", gettext_noop("Map RFC3046 remote-id to tag."), NULL }, { LOPT_SUBSCR, ARG_DUP, "set:,", gettext_noop("Map RFC3993 subscriber-id to tag."), NULL }, { 'J', ARG_DUP, "tag:...", gettext_noop("Don't do DHCP for hosts with tag set."), NULL }, - { LOPT_BROADCAST, ARG_DUP, "[=tag:...]", gettext_noop("Force broadcast replies for hosts with tag set."), NULL }, + { LOPT_BROADCAST, ARG_DUP, "[=tag:...]", gettext_noop("Force broadcast replies for hosts with tag set."), NULL }, { 'k', OPT_NO_FORK, NULL, gettext_noop("Do NOT fork into the background, do NOT run in debug mode."), NULL }, { 'K', OPT_AUTHORITATIVE, NULL, gettext_noop("Assume we are the only DHCP server on the local network."), NULL }, { 'l', ARG_ONE, "", gettext_noop("Specify where to store DHCP leases (defaults to %s)."), LEASEFILE }, { 'L', OPT_LOCALMX, NULL, gettext_noop("Return MX records for local hosts."), NULL }, { 'm', ARG_DUP, ",,", gettext_noop("Specify an MX record."), NULL }, { 'M', ARG_DUP, "", gettext_noop("Specify BOOTP options to DHCP server."), NULL }, - { 'n', OPT_NO_POLL, NULL, gettext_noop("Do NOT poll %s file, reload only on SIGHUP."), RESOLVFILE }, + { 'n', OPT_NO_POLL, NULL, gettext_noop("Do NOT poll %s file, reload only on SIGHUP."), RESOLVFILE }, { 'N', OPT_NO_NEG, NULL, gettext_noop("Do NOT cache failed search results."), NULL }, { 'o', OPT_ORDER, NULL, gettext_noop("Use nameservers strictly in the order given in %s."), RESOLVFILE }, { 'O', ARG_DUP, "", gettext_noop("Specify options to be sent to DHCP clients."), NULL }, @@ -399,7 +405,7 @@ static struct { { 'q', ARG_DUP, NULL, gettext_noop("Log DNS queries."), NULL }, { 'Q', ARG_ONE, "", gettext_noop("Force the originating port for upstream DNS queries."), NULL }, { 'R', OPT_NO_RESOLV, NULL, gettext_noop("Do NOT read resolv.conf."), NULL }, - { 'r', ARG_DUP, "", gettext_noop("Specify path to resolv.conf (defaults to %s)."), RESOLVFILE }, + { 'r', ARG_DUP, "", gettext_noop("Specify path to resolv.conf (defaults to %s)."), RESOLVFILE }, { LOPT_SERVERS_FILE, ARG_ONE, "", gettext_noop("Specify path to file with server= options"), NULL }, { 'S', ARG_DUP, "//", gettext_noop("Specify address(es) of upstream servers with optional domains."), NULL }, { LOPT_REV_SERV, ARG_DUP, "/,", gettext_noop("Specify address of upstream servers for reverse address queries"), NULL }, @@ -411,7 +417,7 @@ static struct { { LOPT_MAXTTL, ARG_ONE, "", gettext_noop("Specify time-to-live in seconds for maximum TTL to send to clients."), NULL }, { LOPT_MAXCTTL, ARG_ONE, "", gettext_noop("Specify time-to-live ceiling for cache."), NULL }, { LOPT_MINCTTL, ARG_ONE, "", gettext_noop("Specify time-to-live floor for cache."), NULL }, - { 'u', ARG_ONE, "", gettext_noop("Change to this user after startup. (defaults to %s)."), CHUSER }, + { 'u', ARG_ONE, "", gettext_noop("Change to this user after startup. (defaults to %s)."), CHUSER }, { 'U', ARG_DUP, "set:,", gettext_noop("Map DHCP vendor class to tag."), NULL }, { 'v', 0, NULL, gettext_noop("Display dnsmasq version and copyright information."), NULL }, { 'V', ARG_DUP, ",,", gettext_noop("Translate IPv4 addresses from upstream servers."), NULL }, @@ -426,11 +432,12 @@ static struct { { 'z', OPT_NOWILD, NULL, gettext_noop("Bind only to interfaces in use."), NULL }, { 'Z', OPT_ETHERS, NULL, gettext_noop("Read DHCP static host information from %s."), ETHERSFILE }, { '1', ARG_ONE, "[=]", gettext_noop("Enable the DBus interface for setting upstream servers, etc."), NULL }, - { LOPT_UBUS, OPT_UBUS, NULL, gettext_noop("Enable the UBus interface."), NULL }, + { LOPT_UBUS, ARG_ONE, "[=]", gettext_noop("Enable the UBus interface."), NULL }, { '2', ARG_DUP, "", gettext_noop("Do not provide DHCP on this interface, only provide DNS."), NULL }, { '3', ARG_DUP, "[=tag:]...", gettext_noop("Enable dynamic address allocation for bootp."), NULL }, { '4', ARG_DUP, "set:,", gettext_noop("Map MAC address (with wildcards) to option set."), NULL }, { LOPT_BRIDGE, ARG_DUP, ",..", gettext_noop("Treat DHCP requests on aliases as arriving from interface."), NULL }, + { LOPT_SHARED_NET, ARG_DUP, "|,", gettext_noop("Specify extra networks sharing a broadcast domain for DHCP"), NULL}, { '5', OPT_NO_PING, NULL, gettext_noop("Disable ICMP echo address checking in the DHCP server."), NULL }, { '6', ARG_ONE, "", gettext_noop("Shell script to run on DHCP lease creation and destruction."), NULL }, { LOPT_LUASCRIPT, ARG_DUP, "path", gettext_noop("Lua script to run on DHCP lease creation and destruction."), NULL }, @@ -439,7 +446,7 @@ static struct { { '7', ARG_DUP, "", gettext_noop("Read configuration from all the files in this directory."), NULL }, { '8', ARG_ONE, "|", gettext_noop("Log to this syslog facility or file. (defaults to DAEMON)"), NULL }, { '9', OPT_LEASE_RO, NULL, gettext_noop("Do not use leasefile."), NULL }, - { '0', ARG_ONE, "", gettext_noop("Maximum number of concurrent DNS queries. (defaults to %s)"), "!" }, + { '0', ARG_ONE, "", gettext_noop("Maximum number of concurrent DNS queries. (defaults to %s)"), "!" }, { LOPT_RELOAD, OPT_RELOAD, NULL, gettext_noop("Clear DNS cache when reloading %s."), RESOLVFILE }, { LOPT_NO_NAMES, ARG_DUP, "[=tag:]...", gettext_noop("Ignore hostnames provided by DHCP clients."), NULL }, { LOPT_OVERRIDE, OPT_NO_OVERRIDE, NULL, gettext_noop("Do NOT reuse filename and server fields for extra DHCP options."), NULL }, @@ -453,6 +460,7 @@ static struct { { LOPT_NOBLOCK, OPT_TFTP_NOBLOCK, NULL, gettext_noop("Disable the TFTP blocksize extension."), NULL }, { LOPT_TFTP_LC, OPT_TFTP_LC, NULL, gettext_noop("Convert TFTP filenames to lowercase"), NULL }, { LOPT_TFTPPORTS, ARG_ONE, ",", gettext_noop("Ephemeral port range for use by TFTP transfers."), NULL }, + { LOPT_SINGLE_PORT, OPT_SINGLE_PORT, NULL, gettext_noop("Use only one port for TFTP server."), NULL }, { LOPT_LOG_OPTS, OPT_LOG_OPTS, NULL, gettext_noop("Extra logging for DHCP."), NULL }, { LOPT_MAX_LOGS, ARG_ONE, "[=]", gettext_noop("Enable async. logging; optionally set queue length."), NULL }, { LOPT_REBIND, OPT_NO_REBIND, NULL, gettext_noop("Stop DNS rebinding. Filter private IP ranges when resolving."), NULL }, @@ -478,12 +486,13 @@ static struct { { LOPT_CPE_ID, ARG_ONE, "", gettext_noop("Add client identification to forwarded DNS queries."), NULL }, { LOPT_DNSSEC, OPT_DNSSEC_PROXY, NULL, gettext_noop("Proxy DNSSEC validation results from upstream nameservers."), NULL }, { LOPT_INCR_ADDR, OPT_CONSEC_ADDR, NULL, gettext_noop("Attempt to allocate sequential IP addresses to DHCP clients."), NULL }, + { LOPT_IGNORE_CLID, OPT_IGNORE_CLID, NULL, gettext_noop("Ignore client identifier option sent by DHCP clients."), NULL }, { LOPT_CONNTRACK, OPT_CONNTRACK, NULL, gettext_noop("Copy connection-track mark from queries to upstream connections."), NULL }, { LOPT_FQDN, OPT_FQDN_UPDATE, NULL, gettext_noop("Allow DHCP clients to do their own DDNS updates."), NULL }, { LOPT_RA, OPT_RA, NULL, gettext_noop("Send router-advertisements for interfaces doing DHCPv6"), NULL }, { LOPT_DUID, ARG_ONE, ",", gettext_noop("Specify DUID_EN-type DHCPv6 server DUID"), NULL }, { LOPT_HOST_REC, ARG_DUP, ",
[,]", gettext_noop("Specify host (A/AAAA and PTR) records"), NULL }, - { LOPT_CAA, ARG_DUP, ",,,", gettext_noop("Specify certification authority authorization record"), NULL }, + { LOPT_CAA, ARG_DUP, ",,,", gettext_noop("Specify certification authority authorization record"), NULL }, { LOPT_RR, ARG_DUP, ",,[]", gettext_noop("Specify arbitrary DNS resource record"), NULL }, { LOPT_CLVERBIND, OPT_CLEVERBIND, NULL, gettext_noop("Bind to interfaces in use - check for new interfaces"), NULL }, { LOPT_AUTHSERV, ARG_ONE, ",", gettext_noop("Export local names to global DNS"), NULL }, @@ -500,30 +509,28 @@ static struct { { LOPT_DNSSEC_CHECK, ARG_DUP, NULL, gettext_noop("Ensure answers without DNSSEC are in unsigned zones."), NULL }, { LOPT_DNSSEC_TIME, OPT_DNSSEC_TIME, NULL, gettext_noop("Don't check DNSSEC signature timestamps until first cache-reload"), NULL }, { LOPT_DNSSEC_STAMP, ARG_ONE, "", gettext_noop("Timestamp file to verify system clock for DNSSEC"), NULL }, -#ifdef OPTION6_PREFIX_CLASS - { LOPT_PREF_CLSS, ARG_DUP, "set:tag,", gettext_noop("Specify DHCPv6 prefix class"), NULL }, -#endif { LOPT_RA_PARAM, ARG_DUP, ",[mtu:||off,][,][,]", gettext_noop("Set MTU, priority, resend-interval and router-lifetime"), NULL }, { LOPT_QUIET_DHCP, OPT_QUIET_DHCP, NULL, gettext_noop("Do not log routine DHCP."), NULL }, { LOPT_QUIET_DHCP6, OPT_QUIET_DHCP6, NULL, gettext_noop("Do not log routine DHCPv6."), NULL }, { LOPT_QUIET_RA, OPT_QUIET_RA, NULL, gettext_noop("Do not log RA."), NULL }, { LOPT_LOCAL_SERVICE, OPT_LOCAL_SERVICE, NULL, gettext_noop("Accept queries only from directly-connected networks."), NULL }, { LOPT_LOOP_DETECT, OPT_LOOP_DETECT, NULL, gettext_noop("Detect and remove DNS forwarding loops."), NULL }, - { LOPT_IGNORE_ADDR, ARG_DUP, "", gettext_noop("Ignore DNS responses containing ipaddr."), NULL }, - { LOPT_DHCPTTL, ARG_ONE, "", gettext_noop("Set TTL in DNS responses with DHCP-derived addresses."), NULL }, + { LOPT_IGNORE_ADDR, ARG_DUP, "", gettext_noop("Ignore DNS responses containing ipaddr."), NULL }, + { LOPT_DHCPTTL, ARG_ONE, "", gettext_noop("Set TTL in DNS responses with DHCP-derived addresses."), NULL }, { LOPT_REPLY_DELAY, ARG_ONE, "", gettext_noop("Delay DHCP replies for at least number of seconds."), NULL }, { LOPT_RAPID_COMMIT, OPT_RAPID_COMMIT, NULL, gettext_noop("Enables DHCPv4 Rapid Commit option."), NULL }, { LOPT_DUMPFILE, ARG_ONE, "", gettext_noop("Path to debug packet dump file"), NULL }, { LOPT_DUMPMASK, ARG_ONE, "", gettext_noop("Mask which packets to dump"), NULL }, + { LOPT_SCRIPT_TIME, OPT_LEASE_RENEW, NULL, gettext_noop("Call dhcp-script when lease expiry changes."), NULL }, { 0, 0, NULL, NULL, NULL } -}; +}; /* We hide metacharacters in quoted strings by mapping them into the ASCII control character space. Note that the \0, \t \b \r \033 and \n characters are carefully placed in the following sequence so that they map to themselves: it is therefore possible to call unhide_metas repeatedly on string without breaking things. - The transformation gets undone by opt_canonicalise, atoi_check and opt_string_alloc, and a - couple of other places. + The transformation gets undone by opt_canonicalise, atoi_check and opt_string_alloc, and a + couple of other places. Note that space is included here so that --dhcp-option=3, string has five characters, whilst @@ -540,17 +547,17 @@ static char hide_meta(char c) for (i = 0; i < (sizeof(meta) - 1); i++) if (c == meta[i]) return (char)i; - + return c; } static char unhide_meta(char cr) -{ +{ unsigned int c = cr; - + if (c < (sizeof(meta) - 1)) cr = meta[c]; - + return cr; } @@ -573,23 +580,24 @@ static void *opt_malloc(size_t size) } else ret = safe_malloc(size); - + return ret; } -static char *opt_string_alloc(char *cp) +static char *opt_string_alloc(const char *cp) { char *ret = NULL; - - if (cp && strlen(cp) != 0) + size_t len; + + if (cp && (len = strlen(cp)) != 0) { - ret = opt_malloc(strlen(cp)+1); - strcpy(ret, cp); - + ret = opt_malloc(len+1); + memcpy(ret, cp, len+1); + /* restore hidden metachars */ unhide_metas(ret); } - + return ret; } @@ -603,15 +611,15 @@ static char *split_chr(char *s, char c) if (!s || !(comma = strchr(s, c))) return NULL; - + p = comma; *comma = ' '; - + for (; *comma == ' '; comma++); - + for (; (p >= s) && *p == ' '; p--) *p = 0; - + return comma; } @@ -648,7 +656,7 @@ static int atoi_check(char *a, int *res) return 0; unhide_metas(a); - + for (p = a; *p; p++) if (*p < '0' || *p > '9') return 0; @@ -723,15 +731,15 @@ static void do_usage(void) printf(_("Use short options only on the command line.\n")); #endif printf(_("Valid options are:\n")); - + for (i = 0; usage[i].opt != 0; i++) { - char *desc = usage[i].flagdesc; + char *desc = usage[i].flagdesc; char *eq = "="; - + if (!desc || *desc == '[') eq = ""; - + if (!desc) desc = ""; @@ -742,10 +750,10 @@ static void do_usage(void) sprintf(buff, "-%c, ", usage[i].opt); else sprintf(buff, " "); - + sprintf(buff+4, "--%s%s%s", opts[j].name, eq, desc); printf("%-55.55s", buff); - + if (usage[i].arg) { strcpy(buff, usage[i].arg); @@ -759,18 +767,18 @@ static void do_usage(void) } #define ret_err(x) do { strcpy(errstr, (x)); return 0; } while (0) +#define ret_err_free(x,m) do { strcpy(errstr, (x)); free((m)); return 0; } while (0) +#define goto_err(x) do { strcpy(errstr, (x)); goto on_error; } while (0) -static char *parse_mysockaddr(char *arg, union mysockaddr *addr) +static char *parse_mysockaddr(char *arg, union mysockaddr *addr) { if (inet_pton(AF_INET, arg, &addr->in.sin_addr) > 0) addr->sa.sa_family = AF_INET; -#ifdef HAVE_IPV6 else if (inet_pton(AF_INET6, arg, &addr->in6.sin6_addr) > 0) addr->sa.sa_family = AF_INET6; -#endif else return _("bad address"); - + return NULL; } @@ -779,11 +787,9 @@ char *parse_server(char *arg, union mysockaddr *addr, union mysockaddr *source_a int source_port = 0, serv_port = NAMESERVER_PORT; char *portno, *source; char *interface_opt = NULL; -#ifdef HAVE_IPV6 int scope_index = 0; char *scope_id; -#endif - + if (!arg || strlen(arg) == 0) { *flags |= SERV_NO_ADDR; @@ -795,15 +801,13 @@ char *parse_server(char *arg, union mysockaddr *addr, union mysockaddr *source_a (portno = split_chr(source, '#')) && !atoi_check16(portno, &source_port)) return _("bad port"); - + if ((portno = split_chr(arg, '#')) && /* is there a port no. */ !atoi_check16(portno, &serv_port)) return _("bad port"); - -#ifdef HAVE_IPV6 + scope_id = split_chr(arg, '%'); -#endif - + if (source) { interface_opt = split_chr(source, '@'); @@ -819,14 +823,14 @@ char *parse_server(char *arg, union mysockaddr *addr, union mysockaddr *source_a if (inet_pton(AF_INET, arg, &addr->in.sin_addr) > 0) { - addr->in.sin_port = htons(serv_port); + addr->in.sin_port = htons(serv_port); addr->sa.sa_family = source_addr->sa.sa_family = AF_INET; #ifdef HAVE_SOCKADDR_SA_LEN source_addr->in.sin_len = addr->in.sin_len = sizeof(struct sockaddr_in); #endif source_addr->in.sin_addr.s_addr = INADDR_ANY; source_addr->in.sin_port = htons(daemon->query_port); - + if (source) { if (flags) @@ -837,7 +841,7 @@ char *parse_server(char *arg, union mysockaddr *addr, union mysockaddr *source_a #if defined(SO_BINDTODEVICE) if (interface_opt) return _("interface can only be specified once"); - + source_addr->in.sin_addr.s_addr = INADDR_ANY; safe_strncpy(interface, source, IF_NAMESIZE); #else @@ -846,15 +850,14 @@ char *parse_server(char *arg, union mysockaddr *addr, union mysockaddr *source_a } } } -#ifdef HAVE_IPV6 else if (inet_pton(AF_INET6, arg, &addr->in6.sin6_addr) > 0) { if (scope_id && (scope_index = if_nametoindex(scope_id)) == 0) return _("bad interface name"); - + addr->in6.sin6_port = htons(serv_port); addr->in6.sin6_scope_id = scope_index; - source_addr->in6.sin6_addr = in6addr_any; + source_addr->in6.sin6_addr = in6addr_any; source_addr->in6.sin6_port = htons(daemon->query_port); source_addr->in6.sin6_scope_id = 0; addr->sa.sa_family = source_addr->sa.sa_family = AF_INET6; @@ -872,7 +875,7 @@ char *parse_server(char *arg, union mysockaddr *addr, union mysockaddr *source_a #if defined(SO_BINDTODEVICE) if (interface_opt) return _("interface can only be specified once"); - + source_addr->in6.sin6_addr = in6addr_any; safe_strncpy(interface, source, IF_NAMESIZE); #else @@ -881,7 +884,6 @@ char *parse_server(char *arg, union mysockaddr *addr, union mysockaddr *source_a } } } -#endif else return _("bad address"); @@ -912,11 +914,13 @@ static struct server *add_rev4(struct in_addr addr, int msize) p += sprintf(p, "%d.", (a >> 24) & 0xff); break; default: + free(serv->domain); + free(serv); return NULL; } p += sprintf(p, "in-addr.arpa"); - + serv->flags = SERV_HAS_DOMAIN; serv->next = daemon->servers; daemon->servers = serv; @@ -930,21 +934,21 @@ static struct server *add_rev6(struct in6_addr *addr, int msize) struct server *serv = opt_malloc(sizeof(struct server)); char *p; int i; - + memset(serv, 0, sizeof(struct server)); p = serv->domain = opt_malloc(73); /* strlen("32*ip6.arpa")+1 */ - + for (i = msize-1; i >= 0; i -= 4) - { + { int dig = ((unsigned char *)addr)[i>>3]; p += sprintf(p, "%.1x.", (i>>2) & 1 ? dig & 15 : dig >> 4); } p += sprintf(p, "ip6.arpa"); - + serv->flags = SERV_HAS_DOMAIN; serv->next = daemon->servers; daemon->servers = serv; - + return serv; } @@ -954,7 +958,7 @@ static int is_tag_prefix(char *arg) { if (arg && (strstr(arg, "net:") == arg || strstr(arg, "tag:") == arg)) return 1; - + return 0; } @@ -962,10 +966,120 @@ static char *set_prefix(char *arg) { if (strstr(arg, "set:") == arg) return arg+4; - + return arg; } +static struct dhcp_netid *dhcp_netid_create(const char *net, struct dhcp_netid *next) +{ + struct dhcp_netid *tt; + tt = opt_malloc(sizeof (struct dhcp_netid)); + tt->net = opt_string_alloc(net); + tt->next = next; + return tt; +} + +static void dhcp_netid_free(struct dhcp_netid *nid) +{ + while (nid) + { + struct dhcp_netid *tmp = nid; + nid = nid->next; + free(tmp->net); + free(tmp); + } +} + +/* Parse one or more tag:s before parameters. + * Moves arg to the end of tags. */ +static struct dhcp_netid * dhcp_tags(char **arg) +{ + struct dhcp_netid *id = NULL; + + while (is_tag_prefix(*arg)) + { + char *comma = split(*arg); + id = dhcp_netid_create((*arg)+4, id); + *arg = comma; + }; + if (!*arg) + { + dhcp_netid_free(id); + id = NULL; + } + return id; +} + +static void dhcp_netid_list_free(struct dhcp_netid_list *netid) +{ + while (netid) + { + struct dhcp_netid_list *tmplist = netid; + netid = netid->next; + dhcp_netid_free(tmplist->list); + free(tmplist); + } +} + +static void dhcp_config_free(struct dhcp_config *config) +{ + if (config) + { + struct hwaddr_config *hwaddr = config->hwaddr; + + while (hwaddr) + { + struct hwaddr_config *tmp = hwaddr; + hwaddr = hwaddr->next; + free(tmp); + } + + dhcp_netid_list_free(config->netid); + dhcp_netid_free(config->filter); + + if (config->flags & CONFIG_CLID) + free(config->clid); + +#ifdef HAVE_DHCP6 + if (config->flags & CONFIG_ADDR6) + { + struct addrlist *addr, *tmp; + + for (addr = config->addr6; addr; addr = tmp) + { + tmp = addr->next; + free(addr); + } + } +#endif + + free(config); + } +} + +static void dhcp_context_free(struct dhcp_context *ctx) +{ + if (ctx) + { + dhcp_netid_free(ctx->filter); + free(ctx->netid.net); +#ifdef HAVE_DHCP6 + free(ctx->template_interface); +#endif + free(ctx); + } +} + +static void dhcp_opt_free(struct dhcp_opt *opt) +{ + if (opt->flags & DHOPT_VENDOR) + free(opt->u.vendor_class); + dhcp_netid_free(opt->netid); + free(opt->val); + free(opt); +} + + /* This is too insanely large to keep in-line in the switch */ static int parse_dhcp_opt(char *errstr, char *arg, int flags) { @@ -973,7 +1087,6 @@ static int parse_dhcp_opt(char *errstr, char *arg, int flags) char lenchar = 0, *cp; int addrs, digs, is_addr, is_addr6, is_hex, is_dec, is_string, dots; char *comma = NULL; - struct dhcp_netid *np = NULL; u16 opt_len = 0; int is6 = 0; int option_ok = 0; @@ -983,15 +1096,15 @@ static int parse_dhcp_opt(char *errstr, char *arg, int flags) new->netid = NULL; new->val = NULL; new->opt = 0; - + while (arg) { - comma = split(arg); + comma = split(arg); for (cp = arg; *cp; cp++) if (*cp < '0' || *cp > '9') break; - + if (!*cp) { new->opt = atoi(arg); @@ -999,7 +1112,7 @@ static int parse_dhcp_opt(char *errstr, char *arg, int flags) option_ok = 1; break; } - + if (strstr(arg, "option:") == arg) { if ((new->opt = lookup_dhcp_opt(AF_INET, arg+7)) != -1) @@ -1017,7 +1130,7 @@ static int parse_dhcp_opt(char *errstr, char *arg, int flags) for (cp = arg+8; *cp; cp++) if (*cp < '0' || *cp > '9') break; - + if (!*cp) { new->opt = atoi(arg+8); @@ -1060,25 +1173,20 @@ static int parse_dhcp_opt(char *errstr, char *arg, int flags) } else { - new->netid = opt_malloc(sizeof (struct dhcp_netid)); /* allow optional "net:" or "tag:" for consistency */ - if (is_tag_prefix(arg)) - new->netid->net = opt_string_alloc(arg+4); - else - new->netid->net = opt_string_alloc(set_prefix(arg)); - new->netid->next = np; - np = new->netid; + const char *name = (is_tag_prefix(arg)) ? arg+4 : set_prefix(arg); + new->netid = dhcp_netid_create(name, new->netid); } - - arg = comma; + + arg = comma; } #ifdef HAVE_DHCP6 if (is6) { if (new->flags & (DHOPT_VENDOR | DHOPT_ENCAPSULATE)) - ret_err(_("unsupported encapsulation for IPv6 option")); - + goto_err(_("unsupported encapsulation for IPv6 option")); + if (opt_len == 0 && !(new->flags & DHOPT_RFC3925)) opt_len = lookup_dhcp_len(AF_INET6, new->opt); @@ -1088,16 +1196,16 @@ static int parse_dhcp_opt(char *errstr, char *arg, int flags) if (opt_len == 0 && !(new->flags & (DHOPT_VENDOR | DHOPT_ENCAPSULATE | DHOPT_RFC3925))) opt_len = lookup_dhcp_len(AF_INET, new->opt); - + /* option may be missing with rfc3925 match */ if (!option_ok) - ret_err(_("bad dhcp-option")); - + goto_err(_("bad dhcp-option")); + if (comma) { /* characterise the value */ char c; - int found_dig = 0; + int found_dig = 0, found_colon = 0; is_addr = is_addr6 = is_hex = is_dec = is_string = 1; addrs = digs = 1; dots = 0; @@ -1111,14 +1219,15 @@ static int parse_dhcp_opt(char *errstr, char *arg, int flags) { digs++; is_dec = is_addr = 0; + found_colon = 1; } - else if (c == '/') + else if (c == '/') { is_addr6 = is_dec = is_hex = 0; if (cp == comma) /* leading / means a pathname */ is_addr = 0; - } - else if (c == '.') + } + else if (c == '.') { is_addr6 = is_dec = is_hex = 0; dots++; @@ -1139,7 +1248,7 @@ static int parse_dhcp_opt(char *errstr, char *arg, int flags) else is_dec = 0; if (!((c >='A' && c <= 'F') || - (c >='a' && c <= 'f') || + (c >='a' && c <= 'f') || (c == '*' && (flags & DHOPT_MATCH)))) { is_hex = 0; @@ -1149,25 +1258,34 @@ static int parse_dhcp_opt(char *errstr, char *arg, int flags) } else found_dig = 1; - + if (!found_dig) is_dec = is_addr = 0; + if (!found_colon) + is_addr6 = 0; + +#ifdef HAVE_DHCP6 + /* NTP server option takes hex, addresses or FQDN */ + if (is6 && new->opt == OPTION6_NTP_SERVER && !is_hex) + opt_len |= is_addr6 ? OT_ADDR_LIST : OT_RFC1035_NAME; +#endif + /* We know that some options take addresses */ if (opt_len & OT_ADDR_LIST) { is_string = is_dec = is_hex = 0; - + if (!is6 && (!is_addr || dots == 0)) - ret_err(_("bad IP address")); + goto_err(_("bad IP address")); if (is6 && !is_addr6) - ret_err(_("bad IPv6 address")); + goto_err(_("bad IPv6 address")); } /* or names */ else if (opt_len & (OT_NAME | OT_RFC1035_NAME | OT_CSTRING)) is_addr6 = is_addr = is_dec = is_hex = 0; - + if (found_dig && (opt_len & OT_TIME) && strlen(comma) > 0) { int val, fac = 1; @@ -1194,12 +1312,12 @@ static int parse_dhcp_opt(char *errstr, char *arg, int flags) case 'S': comma[strlen(comma) - 1] = 0; } - + new->len = 4; new->val = opt_malloc(4); val = atoi(comma); - *((int *)new->val) = htonl(val * fac); - } + *((int *)new->val) = htonl(val * fac); + } else if (is_hex && digs > 1) { new->len = digs; @@ -1228,12 +1346,12 @@ static int parse_dhcp_opt(char *errstr, char *arg, int flags) new->len = 2; else if (lenchar == 'i') new->len = 4; - + new->val = opt_malloc(new->len); for (i=0; ilen; i++) new->val[i] = val>>((new->len - i - 1)*8); } - else if (is_addr && !is6) + else if (is_addr && !is6) { struct in_addr in; unsigned char *op; @@ -1243,19 +1361,19 @@ static int parse_dhcp_opt(char *errstr, char *arg, int flags) new->val = op = opt_malloc((5 * addrs) + 1); new->flags |= DHOPT_ADDR; - if (!(new->flags & (DHOPT_ENCAPSULATE | DHOPT_VENDOR | DHOPT_RFC3925)) && + if (!(new->flags & (DHOPT_ENCAPSULATE | DHOPT_VENDOR | DHOPT_RFC3925)) && new->opt == OPTION_SIP_SERVER) { *(op++) = 1; /* RFC 3361 "enc byte" */ new->flags &= ~DHOPT_ADDR; } - while (addrs--) + while (addrs--) { cp = comma; comma = split(cp); slash = split_chr(cp, '/'); if (!inet_pton(AF_INET, cp, &in)) - ret_err(_("bad IPv4 address")); + goto_err(_("bad IPv4 address")); if (!slash) { memcpy(op, &in, INADDRSZ); @@ -1275,7 +1393,7 @@ static int parse_dhcp_opt(char *errstr, char *arg, int flags) if (netsize > 24) *op++ = *p++; new->flags &= ~DHOPT_ADDR; /* cannot re-write descriptor format */ - } + } } new->len = op - new->val; } @@ -1284,25 +1402,25 @@ static int parse_dhcp_opt(char *errstr, char *arg, int flags) unsigned char *op; new->val = op = opt_malloc(16 * addrs); new->flags |= DHOPT_ADDR6; - while (addrs--) + while (addrs--) { cp = comma; comma = split(cp); - + /* check for [1234::7] */ if (*cp == '[') cp++; if (strlen(cp) > 1 && cp[strlen(cp)-1] == ']') cp[strlen(cp)-1] = 0; - + if (inet_pton(AF_INET6, cp, op)) { op += IN6ADDRSZ; continue; } - ret_err(_("bad IPv6 address")); - } + goto_err(_("bad IPv6 address")); + } new->len = op - new->val; } else if (is_string) @@ -1316,10 +1434,10 @@ static int parse_dhcp_opt(char *errstr, char *arg, int flags) unsigned char *p, *m = NULL, *newp; size_t newlen, len = 0; int header_size = (new->opt == OPTION_DOMAIN_SEARCH) ? 0 : 1; - + arg = comma; comma = split(arg); - + while (arg && *arg) { char *in, *dom = NULL; @@ -1328,11 +1446,11 @@ static int parse_dhcp_opt(char *errstr, char *arg, int flags) if (strcmp (arg, ".") != 0) { if (!(dom = canonicalise_opt(arg))) - ret_err(_("bad domain in dhcp-option")); - + goto_err(_("bad domain in dhcp-option")); + domlen = strlen(dom) + 2; } - + newp = opt_malloc(len + domlen + header_size); if (m) { @@ -1342,9 +1460,9 @@ static int parse_dhcp_opt(char *errstr, char *arg, int flags) m = newp; p = m + header_size; q = p + len; - + /* add string on the end in RFC1035 format */ - for (in = dom; in && *in;) + for (in = dom; in && *in;) { unsigned char *cp = q++; int j; @@ -1356,26 +1474,26 @@ static int parse_dhcp_opt(char *errstr, char *arg, int flags) } *q++ = 0; free(dom); - + /* Now tail-compress using earlier names. */ newlen = q - p; for (tail = p + len; *tail; tail += (*tail) + 1) for (r = p; r - p < (int)len; r += (*r) + 1) if (strcmp((char *)r, (char *)tail) == 0) { - PUTSHORT((r - p) | 0xc000, tail); + PUTSHORT((r - p) | 0xc000, tail); newlen = tail - p; goto end; } end: len = newlen; - + arg = comma; comma = split(arg); } - + /* RFC 3361, enc byte is zero for names */ - if (new->opt == OPTION_SIP_SERVER) + if (new->opt == OPTION_SIP_SERVER && m) m[0] = 0; new->len = (int) len + header_size; new->val = m; @@ -1390,19 +1508,19 @@ static int parse_dhcp_opt(char *errstr, char *arg, int flags) for (i = 0; comma[i]; i++) if (comma[i] == ',') commas++; - - newp = opt_malloc(strlen(comma)+(2*commas)); + + newp = opt_malloc(strlen(comma)+(2*commas)); p = newp; arg = comma; comma = split(arg); - + while (arg && *arg) { u16 len = strlen(arg); unhide_metas(arg); PUTSHORT(len, p); memcpy(p, arg, len); - p += len; + p += len; arg = comma; comma = split(arg); @@ -1413,35 +1531,42 @@ static int parse_dhcp_opt(char *errstr, char *arg, int flags) } else if (comma && (opt_len & OT_RFC1035_NAME)) { - unsigned char *p = NULL, *newp, *end; + unsigned char *p = NULL, *q, *newp, *end; int len = 0; + int header_size = (is6 && new->opt == OPTION6_NTP_SERVER) ? 4 : 0; arg = comma; comma = split(arg); - + while (arg && *arg) { char *dom = canonicalise_opt(arg); if (!dom) - ret_err(_("bad domain in dhcp-option")); - - newp = opt_malloc(len + strlen(dom) + 2); - + goto_err(_("bad domain in dhcp-option")); + + newp = opt_malloc(len + header_size + strlen(dom) + 2); + if (p) { memcpy(newp, p, len); free(p); } - + p = newp; - end = do_rfc1035_name(p + len, dom, NULL); + q = p + len; + end = do_rfc1035_name(q + header_size, dom, NULL); *end++ = 0; + if (is6 && new->opt == OPTION6_NTP_SERVER) + { + PUTSHORT(NTP_SUBOPTION_SRV_FQDN, q); + PUTSHORT(end - q - 2, q); + } len = end - p; free(dom); arg = comma; comma = split(arg); } - + new->val = p; new->len = len; } @@ -1456,19 +1581,19 @@ static int parse_dhcp_opt(char *errstr, char *arg, int flags) } } - if (!is6 && - ((new->len > 255) || + if (!is6 && + ((new->len > 255) || (new->len > 253 && (new->flags & (DHOPT_VENDOR | DHOPT_ENCAPSULATE))) || (new->len > 250 && (new->flags & DHOPT_RFC3925)))) - ret_err(_("dhcp-option too long")); - + goto_err(_("dhcp-option too long")); + if (flags == DHOPT_MATCH) { if ((new->flags & (DHOPT_ENCAPSULATE | DHOPT_VENDOR)) || !new->netid || new->netid->next) - ret_err(_("illegal dhcp-match")); - + goto_err(_("illegal dhcp-match")); + if (is6) { new->next = daemon->dhcp_match6; @@ -1490,41 +1615,48 @@ static int parse_dhcp_opt(char *errstr, char *arg, int flags) new->next = daemon->dhcp_opts; daemon->dhcp_opts = new; } - + return 1; +on_error: + dhcp_opt_free(new); + return 0; } #endif void set_option_bool(unsigned int opt) { - if (opt < 32) - daemon->options |= 1u << opt; - else - daemon->options2 |= 1u << (opt - 32); + option_var(opt) |= option_val(opt); } void reset_option_bool(unsigned int opt) { - if (opt < 32) - daemon->options &= ~(1u << opt); - else - daemon->options2 &= ~(1u << (opt - 32)); + option_var(opt) &= ~(option_val(opt)); } -static int one_opt(int option, char *arg, char *errstr, char *gen_err, int command_line, int servers_only) +static void server_list_free(struct server *list) { + while (list) + { + struct server *tmp = list; + list = list->next; + free(tmp); + } +} + +static int one_opt(int option, char *arg, char *errstr, char *gen_err, int command_line, int servers_only) +{ int i; char *comma; if (option == '?') ret_err(gen_err); - + for (i=0; usage[i].opt != 0; i++) if (usage[i].opt == option) { int rept = usage[i].rept; - + if (command_line) { /* command line */ @@ -1542,17 +1674,17 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma usage[i].rept = ARG_USED_FILE; } - if (rept != ARG_DUP && rept != ARG_ONE && rept != ARG_USED_CL) + if (rept != ARG_DUP && rept != ARG_ONE && rept != ARG_USED_CL) { set_option_bool(rept); return 1; } - + break; } - + switch (option) - { + { case 'C': /* --conf-file */ { char *file = opt_string_alloc(arg); @@ -1564,21 +1696,21 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma break; } - case '7': /* --conf-dir */ + case '7': /* --conf-dir */ { DIR *dir_stream; struct dirent *ent; char *directory, *path; struct list { - char *suffix; + char *name; struct list *next; - } *ignore_suffix = NULL, *match_suffix = NULL, *li; - + } *ignore_suffix = NULL, *match_suffix = NULL, *files = NULL, *li; + comma = split(arg); if (!(directory = opt_string_alloc(arg))) break; - - for (arg = comma; arg; arg = comma) + + for (arg = comma; arg; arg = comma) { comma = split(arg); if (strlen(arg) != 0) @@ -1594,7 +1726,7 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma li->next = match_suffix; match_suffix = li; /* Have to copy: buffer is overwritten */ - li->suffix = opt_string_alloc(arg+1); + li->name = opt_string_alloc(arg+1); } } else @@ -1602,19 +1734,19 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma li->next = ignore_suffix; ignore_suffix = li; /* Have to copy: buffer is overwritten */ - li->suffix = opt_string_alloc(arg); + li->name = opt_string_alloc(arg); } } } - + if (!(dir_stream = opendir(directory))) die(_("cannot access directory %s: %s"), directory, EC_FILE); - + while ((ent = readdir(dir_stream))) { size_t len = strlen(ent->d_name); struct stat buf; - + /* ignore emacs backups and dotfiles */ if (len == 0 || ent->d_name[len - 1] == '~' || @@ -1627,26 +1759,26 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma for (li = match_suffix; li; li = li->next) { /* check for required suffices */ - size_t ls = strlen(li->suffix); + size_t ls = strlen(li->name); if (len > ls && - strcmp(li->suffix, &ent->d_name[len - ls]) == 0) + strcmp(li->name, &ent->d_name[len - ls]) == 0) break; } if (!li) continue; } - + for (li = ignore_suffix; li; li = li->next) { /* check for proscribed suffices */ - size_t ls = strlen(li->suffix); + size_t ls = strlen(li->name); if (len > ls && - strcmp(li->suffix, &ent->d_name[len - ls]) == 0) + strcmp(li->name, &ent->d_name[len - ls]) == 0) break; } if (li) continue; - + path = opt_malloc(strlen(directory) + len + 2); strcpy(path, directory); strcat(path, "/"); @@ -1655,28 +1787,47 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma /* files must be readable */ if (stat(path, &buf) == -1) die(_("cannot access %s: %s"), path, EC_FILE); - + /* only reg files allowed. */ if (S_ISREG(buf.st_mode)) - one_file(path, 0); + { + /* sort files into order. */ + struct list **up, *new = opt_malloc(sizeof(struct list)); + new->name = path; + + for (up = &files, li = files; li; up = &li->next, li = li->next) + if (strcmp(li->name, path) >=0) + break; + + new->next = li; + *up = new; + } - free(path); } + for (li = files; li; li = li->next) + one_file(li->name, 0); + closedir(dir_stream); free(directory); for(; ignore_suffix; ignore_suffix = li) { li = ignore_suffix->next; - free(ignore_suffix->suffix); + free(ignore_suffix->name); free(ignore_suffix); } for(; match_suffix; match_suffix = li) { li = match_suffix->next; - free(match_suffix->suffix); + free(match_suffix->name); free(match_suffix); } + for(; files; files = li) + { + li = files->next; + free(files->name); + free(files); + } break; } @@ -1693,14 +1844,14 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma /* has subnet+len */ err = parse_mysockaddr(arg, &new->addr); if (err) - ret_err(err); + ret_err_free(err, new); if (!atoi_check(end, &new->mask)) - ret_err(gen_err); + ret_err_free(gen_err, new); new->addr_used = 1; - } + } else if (!atoi_check(arg, &new->mask)) - ret_err(gen_err); - + ret_err_free(gen_err, new); + daemon->add_subnet4 = new; if (comma) @@ -1711,17 +1862,17 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma /* has subnet+len */ err = parse_mysockaddr(comma, &new->addr); if (err) - ret_err(err); + ret_err_free(err, new); if (!atoi_check(end, &new->mask)) - ret_err(gen_err); + ret_err_free(gen_err, new); new->addr_used = 1; } else { if (!atoi_check(comma, &new->mask)) - ret_err(gen_err); + ret_err_free(gen_err, new); } - + daemon->add_subnet6 = new; } } @@ -1735,19 +1886,27 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma daemon->dbus_name = DNSMASQ_SERVICE; break; + case LOPT_UBUS: /* --enable-ubus */ + set_option_bool(OPT_UBUS); + if (arg) + daemon->ubus_name = opt_string_alloc(arg); + else + daemon->ubus_name = DNSMASQ_UBUS_NAME; + break; + case '8': /* --log-facility */ /* may be a filename */ if (strchr(arg, '/') || strcmp (arg, "-") == 0) daemon->log_file = opt_string_alloc(arg); else - { + { #ifdef __ANDROID__ ret_err(_("setting log facility is not possible under Android")); #else for (i = 0; facilitynames[i].c_name; i++) if (hostname_isequal((char *)facilitynames[i].c_name, arg)) break; - + if (facilitynames[i].c_name) daemon->log_fac = facilitynames[i].c_val; else @@ -1764,7 +1923,7 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma { char *name = opt_string_alloc(arg); struct resolvc *new, *list = daemon->resolv_files; - + if (list && list->is_default) { /* replace default resolv file - possibly with nothing */ @@ -1793,7 +1952,7 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma case LOPT_SERVERS_FILE: daemon->servers_file = opt_string_alloc(arg); break; - + case 'm': /* --mx-host */ { int pref = 1; @@ -1806,11 +1965,11 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma if ((prefstr = split(comma)) && !atoi_check16(prefstr, &pref)) ret_err(_("bad MX preference")); } - - if (!(name = canonicalise_opt(arg)) || + + if (!(name = canonicalise_opt(arg)) || (comma && !(target = canonicalise_opt(comma)))) ret_err(_("bad MX name")); - + new = opt_malloc(sizeof(struct mx_srv_record)); new->next = daemon->mxnames; daemon->mxnames = new; @@ -1820,7 +1979,7 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma new->weight = pref; break; } - + case 't': /* --mx-target */ if (!(daemon->mxtarget = canonicalise_opt(arg))) ret_err(_("bad MX target")); @@ -1833,18 +1992,16 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma case LOPT_DUMPMASK: /* --dumpmask */ daemon->dump_mask = strtol(arg, NULL, 0); break; - -#ifdef HAVE_DHCP + +#ifdef HAVE_DHCP case 'l': /* --dhcp-leasefile */ daemon->lease_file = opt_string_alloc(arg); break; - + /* Sorry about the gross pre-processor abuse */ case '6': /* --dhcp-script */ case LOPT_LUASCRIPT: /* --dhcp-luascript */ -# if defined(NO_FORK) - ret_err(_("cannot run scripts under uClinux")); -# elif !defined(HAVE_SCRIPT) +# if !defined(HAVE_SCRIPT) ret_err(_("recompile with HAVE_SCRIPT defined to enable lease-change scripts")); # else if (option == LOPT_LUASCRIPT) @@ -1885,11 +2042,11 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma { new->next = daemon->dhcp_opts_file; daemon->dhcp_opts_file = new; - } - else + } + else { new->next = daemon->dynamic_dirs; - daemon->dynamic_dirs = new; + daemon->dynamic_dirs = new; if (option == LOPT_DHCP_INOTIFY) new->flags |= AH_DHCP_HST; else if (option == LOPT_DHOPT_INOTIFY) @@ -1897,15 +2054,15 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma else if (option == LOPT_HOST_INOTIFY) new->flags |= AH_HOSTS; } - + break; } - + case LOPT_AUTHSERV: /* --auth-server */ comma = split(arg); - + daemon->authserver = opt_string_alloc(arg); - + while ((arg = comma)) { struct iname *new = opt_malloc(sizeof(struct iname)); @@ -1914,10 +2071,8 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma unhide_metas(arg); if (inet_pton(AF_INET, arg, &new->addr.in.sin_addr) > 0) new->addr.sa.sa_family = AF_INET; -#ifdef HAVE_IPV6 else if (inet_pton(AF_INET6, arg, &new->addr.in6.sin6_addr) > 0) new->addr.sa.sa_family = AF_INET6; -#endif else { char *fam = split_chr(arg, '/'); @@ -1927,18 +2082,19 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma { if (strcmp(fam, "4") == 0) new->addr.sa.sa_family = AF_INET; -#ifdef HAVE_IPV6 else if (strcmp(fam, "6") == 0) new->addr.sa.sa_family = AF_INET6; -#endif else - ret_err(gen_err); - } + { + free(new->name); + ret_err_free(gen_err, new); + } + } } new->next = daemon->authinterface; daemon->authinterface = new; }; - + break; case LOPT_AUTHSFS: /* --auth-sec-servers */ @@ -1955,13 +2111,13 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma } while (arg); break; } - + case LOPT_AUTHZONE: /* --auth-zone */ { struct auth_zone *new; - + comma = split(arg); - + new = opt_malloc(sizeof(struct auth_zone)); new->domain = opt_string_alloc(arg); new->subnet = NULL; @@ -1976,35 +2132,33 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma int is_exclude = 0; char *prefix; struct addrlist *subnet = NULL; - struct all_addr addr; + union all_addr addr; comma = split(arg); prefix = split_chr(arg, '/'); - + if (prefix && !atoi_check(prefix, &prefixlen)) ret_err(gen_err); - + if (strstr(arg, "exclude:") == arg) { is_exclude = 1; arg = arg+8; } - if (inet_pton(AF_INET, arg, &addr.addr.addr4)) + if (inet_pton(AF_INET, arg, &addr.addr4)) { subnet = opt_malloc(sizeof(struct addrlist)); subnet->prefixlen = (prefixlen == 0) ? 24 : prefixlen; subnet->flags = ADDRLIST_LITERAL; } -#ifdef HAVE_IPV6 - else if (inet_pton(AF_INET6, arg, &addr.addr.addr6)) + else if (inet_pton(AF_INET6, arg, &addr.addr6)) { subnet = opt_malloc(sizeof(struct addrlist)); subnet->prefixlen = (prefixlen == 0) ? 64 : prefixlen; subnet->flags = ADDRLIST_LITERAL | ADDRLIST_IPV6; } -#endif - else + else { struct auth_name_list *name = opt_malloc(sizeof(struct auth_name_list)); name->name = opt_string_alloc(arg); @@ -2015,15 +2169,13 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma { if (prefixlen == 4) name->flags &= ~AUTH6; -#ifdef HAVE_IPV6 else if (prefixlen == 6) name->flags &= ~AUTH4; -#endif else ret_err(gen_err); } } - + if (subnet) { subnet->addr = addr; @@ -2042,7 +2194,7 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma } break; } - + case LOPT_AUTHSOA: /* --auth-soa */ comma = split(arg); daemon->soa_sn = (u32)atoi(arg); @@ -2059,12 +2211,12 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma if (comma) { arg = comma; - comma = split(arg); + comma = split(arg); daemon->soa_refresh = (u32)atoi(arg); if (comma) { arg = comma; - comma = split(arg); + comma = split(arg); daemon->soa_retry = (u32)atoi(arg); if (comma) daemon->soa_expiry = (u32)atoi(comma); @@ -2090,10 +2242,10 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma { struct cond_domain *new = opt_malloc(sizeof(struct cond_domain)); char *netpart; - + new->prefix = NULL; new->indexed = 0; - + unhide_metas(comma); if ((netpart = split_chr(comma, '/'))) { @@ -2101,11 +2253,11 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma arg = split(netpart); if (!atoi_check(netpart, &msize)) - ret_err(gen_err); + ret_err_free(gen_err, new); else if (inet_pton(AF_INET, comma, &new->start)) { int mask = (1 << (32 - msize)) - 1; - new->is6 = 0; + new->is6 = 0; new->start.s_addr = ntohl(htonl(new->start.s_addr) & ~mask); new->end.s_addr = new->start.s_addr | htonl(mask); if (arg) @@ -2114,18 +2266,18 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma { if (!(new->prefix = canonicalise_opt(arg)) || strlen(new->prefix) > MAXLABEL - INET_ADDRSTRLEN) - ret_err(_("bad prefix")); + ret_err_free(_("bad prefix"), new); } else if (strcmp(arg, "local") != 0 || (msize != 8 && msize != 16 && msize != 24)) - ret_err(gen_err); + ret_err_free(gen_err, new); else { /* generate the equivalent of local=/xxx.yyy.zzz.in-addr.arpa/ */ struct server *serv = add_rev4(new->start, msize); if (!serv) - ret_err(_("bad prefix")); + ret_err_free(_("bad prefix"), new); serv->flags |= SERV_NO_ADDR; @@ -2139,40 +2291,39 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma } } } -#ifdef HAVE_IPV6 else if (inet_pton(AF_INET6, comma, &new->start6)) { u64 mask = (1LLU << (128 - msize)) - 1LLU; u64 addrpart = addr6part(&new->start6); new->is6 = 1; - + /* prefix==64 overflows the mask calculation above */ if (msize == 64) mask = (u64)-1LL; - + new->end6 = new->start6; setaddr6part(&new->start6, addrpart & ~mask); setaddr6part(&new->end6, addrpart | mask); - + if (msize < 64) - ret_err(gen_err); + ret_err_free(gen_err, new); else if (arg) { if (option != 's') { if (!(new->prefix = canonicalise_opt(arg)) || strlen(new->prefix) > MAXLABEL - INET6_ADDRSTRLEN) - ret_err(_("bad prefix")); - } + ret_err_free(_("bad prefix"), new); + } else if (strcmp(arg, "local") != 0 || ((msize & 4) != 0)) - ret_err(gen_err); - else + ret_err_free(gen_err, new); + else { /* generate the equivalent of local=/xxx.yyy.zzz.ip6.arpa/ */ struct server *serv = add_rev6(&new->start6, msize); serv->flags |= SERV_NO_ADDR; - + /* local=// */ serv = opt_malloc(sizeof(struct server)); memset(serv, 0, sizeof(struct server)); @@ -2183,9 +2334,8 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma } } } -#endif else - ret_err(gen_err); + ret_err_free(gen_err, new); } else { @@ -2199,26 +2349,24 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma if (!arg) new->end.s_addr = new->start.s_addr; else if (!inet_pton(AF_INET, arg, &new->end)) - ret_err(gen_err); + ret_err_free(gen_err, new); } -#ifdef HAVE_IPV6 else if (inet_pton(AF_INET6, comma, &new->start6)) { new->is6 = 1; if (!arg) memcpy(&new->end6, &new->start6, IN6ADDRSZ); else if (!inet_pton(AF_INET6, arg, &new->end6)) - ret_err(gen_err); + ret_err_free(gen_err, new); } -#endif - else - ret_err(gen_err); + else + ret_err_free(gen_err, new); if (option != 's' && prefstr) { if (!(new->prefix = canonicalise_opt(prefstr)) || strlen(new->prefix) > MAXLABEL - INET_ADDRSTRLEN) - ret_err(_("bad prefix")); + ret_err_free(_("bad prefix"), new); } } @@ -2244,12 +2392,12 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma } else if (option == 's') daemon->domain_suffix = d; - else + else ret_err(gen_err); } } break; - + case LOPT_CPE_ID: /* --add-dns-client */ if (arg) daemon->dns_client_id = opt_string_alloc(arg); @@ -2273,7 +2421,7 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma case 'u': /* --user */ daemon->username = opt_string_alloc(arg); break; - + case 'g': /* --group */ daemon->groupname = opt_string_alloc(arg); daemon->group_set = 1; @@ -2284,7 +2432,7 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma daemon->scriptuser = opt_string_alloc(arg); break; #endif - + case 'i': /* --interface */ do { struct iname *new = opt_malloc(sizeof(struct iname)); @@ -2298,7 +2446,7 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma arg = comma; } while (arg); break; - + case LOPT_TFTP: /* --enable-tftp */ set_option_bool(OPT_TFTP); if (!arg) @@ -2329,7 +2477,7 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma arg = comma; } while (arg); break; - + case 'B': /* --bogus-nxdomain */ case LOPT_IGNORE_ADDR: /* --ignore-address */ { @@ -2352,9 +2500,9 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma } else ret_err(gen_err); /* error */ - break; + break; } - + case 'a': /* --listen-address */ case LOPT_AUTHPEER: /* --auth-peer */ do { @@ -2369,7 +2517,6 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma new->addr.in.sin_len = sizeof(new->addr.in); #endif } -#ifdef HAVE_IPV6 else if (arg && inet_pton(AF_INET6, arg, &new->addr.in6.sin6_addr) > 0) { new->addr.sa.sa_family = AF_INET6; @@ -2380,9 +2527,8 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma new->addr.in6.sin6_len = sizeof(new->addr.in6); #endif } -#endif else - ret_err(gen_err); + ret_err_free(gen_err, new); new->used = 0; if (option == 'a') @@ -2394,20 +2540,20 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma { new->next = daemon->auth_peers; daemon->auth_peers = new; - } + } arg = comma; } while (arg); break; - + case 'S': /* --server */ case LOPT_LOCAL: /* --local */ case 'A': /* --address */ case LOPT_NO_REBIND: /* --rebind-domain-ok */ { struct server *serv, *newlist = NULL; - + unhide_metas(arg); - + if (arg && (*arg == '/' || option == LOPT_NO_REBIND)) { int rebind = !(*arg == '/'); @@ -2445,19 +2591,22 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma newlist->uid = rand32(); #endif } - + if (servers_only && option == 'S') newlist->flags |= SERV_FROM_FILE; - + if (option == 'A') { newlist->flags |= SERV_LITERAL_ADDRESS; if (!(newlist->flags & SERV_TYPE)) - ret_err(gen_err); + { + server_list_free(newlist); + ret_err(gen_err); + } } else if (option == LOPT_NO_REBIND) newlist->flags |= SERV_NO_REBIND; - + if (!arg || !*arg) { if (!(newlist->flags & SERV_NO_REBIND)) @@ -2470,13 +2619,16 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma { char *err = parse_server(arg, &newlist->addr, &newlist->source_addr, newlist->interface, &newlist->flags); if (err) - ret_err(err); + { + server_list_free(newlist); + ret_err(err); + } } - + serv = newlist; while (serv->next) { - serv->next->flags = serv->flags; + serv->next->flags |= serv->flags & ~(SERV_HAS_DOMAIN | SERV_FOR_NODOTS); serv->next->addr = serv->addr; serv->next->source_addr = serv->source_addr; strcpy(serv->next->interface, serv->interface); @@ -2493,35 +2645,36 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma int size; struct server *serv; struct in_addr addr4; -#ifdef HAVE_IPV6 struct in6_addr addr6; -#endif - + unhide_metas(arg); - if (!arg || !(comma=split(arg)) || !(string = split_chr(arg, '/')) || !atoi_check(string, &size)) + if (!arg) ret_err(gen_err); + + comma=split(arg); + if (!(string = split_chr(arg, '/')) || !atoi_check(string, &size)) + ret_err(gen_err); + if (inet_pton(AF_INET, arg, &addr4)) { serv = add_rev4(addr4, size); if (!serv) ret_err(_("bad prefix")); } -#ifdef HAVE_IPV6 else if (inet_pton(AF_INET6, arg, &addr6)) serv = add_rev6(&addr6, size); -#endif else ret_err(gen_err); - + string = parse_server(comma, &serv->addr, &serv->source_addr, serv->interface, &serv->flags); - + if (string) ret_err(string); - + if (servers_only) serv->flags |= SERV_FROM_FILE; - + break; } @@ -2538,10 +2691,10 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma char **sets, **sets_pos; memset(ipsets, 0, sizeof(struct ipsets)); unhide_metas(arg); - if (arg && *arg == '/') + if (arg && *arg == '/') { arg++; - while ((end = split_chr(arg, '/'))) + while ((end = split_chr(arg, '/'))) { char *domain = NULL; /* elide leading dots - they are implied in the search algorithm */ @@ -2558,24 +2711,24 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma ipsets->domain = domain; arg = end; } - } - else + } + else { ipsets->next = opt_malloc(sizeof(struct ipsets)); ipsets = ipsets->next; memset(ipsets, 0, sizeof(struct ipsets)); ipsets->domain = ""; } - + if (!arg || !*arg) ret_err(gen_err); - - for (size = 2, end = arg; *end; ++end) + + for (size = 2, end = arg; *end; ++end) if (*end == ',') ++size; - + sets = sets_pos = opt_malloc(sizeof(char *) * size); - + do { end = split(arg); *sets_pos++ = opt_string_alloc(arg); @@ -2586,34 +2739,42 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma ipsets->next->sets = sets; ipsets->next = daemon->ipsets; daemon->ipsets = ipsets_head.next; - + break; } #endif - + case 'c': /* --cache-size */ { int size; - + if (!atoi_check(arg, &size)) ret_err(gen_err); else { /* zero is OK, and means no caching. */ - + if (size < 0) size = 0; + /* Note that for very large cache sizes, the malloc() + will overflow. For the size of the cache record + at the time this was noted, the value of "very large" + was 46684428. Limit to an order of magnitude less than + that to be safe from changes to the cache record. */ + if (size > 5000000) + size = 5000000; + daemon->cachesize = size; } break; } - + case 'p': /* --port */ if (!atoi_check16(arg, &daemon->port)) ret_err(gen_err); break; - + case LOPT_MINPORT: /* --min-port */ if (!atoi_check16(arg, &daemon->min_port)) ret_err(gen_err); @@ -2627,8 +2788,8 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma case '0': /* --dns-forward-max */ if (!atoi_check(arg, &daemon->ftabsize)) ret_err(gen_err); - break; - + break; + case 'q': /* --log-queries */ set_option_bool(OPT_LOG); if (arg && strcmp(arg, "extra") == 0) @@ -2641,17 +2802,17 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma ret_err(gen_err); else if (daemon->max_logs > 100) daemon->max_logs = 100; - break; + break; case 'P': /* --edns-packet-max */ { int i; if (!atoi_check(arg, &i)) ret_err(gen_err); - daemon->edns_pktsz = (unsigned short)i; + daemon->edns_pktsz = (unsigned short)i; break; } - + case 'Q': /* --query-port */ if (!atoi_check16(arg, &daemon->query_port)) ret_err(gen_err); @@ -2660,7 +2821,7 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma if (daemon->query_port == 0) daemon->osport = 1; break; - + case 'T': /* --local-ttl */ case LOPT_NEGTTL: /* --neg-ttl */ case LOPT_MAXTTL: /* --max-ttl */ @@ -2695,19 +2856,19 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma daemon->local_ttl = (unsigned long)ttl; break; } - + #ifdef HAVE_DHCP case 'X': /* --dhcp-lease-max */ if (!atoi_check(arg, &daemon->dhcp_max)) ret_err(gen_err); break; #endif - + #ifdef HAVE_TFTP case LOPT_TFTP_MAX: /* --tftp-max */ if (!atoi_check(arg, &daemon->tftp_max)) ret_err(gen_err); - break; + break; case LOPT_TFTP_MTU: /* --tftp-mtu */ if (!atoi_check(arg, &daemon->tftp_mtu)) @@ -2729,18 +2890,18 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma break; case LOPT_TFTPPORTS: /* --tftp-port-range */ - if (!(comma = split(arg)) || + if (!(comma = split(arg)) || !atoi_check16(arg, &daemon->start_tftp_port) || !atoi_check16(comma, &daemon->end_tftp_port)) ret_err(_("bad port range")); - + if (daemon->start_tftp_port > daemon->end_tftp_port) { int tmp = daemon->start_tftp_port; daemon->start_tftp_port = daemon->end_tftp_port; daemon->end_tftp_port = tmp; - } - + } + break; case LOPT_APREF: /* --tftp-unique-root */ @@ -2752,7 +2913,7 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma ret_err(gen_err); break; #endif - + case LOPT_BRIDGE: /* --bridge-interface */ { struct dhcp_bridge *new; @@ -2772,59 +2933,95 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma new->next = daemon->bridges; daemon->bridges = new; } - + do { arg = comma; comma = split(arg); if (strlen(arg) != 0 && strlen(arg) <= IF_NAMESIZE - 1) { - struct dhcp_bridge *b = opt_malloc(sizeof(struct dhcp_bridge)); + struct dhcp_bridge *b = opt_malloc(sizeof(struct dhcp_bridge)); b->next = new->alias; new->alias = b; strcpy(b->iface, arg); } } while (comma); - + break; } #ifdef HAVE_DHCP + case LOPT_SHARED_NET: /* --shared-network */ + { + struct shared_network *new = opt_malloc(sizeof(struct shared_network)); + +#ifdef HAVE_DHCP6 + new->shared_addr.s_addr = 0; +#endif + new->if_index = 0; + + if (!(comma = split(arg))) + { + snerr: + free(new); + ret_err(_("bad shared-network")); + } + + if (inet_pton(AF_INET, comma, &new->shared_addr)) + { + if (!inet_pton(AF_INET, arg, &new->match_addr) && + !(new->if_index = if_nametoindex(arg))) + goto snerr; + } +#ifdef HAVE_DHCP6 + else if (inet_pton(AF_INET6, comma, &new->shared_addr6)) + { + if (!inet_pton(AF_INET6, arg, &new->match_addr6) && + !(new->if_index = if_nametoindex(arg))) + goto snerr; + } +#endif + else + goto snerr; + + new->next = daemon->shared_networks; + daemon->shared_networks = new; + break; + } + case 'F': /* --dhcp-range */ { int k, leasepos = 2; char *cp, *a[8] = { NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL }; struct dhcp_context *new = opt_malloc(sizeof(struct dhcp_context)); - + memset (new, 0, sizeof(*new)); new->lease_time = DEFLEASE; - + while(1) { for (cp = arg; *cp; cp++) - if (!(*cp == ' ' || *cp == '.' || *cp == ':' || + if (!(*cp == ' ' || *cp == '.' || *cp == ':' || (*cp >= 'a' && *cp <= 'f') || (*cp >= 'A' && *cp <= 'F') || (*cp >='0' && *cp <= '9'))) break; - + if (*cp != ',' && (comma = split(arg))) { if (is_tag_prefix(arg)) { - struct dhcp_netid *tt = opt_malloc(sizeof (struct dhcp_netid)); - tt->net = opt_string_alloc(arg+4); - tt->next = new->filter; /* ignore empty tag */ - if (tt->net) - new->filter = tt; + if (arg[4]) + new->filter = dhcp_netid_create(arg+4, new->filter); } else { if (new->netid.net) - ret_err(_("only one tag allowed")); - else if (strstr(arg, "set:") == arg) - new->netid.net = opt_string_alloc(arg+4); + { + dhcp_context_free(new); + ret_err(_("only one tag allowed")); + } else - new->netid.net = opt_string_alloc(arg); + new->netid.net = opt_string_alloc(set_prefix(arg)); } arg = comma; } @@ -2834,14 +3031,17 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma break; } } - + for (k = 1; k < 8; k++) if (!(a[k] = split(a[k-1]))) break; - + if (k < 2) - ret_err(_("bad dhcp-range")); - + { + dhcp_context_free(new); + ret_err(_("bad dhcp-range")); + } + if (inet_pton(AF_INET, a[0], &new->start)) { new->next = daemon->dhcp; @@ -2852,25 +3052,31 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma else if (strcmp(a[1], "proxy") == 0) new->flags |= CONTEXT_PROXY; else if (!inet_pton(AF_INET, a[1], &new->end)) - ret_err(_("bad dhcp-range")); - + { + dhcp_context_free(new); + ret_err(_("bad dhcp-range")); + } + if (ntohl(new->start.s_addr) > ntohl(new->end.s_addr)) { struct in_addr tmp = new->start; new->start = new->end; new->end = tmp; } - - if (k >= 3 && strchr(a[2], '.') && + + if (k >= 3 && strchr(a[2], '.') && (inet_pton(AF_INET, a[2], &new->netmask) > 0)) { new->flags |= CONTEXT_NETMASK; leasepos = 3; if (!is_same_net(new->start, new->end, new->netmask)) - ret_err(_("inconsistent DHCP range")); - - - if (k >= 4 && strchr(a[3], '.') && + { + dhcp_context_free(new); + ret_err(_("inconsistent DHCP range")); + } + + + if (k >= 4 && strchr(a[3], '.') && (inet_pton(AF_INET, a[3], &new->broadcast) > 0)) { new->flags |= CONTEXT_BRDCAST; @@ -2881,7 +3087,9 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma #ifdef HAVE_DHCP6 else if (inet_pton(AF_INET6, a[0], &new->start6)) { - new->flags |= CONTEXT_V6; + const char *err = NULL; + + new->flags |= CONTEXT_V6; new->prefix = 64; /* default */ new->end6 = new->start6; new->next = daemon->dhcp6; @@ -2902,16 +3110,16 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma else if (strcmp(a[leasepos], "off-link") == 0) new->flags |= CONTEXT_RA_OFF_LINK; else if (leasepos == 1 && inet_pton(AF_INET6, a[leasepos], &new->end6)) - new->flags |= CONTEXT_DHCP; + new->flags |= CONTEXT_DHCP; else if (strstr(a[leasepos], "constructor:") == a[leasepos]) { new->template_interface = opt_string_alloc(a[leasepos] + 12); new->flags |= CONTEXT_TEMPLATE; } - else + else break; } - + /* bare integer < 128 is prefix value */ if (leasepos < k) { @@ -2925,33 +3133,41 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma leasepos++; } } - - if (new->prefix != 64) + + if (new->prefix > 64) { if (new->flags & CONTEXT_RA) - ret_err(_("prefix length must be exactly 64 for RA subnets")); + err=(_("prefix length must be exactly 64 for RA subnets")); else if (new->flags & CONTEXT_TEMPLATE) - ret_err(_("prefix length must be exactly 64 for subnet constructors")); + err=(_("prefix length must be exactly 64 for subnet constructors")); } + else if (new->prefix < 64) + err=(_("prefix length must be at least 64")); + + if (!err && !is_same_net6(&new->start6, &new->end6, new->prefix)) + err=(_("inconsistent DHCPv6 range")); - if (new->prefix < 64) - ret_err(_("prefix length must be at least 64")); - - if (!is_same_net6(&new->start6, &new->end6, new->prefix)) - ret_err(_("inconsistent DHCPv6 range")); + if (err) + { + dhcp_context_free(new); + ret_err(err); + } /* dhcp-range=:: enables DHCP stateless on any interface */ if (IN6_IS_ADDR_UNSPECIFIED(&new->start6) && !(new->flags & CONTEXT_TEMPLATE)) new->prefix = 0; - + if (new->flags & CONTEXT_TEMPLATE) { struct in6_addr zero; memset(&zero, 0, sizeof(zero)); if (!is_same_net6(&zero, &new->start6, new->prefix)) - ret_err(_("prefix must be zero with \"constructor:\" argument")); + { + dhcp_context_free(new); + ret_err(_("prefix must be zero with \"constructor:\" argument")); + } } - + if (addr6part(&new->start6) > addr6part(&new->end6)) { struct in6_addr tmp = new->start6; @@ -2961,13 +3177,19 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma } #endif else - ret_err(_("bad dhcp-range")); - + { + dhcp_context_free(new); + ret_err(_("bad dhcp-range")); + } + if (leasepos < k) { if (leasepos != k-1) - ret_err(_("bad dhcp-range")); - + { + dhcp_context_free(new); + ret_err(_("bad dhcp-range")); + } + if (strcmp(a[leasepos], "infinite") == 0) new->lease_time = 0xffffffff; else if (strcmp(a[leasepos], "deprecated") == 0) @@ -2999,14 +3221,14 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma case 'S': a[leasepos][strlen(a[leasepos]) - 1] = 0; } - + for (cp = a[leasepos]; *cp; cp++) if (!(*cp >= '0' && *cp <= '9')) break; if (*cp || (leasepos+1 < k)) - ret_err(_("bad dhcp-range")); - + ret_err_free(_("bad dhcp-range"), new); + new->lease_time = atoi(a[leasepos]) * fac; /* Leases of a minute or less confuse some clients, notably Apple's */ @@ -3021,204 +3243,240 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma case LOPT_BANK: case 'G': /* --dhcp-host */ { - int j, k = 0; - char *a[7] = { NULL, NULL, NULL, NULL, NULL, NULL, NULL }; struct dhcp_config *new; struct in_addr in; - + new = opt_malloc(sizeof(struct dhcp_config)); - + new->next = daemon->dhcp_conf; new->flags = (option == LOPT_BANK) ? CONFIG_BANK : 0; new->hwaddr = NULL; new->netid = NULL; - - if ((a[0] = arg)) - for (k = 1; k < 7; k++) - if (!(a[k] = split(a[k-1]))) - break; - - for (j = 0; j < k; j++) - if (strchr(a[j], ':')) /* ethernet address, netid or binary CLID */ - { - char *arg = a[j]; - - if ((arg[0] == 'i' || arg[0] == 'I') && - (arg[1] == 'd' || arg[1] == 'D') && - arg[2] == ':') - { - if (arg[3] == '*') - new->flags |= CONFIG_NOCLID; - else - { - int len; - arg += 3; /* dump id: */ - if (strchr(arg, ':')) - len = parse_hex(arg, (unsigned char *)arg, -1, NULL, NULL); - else - { - unhide_metas(arg); - len = (int) strlen(arg); - } - - if (len == -1) - ret_err(_("bad hex constant")); - else if ((new->clid = opt_malloc(len))) - { - new->flags |= CONFIG_CLID; - new->clid_len = len; - memcpy(new->clid, arg, len); - } - } - } - /* dhcp-host has strange backwards-compat needs. */ - else if (strstr(arg, "net:") == arg || strstr(arg, "set:") == arg) - { - struct dhcp_netid *newtag = opt_malloc(sizeof(struct dhcp_netid)); - struct dhcp_netid_list *newlist = opt_malloc(sizeof(struct dhcp_netid_list)); - newtag->net = opt_malloc(strlen(arg + 4) + 1); - newlist->next = new->netid; - new->netid = newlist; - newlist->list = newtag; - strcpy(newtag->net, arg+4); - unhide_metas(newtag->net); - } - else if (strstr(arg, "tag:") == arg) - ret_err(_("cannot match tags in --dhcp-host")); + new->filter = NULL; + new->clid = NULL; #ifdef HAVE_DHCP6 - else if (arg[0] == '[' && arg[strlen(arg)-1] == ']') - { - arg[strlen(arg)-1] = 0; - arg++; - - if (!inet_pton(AF_INET6, arg, &new->addr6)) - ret_err(_("bad IPv6 address")); - - for (i= 0; i < 8; i++) - if (new->addr6.s6_addr[i] != 0) - break; - - /* set WILDCARD if network part all zeros */ - if (i == 8) - new->flags |= CONFIG_WILDCARD; - - new->flags |= CONFIG_ADDR6; - } + new->addr6 = NULL; #endif - else - { - struct hwaddr_config *newhw = opt_malloc(sizeof(struct hwaddr_config)); - if ((newhw->hwaddr_len = parse_hex(a[j], newhw->hwaddr, DHCP_CHADDR_MAX, - &newhw->wildcard_mask, &newhw->hwaddr_type)) == -1) - ret_err(_("bad hex constant")); - else - { - - newhw->next = new->hwaddr; - new->hwaddr = newhw; - } - } - } - else if (strchr(a[j], '.') && (inet_pton(AF_INET, a[j], &in) > 0)) - { - struct dhcp_config *configs; - new->addr = in; - new->flags |= CONFIG_ADDR; - - /* If the same IP appears in more than one host config, then DISCOVER - for one of the hosts will get the address, but REQUEST will be NAKed, - since the address is reserved by the other one -> protocol loop. */ - for (configs = daemon->dhcp_conf; configs; configs = configs->next) - if ((configs->flags & CONFIG_ADDR) && configs->addr.s_addr == in.s_addr) + while (arg) + { + comma = split(arg); + if (strchr(arg, ':')) /* ethernet address, netid or binary CLID */ + { + if ((arg[0] == 'i' || arg[0] == 'I') && + (arg[1] == 'd' || arg[1] == 'D') && + arg[2] == ':') + { + if (arg[3] == '*') + new->flags |= CONFIG_NOCLID; + else + { + int len; + arg += 3; /* dump id: */ + if (strchr(arg, ':')) + len = parse_hex(arg, (unsigned char *)arg, -1, NULL, NULL); + else + { + unhide_metas(arg); + len = (int) strlen(arg); + } + + if (len == -1) + { + dhcp_config_free(new); + ret_err(_("bad hex constant")); + } + else if ((new->clid = opt_malloc(len))) + { + new->flags |= CONFIG_CLID; + new->clid_len = len; + memcpy(new->clid, arg, len); + } + } + } + /* dhcp-host has strange backwards-compat needs. */ + else if (strstr(arg, "net:") == arg || strstr(arg, "set:") == arg) { - sprintf(errstr, _("duplicate dhcp-host IP address %s"), inet_ntoa(in)); - return 0; + struct dhcp_netid_list *newlist = opt_malloc(sizeof(struct dhcp_netid_list)); + newlist->next = new->netid; + new->netid = newlist; + newlist->list = dhcp_netid_create(arg+4, NULL); } - } - else - { - char *cp, *lastp = NULL, last = 0; - int fac = 1, isdig = 0; + else if (strstr(arg, "tag:") == arg) + new->filter = dhcp_netid_create(arg+4, new->filter); + +#ifdef HAVE_DHCP6 + else if (arg[0] == '[' && arg[strlen(arg)-1] == ']') + { + char *pref; + struct in6_addr in6; + struct addrlist *new_addr; + + arg[strlen(arg)-1] = 0; + arg++; + pref = split_chr(arg, '/'); + + if (!inet_pton(AF_INET6, arg, &in6)) + { + dhcp_config_free(new); + ret_err(_("bad IPv6 address")); + } - if (strlen(a[j]) > 1) - { - lastp = a[j] + strlen(a[j]) - 1; - last = *lastp; - switch (last) + new_addr = opt_malloc(sizeof(struct addrlist)); + new_addr->next = new->addr6; + new_addr->flags = 0; + new_addr->addr.addr6 = in6; + new->addr6 = new_addr; + + if (pref) + { + u64 addrpart = addr6part(&in6); + + if (!atoi_check(pref, &new_addr->prefixlen) || + new_addr->prefixlen > 128 || + ((((u64)1<<(128-new_addr->prefixlen))-1) & addrpart) != 0) + { + dhcp_config_free(new); + ret_err(_("bad IPv6 prefix")); + } + + new_addr->flags |= ADDRLIST_PREFIX; + } + + for (i= 0; i < 8; i++) + if (in6.s6_addr[i] != 0) + break; + + /* set WILDCARD if network part all zeros */ + if (i == 8) + new_addr->flags |= ADDRLIST_WILDCARD; + + new->flags |= CONFIG_ADDR6; + } +#endif + else + { + struct hwaddr_config *newhw = opt_malloc(sizeof(struct hwaddr_config)); + if ((newhw->hwaddr_len = parse_hex(arg, newhw->hwaddr, DHCP_CHADDR_MAX, + &newhw->wildcard_mask, &newhw->hwaddr_type)) == -1) + { + free(newhw); + dhcp_config_free(new); + ret_err(_("bad hex constant")); + } + else + { + newhw->next = new->hwaddr; + new->hwaddr = newhw; + } + } + } + else if (strchr(arg, '.') && (inet_pton(AF_INET, arg, &in) > 0)) + { + struct dhcp_config *configs; + + new->addr = in; + new->flags |= CONFIG_ADDR; + + /* If the same IP appears in more than one host config, then DISCOVER + for one of the hosts will get the address, but REQUEST will be NAKed, + since the address is reserved by the other one -> protocol loop. */ + for (configs = daemon->dhcp_conf; configs; configs = configs->next) + if ((configs->flags & CONFIG_ADDR) && configs->addr.s_addr == in.s_addr) { - case 'w': - case 'W': - fac *= 7; - /* fall through */ - case 'd': - case 'D': - fac *= 24; - /* fall through */ - case 'h': - case 'H': - fac *= 60; - /* fall through */ - case 'm': - case 'M': - fac *= 60; - /* fall through */ - case 's': - case 'S': - *lastp = 0; - } - } - - for (cp = a[j]; *cp; cp++) - if (isdigit((unsigned char)*cp)) - isdig = 1; - else if (*cp != ' ') - break; + sprintf(errstr, _("duplicate dhcp-host IP address %s"), inet_ntoa(in)); + return 0; + } + } + else + { + char *cp, *lastp = NULL, last = 0; + int fac = 1, isdig = 0; + + if (strlen(arg) > 1) + { + lastp = arg + strlen(arg) - 1; + last = *lastp; + switch (last) + { + case 'w': + case 'W': + fac *= 7; + /* fall through */ + case 'd': + case 'D': + fac *= 24; + /* fall through */ + case 'h': + case 'H': + fac *= 60; + /* fall through */ + case 'm': + case 'M': + fac *= 60; + /* fall through */ + case 's': + case 'S': + *lastp = 0; + } + } + + for (cp = arg; *cp; cp++) + if (isdigit((unsigned char)*cp)) + isdig = 1; + else if (*cp != ' ') + break; - if (*cp) - { - if (lastp) - *lastp = last; - if (strcmp(a[j], "infinite") == 0) - { - new->lease_time = 0xffffffff; - new->flags |= CONFIG_TIME; - } - else if (strcmp(a[j], "ignore") == 0) - new->flags |= CONFIG_DISABLE; - else - { - if (!(new->hostname = canonicalise_opt(a[j])) || - !legal_hostname(new->hostname)) - ret_err(_("bad DHCP host name")); + if (*cp) + { + if (lastp) + *lastp = last; + if (strcmp(arg, "infinite") == 0) + { + new->lease_time = 0xffffffff; + new->flags |= CONFIG_TIME; + } + else if (strcmp(arg, "ignore") == 0) + new->flags |= CONFIG_DISABLE; + else + { + if (!(new->hostname = canonicalise_opt(arg)) || + !legal_hostname(new->hostname)) + { + dhcp_config_free(new); + ret_err(_("bad DHCP host name")); + } + + new->flags |= CONFIG_NAME; + new->domain = strip_hostname(new->hostname); + } + } + else if (isdig) + { + new->lease_time = atoi(arg) * fac; + /* Leases of a minute or less confuse + some clients, notably Apple's */ + if (new->lease_time < 120) + new->lease_time = 120; + new->flags |= CONFIG_TIME; + } + } - new->flags |= CONFIG_NAME; - new->domain = strip_hostname(new->hostname); - } - } - else if (isdig) - { - new->lease_time = atoi(a[j]) * fac; - /* Leases of a minute or less confuse - some clients, notably Apple's */ - if (new->lease_time < 120) - new->lease_time = 120; - new->flags |= CONFIG_TIME; - } - } + arg = comma; + } daemon->dhcp_conf = new; break; } - + case LOPT_TAG_IF: /* --tag-if */ { struct tag_if *new = opt_malloc(sizeof(struct tag_if)); - + new->tag = NULL; new->set = NULL; new->next = NULL; - + /* preserve order */ if (!daemon->tag_if) daemon->tag_if = new; @@ -3243,10 +3501,7 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma } else { - struct dhcp_netid *newtag = opt_malloc(sizeof(struct dhcp_netid)); - newtag->net = opt_malloc(len - 3); - strcpy(newtag->net, arg+4); - unhide_metas(newtag->net); + struct dhcp_netid *newtag = dhcp_netid_create(arg+4, NULL); if (strstr(arg, "set:") == arg) { @@ -3260,30 +3515,34 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma newtag->next = new->tag; new->tag = newtag; } - else + else { new->set = NULL; - free(newtag); + dhcp_netid_free(newtag); break; } } - + arg = comma; } if (!new->set) - ret_err(_("bad tag-if")); - + { + dhcp_netid_free(new->tag); + dhcp_netid_list_free(new->set); + ret_err_free(_("bad tag-if"), new); + } + break; } - + case 'O': /* --dhcp-option */ case LOPT_FORCE: /* --dhcp-option-force */ case LOPT_OPTS: case LOPT_MATCH: /* --dhcp-match */ - return parse_dhcp_opt(errstr, arg, - option == LOPT_FORCE ? DHOPT_FORCE : + return parse_dhcp_opt(errstr, arg, + option == LOPT_FORCE ? DHOPT_FORCE : (option == LOPT_MATCH ? DHOPT_MATCH : (option == LOPT_OPTS ? DHOPT_BANK : 0))); @@ -3292,7 +3551,7 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma struct dhcp_match_name *new = opt_malloc(sizeof(struct dhcp_match_name)); struct dhcp_netid *id = opt_malloc(sizeof(struct dhcp_netid)); ssize_t len; - + if (!(comma = split(arg)) || (len = strlen(comma)) == 0) ret_err(gen_err); @@ -3312,23 +3571,16 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma break; } - + case 'M': /* --dhcp-boot */ { - struct dhcp_netid *id = NULL; - while (is_tag_prefix(arg)) - { - struct dhcp_netid *newid = opt_malloc(sizeof(struct dhcp_netid)); - newid->next = id; - id = newid; - comma = split(arg); - newid->net = opt_string_alloc(arg+4); - arg = comma; - }; - + struct dhcp_netid *id = dhcp_tags(&arg); + if (!arg) - ret_err(gen_err); - else + { + ret_err(gen_err); + } + else { char *dhcp_file, *dhcp_sname = NULL, *tftp_sname = NULL; struct in_addr dhcp_next_server; @@ -3350,14 +3602,14 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma * The user may have specified the tftp hostname here. * save it so that it can be resolved/looked up during * actual dhcp_reply(). - */ - + */ + tftp_sname = opt_string_alloc(comma); dhcp_next_server.s_addr = 0; } } } - + new = opt_malloc(sizeof(struct dhcp_boot)); new->file = dhcp_file; new->sname = dhcp_sname; @@ -3367,62 +3619,49 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma new->next = daemon->boot_config; daemon->boot_config = new; } - + break; } case LOPT_REPLY_DELAY: /* --dhcp-reply-delay */ { - struct dhcp_netid *id = NULL; - while (is_tag_prefix(arg)) - { - struct dhcp_netid *newid = opt_malloc(sizeof(struct dhcp_netid)); - newid->next = id; - id = newid; - comma = split(arg); - newid->net = opt_string_alloc(arg+4); - arg = comma; - }; - + struct dhcp_netid *id = dhcp_tags(&arg); + if (!arg) - ret_err(gen_err); + { + ret_err(gen_err); + } else { struct delay_config *new; int delay; if (!atoi_check(arg, &delay)) ret_err(gen_err); - + new = opt_malloc(sizeof(struct delay_config)); new->delay = delay; new->netid = id; new->next = daemon->delay_conf; daemon->delay_conf = new; } - + break; } - + case LOPT_PXE_PROMT: /* --pxe-prompt */ { struct dhcp_opt *new = opt_malloc(sizeof(struct dhcp_opt)); int timeout; - + new->netid = NULL; new->opt = 10; /* PXE_MENU_PROMPT */ - - while (is_tag_prefix(arg)) - { - struct dhcp_netid *nn = opt_malloc(sizeof (struct dhcp_netid)); - comma = split(arg); - nn->next = new->netid; - new->netid = nn; - nn->net = opt_string_alloc(arg+4); - arg = comma; - } - + new->netid = dhcp_tags(&arg); + if (!arg) - ret_err(gen_err); + { + dhcp_opt_free(new); + ret_err(gen_err); + } else { comma = split(arg); @@ -3430,10 +3669,10 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma new->len = strlen(arg) + 1; new->val = opt_malloc(new->len); memcpy(new->val + 1, arg, new->len - 1); - + new->u.vendor_class = (unsigned char *)"PXEClient"; new->flags = DHOPT_VENDOR; - + if (comma && atoi_check(comma, &timeout)) *(new->val) = timeout; else @@ -3443,46 +3682,37 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma daemon->dhcp_opts = new; daemon->enable_pxe = 1; } - + break; } - + case LOPT_PXE_SERV: /* --pxe-service */ { struct pxe_service *new = opt_malloc(sizeof(struct pxe_service)); char *CSA[] = { "x86PC", "PC98", "IA64_EFI", "Alpha", "Arc_x86", "Intel_Lean_Client", "IA32_EFI", "x86-64_EFI", "Xscale_EFI", "BC_EFI", - "ARM32_EFI", "ARM64_EFI", NULL }; + "ARM32_EFI", "ARM64_EFI", NULL }; static int boottype = 32768; - + new->netid = NULL; new->sname = NULL; new->server.s_addr = 0; - - while (is_tag_prefix(arg)) - { - struct dhcp_netid *nn = opt_malloc(sizeof (struct dhcp_netid)); - comma = split(arg); - nn->next = new->netid; - new->netid = nn; - nn->net = opt_string_alloc(arg+4); - arg = comma; - } + new->netid = dhcp_tags(&arg); if (arg && (comma = split(arg))) { for (i = 0; CSA[i]; i++) if (strcasecmp(CSA[i], arg) == 0) break; - + if (CSA[i] || atoi_check(arg, &i)) { arg = comma; comma = split(arg); - + new->CSA = i; new->menu = opt_string_alloc(arg); - + if (!comma) { new->type = 0; /* local boot */ @@ -3502,7 +3732,7 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma new->type = boottype++; new->basename = opt_string_alloc(arg); } - + if (comma) { if (!inet_pton(AF_INET, comma, &new->server)) @@ -3510,30 +3740,30 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma new->server.s_addr = 0; new->sname = opt_string_alloc(comma); } - + } } - + /* Order matters */ new->next = NULL; if (!daemon->pxe_services) - daemon->pxe_services = new; + daemon->pxe_services = new; else { struct pxe_service *s; for (s = daemon->pxe_services; s->next; s = s->next); s->next = new; } - + daemon->enable_pxe = 1; break; - + } } - + ret_err(gen_err); } - + case '4': /* --dhcp-mac */ { if (!(comma = split(arg))) @@ -3545,7 +3775,10 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma unhide_metas(comma); new->hwaddr_len = parse_hex(comma, new->hwaddr, DHCP_CHADDR_MAX, &new->mask, &new->hwaddr_type); if (new->hwaddr_len == -1) - ret_err(gen_err); + { + free(new->netid.net); + ret_err_free(gen_err, new); + } else { new->next = daemon->dhcp_macs; @@ -3555,24 +3788,6 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma } break; -#ifdef OPTION6_PREFIX_CLASS - case LOPT_PREF_CLSS: /* --dhcp-prefix-class */ - { - struct prefix_class *new = opt_malloc(sizeof(struct prefix_class)); - - if (!(comma = split(arg)) || - !atoi_check16(comma, &new->class)) - ret_err(gen_err); - - new->tag.net = opt_string_alloc(set_prefix(arg)); - new->next = daemon->prefix_classes; - daemon->prefix_classes = new; - - break; - } -#endif - - case 'U': /* --dhcp-vendorclass */ case 'j': /* --dhcp-userclass */ case LOPT_CIRCUIT: /* --dhcp-circuitid */ @@ -3582,25 +3797,28 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma unsigned char *p; int dig = 0; struct dhcp_vendor *new = opt_malloc(sizeof(struct dhcp_vendor)); - + if (!(comma = split(arg))) - ret_err(gen_err); - + ret_err_free(gen_err, new); + new->netid.net = opt_string_alloc(set_prefix(arg)); - /* check for hex string - must digits may include : must not have nothing else, + /* check for hex string - must digits may include : must not have nothing else, only allowed for agent-options. */ - + arg = comma; if ((comma = split(arg))) { if (option != 'U' || strstr(arg, "enterprise:") != arg) - ret_err(gen_err); + { + free(new->netid.net); + ret_err_free(gen_err, new); + } else new->enterprise = atoi(arg+11); } else comma = arg; - + for (p = (unsigned char *)comma; *p; p++) if (isxdigit(*p)) dig = 1; @@ -3609,7 +3827,7 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma unhide_metas(comma); if (option == 'U' || option == 'j' || *p || !dig) { - new->len = strlen(comma); + new->len = strlen(comma); new->data = opt_malloc(new->len); memcpy(new->data, comma, new->len); } @@ -3619,7 +3837,7 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma new->data = opt_malloc(new->len); memcpy(new->data, comma, new->len); } - + switch (option) { case 'j': @@ -3627,7 +3845,7 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma break; case 'U': new->match_type = MATCH_VENDOR; - break; + break; case LOPT_CIRCUIT: new->match_type = MATCH_CIRCUIT; break; @@ -3643,7 +3861,7 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma break; } - + case LOPT_ALTPORT: /* --dhcp-alternate-port */ if (!arg) { @@ -3653,11 +3871,11 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma else { comma = split(arg); - if (!atoi_check16(arg, &daemon->dhcp_server_port) || + if (!atoi_check16(arg, &daemon->dhcp_server_port) || (comma && !atoi_check16(comma, &daemon->dhcp_client_port))) ret_err(_("invalid port number")); if (!comma) - daemon->dhcp_client_port = daemon->dhcp_server_port+1; + daemon->dhcp_client_port = daemon->dhcp_server_port+1; } break; @@ -3694,19 +3912,13 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma new->next = daemon->dhcp_ignore_names; daemon->dhcp_ignore_names = new; } - + while (arg) { - struct dhcp_netid *member = opt_malloc(sizeof(struct dhcp_netid)); comma = split(arg); - member->next = list; - list = member; - if (is_tag_prefix(arg)) - member->net = opt_string_alloc(arg+4); - else - member->net = opt_string_alloc(arg); + list = dhcp_netid_create(is_tag_prefix(arg) ? arg+4 :arg, list); arg = comma; } - + new->list = list; break; } @@ -3717,7 +3929,7 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma struct addr_list *new = opt_malloc(sizeof(struct addr_list)); comma = split(arg); if (!(inet_pton(AF_INET, arg, &new->addr) > 0)) - ret_err(_("bad dhcp-proxy address")); + ret_err_free(_("bad dhcp-proxy address"), new); new->next = daemon->override_relays; daemon->override_relays = new; arg = comma; @@ -3743,13 +3955,16 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma } #endif else - ret_err(_("Bad dhcp-relay")); - + { + free(new->interface); + ret_err_free(_("Bad dhcp-relay"), new); + } + break; } #endif - + #ifdef HAVE_DHCP6 case LOPT_RA_PARAM: /* --ra-param */ if ((comma = split(arg))) @@ -3781,16 +3996,19 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma comma = split(comma); } arg = split(comma); - if (!atoi_check(comma, &new->interval) || + if (!atoi_check(comma, &new->interval) || (arg && !atoi_check(arg, &new->lifetime))) + { err: - ret_err(_("bad RA-params")); - + free(new->name); + ret_err_free(_("bad RA-params"), new); + } + new->next = daemon->ra_interfaces; daemon->ra_interfaces = new; } break; - + case LOPT_DUID: /* --dhcp-duid */ if (!(comma = split(arg)) || !atoi_check(arg, (int *)&daemon->duid_enterprise)) ret_err(_("bad DUID")); @@ -3820,38 +4038,38 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma break; unhide_metas(a[k]); } - + dash = split_chr(a[0], '-'); - if ((k < 2) || + if ((k < 2) || (!(inet_pton(AF_INET, a[0], &new->in) > 0)) || (!(inet_pton(AF_INET, a[1], &new->out) > 0)) || (k == 3 && !inet_pton(AF_INET, a[2], &new->mask))) ret_err(_("missing address in alias")); - - if (dash && + + if (dash && (!(inet_pton(AF_INET, dash, &new->end) > 0) || !is_same_net(new->in, new->end, new->mask) || ntohl(new->in.s_addr) > ntohl(new->end.s_addr))) - ret_err(_("invalid alias range")); - + ret_err_free(_("invalid alias range"), new); + break; } - + case LOPT_INTNAME: /* --interface-name */ { struct interface_name *new, **up; char *domain = NULL; comma = split(arg); - + if (!comma || !(domain = canonicalise_opt(arg))) ret_err(_("bad interface name")); - + new = opt_malloc(sizeof(struct interface_name)); new->next = NULL; new->addr = NULL; - + /* Add to the end of the list, so that first name of an interface is used for PTR lookups. */ for (up = &daemon->int_names; *up; up = &((*up)->next)); @@ -3863,17 +4081,15 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma { if (strcmp(arg, "4") == 0) new->family = AF_INET; -#ifdef HAVE_IPV6 else if (strcmp(arg, "6") == 0) new->family = AF_INET6; -#endif else - ret_err(gen_err); - } + ret_err_free(gen_err, new); + } new->intr = opt_string_alloc(comma); break; } - + case LOPT_CNAME: /* --cname */ { struct cname *new; @@ -3888,10 +4104,10 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma if (!pen) ret_err(_("bad CNAME")); - + if (pen != arg && atoi_check(last, &ttl)) last = pen; - + target = canonicalise_opt(last); while (arg != last) @@ -3900,11 +4116,19 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma alias = canonicalise_opt(arg); if (!alias || !target) - ret_err(_("bad CNAME")); - + { + free(target); + free(alias); + ret_err(_("bad CNAME")); + } + for (new = daemon->cnames; new; new = new->next) if (hostname_isequal(new->alias, alias)) - ret_err(_("duplicate CNAME")); + { + free(target); + free(alias); + ret_err(_("duplicate CNAME")); + } new = opt_malloc(sizeof(struct cname)); new->next = daemon->cnames; daemon->cnames = new; @@ -3914,7 +4138,7 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma for (arg += arglen+1; *arg && isspace(*arg); arg++); } - + break; } @@ -3924,10 +4148,14 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma char *dom, *target = NULL; comma = split(arg); - + if (!(dom = canonicalise_opt(arg)) || (comma && !(target = canonicalise_opt(comma)))) - ret_err(_("bad PTR record")); + { + free(dom); + free(target); + ret_err(_("bad PTR record")); + } else { new = opt_malloc(sizeof(struct ptr_record)); @@ -3945,20 +4173,24 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma int k = 0; struct naptr *new; int order, pref; - char *name, *replace = NULL; + char *name=NULL, *replace = NULL; if ((a[0] = arg)) for (k = 1; k < 7; k++) if (!(a[k] = split(a[k-1]))) break; - - - if (k < 6 || + + + if (k < 6 || !(name = canonicalise_opt(a[0])) || - !atoi_check16(a[1], &order) || + !atoi_check16(a[1], &order) || !atoi_check16(a[2], &pref) || (k == 7 && !(replace = canonicalise_opt(a[6])))) - ret_err(_("bad NAPTR record")); + { + free(name); + free(replace); + ret_err(_("bad NAPTR record")); + } else { new = opt_malloc(sizeof(struct naptr)); @@ -3980,30 +4212,34 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma struct txt_record *new; size_t len = 0; char *data; - int val; + int class; comma = split(arg); data = split(comma); - + new = opt_malloc(sizeof(struct txt_record)); - new->next = daemon->rr; - daemon->rr = new; - - if (!atoi_check(comma, &val) || + new->name = NULL; + + if (!atoi_check(comma, &class) || !(new->name = canonicalise_opt(arg)) || (data && (len = parse_hex(data, (unsigned char *)data, -1, NULL, NULL)) == -1U)) - ret_err(_("bad RR record")); + { + free(new->name); + ret_err_free(_("bad RR record"), new); + } - new->class = val; new->len = 0; - + new->class = class; + new->next = daemon->rr; + daemon->rr = new; + if (data) { new->txt = opt_malloc(len); new->len = len; memcpy(new->txt, data, len); } - + break; } @@ -4012,18 +4248,18 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma struct txt_record *new; char *tag, *value; int flags; - + comma = split(arg); tag = split(comma); value = split(tag); - + new = opt_malloc(sizeof(struct txt_record)); new->next = daemon->rr; daemon->rr = new; if (!atoi_check(comma, &flags) || !tag || !value || !(new->name = canonicalise_opt(arg))) ret_err(_("bad CAA record")); - + unhide_metas(tag); unhide_metas(value); @@ -4034,10 +4270,10 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma memcpy(&new->txt[2], tag, strlen(tag)); memcpy(&new->txt[2 + strlen(tag)], value, strlen(value)); new->class = T_CAA; - + break; } - + case 'Y': /* --txt-record */ { struct txt_record *new; @@ -4045,23 +4281,23 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma size_t len; comma = split(arg); - + new = opt_malloc(sizeof(struct txt_record)); - new->next = daemon->txt; - daemon->txt = new; new->class = C_IN; new->stat = 0; if (!(new->name = canonicalise_opt(arg))) - ret_err(_("bad TXT record")); - + ret_err_free(_("bad TXT record"), new); + + new->next = daemon->txt; + daemon->txt = new; len = comma ? strlen(comma) : 0; len += (len/255) + 1; /* room for extra counts */ new->txt = p = opt_malloc(len); cnt = p++; *cnt = 0; - + while (comma && *comma) { unsigned char c = (unsigned char)*comma++; @@ -4084,45 +4320,53 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma break; } - + case 'W': /* --srv-host */ { int port = 1, priority = 0, weight = 0; char *name, *target = NULL; struct mx_srv_record *new; - + comma = split(arg); - + if (!(name = canonicalise_opt(arg))) ret_err(_("bad SRV record")); - + if (comma) { arg = comma; comma = split(arg); if (!(target = canonicalise_opt(arg))) - ret_err(_("bad SRV target")); - + ret_err_free(_("bad SRV target"), name); + if (comma) { arg = comma; comma = split(arg); if (!atoi_check16(arg, &port)) - ret_err(_("invalid port number")); - + { + free(name); + ret_err_free(_("invalid port number"), target); + } + if (comma) { arg = comma; comma = split(arg); if (!atoi_check16(arg, &priority)) - ret_err(_("invalid priority")); - + { + free(name); + ret_err_free(_("invalid priority"), target); + } if (comma && !atoi_check16(comma, &weight)) - ret_err(_("invalid weight")); + { + free(name); + ret_err_free(_("invalid weight"), target); + } } } } - + new = opt_malloc(sizeof(struct mx_srv_record)); new->next = daemon->mxnames; daemon->mxnames = new; @@ -4134,19 +4378,22 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma new->weight = weight; break; } - + case LOPT_HOST_REC: /* --host-record */ { - struct host_record *new = opt_malloc(sizeof(struct host_record)); - memset(new, 0, sizeof(struct host_record)); - new->ttl = -1; + struct host_record *new; if (!arg || !(comma = split(arg))) ret_err(_("Bad host-record")); + + new = opt_malloc(sizeof(struct host_record)); + memset(new, 0, sizeof(struct host_record)); + new->ttl = -1; + new->flags = 0; while (arg) { - struct all_addr addr; + union all_addr addr; char *dig; for (dig = arg; *dig != 0; dig++) @@ -4154,33 +4401,46 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma break; if (*dig == 0) new->ttl = atoi(arg); - else if (inet_pton(AF_INET, arg, &addr)) - new->addr = addr.addr.addr4; -#ifdef HAVE_IPV6 - else if (inet_pton(AF_INET6, arg, &addr)) - new->addr6 = addr.addr.addr6; -#endif + else if (inet_pton(AF_INET, arg, &addr.addr4)) + { + new->addr = addr.addr4; + new->flags |= HR_4; + } + else if (inet_pton(AF_INET6, arg, &addr.addr6)) + { + new->addr6 = addr.addr6; + new->flags |= HR_6; + } else { int nomem; char *canon = canonicalise(arg, &nomem); - struct name_list *nl = opt_malloc(sizeof(struct name_list)); + struct name_list *nl; if (!canon) - ret_err(_("Bad name in host-record")); + { + struct name_list *tmp = new->names, *next; + for (tmp = new->names; tmp; tmp = next) + { + next = tmp->next; + free(tmp); + } + ret_err_free(_("Bad name in host-record"), new); + } + nl = opt_malloc(sizeof(struct name_list)); nl->name = canon; /* keep order, so that PTR record goes to first name */ nl->next = NULL; if (!new->names) new->names = nl; else - { + { struct name_list *tmp; for (tmp = new->names; tmp->next; tmp = tmp->next); tmp->next = nl; } } - + arg = comma; comma = split(arg); } @@ -4197,7 +4457,7 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma #ifdef HAVE_DNSSEC case LOPT_DNSSEC_STAMP: /* --dnssec-timestamp */ - daemon->timestamp_file = opt_string_alloc(arg); + daemon->timestamp_file = opt_string_alloc(arg); break; case LOPT_DNSSEC_CHECK: /* --dnssec-check-unsigned */ @@ -4209,14 +4469,15 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma ret_err(_("bad value for dnssec-check-unsigned")); } break; - + case LOPT_TRUST_ANCHOR: /* --trust-anchor */ { struct ds_config *new = opt_malloc(sizeof(struct ds_config)); char *cp, *cp1, *keyhex, *digest, *algo = NULL; int len; - + new->class = C_IN; + new->name = NULL; if ((comma = split(arg)) && (algo = split(comma))) { @@ -4227,7 +4488,7 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma class = C_CHAOS; else if (strcmp(comma, "HS") == 0) class = C_HESIOD; - + if (class != 0) { new->class = class; @@ -4235,14 +4496,14 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma algo = split(comma); } } - + if (!comma || !algo || !(digest = split(algo)) || !(keyhex = split(digest)) || - !atoi_check16(comma, &new->keytag) || + !atoi_check16(comma, &new->keytag) || !atoi_check8(algo, &new->algo) || !atoi_check8(digest, &new->digest_type) || !(new->name = canonicalise_opt(arg))) - ret_err(_("bad trust anchor")); - + ret_err_free(_("bad trust anchor"), new); + /* Upper bound on length */ len = (2*strlen(keyhex))+1; new->digest = opt_malloc(len); @@ -4255,28 +4516,31 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma else cp++; if ((new->digestlen = parse_hex(keyhex, (unsigned char *)new->digest, len, NULL, NULL)) == -1) - ret_err(_("bad HEX in trust anchor")); - + { + free(new->name); + ret_err_free(_("bad HEX in trust anchor"), new); + } + new->next = daemon->ds; daemon->ds = new; - + break; } #endif - + default: ret_err(_("unsupported option (check that dnsmasq was compiled with DHCP/TFTP/DNSSEC/DBus support)")); - + } - + return 1; } -static void read_file(char *file, FILE *f, int hard_opt) +static void read_file(char *file, FILE *f, int hard_opt) { volatile int lineno = 0; char *buff = daemon->namebuff; - + while (fgets(buff, MAXDNAME, f)) { int white, i; @@ -4284,19 +4548,19 @@ static void read_file(char *file, FILE *f, int hard_opt) char *errmess, *p, *arg, *start; size_t len; - /* Memory allocation failure longjmps here if mem_recover == 1 */ + /* Memory allocation failure longjmps here if mem_recover == 1 */ if (option != 0 || hard_opt == LOPT_REV_SERV) { if (setjmp(mem_jmp)) continue; mem_recover = 1; } - + arg = NULL; lineno++; errmess = NULL; - - /* Implement quotes, inside quotes we allow \\ \" \n and \t + + /* Implement quotes, inside quotes we allow \\ \" \n and \t metacharacters get hidden also strip comments */ for (white = 1, p = buff; *p; p++) { @@ -4323,10 +4587,10 @@ static void read_file(char *file, FILE *f, int hard_opt) *p = hide_meta(*p); } - if (*p == 0) + if (*p == 0) { errmess = _("missing \""); - goto oops; + goto oops; } memmove(p, p+1, strlen(p+1)+1); @@ -4337,29 +4601,29 @@ static void read_file(char *file, FILE *f, int hard_opt) *p = ' '; white = 1; } - else + else { if (white && *p == '#') - { + { *p = 0; break; } white = 0; - } + } } - + /* strip leading spaces */ for (start = buff; *start && *start == ' '; start++); - + /* strip trailing spaces */ for (len = strlen(start); (len != 0) && (start[len-1] == ' '); len--); - + if (len == 0) - continue; + continue; else start[len] = 0; - + if (option != 0) arg = start; else if ((p=strchr(start, '='))) @@ -4374,13 +4638,13 @@ static void read_file(char *file, FILE *f, int hard_opt) if (option == 0) { - for (option = 0, i = 0; opts[i].name; i++) + for (option = 0, i = 0; opts[i].name; i++) if (strcmp(opts[i].name, start) == 0) { option = opts[i].val; break; } - + if (!option) errmess = _("bad option"); else if (opts[i].has_arg == 0 && arg) @@ -4394,7 +4658,7 @@ static void read_file(char *file, FILE *f, int hard_opt) oops: if (errmess) strcpy(daemon->namebuff, errmess); - + if (errmess || !one_opt(option, arg, daemon->namebuff, _("error"), 0, hard_opt == LOPT_REV_SERV)) { sprintf(daemon->namebuff + strlen(daemon->namebuff), _(" at line %d of %s"), lineno, file); @@ -4413,12 +4677,12 @@ static void read_file(char *file, FILE *f, int hard_opt) int option_read_dynfile(char *file, int flags) { my_syslog(MS_DHCP | LOG_INFO, _("read %s"), file); - + if (flags & AH_DHCP_HST) return one_file(file, LOPT_BANK); else if (flags & AH_DHCP_OPT) return one_file(file, LOPT_OPTS); - + return 0; } #endif @@ -4433,7 +4697,7 @@ static int one_file(char *file, int hard_opt) ino_t ino; struct fileread *next; } *filesread = NULL; - + if (hard_opt == '7') { /* default conf-file reading */ @@ -4453,24 +4717,24 @@ static int one_file(char *file, int hard_opt) { /* ignore repeated files. */ struct stat statbuf; - + if (hard_opt == 0 && stat(file, &statbuf) == 0) { struct fileread *r; - + for (r = filesread; r; r = r->next) if (r->dev == statbuf.st_dev && r->ino == statbuf.st_ino) return 1; - + r = safe_malloc(sizeof(struct fileread)); r->next = filesread; filesread = r; r->dev = statbuf.st_dev; r->ino = statbuf.st_ino; } - + if (!(f = fopen(file, "r"))) - { + { if (errno == ENOENT && nofile_ok) return 1; /* No conffile, all done. */ else @@ -4484,9 +4748,9 @@ static int one_file(char *file, int hard_opt) else die(str, file, EC_FILE); } - } + } } - + read_file(file, f, hard_opt); return 1; } @@ -4517,12 +4781,12 @@ struct hostsfile *expand_filelist(struct hostsfile *list) { DIR *dir_stream; struct dirent *ent; - + /* don't read this as a file */ ah->flags |= AH_INACTIVE; - + if (!(dir_stream = opendir(ah->fname))) - my_syslog(LOG_ERR, _("cannot access directory %s: %s"), + my_syslog(LOG_ERR, _("cannot access directory %s: %s"), ah->fname, strerror(errno)); else { @@ -4532,19 +4796,19 @@ struct hostsfile *expand_filelist(struct hostsfile *list) size_t lenfile = strlen(ent->d_name); struct hostsfile *ah1; char *path; - + /* ignore emacs backups and dotfiles */ - if (lenfile == 0 || + if (lenfile == 0 || ent->d_name[lenfile - 1] == '~' || (ent->d_name[0] == '#' && ent->d_name[lenfile - 1] == '#') || ent->d_name[0] == '.') continue; - + /* see if we have an existing record. - dir is ah->fname + dir is ah->fname file is ent->d_name path to match is ah1->fname */ - + for (ah1 = list; ah1; ah1 = ah1->next) { if (lendir < strlen(ah1->fname) && @@ -4556,19 +4820,19 @@ struct hostsfile *expand_filelist(struct hostsfile *list) break; } } - + /* make new record */ if (!ah1) { if (!(ah1 = whine_malloc(sizeof(struct hostsfile)))) continue; - + if (!(path = whine_malloc(lendir + lenfile + 2))) { free(ah1); continue; } - + strcpy(path, ah->fname); strcat(path, "/"); strcat(path, ent->d_name); @@ -4578,17 +4842,17 @@ struct hostsfile *expand_filelist(struct hostsfile *list) ah1->next = list; list = ah1; } - + /* inactivate record if not regular file */ if ((ah1->flags & AH_DIR) && stat(ah1->fname, &buf) != -1 && !S_ISREG(buf.st_mode)) - ah1->flags |= AH_INACTIVE; - + ah1->flags |= AH_INACTIVE; + } closedir(dir_stream); } } } - + return list; } @@ -4601,48 +4865,48 @@ void read_servers_file(void) my_syslog(LOG_ERR, _("cannot read %s: %s"), daemon->servers_file, strerror(errno)); return; } - + mark_servers(SERV_FROM_FILE); cleanup_servers(); - + read_file(daemon->servers_file, f, LOPT_REV_SERV); } - + #ifdef HAVE_DHCP static void clear_dynamic_conf(void) { struct dhcp_config *configs, *cp, **up; - + /* remove existing... */ for (up = &daemon->dhcp_conf, configs = daemon->dhcp_conf; configs; configs = cp) { cp = configs->next; - + if (configs->flags & CONFIG_BANK) { struct hwaddr_config *mac, *tmp; struct dhcp_netid_list *list, *tmplist; - + for (mac = configs->hwaddr; mac; mac = tmp) { tmp = mac->next; free(mac); } - + if (configs->flags & CONFIG_CLID) free(configs->clid); - + for (list = configs->netid; list; list = tmplist) { free(list->list); tmplist = list->next; free(list); } - + if (configs->flags & CONFIG_NAME) free(configs->hostname); - + *up = configs->next; free(configs); } @@ -4659,7 +4923,7 @@ static void clear_dynamic_opt(void) for (up = &daemon->dhcp_opts, opts = daemon->dhcp_opts; opts; opts = cp) { cp = opts->next; - + if (opts->flags & DHOPT_BANK) { if ((opts->flags & DHOPT_VENDOR)) @@ -4686,7 +4950,7 @@ void reread_dhcp(void) /* Do these even if there is no daemon->dhcp_hosts_file or daemon->dhcp_opts_file since entries may have been created by the inotify dynamic file reading system. */ - + clear_dynamic_conf(); clear_dynamic_opt(); @@ -4696,7 +4960,7 @@ void reread_dhcp(void) for (hf = daemon->dhcp_hosts_file; hf; hf = hf->next) if (!(hf->flags & AH_INACTIVE)) { - if (one_file(hf->fname, LOPT_BANK)) + if (one_file(hf->fname, LOPT_BANK)) my_syslog(MS_DHCP | LOG_INFO, _("read %s"), hf->fname); } } @@ -4707,7 +4971,7 @@ void reread_dhcp(void) for (hf = daemon->dhcp_opts_file; hf; hf = hf->next) if (!(hf->flags & AH_INACTIVE)) { - if (one_file(hf->fname, LOPT_OPTS)) + if (one_file(hf->fname, LOPT_OPTS)) my_syslog(MS_DHCP | LOG_INFO, _("read %s"), hf->fname); } } @@ -4724,9 +4988,9 @@ void read_opts(int argc, char **argv, char *compile_opts) size_t argbuf_size = MAXDNAME; char *argbuf = opt_malloc(argbuf_size); char *buff = opt_malloc(MAXDNAME); - int option, conffile_opt = '7', testmode = 0; - char *arg, *conffile = CONFFILE; - + int option, testmode = 0; + char *arg, *conffile = NULL; + opterr = 0; daemon = opt_malloc(sizeof(struct daemon)); @@ -4748,7 +5012,7 @@ void read_opts(int argc, char **argv, char *compile_opts) daemon->tftp_max = TFTP_MAX_CONNECTIONS; daemon->edns_pktsz = EDNS_PKTSZ; daemon->log_fac = -1; - daemon->auth_ttl = AUTH_TTL; + daemon->auth_ttl = AUTH_TTL; daemon->soa_refresh = SOA_REFRESH; daemon->soa_retry = SOA_RETRY; daemon->soa_expiry = SOA_EXPIRY; @@ -4772,15 +5036,18 @@ void read_opts(int argc, char **argv, char *compile_opts) add_txt("privacylevel.pihole", NULL, TXT_PRIVACYLEVEL); /************************/ #endif + /******** Pi-hole modification ********/ + add_txt("version.FTL", get_FTL_version(), 0 ); + /**************************************/ - while (1) + while (1) { #ifdef HAVE_GETOPT_LONG option = getopt_long(argc, argv, OPTSTRING, opts, NULL); #else option = getopt(argc, argv, OPTSTRING); #endif - + if (option == -1) { for (; optind < argc; optind++) @@ -4807,7 +5074,7 @@ void read_opts(int argc, char **argv, char *compile_opts) } else arg = NULL; - + /* command-line only stuff */ if (option == LOPT_TEST) testmode = 1; @@ -4829,7 +5096,7 @@ void read_opts(int argc, char **argv, char *compile_opts) else if (option == 'v') { printf(_("Dnsmasq version %s %s\n"), VERSION, COPYRIGHT); - printf(_("Compile time options: %s\n\n"), compile_opts); + printf(_("Compile time options: %s\n\n"), compile_opts); printf(_("This software comes with ABSOLUTELY NO WARRANTY.\n")); printf(_("Dnsmasq is free software, and you are welcome to redistribute it\n")); printf(_("under the terms of the GNU General Public License, version 2 or 3.\n")); @@ -4837,16 +5104,22 @@ void read_opts(int argc, char **argv, char *compile_opts) } else if (option == 'C') { - conffile_opt = 0; /* file must exist */ - conffile = opt_string_alloc(arg); + if (!conffile) + conffile = opt_string_alloc(arg); + else + { + char *extra = opt_string_alloc(arg); + one_file(extra, 0); + free(extra); + } } else { #ifdef HAVE_GETOPT_LONG if (!one_opt(option, arg, daemon->namebuff, _("try --help"), 1, 0)) -#else - if (!one_opt(option, arg, daemon->namebuff, _("try -w"), 1, 0)) -#endif +#else + if (!one_opt(option, arg, daemon->namebuff, _("try -w"), 1, 0)) +#endif die(_("bad command line options: %s"), daemon->namebuff, EC_BADCONF); } } @@ -4855,10 +5128,11 @@ void read_opts(int argc, char **argv, char *compile_opts) if (conffile) { - one_file(conffile, conffile_opt); - if (conffile_opt == 0) - free(conffile); + one_file(conffile, 0); + free(conffile); } + else + one_file(CONFFILE, '7'); /* port might not be known when the address is parsed - fill in here */ if (daemon->servers) @@ -4869,17 +5143,15 @@ void read_opts(int argc, char **argv, char *compile_opts) { if (tmp->source_addr.sa.sa_family == AF_INET) tmp->source_addr.in.sin_port = htons(daemon->query_port); -#ifdef HAVE_IPV6 else if (tmp->source_addr.sa.sa_family == AF_INET6) tmp->source_addr.in6.sin6_port = htons(daemon->query_port); -#endif } - } - + } + if (daemon->host_records) { struct host_record *hr; - + for (hr = daemon->host_records; hr; hr = hr->next) if (hr->ttl == -1) hr->ttl = daemon->local_ttl; @@ -4890,9 +5162,9 @@ void read_opts(int argc, char **argv, char *compile_opts) struct cname *cn, *cn2, *cn3; #define NOLOOP 1 -#define TESTLOOP 2 +#define TESTLOOP 2 - /* Fill in TTL for CNAMES noe we have local_ttl. + /* Fill in TTL for CNAMES now we have local_ttl. Also prepare to do loop detection. */ for (cn = daemon->cnames; cn; cn = cn->next) { @@ -4907,7 +5179,7 @@ void read_opts(int argc, char **argv, char *compile_opts) break; } } - + /* Find any CNAME loops.*/ for (cn = daemon->cnames; cn; cn = cn->next) { @@ -4915,30 +5187,28 @@ void read_opts(int argc, char **argv, char *compile_opts) { if (cn2->flag == NOLOOP) break; - + if (cn2->flag == TESTLOOP) die(_("CNAME loop involving %s"), cn->alias, EC_BADCONF); - + cn2->flag = TESTLOOP; } - + for (cn3 = cn->targetp; cn3 != cn2; cn3 = cn3->targetp) cn3->flag = NOLOOP; } } if (daemon->if_addrs) - { + { struct iname *tmp; for(tmp = daemon->if_addrs; tmp; tmp = tmp->next) if (tmp->addr.sa.sa_family == AF_INET) tmp->addr.in.sin_port = htons(daemon->port); -#ifdef HAVE_IPV6 else if (tmp->addr.sa.sa_family == AF_INET6) tmp->addr.in6.sin6_port = htons(daemon->port); -#endif /* IPv6 */ } - + /* create default, if not specified */ if (daemon->authserver && !daemon->hostmaster) { @@ -4946,19 +5216,19 @@ void read_opts(int argc, char **argv, char *compile_opts) strcat(buff, daemon->authserver); daemon->hostmaster = opt_string_alloc(buff); } - + /* only one of these need be specified: the other defaults to the host-name */ if (option_bool(OPT_LOCALMX) || daemon->mxnames || daemon->mxtarget) { struct mx_srv_record *mx; - + if (gethostname(buff, MAXDNAME) == -1) die(_("cannot get host-name: %s"), NULL, EC_MISC); - + for (mx = daemon->mxnames; mx; mx = mx->next) if (!mx->issrv && hostname_isequal(mx->name, buff)) break; - + if ((daemon->mxtarget || option_bool(OPT_LOCALMX)) && !mx) { mx = opt_malloc(sizeof(struct mx_srv_record)); @@ -4968,7 +5238,7 @@ void read_opts(int argc, char **argv, char *compile_opts) mx->name = opt_string_alloc(buff); daemon->mxnames = mx; } - + if (!daemon->mxtarget) daemon->mxtarget = opt_string_alloc(buff); @@ -4978,32 +5248,32 @@ void read_opts(int argc, char **argv, char *compile_opts) } if (!option_bool(OPT_NO_RESOLV) && - daemon->resolv_files && - daemon->resolv_files->next && + daemon->resolv_files && + daemon->resolv_files->next && option_bool(OPT_NO_POLL)) die(_("only one resolv.conf file allowed in no-poll mode."), NULL, EC_BADCONF); - + if (option_bool(OPT_RESOLV_DOMAIN)) { char *line; FILE *f; if (option_bool(OPT_NO_RESOLV) || - !daemon->resolv_files || + !daemon->resolv_files || (daemon->resolv_files)->next) die(_("must have exactly one resolv.conf to read domain from."), NULL, EC_BADCONF); - + if (!(f = fopen((daemon->resolv_files)->name, "r"))) die(_("failed to read %s: %s"), (daemon->resolv_files)->name, EC_FILE); - + while ((line = fgets(buff, MAXDNAME, f))) { char *token = strtok(line, " \t\n\r"); - + if (!token || strcmp(token, "search") != 0) continue; - - if ((token = strtok(NULL, " \t\n\r")) && + + if ((token = strtok(NULL, " \t\n\r")) && (daemon->domain_suffix = canonicalise_opt(token))) break; } @@ -5018,10 +5288,10 @@ void read_opts(int argc, char **argv, char *compile_opts) { /* add domain for any srv record without one. */ struct mx_srv_record *srv; - + for (srv = daemon->mxnames; srv; srv = srv->next) if (srv->issrv && - strchr(srv->name, '.') && + strchr(srv->name, '.') && strchr(srv->name, '.') == strrchr(srv->name, '.')) { strcpy(buff, srv->name); @@ -5037,11 +5307,11 @@ void read_opts(int argc, char **argv, char *compile_opts) /* If there's access-control config, then ignore --local-service, it's intended as a system default to keep otherwise unconfigured installations safe. */ if (daemon->if_names || daemon->if_except || daemon->if_addrs || daemon->authserver) - reset_option_bool(OPT_LOCAL_SERVICE); + reset_option_bool(OPT_LOCAL_SERVICE); if (testmode) { fprintf(stderr, "dnsmasq: %s.\n", _("syntax check OK")); exit(0); } -} +} diff --git a/dnsmasq/outpacket.c b/src/dnsmasq/outpacket.c similarity index 97% rename from dnsmasq/outpacket.c rename to src/dnsmasq/outpacket.c index d20bd33a6..50513dc33 100644 --- a/dnsmasq/outpacket.c +++ b/src/dnsmasq/outpacket.c @@ -1,4 +1,4 @@ -/* dnsmasq is Copyright (c) 2000-2018 Simon Kelley +/* dnsmasq is Copyright (c) 2000-2020 Simon Kelley This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by diff --git a/dnsmasq/poll.c b/src/dnsmasq/poll.c similarity index 98% rename from dnsmasq/poll.c rename to src/dnsmasq/poll.c index 0d8382f49..24d01bb1a 100644 --- a/dnsmasq/poll.c +++ b/src/dnsmasq/poll.c @@ -1,4 +1,4 @@ -/* dnsmasq is Copyright (c) 2000-2018 Simon Kelley +/* dnsmasq is Copyright (c) 2000-2020 Simon Kelley This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by diff --git a/dnsmasq/radv-protocol.h b/src/dnsmasq/radv-protocol.h similarity index 96% rename from dnsmasq/radv-protocol.h rename to src/dnsmasq/radv-protocol.h index 2ca9ec7a2..edc153278 100644 --- a/dnsmasq/radv-protocol.h +++ b/src/dnsmasq/radv-protocol.h @@ -1,4 +1,4 @@ -/* dnsmasq is Copyright (c) 2000-2018 Simon Kelley +/* dnsmasq is Copyright (c) 2000-2020 Simon Kelley This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by diff --git a/dnsmasq/radv.c b/src/dnsmasq/radv.c similarity index 97% rename from dnsmasq/radv.c rename to src/dnsmasq/radv.c index 4f31457db..7ec8cd0f2 100644 --- a/dnsmasq/radv.c +++ b/src/dnsmasq/radv.c @@ -1,4 +1,4 @@ -/* dnsmasq is Copyright (c) 2000-2018 Simon Kelley +/* dnsmasq is Copyright (c) 2000-2020 Simon Kelley This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -288,7 +288,10 @@ static void send_ra_alias(time_t now, int iface, char *iface_name, struct in6_ad context->netid.next = &context->netid; } - if (!iface_enumerate(AF_INET6, &parm, add_prefixes)) + /* If no link-local address then we can't advertise since source address of + advertisement must be link local address: RFC 4861 para 6.1.2. */ + if (!iface_enumerate(AF_INET6, &parm, add_prefixes) || + parm.link_pref_time == 0) return; /* Find smallest preferred time within address classes, @@ -412,7 +415,7 @@ static void send_ra_alias(time_t now, int iface, char *iface_name, struct in6_ad if (mtu == 0) { char *mtu_name = ra_param ? ra_param->mtu_name : NULL; - sprintf(daemon->namebuff, "/proc/sys/net/ipv6/conf/%s/mtu", mtu_name ? : iface_name); + sprintf(daemon->namebuff, "/proc/sys/net/ipv6/conf/%s/mtu", mtu_name ? mtu_name : iface_name); if ((f = fopen(daemon->namebuff, "r"))) { if (fgets(daemon->namebuff, MAXDNAME, f)) @@ -888,11 +891,21 @@ static int iface_search(struct in6_addr *local, int prefix, { struct search_param *param = vparam; struct dhcp_context *context; - + struct iname *tmp; + (void)scope; (void)preferred; (void)valid; - + + /* ignore interfaces we're not doing DHCP on. */ + if (!indextoname(daemon->icmp6fd, if_index, param->name) || + !iface_check(AF_LOCAL, NULL, param->name, NULL)) + return 1; + + for (tmp = daemon->dhcp_except; tmp; tmp = tmp->next) + if (tmp->name && wildcard_match(tmp->name, param->name)) + return 1; + for (context = daemon->dhcp6; context; context = context->next) if (!(context->flags & (CONTEXT_TEMPLATE | CONTEXT_OLD)) && prefix <= context->prefix && @@ -904,17 +917,9 @@ static int iface_search(struct in6_addr *local, int prefix, /* found an interface that's overdue for RA determine new timeout value and arrange for RA to be sent unless interface is still doing DAD.*/ - if (!(flags & IFACE_TENTATIVE)) param->iface = if_index; - /* should never fail */ - if (!indextoname(daemon->icmp6fd, if_index, param->name)) - { - param->iface = 0; - return 0; - } - new_timeout(context, param->name, param->now); /* zero timers for other contexts on the same subnet, so they don't timeout diff --git a/dnsmasq/rfc1035.c b/src/dnsmasq/rfc1035.c similarity index 79% rename from dnsmasq/rfc1035.c rename to src/dnsmasq/rfc1035.c index 469bd3325..e2f410ebd 100644 --- a/dnsmasq/rfc1035.c +++ b/src/dnsmasq/rfc1035.c @@ -1,15 +1,15 @@ -/* dnsmasq is Copyright (c) 2000-2018 Simon Kelley +/* dnsmasq is Copyright (c) 2000-2020 Simon Kelley This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; version 2 dated June, 1991, or (at your option) version 3 dated 29 June, 2007. - + This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. - + You should have received a copy of the GNU General Public License along with this program. If not, see . */ @@ -17,30 +17,30 @@ #include "dnsmasq.h" #include "../dnsmasq_interface.h" -int extract_name(struct dns_header *header, size_t plen, unsigned char **pp, +int extract_name(struct dns_header *header, size_t plen, unsigned char **pp, char *name, int isExtract, int extrabytes) { unsigned char *cp = (unsigned char *)name, *p = *pp, *p1 = NULL; unsigned int j, l, namelen = 0, hops = 0; int retvalue = 1; - + if (isExtract) *cp = 0; while (1) - { + { unsigned int label_type; if (!CHECK_LEN(header, p, plen, 1)) return 0; - - if ((l = *p++) == 0) + + if ((l = *p++) == 0) /* end marker */ { /* check that there are the correct no. of bytes after the name */ if (!CHECK_LEN(header, p1 ? p1 : p, plen, extrabytes)) return 0; - + if (isExtract) { if (cp != (unsigned char *)name) @@ -49,33 +49,33 @@ int extract_name(struct dns_header *header, size_t plen, unsigned char **pp, } else if (*cp != 0) retvalue = 2; - + if (p1) /* we jumped via compression */ *pp = p1; else *pp = p; - + return retvalue; } label_type = l & 0xc0; - + if (label_type == 0xc0) /* pointer */ - { + { if (!CHECK_LEN(header, p, plen, 1)) return 0; - + /* get offset */ l = (l&0x3f) << 8; l |= *p++; - + if (!p1) /* first jump, save location to go back to */ p1 = p; - + hops++; /* break malicious infinite loops */ if (hops > 255) return 0; - + p = l + (unsigned char *)header; } else if (label_type == 0x00) @@ -85,7 +85,7 @@ int extract_name(struct dns_header *header, size_t plen, unsigned char **pp, return 0; if (!CHECK_LEN(header, p, plen, l)) return 0; - + for(j=0; j= 'A' && c1 <= 'Z') @@ -123,15 +123,15 @@ int extract_name(struct dns_header *header, size_t plen, unsigned char **pp, if (option_bool(OPT_DNSSEC_VALID) && c1 == NAME_ESCAPE) c1 = (*cp++)-1; #endif - + if (c2 >= 'A' && c2 <= 'Z') c2 += 'a' - 'A'; - + if (c1 != c2) retvalue = 2; } } - + if (isExtract) *cp++ = '.'; else if (*cp != 0 && *cp++ != '.') @@ -141,20 +141,20 @@ int extract_name(struct dns_header *header, size_t plen, unsigned char **pp, return 0; /* label types 0x40 and 0x80 not supported */ } } - + /* Max size of input string (for IPv6) is 75 chars.) */ #define MAXARPANAME 75 -int in_arpa_name_2_addr(char *namein, struct all_addr *addrp) +int in_arpa_name_2_addr(char *namein, union all_addr *addrp) { int j; char name[MAXARPANAME+1], *cp1; unsigned char *addr = (unsigned char *)addrp; char *lastchunk = NULL, *penchunk = NULL; - + if (strlen(namein) > MAXARPANAME) return 0; - memset(addrp, 0, sizeof(struct all_addr)); + memset(addrp, 0, sizeof(union all_addr)); /* turn name into a series of asciiz strings */ /* j counts no. of labels */ @@ -168,7 +168,7 @@ int in_arpa_name_2_addr(char *namein, struct all_addr *addrp) } else *cp1 = *namein; - + *cp1 = 0; if (j<3) @@ -184,13 +184,13 @@ int in_arpa_name_2_addr(char *namein, struct all_addr *addrp) for (cp1 = name; cp1 != penchunk; cp1 += strlen(cp1)+1) { /* check for digits only (weeds out things like - 50.0/24.67.28.64.in-addr.arpa which are used + 50.0/24.67.28.64.in-addr.arpa which are used as CNAME targets according to RFC 2317 */ char *cp; for (cp = cp1; *cp; cp++) if (!isdigit((unsigned char)*cp)) return 0; - + addr[3] = addr[2]; addr[2] = addr[1]; addr[1] = addr[0]; @@ -199,22 +199,21 @@ int in_arpa_name_2_addr(char *namein, struct all_addr *addrp) return F_IPV4; } -#ifdef HAVE_IPV6 - else if (hostname_isequal(penchunk, "ip6") && + else if (hostname_isequal(penchunk, "ip6") && (hostname_isequal(lastchunk, "int") || hostname_isequal(lastchunk, "arpa"))) { /* IP v6: Address arrives as 0.1.2.3.4.5.6.7.8.9.a.b.c.d.e.f.ip6.[int|arpa] or \[xfedcba9876543210fedcba9876543210/128].ip6.[int|arpa] - - Note that most of these the various representations are obsolete and + + Note that most of these the various representations are obsolete and left-over from the many DNS-for-IPv6 wars. We support all the formats that we can since there is no reason not to. */ - if (*name == '\\' && *(name+1) == '[' && + if (*name == '\\' && *(name+1) == '[' && (*(name+2) == 'x' || *(name+2) == 'X')) - { + { for (j = 0, cp1 = name+3; *cp1 && isxdigit((unsigned char) *cp1) && j < 32; cp1++, j++) { char xdig[2]; @@ -225,7 +224,7 @@ int in_arpa_name_2_addr(char *namein, struct all_addr *addrp) else addr[j/2] = strtol(xdig, NULL, 16) << 4; } - + if (*cp1 == '/' && j == 32) return F_IPV6; } @@ -235,17 +234,16 @@ int in_arpa_name_2_addr(char *namein, struct all_addr *addrp) { if (*(cp1+1) || !isxdigit((unsigned char)*cp1)) return 0; - - for (j = sizeof(struct all_addr)-1; j>0; j--) + + for (j = sizeof(struct in6_addr)-1; j>0; j--) addr[j] = (addr[j] >> 4) | (addr[j-1] << 4); addr[0] = (addr[0] >> 4) | (strtol(cp1, NULL, 16) << 4); } - + return F_IPV6; } } -#endif - + return 0; } @@ -254,16 +252,16 @@ unsigned char *skip_name(unsigned char *ansp, struct dns_header *header, size_t while(1) { unsigned int label_type; - + if (!CHECK_LEN(header, ansp, plen, 1)) return NULL; - + label_type = (*ansp) & 0xc0; if (label_type == 0xc0) { /* pointer for compression. */ - ansp += 2; + ansp += 2; break; } else if (label_type == 0x80) @@ -272,15 +270,15 @@ unsigned char *skip_name(unsigned char *ansp, struct dns_header *header, size_t { /* Extended label type */ unsigned int count; - + if (!CHECK_LEN(header, ansp, plen, 2)) return NULL; - + if (((*ansp++) & 0x3f) != 1) return NULL; /* we only understand bitstrings */ - + count = *(ansp++); /* Bits in bitstring */ - + if (count == 0) /* count == 0 means 256 bits */ ansp += 32; else @@ -289,7 +287,7 @@ unsigned char *skip_name(unsigned char *ansp, struct dns_header *header, size_t else { /* label type == 0 Bottom six bits is length */ unsigned int len = (*ansp++) & 0x3f; - + if (!ADD_RDLEN(header, ansp, plen, len)) return NULL; @@ -300,7 +298,7 @@ unsigned char *skip_name(unsigned char *ansp, struct dns_header *header, size_t if (!CHECK_LEN(header, ansp, plen, extrabytes)) return NULL; - + return ansp; } @@ -315,18 +313,18 @@ unsigned char *skip_questions(struct dns_header *header, size_t plen) return NULL; ansp += 4; /* class and type */ } - + return ansp; } unsigned char *skip_section(unsigned char *ansp, int count, struct dns_header *header, size_t plen) { int i, rdlen; - + for (i = 0; i < count; i++) { if (!(ansp = skip_name(ansp, header, plen, 10))) - return NULL; + return NULL; ansp += 8; /* type, class, TTL */ GETSHORT(rdlen, ansp); if (!ADD_RDLEN(header, ansp, plen, rdlen)) @@ -336,10 +334,10 @@ unsigned char *skip_section(unsigned char *ansp, int count, struct dns_header *h return ansp; } -/* CRC the question section. This is used to safely detect query - retransmission and to detect answers to questions we didn't ask, which - might be poisoning attacks. Note that we decode the name rather - than CRC the raw bytes, since replies might be compressed differently. +/* CRC the question section. This is used to safely detect query + retransmission and to detect answers to questions we didn't ask, which + might be poisoning attacks. Note that we decode the name rather + than CRC the raw bytes, since replies might be compressed differently. We ignore case in the names for the same reason. Return all-ones if there is not question section. */ #ifndef HAVE_DNSSEC @@ -349,11 +347,11 @@ unsigned int questions_crc(struct dns_header *header, size_t plen, char *name) unsigned int crc = 0xffffffff; unsigned char *p1, *p = (unsigned char *)(header+1); - for (q = ntohs(header->qdcount); q != 0; q--) + for (q = ntohs(header->qdcount); q != 0; q--) { if (!extract_name(header, plen, &p, name, 1, 4)) return crc; /* bad packet */ - + for (p1 = (unsigned char *)name; *p1; p1++) { int i = 8; @@ -366,7 +364,7 @@ unsigned int questions_crc(struct dns_header *header, size_t plen, char *name) while (i--) crc = crc & 0x80000000 ? (crc << 1) ^ 0x04c11db7 : crc << 1; } - + /* CRC the class and type as well */ for (p1 = p; p1 < p+4; p1++) { @@ -388,15 +386,15 @@ unsigned int questions_crc(struct dns_header *header, size_t plen, char *name) size_t resize_packet(struct dns_header *header, size_t plen, unsigned char *pheader, size_t hlen) { unsigned char *ansp = skip_questions(header, plen); - + /* if packet is malformed, just return as-is. */ if (!ansp) return plen; - + if (!(ansp = skip_section(ansp, ntohs(header->ancount) + ntohs(header->nscount) + ntohs(header->arcount), header, plen))) return plen; - + /* restore pseudoheader */ if (pheader && ntohs(header->arcount) == 0) { @@ -409,8 +407,8 @@ size_t resize_packet(struct dns_header *header, size_t plen, unsigned char *phea return ansp - (unsigned char *)header; } -/* is addr in the non-globally-routed IP space? */ -int private_net(struct in_addr addr, int ban_localhost) +/* is addr in the non-globally-routed IP space? */ +int private_net(struct in_addr addr, int ban_localhost) { in_addr_t ip_addr = ntohl(addr.s_addr); @@ -427,18 +425,15 @@ int private_net(struct in_addr addr, int ban_localhost) ((ip_addr & 0xFFFFFFFF) == 0xFFFFFFFF) /* 255.255.255.255/32 (broadcast)*/ ; } -#ifdef HAVE_IPV6 static int private_net6(struct in6_addr *a) { - return + return IN6_IS_ADDR_UNSPECIFIED(a) || /* RFC 6303 4.3 */ IN6_IS_ADDR_LOOPBACK(a) || /* RFC 6303 4.3 */ IN6_IS_ADDR_LINKLOCAL(a) || /* RFC 6303 4.5 */ ((unsigned char *)a)[0] == 0xfd || /* RFC 6303 4.4 */ ((u32 *)a)[0] == htonl(0x20010db8); /* RFC 6303 4.6 */ } -#endif - static unsigned char *do_doctor(unsigned char *p, int count, struct dns_header *header, size_t qlen, char *name, int *doctored) { @@ -453,23 +448,23 @@ static unsigned char *do_doctor(unsigned char *p, int count, struct dns_header * } else if (!(p = skip_name(p, header, qlen, 10))) return 0; /* bad packet */ - - GETSHORT(qtype, p); + + GETSHORT(qtype, p); GETSHORT(qclass, p); p += 4; /* ttl */ GETSHORT(rdlen, p); - + if (qclass == C_IN && qtype == T_A) { struct doctor *doctor; struct in_addr addr; - + if (!CHECK_LEN(header, p, qlen, INADDRSZ)) return 0; - + /* alignment */ memcpy(&addr, p, INADDRSZ); - + for (doctor = daemon->doctors; doctor; doctor = doctor->next) { if (doctor->end.s_addr == 0) @@ -477,10 +472,10 @@ static unsigned char *do_doctor(unsigned char *p, int count, struct dns_header * if (!is_same_net(doctor->in, addr, doctor->mask)) continue; } - else if (ntohl(doctor->in.s_addr) > ntohl(addr.s_addr) || + else if (ntohl(doctor->in.s_addr) > ntohl(addr.s_addr) || ntohl(doctor->end.s_addr) < ntohl(addr.s_addr)) continue; - + addr.s_addr &= ~doctor->mask.s_addr; addr.s_addr |= (doctor->out.s_addr & doctor->mask.s_addr); /* Since we munged the data, the server it came from is no longer authoritative */ @@ -506,7 +501,7 @@ static unsigned char *do_doctor(unsigned char *p, int count, struct dns_header * { if (!isprint((int)*(p2+1))) break; - + *p2 = *(p2+1); p2++; } @@ -517,13 +512,13 @@ static unsigned char *do_doctor(unsigned char *p, int count, struct dns_header * *p1 = len; p1 += len+1; } - } - + } + if (!ADD_RDLEN(header, p, qlen, rdlen)) return 0; /* bad packet */ } - - return p; + + return p; } static int find_soa(struct dns_header *header, size_t qlen, char *name, int *doctored) @@ -532,22 +527,22 @@ static int find_soa(struct dns_header *header, size_t qlen, char *name, int *doc int qtype, qclass, rdlen; unsigned long ttl, minttl = ULONG_MAX; int i, found_soa = 0; - + /* first move to NS section and find TTL from any SOA section */ if (!(p = skip_questions(header, qlen)) || !(p = do_doctor(p, ntohs(header->ancount), header, qlen, name, doctored))) return 0; /* bad packet */ - + for (i = ntohs(header->nscount); i != 0; i--) { if (!(p = skip_name(p, header, qlen, 10))) return 0; /* bad packet */ - - GETSHORT(qtype, p); + + GETSHORT(qtype, p); GETSHORT(qclass, p); GETLONG(ttl, p); GETSHORT(rdlen, p); - + if ((qclass == C_IN) && (qtype == T_SOA)) { found_soa = 1; @@ -561,7 +556,7 @@ static int find_soa(struct dns_header *header, size_t qlen, char *name, int *doc if (!(p = skip_name(p, header, qlen, 20))) return 0; p += 16; /* SERIAL REFRESH RETRY EXPIRE */ - + GETLONG(ttl, p); /* minTTL */ if (ttl < minttl) minttl = ttl; @@ -569,11 +564,11 @@ static int find_soa(struct dns_header *header, size_t qlen, char *name, int *doc else if (!ADD_RDLEN(header, p, qlen, rdlen)) return 0; /* bad packet */ } - + /* rewrite addresses in additional section too */ if (!do_doctor(p, ntohs(header->arcount), header, qlen, NULL, doctored)) return 0; - + if (!found_soa) minttl = daemon->neg_ttl; @@ -581,24 +576,25 @@ static int find_soa(struct dns_header *header, size_t qlen, char *name, int *doc } /* Note that the following code can create CNAME chains that don't point to a real record, - either because of lack of memory, or lack of SOA records. These are treated by the cache code as - expired and cleaned out that way. - Return 1 if we reject an address because it look like part of dns-rebinding attack. */ -int extract_addresses(struct dns_header *header, size_t qlen, char *name, time_t now, + either because of lack of memory, or lack of SOA records. These are treated by the cache code as + expired and cleaned out that way. + Return 1 if we reject an address because it look like part of dns-rebinding attack. + **** Pi-hole: Return 2 if we reject a part of a CNAME chain **** */ +int extract_addresses(struct dns_header *header, size_t qlen, char *name, time_t now, char **ipsets, int is_sign, int check_rebind, int no_cache_dnssec, int secure, int *doctored) { unsigned char *p, *p1, *endrr, *namep; int i, j, qtype, qclass, aqtype, aqclass, ardlen, res, searched_soa = 0; unsigned long ttl = 0; - struct all_addr addr; + union all_addr addr; #ifdef HAVE_IPSET char **ipsets_cur; #else (void)ipsets; /* unused */ #endif - + cache_start_insert(); /* find_soa is needed for dns_doctor and logging side-effects, so don't call it lazily if there are any. */ @@ -614,15 +610,15 @@ int extract_addresses(struct dns_header *header, size_t qlen, char *name, time_t #ifdef HAVE_DNSSEC if (option_bool(OPT_DNSSEC_VALID)) for (i = 0; i < ntohs(header->ancount); i++) - if (daemon->rr_status[i]) + if (daemon->rr_status[i] != 0) return 0; #endif } } - + /* go through the questions. */ p = (unsigned char *)(header+1); - + for (i = ntohs(header->qdcount); i != 0; i--) { int found = 0, cname_count = CNAME_CHAIN; @@ -636,19 +632,19 @@ int extract_addresses(struct dns_header *header, size_t qlen, char *name, time_t namep = p; if (!extract_name(header, qlen, &p, name, 1, 4)) return 0; /* bad packet */ - - GETSHORT(qtype, p); + + GETSHORT(qtype, p); GETSHORT(qclass, p); - + if (qclass != C_IN) continue; - /* PTRs: we chase CNAMEs here, since we have no way to + /* PTRs: we chase CNAMEs here, since we have no way to represent them in the cache. */ if (qtype == T_PTR) - { + { int name_encoding = in_arpa_name_2_addr(name, &addr); - + if (!name_encoding) continue; @@ -657,8 +653,8 @@ int extract_addresses(struct dns_header *header, size_t qlen, char *name, time_t cname_loop: if (!(p1 = skip_questions(header, qlen))) return 0; - - for (j = 0; j < ntohs(header->ancount); j++) + + for (j = 0; j < ntohs(header->ancount); j++) { int secflag = 0; unsigned char *tmp = namep; @@ -666,8 +662,8 @@ int extract_addresses(struct dns_header *header, size_t qlen, char *name, time_t if (!extract_name(header, qlen, &tmp, name, 1, 0) || !(res = extract_name(header, qlen, &p1, name, 0, 10))) return 0; /* bad packet */ - - GETSHORT(aqtype, p1); + + GETSHORT(aqtype, p1); GETSHORT(aqclass, p1); GETLONG(attl, p1); if ((daemon->max_ttl != 0) && (attl > daemon->max_ttl) && !is_sign) @@ -677,7 +673,7 @@ int extract_addresses(struct dns_header *header, size_t qlen, char *name, time_t } GETSHORT(ardlen, p1); endrr = p1+ardlen; - + /* TTL of record is minimum of CNAMES and PTR */ if (attl < cttl) cttl = attl; @@ -687,13 +683,16 @@ int extract_addresses(struct dns_header *header, size_t qlen, char *name, time_t if (!extract_name(header, qlen, &p1, name, 1, 0)) return 0; #ifdef HAVE_DNSSEC - if (option_bool(OPT_DNSSEC_VALID) && daemon->rr_status[j]) + if (option_bool(OPT_DNSSEC_VALID) && daemon->rr_status[j] != 0) { /* validated RR anywhere in CNAME chain, don't cache. */ if (cname_short || aqtype == T_CNAME) return 0; secflag = F_DNSSECOK; + /* limit TTL based on signature. */ + if (daemon->rr_status[j] < cttl) + cttl = daemon->rr_status[j]; } #endif @@ -706,17 +705,17 @@ int extract_addresses(struct dns_header *header, size_t qlen, char *name, time_t #endif goto cname_loop; } - - cache_insert(name, &addr, now, cttl, name_encoding | secflag | F_REVERSE); - found = 1; + + cache_insert(name, &addr, C_IN, now, cttl, name_encoding | secflag | F_REVERSE); + found = 1; } - + p1 = endrr; if (!CHECK_LEN(header, p1, qlen, 0)) return 0; /* bad packet */ } } - + if (!found && !option_bool(OPT_NO_NEG)) { if (!searched_soa) @@ -725,42 +724,42 @@ int extract_addresses(struct dns_header *header, size_t qlen, char *name, time_t ttl = find_soa(header, qlen, NULL, doctored); } if (ttl) - cache_insert(NULL, &addr, now, ttl, name_encoding | F_REVERSE | F_NEG | flags | (secure ? F_DNSSECOK : 0)); + cache_insert(NULL, &addr, C_IN, now, ttl, name_encoding | F_REVERSE | F_NEG | flags | (secure ? F_DNSSECOK : 0)); } } else { /* everything other than PTR */ struct crec *newc; - int addrlen; + int addrlen = 0; if (qtype == T_A) { addrlen = INADDRSZ; flags |= F_IPV4; } -#ifdef HAVE_IPV6 else if (qtype == T_AAAA) { addrlen = IN6ADDRSZ; flags |= F_IPV6; } -#endif + else if (qtype == T_SRV) + flags |= F_SRV; else continue; - + cname_loop1: if (!(p1 = skip_questions(header, qlen))) return 0; - - for (j = 0; j < ntohs(header->ancount); j++) + + for (j = 0; j < ntohs(header->ancount); j++) { int secflag = 0; - + if (!(res = extract_name(header, qlen, &p1, name, 0, 10))) return 0; /* bad packet */ - - GETSHORT(aqtype, p1); + + GETSHORT(aqtype, p1); GETSHORT(aqclass, p1); GETLONG(attl, p1); if ((daemon->max_ttl != 0) && (attl > daemon->max_ttl) && !is_sign) @@ -770,23 +769,36 @@ int extract_addresses(struct dns_header *header, size_t qlen, char *name, time_t } GETSHORT(ardlen, p1); endrr = p1+ardlen; - + if (aqclass == C_IN && res != 2 && (aqtype == T_CNAME || aqtype == qtype)) { #ifdef HAVE_DNSSEC - if (option_bool(OPT_DNSSEC_VALID) && daemon->rr_status[j]) + if (option_bool(OPT_DNSSEC_VALID) && daemon->rr_status[j] != 0) + { secflag = F_DNSSECOK; -#endif + + /* limit TTl based on sig. */ + if (daemon->rr_status[j] < attl) + attl = daemon->rr_status[j]; + } +#endif + // ****************************** Pi-hole modification ****************************** + if(FTL_CNAME(name, cpp, daemon->log_display_id)) + { + // This query is to be blocked as we found a blocked domain while walking + // the CNAME path. + return 2; + } + // ********************************************************************************** if (aqtype == T_CNAME) { if (!cname_count--) return 0; /* looped CNAMES */ - newc = cache_insert(name, NULL, now, attl, F_CNAME | F_FORWARD | secflag); - if (newc) + + if ((newc = cache_insert(name, NULL, C_IN, now, attl, F_CNAME | F_FORWARD | secflag))) { newc->addr.cname.target.cache = NULL; - /* anything other than zero, to avoid being mistaken for CNAME to interface-name */ - newc->addr.cname.uid = 1; + newc->addr.cname.is_name_ptr = 0; if (cpp) { next_uid(newc); @@ -794,56 +806,92 @@ int extract_addresses(struct dns_header *header, size_t qlen, char *name, time_t cpp->addr.cname.uid = newc->uid; } } - + cpp = newc; if (attl < cttl) cttl = attl; - + + namep = p1; if (!extract_name(header, qlen, &p1, name, 1, 0)) return 0; + goto cname_loop1; } else if (!(flags & F_NXDOMAIN)) { found = 1; - - /* copy address into aligned storage */ - if (!CHECK_LEN(header, p1, qlen, addrlen)) - return 0; /* bad packet */ - memcpy(&addr, p1, addrlen); - - /* check for returned address in private space */ - if (check_rebind) + + if (flags & F_SRV) { - if ((flags & F_IPV4) && - private_net(addr.addr.addr4, !option_bool(OPT_LOCAL_REBIND))) - return 1; - -#ifdef HAVE_IPV6 - if ((flags & F_IPV6) && - IN6_IS_ADDR_V4MAPPED(&addr.addr.addr6)) + unsigned char *tmp = namep; + + if (!CHECK_LEN(header, p1, qlen, 6)) + return 0; /* bad packet */ + GETSHORT(addr.srv.priority, p1); + GETSHORT(addr.srv.weight, p1); + GETSHORT(addr.srv.srvport, p1); + if (!extract_name(header, qlen, &p1, name, 1, 0)) + return 0; + addr.srv.targetlen = strlen(name) + 1; /* include terminating zero */ + if (!(addr.srv.target = blockdata_alloc(name, addr.srv.targetlen))) + return 0; + + /* we overwrote the original name, so get it back here. */ + if (!extract_name(header, qlen, &tmp, name, 1, 0)) + return 0; + } + else + { + /* copy address into aligned storage */ + if (!CHECK_LEN(header, p1, qlen, addrlen)) + return 0; /* bad packet */ + memcpy(&addr, p1, addrlen); + + /* check for returned address in private space */ + if (check_rebind) { - struct in_addr v4; - v4.s_addr = ((const uint32_t *) (&addr.addr.addr6))[3]; - if (private_net(v4, !option_bool(OPT_LOCAL_REBIND))) + if ((flags & F_IPV4) && + private_net(addr.addr4, !option_bool(OPT_LOCAL_REBIND))) return 1; + + /* Block IPv4-mapped IPv6 addresses in private IPv4 address space */ + if (flags & F_IPV6) + { + if (IN6_IS_ADDR_V4MAPPED(&addr.addr6)) + { + struct in_addr v4; + v4.s_addr = ((const uint32_t *) (&addr.addr6))[3]; + if (private_net(v4, !option_bool(OPT_LOCAL_REBIND))) + return 1; + } + + /* Check for link-local (LL) and site-local (ULA) IPv6 addresses */ + if (IN6_IS_ADDR_LINKLOCAL(&addr.addr6) || + IN6_IS_ADDR_SITELOCAL(&addr.addr6)) + return 1; + + /* Check for the IPv6 loopback address (::1) when + option rebind-localhost-ok is NOT set */ + if (!option_bool(OPT_LOCAL_REBIND) && + IN6_IS_ADDR_LOOPBACK(&addr.addr6)) + return 1; + } } -#endif - } #ifdef HAVE_IPSET - if (ipsets && (flags & (F_IPV4 | F_IPV6))) - { - ipsets_cur = ipsets; - while (*ipsets_cur) + if (ipsets && (flags & (F_IPV4 | F_IPV6))) { - log_query((flags & (F_IPV4 | F_IPV6)) | F_IPSET, name, &addr, *ipsets_cur); - add_to_ipset(*ipsets_cur++, &addr, flags, 0); + ipsets_cur = ipsets; + while (*ipsets_cur) + { + log_query((flags & (F_IPV4 | F_IPV6)) | F_IPSET, name, &addr, *ipsets_cur); + add_to_ipset(*ipsets_cur++, &addr, flags, 0); + } } - } #endif - - newc = cache_insert(name, &addr, now, attl, flags | F_FORWARD | secflag); + } + + newc = cache_insert(name, &addr, C_IN, now, attl, flags | F_FORWARD | secflag); if (newc && cpp) { next_uid(newc); @@ -853,12 +901,12 @@ int extract_addresses(struct dns_header *header, size_t qlen, char *name, time_t cpp = NULL; } } - + p1 = endrr; if (!CHECK_LEN(header, p1, qlen, 0)) return 0; /* bad packet */ } - + if (!found && !option_bool(OPT_NO_NEG)) { if (!searched_soa) @@ -866,11 +914,11 @@ int extract_addresses(struct dns_header *header, size_t qlen, char *name, time_t searched_soa = 1; ttl = find_soa(header, qlen, NULL, doctored); } - /* If there's no SOA to get the TTL from, but there is a CNAME + /* If there's no SOA to get the TTL from, but there is a CNAME pointing at this, inherit its TTL */ if (ttl || cpp) { - newc = cache_insert(name, NULL, now, ttl ? ttl : cttl, F_FORWARD | F_NEG | flags | (secure ? F_DNSSECOK : 0)); + newc = cache_insert(name, NULL, C_IN, now, ttl ? ttl : cttl, F_FORWARD | F_NEG | flags | (secure ? F_DNSSECOK : 0)); if (newc && cpp) { next_uid(newc); @@ -881,12 +929,12 @@ int extract_addresses(struct dns_header *header, size_t qlen, char *name, time_t } } } - + /* Don't put stuff from a truncated packet into the cache. - Don't cache replies from non-recursive nameservers, since we may get a - reply containing a CNAME but not its target, even though the target + Don't cache replies from non-recursive nameservers, since we may get a + reply containing a CNAME but not its target, even though the target does exist. */ - if (!(header->hb3 & HB3_TC) && + if (!(header->hb3 & HB3_TC) && !(header->hb4 & HB4_CD) && (header->hb4 & HB4_RA) && !no_cache_dnssec) @@ -907,11 +955,11 @@ unsigned int extract_request(struct dns_header *header, size_t qlen, char *name, if (ntohs(header->qdcount) != 1 || OPCODE(header) != QUERY) return 0; /* must be exactly one query. */ - + if (!extract_name(header, qlen, &p, name, 1, 4)) return 0; /* bad packet */ - - GETSHORT(qtype, p); + + GETSHORT(qtype, p); GETSHORT(qclass, p); if (typep) @@ -927,21 +975,28 @@ unsigned int extract_request(struct dns_header *header, size_t qlen, char *name, return F_IPV4 | F_IPV6; } + /* F_DNSSECOK as agument to search_servers() inhibits forwarding + to servers for domains without a trust anchor. This make the + behaviour for DS and DNSKEY queries we forward the same + as for DS and DNSKEY queries we originate. */ + if (qtype == T_DS || qtype == T_DNSKEY) + return F_DNSSECOK; + return F_QUERY; } size_t setup_reply(struct dns_header *header, size_t qlen, - struct all_addr *addrp, unsigned int flags, unsigned long ttl) + union all_addr *addrp, unsigned int flags, unsigned long ttl) { unsigned char *p; - + if (!(p = skip_questions(header, qlen))) return 0; - + /* clear authoritative and truncated flags, set QR flag */ - header->hb3 = (header->hb3 & ~(HB3_AA | HB3_TC)) | HB3_QR; - /* set RA flag */ - header->hb4 |= HB4_RA; + header->hb3 = (header->hb3 & ~(HB3_AA | HB3_TC )) | HB3_QR; + /* clear AD flag, set RA flag */ + header->hb4 = (header->hb4 & ~HB4_AD) | HB4_RA; header->nscount = htons(0); header->arcount = htons(0); @@ -952,8 +1007,8 @@ size_t setup_reply(struct dns_header *header, size_t qlen, SET_RCODE(header, NXDOMAIN); else if (flags == F_SERVFAIL) { - struct all_addr a; - a.addr.rcode.rcode = SERVFAIL; + union all_addr a; + a.log.rcode = SERVFAIL; log_query(F_CONFIG | F_RCODE, "error", &a, NULL); SET_RCODE(header, SERVFAIL); } @@ -966,8 +1021,7 @@ size_t setup_reply(struct dns_header *header, size_t qlen, header->hb3 |= HB3_AA; add_resource_record(header, NULL, NULL, sizeof(struct dns_header), &p, ttl, NULL, T_A, C_IN, "4", addrp); } - -#ifdef HAVE_IPV6 + if (flags & F_IPV6) { SET_RCODE(header, NOERROR); @@ -975,16 +1029,15 @@ size_t setup_reply(struct dns_header *header, size_t qlen, header->hb3 |= HB3_AA; add_resource_record(header, NULL, NULL, sizeof(struct dns_header), &p, ttl, NULL, T_AAAA, C_IN, "6", addrp); } -#endif } else /* nowhere to forward to */ { - struct all_addr a; - a.addr.rcode.rcode = REFUSED; + union all_addr a; + a.log.rcode = REFUSED; log_query(F_CONFIG | F_RCODE, "error", &a, NULL); SET_RCODE(header, REFUSED); } - + return p - (unsigned char *)header; } @@ -1026,7 +1079,7 @@ int check_for_local_domain(char *name, time_t now) /* Is the packet a reply with the answer address equal to addr? If so mung is into an NXDOMAIN reply and also put that information in the cache. */ -int check_for_bogus_wildcard(struct dns_header *header, size_t qlen, char *name, +int check_for_bogus_wildcard(struct dns_header *header, size_t qlen, char *name, struct bogus_addr *baddr, time_t now) { unsigned char *p; @@ -1042,34 +1095,34 @@ int check_for_bogus_wildcard(struct dns_header *header, size_t qlen, char *name, { if (!extract_name(header, qlen, &p, name, 1, 10)) return 0; /* bad packet */ - - GETSHORT(qtype, p); + + GETSHORT(qtype, p); GETSHORT(qclass, p); GETLONG(ttl, p); GETSHORT(rdlen, p); - + if (qclass == C_IN && qtype == T_A) { if (!CHECK_LEN(header, p, qlen, INADDRSZ)) return 0; - + for (baddrp = baddr; baddrp; baddrp = baddrp->next) if (memcmp(&baddrp->addr, p, INADDRSZ) == 0) { /* Found a bogus address. Insert that info here, since there no SOA record to get the ttl from in the normal processing */ cache_start_insert(); - cache_insert(name, NULL, now, ttl, F_IPV4 | F_FORWARD | F_NEG | F_NXDOMAIN); + cache_insert(name, NULL, C_IN, now, ttl, F_IPV4 | F_FORWARD | F_NEG | F_NXDOMAIN); cache_end_insert(); - + return 1; } } - + if (!ADD_RDLEN(header, p, qlen, rdlen)) return 0; } - + return 0; } @@ -1087,31 +1140,31 @@ int check_for_ignored_address(struct dns_header *header, size_t qlen, struct bog { if (!(p = skip_name(p, header, qlen, 10))) return 0; /* bad packet */ - - GETSHORT(qtype, p); + + GETSHORT(qtype, p); GETSHORT(qclass, p); p += 4; /* TTL */ GETSHORT(rdlen, p); - + if (qclass == C_IN && qtype == T_A) { if (!CHECK_LEN(header, p, qlen, INADDRSZ)) return 0; - + for (baddrp = baddr; baddrp; baddrp = baddrp->next) if (memcmp(&baddrp->addr, p, INADDRSZ) == 0) return 1; } - + if (!ADD_RDLEN(header, p, qlen, rdlen)) return 0; } - + return 0; } -int add_resource_record(struct dns_header *header, char *limit, int *truncp, int nameoffset, unsigned char **pp, +int add_resource_record(struct dns_header *header, char *limit, int *truncp, int nameoffset, unsigned char **pp, unsigned long ttl, int *offset, unsigned short type, unsigned short class, char *format, ...) { va_list ap; @@ -1120,15 +1173,15 @@ int add_resource_record(struct dns_header *header, char *limit, int *truncp, int unsigned short usval; long lval; char *sval; - + #define CHECK_LIMIT(size) \ if (limit && p + (size) > (unsigned char*)limit) goto truncated; va_start(ap, format); /* make ap point to 1st unamed argument */ - + if (truncp && *truncp) goto truncated; - + if (nameoffset > 0) { CHECK_LIMIT(2); @@ -1139,7 +1192,7 @@ int add_resource_record(struct dns_header *header, char *limit, int *truncp, int char *name = va_arg(ap, char *); if (name && !(p = do_rfc1035_name(p, name, limit))) goto truncated; - + if (nameoffset < 0) { CHECK_LIMIT(2); @@ -1154,7 +1207,7 @@ int add_resource_record(struct dns_header *header, char *limit, int *truncp, int /* type (2) + class (2) + ttl (4) + rdlen (2) */ CHECK_LIMIT(10); - + PUTSHORT(type, p); PUTSHORT(class, p); PUTLONG(ttl, p); /* TTL */ @@ -1165,40 +1218,38 @@ int add_resource_record(struct dns_header *header, char *limit, int *truncp, int for (; *format; format++) switch (*format) { -#ifdef HAVE_IPV6 case '6': CHECK_LIMIT(IN6ADDRSZ); - sval = va_arg(ap, char *); + sval = va_arg(ap, char *); memcpy(p, sval, IN6ADDRSZ); p += IN6ADDRSZ; break; -#endif - + case '4': CHECK_LIMIT(INADDRSZ); - sval = va_arg(ap, char *); + sval = va_arg(ap, char *); memcpy(p, sval, INADDRSZ); p += INADDRSZ; break; - + case 'b': CHECK_LIMIT(1); usval = va_arg(ap, int); *p++ = usval; break; - + case 's': CHECK_LIMIT(2); usval = va_arg(ap, int); PUTSHORT(usval, p); break; - + case 'l': CHECK_LIMIT(4); lval = va_arg(ap, long); PUTLONG(lval, p); break; - + case 'd': /* get domain-name answer arg and store it in RDATA field */ if (offset) @@ -1208,7 +1259,7 @@ int add_resource_record(struct dns_header *header, char *limit, int *truncp, int CHECK_LIMIT(1); *p++ = 0; break; - + case 't': usval = va_arg(ap, int); CHECK_LIMIT(usval); @@ -1231,14 +1282,14 @@ int add_resource_record(struct dns_header *header, char *limit, int *truncp, int } va_end(ap); /* clean up variable argument pointer */ - + /* Now, store real RDLength. sav already checked against limit. */ j = p - sav - 2; PUTSHORT(j, sav); - + *pp = p; return 1; - + truncated: va_end(ap); if (truncp) @@ -1256,14 +1307,14 @@ static unsigned long crec_ttl(struct crec *crecp, time_t now) if (crecp->flags & F_DHCP) { int conf_ttl = daemon->use_dhcp_ttl ? daemon->dhcp_ttl : daemon->local_ttl; - + /* Apply ceiling of actual lease length to configured TTL. */ if (!(crecp->flags & F_IMMORTAL) && (crecp->ttd - now) < conf_ttl) return crecp->ttd - now; - + return conf_ttl; - } - + } + /* Immortal entries other than DHCP are local, and hold TTL in TTD field. */ if (crecp->flags & F_IMMORTAL) return crecp->ttd; @@ -1275,28 +1326,32 @@ static unsigned long crec_ttl(struct crec *crecp, time_t now) return daemon->max_ttl; } +static int cache_validated(const struct crec *crecp) +{ + return (option_bool(OPT_DNSSEC_VALID) && !(crecp->flags & F_DNSSECOK)); +} /* return zero if we can't answer from cache, or packet size if we can */ -size_t answer_request(struct dns_header *header, char *limit, size_t qlen, - struct in_addr local_addr, struct in_addr local_netmask, - time_t now, int ad_reqd, int do_bit, int have_pseudoheader) +size_t answer_request(struct dns_header *header, char *limit, size_t qlen, + struct in_addr local_addr, struct in_addr local_netmask, + time_t now, int ad_reqd, int do_bit, int have_pseudoheader) { char *name = daemon->namebuff; unsigned char *p, *ansp; unsigned int qtype, qclass; - struct all_addr addr; + union all_addr addr; int nameoffset; unsigned short flag; int q, ans, anscount = 0, addncount = 0; int dryrun = 0; struct crec *crecp; - int nxdomain = 0, auth = 1, trunc = 0, sec_data = 1; + int nxdomain = 0, notimp = 0, auth = 1, trunc = 0, sec_data = 1; struct mx_srv_record *rec; size_t len; + int rd_bit = (header->hb3 & HB3_RD); /* never answer queries with RD unset, to avoid cache snooping. */ - if (!(header->hb3 & HB3_RD) || - ntohs(header->ancount) != 0 || + if (ntohs(header->ancount) != 0 || ntohs(header->nscount) != 0 || ntohs(header->qdcount) == 0 || OPCODE(header) != QUERY ) @@ -1305,7 +1360,7 @@ size_t answer_request(struct dns_header *header, char *limit, size_t qlen, /* Don't return AD set if checking disabled. */ if (header->hb4 & HB4_CD) sec_data = 0; - + /* If there is an additional data section then it will be overwritten by partial replies, so we have to do a dry run to see if we can answer the query. */ @@ -1314,29 +1369,59 @@ size_t answer_request(struct dns_header *header, char *limit, size_t qlen, for (rec = daemon->mxnames; rec; rec = rec->next) rec->offset = 0; - + rerun: /* determine end of question section (we put answers there) */ if (!(ansp = skip_questions(header, qlen))) return 0; /* bad packet */ - + /* now process each question, answers go in RRs after the question */ p = (unsigned char *)(header+1); for (q = ntohs(header->qdcount); q != 0; q--) { + int count = 255; /* catch loops */ + /* save pointer to name for copying into answers */ nameoffset = p - (unsigned char *)header; /* now extract name as .-concatenated string into name */ if (!extract_name(header, qlen, &p, name, 1, 4)) return 0; /* bad packet */ - - GETSHORT(qtype, p); + + GETSHORT(qtype, p); GETSHORT(qclass, p); ans = 0; /* have we answered this question */ + while (--count != 0 && (crecp = cache_find_by_name(NULL, name, now, F_CNAME))) + { + char *cname_target = cache_get_cname_target(crecp); + + /* If the client asked for DNSSEC don't use cached data. */ + if ((crecp->flags & (F_HOSTS | F_DHCP | F_CONFIG)) || + (rd_bit && (!do_bit || cache_validated(crecp)))) + { + if (crecp->flags & F_CONFIG || qtype == T_CNAME) + ans = 1; + + if (!(crecp->flags & F_DNSSECOK)) + sec_data = 0; + + if (!dryrun) + { + log_query(crecp->flags, name, NULL, record_source(crecp->uid)); + if (add_resource_record(header, limit, &trunc, nameoffset, &ansp, + crec_ttl(crecp, now), &nameoffset, + T_CNAME, C_IN, "d", cname_target)) + anscount++; + } + + } + + strcpy(name, cname_target); + } + if (qtype == T_TXT || qtype == T_ANY) { struct txt_record *t; @@ -1344,13 +1429,11 @@ size_t answer_request(struct dns_header *header, char *limit, size_t qlen, { if (t->class == qclass && hostname_isequal(name, t->name)) { - ans = 1; + ans = 1, sec_data = 0; if (!dryrun) { unsigned long ttl = daemon->local_ttl; int ok = 1; - log_query(F_CONFIG | F_RRNAME, name, NULL, ""); - FTL_cache(F_CONFIG | F_RRNAME, name, NULL, "", daemon->log_display_id); #ifndef NO_ID /* Dynamically generate stat record */ if (t->stat != 0) @@ -1360,12 +1443,35 @@ size_t answer_request(struct dns_header *header, char *limit, size_t qlen, ok = 0; } #endif - if (ok && add_resource_record(header, limit, &trunc, nameoffset, &ansp, - ttl, NULL, - T_TXT, t->class, "t", t->len, t->txt)) - anscount++; + if (ok) + { + log_query(F_CONFIG | F_RRNAME, name, NULL, ""); + FTL_cache(F_CONFIG | F_RRNAME, name, NULL, "", daemon->log_display_id); + if (add_resource_record(header, limit, &trunc, nameoffset, &ansp, + ttl, NULL, + T_TXT, t->class, "t", t->len, t->txt)) + anscount++; + } + } + } + } + } + if (qclass == C_CHAOS) + { + /* don't forward *.bind and *.server chaos queries - always reply with NOTIMP */ + if (hostname_issubdomain("bind", name) || hostname_issubdomain("server", name)) + { + if (!ans) + { + notimp = 1, auth = 0; + if (!dryrun) + { + addr.log.rcode = NOTIMP; + log_query(F_CONFIG | F_RCODE, name, &addr, NULL); + FTL_cache(F_CONFIG | F_RCODE, name, &addr, NULL, daemon->log_display_id); } + ans = 1, sec_data = 0; } } } @@ -1383,13 +1489,13 @@ size_t answer_request(struct dns_header *header, char *limit, size_t qlen, { log_query(F_CONFIG | F_RRNAME, name, NULL, querystr(NULL, t->class)); FTL_cache(F_CONFIG | F_RRNAME, name, NULL, querystr(NULL, t->class), daemon->log_display_id); - if (add_resource_record(header, limit, &trunc, nameoffset, &ansp, + if (add_resource_record(header, limit, &trunc, nameoffset, &ansp, daemon->local_ttl, NULL, t->class, C_IN, "t", t->len, t->txt)) anscount++; } } - + if (qtype == T_PTR || qtype == T_ANY) { /* see if it's w.z.y.z.in-addr.arpa format */ @@ -1405,35 +1511,33 @@ size_t answer_request(struct dns_header *header, char *limit, size_t qlen, for (intr = daemon->int_names; intr; intr = intr->next) { struct addrlist *addrlist; - + for (addrlist = intr->addr; addrlist; addrlist = addrlist->next) - if (!(addrlist->flags & ADDRLIST_IPV6) && addr.addr.addr4.s_addr == addrlist->addr.addr.addr4.s_addr) + if (!(addrlist->flags & ADDRLIST_IPV6) && addr.addr4.s_addr == addrlist->addr.addr4.s_addr) break; - + if (addrlist) break; else while (intr->next && strcmp(intr->intr, intr->next->intr) == 0) intr = intr->next; } -#ifdef HAVE_IPV6 else if (is_arpa == F_IPV6) for (intr = daemon->int_names; intr; intr = intr->next) { struct addrlist *addrlist; - + for (addrlist = intr->addr; addrlist; addrlist = addrlist->next) - if ((addrlist->flags & ADDRLIST_IPV6) && IN6_ARE_ADDR_EQUAL(&addr.addr.addr6, &addrlist->addr.addr.addr6)) + if ((addrlist->flags & ADDRLIST_IPV6) && IN6_ARE_ADDR_EQUAL(&addr.addr6, &addrlist->addr.addr6)) break; - + if (addrlist) break; else while (intr->next && strcmp(intr->intr, intr->next->intr) == 0) intr = intr->next; } -#endif - + if (intr) { sec_data = 0; @@ -1442,7 +1546,7 @@ size_t answer_request(struct dns_header *header, char *limit, size_t qlen, { log_query(is_arpa | F_REVERSE | F_CONFIG, intr->name, &addr, NULL); FTL_cache(is_arpa | F_REVERSE | F_CONFIG, intr->name, &addr, NULL, daemon->log_display_id); - if (add_resource_record(header, limit, &trunc, nameoffset, &ansp, + if (add_resource_record(header, limit, &trunc, nameoffset, &ansp, daemon->local_ttl, NULL, T_PTR, C_IN, "d", intr->name)) anscount++; @@ -1458,11 +1562,11 @@ size_t answer_request(struct dns_header *header, char *limit, size_t qlen, FTL_cache(F_CONFIG | F_RRNAME, name, NULL, "", daemon->log_display_id); for (ptr = daemon->ptr; ptr; ptr = ptr->next) if (hostname_isequal(name, ptr->name) && - add_resource_record(header, limit, &trunc, nameoffset, &ansp, + add_resource_record(header, limit, &trunc, nameoffset, &ansp, daemon->local_ttl, NULL, T_PTR, C_IN, "d", ptr->ptr)) anscount++; - + } } else if ((crecp = cache_find_by_addr(NULL, &addr, now, is_arpa))) @@ -1471,20 +1575,19 @@ size_t answer_request(struct dns_header *header, char *limit, size_t qlen, the zone is unsigned, which implies that we're doing validation. */ if ((crecp->flags & (F_HOSTS | F_DHCP | F_CONFIG)) || - !do_bit || - (option_bool(OPT_DNSSEC_VALID) && !(crecp->flags & F_DNSSECOK))) + (rd_bit && (!do_bit || cache_validated(crecp)) )) { - do - { + do + { /* don't answer wildcard queries with data not from /etc/hosts or dhcp leases */ if (qtype == T_ANY && !(crecp->flags & (F_HOSTS | F_DHCP))) continue; - + if (!(crecp->flags & F_DNSSECOK)) sec_data = 0; - + ans = 1; - + if (crecp->flags & F_NEG) { auth = 0; @@ -1502,12 +1605,12 @@ size_t answer_request(struct dns_header *header, char *limit, size_t qlen, auth = 0; if (!dryrun) { - log_query(crecp->flags & ~F_FORWARD, cache_get_name(crecp), &addr, + log_query(crecp->flags & ~F_FORWARD, cache_get_name(crecp), &addr, record_source(crecp->uid)); FTL_cache(crecp->flags & ~F_FORWARD, cache_get_name(crecp), &addr, - record_source(crecp->uid), daemon->log_display_id); + record_source(crecp->uid), daemon->log_display_id); - if (add_resource_record(header, limit, &trunc, nameoffset, &ansp, + if (add_resource_record(header, limit, &trunc, nameoffset, &ansp, crec_ttl(crecp, now), NULL, T_PTR, C_IN, "d", cache_get_name(crecp))) anscount++; @@ -1522,20 +1625,18 @@ size_t answer_request(struct dns_header *header, char *limit, size_t qlen, sec_data = 0; if (!dryrun) { - log_query(F_CONFIG | F_REVERSE | is_arpa, name, &addr, NULL); + log_query(F_CONFIG | F_REVERSE | is_arpa, name, &addr, NULL); FTL_cache(F_CONFIG | F_REVERSE | is_arpa, name, &addr, NULL, daemon->log_display_id); - - if (add_resource_record(header, limit, &trunc, nameoffset, &ansp, + + if (add_resource_record(header, limit, &trunc, nameoffset, &ansp, daemon->local_ttl, NULL, T_PTR, C_IN, "d", name)) anscount++; } } else if (option_bool(OPT_BOGUSPRIV) && ( -#ifdef HAVE_IPV6 - (is_arpa == F_IPV6 && private_net6(&addr.addr.addr6)) || -#endif - (is_arpa == F_IPV4 && private_net(addr.addr.addr4, 1)))) + (is_arpa == F_IPV6 && private_net6(&addr.addr6)) || + (is_arpa == F_IPV4 && private_net(addr.addr4, 1)))) { struct server *serv; unsigned int namelen = strlen(name); @@ -1579,92 +1680,78 @@ size_t answer_request(struct dns_header *header, char *limit, size_t qlen, for (flag = F_IPV4; flag; flag = (flag == F_IPV4) ? F_IPV6 : 0) { - unsigned short type = T_A; + unsigned short type = (flag == F_IPV6) ? T_AAAA : T_A; struct interface_name *intr; - if (flag == F_IPV6) -#ifdef HAVE_IPV6 - type = T_AAAA; -#else - break; -#endif - if (qtype != type && qtype != T_ANY) continue; - + /* interface name stuff */ - intname_restart: for (intr = daemon->int_names; intr; intr = intr->next) if (hostname_isequal(name, intr->name)) break; - + if (intr) { struct addrlist *addrlist; int gotit = 0, localise = 0; enumerate_interfaces(0); - + /* See if a putative address is on the network from which we received the query, is so we'll filter other answers. */ if (local_addr.s_addr != 0 && option_bool(OPT_LOCALISE) && type == T_A) for (intr = daemon->int_names; intr; intr = intr->next) if (hostname_isequal(name, intr->name)) for (addrlist = intr->addr; addrlist; addrlist = addrlist->next) -#ifdef HAVE_IPV6 - if (!(addrlist->flags & ADDRLIST_IPV6)) -#endif - if (is_same_net(*((struct in_addr *)&addrlist->addr), local_addr, local_netmask)) - { - localise = 1; - break; - } - + if (!(addrlist->flags & ADDRLIST_IPV6) && + is_same_net(addrlist->addr.addr4, local_addr, local_netmask)) + { + localise = 1; + break; + } + for (intr = daemon->int_names; intr; intr = intr->next) if (hostname_isequal(name, intr->name)) { for (addrlist = intr->addr; addrlist; addrlist = addrlist->next) -#ifdef HAVE_IPV6 if (((addrlist->flags & ADDRLIST_IPV6) ? T_AAAA : T_A) == type) -#endif { - if (localise && - !is_same_net(*((struct in_addr *)&addrlist->addr), local_addr, local_netmask)) + if (localise && + !is_same_net(addrlist->addr.addr4, local_addr, local_netmask)) continue; -#ifdef HAVE_IPV6 if (addrlist->flags & ADDRLIST_REVONLY) continue; -#endif - ans = 1; + + ans = 1; sec_data = 0; if (!dryrun) { gotit = 1; log_query(F_FORWARD | F_CONFIG | flag, name, &addrlist->addr, NULL); FTL_cache(F_FORWARD | F_CONFIG | flag, name, &addrlist->addr, NULL, daemon->log_display_id); - if (add_resource_record(header, limit, &trunc, nameoffset, &ansp, - daemon->local_ttl, NULL, type, C_IN, + if (add_resource_record(header, limit, &trunc, nameoffset, &ansp, + daemon->local_ttl, NULL, type, C_IN, type == T_A ? "4" : "6", &addrlist->addr)) anscount++; } } } - + if (!dryrun && !gotit) { log_query(F_FORWARD | F_CONFIG | flag | F_NEG, name, NULL, NULL); FTL_cache(F_FORWARD | F_CONFIG | flag | F_NEG, name, NULL, NULL, daemon->log_display_id); } - + continue; } - cname_restart: - if ((crecp = cache_find_by_name(NULL, name, now, flag | F_CNAME | (dryrun ? F_NO_RR : 0)))) + if ((crecp = cache_find_by_name(NULL, name, now, flag | (dryrun ? F_NO_RR : 0)))) { int localise = 0; - + /* See if a putative address is on the network from which we received the query, is so we'll filter other answers. */ if (local_addr.s_addr != 0 && option_bool(OPT_LOCALISE) && flag == F_IPV4) @@ -1672,51 +1759,28 @@ size_t answer_request(struct dns_header *header, char *limit, size_t qlen, struct crec *save = crecp; do { if ((crecp->flags & F_HOSTS) && - is_same_net(*((struct in_addr *)&crecp->addr), local_addr, local_netmask)) + is_same_net(crecp->addr.addr4, local_addr, local_netmask)) { localise = 1; break; - } - } while ((crecp = cache_find_by_name(crecp, name, now, flag | F_CNAME))); + } + } while ((crecp = cache_find_by_name(crecp, name, now, flag))); crecp = save; } /* If the client asked for DNSSEC don't use cached data. */ if ((crecp->flags & (F_HOSTS | F_DHCP | F_CONFIG)) || - !do_bit || - (option_bool(OPT_DNSSEC_VALID) && !(crecp->flags & F_DNSSECOK))) + (rd_bit && (!do_bit || cache_validated(crecp)) )) do - { + { /* don't answer wildcard queries with data not from /etc/hosts or DHCP leases */ if (qtype == T_ANY && !(crecp->flags & (F_HOSTS | F_DHCP | F_CONFIG))) break; - + if (!(crecp->flags & F_DNSSECOK)) sec_data = 0; - - if (crecp->flags & F_CNAME) - { - char *cname_target = cache_get_cname_target(crecp); - - if (!dryrun) - { - log_query(crecp->flags, name, NULL, record_source(crecp->uid)); - FTL_cache(crecp->flags, name, NULL, record_source(crecp->uid), daemon->log_display_id); - if (add_resource_record(header, limit, &trunc, nameoffset, &ansp, - crec_ttl(crecp, now), &nameoffset, - T_CNAME, C_IN, "d", cname_target)) - anscount++; - } - - strcpy(name, cname_target); - /* check if target interface_name */ - if (crecp->addr.cname.uid == SRC_INTERFACE) - goto intname_restart; - else - goto cname_restart; - } - + if (crecp->flags & F_NEG) { ans = 1; @@ -1731,110 +1795,89 @@ size_t answer_request(struct dns_header *header, char *limit, size_t qlen, FTL_cache(crecp->flags, name, NULL, record_source(crecp->uid), daemon->log_display_id); } } - else + else { /* If we are returning local answers depending on network, filter here. */ - if (localise && + if (localise && (crecp->flags & F_HOSTS) && - !is_same_net(*((struct in_addr *)&crecp->addr), local_addr, local_netmask)) + !is_same_net(crecp->addr.addr4, local_addr, local_netmask)) continue; - + if (!(crecp->flags & (F_HOSTS | F_DHCP))) auth = 0; - + ans = 1; if (!dryrun) { - log_query(crecp->flags & ~F_REVERSE, name, &crecp->addr.addr, + log_query(crecp->flags & ~F_REVERSE, name, &crecp->addr, record_source(crecp->uid)); - FTL_cache(crecp->flags & ~F_REVERSE, name, &crecp->addr.addr, - record_source(crecp->uid), - daemon->log_display_id); - - if (add_resource_record(header, limit, &trunc, nameoffset, &ansp, - crec_ttl(crecp, now), NULL, type, C_IN, + FTL_cache(crecp->flags & ~F_REVERSE, name, &crecp->addr, + record_source(crecp->uid), + daemon->log_display_id); + + if (add_resource_record(header, limit, &trunc, nameoffset, &ansp, + crec_ttl(crecp, now), NULL, type, C_IN, type == T_A ? "4" : "6", &crecp->addr)) anscount++; } } - } while ((crecp = cache_find_by_name(crecp, name, now, flag | F_CNAME))); + } while ((crecp = cache_find_by_name(crecp, name, now, flag))); } else if (is_name_synthetic(flag, name, &addr)) { - ans = 1; + ans = 1, sec_data = 0; if (!dryrun) { log_query(F_FORWARD | F_CONFIG | flag, name, &addr, NULL); FTL_cache(F_FORWARD | F_CONFIG | flag, name, &addr, NULL, daemon->log_display_id); - if (add_resource_record(header, limit, &trunc, nameoffset, &ansp, + if (add_resource_record(header, limit, &trunc, nameoffset, &ansp, daemon->local_ttl, NULL, type, C_IN, type == T_A ? "4" : "6", &addr)) anscount++; } } } - if (qtype == T_CNAME || qtype == T_ANY) - { - if ((crecp = cache_find_by_name(NULL, name, now, F_CNAME | (dryrun ? F_NO_RR : 0))) && - (qtype == T_CNAME || (crecp->flags & F_CONFIG)) && - ((crecp->flags & F_CONFIG) || !do_bit || (option_bool(OPT_DNSSEC_VALID) && !(crecp->flags & F_DNSSECOK)))) - { - if (!(crecp->flags & F_DNSSECOK)) - sec_data = 0; - - ans = 1; - if (!dryrun) - { - log_query(crecp->flags, name, NULL, record_source(crecp->uid)); - FTL_cache(crecp->flags, name, NULL, record_source(crecp->uid), - daemon->log_display_id); - if (add_resource_record(header, limit, &trunc, nameoffset, &ansp, - crec_ttl(crecp, now), &nameoffset, - T_CNAME, C_IN, "d", cache_get_cname_target(crecp))) - anscount++; - } - } - } - if (qtype == T_MX || qtype == T_ANY) { int found = 0; for (rec = daemon->mxnames; rec; rec = rec->next) if (!rec->issrv && hostname_isequal(name, rec->name)) { - ans = found = 1; - if (!dryrun) - { - int offset; - log_query(F_CONFIG | F_RRNAME, name, NULL, ""); - FTL_cache(F_CONFIG | F_RRNAME, name, NULL, "", daemon->log_display_id); - if (add_resource_record(header, limit, &trunc, nameoffset, &ansp, daemon->local_ttl, - &offset, T_MX, C_IN, "sd", rec->weight, rec->target)) - { - anscount++; - if (rec->target) - rec->offset = offset; - } - } + ans = found = 1; + sec_data = 0; + if (!dryrun) + { + int offset; + log_query(F_CONFIG | F_RRNAME, name, NULL, ""); + FTL_cache(F_CONFIG | F_RRNAME, name, NULL, "", daemon->log_display_id); + if (add_resource_record(header, limit, &trunc, nameoffset, &ansp, daemon->local_ttl, + &offset, T_MX, C_IN, "sd", rec->weight, rec->target)) + { + anscount++; + if (rec->target) + rec->offset = offset; + } + } } - + if (!found && (option_bool(OPT_SELFMX) || option_bool(OPT_LOCALMX)) && cache_find_by_name(NULL, name, now, F_HOSTS | F_DHCP | F_NO_RR)) - { + { ans = 1; + sec_data = 0; if (!dryrun) { log_query(F_CONFIG | F_RRNAME, name, NULL, ""); FTL_cache(F_CONFIG | F_RRNAME, name, NULL, "", daemon->log_display_id); - if (add_resource_record(header, limit, &trunc, nameoffset, &ansp, daemon->local_ttl, NULL, - T_MX, C_IN, "sd", 1, + if (add_resource_record(header, limit, &trunc, nameoffset, &ansp, daemon->local_ttl, NULL, + T_MX, C_IN, "sd", 1, option_bool(OPT_SELFMX) ? name : daemon->mxtarget)) anscount++; } } } - + if (qtype == T_SRV || qtype == T_ANY) { int found = 0; @@ -1844,13 +1887,14 @@ size_t answer_request(struct dns_header *header, char *limit, size_t qlen, if (rec->issrv && hostname_isequal(name, rec->name)) { found = ans = 1; + sec_data = 0; if (!dryrun) { int offset; log_query(F_CONFIG | F_RRNAME, name, NULL, ""); FTL_cache(F_CONFIG | F_RRNAME, name, NULL, "", daemon->log_display_id); - if (add_resource_record(header, limit, &trunc, nameoffset, &ansp, daemon->local_ttl, - &offset, T_SRV, C_IN, "sssd", + if (add_resource_record(header, limit, &trunc, nameoffset, &ansp, daemon->local_ttl, + &offset, T_SRV, C_IN, "sssd", rec->priority, rec->weight, rec->srvport, rec->target)) { anscount++; @@ -1858,7 +1902,7 @@ size_t answer_request(struct dns_header *header, char *limit, size_t qlen, rec->offset = offset; } } - + /* unlink first SRV record found */ if (!move) { @@ -1866,7 +1910,7 @@ size_t answer_request(struct dns_header *header, char *limit, size_t qlen, *up = rec->next; } else - up = &rec->next; + up = &rec->next; } else up = &rec->next; @@ -1878,9 +1922,44 @@ size_t answer_request(struct dns_header *header, char *limit, size_t qlen, move->next = NULL; } + if (!found) + { + if ((crecp = cache_find_by_name(NULL, name, now, F_SRV | (dryrun ? F_NO_RR : 0))) && + rd_bit && (!do_bit || (option_bool(OPT_DNSSEC_VALID) && !(crecp->flags & F_DNSSECOK)))) + { + if (!(crecp->flags & F_DNSSECOK)) + sec_data = 0; + + auth = 0; + found = ans = 1; + + do { + if (crecp->flags & F_NEG) + { + if (crecp->flags & F_NXDOMAIN) + nxdomain = 1; + if (!dryrun) + log_query(crecp->flags, name, NULL, NULL); + } + else if (!dryrun) + { + char *target = blockdata_retrieve(crecp->addr.srv.target, crecp->addr.srv.targetlen, NULL); + log_query(crecp->flags, name, NULL, 0); + + if (add_resource_record(header, limit, &trunc, nameoffset, &ansp, + crec_ttl(crecp, now), NULL, T_SRV, C_IN, "sssd", + crecp->addr.srv.priority, crecp->addr.srv.weight, crecp->addr.srv.srvport, + target)) + anscount++; + } + } while ((crecp = cache_find_by_name(crecp, name, now, F_SRV))); + } + } + if (!found && option_bool(OPT_FILTER) && (qtype == T_SRV || (qtype == T_ANY && strchr(name, '_')))) { ans = 1; + sec_data = 0; if (!dryrun) { log_query(F_CONFIG | F_NEG, name, NULL, NULL); @@ -1896,24 +1975,26 @@ size_t answer_request(struct dns_header *header, char *limit, size_t qlen, if (hostname_isequal(name, na->name)) { ans = 1; + sec_data = 0; if (!dryrun) { log_query(F_CONFIG | F_RRNAME, name, NULL, ""); FTL_cache(F_CONFIG | F_NEG, name, NULL, "", daemon->log_display_id); - if (add_resource_record(header, limit, &trunc, nameoffset, &ansp, daemon->local_ttl, - NULL, T_NAPTR, C_IN, "sszzzd", + if (add_resource_record(header, limit, &trunc, nameoffset, &ansp, daemon->local_ttl, + NULL, T_NAPTR, C_IN, "sszzzd", na->order, na->pref, na->flags, na->services, na->regexp, na->replace)) anscount++; } } } - + if (qtype == T_MAILB) - ans = 1, nxdomain = 1; + ans = 1, nxdomain = 1, sec_data = 0; if (qtype == T_SOA && option_bool(OPT_FILTER)) { ans = 1; + sec_data = 0; if (!dryrun) { log_query(F_CONFIG | F_NEG, name, &addr, NULL); @@ -1925,13 +2006,13 @@ size_t answer_request(struct dns_header *header, char *limit, size_t qlen, if (!ans) return 0; /* failed to answer a question */ } - + if (dryrun) { dryrun = 0; goto rerun; } - + /* create an additional data section, for stuff in SRV and MX record replies. */ for (rec = daemon->mxnames; rec; rec = rec->next) if (rec->offset != 0) @@ -1941,41 +2022,40 @@ size_t answer_request(struct dns_header *header, char *limit, size_t qlen, for (tmp = rec->next; tmp; tmp = tmp->next) if (tmp->offset != 0 && hostname_isequal(rec->target, tmp->target)) tmp->offset = 0; - + crecp = NULL; while ((crecp = cache_find_by_name(crecp, rec->target, now, F_IPV4 | F_IPV6))) { -#ifdef HAVE_IPV6 int type = crecp->flags & F_IPV4 ? T_A : T_AAAA; -#else - int type = T_A; -#endif + if (crecp->flags & F_NEG) continue; - if (add_resource_record(header, limit, NULL, rec->offset, &ansp, - crec_ttl(crecp, now), NULL, type, C_IN, + if (add_resource_record(header, limit, NULL, rec->offset, &ansp, + crec_ttl(crecp, now), NULL, type, C_IN, crecp->flags & F_IPV4 ? "4" : "6", &crecp->addr)) addncount++; } } - + /* done all questions, set up header and return length of result */ /* clear authoritative and truncated flags, set QR flag */ header->hb3 = (header->hb3 & ~(HB3_AA | HB3_TC)) | HB3_QR; /* set RA flag */ header->hb4 |= HB4_RA; - + /* authoritative - only hosts and DHCP derived names. */ if (auth) header->hb3 |= HB3_AA; - + /* truncation */ if (trunc) header->hb3 |= HB3_TC; - + if (nxdomain) SET_RCODE(header, NXDOMAIN); + else if (notimp) + SET_RCODE(header, NOTIMP); else SET_RCODE(header, NOERROR); /* no error */ header->ancount = htons(anscount); @@ -1983,15 +2063,15 @@ size_t answer_request(struct dns_header *header, char *limit, size_t qlen, header->arcount = htons(addncount); len = ansp - (unsigned char *)header; - + /* Advertise our packet size limit in our reply */ if (have_pseudoheader) len = add_pseudoheader(header, len, (unsigned char *)limit, daemon->edns_pktsz, 0, NULL, 0, do_bit, 0); - + if (ad_reqd && sec_data) header->hb4 |= HB4_AD; else header->hb4 &= ~HB4_AD; - + return len; } diff --git a/dnsmasq/rfc2131.c b/src/dnsmasq/rfc2131.c similarity index 90% rename from dnsmasq/rfc2131.c rename to src/dnsmasq/rfc2131.c index 1b081578e..fc54aab4b 100644 --- a/dnsmasq/rfc2131.c +++ b/src/dnsmasq/rfc2131.c @@ -1,15 +1,15 @@ -/* dnsmasq is Copyright (c) 2000-2018 Simon Kelley +/* dnsmasq is Copyright (c) 2000-2020 Simon Kelley This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; version 2 dated June, 1991, or (at your option) version 3 dated 29 June, 2007. - + This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. - + You should have received a copy of the GNU General Public License along with this program. If not, see . */ @@ -29,11 +29,11 @@ static int sanitise(unsigned char *opt, char *buf); static struct in_addr server_id(struct dhcp_context *context, struct in_addr override, struct in_addr fallback); static unsigned int calc_time(struct dhcp_context *context, struct dhcp_config *config, unsigned char *opt); static void option_put(struct dhcp_packet *mess, unsigned char *end, int opt, int len, unsigned int val); -static void option_put_string(struct dhcp_packet *mess, unsigned char *end, +static void option_put_string(struct dhcp_packet *mess, unsigned char *end, int opt, char *string, int null_term); static struct in_addr option_addr(unsigned char *opt); static unsigned int option_uint(unsigned char *opt, int offset, int size); -static void log_packet(char *type, void *addr, unsigned char *ext_mac, +static void log_packet(char *type, void *addr, unsigned char *ext_mac, int mac_len, char *interface, char *string, char *err, u32 xid); static unsigned char *option_find(struct dhcp_packet *mess, size_t size, int opt_type, int minsize); static unsigned char *option_find1(unsigned char *p, unsigned char *end, int opt, int minsize); @@ -44,10 +44,10 @@ static void do_options(struct dhcp_context *context, struct dhcp_packet *mess, unsigned char *end, unsigned char *req_options, - char *hostname, + char *hostname, char *domain, struct dhcp_netid *netid, - struct in_addr subnet_addr, + struct in_addr subnet_addr, unsigned char fqdn_flags, int null_term, int pxe_arch, unsigned char *uuid, @@ -57,7 +57,7 @@ static void do_options(struct dhcp_context *context, unsigned short fuzz); -static void match_vendor_opts(unsigned char *opt, struct dhcp_opt *dopt); +static void match_vendor_opts(unsigned char *opt, struct dhcp_opt *dopt); static int do_encap_opts(struct dhcp_opt *opt, int encap, int flag, struct dhcp_packet *mess, unsigned char *end, int null_term); static void pxe_misc(struct dhcp_packet *mess, unsigned char *end, unsigned char *uuid); static int prune_vendor_opts(struct dhcp_netid *netid); @@ -77,8 +77,8 @@ size_t dhcp_reply(struct dhcp_context *context, char *iface_name, int int_index, struct dhcp_netid_list *id_list; int clid_len = 0, ignore = 0, do_classes = 0, rapid_commit = 0, selecting = 0, pxearch = -1; struct dhcp_packet *mess = (struct dhcp_packet *)daemon->dhcp_packet.iov_base; - unsigned char *end = (unsigned char *)(mess + 1); - unsigned char *real_end = (unsigned char *)(mess + 1); + unsigned char *end = (unsigned char *)(mess + 1); + unsigned char *real_end = (unsigned char *)(mess + 1); char *hostname = NULL, *offer_hostname = NULL, *client_hostname = NULL, *domain = NULL; int hostname_auth = 0, borken_opt = 0; unsigned char *req_options = NULL; @@ -106,11 +106,11 @@ size_t dhcp_reply(struct dhcp_context *context, char *iface_name, int int_index, /* set tag with name == interface */ iface_id.net = iface_name; iface_id.next = NULL; - netid = &iface_id; - + netid = &iface_id; + if (mess->op != BOOTREQUEST || mess->hlen > DHCP_CHADDR_MAX) return 0; - + if (mess->htype == 0 && mess->hlen != 0) return 0; @@ -118,25 +118,25 @@ size_t dhcp_reply(struct dhcp_context *context, char *iface_name, int int_index, if ((opt = option_find(mess, sz, OPTION_MESSAGE_TYPE, 1))) { u32 cookie = htonl(DHCP_COOKIE); - + /* only insist on a cookie for DHCP. */ if (memcmp(mess->options, &cookie, sizeof(u32)) != 0) return 0; - + mess_type = option_uint(opt, 0, 1); - + /* two things to note here: expand_buf may move the packet, so reassign mess from daemon->packet. Also, the size sent includes the IP and UDP headers, hence the magic "-28" */ if ((opt = option_find(mess, sz, OPTION_MAXMESSAGE, 2))) { size_t size = (size_t)option_uint(opt, 0, 2) - 28; - + if (size > DHCP_PACKET_MAX) size = DHCP_PACKET_MAX; else if (size < sizeof(struct dhcp_packet)) size = sizeof(struct dhcp_packet); - + if (expand_buf(&daemon->dhcp_packet, size)) { mess = (struct dhcp_packet *)daemon->dhcp_packet.iov_base; @@ -153,7 +153,7 @@ size_t dhcp_reply(struct dhcp_context *context, char *iface_name, int int_index, if ((opt = option_find(mess, sz, OPTION_VENDOR_IDENT_OPT, 5))) { unsigned int elen, offset, len = option_len(opt); - + for (offset = 0; offset < (len - 5); offset += elen + 5) { elen = option_uint(opt, offset + 4 , 1); @@ -164,9 +164,9 @@ size_t dhcp_reply(struct dhcp_context *context, char *iface_name, int int_index, oui = option_find1(x, y, 1, 1); serial = option_find1(x, y, 2, 1); #ifdef HAVE_SCRIPT - class = option_find1(x, y, 3, 1); + class = option_find1(x, y, 3, 1); #endif - /* If TR069-id is present set the tag "cpewan-id" to facilitate echoing + /* If TR069-id is present set the tag "cpewan-id" to facilitate echoing the gateway id back. Note that the device class is optional */ if (oui && serial) { @@ -178,13 +178,13 @@ size_t dhcp_reply(struct dhcp_context *context, char *iface_name, int int_index, } } } - + if ((opt = option_find(mess, sz, OPTION_AGENT_ID, 1))) { /* Any agent-id needs to be copied back out, verbatim, as the last option in the packet. Here, we shift it to the very end of the buffer, if it doesn't get overwritten, then it will be shuffled back at the end of processing. - Note that the incoming options must not be overwritten here, so there has to + Note that the incoming options must not be overwritten here, so there has to be enough free space at the end of the packet to copy the option. */ unsigned char *sopt; unsigned int total = option_len(opt) + 2; @@ -204,19 +204,19 @@ size_t dhcp_reply(struct dhcp_context *context, char *iface_name, int int_index, /* look for RFC5107 server-identifier-override */ if ((sopt = option_find1(option_ptr(opt, 0), option_ptr(opt, option_len(opt)), SUBOPT_SERVER_OR, INADDRSZ))) override = option_addr(sopt); - - /* if a circuit-id or remote-is option is provided, exact-match to options. */ + + /* if a circuit-id or remote-is option is provided, exact-match to options. */ for (vendor = daemon->dhcp_vendors; vendor; vendor = vendor->next) { int search; - + if (vendor->match_type == MATCH_CIRCUIT) search = SUBOPT_CIRCUIT_ID; else if (vendor->match_type == MATCH_REMOTE) search = SUBOPT_REMOTE_ID; else if (vendor->match_type == MATCH_SUBSCRIBER) search = SUBOPT_SUBSCR_ID; - else + else continue; if ((sopt = option_find1(option_ptr(opt, 0), option_ptr(opt, option_len(opt)), search, 1)) && @@ -225,16 +225,16 @@ size_t dhcp_reply(struct dhcp_context *context, char *iface_name, int int_index, { vendor->netid.next = netid; netid = &vendor->netid; - } + } } } /* Check for RFC3011 subnet selector - only if RFC3527 one not present */ if (subnet_addr.s_addr == 0 && (opt = option_find(mess, sz, OPTION_SUBNET_SELECT, INADDRSZ))) subnet_addr = option_addr(opt); - + /* If there is no client identifier option, use the hardware address */ - if ((opt = option_find(mess, sz, OPTION_CLIENT_ID, 1))) + if (!option_bool(OPT_IGNORE_CLID) && (opt = option_find(mess, sz, OPTION_CLIENT_ID, 1))) { clid_len = option_len(opt); clid = option_ptr(opt, 0); @@ -243,7 +243,7 @@ size_t dhcp_reply(struct dhcp_context *context, char *iface_name, int int_index, /* do we have a lease in store? */ lease = lease_find_by_client(mess->chaddr, mess->hlen, mess->htype, clid, clid_len); - /* If this request is missing a clid, but we've seen one before, + /* If this request is missing a clid, but we've seen one before, use it again for option matching etc. */ if (lease && !clid && lease->clid) { @@ -254,7 +254,7 @@ size_t dhcp_reply(struct dhcp_context *context, char *iface_name, int int_index, /* find mac to use for logging and hashing */ emac = extended_hwaddr(mess->htype, mess->hlen, mess->chaddr, clid_len, clid, &emac_len); } - + for (mac = daemon->dhcp_macs; mac; mac = mac->next) if (mac->hwaddr_len == mess->hlen && (mac->hwaddr_type == mess->htype || mac->hwaddr_type == 0) && @@ -263,20 +263,21 @@ size_t dhcp_reply(struct dhcp_context *context, char *iface_name, int int_index, mac->netid.next = netid; netid = &mac->netid; } - - /* Determine network for this packet. Our caller will have already linked all the - contexts which match the addresses of the receiving interface but if the - machine has an address already, or came via a relay, or we have a subnet selector, - we search again. If we don't have have a giaddr or explicit subnet selector, - use the ciaddr. This is necessary because a machine which got a lease via a - relay won't use the relay to renew. If matching a ciaddr fails but we have a context + + /* Determine network for this packet. Our caller will have already linked all the + contexts which match the addresses of the receiving interface but if the + machine has an address already, or came via a relay, or we have a subnet selector, + we search again. If we don't have have a giaddr or explicit subnet selector, + use the ciaddr. This is necessary because a machine which got a lease via a + relay won't use the relay to renew. If matching a ciaddr fails but we have a context from the physical network, continue using that to allow correct DHCPNAK generation later. */ if (mess->giaddr.s_addr || subnet_addr.s_addr || mess->ciaddr.s_addr) { struct dhcp_context *context_tmp, *context_new = NULL; + struct shared_network *share = NULL; struct in_addr addr; - int force = 0; - + int force = 0, via_relay = 0; + if (subnet_addr.s_addr) { addr = subnet_addr; @@ -286,65 +287,89 @@ size_t dhcp_reply(struct dhcp_context *context, char *iface_name, int int_index, { addr = mess->giaddr; force = 1; + via_relay = 1; } else { /* If ciaddr is in the hardware derived set of contexts, leave that unchanged */ addr = mess->ciaddr; for (context_tmp = context; context_tmp; context_tmp = context_tmp->current) - if (context_tmp->netmask.s_addr && + if (context_tmp->netmask.s_addr && is_same_net(addr, context_tmp->start, context_tmp->netmask) && is_same_net(addr, context_tmp->end, context_tmp->netmask)) { context_new = context; break; } - } - + } + if (!context_new) - for (context_tmp = daemon->dhcp; context_tmp; context_tmp = context_tmp->next) - { - struct in_addr netmask = context_tmp->netmask; - - /* guess the netmask for relayed networks */ - if (!(context_tmp->flags & CONTEXT_NETMASK) && context_tmp->netmask.s_addr == 0) - { - if (IN_CLASSA(ntohl(context_tmp->start.s_addr)) && IN_CLASSA(ntohl(context_tmp->end.s_addr))) - netmask.s_addr = htonl(0xff000000); - else if (IN_CLASSB(ntohl(context_tmp->start.s_addr)) && IN_CLASSB(ntohl(context_tmp->end.s_addr))) - netmask.s_addr = htonl(0xffff0000); - else if (IN_CLASSC(ntohl(context_tmp->start.s_addr)) && IN_CLASSC(ntohl(context_tmp->end.s_addr))) - netmask.s_addr = htonl(0xffffff00); - } - - /* This section fills in context mainly when a client which is on a remote (relayed) - network renews a lease without using the relay, after dnsmasq has restarted. */ - if (netmask.s_addr != 0 && - is_same_net(addr, context_tmp->start, netmask) && - is_same_net(addr, context_tmp->end, netmask)) - { - context_tmp->netmask = netmask; - if (context_tmp->local.s_addr == 0) - context_tmp->local = fallback; - if (context_tmp->router.s_addr == 0) - context_tmp->router = mess->giaddr; - - /* fill in missing broadcast addresses for relayed ranges */ - if (!(context_tmp->flags & CONTEXT_BRDCAST) && context_tmp->broadcast.s_addr == 0 ) - context_tmp->broadcast.s_addr = context_tmp->start.s_addr | ~context_tmp->netmask.s_addr; - - context_tmp->current = context_new; - context_new = context_tmp; - } - } + { + for (context_tmp = daemon->dhcp; context_tmp; context_tmp = context_tmp->next) + { + struct in_addr netmask = context_tmp->netmask; + + /* guess the netmask for relayed networks */ + if (!(context_tmp->flags & CONTEXT_NETMASK) && context_tmp->netmask.s_addr == 0) + { + if (IN_CLASSA(ntohl(context_tmp->start.s_addr)) && IN_CLASSA(ntohl(context_tmp->end.s_addr))) + netmask.s_addr = htonl(0xff000000); + else if (IN_CLASSB(ntohl(context_tmp->start.s_addr)) && IN_CLASSB(ntohl(context_tmp->end.s_addr))) + netmask.s_addr = htonl(0xffff0000); + else if (IN_CLASSC(ntohl(context_tmp->start.s_addr)) && IN_CLASSC(ntohl(context_tmp->end.s_addr))) + netmask.s_addr = htonl(0xffffff00); + } + /* check to see is a context is OK because of a shared address on + the relayed subnet. */ + if (via_relay) + for (share = daemon->shared_networks; share; share = share->next) + { +#ifdef HAVE_DHCP6 + if (share->shared_addr.s_addr == 0) + continue; +#endif + if (share->if_index != 0 || + share->match_addr.s_addr != mess->giaddr.s_addr) + continue; + + if (netmask.s_addr != 0 && + is_same_net(share->shared_addr, context_tmp->start, netmask) && + is_same_net(share->shared_addr, context_tmp->end, netmask)) + break; + } + + /* This section fills in context mainly when a client which is on a remote (relayed) + network renews a lease without using the relay, after dnsmasq has restarted. */ + if (share || + (netmask.s_addr != 0 && + is_same_net(addr, context_tmp->start, netmask) && + is_same_net(addr, context_tmp->end, netmask))) + { + context_tmp->netmask = netmask; + if (context_tmp->local.s_addr == 0) + context_tmp->local = fallback; + if (context_tmp->router.s_addr == 0 && !share) + context_tmp->router = mess->giaddr; + + /* fill in missing broadcast addresses for relayed ranges */ + if (!(context_tmp->flags & CONTEXT_BRDCAST) && context_tmp->broadcast.s_addr == 0 ) + context_tmp->broadcast.s_addr = context_tmp->start.s_addr | ~context_tmp->netmask.s_addr; + + context_tmp->current = context_new; + context_new = context_tmp; + } + + } + } + if (context_new || force) - context = context_new; + context = context_new; } - + if (!context) { - my_syslog(MS_DHCP | LOG_WARNING, _("no address range available for DHCP request %s %s"), + my_syslog(MS_DHCP | LOG_WARNING, _("no address range available for DHCP request %s %s"), subnet_addr.s_addr ? _("with subnet selector") : _("via"), subnet_addr.s_addr ? inet_ntoa(subnet_addr) : (mess->giaddr.s_addr ? inet_ntoa(mess->giaddr) : iface_name)); return 0; @@ -360,14 +385,14 @@ size_t dhcp_reply(struct dhcp_context *context, char *iface_name, int int_index, my_syslog(MS_DHCP | LOG_INFO, _("%u available DHCP subnet: %s/%s"), ntohl(mess->xid), daemon->namebuff, inet_ntoa(context_tmp->netmask)); else - my_syslog(MS_DHCP | LOG_INFO, _("%u available DHCP range: %s -- %s"), + my_syslog(MS_DHCP | LOG_INFO, _("%u available DHCP range: %s -- %s"), ntohl(mess->xid), daemon->namebuff, inet_ntoa(context_tmp->end)); } } - + /* dhcp-match. If we have hex-and-wildcards, look for a left-anchored match. - Otherwise assume the option is an array, and look for a matching element. - If no data given, existence of the option is enough. This code handles + Otherwise assume the option is an array, and look for a matching element. + If no data given, existence of the option is enough. This code handles rfc3925 V-I classes too. */ for (o = daemon->dhcp_match; o; o = o->next) { @@ -378,7 +403,7 @@ size_t dhcp_reply(struct dhcp_context *context, char *iface_name, int int_index, { if (!(opt = option_find(mess, sz, OPTION_VENDOR_IDENT, 5))) continue; - + for (offset = 0; offset < (option_len(opt) - 5u); offset += len + 5) { len = option_uint(opt, offset + 4 , 1); @@ -386,23 +411,23 @@ size_t dhcp_reply(struct dhcp_context *context, char *iface_name, int int_index, if ((offset + len + 5 <= (unsigned)(option_len(opt))) && (option_uint(opt, offset, 4) == (unsigned int)o->u.encap)) for (o2 = offset + 5; o2 < offset + len + 5; o2 += elen + 1) - { + { elen = option_uint(opt, o2, 1); - if ((o2 + elen + 1 <= option_len(opt)) && + if ((o2 + elen + 1 <= (unsigned)option_len(opt)) && (match = match_bytes(o, option_ptr(opt, o2 + 1), elen))) break; } - if (match) + if (match) break; - } + } } else { if (!(opt = option_find(mess, sz, o->opt, 1))) continue; - + match = match_bytes(o, option_ptr(opt, 0), option_len(opt)); - } + } if (match) { @@ -410,13 +435,13 @@ size_t dhcp_reply(struct dhcp_context *context, char *iface_name, int int_index, netid = o->netid; } } - + /* user-class options are, according to RFC3004, supposed to contain a set of counted strings. Here we check that this is so (by seeing if the counts are consistent with the overall option length) and if - so zero the counts so that we don't get spurious matches between + so zero the counts so that we don't get spurious matches between the vendor string and the counts. If the lengths don't add up, we - assume that the option is a single string and non RFC3004 compliant + assume that the option is a single string and non RFC3004 compliant and just do the substring match. dhclient provides these broken options. The code, later, which sends user-class data to the lease-change script relies on the transformation done here. @@ -434,15 +459,15 @@ size_t dhcp_reply(struct dhcp_context *context, char *iface_name, int int_index, ucp[j] = 0; } } - + for (vendor = daemon->dhcp_vendors; vendor; vendor = vendor->next) { int mopt; - + if (vendor->match_type == MATCH_VENDOR) mopt = OPTION_VENDOR_ID; else if (vendor->match_type == MATCH_USER) - mopt = OPTION_USER_CLASS; + mopt = OPTION_USER_CLASS; else continue; @@ -467,7 +492,7 @@ size_t dhcp_reply(struct dhcp_context *context, char *iface_name, int int_index, vendor_class_len = option_len(opt); } match_vendor_opts(opt, daemon->dhcp_opts); - + if (option_bool(OPT_LOG_OPTS)) { if (sanitise(opt, daemon->namebuff)) @@ -477,9 +502,9 @@ size_t dhcp_reply(struct dhcp_context *context, char *iface_name, int int_index, } mess->op = BOOTREPLY; - - config = find_config(daemon->dhcp_conf, context, clid, clid_len, - mess->chaddr, mess->hlen, mess->htype, NULL); + + config = find_config(daemon->dhcp_conf, context, clid, clid_len, + mess->chaddr, mess->hlen, mess->htype, NULL, run_tag_if(netid)); /* set "known" tag for known hosts */ if (config) @@ -488,14 +513,14 @@ size_t dhcp_reply(struct dhcp_context *context, char *iface_name, int int_index, known_id.next = netid; netid = &known_id; } - else if (find_config(daemon->dhcp_conf, NULL, clid, clid_len, - mess->chaddr, mess->hlen, mess->htype, NULL)) + else if (find_config(daemon->dhcp_conf, NULL, clid, clid_len, + mess->chaddr, mess->hlen, mess->htype, NULL, run_tag_if(netid))) { known_id.net = "known-othernet"; known_id.next = netid; netid = &known_id; } - + if (mess_type == 0 && !pxe) { /* BOOTP request */ @@ -505,12 +530,12 @@ size_t dhcp_reply(struct dhcp_context *context, char *iface_name, int int_index, /* must have a MAC addr for bootp */ if (mess->htype == 0 || mess->hlen == 0 || (context->flags & CONTEXT_PROXY)) return 0; - + if (have_config(config, CONFIG_DISABLE)) message = _("disabled"); end = mess->options + 64; /* BOOTP vend area is only 64 bytes */ - + if (have_config(config, CONFIG_NAME)) { hostname = config->hostname; @@ -543,13 +568,13 @@ size_t dhcp_reply(struct dhcp_context *context, char *iface_name, int int_index, bootp_id.net = "bootp"; bootp_id.next = netid; netid = &bootp_id; - + tagif_netid = run_tag_if(netid); for (id_list = daemon->dhcp_ignore; id_list; id_list = id_list->next) if (match_netid(id_list->list, tagif_netid, 0)) message = _("ignored"); - + if (!message) { int nailed = 0; @@ -582,7 +607,7 @@ size_t dhcp_reply(struct dhcp_context *context, char *iface_name, int int_index, else mess->yiaddr = lease->addr; } - + if (!message && !(context = narrow_context(context, mess->yiaddr, netid))) message = _("wrong network"); else if (context->netid.net) @@ -592,7 +617,7 @@ size_t dhcp_reply(struct dhcp_context *context, char *iface_name, int int_index, } log_tags(tagif_netid, ntohl(mess->xid)); - + if (!message && !nailed) { for (id_list = daemon->bootp_dynamic; id_list; id_list = id_list->next) @@ -602,48 +627,48 @@ size_t dhcp_reply(struct dhcp_context *context, char *iface_name, int int_index, message = _("no address configured"); } - if (!message && - !lease && + if (!message && + !lease && (!(lease = lease4_allocate(mess->yiaddr)))) message = _("no leases left"); - + if (!message) { logaddr = &mess->yiaddr; - + lease_set_hwaddr(lease, mess->chaddr, NULL, mess->hlen, mess->htype, 0, now, 1); if (hostname) - lease_set_hostname(lease, hostname, 1, get_domain(lease->addr), domain); + lease_set_hostname(lease, hostname, 1, get_domain(lease->addr), domain); /* infinite lease unless nailed in dhcp-host line. */ - lease_set_expires(lease, - have_config(config, CONFIG_TIME) ? config->lease_time : 0xffffffff, - now); + lease_set_expires(lease, + have_config(config, CONFIG_TIME) ? config->lease_time : 0xffffffff, + now); lease_set_interface(lease, int_index, now); - + clear_packet(mess, end); - do_options(context, mess, end, NULL, hostname, get_domain(mess->yiaddr), + do_options(context, mess, end, NULL, hostname, get_domain(mess->yiaddr), netid, subnet_addr, 0, 0, -1, NULL, vendor_class_len, now, 0xffffffff, 0); } } - + daemon->metrics[METRIC_BOOTP]++; log_packet("BOOTP", logaddr, mess->chaddr, mess->hlen, iface_name, NULL, message, mess->xid); - + return message ? 0 : dhcp_packet_size(mess, agent_id, real_end); } - + if ((opt = option_find(mess, sz, OPTION_CLIENT_FQDN, 3))) { /* http://tools.ietf.org/wg/dhc/draft-ietf-dhc-fqdn-option/draft-ietf-dhc-fqdn-option-10.txt */ int len = option_len(opt); char *pq = daemon->dhcp_buff; unsigned char *pp, *op = option_ptr(opt, 0); - + fqdn_flags = *op; len -= 3; op += 3; pp = op; - + /* NB, the following always sets at least one bit */ if (option_bool(OPT_FQDN_UPDATE)) { @@ -654,13 +679,13 @@ size_t dhcp_reply(struct dhcp_context *context, char *iface_name, int int_index, } fqdn_flags |= 0x08; /* set N */ } - else + else { if (!(fqdn_flags & 0x01)) fqdn_flags |= 0x03; /* set S and O */ fqdn_flags &= ~0x08; /* clear N */ } - + if (fqdn_flags & 0x04) while (*op != 0 && ((op + (*op)) - pp) < len) { @@ -676,12 +701,12 @@ size_t dhcp_reply(struct dhcp_context *context, char *iface_name, int int_index, borken_opt = 1; pq += len + 1; } - + if (pq != daemon->dhcp_buff) pq--; - + *pq = 0; - + if (legal_hostname(daemon->dhcp_buff)) offer_hostname = client_hostname = daemon->dhcp_buff; } @@ -700,10 +725,38 @@ size_t dhcp_reply(struct dhcp_context *context, char *iface_name, int int_index, client_hostname = daemon->dhcp_buff; } - if (client_hostname && option_bool(OPT_LOG_OPTS)) - my_syslog(MS_DHCP | LOG_INFO, _("%u client provides name: %s"), ntohl(mess->xid), client_hostname); - - + if (client_hostname) + { + struct dhcp_match_name *m; + size_t nl = strlen(client_hostname); + + if (option_bool(OPT_LOG_OPTS)) + my_syslog(MS_DHCP | LOG_INFO, _("%u client provides name: %s"), ntohl(mess->xid), client_hostname); + for (m = daemon->dhcp_name_match; m; m = m->next) + { + size_t ml = strlen(m->name); + char save = 0; + + if (nl < ml) + continue; + if (nl > ml) + { + save = client_hostname[ml]; + client_hostname[ml] = 0; + } + + if (hostname_isequal(client_hostname, m->name) && + (save == 0 || m->wildcard)) + { + m->netid->next = netid; + netid = m->netid; + } + + if (save != 0) + client_hostname[ml] = save; + } + } + if (have_config(config, CONFIG_NAME)) { hostname = config->hostname; @@ -715,23 +768,20 @@ size_t dhcp_reply(struct dhcp_context *context, char *iface_name, int int_index, } else if (client_hostname) { - struct dhcp_match_name *m; - size_t nl; - domain = strip_hostname(client_hostname); - - if ((nl = strlen(client_hostname)) != 0) + + if (strlen(client_hostname) != 0) { hostname = client_hostname; - + if (!config) { - /* Search again now we have a hostname. + /* Search again now we have a hostname. Only accept configs without CLID and HWADDR here, (they won't match) to avoid impersonation by name. */ struct dhcp_config *new = find_config(daemon->dhcp_conf, context, NULL, 0, - mess->chaddr, mess->hlen, - mess->htype, hostname); + mess->chaddr, mess->hlen, + mess->htype, hostname, run_tag_if(netid)); if (new && !have_config(new, CONFIG_CLID) && !new->hwaddr) { config = new; @@ -741,52 +791,28 @@ size_t dhcp_reply(struct dhcp_context *context, char *iface_name, int int_index, netid = &known_id; } } - - for (m = daemon->dhcp_name_match; m; m = m->next) - { - size_t ml = strlen(m->name); - char save = 0; - - if (nl < ml) - continue; - if (nl > ml) - { - save = client_hostname[ml]; - client_hostname[ml] = 0; - } - - if (hostname_isequal(client_hostname, m->name) && - (save == 0 || m->wildcard)) - { - m->netid->next = netid; - netid = m->netid; - } - - if (save != 0) - client_hostname[ml] = save; - } } } - + if (config) { struct dhcp_netid_list *list; - + for (list = config->netid; list; list = list->next) { list->list->next = netid; netid = list->list; } } - + tagif_netid = run_tag_if(netid); - + /* if all the netids in the ignore list are present, ignore this client */ for (id_list = daemon->dhcp_ignore; id_list; id_list = id_list->next) if (match_netid(id_list->list, tagif_netid, 0)) ignore = 1; - /* If configured, we can override the server-id to be the address of the relay, + /* If configured, we can override the server-id to be the address of the relay, so that all traffic goes via the relay and can pick up agent-id info. This can be configured for all relays, or by address. */ if (daemon->override && mess->giaddr.s_addr != 0 && override.s_addr == 0) @@ -807,10 +833,10 @@ size_t dhcp_reply(struct dhcp_context *context, char *iface_name, int int_index, /* Can have setting to ignore the client ID for a particular MAC address or hostname */ if (have_config(config, CONFIG_NOCLID)) clid = NULL; - + /* Check if client is PXE client. */ - if (daemon->enable_pxe && - (opt = option_find(mess, sz, OPTION_VENDOR_ID, 9)) && + if (daemon->enable_pxe && + (opt = option_find(mess, sz, OPTION_VENDOR_ID, 9)) && strncmp(option_ptr(opt, 0), "PXEClient", 9) == 0) { if ((opt = option_find(mess, sz, OPTION_PXE_UUID, 17))) @@ -840,41 +866,41 @@ size_t dhcp_reply(struct dhcp_context *context, char *iface_name, int int_index, } memcpy(save71, option_ptr(opt, 0), 4); - + for (service = daemon->pxe_services; service; service = service->next) if (service->type == type) break; - + for (; context; context = context->current) if (match_netid(context->filter, tagif_netid, 1) && is_same_net(mess->ciaddr, context->start, context->netmask)) break; - + if (!service || !service->basename || !context) return 0; - + clear_packet(mess, end); - + mess->yiaddr = mess->ciaddr; mess->ciaddr.s_addr = 0; if (service->sname) mess->siaddr = a_record_from_hosts(service->sname, now); else if (service->server.s_addr != 0) - mess->siaddr = service->server; + mess->siaddr = service->server; else - mess->siaddr = context->local; - + mess->siaddr = context->local; + if (strchr(service->basename, '.')) snprintf((char *)mess->file, sizeof(mess->file), "%s", service->basename); else snprintf((char *)mess->file, sizeof(mess->file), "%s.%d", service->basename, layer); - + option_put(mess, end, OPTION_MESSAGE_TYPE, 1, DHCPACK); option_put(mess, end, OPTION_SERVER_IDENTIFIER, INADDRSZ, htonl(context->local.s_addr)); pxe_misc(mess, end, uuid); - + prune_vendor_opts(tagif_netid); opt71.val = save71; opt71.opt = SUBOPT_PXE_BOOT_ITEM; @@ -883,12 +909,12 @@ size_t dhcp_reply(struct dhcp_context *context, char *iface_name, int int_index, opt71.netid = NULL; opt71.next = daemon->dhcp_opts; do_encap_opts(&opt71, OPTION_VENDOR_CLASS_OPT, DHOPT_VENDOR_MATCH, mess, end, 0); - + log_packet("PXE", &mess->yiaddr, emac, emac_len, iface_name, (char *)mess->file, NULL, mess->xid); log_tags(tagif_netid, ntohl(mess->xid)); - return dhcp_packet_size(mess, agent_id, real_end); + return dhcp_packet_size(mess, agent_id, real_end); } - + if ((opt = option_find(mess, sz, OPTION_ARCH, 2))) { pxearch = option_uint(opt, 0, 2); @@ -898,12 +924,12 @@ size_t dhcp_reply(struct dhcp_context *context, char *iface_name, int int_index, { struct dhcp_context *tmp; int workaround = 0; - + for (tmp = context; tmp; tmp = tmp->current) if ((tmp->flags & CONTEXT_PROXY) && match_netid(tmp->filter, tagif_netid, 1)) break; - + if (tmp) { struct dhcp_boot *boot; @@ -914,56 +940,56 @@ size_t dhcp_reply(struct dhcp_context *context, char *iface_name, int int_index, tmp->netid.next = netid; tagif_netid = run_tag_if(&tmp->netid); } - + boot = find_boot(tagif_netid); - + mess->yiaddr.s_addr = 0; if (mess_type == DHCPDISCOVER || mess->ciaddr.s_addr == 0) { mess->ciaddr.s_addr = 0; mess->flags |= htons(0x8000); /* broadcast */ } - + clear_packet(mess, end); - + /* Redirect EFI clients to port 4011 */ if (pxearch >= 6) { redirect4011 = 1; mess->siaddr = tmp->local; } - - /* Returns true if only one matching service is available. On port 4011, + + /* Returns true if only one matching service is available. On port 4011, it also inserts the boot file and server name. */ workaround = pxe_uefi_workaround(pxearch, tagif_netid, mess, tmp->local, now, pxe); - + if (!workaround && boot) { /* Provide the bootfile here, for iPXE, and in case we have no menu items and set discovery_control = 8 */ - if (boot->next_server.s_addr) + if (boot->next_server.s_addr) mess->siaddr = boot->next_server; - else if (boot->tftp_sname) + else if (boot->tftp_sname) mess->siaddr = a_record_from_hosts(boot->tftp_sname, now); - + if (boot->file) safe_strncpy((char *)mess->file, boot->file, sizeof(mess->file)); } - - option_put(mess, end, OPTION_MESSAGE_TYPE, 1, + + option_put(mess, end, OPTION_MESSAGE_TYPE, 1, mess_type == DHCPDISCOVER ? DHCPOFFER : DHCPACK); option_put(mess, end, OPTION_SERVER_IDENTIFIER, INADDRSZ, htonl(tmp->local.s_addr)); pxe_misc(mess, end, uuid); prune_vendor_opts(tagif_netid); if ((pxe && !workaround) || !redirect4011) do_encap_opts(pxe_opts(pxearch, tagif_netid, tmp->local, now), OPTION_VENDOR_CLASS_OPT, DHOPT_VENDOR_MATCH, mess, end, 0); - + daemon->metrics[METRIC_PXE]++; log_packet("PXE", NULL, emac, emac_len, iface_name, ignore ? "proxy-ignored" : "proxy", NULL, mess->xid); log_tags(tagif_netid, ntohl(mess->xid)); if (!ignore) apply_delay(mess->xid, recvtime, tagif_netid); - return ignore ? 0 : dhcp_packet_size(mess, agent_id, real_end); + return ignore ? 0 : dhcp_packet_size(mess, agent_id, real_end); } } } @@ -972,38 +998,38 @@ size_t dhcp_reply(struct dhcp_context *context, char *iface_name, int int_index, /* if we're just a proxy server, go no further */ if ((context->flags & CONTEXT_PROXY) || pxe) return 0; - + if ((opt = option_find(mess, sz, OPTION_REQUESTED_OPTIONS, 0))) { req_options = (unsigned char *)daemon->dhcp_buff2; memcpy(req_options, option_ptr(opt, 0), option_len(opt)); req_options[option_len(opt)] = OPTION_END; } - + switch (mess_type) { case DHCPDECLINE: if (!(opt = option_find(mess, sz, OPTION_SERVER_IDENTIFIER, INADDRSZ)) || option_addr(opt).s_addr != server_id(context, override, fallback).s_addr) return 0; - + /* sanitise any message. Paranoid? Moi? */ sanitise(option_find(mess, sz, OPTION_MESSAGE, 1), daemon->dhcp_buff); - + if (!(opt = option_find(mess, sz, OPTION_REQUESTED_IP, INADDRSZ))) return 0; - + daemon->metrics[METRIC_DHCPDECLINE]++; log_packet("DHCPDECLINE", option_ptr(opt, 0), emac, emac_len, iface_name, NULL, daemon->dhcp_buff, mess->xid); - + if (lease && lease->addr.s_addr == option_addr(opt).s_addr) lease_prune(lease, now); - - if (have_config(config, CONFIG_ADDR) && + + if (have_config(config, CONFIG_ADDR) && config->addr.s_addr == option_addr(opt).s_addr) { prettyprint_time(daemon->dhcp_buff, DECLINE_BACKOFF); - my_syslog(MS_DHCP | LOG_WARNING, _("disabling DHCP static address %s for %s"), + my_syslog(MS_DHCP | LOG_WARNING, _("disabling DHCP static address %s for %s"), inet_ntoa(config->addr), daemon->dhcp_buff); config->flags |= CONFIG_DECLINED; config->decline_time = now; @@ -1012,7 +1038,7 @@ size_t dhcp_reply(struct dhcp_context *context, char *iface_name, int int_index, /* make sure this host gets a different address next time. */ for (; context; context = context->current) context->addr_epoch++; - + return 0; case DHCPRELEASE: @@ -1020,7 +1046,7 @@ size_t dhcp_reply(struct dhcp_context *context, char *iface_name, int int_index, !(opt = option_find(mess, sz, OPTION_SERVER_IDENTIFIER, INADDRSZ)) || option_addr(opt).s_addr != server_id(context, override, fallback).s_addr) return 0; - + if (lease && lease->addr.s_addr == mess->ciaddr.s_addr) lease_prune(lease, now); else @@ -1028,9 +1054,9 @@ size_t dhcp_reply(struct dhcp_context *context, char *iface_name, int int_index, daemon->metrics[METRIC_DHCPRELEASE]++; log_packet("DHCPRELEASE", &mess->ciaddr, emac, emac_len, iface_name, NULL, message, mess->xid); - + return 0; - + case DHCPDISCOVER: if (ignore || have_config(config, CONFIG_DISABLE)) { @@ -1039,20 +1065,20 @@ size_t dhcp_reply(struct dhcp_context *context, char *iface_name, int int_index, message = _("ignored"); opt = NULL; } - else + else { struct in_addr addr, conf; - + addr.s_addr = conf.s_addr = 0; - if ((opt = option_find(mess, sz, OPTION_REQUESTED_IP, INADDRSZ))) + if ((opt = option_find(mess, sz, OPTION_REQUESTED_IP, INADDRSZ))) addr = option_addr(opt); - + if (have_config(config, CONFIG_ADDR)) { char *addrs = inet_ntoa(config->addr); - - if ((ltmp = lease_find_by_addr(config->addr)) && + + if ((ltmp = lease_find_by_addr(config->addr)) && ltmp != lease && !config_has_mac(config, ltmp->hwaddr, ltmp->hwaddr_len, ltmp->hwaddr_type)) { @@ -1077,24 +1103,24 @@ size_t dhcp_reply(struct dhcp_context *context, char *iface_name, int int_index, conf = config->addr; } } - + if (conf.s_addr) mess->yiaddr = conf; - else if (lease && - address_available(context, lease->addr, tagif_netid) && + else if (lease && + address_available(context, lease->addr, tagif_netid) && !config_find_by_address(daemon->dhcp_conf, lease->addr)) mess->yiaddr = lease->addr; - else if (opt && address_available(context, addr, tagif_netid) && !lease_find_by_addr(addr) && + else if (opt && address_available(context, addr, tagif_netid) && !lease_find_by_addr(addr) && !config_find_by_address(daemon->dhcp_conf, addr) && do_icmp_ping(now, addr, 0, loopback)) mess->yiaddr = addr; else if (emac_len == 0) message = _("no unique-id"); else if (!address_allocate(context, &mess->yiaddr, emac, emac_len, tagif_netid, now, loopback)) - message = _("no address available"); + message = _("no address available"); } - + daemon->metrics[METRIC_DHCPDISCOVER]++; - log_packet("DHCPDISCOVER", opt ? option_ptr(opt, 0) : NULL, emac, emac_len, iface_name, NULL, message, mess->xid); + log_packet("DHCPDISCOVER", opt ? option_ptr(opt, 0) : NULL, emac, emac_len, iface_name, NULL, message, mess->xid); if (message || !(context = narrow_context(context, mess->yiaddr, tagif_netid))) return 0; @@ -1113,21 +1139,21 @@ size_t dhcp_reply(struct dhcp_context *context, char *iface_name, int int_index, rapid_commit = 1; goto rapid_commit; } - + daemon->metrics[METRIC_DHCPOFFER]++; log_packet("DHCPOFFER" , &mess->yiaddr, emac, emac_len, iface_name, NULL, NULL, mess->xid); - + time = calc_time(context, config, option_find(mess, sz, OPTION_LEASE_TIME, 4)); clear_packet(mess, end); option_put(mess, end, OPTION_MESSAGE_TYPE, 1, DHCPOFFER); option_put(mess, end, OPTION_SERVER_IDENTIFIER, INADDRSZ, ntohl(server_id(context, override, fallback).s_addr)); option_put(mess, end, OPTION_LEASE_TIME, 4, time); /* T1 and T2 are required in DHCPOFFER by HP's wacky Jetdirect client. */ - do_options(context, mess, end, req_options, offer_hostname, get_domain(mess->yiaddr), + do_options(context, mess, end, req_options, offer_hostname, get_domain(mess->yiaddr), netid, subnet_addr, fqdn_flags, borken_opt, pxearch, uuid, vendor_class_len, now, time, fuzz); - + return dhcp_packet_size(mess, agent_id, real_end); - + case DHCPREQUEST: if (ignore || have_config(config, CONFIG_DISABLE)) @@ -1136,33 +1162,33 @@ size_t dhcp_reply(struct dhcp_context *context, char *iface_name, int int_index, { /* SELECTING or INIT_REBOOT */ mess->yiaddr = option_addr(opt); - + /* send vendor and user class info for new or recreated lease */ do_classes = 1; - + if ((opt = option_find(mess, sz, OPTION_SERVER_IDENTIFIER, INADDRSZ))) { /* SELECTING */ selecting = 1; - + if (override.s_addr != 0) { if (option_addr(opt).s_addr != override.s_addr) return 0; } - else + else { for (; context; context = context->current) if (context->local.s_addr == option_addr(opt).s_addr) break; - + if (!context) { /* Handle very strange configs where clients have more than one route to the server. If a clients idea of its server-id matches any of our DHCP interfaces, we let it pass. Have to set override to make sure we echo back the correct server-id */ struct irec *intr; - + enumerate_interfaces(0); for (intr = daemon->interfaces; intr; intr = intr->next) @@ -1176,7 +1202,7 @@ size_t dhcp_reply(struct dhcp_context *context, char *iface_name, int int_index, else { /* In auth mode, a REQUEST sent to the wrong server - should be faulted, so that the client establishes + should be faulted, so that the client establishes communication with us, otherwise, silently ignore. */ if (!option_bool(OPT_AUTHORITATIVE)) return 0; @@ -1197,14 +1223,14 @@ size_t dhcp_reply(struct dhcp_context *context, char *iface_name, int int_index, /* INIT-REBOOT */ if (!lease && !option_bool(OPT_AUTHORITATIVE)) return 0; - + if (lease && lease->addr.s_addr != mess->yiaddr.s_addr) message = _("wrong address"); } } else { - /* RENEWING or REBINDING */ + /* RENEWING or REBINDING */ /* Check existing lease for this address. We allow it to be missing if dhcp-authoritative mode as long as we can allocate the lease now - checked below. @@ -1212,8 +1238,8 @@ size_t dhcp_reply(struct dhcp_context *context, char *iface_name, int int_index, if ((lease && mess->ciaddr.s_addr != lease->addr.s_addr) || (!lease && !option_bool(OPT_AUTHORITATIVE))) { - /* A client rebinding will broadcast the request, so we may see it even - if the lease is held by another server. Just ignore it in that case. + /* A client rebinding will broadcast the request, so we may see it even + if the lease is held by another server. Just ignore it in that case. If the request is unicast to us, then somethings wrong, NAK */ if (!unicast_dest) return 0; @@ -1229,18 +1255,18 @@ size_t dhcp_reply(struct dhcp_context *context, char *iface_name, int int_index, daemon->metrics[METRIC_DHCPREQUEST]++; log_packet("DHCPREQUEST", &mess->yiaddr, emac, emac_len, iface_name, NULL, NULL, mess->xid); - + rapid_commit: if (!message) { struct dhcp_config *addr_config; struct dhcp_context *tmp = NULL; - + if (have_config(config, CONFIG_ADDR)) for (tmp = context; tmp; tmp = tmp->current) if (context->router.s_addr == config->addr.s_addr) break; - + if (!(context = narrow_context(context, mess->yiaddr, tagif_netid))) { /* If a machine moves networks whilst it has a lease, we catch that here. */ @@ -1248,17 +1274,17 @@ size_t dhcp_reply(struct dhcp_context *context, char *iface_name, int int_index, /* ensure we broadcast NAK */ unicast_dest = 0; } - + /* Check for renewal of a lease which is outside the allowed range. */ else if (!address_available(context, mess->yiaddr, tagif_netid) && (!have_config(config, CONFIG_ADDR) || config->addr.s_addr != mess->yiaddr.s_addr)) message = _("address not available"); - + /* Check if a new static address has been configured. Be very sure that when the client does DISCOVER, it will get the static address, otherwise an endless protocol loop will ensue. */ else if (!tmp && !selecting && - have_config(config, CONFIG_ADDR) && + have_config(config, CONFIG_ADDR) && (!have_config(config, CONFIG_DECLINED) || difftime(now, config->decline_time) > (float)DECLINE_BACKOFF) && config->addr.s_addr != mess->yiaddr.s_addr && @@ -1271,12 +1297,12 @@ size_t dhcp_reply(struct dhcp_context *context, char *iface_name, int int_index, else if (!lease && (ltmp = lease_find_by_addr(mess->yiaddr))) { - /* If a host is configured with more than one MAC address, it's OK to 'nix + /* If a host is configured with more than one MAC address, it's OK to 'nix a lease from one of it's MACs to give the address to another. */ if (config && config_has_mac(config, ltmp->hwaddr, ltmp->hwaddr_len, ltmp->hwaddr_type)) { my_syslog(MS_DHCP | LOG_INFO, _("abandoning lease to %s of %s"), - print_mac(daemon->namebuff, ltmp->hwaddr, ltmp->hwaddr_len), + print_mac(daemon->namebuff, ltmp->hwaddr, ltmp->hwaddr_len), inet_ntoa(ltmp->addr)); lease = ltmp; } @@ -1288,9 +1314,9 @@ size_t dhcp_reply(struct dhcp_context *context, char *iface_name, int int_index, { if (emac_len == 0) message = _("no unique-id"); - + else if (!lease) - { + { if ((lease = lease4_allocate(mess->yiaddr))) do_classes = 1; else @@ -1307,15 +1333,15 @@ size_t dhcp_reply(struct dhcp_context *context, char *iface_name, int int_index, /* rapid commit case: lease allocate failed but don't send DHCPNAK */ if (rapid_commit) return 0; - + mess->yiaddr.s_addr = 0; clear_packet(mess, end); option_put(mess, end, OPTION_MESSAGE_TYPE, 1, DHCPNAK); option_put(mess, end, OPTION_SERVER_IDENTIFIER, INADDRSZ, ntohl(server_id(context, override, fallback).s_addr)); option_put_string(mess, end, OPTION_MESSAGE, message, borken_opt); - /* This fixes a problem with the DHCP spec, broadcasting a NAK to a host on + /* This fixes a problem with the DHCP spec, broadcasting a NAK to a host on a distant subnet which unicast a REQ to us won't work. */ - if (!unicast_dest || mess->giaddr.s_addr != 0 || + if (!unicast_dest || mess->giaddr.s_addr != 0 || mess->ciaddr.s_addr == 0 || is_same_net(context->local, mess->ciaddr, context->netmask)) { mess->flags |= htons(0x8000); /* broadcast */ @@ -1331,7 +1357,7 @@ size_t dhcp_reply(struct dhcp_context *context, char *iface_name, int int_index, } log_tags(tagif_netid, ntohl(mess->xid)); - + if (do_classes) { /* pick up INIT-REBOOT events. */ @@ -1341,14 +1367,14 @@ size_t dhcp_reply(struct dhcp_context *context, char *iface_name, int int_index, if (daemon->lease_change_command) { struct dhcp_netid *n; - + if (mess->giaddr.s_addr) lease->giaddr = mess->giaddr; - + free(lease->extradata); lease->extradata = NULL; lease->extradata_size = lease->extradata_len = 0; - + add_extradata_opt(lease, option_find(mess, sz, OPTION_VENDOR_ID, 1)); add_extradata_opt(lease, option_find(mess, sz, OPTION_HOSTNAME, 1)); add_extradata_opt(lease, oui); @@ -1379,7 +1405,7 @@ size_t dhcp_reply(struct dhcp_context *context, char *iface_name, int int_index, { q += snprintf(q, MAXDNAME - (q - daemon->namebuff), "%d%s", rop[i], i + 1 == len ? "" : ","); } - lease_add_extradata(lease, (unsigned char *)daemon->namebuff, (q - daemon->namebuff), 0); + lease_add_extradata(lease, (unsigned char *)daemon->namebuff, (q - daemon->namebuff), 0); } else { @@ -1398,9 +1424,9 @@ size_t dhcp_reply(struct dhcp_context *context, char *iface_name, int int_index, if (strcmp(n->net, n1->net) == 0) break; if (!n1) - lease_add_extradata(lease, (unsigned char *)n->net, strlen(n->net), n->next ? ' ' : 0); + lease_add_extradata(lease, (unsigned char *)n->net, strlen(n->net), n->next ? ' ' : 0); } - + if ((opt = option_find(mess, sz, OPTION_USER_CLASS, 1))) { int len = option_len(opt); @@ -1413,17 +1439,17 @@ size_t dhcp_reply(struct dhcp_context *context, char *iface_name, int int_index, } #endif } - + if (!hostname_auth && (client_hostname = host_from_dns(mess->yiaddr))) { domain = get_domain(mess->yiaddr); hostname = client_hostname; hostname_auth = 1; } - + time = calc_time(context, config, option_find(mess, sz, OPTION_LEASE_TIME, 4)); lease_set_hwaddr(lease, mess->chaddr, clid, mess->hlen, mess->htype, clid_len, now, do_classes); - + /* if all the netids in the ignore_name list are present, ignore client-supplied name */ if (!hostname_auth) { @@ -1433,7 +1459,7 @@ size_t dhcp_reply(struct dhcp_context *context, char *iface_name, int int_index, if (id_list) hostname = NULL; } - + /* Last ditch, if configured, generate hostname from mac address */ if (!hostname && emac_len != 0) { @@ -1454,7 +1480,7 @@ size_t dhcp_reply(struct dhcp_context *context, char *iface_name, int int_index, if (hostname) lease_set_hostname(lease, hostname, hostname_auth, get_domain(lease->addr), domain); - + lease_set_expires(lease, time, now); lease_set_interface(lease, int_index, now); @@ -1464,7 +1490,7 @@ size_t dhcp_reply(struct dhcp_context *context, char *iface_name, int int_index, override = lease->override; daemon->metrics[METRIC_DHCPACK]++; - log_packet("DHCPACK", &mess->yiaddr, emac, emac_len, iface_name, hostname, NULL, mess->xid); + log_packet("DHCPACK", &mess->yiaddr, emac, emac_len, iface_name, hostname, NULL, mess->xid); clear_packet(mess, end); option_put(mess, end, OPTION_MESSAGE_TYPE, 1, DHCPACK); @@ -1472,35 +1498,35 @@ size_t dhcp_reply(struct dhcp_context *context, char *iface_name, int int_index, option_put(mess, end, OPTION_LEASE_TIME, 4, time); if (rapid_commit) option_put(mess, end, OPTION_RAPID_COMMIT, 0, 0); - do_options(context, mess, end, req_options, hostname, get_domain(mess->yiaddr), + do_options(context, mess, end, req_options, hostname, get_domain(mess->yiaddr), netid, subnet_addr, fqdn_flags, borken_opt, pxearch, uuid, vendor_class_len, now, time, fuzz); } - return dhcp_packet_size(mess, agent_id, real_end); - + return dhcp_packet_size(mess, agent_id, real_end); + case DHCPINFORM: if (ignore || have_config(config, CONFIG_DISABLE)) message = _("ignored"); - + daemon->metrics[METRIC_DHCPINFORM]++; log_packet("DHCPINFORM", &mess->ciaddr, emac, emac_len, iface_name, message, NULL, mess->xid); - + if (message || mess->ciaddr.s_addr == 0) return 0; /* For DHCPINFORM only, cope without a valid context */ context = narrow_context(context, mess->ciaddr, tagif_netid); - + /* Find a least based on IP address if we didn't get one from MAC address/client-d */ if (!lease && - (lease = lease_find_by_addr(mess->ciaddr)) && + (lease = lease_find_by_addr(mess->ciaddr)) && lease->hostname) hostname = lease->hostname; - + if (!hostname) hostname = host_from_dns(mess->ciaddr); - + if (context && context->netid.net) { context->netid.next = netid; @@ -1508,10 +1534,10 @@ size_t dhcp_reply(struct dhcp_context *context, char *iface_name, int int_index, } log_tags(tagif_netid, ntohl(mess->xid)); - + daemon->metrics[METRIC_DHCPACK]++; log_packet("DHCPACK", &mess->ciaddr, emac, emac_len, iface_name, hostname, NULL, mess->xid); - + if (lease) { lease_set_interface(lease, int_index, now); @@ -1524,11 +1550,11 @@ size_t dhcp_reply(struct dhcp_context *context, char *iface_name, int int_index, clear_packet(mess, end); option_put(mess, end, OPTION_MESSAGE_TYPE, 1, DHCPACK); option_put(mess, end, OPTION_SERVER_IDENTIFIER, INADDRSZ, ntohl(server_id(context, override, fallback).s_addr)); - - /* RFC 2131 says that DHCPINFORM shouldn't include lease-time parameters, but + + /* RFC 2131 says that DHCPINFORM shouldn't include lease-time parameters, but we supply a utility which makes DHCPINFORM requests to get this information. Only include lease time if OPTION_LEASE_TIME is in the parameter request list, - which won't be true for ordinary clients, but will be true for the + which won't be true for ordinary clients, but will be true for the dhcp_lease_time utility. */ if (lease && in_list(req_options, OPTION_LEASE_TIME)) { @@ -1541,23 +1567,23 @@ size_t dhcp_reply(struct dhcp_context *context, char *iface_name, int int_index, do_options(context, mess, end, req_options, hostname, get_domain(mess->ciaddr), netid, subnet_addr, fqdn_flags, borken_opt, pxearch, uuid, vendor_class_len, now, 0xffffffff, 0); - + *is_inform = 1; /* handle reply differently */ - return dhcp_packet_size(mess, agent_id, real_end); + return dhcp_packet_size(mess, agent_id, real_end); } - + return 0; } /* find a good value to use as MAC address for logging and address-allocation hashing. This is normally just the chaddr field from the DHCP packet, - but eg Firewire will have hlen == 0 and use the client-id instead. + but eg Firewire will have hlen == 0 and use the client-id instead. This could be anything, but will normally be EUI64 for Firewire. We assume that if the first byte of the client-id equals the htype byte - then the client-id is using the usual encoding and use the rest of the + then the client-id is using the usual encoding and use the rest of the client-id: if not we can use the whole client-id. This should give sane MAC address logs. */ -unsigned char *extended_hwaddr(int hwtype, int hwlen, unsigned char *hwaddr, +unsigned char *extended_hwaddr(int hwtype, int hwlen, unsigned char *hwaddr, int clid_len, unsigned char *clid, int *len_out) { if (hwlen == 0 && clid && clid_len > 3) @@ -1575,11 +1601,11 @@ unsigned char *extended_hwaddr(int hwtype, int hwlen, unsigned char *hwaddr, return clid + 1; } #endif - + *len_out = clid_len; return clid; } - + *len_out = hwlen; return hwaddr; } @@ -1587,9 +1613,9 @@ unsigned char *extended_hwaddr(int hwtype, int hwlen, unsigned char *hwaddr, static unsigned int calc_time(struct dhcp_context *context, struct dhcp_config *config, unsigned char *opt) { unsigned int time = have_config(config, CONFIG_TIME) ? config->lease_time : context->lease_time; - + if (opt) - { + { unsigned int req_time = option_uint(opt, 0, 4); if (req_time < 120 ) req_time = 120; /* sanity */ @@ -1614,9 +1640,9 @@ static int sanitise(unsigned char *opt, char *buf) { char *p; int i; - + *buf = 0; - + if (!opt) return 0; @@ -1629,7 +1655,7 @@ static int sanitise(unsigned char *opt, char *buf) *buf++ = c; } *buf = 0; /* add terminator */ - + return 1; } @@ -1639,29 +1665,29 @@ static void add_extradata_opt(struct dhcp_lease *lease, unsigned char *opt) if (!opt) lease_add_extradata(lease, NULL, 0, 0); else - lease_add_extradata(lease, option_ptr(opt, 0), option_len(opt), 0); + lease_add_extradata(lease, option_ptr(opt, 0), option_len(opt), 0); } #endif -static void log_packet(char *type, void *addr, unsigned char *ext_mac, +static void log_packet(char *type, void *addr, unsigned char *ext_mac, int mac_len, char *interface, char *string, char *err, u32 xid) { struct in_addr a; - + if (!err && !option_bool(OPT_LOG_OPTS) && option_bool(OPT_QUIET_DHCP)) return; - + /* addr may be misaligned */ if (addr) memcpy(&a, addr, sizeof(a)); - + print_mac(daemon->namebuff, ext_mac, mac_len); - + if(option_bool(OPT_LOG_OPTS)) my_syslog(MS_DHCP | LOG_INFO, "%u %s(%s) %s%s%s %s%s", - ntohl(xid), + ntohl(xid), type, - interface, + interface, addr ? inet_ntoa(a) : "", addr ? " " : "", daemon->namebuff, @@ -1670,7 +1696,7 @@ static void log_packet(char *type, void *addr, unsigned char *ext_mac, else my_syslog(MS_DHCP | LOG_INFO, "%s(%s) %s%s%s %s%s", type, - interface, + interface, addr ? inet_ntoa(a) : "", addr ? " " : "", daemon->namebuff, @@ -1690,8 +1716,8 @@ static void log_options(unsigned char *start, u32 xid) while (*start != OPTION_END) { char *optname = option_string(AF_INET, start[0], option_ptr(start, 0), option_len(start), daemon->namebuff, MAXDNAME); - - my_syslog(MS_DHCP | LOG_INFO, "%u sent size:%3d option:%3d %s %s", + + my_syslog(MS_DHCP | LOG_INFO, "%u sent size:%3d option:%3d %s %s", ntohl(xid), option_len(start), start[0], optname, daemon->namebuff); start += start[1] + 2; } @@ -1699,7 +1725,7 @@ static void log_options(unsigned char *start, u32 xid) static unsigned char *option_find1(unsigned char *p, unsigned char *end, int opt, int minsize) { - while (1) + while (1) { if (p >= end) return NULL; @@ -1707,8 +1733,8 @@ static unsigned char *option_find1(unsigned char *p, unsigned char *end, int opt return opt == OPTION_END ? p : NULL; else if (*p == OPTION_PAD) p++; - else - { + else + { int opt_len; if (p > end - 2) return NULL; /* malformed packet */ @@ -1721,11 +1747,11 @@ static unsigned char *option_find1(unsigned char *p, unsigned char *end, int opt } } } - + static unsigned char *option_find(struct dhcp_packet *mess, size_t size, int opt_type, int minsize) { unsigned char *ret, *overload; - + /* skip over DHCP cookie; */ if ((ret = option_find1(&mess->options[0] + sizeof(u32), ((unsigned char *)mess) + size, opt_type, minsize))) return ret; @@ -1733,7 +1759,7 @@ static unsigned char *option_find(struct dhcp_packet *mess, size_t size, int opt /* look for overload option. */ if (!(overload = option_find1(&mess->options[0] + sizeof(u32), ((unsigned char *)mess) + size, OPTION_OVERLOAD, 1))) return NULL; - + /* Can we look in filename area ? */ if ((overload[2] & 1) && (ret = option_find1(&mess->file[0], &mess->file[128], opt_type, minsize))) @@ -1764,7 +1790,7 @@ static unsigned int option_uint(unsigned char *opt, int offset, int size) unsigned int ret = 0; int i; unsigned char *p = option_ptr(opt, offset); - + for (i = 0; i < size; i++) ret = (ret << 8) | *p++; @@ -1778,11 +1804,11 @@ static unsigned char *dhcp_skip_opts(unsigned char *start) return start; } -/* only for use when building packet: doesn't check for bad data. */ +/* only for use when building packet: doesn't check for bad data. */ static unsigned char *find_overload(struct dhcp_packet *mess) { unsigned char *p = &mess->options[0] + sizeof(u32); - + while (*p != 0) { if (*p == OPTION_OVERLOAD) @@ -1797,7 +1823,7 @@ static size_t dhcp_packet_size(struct dhcp_packet *mess, unsigned char *agent_id unsigned char *p = dhcp_skip_opts(&mess->options[0] + sizeof(u32)); unsigned char *overload; size_t ret; - + /* move agent_id back down to the end of the packet */ if (agent_id) { @@ -1805,10 +1831,10 @@ static size_t dhcp_packet_size(struct dhcp_packet *mess, unsigned char *agent_id p += real_end - agent_id; memset(p, 0, real_end - p); /* in case of overlap */ } - + /* add END options to the regions. */ overload = find_overload(mess); - + if (overload && (option_uint(overload, 0, 1) & 1)) { *dhcp_skip_opts(mess->file) = OPTION_END; @@ -1817,7 +1843,7 @@ static size_t dhcp_packet_size(struct dhcp_packet *mess, unsigned char *agent_id } else if (option_bool(OPT_LOG_OPTS) && strlen((char *)mess->file) != 0) my_syslog(MS_DHCP | LOG_INFO, _("%u bootfile name: %s"), ntohl(mess->xid), (char *)mess->file); - + if (overload && (option_uint(overload, 0, 1) & 2)) { *dhcp_skip_opts(mess->sname) = OPTION_END; @@ -1829,35 +1855,35 @@ static size_t dhcp_packet_size(struct dhcp_packet *mess, unsigned char *agent_id *p++ = OPTION_END; - + if (option_bool(OPT_LOG_OPTS)) { if (mess->siaddr.s_addr != 0) my_syslog(MS_DHCP | LOG_INFO, _("%u next server: %s"), ntohl(mess->xid), inet_ntoa(mess->siaddr)); - + if ((mess->flags & htons(0x8000)) && mess->ciaddr.s_addr == 0) my_syslog(MS_DHCP | LOG_INFO, _("%u broadcast response"), ntohl(mess->xid)); - + log_options(&mess->options[0] + sizeof(u32), mess->xid); - } - + } + ret = (size_t)(p - (unsigned char *)mess); - + if (ret < MIN_PACKETSZ) ret = MIN_PACKETSZ; - + return ret; } static unsigned char *free_space(struct dhcp_packet *mess, unsigned char *end, int opt, int len) { unsigned char *p = dhcp_skip_opts(&mess->options[0] + sizeof(u32)); - + if (p + len + 3 >= end) /* not enough space in options area, try and use overload, if poss */ { unsigned char *overload; - + if (!(overload = find_overload(mess)) && (mess->file[0] == 0 || mess->sname[0] == 0)) { @@ -1867,28 +1893,28 @@ static unsigned char *free_space(struct dhcp_packet *mess, unsigned char *end, i *(p++) = OPTION_OVERLOAD; *(p++) = 1; } - + p = NULL; - + /* using filename field ? */ if (overload) { if (mess->file[0] == 0) overload[2] |= 1; - + if (overload[2] & 1) { p = dhcp_skip_opts(mess->file); if (p + len + 3 >= mess->file + sizeof(mess->file)) p = NULL; } - + if (!p) { /* try to bring sname into play (it may be already) */ if (mess->sname[0] == 0) overload[2] |= 2; - + if (overload[2] & 2) { p = dhcp_skip_opts(mess->sname); @@ -1897,11 +1923,11 @@ static unsigned char *free_space(struct dhcp_packet *mess, unsigned char *end, i } } } - + if (!p) my_syslog(MS_DHCP | LOG_WARNING, _("cannot send DHCP/BOOTP option %d: no space left in packet"), opt); } - + if (p) { *(p++) = opt; @@ -1910,18 +1936,18 @@ static unsigned char *free_space(struct dhcp_packet *mess, unsigned char *end, i return p; } - + static void option_put(struct dhcp_packet *mess, unsigned char *end, int opt, int len, unsigned int val) { int i; unsigned char *p = free_space(mess, end, opt, len); - - if (p) + + if (p) for (i = 0; i < len; i++) *(p++) = val >> (8 * (len - (i + 1))); } -static void option_put_string(struct dhcp_packet *mess, unsigned char *end, int opt, +static void option_put_string(struct dhcp_packet *mess, unsigned char *end, int opt, char *string, int null_term) { unsigned char *p; @@ -1938,7 +1964,7 @@ static void option_put_string(struct dhcp_packet *mess, unsigned char *end, int static int do_opt(struct dhcp_opt *opt, unsigned char *p, struct dhcp_context *context, int null_term) { int len = opt->len; - + if ((opt->flags & DHOPT_STRING) && null_term && len != 255) len++; @@ -1961,7 +1987,7 @@ static int do_opt(struct dhcp_opt *opt, unsigned char *p, struct dhcp_context *c else /* empty string may be extended to "\0" by null_term */ memcpy(p, opt->val ? opt->val : (unsigned char *)"", len); - } + } return len; } @@ -1972,7 +1998,7 @@ static int in_list(unsigned char *list, int opt) /* If no requested options, send everything, not nothing. */ if (!list) return 1; - + for (i = 0; list[i] != OPTION_END; i++) if (opt == list[i]) return 1; @@ -1983,11 +2009,11 @@ static int in_list(unsigned char *list, int opt) static struct dhcp_opt *option_find2(int opt) { struct dhcp_opt *opts; - + for (opts = daemon->dhcp_opts; opts; opts = opts->next) if (opts->opt == opt && (opts->flags & DHOPT_TAGOK)) return opts; - + return NULL; } @@ -2013,13 +2039,13 @@ static void match_vendor_opts(unsigned char *opt, struct dhcp_opt *dopt) } } -static int do_encap_opts(struct dhcp_opt *opt, int encap, int flag, +static int do_encap_opts(struct dhcp_opt *opt, int encap, int flag, struct dhcp_packet *mess, unsigned char *end, int null_term) { int len, enc_len, ret = 0; struct dhcp_opt *start; unsigned char *p; - + /* find size in advance */ for (enc_len = 0, start = opt; opt; opt = opt->next) if (opt->flags & flag) @@ -2043,7 +2069,7 @@ static int do_encap_opts(struct dhcp_opt *opt, int encap, int flag, start = opt; } } - + if (enc_len != 0 && (p = free_space(mess, end, encap, enc_len + 1))) { @@ -2100,64 +2126,64 @@ static int pxe_uefi_workaround(int pxe_arch, struct dhcp_netid *netid, struct dh /* Only workaround UEFI archs. */ if (pxe_arch < 6) return 0; - + for (found = NULL, service = daemon->pxe_services; service; service = service->next) if (pxe_arch == service->CSA && service->basename && match_netid(service->netid, netid, 1)) { if (found) return 0; /* More than one relevant menu item */ - + found = service; } if (!found) return 0; /* No relevant menu items. */ - + if (!pxe) return 1; - + if (found->sname) { mess->siaddr = a_record_from_hosts(found->sname, now); snprintf((char *)mess->sname, sizeof(mess->sname), "%s", found->sname); } - else + else { if (found->server.s_addr != 0) - mess->siaddr = found->server; + mess->siaddr = found->server; else mess->siaddr = local; - + inet_ntop(AF_INET, &mess->siaddr, (char *)mess->sname, INET_ADDRSTRLEN); } - - snprintf((char *)mess->file, sizeof(mess->file), + + snprintf((char *)mess->file, sizeof(mess->file), strchr(found->basename, '.') ? "%s" : "%s.0", found->basename); - + return 1; } static struct dhcp_opt *pxe_opts(int pxe_arch, struct dhcp_netid *netid, struct in_addr local, time_t now) { -#define NUM_OPTS 4 +#define NUM_OPTS 4 unsigned char *p, *q; struct pxe_service *service; static struct dhcp_opt *o, *ret; int i, j = NUM_OPTS - 1; struct in_addr boot_server; - + /* We pass back references to these, hence they are declared static */ static unsigned char discovery_control; - static unsigned char fake_prompt[] = { 0, 'P', 'X', 'E' }; + static unsigned char fake_prompt[] = { 0, 'P', 'X', 'E' }; static struct dhcp_opt *fake_opts = NULL; - + /* Disable multicast, since we don't support it, and broadcast unless we need it */ discovery_control = 3; - + ret = daemon->dhcp_opts; - + if (!fake_opts && !(fake_opts = whine_malloc(NUM_OPTS * sizeof(struct dhcp_opt)))) return ret; @@ -2167,7 +2193,7 @@ static struct dhcp_opt *pxe_opts(int pxe_arch, struct dhcp_netid *netid, struct fake_opts[i].netid = NULL; fake_opts[i].next = i == (NUM_OPTS - 1) ? ret : &fake_opts[i+1]; } - + /* create the data for the PXE_MENU and PXE_SERVERS options. */ p = (unsigned char *)daemon->dhcp_buff; q = (unsigned char *)daemon->dhcp_buff3; @@ -2193,15 +2219,15 @@ static struct dhcp_opt *pxe_opts(int pxe_arch, struct dhcp_netid *netid, struct my_syslog(MS_DHCP | LOG_ERR, _("PXE menu too large")); return daemon->dhcp_opts; } - - boot_server = service->basename ? local : + + boot_server = service->basename ? local : (service->sname ? a_record_from_hosts(service->sname, now) : service->server); - + if (boot_server.s_addr != 0) { if (q - (unsigned char *)daemon->dhcp_buff3 + 3 + INADDRSZ >= 253) goto toobig; - + /* Boot service with known address - give it */ *(q++) = service->type >> 8; *(q++) = service->type; @@ -2218,7 +2244,7 @@ static struct dhcp_opt *pxe_opts(int pxe_arch, struct dhcp_netid *netid, struct /* if no prompt, wait forever if there's a choice */ fake_prompt[0] = (i > 1) ? 255 : 0; - + if (i == 0) discovery_control = 8; /* no menu - just use use mess->filename */ else @@ -2230,7 +2256,7 @@ static struct dhcp_opt *pxe_opts(int pxe_arch, struct dhcp_netid *netid, struct if (q - (unsigned char *)daemon->dhcp_buff3 != 0) { - ret = &fake_opts[j--]; + ret = &fake_opts[j--]; ret->len = q - (unsigned char *)daemon->dhcp_buff3; ret->val = (unsigned char *)daemon->dhcp_buff3; ret->opt = SUBOPT_PXE_SERVERS; @@ -2240,23 +2266,23 @@ static struct dhcp_opt *pxe_opts(int pxe_arch, struct dhcp_netid *netid, struct for (o = daemon->dhcp_opts; o; o = o->next) if ((o->flags & DHOPT_VENDOR_MATCH) && o->opt == SUBOPT_PXE_MENU_PROMPT) break; - + if (!o) { - ret = &fake_opts[j--]; + ret = &fake_opts[j--]; ret->len = sizeof(fake_prompt); ret->val = fake_prompt; ret->opt = SUBOPT_PXE_MENU_PROMPT; } - - ret = &fake_opts[j--]; + + ret = &fake_opts[j--]; ret->len = 1; ret->opt = SUBOPT_PXE_DISCOVERY; ret->val= &discovery_control; - + return ret; } - + static void clear_packet(struct dhcp_packet *mess, unsigned char *end) { memset(mess->sname, 0, sizeof(mess->sname)); @@ -2284,9 +2310,9 @@ struct dhcp_boot *find_boot(struct dhcp_netid *netid) static void do_options(struct dhcp_context *context, struct dhcp_packet *mess, - unsigned char *end, + unsigned char *end, unsigned char *req_options, - char *hostname, + char *hostname, char *domain, struct dhcp_netid *netid, struct in_addr subnet_addr, @@ -2312,7 +2338,7 @@ static void do_options(struct dhcp_context *context, if (context) context->netid.next = NULL; tagif = option_filter(netid, context && context->netid.net ? &context->netid : NULL, config_opts); - + /* logging */ if (option_bool(OPT_LOG_OPTS) && req_options) { @@ -2321,10 +2347,10 @@ static void do_options(struct dhcp_context *context, { char *s = option_string(AF_INET, req_options[i], NULL, 0, NULL, 0); q += snprintf(q, MAXDNAME - (q - daemon->namebuff), - "%d%s%s%s", + "%d%s%s%s", req_options[i], strlen(s) != 0 ? ":" : "", - s, + s, req_options[i+1] == OPTION_END ? "" : ", "); if (req_options[i+1] == OPTION_END || (q - daemon->namebuff) > 40) { @@ -2333,45 +2359,45 @@ static void do_options(struct dhcp_context *context, } } } - + for (id_list = daemon->force_broadcast; id_list; id_list = id_list->next) if ((!id_list->list) || match_netid(id_list->list, netid, 0)) break; if (id_list) mess->flags |= htons(0x8000); /* force broadcast */ - + if (context) mess->siaddr = context->local; - + /* See if we can send the boot stuff as options. To do this we need a requested option list, BOOTP - and very old DHCP clients won't have this, we also + and very old DHCP clients won't have this, we also provide a manual option to disable it. - Some PXE ROMs have bugs (surprise!) and need zero-terminated + Some PXE ROMs have bugs (surprise!) and need zero-terminated names, so we always send those. */ if ((boot = find_boot(tagif))) { if (boot->sname) - { + { if (!option_bool(OPT_NO_OVERRIDE) && - req_options && + req_options && in_list(req_options, OPTION_SNAME)) option_put_string(mess, end, OPTION_SNAME, boot->sname, 1); else safe_strncpy((char *)mess->sname, boot->sname, sizeof(mess->sname)); } - + if (boot->file) { if (!option_bool(OPT_NO_OVERRIDE) && - req_options && + req_options && in_list(req_options, OPTION_FILENAME)) option_put_string(mess, end, OPTION_FILENAME, boot->file, 1); else safe_strncpy((char *)mess->file, boot->file, sizeof(mess->file)); } - - if (boot->next_server.s_addr) + + if (boot->next_server.s_addr) mess->siaddr = boot->next_server; else if (boot->tftp_sname) mess->siaddr = a_record_from_hosts(boot->tftp_sname, now); @@ -2388,18 +2414,18 @@ static void do_options(struct dhcp_context *context, safe_strncpy((char *)mess->file, (char *)opt->val, sizeof(mess->file)); done_file = 1; } - + if ((!req_options || !in_list(req_options, OPTION_SNAME)) && (opt = option_find2(OPTION_SNAME)) && !(opt->flags & DHOPT_FORCE)) { safe_strncpy((char *)mess->sname, (char *)opt->val, sizeof(mess->sname)); done_server = 1; } - + if ((opt = option_find2(OPTION_END))) - mess->siaddr.s_addr = ((struct in_addr *)opt->val)->s_addr; + mess->siaddr.s_addr = ((struct in_addr *)opt->val)->s_addr; } - + /* We don't want to do option-overload for BOOTP, so make the file and sname fields look like they are in use, even when they aren't. This gets restored at the end of this function. */ @@ -2411,7 +2437,7 @@ static void do_options(struct dhcp_context *context, s0 = mess->sname[0]; mess->sname[0] = 1; } - + /* At this point, if mess->sname or mess->file are zeroed, they are available for option overload, reserve space for the overload option. */ if (mess->file[0] == 0 || mess->sname[0] == 0) @@ -2420,13 +2446,13 @@ static void do_options(struct dhcp_context *context, /* rfc3011 says this doesn't need to be in the requested options list. */ if (subnet_addr.s_addr) option_put(mess, end, OPTION_SUBNET_SELECT, INADDRSZ, ntohl(subnet_addr.s_addr)); - + if (lease_time != 0xffffffff) - { - unsigned int t1val = lease_time/2; + { + unsigned int t1val = lease_time/2; unsigned int t2val = (lease_time*7)/8; unsigned int hval; - + /* If set by user, sanity check, so not longer than lease. */ if ((opt = option_find2(OPTION_T1))) { @@ -2441,17 +2467,17 @@ static void do_options(struct dhcp_context *context, if (hval < lease_time && hval > 2) t2val = hval; } - + /* ensure T1 is still < T2 */ if (t2val <= t1val) - t1val = t2val - 1; + t1val = t2val - 1; while (fuzz > (t1val/8)) fuzz = fuzz/2; - + t1val -= fuzz; t2val -= fuzz; - + option_put(mess, end, OPTION_T1, 4, t1val); option_put(mess, end, OPTION_T2, 4, t2val); } @@ -2461,41 +2487,41 @@ static void do_options(struct dhcp_context *context, { if (!option_find2(OPTION_NETMASK)) option_put(mess, end, OPTION_NETMASK, INADDRSZ, ntohl(context->netmask.s_addr)); - + /* May not have a "guessed" broadcast address if we got no packets via a relay from this net yet (ie just unicast renewals after a restart */ if (context->broadcast.s_addr && !option_find2(OPTION_BROADCAST)) option_put(mess, end, OPTION_BROADCAST, INADDRSZ, ntohl(context->broadcast.s_addr)); - + /* Same comments as broadcast apply, and also may not be able to get a sensible default when using subnet select. User must configure by steam in that case. */ if (context->router.s_addr && in_list(req_options, OPTION_ROUTER) && !option_find2(OPTION_ROUTER)) option_put(mess, end, OPTION_ROUTER, INADDRSZ, ntohl(context->router.s_addr)); - + if (daemon->port == NAMESERVER_PORT && in_list(req_options, OPTION_DNSSERVER) && !option_find2(OPTION_DNSSERVER)) option_put(mess, end, OPTION_DNSSERVER, INADDRSZ, ntohl(context->local.s_addr)); } - if (domain && in_list(req_options, OPTION_DOMAINNAME) && + if (domain && in_list(req_options, OPTION_DOMAINNAME) && !option_find2(OPTION_DOMAINNAME)) option_put_string(mess, end, OPTION_DOMAINNAME, domain, null_term); - + /* Note that we ignore attempts to set the fqdn using --dhc-option=81, */ if (hostname) { if (in_list(req_options, OPTION_HOSTNAME) && !option_find2(OPTION_HOSTNAME)) option_put_string(mess, end, OPTION_HOSTNAME, hostname, null_term); - + if (fqdn_flags != 0) { len = strlen(hostname) + 3; - + if (fqdn_flags & 0x04) len += 2; else if (null_term) @@ -2508,7 +2534,7 @@ static void do_options(struct dhcp_context *context, if ((p = free_space(mess, end, OPTION_CLIENT_FQDN, len))) { - *(p++) = fqdn_flags & 0x0f; /* MBZ bits to zero */ + *(p++) = fqdn_flags & 0x0f; /* MBZ bits to zero */ *(p++) = 255; *(p++) = 255; @@ -2536,7 +2562,7 @@ static void do_options(struct dhcp_context *context, } } } - } + } for (opt = config_opts; opt; opt = opt->next) { @@ -2545,11 +2571,11 @@ static void do_options(struct dhcp_context *context, /* netids match and not encapsulated? */ if (!(opt->flags & DHOPT_TAGOK)) continue; - + /* was it asked for, or are we sending it anyway? */ if (!(opt->flags & DHOPT_FORCE) && !in_list(req_options, optno)) continue; - + /* prohibit some used-internally options. T1 and T2 already handled. */ if (optno == OPTION_CLIENT_FQDN || optno == OPTION_MAXMESSAGE || @@ -2565,15 +2591,15 @@ static void do_options(struct dhcp_context *context, if (optno == OPTION_FILENAME && done_file) continue; - + /* For the options we have default values on dhc-option= means "don't include this option" not "include a zero-length option" */ - if (opt->len == 0 && + if (opt->len == 0 && (optno == OPTION_NETMASK || optno == OPTION_BROADCAST || optno == OPTION_ROUTER || - optno == OPTION_DNSSERVER || + optno == OPTION_DNSSERVER || optno == OPTION_DOMAINNAME || optno == OPTION_HOSTNAME)) continue; @@ -2581,27 +2607,27 @@ static void do_options(struct dhcp_context *context, /* vendor-class comes from elsewhere for PXE */ if (pxe_arch != -1 && optno == OPTION_VENDOR_ID) continue; - + /* always force null-term for filename and servername - buggy PXE again. */ - len = do_opt(opt, NULL, context, + len = do_opt(opt, NULL, context, (optno == OPTION_SNAME || optno == OPTION_FILENAME) ? 1 : null_term); if ((p = free_space(mess, end, optno, len))) { - do_opt(opt, p, context, + do_opt(opt, p, context, (optno == OPTION_SNAME || optno == OPTION_FILENAME) ? 1 : null_term); - - /* If we send a vendor-id, revisit which vendor-ops we consider + + /* If we send a vendor-id, revisit which vendor-ops we consider it appropriate to send. */ if (optno == OPTION_VENDOR_ID) { match_vendor_opts(p - 2, config_opts); done_vendor_class = 1; } - } + } } - /* Now send options to be encapsulated in arbitrary options, + /* Now send options to be encapsulated in arbitrary options, eg dhcp-option=encap:172,17,....... Also handle vendor-identifying vendor-encapsulated options, dhcp-option = vi-encap:13,17,....... @@ -2609,11 +2635,11 @@ static void do_options(struct dhcp_context *context, all the options which match each outer in turn. */ for (opt = config_opts; opt; opt = opt->next) opt->flags &= ~DHOPT_ENCAP_DONE; - + for (opt = config_opts; opt; opt = opt->next) { int flags; - + if ((flags = (opt->flags & (DHOPT_ENCAPSULATE | DHOPT_RFC3925)))) { int found = 0; @@ -2627,10 +2653,10 @@ static void do_options(struct dhcp_context *context, int outer = flags & DHOPT_ENCAPSULATE ? o->u.encap : OPTION_VENDOR_IDENT_OPT; o->flags &= ~DHOPT_ENCAP_MATCH; - + if (!(o->flags & flags) || opt->u.encap != o->u.encap) continue; - + o->flags |= DHOPT_ENCAP_DONE; if (match_netid(o->netid, tagif, 1) && ((o->flags & DHOPT_FORCE) || in_list(req_options, outer))) @@ -2639,10 +2665,10 @@ static void do_options(struct dhcp_context *context, found = 1; len += do_opt(o, NULL, NULL, 0) + 2; } - } - + } + if (found) - { + { if (flags & DHOPT_ENCAPSULATE) do_encap_opts(config_opts, opt->u.encap, DHOPT_ENCAP_MATCH, mess, end, null_term); else if (len > 250) @@ -2660,14 +2686,14 @@ static void do_options(struct dhcp_context *context, *(p++) = o->opt; *(p++) = len; p += len; - } + } } } } - } + } force_encap = prune_vendor_opts(tagif); - + if (context && pxe_arch != -1) { pxe_misc(mess, end, uuid); @@ -2676,13 +2702,13 @@ static void do_options(struct dhcp_context *context, } if ((force_encap || in_list(req_options, OPTION_VENDOR_CLASS_OPT)) && - do_encap_opts(config_opts, OPTION_VENDOR_CLASS_OPT, DHOPT_VENDOR_MATCH, mess, end, null_term) && + do_encap_opts(config_opts, OPTION_VENDOR_CLASS_OPT, DHOPT_VENDOR_MATCH, mess, end, null_term) && pxe_arch == -1 && !done_vendor_class && vendor_class_len != 0 && (p = free_space(mess, end, OPTION_VENDOR_ID, vendor_class_len))) /* If we send vendor encapsulated options, and haven't already sent option 60, echo back the value we got from the client. */ - memcpy(p, daemon->dhcp_buff3, vendor_class_len); - + memcpy(p, daemon->dhcp_buff3, vendor_class_len); + /* restore BOOTP anti-overload hack */ if (!req_options || option_bool(OPT_NO_OVERRIDE)) { @@ -2694,12 +2720,12 @@ static void do_options(struct dhcp_context *context, static void apply_delay(u32 xid, time_t recvtime, struct dhcp_netid *netid) { struct delay_config *delay_conf; - + /* Decide which delay_config option we're using */ for (delay_conf = daemon->delay_conf; delay_conf; delay_conf = delay_conf->next) if (match_netid(delay_conf->netid, netid, 0)) break; - + if (!delay_conf) /* No match, look for one without a netid */ for (delay_conf = daemon->delay_conf; delay_conf; delay_conf = delay_conf->next) @@ -2715,10 +2741,10 @@ static void apply_delay(u32 xid, time_t recvtime, struct dhcp_netid *netid) } #endif + + + - - - - + diff --git a/dnsmasq/rfc3315.c b/src/dnsmasq/rfc3315.c similarity index 84% rename from dnsmasq/rfc3315.c rename to src/dnsmasq/rfc3315.c index ac9db5fed..b3f0a0a0d 100644 --- a/dnsmasq/rfc3315.c +++ b/src/dnsmasq/rfc3315.c @@ -1,15 +1,15 @@ -/* dnsmasq is Copyright (c) 2000-2018 Simon Kelley +/* dnsmasq is Copyright (c) 2000-2020 Simon Kelley This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; version 2 dated June, 1991, or (at your option) version 3 dated 29 June, 2007. - + This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. - + You should have received a copy of the GNU General Public License along with this program. If not, see . */ @@ -21,22 +21,19 @@ struct state { unsigned char *clid; - int clid_len, iaid, ia_type, interface, hostname_auth, lease_allocate; + int clid_len, ia_type, interface, hostname_auth, lease_allocate; char *client_hostname, *hostname, *domain, *send_domain; struct dhcp_context *context; struct in6_addr *link_address, *fallback, *ll_addr, *ula_addr; - unsigned int xid, fqdn_flags; + unsigned int xid, fqdn_flags, iaid; char *iface_name; void *packet_options, *end; struct dhcp_netid *tags, *context_tags; unsigned char mac[DHCP_CHADDR_MAX]; unsigned int mac_len, mac_type; -#ifdef OPTION6_PREFIX_CLASS - struct prefix_class *send_prefix_class; -#endif }; -static int dhcp6_maybe_relay(struct state *state, void *inbuff, size_t sz, +static int dhcp6_maybe_relay(struct state *state, void *inbuff, size_t sz, struct in6_addr *client_addr, int is_unicast, time_t now); static int dhcp6_no_relay(struct state *state, int msg_type, void *inbuff, size_t sz, int is_unicast, time_t now); static void log6_opts(int nest, unsigned int xid, void *start_opts, void *end_opts); @@ -49,18 +46,17 @@ static void get_context_tag(struct state *state, struct dhcp_context *context); static int check_ia(struct state *state, void *opt, void **endp, void **ia_option); static int build_ia(struct state *state, int *t1cntr); static void end_ia(int t1cntr, unsigned int min_time, int do_fuzz); -#ifdef OPTION6_PREFIX_CLASS -static struct prefix_class *prefix_class_from_context(struct dhcp_context *context); -#endif static void mark_context_used(struct state *state, struct in6_addr *addr); static void mark_config_used(struct dhcp_context *context, struct in6_addr *addr); static int check_address(struct state *state, struct in6_addr *addr); -static void add_address(struct state *state, struct dhcp_context *context, unsigned int lease_time, void *ia_option, +static int config_valid(struct dhcp_config *config, struct dhcp_context *context, struct in6_addr *addr, struct state *state, time_t now); +static struct addrlist *config_implies(struct dhcp_config *config, struct dhcp_context *context, struct in6_addr *addr); +static void add_address(struct state *state, struct dhcp_context *context, unsigned int lease_time, void *ia_option, unsigned int *min_time, struct in6_addr *addr, time_t now); static void update_leases(struct state *state, struct dhcp_context *context, struct in6_addr *addr, unsigned int lease_time, time_t now); static int add_local_addrs(struct dhcp_context *context); static struct dhcp_netid *add_options(struct state *state, int do_refresh); -static void calculate_times(struct dhcp_context *context, unsigned int *min_time, unsigned int *valid_timep, +static void calculate_times(struct dhcp_context *context, unsigned int *min_time, unsigned int *valid_timep, unsigned int *preferred_timep, unsigned int lease_time); #define opt6_len(opt) ((int)(opt6_uint(opt, -2, 2))) @@ -70,7 +66,7 @@ static void calculate_times(struct dhcp_context *context, unsigned int *min_time #define opt6_user_vendor_ptr(opt, i) ((void *)&(((unsigned char *)(opt))[2+(i)])) #define opt6_user_vendor_len(opt) ((int)(opt6_uint(opt, -4, 2))) #define opt6_user_vendor_next(opt, end) (opt6_next(((void *) opt) - 2, end)) - + unsigned short dhcp6_reply(struct dhcp_context *context, int interface, char *iface_name, struct in6_addr *fallback, struct in6_addr *ll_addr, struct in6_addr *ula_addr, @@ -79,16 +75,16 @@ unsigned short dhcp6_reply(struct dhcp_context *context, int interface, char *if struct dhcp_vendor *vendor; int msg_type; struct state state; - + if (sz <= 4) return 0; - + msg_type = *((unsigned char *)daemon->dhcp_packet.iov_base); - + /* Mark these so we only match each at most once, to avoid tangled linked lists */ for (vendor = daemon->dhcp_vendors; vendor; vendor = vendor->next) vendor->netid.next = &vendor->netid; - + reset_counter(); state.context = context; state.interface = interface; @@ -100,7 +96,7 @@ unsigned short dhcp6_reply(struct dhcp_context *context, int interface, char *if state.tags = NULL; state.link_address = NULL; - if (dhcp6_maybe_relay(&state, daemon->dhcp_packet.iov_base, sz, client_addr, + if (dhcp6_maybe_relay(&state, daemon->dhcp_packet.iov_base, sz, client_addr, IN6_IS_ADDR_MULTICAST(client_addr), now)) return msg_type == DHCP6RELAYFORW ? DHCPV6_SERVER_PORT : DHCPV6_CLIENT_PORT; @@ -108,7 +104,7 @@ unsigned short dhcp6_reply(struct dhcp_context *context, int interface, char *if } /* This cost me blood to write, it will probably cost you blood to understand - srk. */ -static int dhcp6_maybe_relay(struct state *state, void *inbuff, size_t sz, +static int dhcp6_maybe_relay(struct state *state, void *inbuff, size_t sz, struct in6_addr *client_addr, int is_unicast, time_t now) { void *end = inbuff + sz; @@ -121,48 +117,68 @@ static int dhcp6_maybe_relay(struct state *state, void *inbuff, size_t sz, /* if not an encapsulated relayed message, just do the stuff */ if (msg_type != DHCP6RELAYFORW) { - /* if link_address != NULL if points to the link address field of the + /* if link_address != NULL if points to the link address field of the innermost nested RELAYFORW message, which is where we find the address of the network on which we can allocate an address. - Recalculate the available contexts using that information. + Recalculate the available contexts using that information. - link_address == NULL means there's no relay in use, so we try and find the client's + link_address == NULL means there's no relay in use, so we try and find the client's MAC address from the local ND cache. */ - + if (!state->link_address) get_client_mac(client_addr, state->interface, state->mac, &state->mac_len, &state->mac_type, now); else { struct dhcp_context *c; + struct shared_network *share = NULL; state->context = NULL; if (!IN6_IS_ADDR_LOOPBACK(state->link_address) && !IN6_IS_ADDR_LINKLOCAL(state->link_address) && !IN6_IS_ADDR_MULTICAST(state->link_address)) for (c = daemon->dhcp6; c; c = c->next) - if ((c->flags & CONTEXT_DHCP) && - !(c->flags & (CONTEXT_TEMPLATE | CONTEXT_OLD)) && - is_same_net6(state->link_address, &c->start6, c->prefix) && - is_same_net6(state->link_address, &c->end6, c->prefix)) - { - c->preferred = c->valid = 0xffffffff; - c->current = state->context; - state->context = c; - } - + { + for (share = daemon->shared_networks; share; share = share->next) + { + if (share->shared_addr.s_addr != 0) + continue; + + if (share->if_index != 0 || + !IN6_ARE_ADDR_EQUAL(state->link_address, &share->match_addr6)) + continue; + + if ((c->flags & CONTEXT_DHCP) && + !(c->flags & (CONTEXT_TEMPLATE | CONTEXT_OLD)) && + is_same_net6(&share->shared_addr6, &c->start6, c->prefix) && + is_same_net6(&share->shared_addr6, &c->end6, c->prefix)) + break; + } + + if (share || + ((c->flags & CONTEXT_DHCP) && + !(c->flags & (CONTEXT_TEMPLATE | CONTEXT_OLD)) && + is_same_net6(state->link_address, &c->start6, c->prefix) && + is_same_net6(state->link_address, &c->end6, c->prefix))) + { + c->preferred = c->valid = 0xffffffff; + c->current = state->context; + state->context = c; + } + } + if (!state->context) { - inet_ntop(AF_INET6, state->link_address, daemon->addrbuff, ADDRSTRLEN); - my_syslog(MS_DHCP | LOG_WARNING, + inet_ntop(AF_INET6, state->link_address, daemon->addrbuff, ADDRSTRLEN); + my_syslog(MS_DHCP | LOG_WARNING, _("no address range available for DHCPv6 request from relay at %s"), daemon->addrbuff); return 0; } } - + if (!state->context) { - my_syslog(MS_DHCP | LOG_WARNING, + my_syslog(MS_DHCP | LOG_WARNING, _("no address range available for DHCPv6 request via %s"), state->iface_name); return 0; } @@ -174,7 +190,7 @@ static int dhcp6_maybe_relay(struct state *state, void *inbuff, size_t sz, which is 1 + 1 + 16 + 16 + 2 + 2 = 38 */ if (sz < 38) return 0; - + /* copy header stuff into reply message and set type to reply */ if (!(outmsgtypep = put_opt6(inbuff, 34))) return 0; @@ -184,11 +200,11 @@ static int dhcp6_maybe_relay(struct state *state, void *inbuff, size_t sz, for (vendor = daemon->dhcp_vendors; vendor; vendor = vendor->next) { int mopt; - + if (vendor->match_type == MATCH_SUBSCRIBER) mopt = OPTION6_SUBSCRIBER_ID; else if (vendor->match_type == MATCH_REMOTE) - mopt = OPTION6_REMOTE_ID; + mopt = OPTION6_REMOTE_ID; else continue; @@ -202,7 +218,7 @@ static int dhcp6_maybe_relay(struct state *state, void *inbuff, size_t sz, break; } } - + /* RFC-6939 */ if ((opt = opt6_find(opts, end, OPTION6_CLIENT_MAC, 3))) { @@ -213,29 +229,33 @@ static int dhcp6_maybe_relay(struct state *state, void *inbuff, size_t sz, state->mac_len = opt6_len(opt) - 2; memcpy(&state->mac[0], opt6_ptr(opt, 2), state->mac_len); } - + for (opt = opts; opt; opt = opt6_next(opt, end)) { - if (opt6_ptr(opt, 0) + opt6_len(opt) > end) + if (opt6_ptr(opt, 0) + opt6_len(opt) > end) return 0; - - int o = new_opt6(opt6_type(opt)); - if (opt6_type(opt) == OPTION6_RELAY_MSG) + + /* Don't copy MAC address into reply. */ + if (opt6_type(opt) != OPTION6_CLIENT_MAC) { - struct in6_addr align; - /* the packet data is unaligned, copy to aligned storage */ - memcpy(&align, inbuff + 2, IN6ADDRSZ); - state->link_address = &align; - /* zero is_unicast since that is now known to refer to the - relayed packet, not the original sent by the client */ - if (!dhcp6_maybe_relay(state, opt6_ptr(opt, 0), opt6_len(opt), client_addr, 0, now)) - return 0; + int o = new_opt6(opt6_type(opt)); + if (opt6_type(opt) == OPTION6_RELAY_MSG) + { + struct in6_addr align; + /* the packet data is unaligned, copy to aligned storage */ + memcpy(&align, inbuff + 2, IN6ADDRSZ); + state->link_address = &align; + /* zero is_unicast since that is now known to refer to the + relayed packet, not the original sent by the client */ + if (!dhcp6_maybe_relay(state, opt6_ptr(opt, 0), opt6_len(opt), client_addr, 0, now)) + return 0; + } + else + put_opt6(opt6_ptr(opt, 0), opt6_len(opt)); + end_opt6(o); } - else if (opt6_type(opt) != OPTION6_CLIENT_MAC) - put_opt6(opt6_ptr(opt, 0), opt6_len(opt)); - end_opt6(o); } - + return 1; } @@ -252,10 +272,6 @@ static int dhcp6_no_relay(struct state *state, int msg_type, void *inbuff, size_ struct dhcp_context *context_tmp; struct dhcp_mac *mac_opt; unsigned int ignore = 0; -#ifdef OPTION6_PREFIX_CLASS - struct prefix_class *p; - int dump_all_prefix_classes = 0; -#endif state->packet_options = inbuff + 4; state->end = inbuff + sz; @@ -269,14 +285,11 @@ static int dhcp6_no_relay(struct state *state, int msg_type, void *inbuff, size_ state->hostname = NULL; state->client_hostname = NULL; state->fqdn_flags = 0x01; /* default to send if we receive no FQDN option */ -#ifdef OPTION6_PREFIX_CLASS - state->send_prefix_class = NULL; -#endif /* set tag with name == interface */ iface_id.net = state->iface_name; iface_id.next = state->tags; - state->tags = &iface_id; + state->tags = &iface_id; /* set tag "dhcpv6" */ v6_id.net = "dhcpv6"; @@ -288,8 +301,8 @@ static int dhcp6_no_relay(struct state *state, int msg_type, void *inbuff, size_ return 0; start_opts = save_counter(-1); state->xid = outmsgtypep[3] | outmsgtypep[2] << 8 | outmsgtypep[1] << 16; - - /* We're going to be linking tags from all context we use. + + /* We're going to be linking tags from all context we use. mark them as unused so we don't link one twice and break the list */ for (context_tmp = state->context; context_tmp; context_tmp = context_tmp->current) { @@ -297,13 +310,13 @@ static int dhcp6_no_relay(struct state *state, int msg_type, void *inbuff, size_ if (option_bool(OPT_LOG_OPTS)) { - inet_ntop(AF_INET6, &context_tmp->start6, daemon->dhcp_buff, ADDRSTRLEN); - inet_ntop(AF_INET6, &context_tmp->end6, daemon->dhcp_buff2, ADDRSTRLEN); + inet_ntop(AF_INET6, &context_tmp->start6, daemon->dhcp_buff, ADDRSTRLEN); + inet_ntop(AF_INET6, &context_tmp->end6, daemon->dhcp_buff2, ADDRSTRLEN); if (context_tmp->flags & (CONTEXT_STATIC)) my_syslog(MS_DHCP | LOG_INFO, _("%u available DHCPv6 subnet: %s/%d"), state->xid, daemon->dhcp_buff, context_tmp->prefix); else - my_syslog(MS_DHCP | LOG_INFO, _("%u available DHCP range: %s -- %s"), + my_syslog(MS_DHCP | LOG_INFO, _("%u available DHCP range: %s -- %s"), state->xid, daemon->dhcp_buff, daemon->dhcp_buff2); } } @@ -325,15 +338,15 @@ static int dhcp6_no_relay(struct state *state, int msg_type, void *inbuff, size_ opt6_len(opt) != daemon->duid_len || memcmp(opt6_ptr(opt, 0), daemon->duid, daemon->duid_len) != 0)) return 0; - + o = new_opt6(OPTION6_SERVER_ID); put_opt6(daemon->duid, daemon->duid_len); end_opt6(o); if (is_unicast && (msg_type == DHCP6REQUEST || msg_type == DHCP6RENEW || msg_type == DHCP6RELEASE || msg_type == DHCP6DECLINE)) - - { + + { *outmsgtypep = DHCP6REPLY; o1 = new_opt6(OPTION6_STATUS_CODE); put_opt6_short(DHCP6USEMULTI); @@ -346,11 +359,11 @@ static int dhcp6_no_relay(struct state *state, int msg_type, void *inbuff, size_ for (vendor = daemon->dhcp_vendors; vendor; vendor = vendor->next) { int mopt; - + if (vendor->match_type == MATCH_VENDOR) mopt = OPTION6_VENDOR_CLASS; else if (vendor->match_type == MATCH_USER) - mopt = OPTION6_USER_CLASS; + mopt = OPTION6_USER_CLASS; else continue; @@ -358,18 +371,18 @@ static int dhcp6_no_relay(struct state *state, int msg_type, void *inbuff, size_ { void *enc_opt, *enc_end = opt6_ptr(opt, opt6_len(opt)); int offset = 0; - + if (mopt == OPTION6_VENDOR_CLASS) { if (opt6_len(opt) < 4) continue; - + if (vendor->enterprise != opt6_uint(opt, 0, 4)) continue; - + offset = 4; } - + /* Note that format if user/vendor classes is different to DHCP options - no option types. */ for (enc_opt = opt6_ptr(opt, offset); enc_opt; enc_opt = opt6_user_vendor_next(enc_opt, enc_end)) for (i = 0; i <= (opt6_user_vendor_len(enc_opt) - vendor->len); i++) @@ -384,15 +397,15 @@ static int dhcp6_no_relay(struct state *state, int msg_type, void *inbuff, size_ if (option_bool(OPT_LOG_OPTS) && (opt = opt6_find(state->packet_options, state->end, OPTION6_VENDOR_CLASS, 4))) my_syslog(MS_DHCP | LOG_INFO, _("%u vendor class: %u"), state->xid, opt6_uint(opt, 0, 4)); - + /* dhcp-match. If we have hex-and-wildcards, look for a left-anchored match. - Otherwise assume the option is an array, and look for a matching element. - If no data given, existence of the option is enough. This code handles + Otherwise assume the option is an array, and look for a matching element. + If no data given, existence of the option is enough. This code handles V-I opts too. */ for (opt_cfg = daemon->dhcp_match6; opt_cfg; opt_cfg = opt_cfg->next) { int match = 0; - + if (opt_cfg->flags & DHOPT_RFC3925) { for (opt = opt6_find(state->packet_options, state->end, OPTION6_VENDOR_OPTS, 4); @@ -401,7 +414,7 @@ static int dhcp6_no_relay(struct state *state, int msg_type, void *inbuff, size_ { void *vopt; void *vend = opt6_ptr(opt, opt6_len(opt)); - + for (vopt = opt6_find(opt6_ptr(opt, 4), vend, opt_cfg->opt, 0); vopt; vopt = opt6_find(opt6_next(vopt, vend), vend, opt_cfg->opt, 0)) @@ -415,10 +428,10 @@ static int dhcp6_no_relay(struct state *state, int msg_type, void *inbuff, size_ { if (!(opt = opt6_find(state->packet_options, state->end, opt_cfg->opt, 1))) continue; - + match = match_bytes(opt_cfg, opt6_ptr(opt, 0), opt6_len(opt)); - } - + } + if (match) { opt_cfg->netid->next = state->tags; @@ -443,25 +456,25 @@ static int dhcp6_no_relay(struct state *state, int msg_type, void *inbuff, size_ state->tags = &mac_opt->netid; } } - + if ((opt = opt6_find(state->packet_options, state->end, OPTION6_FQDN, 1))) { /* RFC4704 refers */ int len = opt6_len(opt) - 1; - + state->fqdn_flags = opt6_uint(opt, 0, 1); - + /* Always force update, since the client has no way to do it itself. */ if (!option_bool(OPT_FQDN_UPDATE) && !(state->fqdn_flags & 0x01)) state->fqdn_flags |= 0x03; - + state->fqdn_flags &= ~0x04; if (len != 0 && len < 255) { unsigned char *pp, *op = opt6_ptr(opt, 1); char *pq = daemon->dhcp_buff; - + pp = op; while (*op != 0 && ((op + (*op)) - pp) < len) { @@ -470,75 +483,73 @@ static int dhcp6_no_relay(struct state *state, int msg_type, void *inbuff, size_ op += (*op)+1; *(pq++) = '.'; } - + if (pq != daemon->dhcp_buff) pq--; *pq = 0; - + if (legal_hostname(daemon->dhcp_buff)) { + struct dhcp_match_name *m; + size_t nl = strlen(daemon->dhcp_buff); + state->client_hostname = daemon->dhcp_buff; + if (option_bool(OPT_LOG_OPTS)) my_syslog(MS_DHCP | LOG_INFO, _("%u client provides name: %s"), state->xid, state->client_hostname); + + for (m = daemon->dhcp_name_match; m; m = m->next) + { + size_t ml = strlen(m->name); + char save = 0; + + if (nl < ml) + continue; + if (nl > ml) + { + save = state->client_hostname[ml]; + state->client_hostname[ml] = 0; + } + + if (hostname_isequal(state->client_hostname, m->name) && + (save == 0 || m->wildcard)) + { + m->netid->next = state->tags; + state->tags = m->netid; + } + + if (save != 0) + state->client_hostname[ml] = save; + } } } + } + + if (state->clid && + (config = find_config(daemon->dhcp_conf, state->context, state->clid, state->clid_len, + state->mac, state->mac_len, state->mac_type, NULL, run_tag_if(state->tags))) && + have_config(config, CONFIG_NAME)) + { + state->hostname = config->hostname; + state->domain = config->domain; + state->hostname_auth = 1; } - - if (state->clid) + else if (state->client_hostname) { - config = find_config(daemon->dhcp_conf, state->context, state->clid, state->clid_len, state->mac, state->mac_len, state->mac_type, NULL); - - if (have_config(config, CONFIG_NAME)) - { - state->hostname = config->hostname; - state->domain = config->domain; - state->hostname_auth = 1; - } - else if (state->client_hostname) + state->domain = strip_hostname(state->client_hostname); + + if (strlen(state->client_hostname) != 0) { - struct dhcp_match_name *m; - size_t nl; - - state->domain = strip_hostname(state->client_hostname); - nl = strlen(state->client_hostname); - - if (strlen(state->client_hostname) != 0) + state->hostname = state->client_hostname; + + if (!config) { - state->hostname = state->client_hostname; - - if (!config) - { - /* Search again now we have a hostname. - Only accept configs without CLID here, (it won't match) - to avoid impersonation by name. */ - struct dhcp_config *new = find_config(daemon->dhcp_conf, state->context, NULL, 0, NULL, 0, 0, state->hostname); - if (new && !have_config(new, CONFIG_CLID) && !new->hwaddr) - config = new; - } - - for (m = daemon->dhcp_name_match; m; m = m->next) - { - size_t ml = strlen(m->name); - char save = 0; - - if (nl < ml) - continue; - if (nl > ml) - { - save = state->client_hostname[ml]; - state->client_hostname[ml] = 0; - } - - if (hostname_isequal(state->client_hostname, m->name) && - (save == 0 || m->wildcard)) - { - m->netid->next = state->tags; - state->tags = m->netid; - } - - if (save != 0) - state->client_hostname[ml] = save; - } + /* Search again now we have a hostname. + Only accept configs without CLID here, (it won't match) + to avoid impersonation by name. */ + struct dhcp_config *new = find_config(daemon->dhcp_conf, state->context, NULL, 0, NULL, 0, 0, state->hostname, run_tag_if(state->tags)); + if (new && !have_config(new, CONFIG_CLID) && !new->hwaddr) + config = new; } } } @@ -546,7 +557,7 @@ static int dhcp6_no_relay(struct state *state, int msg_type, void *inbuff, size_ if (config) { struct dhcp_netid_list *list; - + for (list = config->netid; list; list = list->next) { list->list->next = state->tags; @@ -562,78 +573,54 @@ static int dhcp6_no_relay(struct state *state, int msg_type, void *inbuff, size_ ignore = 1; } else if (state->clid && - find_config(daemon->dhcp_conf, NULL, state->clid, state->clid_len, state->mac, state->mac_len, state->mac_type, NULL)) + find_config(daemon->dhcp_conf, NULL, state->clid, state->clid_len, + state->mac, state->mac_len, state->mac_type, NULL, run_tag_if(state->tags))) { known_id.net = "known-othernet"; known_id.next = state->tags; state->tags = &known_id; } - -#ifdef OPTION6_PREFIX_CLASS - /* OPTION_PREFIX_CLASS in ORO, send addresses in all prefix classes */ - if (daemon->prefix_classes && (msg_type == DHCP6SOLICIT || msg_type == DHCP6REQUEST)) - { - void *oro; - - if ((oro = opt6_find(state->packet_options, state->end, OPTION6_ORO, 0))) - for (i = 0; i < opt6_len(oro) - 1; i += 2) - if (opt6_uint(oro, i, 2) == OPTION6_PREFIX_CLASS) - { - dump_all_prefix_classes = 1; - break; - } - - if (msg_type != DHCP6SOLICIT || dump_all_prefix_classes) - /* Add the tags associated with prefix classes so we can use the DHCP ranges. - Not done for SOLICIT as we add them one-at-time. */ - for (p = daemon->prefix_classes; p ; p = p->next) - { - p->tag.next = state->tags; - state->tags = &p->tag; - } - } -#endif - + tagif = run_tag_if(state->tags); - + /* if all the netids in the ignore list are present, ignore this client */ if (daemon->dhcp_ignore) { struct dhcp_netid_list *id_list; - + for (id_list = daemon->dhcp_ignore; id_list; id_list = id_list->next) if (match_netid(id_list->list, tagif, 0)) ignore = 1; } - + /* if all the netids in the ignore_name list are present, ignore client-supplied name */ if (!state->hostname_auth) { struct dhcp_netid_list *id_list; - + for (id_list = daemon->dhcp_ignore_names; id_list; id_list = id_list->next) if ((!id_list->list) || match_netid(id_list->list, tagif, 0)) break; if (id_list) state->hostname = NULL; } - + switch (msg_type) { default: return 0; - - + + case DHCP6SOLICIT: { int address_assigned = 0; /* tags without all prefix-class tags */ struct dhcp_netid *solicit_tags; struct dhcp_context *c; - + *outmsgtypep = DHCP6ADVERTISE; - + if (opt6_find(state->packet_options, state->end, OPTION6_RAPID_COMMIT, 0)) { *outmsgtypep = DHCP6REPLY; @@ -641,15 +628,15 @@ static int dhcp6_no_relay(struct state *state, int msg_type, void *inbuff, size_ o = new_opt6(OPTION6_RAPID_COMMIT); end_opt6(o); } - + log6_quiet(state, "DHCPSOLICIT", NULL, ignore ? _("ignored") : NULL); request_no_address: solicit_tags = tagif; - + if (ignore) return 0; - + /* reset USED bits in leases */ lease6_reset(); @@ -658,7 +645,7 @@ static int dhcp6_no_relay(struct state *state, int msg_type, void *inbuff, size_ c->flags &= ~CONTEXT_CONF_USED; for (opt = state->packet_options; opt; opt = opt6_next(opt, state->end)) - { + { void *ia_option, *ia_end; unsigned int min_time = 0xffffffff; int t1cntr; @@ -669,68 +656,14 @@ static int dhcp6_no_relay(struct state *state, int msg_type, void *inbuff, size_ u32 lease_time; struct dhcp_lease *ltmp; struct in6_addr req_addr, addr; - + if (!check_ia(state, opt, &ia_end, &ia_option)) continue; - + /* reset USED bits in contexts - one address per prefix per IAID */ for (c = state->context; c; c = c->current) c->flags &= ~CONTEXT_USED; -#ifdef OPTION6_PREFIX_CLASS - if (daemon->prefix_classes && state->ia_type == OPTION6_IA_NA) - { - void *prefix_opt; - int prefix_class; - - if (dump_all_prefix_classes) - /* OPTION_PREFIX_CLASS in ORO, send addresses in all prefix classes */ - plain_range = 0; - else - { - if ((prefix_opt = opt6_find(opt6_ptr(opt, 12), ia_end, OPTION6_PREFIX_CLASS, 2))) - { - - prefix_class = opt6_uint(prefix_opt, 0, 2); - - for (p = daemon->prefix_classes; p ; p = p->next) - if (p->class == prefix_class) - break; - - if (!p) - my_syslog(MS_DHCP | LOG_WARNING, _("unknown prefix-class %d"), prefix_class); - else - { - /* add tag to list, and exclude undecorated dhcp-ranges */ - p->tag.next = state->tags; - solicit_tags = run_tag_if(&p->tag); - plain_range = 0; - state->send_prefix_class = p; - } - } - else - { - /* client didn't ask for a prefix class, lets see if we can find one. */ - for (p = daemon->prefix_classes; p ; p = p->next) - { - p->tag.next = NULL; - if (match_netid(&p->tag, solicit_tags, 1)) - break; - } - - if (p) - { - plain_range = 0; - state->send_prefix_class = p; - } - } - - if (p && option_bool(OPT_LOG_OPTS)) - my_syslog(MS_DHCP | LOG_INFO, "%u prefix class %d tag:%s", state->xid, p->class, p->tag.net); - } - } -#endif - o = build_ia(state, &t1cntr); if (address_assigned) address_assigned = 2; @@ -739,14 +672,14 @@ static int dhcp6_no_relay(struct state *state, int msg_type, void *inbuff, size_ { /* worry about alignment here. */ memcpy(&req_addr, opt6_ptr(ia_option, 0), IN6ADDRSZ); - + if ((c = address6_valid(state->context, &req_addr, solicit_tags, plain_range))) { lease_time = c->lease_time; - /* If the client asks for an address on the same network as a configured address, + /* If the client asks for an address on the same network as a configured address, offer the configured address instead, to make moving to newly-configured addresses automatic. */ - if (!(c->flags & CONTEXT_CONF_USED) && config_valid(config, c, &addr) && check_address(state, &addr)) + if (!(c->flags & CONTEXT_CONF_USED) && config_valid(config, c, &addr, state, now)) { req_addr = addr; mark_config_used(c, &addr); @@ -757,42 +690,34 @@ static int dhcp6_no_relay(struct state *state, int msg_type, void *inbuff, size_ continue; /* not an address we're allowed */ else if (!check_address(state, &req_addr)) continue; /* address leased elsewhere */ - + /* add address to output packet */ -#ifdef OPTION6_PREFIX_CLASS - if (dump_all_prefix_classes && state->ia_type == OPTION6_IA_NA) - state->send_prefix_class = prefix_class_from_context(c); -#endif add_address(state, c, lease_time, ia_option, &min_time, &req_addr, now); mark_context_used(state, &req_addr); get_context_tag(state, c); address_assigned = 1; } } - + /* Suggest configured address(es) */ - for (c = state->context; c; c = c->current) + for (c = state->context; c; c = c->current) if (!(c->flags & CONTEXT_CONF_USED) && match_netid(c->filter, solicit_tags, plain_range) && - config_valid(config, c, &addr) && - check_address(state, &addr)) + config_valid(config, c, &addr, state, now)) { mark_config_used(state->context, &addr); if (have_config(config, CONFIG_TIME)) lease_time = config->lease_time; else lease_time = c->lease_time; + /* add address to output packet */ -#ifdef OPTION6_PREFIX_CLASS - if (dump_all_prefix_classes && state->ia_type == OPTION6_IA_NA) - state->send_prefix_class = prefix_class_from_context(c); -#endif add_address(state, c, lease_time, NULL, &min_time, &addr, now); mark_context_used(state, &addr); get_context_tag(state, c); address_assigned = 1; } - + /* return addresses for existing leases */ ltmp = NULL; while ((ltmp = lease6_find_by_client(ltmp, state->ia_type == OPTION6_IA_NA ? LEASE_NA : LEASE_TA, state->clid, state->clid_len, state->iaid))) @@ -800,31 +725,23 @@ static int dhcp6_no_relay(struct state *state, int msg_type, void *inbuff, size_ req_addr = ltmp->addr6; if ((c = address6_available(state->context, &req_addr, solicit_tags, plain_range))) { -#ifdef OPTION6_PREFIX_CLASS - if (dump_all_prefix_classes && state->ia_type == OPTION6_IA_NA) - state->send_prefix_class = prefix_class_from_context(c); -#endif add_address(state, c, c->lease_time, NULL, &min_time, &req_addr, now); mark_context_used(state, &req_addr); get_context_tag(state, c); address_assigned = 1; } } - + /* Return addresses for all valid contexts which don't yet have one */ while ((c = address6_allocate(state->context, state->clid, state->clid_len, state->ia_type == OPTION6_IA_TA, state->iaid, ia_counter, solicit_tags, plain_range, &addr))) { -#ifdef OPTION6_PREFIX_CLASS - if (dump_all_prefix_classes && state->ia_type == OPTION6_IA_NA) - state->send_prefix_class = prefix_class_from_context(c); -#endif add_address(state, c, c->lease_time, NULL, &min_time, &addr, now); mark_context_used(state, &addr); get_context_tag(state, c); address_assigned = 1; } - + if (address_assigned != 1) { /* If the server will not assign any addresses to any IAs in a @@ -835,7 +752,7 @@ static int dhcp6_no_relay(struct state *state, int msg_type, void *inbuff, size_ save_counter(o); continue; } - + /* If the server cannot assign any addresses to an IA in the message from the client, the server MUST include the IA in the Reply message with no addresses in the IA and a Status Code option in the IA @@ -845,18 +762,18 @@ static int dhcp6_no_relay(struct state *state, int msg_type, void *inbuff, size_ put_opt6_string(_("address unavailable")); end_opt6(o1); } - + end_ia(t1cntr, min_time, 0); - end_opt6(o); + end_opt6(o); } - if (address_assigned) + if (address_assigned) { o1 = new_opt6(OPTION6_STATUS_CODE); put_opt6_short(DHCP6SUCCESS); put_opt6_string(_("success")); end_opt6(o1); - + /* If --dhcp-authoritative is set, we can tell client not to wait for other possible servers */ o = new_opt6(OPTION6_PREFERENCE); @@ -865,7 +782,7 @@ static int dhcp6_no_relay(struct state *state, int msg_type, void *inbuff, size_ tagif = add_options(state, 0); } else - { + { /* no address, return error */ o1 = new_opt6(OPTION6_STATUS_CODE); put_opt6_short(DHCP6NOADDRS); @@ -885,7 +802,7 @@ static int dhcp6_no_relay(struct state *state, int msg_type, void *inbuff, size_ break; } - + case DHCP6REQUEST: { int address_assigned = 0; @@ -896,16 +813,16 @@ static int dhcp6_no_relay(struct state *state, int msg_type, void *inbuff, size_ state->lease_allocate = 1; log6_quiet(state, "DHCPREQUEST", NULL, ignore ? _("ignored") : NULL); - + if (ignore) return 0; - + for (opt = state->packet_options; opt; opt = opt6_next(opt, state->end)) - { + { void *ia_option, *ia_end; unsigned int min_time = 0xffffffff; int t1cntr; - + if (!check_ia(state, opt, &ia_end, &ia_option)) continue; @@ -914,25 +831,24 @@ static int dhcp6_no_relay(struct state *state, int msg_type, void *inbuff, size_ /* If we get a request with an IA_*A without addresses, treat it exactly like a SOLICT with rapid commit set. */ save_counter(start); - goto request_no_address; + goto request_no_address; } o = build_ia(state, &t1cntr); - + for (; ia_option; ia_option = opt6_find(opt6_next(ia_option, ia_end), ia_end, OPTION6_IAADDR, 24)) { struct in6_addr req_addr; struct dhcp_context *dynamic, *c; unsigned int lease_time; - struct in6_addr addr; int config_ok = 0; /* align. */ memcpy(&req_addr, opt6_ptr(ia_option, 0), IN6ADDRSZ); - + if ((c = address6_valid(state->context, &req_addr, tagif, 1))) - config_ok = config_valid(config, c, &addr) && IN6_ARE_ADDR_EQUAL(&addr, &req_addr); - + config_ok = (config_implies(config, c, &req_addr) != NULL); + if ((dynamic = address6_available(state->context, &req_addr, tagif, 1)) || c) { if (!dynamic && !config_ok) @@ -950,27 +866,23 @@ static int dhcp6_no_relay(struct state *state, int msg_type, void *inbuff, size_ put_opt6_short(DHCP6UNSPEC); put_opt6_string(_("address in use")); end_opt6(o1); - } - else + } + else { if (!dynamic) dynamic = c; lease_time = dynamic->lease_time; - + if (config_ok && have_config(config, CONFIG_TIME)) lease_time = config->lease_time; -#ifdef OPTION6_PREFIX_CLASS - if (dump_all_prefix_classes && state->ia_type == OPTION6_IA_NA) - state->send_prefix_class = prefix_class_from_context(c); -#endif add_address(state, dynamic, lease_time, ia_option, &min_time, &req_addr, now); get_context_tag(state, dynamic); address_assigned = 1; } } - else + else { /* requested address not on the correct link */ o1 = new_opt6(OPTION6_STATUS_CODE); @@ -979,12 +891,12 @@ static int dhcp6_no_relay(struct state *state, int msg_type, void *inbuff, size_ end_opt6(o1); } } - + end_ia(t1cntr, min_time, 0); - end_opt6(o); + end_opt6(o); } - if (address_assigned) + if (address_assigned) { o1 = new_opt6(OPTION6_STATUS_CODE); put_opt6_short(DHCP6SUCCESS); @@ -992,7 +904,7 @@ static int dhcp6_no_relay(struct state *state, int msg_type, void *inbuff, size_ end_opt6(o1); } else - { + { /* no address, return error */ o1 = new_opt6(OPTION6_STATUS_CODE); put_opt6_short(DHCP6NOADDRS); @@ -1004,13 +916,13 @@ static int dhcp6_no_relay(struct state *state, int msg_type, void *inbuff, size_ tagif = add_options(state, 0); break; } - - + + case DHCP6RENEW: { /* set reply message type */ *outmsgtypep = DHCP6REPLY; - + log6_quiet(state, "DHCPRENEW", NULL, NULL); for (opt = state->packet_options; opt; opt = opt6_next(opt, state->end)) @@ -1018,13 +930,13 @@ static int dhcp6_no_relay(struct state *state, int msg_type, void *inbuff, size_ void *ia_option, *ia_end; unsigned int min_time = 0xffffffff; int t1cntr, iacntr; - + if (!check_ia(state, opt, &ia_end, &ia_option)) continue; - + o = build_ia(state, &t1cntr); - iacntr = save_counter(-1); - + iacntr = save_counter(-1); + for (; ia_option; ia_option = opt6_find(opt6_next(ia_option, ia_end), ia_end, OPTION6_IAADDR, 24)) { struct dhcp_lease *lease = NULL; @@ -1034,10 +946,10 @@ static int dhcp6_no_relay(struct state *state, int msg_type, void *inbuff, size_ char *message = NULL; struct dhcp_context *this_context; - memcpy(&req_addr, opt6_ptr(ia_option, 0), IN6ADDRSZ); - + memcpy(&req_addr, opt6_ptr(ia_option, 0), IN6ADDRSZ); + if (!(lease = lease6_find(state->clid, state->clid_len, - state->ia_type == OPTION6_IA_NA ? LEASE_NA : LEASE_TA, + state->ia_type == OPTION6_IA_NA ? LEASE_NA : LEASE_TA, state->iaid, &req_addr))) { /* If the server cannot find a client entry for the IA the server @@ -1045,9 +957,9 @@ static int dhcp6_no_relay(struct state *state, int msg_type, void *inbuff, size_ to NoBinding in the Reply message. */ save_counter(iacntr); t1cntr = 0; - + log6_packet(state, "DHCPREPLY", &req_addr, _("lease not found")); - + o1 = new_opt6(OPTION6_STATUS_CODE); put_opt6_short(DHCP6NOBINDING); put_opt6_string(_("no binding found")); @@ -1056,23 +968,22 @@ static int dhcp6_no_relay(struct state *state, int msg_type, void *inbuff, size_ preferred_time = valid_time = 0; break; } - - + + if ((this_context = address6_available(state->context, &req_addr, tagif, 1)) || (this_context = address6_valid(state->context, &req_addr, tagif, 1))) { - struct in6_addr addr; unsigned int lease_time; get_context_tag(state, this_context); - - if (config_valid(config, this_context, &addr) && IN6_ARE_ADDR_EQUAL(&addr, &req_addr) && have_config(config, CONFIG_TIME)) + + if (config_implies(config, this_context, &req_addr) && have_config(config, CONFIG_TIME)) lease_time = config->lease_time; - else + else lease_time = this_context->lease_time; - - calculate_times(this_context, &min_time, &valid_time, &preferred_time, lease_time); - + + calculate_times(this_context, &min_time, &valid_time, &preferred_time, lease_time); + lease_set_expires(lease, valid_time, now); /* Update MAC record in case it's new information. */ if (state->mac_len != 0) @@ -1082,11 +993,11 @@ static int dhcp6_no_relay(struct state *state, int msg_type, void *inbuff, size_ char *addr_domain = get_domain6(&req_addr); if (!state->send_domain) state->send_domain = addr_domain; - lease_set_hostname(lease, state->hostname, state->hostname_auth, addr_domain, state->domain); + lease_set_hostname(lease, state->hostname, state->hostname_auth, addr_domain, state->domain); message = state->hostname; } - - + + if (preferred_time == 0) message = _("deprecated"); } @@ -1094,42 +1005,42 @@ static int dhcp6_no_relay(struct state *state, int msg_type, void *inbuff, size_ { preferred_time = valid_time = 0; message = _("address invalid"); - } + } if (message && (message != state->hostname)) - log6_packet(state, "DHCPREPLY", &req_addr, message); + log6_packet(state, "DHCPREPLY", &req_addr, message); else log6_quiet(state, "DHCPREPLY", &req_addr, message); - + o1 = new_opt6(OPTION6_IAADDR); put_opt6(&req_addr, sizeof(req_addr)); put_opt6_long(preferred_time); put_opt6_long(valid_time); end_opt6(o1); } - + end_ia(t1cntr, min_time, 1); end_opt6(o); } - + tagif = add_options(state, 0); break; - + } - + case DHCP6CONFIRM: { int good_addr = 0; /* set reply message type */ *outmsgtypep = DHCP6REPLY; - + log6_quiet(state, "DHCPCONFIRM", NULL, NULL); - + for (opt = state->packet_options; opt; opt = opt6_next(opt, state->end)) { void *ia_option, *ia_end; - + for (check_ia(state, opt, &ia_end, &ia_option); ia_option; ia_option = opt6_find(opt6_next(ia_option, ia_end), ia_end, OPTION6_IAADDR, 24)) @@ -1138,7 +1049,7 @@ static int dhcp6_no_relay(struct state *state, int msg_type, void *inbuff, size_ /* alignment */ memcpy(&req_addr, opt6_ptr(ia_option, 0), IN6ADDRSZ); - + if (!address6_valid(state->context, &req_addr, tagif, 1)) { o1 = new_opt6(OPTION6_STATUS_CODE); @@ -1152,8 +1063,8 @@ static int dhcp6_no_relay(struct state *state, int msg_type, void *inbuff, size_ good_addr = 1; log6_quiet(state, "DHCPREPLY", &req_addr, state->hostname); } - } - + } + /* No addresses, no reply: RFC 3315 18.2.2 */ if (!good_addr) return 0; @@ -1164,7 +1075,7 @@ static int dhcp6_no_relay(struct state *state, int msg_type, void *inbuff, size_ end_opt6(o1); break; } - + case DHCP6IREQ: { /* We can't discriminate contexts based on address, as we don't know it. @@ -1176,7 +1087,7 @@ static int dhcp6_no_relay(struct state *state, int msg_type, void *inbuff, size_ } /* Similarly, we can't determine domain from address, but if the FQDN is - given in --dhcp-host, we can use that, and failing that we can use the + given in --dhcp-host, we can use that, and failing that we can use the unqualified configured domain, if any. */ if (state->hostname_auth) state->send_domain = state->domain; @@ -1190,8 +1101,8 @@ static int dhcp6_no_relay(struct state *state, int msg_type, void *inbuff, size_ tagif = add_options(state, 1); break; } - - + + case DHCP6RELEASE: { /* set reply message type */ @@ -1203,10 +1114,10 @@ static int dhcp6_no_relay(struct state *state, int msg_type, void *inbuff, size_ { void *ia_option, *ia_end; int made_ia = 0; - + for (check_ia(state, opt, &ia_end, &ia_option); ia_option; - ia_option = opt6_find(opt6_next(ia_option, ia_end), ia_end, OPTION6_IAADDR, 24)) + ia_option = opt6_find(opt6_next(ia_option, ia_end), ia_end, OPTION6_IAADDR, 24)) { struct dhcp_lease *lease; struct in6_addr addr; @@ -1225,11 +1136,11 @@ static int dhcp6_no_relay(struct state *state, int msg_type, void *inbuff, size_ if (state->ia_type == OPTION6_IA_NA) { put_opt6_long(0); - put_opt6_long(0); + put_opt6_long(0); } made_ia = 1; } - + o1 = new_opt6(OPTION6_IAADDR); put_opt6(&addr, IN6ADDRSZ); put_opt6_long(0); @@ -1237,23 +1148,23 @@ static int dhcp6_no_relay(struct state *state, int msg_type, void *inbuff, size_ end_opt6(o1); } } - + if (made_ia) { o1 = new_opt6(OPTION6_STATUS_CODE); put_opt6_short(DHCP6NOBINDING); put_opt6_string(_("no binding found")); end_opt6(o1); - + end_opt6(o); } } - + o1 = new_opt6(OPTION6_STATUS_CODE); put_opt6_short(DHCP6SUCCESS); put_opt6_string(_("release received")); end_opt6(o1); - + break; } @@ -1261,38 +1172,39 @@ static int dhcp6_no_relay(struct state *state, int msg_type, void *inbuff, size_ { /* set reply message type */ *outmsgtypep = DHCP6REPLY; - + log6_quiet(state, "DHCPDECLINE", NULL, NULL); for (opt = state->packet_options; opt; opt = opt6_next(opt, state->end)) { void *ia_option, *ia_end; int made_ia = 0; - + for (check_ia(state, opt, &ia_end, &ia_option); ia_option; - ia_option = opt6_find(opt6_next(ia_option, ia_end), ia_end, OPTION6_IAADDR, 24)) + ia_option = opt6_find(opt6_next(ia_option, ia_end), ia_end, OPTION6_IAADDR, 24)) { struct dhcp_lease *lease; struct in6_addr addr; - + struct addrlist *addr_list; + /* align */ memcpy(&addr, opt6_ptr(ia_option, 0), IN6ADDRSZ); - if (have_config(config, CONFIG_ADDR6) && IN6_ARE_ADDR_EQUAL(&config->addr6, &addr)) + if ((addr_list = config_implies(config, state->context, &addr))) { prettyprint_time(daemon->dhcp_buff3, DECLINE_BACKOFF); inet_ntop(AF_INET6, &addr, daemon->addrbuff, ADDRSTRLEN); - my_syslog(MS_DHCP | LOG_WARNING, _("disabling DHCP static address %s for %s"), + my_syslog(MS_DHCP | LOG_WARNING, _("disabling DHCP static address %s for %s"), daemon->addrbuff, daemon->dhcp_buff3); - config->flags |= CONFIG_DECLINED; - config->decline_time = now; + addr_list->flags |= ADDRLIST_DECLINED; + addr_list->decline_time = now; } else /* make sure this host gets a different address next time. */ for (context_tmp = state->context; context_tmp; context_tmp = context_tmp->current) context_tmp->addr_epoch++; - + if ((lease = lease6_find(state->clid, state->clid_len, state->ia_type == OPTION6_IA_NA ? LEASE_NA : LEASE_TA, state->iaid, &addr))) lease_prune(lease, now); @@ -1305,11 +1217,11 @@ static int dhcp6_no_relay(struct state *state, int msg_type, void *inbuff, size_ if (state->ia_type == OPTION6_IA_NA) { put_opt6_long(0); - put_opt6_long(0); + put_opt6_long(0); } made_ia = 1; } - + o1 = new_opt6(OPTION6_IAADDR); put_opt6(&addr, IN6ADDRSZ); put_opt6_long(0); @@ -1317,17 +1229,17 @@ static int dhcp6_no_relay(struct state *state, int msg_type, void *inbuff, size_ end_opt6(o1); } } - + if (made_ia) { o1 = new_opt6(OPTION6_STATUS_CODE); put_opt6_short(DHCP6NOBINDING); put_opt6_string(_("no binding found")); end_opt6(o1); - + end_opt6(o); } - + } /* We must answer with 'success' in global section anyway */ @@ -1339,15 +1251,15 @@ static int dhcp6_no_relay(struct state *state, int msg_type, void *inbuff, size_ } } - + log_tags(tagif, state->xid); log6_opts(0, state->xid, daemon->outpacket.iov_base + start_opts, daemon->outpacket.iov_base + save_counter(-1)); - + return 1; } -static struct dhcp_netid *add_options(struct state *state, int do_refresh) +static struct dhcp_netid *add_options(struct state *state, int do_refresh) { void *oro; /* filter options based on tags, those we want get DHOPT_TAGOK bit set */ @@ -1357,65 +1269,81 @@ static struct dhcp_netid *add_options(struct state *state, int do_refresh) int i, o, o1; oro = opt6_find(state->packet_options, state->end, OPTION6_ORO, 0); - + for (opt_cfg = daemon->dhcp_opts6; opt_cfg; opt_cfg = opt_cfg->next) { /* netids match and not encapsulated? */ if (!(opt_cfg->flags & DHOPT_TAGOK)) continue; - + if (!(opt_cfg->flags & DHOPT_FORCE) && oro) { for (i = 0; i < opt6_len(oro) - 1; i += 2) if (opt6_uint(oro, i, 2) == (unsigned)opt_cfg->opt) break; - + /* option not requested */ if (i >= opt6_len(oro) - 1) continue; } - + if (opt_cfg->opt == OPTION6_REFRESH_TIME) done_refresh = 1; - + if (opt_cfg->opt == OPTION6_DNS_SERVER) done_dns = 1; - + if (opt_cfg->flags & DHOPT_ADDR6) { int len, j; struct in6_addr *a; - - for (a = (struct in6_addr *)opt_cfg->val, len = opt_cfg->len, j = 0; + + for (a = (struct in6_addr *)opt_cfg->val, len = opt_cfg->len, j = 0; j < opt_cfg->len; j += IN6ADDRSZ, a++) if ((IN6_IS_ADDR_ULA_ZERO(a) && IN6_IS_ADDR_UNSPECIFIED(state->ula_addr)) || (IN6_IS_ADDR_LINK_LOCAL_ZERO(a) && IN6_IS_ADDR_UNSPECIFIED(state->ll_addr))) len -= IN6ADDRSZ; - + if (len != 0) { - + o = new_opt6(opt_cfg->opt); - + for (a = (struct in6_addr *)opt_cfg->val, j = 0; j < opt_cfg->len; j+=IN6ADDRSZ, a++) { + struct in6_addr *p = NULL; + if (IN6_IS_ADDR_UNSPECIFIED(a)) { if (!add_local_addrs(state->context)) - put_opt6(state->fallback, IN6ADDRSZ); + p = state->fallback; } else if (IN6_IS_ADDR_ULA_ZERO(a)) { if (!IN6_IS_ADDR_UNSPECIFIED(state->ula_addr)) - put_opt6(state->ula_addr, IN6ADDRSZ); + p = state->ula_addr; } else if (IN6_IS_ADDR_LINK_LOCAL_ZERO(a)) { if (!IN6_IS_ADDR_UNSPECIFIED(state->ll_addr)) - put_opt6(state->ll_addr, IN6ADDRSZ); + p = state->ll_addr; } else - put_opt6(a, IN6ADDRSZ); + p = a; + + if (!p) + continue; + else if (opt_cfg->opt == OPTION6_NTP_SERVER) + { + if (IN6_IS_ADDR_MULTICAST(p)) + o1 = new_opt6(NTP_SUBOPTION_MC_ADDR); + else + o1 = new_opt6(NTP_SUBOPTION_SRV_ADDR); + put_opt6(p, IN6ADDRSZ); + end_opt6(o1); + } + else + put_opt6(p, IN6ADDRSZ); } end_opt6(o); @@ -1429,22 +1357,22 @@ static struct dhcp_netid *add_options(struct state *state, int do_refresh) end_opt6(o); } } - + if (daemon->port == NAMESERVER_PORT && !done_dns) { o = new_opt6(OPTION6_DNS_SERVER); if (!add_local_addrs(state->context)) put_opt6(state->fallback, IN6ADDRSZ); - end_opt6(o); + end_opt6(o); } if (state->context && !done_refresh) { struct dhcp_context *c; unsigned int lease_time = 0xffffffff; - + /* Find the smallest lease tie of all contexts, - subject to the RFC-4242 stipulation that this must not + subject to the RFC-4242 stipulation that this must not be less than 600. */ for (c = state->context; c; c = c->next) if (c->lease_time < lease_time) @@ -1457,36 +1385,36 @@ static struct dhcp_netid *add_options(struct state *state, int do_refresh) o = new_opt6(OPTION6_REFRESH_TIME); put_opt6_long(lease_time); - end_opt6(o); + end_opt6(o); } - + /* handle vendor-identifying vendor-encapsulated options, dhcp-option = vi-encap:13,17,....... */ for (opt_cfg = daemon->dhcp_opts6; opt_cfg; opt_cfg = opt_cfg->next) opt_cfg->flags &= ~DHOPT_ENCAP_DONE; - + if (oro) for (i = 0; i < opt6_len(oro) - 1; i += 2) if (opt6_uint(oro, i, 2) == OPTION6_VENDOR_OPTS) do_encap = 1; - + for (opt_cfg = daemon->dhcp_opts6; opt_cfg; opt_cfg = opt_cfg->next) - { + { if (opt_cfg->flags & DHOPT_RFC3925) { int found = 0; struct dhcp_opt *oc; - + if (opt_cfg->flags & DHOPT_ENCAP_DONE) continue; - + for (oc = daemon->dhcp_opts6; oc; oc = oc->next) { oc->flags &= ~DHOPT_ENCAP_MATCH; - + if (!(oc->flags & DHOPT_RFC3925) || opt_cfg->u.encap != oc->u.encap) continue; - + oc->flags |= DHOPT_ENCAP_DONE; if (match_netid(oc->netid, tagif, 1)) { @@ -1496,14 +1424,14 @@ static struct dhcp_netid *add_options(struct state *state, int do_refresh) oc->flags |= DHOPT_ENCAP_MATCH; found = 1; } - } + } } - + if (found) - { - o = new_opt6(OPTION6_VENDOR_OPTS); - put_opt6_long(opt_cfg->u.encap); - + { + o = new_opt6(OPTION6_VENDOR_OPTS); + put_opt6_long(opt_cfg->u.encap); + for (oc = daemon->dhcp_opts6; oc; oc = oc->next) if (oc->flags & DHOPT_ENCAP_MATCH) { @@ -1514,14 +1442,14 @@ static struct dhcp_netid *add_options(struct state *state, int do_refresh) end_opt6(o); } } - } + } if (state->hostname) { unsigned char *p; size_t len = strlen(state->hostname); - + if (state->send_domain) len += strlen(state->send_domain) + 2; @@ -1548,10 +1476,10 @@ static struct dhcp_netid *add_options(struct state *state, int do_refresh) { char *s = option_string(AF_INET6, opt6_uint(oro, i, 2), NULL, 0, NULL, 0); q += snprintf(q, MAXDNAME - (q - daemon->namebuff), - "%d%s%s%s", + "%d%s%s%s", opt6_uint(oro, i, 2), strlen(s) != 0 ? ":" : "", - s, + s, (i > opt6_len(oro) - 3) ? "" : ", "); if ( i > opt6_len(oro) - 3 || (q - daemon->namebuff) > 40) { @@ -1559,15 +1487,15 @@ static struct dhcp_netid *add_options(struct state *state, int do_refresh) my_syslog(MS_DHCP | LOG_INFO, _("%u requested options: %s"), state->xid, daemon->namebuff); } } - } + } return tagif; } - + static int add_local_addrs(struct dhcp_context *context) { int done = 0; - + for (; context; context = context->current) if ((context->flags & CONTEXT_USED) && !IN6_IS_ADDR_UNSPECIFIED(&context->local6)) { @@ -1577,9 +1505,9 @@ static int add_local_addrs(struct dhcp_context *context) if ((c->flags & CONTEXT_USED) && IN6_ARE_ADDR_EQUAL(&context->local6, &c->local6)) break; - + if (!c) - { + { done = 1; put_opt6(&context->local6, IN6ADDRSZ); } @@ -1599,7 +1527,7 @@ static void get_context_tag(struct state *state, struct dhcp_context *context) if (!state->hostname_auth) { struct dhcp_netid_list *id_list; - + for (id_list = daemon->dhcp_ignore_names; id_list; id_list = id_list->next) if ((!id_list->list) || match_netid(id_list->list, &context->netid, 0)) break; @@ -1607,22 +1535,7 @@ static void get_context_tag(struct state *state, struct dhcp_context *context) state->hostname = NULL; } } -} - -#ifdef OPTION6_PREFIX_CLASS -static struct prefix_class *prefix_class_from_context(struct dhcp_context *context) -{ - struct prefix_class *p; - struct dhcp_netid *t; - - for (p = daemon->prefix_classes; p ; p = p->next) - for (t = context->filter; t; t = t->next) - if (strcmp(p->tag.net, t->net) == 0) - return p; - - return NULL; -} -#endif +} static int check_ia(struct state *state, void *opt, void **endp, void **ia_option) { @@ -1631,13 +1544,13 @@ static int check_ia(struct state *state, void *opt, void **endp, void **ia_optio if (state->ia_type != OPTION6_IA_NA && state->ia_type != OPTION6_IA_TA) return 0; - + if (state->ia_type == OPTION6_IA_NA && opt6_len(opt) < 12) return 0; - + if (state->ia_type == OPTION6_IA_TA && opt6_len(opt) < 4) return 0; - + *endp = opt6_ptr(opt, opt6_len(opt)); state->iaid = opt6_uint(opt, 0, 4); *ia_option = opt6_find(opt6_ptr(opt, state->ia_type == OPTION6_IA_NA ? 12 : 4), *endp, OPTION6_IAADDR, 24); @@ -1649,17 +1562,17 @@ static int check_ia(struct state *state, void *opt, void **endp, void **ia_optio static int build_ia(struct state *state, int *t1cntr) { int o = new_opt6(state->ia_type); - + put_opt6_long(state->iaid); *t1cntr = 0; - + if (state->ia_type == OPTION6_IA_NA) { /* save pointer */ *t1cntr = save_counter(-1); /* so we can fill these in later */ put_opt6_long(0); - put_opt6_long(0); + put_opt6_long(0); } return o; @@ -1676,20 +1589,20 @@ static void end_ia(int t1cntr, unsigned int min_time, int do_fuzz) if (do_fuzz) { fuzz = rand16(); - + while (fuzz > (min_time/16)) fuzz = fuzz/2; } - + t1 = (min_time == 0xffffffff) ? 0xffffffff : min_time/2 - fuzz; t2 = (min_time == 0xffffffff) ? 0xffffffff : ((min_time/8)*7) - fuzz; put_opt6_long(t1); put_opt6_long(t2); save_counter(sav); - } + } } -static void add_address(struct state *state, struct dhcp_context *context, unsigned int lease_time, void *ia_option, +static void add_address(struct state *state, struct dhcp_context *context, unsigned int lease_time, void *ia_option, unsigned int *min_time, struct in6_addr *addr, time_t now) { unsigned int valid_time = 0, preferred_time = 0; @@ -1703,23 +1616,13 @@ static void add_address(struct state *state, struct dhcp_context *context, unsig valid_time = opt6_uint(ia_option, 20, 4); } - calculate_times(context, min_time, &valid_time, &preferred_time, lease_time); - + calculate_times(context, min_time, &valid_time, &preferred_time, lease_time); + put_opt6(addr, sizeof(*addr)); put_opt6_long(preferred_time); - put_opt6_long(valid_time); - -#ifdef OPTION6_PREFIX_CLASS - if (state->send_prefix_class) - { - int o1 = new_opt6(OPTION6_PREFIX_CLASS); - put_opt6_short(state->send_prefix_class->class); - end_opt6(o1); - } -#endif - + put_opt6_long(valid_time); end_opt6(o); - + if (state->lease_allocate) update_leases(state, context, addr, valid_time, now); @@ -1731,11 +1634,11 @@ static void add_address(struct state *state, struct dhcp_context *context, unsig { context->netid.next = state->context_tags; state->context_tags = &context->netid; - + if (!state->hostname_auth) { struct dhcp_netid_list *id_list; - + for (id_list = daemon->dhcp_ignore_names; id_list; id_list = id_list->next) if ((!id_list->list) || match_netid(id_list->list, &context->netid, 0)) break; @@ -1753,16 +1656,9 @@ static void mark_context_used(struct state *state, struct in6_addr *addr) struct dhcp_context *context; /* Mark that we have an address for this prefix. */ -#ifdef OPTION6_PREFIX_CLASS - for (context = state->context; context; context = context->current) - if (is_same_net6(addr, &context->start6, context->prefix) && - (!state->send_prefix_class || state->send_prefix_class == prefix_class_from_context(context))) - context->flags |= CONTEXT_USED; -#else for (context = state->context; context; context = context->current) if (is_same_net6(addr, &context->start6, context->prefix)) context->flags |= CONTEXT_USED; -#endif } static void mark_config_used(struct dhcp_context *context, struct in6_addr *addr) @@ -1774,13 +1670,13 @@ static void mark_config_used(struct dhcp_context *context, struct in6_addr *addr /* make sure address not leased to another CLID/IAID */ static int check_address(struct state *state, struct in6_addr *addr) -{ +{ struct dhcp_lease *lease; if (!(lease = lease6_find_by_addr(addr, 128, 0))) return 1; - if (lease->clid_len != state->clid_len || + if (lease->clid_len != state->clid_len || memcmp(lease->clid, state->clid, state->clid_len) != 0 || lease->iaid != state->iaid) return 0; @@ -1789,7 +1685,79 @@ static int check_address(struct state *state, struct in6_addr *addr) } -/* Calculate valid and preferred times to send in leases/renewals. +/* return true of *addr could have been generated from config. */ +static struct addrlist *config_implies(struct dhcp_config *config, struct dhcp_context *context, struct in6_addr *addr) +{ + int prefix; + struct in6_addr wild_addr; + struct addrlist *addr_list; + + if (!config || !(config->flags & CONFIG_ADDR6)) + return NULL; + + for (addr_list = config->addr6; addr_list; addr_list = addr_list->next) + { + prefix = (addr_list->flags & ADDRLIST_PREFIX) ? addr_list->prefixlen : 128; + wild_addr = addr_list->addr.addr6; + + if ((addr_list->flags & ADDRLIST_WILDCARD) && context->prefix == 64) + { + wild_addr = context->start6; + setaddr6part(&wild_addr, addr6part(&addr_list->addr.addr6)); + } + else if (!is_same_net6(&context->start6, addr, context->prefix)) + continue; + + if (is_same_net6(&wild_addr, addr, prefix)) + return addr_list; + } + + return NULL; +} + +static int config_valid(struct dhcp_config *config, struct dhcp_context *context, struct in6_addr *addr, struct state *state, time_t now) +{ + u64 addrpart, i, addresses; + struct addrlist *addr_list; + + if (!config || !(config->flags & CONFIG_ADDR6)) + return 0; + + for (addr_list = config->addr6; addr_list; addr_list = addr_list->next) + if (!(addr_list->flags & ADDRLIST_DECLINED) || + difftime(now, addr_list->decline_time) >= (float)DECLINE_BACKOFF) + { + addrpart = addr6part(&addr_list->addr.addr6); + addresses = 1; + + if (addr_list->flags & ADDRLIST_PREFIX) + addresses = (u64)1<<(128-addr_list->prefixlen); + + if ((addr_list->flags & ADDRLIST_WILDCARD)) + { + if (context->prefix != 64) + continue; + + *addr = context->start6; + } + else if (is_same_net6(&context->start6, &addr_list->addr.addr6, context->prefix)) + *addr = addr_list->addr.addr6; + else + continue; + + for (i = 0 ; i < addresses; i++) + { + setaddr6part(addr, addrpart+i); + + if (check_address(state, addr)) + return 1; + } + } + + return 0; +} + +/* Calculate valid and preferred times to send in leases/renewals. Inputs are: @@ -1800,17 +1768,17 @@ static int check_address(struct state *state, struct in6_addr *addr) *min_time - smallest valid time sent so far. Outputs are : - + *valid_timep, *preferred_timep - times to be send in IAADDR option. *min_time - smallest valid time sent so far, to calculate T1 and T2. - + */ -static void calculate_times(struct dhcp_context *context, unsigned int *min_time, unsigned int *valid_timep, +static void calculate_times(struct dhcp_context *context, unsigned int *min_time, unsigned int *valid_timep, unsigned int *preferred_timep, unsigned int lease_time) { unsigned int req_preferred = *preferred_timep, req_valid = *valid_timep; unsigned int valid_time = lease_time, preferred_time = lease_time; - + /* RFC 3315: "A server ignores the lifetimes set by the client if the preferred lifetime is greater than the valid lifetime. */ @@ -1821,33 +1789,33 @@ static void calculate_times(struct dhcp_context *context, unsigned int *min_time /* 0 == "no preference from client" */ if (req_preferred < 120u) req_preferred = 120u; /* sanity */ - + if (req_preferred < preferred_time) preferred_time = req_preferred; } - + if (req_valid != 0) /* 0 == "no preference from client" */ { if (req_valid < 120u) req_valid = 120u; /* sanity */ - + if (req_valid < valid_time) valid_time = req_valid; } } - /* deprecate (preferred == 0) which configured, or when local address + /* deprecate (preferred == 0) which configured, or when local address is deprecated */ if ((context->flags & CONTEXT_DEPRECATE) || context->preferred == 0) preferred_time = 0; - + if (preferred_time != 0 && preferred_time < *min_time) *min_time = preferred_time; - + if (valid_time != 0 && valid_time < *min_time) *min_time = valid_time; - + *valid_timep = valid_time; *preferred_timep = preferred_time; } @@ -1863,11 +1831,11 @@ static void update_leases(struct state *state, struct dhcp_context *context, str if (!lease) lease = lease6_allocate(addr, state->ia_type == OPTION6_IA_NA ? LEASE_NA : LEASE_TA); - + if (lease) { lease_set_expires(lease, lease_time, now); - lease_set_iaid(lease, state->iaid); + lease_set_iaid(lease, state->iaid); lease_set_hwaddr(lease, state->mac, state->clid, state->mac_len, state->mac_type, state->clid_len, now, 0); lease_set_interface(lease, state->interface, now); if (state->hostname && state->ia_type == OPTION6_IA_NA) @@ -1877,7 +1845,7 @@ static void update_leases(struct state *state, struct dhcp_context *context, str state->send_domain = addr_domain; lease_set_hostname(lease, state->hostname, state->hostname_auth, addr_domain, state->domain); } - + #ifdef HAVE_SCRIPT if (daemon->lease_change_command) { @@ -1886,8 +1854,8 @@ static void update_leases(struct state *state, struct dhcp_context *context, str free(lease->extradata); lease->extradata = NULL; lease->extradata_size = lease->extradata_len = 0; - lease->vendorclass_count = 0; - + lease->vendorclass_count = 0; + if ((class_opt = opt6_find(state->packet_options, state->end, OPTION6_VENDOR_CLASS, 4))) { void *enc_opt, *enc_end = opt6_ptr(class_opt, opt6_len(class_opt)); @@ -1895,18 +1863,18 @@ static void update_leases(struct state *state, struct dhcp_context *context, str /* send enterprise number first */ sprintf(daemon->dhcp_buff2, "%u", opt6_uint(class_opt, 0, 4)); lease_add_extradata(lease, (unsigned char *)daemon->dhcp_buff2, strlen(daemon->dhcp_buff2), 0); - - if (opt6_len(class_opt) >= 6) + + if (opt6_len(class_opt) >= 6) for (enc_opt = opt6_ptr(class_opt, 4); enc_opt; enc_opt = opt6_next(enc_opt, enc_end)) { lease->vendorclass_count++; lease_add_extradata(lease, opt6_ptr(enc_opt, 0), opt6_len(enc_opt), 0); } } - - lease_add_extradata(lease, (unsigned char *)state->client_hostname, - state->client_hostname ? strlen(state->client_hostname) : 0, 0); - + + lease_add_extradata(lease, (unsigned char *)state->client_hostname, + state->client_hostname ? strlen(state->client_hostname) : 0, 0); + /* space-concat tag set */ if (!tagif && !context->netid.net) lease_add_extradata(lease, NULL, 0, 0); @@ -1914,7 +1882,7 @@ static void update_leases(struct state *state, struct dhcp_context *context, str { if (context->netid.net) lease_add_extradata(lease, (unsigned char *)context->netid.net, strlen(context->netid.net), tagif ? ' ' : 0); - + if (tagif) { struct dhcp_netid *n; @@ -1926,16 +1894,16 @@ static void update_leases(struct state *state, struct dhcp_context *context, str if (strcmp(n->net, n1->net) == 0) break; if (!n1) - lease_add_extradata(lease, (unsigned char *)n->net, strlen(n->net), n->next ? ' ' : 0); + lease_add_extradata(lease, (unsigned char *)n->net, strlen(n->net), n->next ? ' ' : 0); } } } - + if (state->link_address) inet_ntop(AF_INET6, state->link_address, daemon->addrbuff, ADDRSTRLEN); - + lease_add_extradata(lease, (unsigned char *)daemon->addrbuff, state->link_address ? strlen(daemon->addrbuff) : 0, 0); - + if ((class_opt = opt6_find(state->packet_options, state->end, OPTION6_USER_CLASS, 2))) { void *enc_opt, *enc_end = opt6_ptr(class_opt, opt6_len(class_opt)); @@ -1943,27 +1911,27 @@ static void update_leases(struct state *state, struct dhcp_context *context, str lease_add_extradata(lease, opt6_ptr(enc_opt, 0), opt6_len(enc_opt), 0); } } -#endif - +#endif + } } - - - + + + static void log6_opts(int nest, unsigned int xid, void *start_opts, void *end_opts) { void *opt; char *desc = nest ? "nest" : "sent"; - + if (!option_bool(OPT_LOG_OPTS) || start_opts == end_opts) return; - + for (opt = start_opts; opt; opt = opt6_next(opt, end_opts)) { int type = opt6_type(opt); void *ia_options = NULL; char *optname; - + if (type == OPTION6_IA_NA) { sprintf(daemon->namebuff, "IAID=%u T1=%u T2=%u", @@ -1984,18 +1952,11 @@ static void log6_opts(int nest, unsigned int xid, void *start_opts, void *end_op /* align */ memcpy(&addr, opt6_ptr(opt, 0), IN6ADDRSZ); inet_ntop(AF_INET6, &addr, daemon->addrbuff, ADDRSTRLEN); - sprintf(daemon->namebuff, "%s PL=%u VL=%u", + sprintf(daemon->namebuff, "%s PL=%u VL=%u", daemon->addrbuff, opt6_uint(opt, 16, 4), opt6_uint(opt, 20, 4)); optname = "iaaddr"; ia_options = opt6_ptr(opt, 24); } -#ifdef OPTION6_PREFIX_CLASS - else if (type == OPTION6_PREFIX_CLASS) - { - optname = "prefix-class"; - sprintf(daemon->namebuff, "class=%u", opt6_uint(opt, 0, 2)); - } -#endif else if (type == OPTION6_STATUS_CODE) { int len = sprintf(daemon->namebuff, "%u ", opt6_uint(opt, 0, 2)); @@ -2009,15 +1970,15 @@ static void log6_opts(int nest, unsigned int xid, void *start_opts, void *end_op int offset = type == OPTION6_FQDN ? 1 : 0; optname = option_string(AF_INET6, type, opt6_ptr(opt, offset), opt6_len(opt) - offset, daemon->namebuff, MAXDNAME); } - - my_syslog(MS_DHCP | LOG_INFO, "%u %s size:%3d option:%3d %s %s", + + my_syslog(MS_DHCP | LOG_INFO, "%u %s size:%3d option:%3d %s %s", xid, desc, opt6_len(opt), type, optname, daemon->namebuff); - + if (ia_options) log6_opts(1, xid, ia_options, opt6_ptr(opt, opt6_len(opt))); } -} - +} + static void log6_quiet(struct state *state, char *type, struct in6_addr *addr, char *string) { if (option_bool(OPT_LOG_OPTS) || !option_bool(OPT_QUIET_DHCP6)) @@ -2031,7 +1992,7 @@ static void log6_packet(struct state *state, char *type, struct in6_addr *addr, /* avoid buffer overflow */ if (clid_len > 100) clid_len = 100; - + print_mac(daemon->namebuff, state->clid, clid_len); if (addr) @@ -2044,16 +2005,16 @@ static void log6_packet(struct state *state, char *type, struct in6_addr *addr, if(option_bool(OPT_LOG_OPTS)) my_syslog(MS_DHCP | LOG_INFO, "%u %s(%s) %s%s %s", - state->xid, + state->xid, type, - state->iface_name, + state->iface_name, daemon->dhcp_buff2, daemon->namebuff, string ? string : ""); else my_syslog(MS_DHCP | LOG_INFO, "%s(%s) %s%s %s", type, - state->iface_name, + state->iface_name, daemon->dhcp_buff2, daemon->namebuff, string ? string : ""); @@ -2063,25 +2024,25 @@ static void *opt6_find (void *opts, void *end, unsigned int search, unsigned int { u16 opt, opt_len; void *start; - + if (!opts) return NULL; - + while (1) { - if (end - opts < 4) + if (end - opts < 4) return NULL; - + start = opts; GETSHORT(opt, opts); GETSHORT(opt_len, opts); - + if (opt_len > (end - opts)) return NULL; - + if (opt == search && (opt_len >= minsize)) return start; - + opts += opt_len; } } @@ -2089,16 +2050,16 @@ static void *opt6_find (void *opts, void *end, unsigned int search, unsigned int static void *opt6_next(void *opts, void *end) { u16 opt_len; - - if (end - opts < 4) + + if (end - opts < 4) return NULL; - + opts += 2; GETSHORT(opt_len, opts); - + if (opt_len >= (end - opts)) return NULL; - + return opts + opt_len; } @@ -2108,19 +2069,19 @@ static unsigned int opt6_uint(unsigned char *opt, int offset, int size) unsigned int ret = 0; int i; unsigned char *p = opt6_ptr(opt, offset); - + for (i = 0; i < size; i++) ret = (ret << 8) | *p++; - + return ret; -} +} -void relay_upstream6(struct dhcp_relay *relay, ssize_t sz, +void relay_upstream6(struct dhcp_relay *relay, ssize_t sz, struct in6_addr *peer_address, u32 scope_id, time_t now) { /* ->local is same value for all relays on ->current chain */ - - struct all_addr from; + + union all_addr from; unsigned char *header; unsigned char *inbuff = daemon->dhcp_packet.iov_base; int msg_type = *inbuff; @@ -2133,9 +2094,9 @@ void relay_upstream6(struct dhcp_relay *relay, ssize_t sz, get_client_mac(peer_address, scope_id, mac, &maclen, &mactype, now); /* source address == relay address */ - from.addr.addr6 = relay->local.addr.addr6; - - /* Get hop count from nested relayed message */ + from.addr6 = relay->local.addr6; + + /* Get hop count from nested relayed message */ if (msg_type == DHCP6RELAYFORW) hopcount = *((unsigned char *)inbuff+1) + 1; else @@ -2153,9 +2114,9 @@ void relay_upstream6(struct dhcp_relay *relay, ssize_t sz, header[0] = DHCP6RELAYFORW; header[1] = hopcount; - memcpy(&header[2], &relay->local.addr.addr6, IN6ADDRSZ); + memcpy(&header[2], &relay->local.addr6, IN6ADDRSZ); memcpy(&header[18], peer_address, IN6ADDRSZ); - + /* RFC-6939 */ if (maclen != 0) { @@ -2164,22 +2125,22 @@ void relay_upstream6(struct dhcp_relay *relay, ssize_t sz, put_opt6(mac, maclen); end_opt6(o); } - + o = new_opt6(OPTION6_RELAY_MSG); put_opt6(inbuff, sz); end_opt6(o); - + for (; relay; relay = relay->current) { union mysockaddr to; - + to.sa.sa_family = AF_INET6; - to.in6.sin6_addr = relay->server.addr.addr6; + to.in6.sin6_addr = relay->server.addr6; to.in6.sin6_port = htons(DHCPV6_SERVER_PORT); to.in6.sin6_flowinfo = 0; to.in6.sin6_scope_id = 0; - if (IN6_ARE_ADDR_EQUAL(&relay->server.addr.addr6, &multicast)) + if (IN6_ARE_ADDR_EQUAL(&relay->server.addr6, &multicast)) { int multicast_iface; if (!relay->interface || strchr(relay->interface, '*') || @@ -2187,9 +2148,9 @@ void relay_upstream6(struct dhcp_relay *relay, ssize_t sz, setsockopt(daemon->dhcp6fd, IPPROTO_IPV6, IPV6_MULTICAST_IF, &multicast_iface, sizeof(multicast_iface)) == -1) my_syslog(MS_DHCP | LOG_ERR, _("Cannot multicast to DHCPv6 server without correct interface")); } - + send_from(daemon->dhcp6fd, 0, daemon->outpacket.iov_base, save_counter(-1), &to, &from, 0); - + if (option_bool(OPT_LOG_OPTS)) { inet_ntop(AF_INET6, &relay->local, daemon->addrbuff, ADDRSTRLEN); @@ -2208,20 +2169,20 @@ unsigned short relay_reply6(struct sockaddr_in6 *peer, ssize_t sz, char *arrival struct dhcp_relay *relay; struct in6_addr link; unsigned char *inbuff = daemon->dhcp_packet.iov_base; - + /* must have at least msg_type+hopcount+link_address+peer_address+minimal size option which is 1 + 1 + 16 + 16 + 2 + 2 = 38 */ - + if (sz < 38 || *inbuff != DHCP6RELAYREPL) return 0; - - memcpy(&link, &inbuff[2], IN6ADDRSZ); - + + memcpy(&link, &inbuff[2], IN6ADDRSZ); + for (relay = daemon->relay6; relay; relay = relay->next) - if (IN6_ARE_ADDR_EQUAL(&link, &relay->local.addr.addr6) && + if (IN6_ARE_ADDR_EQUAL(&link, &relay->local.addr6) && (!relay->interface || wildcard_match(relay->interface, arrival_interface))) break; - + reset_counter(); if (relay) @@ -2233,7 +2194,7 @@ unsigned short relay_reply6(struct sockaddr_in6 *peer, ssize_t sz, char *arrival { int encap_type = *((unsigned char *)opt6_ptr(opt, 0)); put_opt6(opt6_ptr(opt, 0), opt6_len(opt)); - memcpy(&peer->sin6_addr, &inbuff[18], IN6ADDRSZ); + memcpy(&peer->sin6_addr, &inbuff[18], IN6ADDRSZ); peer->sin6_scope_id = relay->iface_index; return encap_type == DHCP6RELAYREPL ? DHCPV6_SERVER_PORT : DHCPV6_CLIENT_PORT; } diff --git a/dnsmasq/rrfilter.c b/src/dnsmasq/rrfilter.c similarity index 98% rename from dnsmasq/rrfilter.c rename to src/dnsmasq/rrfilter.c index 0d9306d3f..16e6b554a 100644 --- a/dnsmasq/rrfilter.c +++ b/src/dnsmasq/rrfilter.c @@ -1,4 +1,4 @@ -/* dnsmasq is Copyright (c) 2000-2018 Simon Kelley +/* dnsmasq is Copyright (c) 2000-2020 Simon Kelley This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -14,7 +14,7 @@ along with this program. If not, see . */ -/* Code to safely remove RRs from a DNS answer */ +/* Code to safely remove RRs from a DNS answer */ #include "dnsmasq.h" @@ -244,7 +244,7 @@ size_t rrfilter(struct dns_header *header, size_t plen, int mode) check_name(&p, header, plen, 1, rrs, rr_found); p += 4; /* qclass, qtype */ - + check_rrs(p, header, plen, 1, rrs, rr_found); /* Fourth pass, elide records */ diff --git a/dnsmasq/slaac.c b/src/dnsmasq/slaac.c similarity index 94% rename from dnsmasq/slaac.c rename to src/dnsmasq/slaac.c index 9889e9dbf..4a3e01d29 100644 --- a/dnsmasq/slaac.c +++ b/src/dnsmasq/slaac.c @@ -1,15 +1,15 @@ -/* dnsmasq is Copyright (c) 2000-2018 Simon Kelley +/* dnsmasq is Copyright (c) 2000-2020 Simon Kelley This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; version 2 dated June, 1991, or (at your option) version 3 dated 29 June, 2007. - + This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. - + You should have received a copy of the GNU General Public License along with this program. If not, see . */ @@ -27,18 +27,18 @@ void slaac_add_addrs(struct dhcp_lease *lease, time_t now, int force) struct slaac_address *slaac, *old, **up; struct dhcp_context *context; int dns_dirty = 0; - - if (!(lease->flags & LEASE_HAVE_HWADDR) || + + if (!(lease->flags & LEASE_HAVE_HWADDR) || (lease->flags & (LEASE_TA | LEASE_NA)) || lease->last_interface == 0 || !lease->hostname) return ; - + old = lease->slaac_address; lease->slaac_address = NULL; - for (context = daemon->dhcp6; context; context = context->next) - if ((context->flags & CONTEXT_RA_NAME) && + for (context = daemon->dhcp6; context; context = context->next) + if ((context->flags & CONTEXT_RA_NAME) && !(context->flags & CONTEXT_OLD) && lease->last_interface == context->if_index) { @@ -58,7 +58,7 @@ void slaac_add_addrs(struct dhcp_lease *lease, time_t now, int force) memcpy(&addr.s6_addr[8], lease->hwaddr, 8); #endif #if defined(ARPHRD_IEEE1394) && defined(ARPHRD_EUI64) - else if (lease->clid_len == 9 && + else if (lease->clid_len == 9 && lease->clid[0] == ARPHRD_EUI64 && lease->hwaddr_type == ARPHRD_IEEE1394) /* firewire has EUI-64 identifier as clid */ @@ -66,9 +66,9 @@ void slaac_add_addrs(struct dhcp_lease *lease, time_t now, int force) #endif else continue; - + addr.s6_addr[8] ^= 0x02; - + /* check if we already have this one */ for (up = &old, slaac = old; slaac; slaac = slaac->next) { @@ -86,7 +86,7 @@ void slaac_add_addrs(struct dhcp_lease *lease, time_t now, int force) } up = &slaac->next; } - + /* No, make new one */ if (!slaac && (slaac = whine_malloc(sizeof(struct slaac_address)))) { @@ -96,17 +96,17 @@ void slaac_add_addrs(struct dhcp_lease *lease, time_t now, int force) /* Do RA's to prod it */ ra_start_unsolicited(now, context); } - + if (slaac) { slaac->next = lease->slaac_address; lease->slaac_address = slaac; } } - + if (old || dns_dirty) lease_update_dns(1); - + /* Free any no reused */ for (; old; old = slaac) { @@ -122,7 +122,7 @@ time_t periodic_slaac(time_t now, struct dhcp_lease *leases) struct dhcp_lease *lease; struct slaac_address *slaac; time_t next_event = 0; - + for (context = daemon->dhcp6; context; context = context->next) if ((context->flags & CONTEXT_RA_NAME) && !(context->flags & CONTEXT_OLD)) break; @@ -140,12 +140,12 @@ time_t periodic_slaac(time_t now, struct dhcp_lease *leases) /* confirmed or given up? */ if (slaac->backoff == 0 || slaac->ping_time == 0) continue; - + if (difftime(slaac->ping_time, now) <= 0.0) { struct ping_packet *ping; struct sockaddr_in6 addr; - + reset_counter(); if (!(ping = expand(sizeof(struct ping_packet)))) @@ -155,7 +155,7 @@ time_t periodic_slaac(time_t now, struct dhcp_lease *leases) ping->code = 0; ping->identifier = ping_id; ping->sequence_no = slaac->backoff; - + memset(&addr, 0, sizeof(addr)); #ifdef HAVE_SOCKADDR_SA_LEN addr.sin6_len = sizeof(struct sockaddr_in6); @@ -163,12 +163,12 @@ time_t periodic_slaac(time_t now, struct dhcp_lease *leases) addr.sin6_family = AF_INET6; addr.sin6_port = htons(IPPROTO_ICMPV6); addr.sin6_addr = slaac->addr; - + if (sendto(daemon->icmp6fd, daemon->outpacket.iov_base, save_counter(-1), 0, (struct sockaddr *)&addr, sizeof(addr)) == -1 && errno == EHOSTUNREACH && slaac->backoff == 12) - slaac->ping_time = 0; /* Give up */ + slaac->ping_time = 0; /* Give up */ else { slaac->ping_time += (1 << (slaac->backoff - 1)) + (rand16()/21785); /* 0 - 3 */ @@ -178,7 +178,7 @@ time_t periodic_slaac(time_t now, struct dhcp_lease *leases) slaac->backoff++; } } - + if (slaac->ping_time != 0 && (next_event == 0 || difftime(next_event, slaac->ping_time) >= 0.0)) next_event = slaac->ping_time; @@ -194,7 +194,7 @@ void slaac_ping_reply(struct in6_addr *sender, unsigned char *packet, char *inte struct slaac_address *slaac; struct ping_packet *ping = (struct ping_packet *)packet; int gotone = 0; - + if (ping->identifier == ping_id) for (lease = leases; lease; lease = lease->next) for (slaac = lease->slaac_address; slaac; slaac = slaac->next) @@ -204,10 +204,10 @@ void slaac_ping_reply(struct in6_addr *sender, unsigned char *packet, char *inte gotone = 1; inet_ntop(AF_INET6, sender, daemon->addrbuff, ADDRSTRLEN); if (!option_bool(OPT_QUIET_DHCP6)) - my_syslog(MS_DHCP | LOG_INFO, "SLAAC-CONFIRM(%s) %s %s", interface, daemon->addrbuff, lease->hostname); + my_syslog(MS_DHCP | LOG_INFO, "SLAAC-CONFIRM(%s) %s %s", interface, daemon->addrbuff, lease->hostname); } - + lease_update_dns(gotone); } - + #endif diff --git a/dnsmasq/tables.c b/src/dnsmasq/tables.c similarity index 93% rename from dnsmasq/tables.c rename to src/dnsmasq/tables.c index a3382ce56..b34ea7713 100644 --- a/dnsmasq/tables.c +++ b/src/dnsmasq/tables.c @@ -62,7 +62,7 @@ void ipset_init(void) } } -int add_to_ipset(const char *setname, const struct all_addr *ipaddr, +int add_to_ipset(const char *setname, const union all_addr *ipaddr, int flags, int remove) { struct pfr_addr addr; @@ -108,19 +108,18 @@ int add_to_ipset(const char *setname, const struct all_addr *ipaddr, my_syslog(LOG_INFO, _("info: table created")); bzero(&addr, sizeof(addr)); -#ifdef HAVE_IPV6 + if (flags & F_IPV6) { addr.pfra_af = AF_INET6; addr.pfra_net = 0x80; - memcpy(&(addr.pfra_ip6addr), &(ipaddr->addr), sizeof(struct in6_addr)); + memcpy(&(addr.pfra_ip6addr), ipaddr, sizeof(struct in6_addr)); } else -#endif { addr.pfra_af = AF_INET; addr.pfra_net = 0x20; - addr.pfra_ip4addr.s_addr = ipaddr->addr.addr4.s_addr; + addr.pfra_ip4addr.s_addr = ipaddr->addr4.s_addr; } bzero(&io, sizeof(io)); diff --git a/dnsmasq/tftp.c b/src/dnsmasq/tftp.c similarity index 84% rename from dnsmasq/tftp.c rename to src/dnsmasq/tftp.c index f2eccbc47..d6f35637d 100644 --- a/dnsmasq/tftp.c +++ b/src/dnsmasq/tftp.c @@ -1,4 +1,4 @@ -/* dnsmasq is Copyright (c) 2000-2018 Simon Kelley +/* dnsmasq is Copyright (c) 2000-2020 Simon Kelley This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -18,6 +18,7 @@ #ifdef HAVE_TFTP +static void handle_tftp(time_t now, struct tftp_transfer *transfer, ssize_t len); static struct tftp_file *check_tftp_fileperm(ssize_t *len, char *prefix); static void free_transfer(struct tftp_transfer *transfer); static ssize_t tftp_err(int err, char *packet, char *message, char *file); @@ -50,7 +51,7 @@ void tftp_request(struct listener *listen, time_t now) struct ifreq ifr; int is_err = 1, if_index = 0, mtu = 0; struct iname *tmp; - struct tftp_transfer *transfer; + struct tftp_transfer *transfer = NULL, **up; int port = daemon->start_tftp_port; /* may be zero to use ephemeral port */ #if defined(IP_MTU_DISCOVER) && defined(IP_PMTUDISC_DONT) int mtuflag = IP_PMTUDISC_DONT; @@ -59,24 +60,20 @@ void tftp_request(struct listener *listen, time_t now) char *name = NULL; char *prefix = daemon->tftp_prefix; struct tftp_prefix *pref; - struct all_addr addra; -#ifdef HAVE_IPV6 + union all_addr addra; /* Can always get recvd interface for IPv6 */ int check_dest = !option_bool(OPT_NOWILD) || listen->family == AF_INET6; -#else - int check_dest = !option_bool(OPT_NOWILD); -#endif union { struct cmsghdr align; /* this ensures alignment */ -#ifdef HAVE_IPV6 char control6[CMSG_SPACE(sizeof(struct in6_pktinfo))]; -#endif #if defined(HAVE_LINUX_NETWORK) char control[CMSG_SPACE(sizeof(struct in_pktinfo))]; #elif defined(HAVE_SOLARIS_NETWORK) - char control[CMSG_SPACE(sizeof(unsigned int))]; + char control[CMSG_SPACE(sizeof(struct in_addr)) + + CMSG_SPACE(sizeof(unsigned int))]; #elif defined(IP_RECVDSTADDR) && defined(IP_RECVIF) - char control[CMSG_SPACE(sizeof(struct sockaddr_dl))]; + char control[CMSG_SPACE(sizeof(struct in_addr)) + + CMSG_SPACE(sizeof(struct sockaddr_dl))]; #endif } control_u; @@ -174,7 +171,6 @@ void tftp_request(struct listener *listen, time_t now) #endif -#ifdef HAVE_IPV6 if (listen->family == AF_INET6) { for (cmptr = CMSG_FIRSTHDR(&msg); cmptr; cmptr = CMSG_NXTHDR(&msg, cmptr)) @@ -190,19 +186,16 @@ void tftp_request(struct listener *listen, time_t now) if_index = p.p->ipi6_ifindex; } } -#endif if (!indextoname(listen->tftpfd, if_index, namebuff)) return; name = namebuff; - addra.addr.addr4 = addr.in.sin_addr; + addra.addr4 = addr.in.sin_addr; -#ifdef HAVE_IPV6 if (listen->family == AF_INET6) - addra.addr.addr6 = addr.in6.sin6_addr; -#endif + addra.addr6 = addr.in6.sin6_addr; if (daemon->tftp_interfaces) { @@ -222,7 +215,7 @@ void tftp_request(struct listener *listen, time_t now) if (!option_bool(OPT_CLEVERBIND)) enumerate_interfaces(0); if (!loopback_exception(listen->tftpfd, listen->family, &addra, name) && - !label_exception(if_index, listen->family, &addra) ) + !label_exception(if_index, listen->family, &addra)) return; } @@ -247,6 +240,39 @@ void tftp_request(struct listener *listen, time_t now) if (mtu == 0) mtu = daemon->tftp_mtu; + /* data transfer via server listening socket */ + if (option_bool(OPT_SINGLE_PORT)) + { + int tftp_cnt; + + for (tftp_cnt = 0, transfer = daemon->tftp_trans, up = &daemon->tftp_trans; transfer; up = &transfer->next, transfer = transfer->next) + { + tftp_cnt++; + + if (sockaddr_isequal(&peer, &transfer->peer)) + { + if (ntohs(*((unsigned short *)packet)) == OP_RRQ) + { + /* Handle repeated RRQ or abandoned transfer from same host and port + by unlinking and reusing the struct transfer. */ + *up = transfer->next; + break; + } + else + { + handle_tftp(now, transfer, len); + return; + } + } + } + + /* Enforce simultaneous transfer limit. In non-single-port mode + this is doene by not listening on the server socket when + too many transfers are in progress. */ + if (!transfer && tftp_cnt >= daemon->tftp_max) + return; + } + if (name) { /* check for per-interface prefix */ @@ -262,7 +288,6 @@ void tftp_request(struct listener *listen, time_t now) addr.in.sin_len = sizeof(addr.in); #endif } -#ifdef HAVE_IPV6 else { addr.in6.sin6_port = htons(port); @@ -272,18 +297,22 @@ void tftp_request(struct listener *listen, time_t now) addr.in6.sin6_len = sizeof(addr.in6); #endif } -#endif - if (!(transfer = whine_malloc(sizeof(struct tftp_transfer)))) + /* May reuse struct transfer from abandoned transfer in single port mode. */ + if (!transfer && !(transfer = whine_malloc(sizeof(struct tftp_transfer)))) return; - if ((transfer->sockfd = socket(listen->family, SOCK_DGRAM, 0)) == -1) + if (option_bool(OPT_SINGLE_PORT)) + transfer->sockfd = listen->tftpfd; + else if ((transfer->sockfd = socket(listen->family, SOCK_DGRAM, 0)) == -1) { free(transfer); return; } transfer->peer = peer; + transfer->source = addra; + transfer->if_index = if_index; transfer->timeout = now + 2; transfer->backoff = 1; transfer->block = 1; @@ -296,7 +325,7 @@ void tftp_request(struct listener *listen, time_t now) prettyprint_addr(&peer, daemon->addrbuff); /* if we have a nailed-down range, iterate until we find a free one. */ - while (1) + while (!option_bool(OPT_SINGLE_PORT)) { if (bind(transfer->sockfd, &addr.sa, sa_len(&addr)) == -1 || #if defined(IP_MTU_DISCOVER) && defined(IP_PMTUDISC_DONT) @@ -310,10 +339,9 @@ void tftp_request(struct listener *listen, time_t now) { if (listen->family == AF_INET) addr.in.sin_port = htons(port); -#ifdef HAVE_IPV6 else - addr.in6.sin6_port = htons(port); -#endif + addr.in6.sin6_port = htons(port); + continue; } my_syslog(MS_TFTP | LOG_ERR, _("unable to get free port for TFTP")); @@ -326,7 +354,7 @@ void tftp_request(struct listener *listen, time_t now) p = packet + 2; end = packet + len; - + if (ntohs(*((unsigned short *)packet)) != OP_RRQ || !(filename = next(&p, end)) || !(mode = next(&p, end)) || @@ -450,9 +478,8 @@ void tftp_request(struct listener *listen, time_t now) is_err = 0; } } - - while (sendto(transfer->sockfd, packet, len, 0, - (struct sockaddr *)&peer, sa_len(&peer)) == -1 && errno == EINTR); + + send_from(transfer->sockfd, !option_bool(OPT_SINGLE_PORT), packet, len, &peer, &addra, if_index); if (is_err) free_transfer(transfer); @@ -549,63 +576,28 @@ static struct tftp_file *check_tftp_fileperm(ssize_t *len, char *prefix) void check_tftp_listeners(time_t now) { struct tftp_transfer *transfer, *tmp, **up; - ssize_t len; - struct ack { - unsigned short op, block; - } *mess = (struct ack *)daemon->packet; - - /* Check for activity on any existing transfers */ - for (transfer = daemon->tftp_trans, up = &daemon->tftp_trans; transfer; transfer = tmp) - { - tmp = transfer->next; - - prettyprint_addr(&transfer->peer, daemon->addrbuff); - + /* In single port mode, all packets come via port 69 and tftp_request() */ + if (!option_bool(OPT_SINGLE_PORT)) + for (transfer = daemon->tftp_trans; transfer; transfer = transfer->next) if (poll_check(transfer->sockfd, POLLIN)) { /* we overwrote the buffer... */ daemon->srv_save = NULL; - - if ((len = recv(transfer->sockfd, daemon->packet, daemon->packet_buff_sz, 0)) >= (ssize_t)sizeof(struct ack)) - { - if (ntohs(mess->op) == OP_ACK && ntohs(mess->block) == (unsigned short)transfer->block) - { - /* Got ack, ensure we take the (re)transmit path */ - transfer->timeout = now; - transfer->backoff = 0; - if (transfer->block++ != 0) - transfer->offset += transfer->blocksize - transfer->expansion; - } - else if (ntohs(mess->op) == OP_ERR) - { - char *p = daemon->packet + sizeof(struct ack); - char *end = daemon->packet + len; - char *err = next(&p, end); - - /* Sanitise error message */ - if (!err) - err = ""; - else - sanitise(err); - - my_syslog(MS_TFTP | LOG_ERR, _("error %d %s received from %s"), - (int)ntohs(mess->block), err, - daemon->addrbuff); - - /* Got err, ensure we take abort */ - transfer->timeout = now; - transfer->backoff = 100; - } - } + handle_tftp(now, transfer, recv(transfer->sockfd, daemon->packet, daemon->packet_buff_sz, 0)); } + + for (transfer = daemon->tftp_trans, up = &daemon->tftp_trans; transfer; transfer = tmp) + { + tmp = transfer->next; if (difftime(now, transfer->timeout) >= 0.0) { int endcon = 0; + ssize_t len; /* timeout, retransmit */ - transfer->timeout += 1 + (1<backoff); + transfer->timeout += 1 + (1<<(transfer->backoff/2)); /* we overwrote the buffer... */ daemon->srv_save = NULL; @@ -615,22 +607,24 @@ void check_tftp_listeners(time_t now) len = tftp_err_oops(daemon->packet, transfer->file->filename); endcon = 1; } - /* don't complain about timeout when we're awaiting the last - ACK, some clients never send it */ - else if (++transfer->backoff > 7 && len != 0) + else if (++transfer->backoff > 7) { - endcon = 1; + /* don't complain about timeout when we're awaiting the last + ACK, some clients never send it */ + if ((unsigned)len == transfer->blocksize + 4) + endcon = 1; len = 0; } if (len != 0) - while(sendto(transfer->sockfd, daemon->packet, len, 0, - (struct sockaddr *)&transfer->peer, sa_len(&transfer->peer)) == -1 && errno == EINTR); - + send_from(transfer->sockfd, !option_bool(OPT_SINGLE_PORT), daemon->packet, len, + &transfer->peer, &transfer->source, transfer->if_index); + if (endcon || len == 0) { strcpy(daemon->namebuff, transfer->file->filename); sanitise(daemon->namebuff); + prettyprint_addr(&transfer->peer, daemon->addrbuff); my_syslog(MS_TFTP | LOG_INFO, endcon ? _("failed sending %s to %s") : _("sent %s to %s"), daemon->namebuff, daemon->addrbuff); /* unlink */ *up = tmp; @@ -649,15 +643,60 @@ void check_tftp_listeners(time_t now) up = &transfer->next; } } + +/* packet in daemon->packet as this is called. */ +static void handle_tftp(time_t now, struct tftp_transfer *transfer, ssize_t len) +{ + struct ack { + unsigned short op, block; + } *mess = (struct ack *)daemon->packet; + + if (len >= (ssize_t)sizeof(struct ack)) + { + if (ntohs(mess->op) == OP_ACK && ntohs(mess->block) == (unsigned short)transfer->block) + { + /* Got ack, ensure we take the (re)transmit path */ + transfer->timeout = now; + transfer->backoff = 0; + if (transfer->block++ != 0) + transfer->offset += transfer->blocksize - transfer->expansion; + } + else if (ntohs(mess->op) == OP_ERR) + { + char *p = daemon->packet + sizeof(struct ack); + char *end = daemon->packet + len; + char *err = next(&p, end); + + prettyprint_addr(&transfer->peer, daemon->addrbuff); + + /* Sanitise error message */ + if (!err) + err = ""; + else + sanitise(err); + + my_syslog(MS_TFTP | LOG_ERR, _("error %d %s received from %s"), + (int)ntohs(mess->block), err, + daemon->addrbuff); + + /* Got err, ensure we take abort */ + transfer->timeout = now; + transfer->backoff = 100; + } + } +} static void free_transfer(struct tftp_transfer *transfer) { - close(transfer->sockfd); + if (!option_bool(OPT_SINGLE_PORT)) + close(transfer->sockfd); + if (transfer->file && (--transfer->file->refcount) == 0) { close(transfer->file->fd); free(transfer->file); } + free(transfer); } diff --git a/src/dnsmasq/ubus.c b/src/dnsmasq/ubus.c new file mode 100644 index 000000000..5f8128779 --- /dev/null +++ b/src/dnsmasq/ubus.c @@ -0,0 +1,205 @@ +/* dnsmasq is Copyright (c) 2000-2020 Simon Kelley + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 dated June, 1991, or + (at your option) version 3 dated 29 June, 2007. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . +*/ + +#include "dnsmasq.h" + +#ifdef HAVE_UBUS + +#include + +static struct blob_buf b; +static int notify; +static int error_logged = 0; + +static int ubus_handle_metrics(struct ubus_context *ctx, struct ubus_object *obj, + struct ubus_request_data *req, const char *method, + struct blob_attr *msg); + +static void ubus_subscribe_cb(struct ubus_context *ctx, struct ubus_object *obj); + +static const struct ubus_method ubus_object_methods[] = { + UBUS_METHOD_NOARG("metrics", ubus_handle_metrics), +}; + +static struct ubus_object_type ubus_object_type = + UBUS_OBJECT_TYPE("dnsmasq", ubus_object_methods); + +static struct ubus_object ubus_object = { + .name = NULL, + .type = &ubus_object_type, + .methods = ubus_object_methods, + .n_methods = ARRAY_SIZE(ubus_object_methods), + .subscribe_cb = ubus_subscribe_cb, +}; + +static void ubus_subscribe_cb(struct ubus_context *ctx, struct ubus_object *obj) +{ + (void)ctx; + + my_syslog(LOG_DEBUG, _("UBus subscription callback: %s subscriber(s)"), obj->has_subscribers ? "1" : "0"); + notify = obj->has_subscribers; +} + +static void ubus_destroy(struct ubus_context *ubus) +{ + // Forces re-initialization when we're reusing the same definitions later on. + ubus_object.id = 0; + ubus_object_type.id = 0; + + ubus_free(ubus); + daemon->ubus = NULL; +} + +static void ubus_disconnect_cb(struct ubus_context *ubus) +{ + int ret; + + ret = ubus_reconnect(ubus, NULL); + if (ret) + { + my_syslog(LOG_ERR, _("Cannot reconnect to UBus: %s"), ubus_strerror(ret)); + + ubus_destroy(ubus); + } +} + +void ubus_init() +{ + struct ubus_context *ubus = NULL; + int ret = 0; + + ubus = ubus_connect(NULL); + if (!ubus) + { + if (!error_logged) + { + my_syslog(LOG_ERR, _("Cannot initialize UBus: connection failed")); + error_logged = 1; + } + + ubus_destroy(ubus); + return; + } + + ubus_object.name = daemon->ubus_name; + ret = ubus_add_object(ubus, &ubus_object); + if (ret) + { + if (!error_logged) + { + my_syslog(LOG_ERR, _("Cannot add object to UBus: %s"), ubus_strerror(ret)); + error_logged = 1; + } + ubus_destroy(ubus); + return; + } + + ubus->connection_lost = ubus_disconnect_cb; + daemon->ubus = ubus; + error_logged = 0; + + my_syslog(LOG_INFO, _("Connected to system UBus")); +} + +void set_ubus_listeners() +{ + struct ubus_context *ubus = (struct ubus_context *)daemon->ubus; + if (!ubus) + { + if (!error_logged) + { + my_syslog(LOG_ERR, _("Cannot set UBus listeners: no connection")); + error_logged = 1; + } + return; + } + + error_logged = 0; + + poll_listen(ubus->sock.fd, POLLIN); + poll_listen(ubus->sock.fd, POLLERR); + poll_listen(ubus->sock.fd, POLLHUP); +} + +void check_ubus_listeners() +{ + struct ubus_context *ubus = (struct ubus_context *)daemon->ubus; + if (!ubus) + { + if (!error_logged) + { + my_syslog(LOG_ERR, _("Cannot poll UBus listeners: no connection")); + error_logged = 1; + } + return; + } + + error_logged = 0; + + if (poll_check(ubus->sock.fd, POLLIN)) + ubus_handle_event(ubus); + + if (poll_check(ubus->sock.fd, POLLHUP | POLLERR)) + { + my_syslog(LOG_INFO, _("Disconnecting from UBus")); + + ubus_destroy(ubus); + } +} + +static int ubus_handle_metrics(struct ubus_context *ctx, struct ubus_object *obj, + struct ubus_request_data *req, const char *method, + struct blob_attr *msg) +{ + int i; + + (void)obj; + (void)method; + (void)msg; + + blob_buf_init(&b, BLOBMSG_TYPE_TABLE); + + for (i=0; i < __METRIC_MAX; i++) + blobmsg_add_u32(&b, get_metric_name(i), daemon->metrics[i]); + + return ubus_send_reply(ctx, req, b.head); +} + +void ubus_event_bcast(const char *type, const char *mac, const char *ip, const char *name, const char *interface) +{ + struct ubus_context *ubus = (struct ubus_context *)daemon->ubus; + int ret; + + if (!ubus || !notify) + return; + + blob_buf_init(&b, BLOBMSG_TYPE_TABLE); + if (mac) + blobmsg_add_string(&b, "mac", mac); + if (ip) + blobmsg_add_string(&b, "ip", ip); + if (name) + blobmsg_add_string(&b, "name", name); + if (interface) + blobmsg_add_string(&b, "interface", interface); + + ret = ubus_notify(ubus, &ubus_object, type, b.head, -1); + if (!ret) + my_syslog(LOG_ERR, _("Failed to send UBus event: %s"), ubus_strerror(ret)); +} + + +#endif /* HAVE_UBUS */ diff --git a/dnsmasq/util.c b/src/dnsmasq/util.c similarity index 85% rename from dnsmasq/util.c rename to src/dnsmasq/util.c index 3b725cc57..5f13027d2 100644 --- a/dnsmasq/util.c +++ b/src/dnsmasq/util.c @@ -1,20 +1,20 @@ -/* dnsmasq is Copyright (c) 2000-2018 Simon Kelley +/* dnsmasq is Copyright (c) 2000-2020 Simon Kelley This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; version 2 dated June, 1991, or (at your option) version 3 dated 29 June, 2007. - + This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. - + You should have received a copy of the GNU General Public License along with this program. If not, see . */ -/* The SURF random number generator was taken from djbdns-1.05, by +/* The SURF random number generator was taken from djbdns-1.05, by Daniel J Bernstein, which is public domain. */ @@ -30,6 +30,10 @@ #include #endif +#ifdef HAVE_LINUX_NETWORK +#include +#endif + /* SURF random number generator */ static u32 seed[32]; @@ -40,12 +44,12 @@ static int outleft = 0; void rand_init() { int fd = open(RANDFILE, O_RDONLY); - + if (fd == -1 || !read_write(fd, (unsigned char *)&seed, sizeof(seed), 1) || !read_write(fd, (unsigned char *)&in, sizeof(in), 1)) die(_("failed to seed the random number generator: %s"), NULL, EC_MISC); - + close(fd); } @@ -73,26 +77,26 @@ static void surf(void) unsigned short rand16(void) { - if (!outleft) + if (!outleft) { if (!++in[0]) if (!++in[1]) if (!++in[2]) ++in[3]; surf(); outleft = 8; } - + return (unsigned short) out[--outleft]; } u32 rand32(void) { - if (!outleft) + if (!outleft) { if (!++in[0]) if (!++in[1]) if (!++in[2]) ++in[3]; surf(); outleft = 8; } - - return out[--outleft]; + + return out[--outleft]; } u64 rand64(void) @@ -105,7 +109,7 @@ u64 rand64(void) surf(); outleft = 8; } - + outleft -= 2; return (u64)out[outleft+1] + (((u64)out[outleft]) << 32); @@ -114,15 +118,15 @@ u64 rand64(void) /* returns 2 if names is OK but contains one or more underscores */ static int check_name(char *in) { - /* remove trailing . + /* remove trailing . also fail empty string and label > 63 chars */ size_t dotgap = 0, l = strlen(in); char c; int nowhite = 0; int hasuscore = 0; - + if (l == 0 || l > MAXDNAME) return 0; - + if (in[l-1] == '.') { in[l-1] = 0; @@ -135,7 +139,7 @@ static int check_name(char *in) dotgap = 0; else if (++dotgap > MAXLABEL) return 0; - else if (isascii((unsigned char)c) && iscntrl((unsigned char)c)) + else if (isascii((unsigned char)c) && iscntrl((unsigned char)c)) /* iscntrl only gives expected results for ascii */ return 0; #if !defined(HAVE_IDN) && !defined(HAVE_LIBIDN2) @@ -157,8 +161,8 @@ static int check_name(char *in) } /* Hostnames have a more limited valid charset than domain names - so check for legal char a-z A-Z 0-9 - _ - Note that this may receive a FQDN, so only check the first label + so check for legal char a-z A-Z 0-9 - _ + Note that this may receive a FQDN, so only check the first label for the tighter criteria. */ int legal_hostname(char *name) { @@ -178,28 +182,28 @@ int legal_hostname(char *name) if (!first && (c == '-' || c == '_')) continue; - + /* end of hostname part */ if (c == '.') return 1; - + return 0; } - + return 1; } - + char *canonicalise(char *in, int *nomem) { char *ret = NULL; int rc; - + if (nomem) *nomem = 0; - + if (!(rc = check_name(in))) return NULL; - + #if defined(HAVE_LIBIDN2) && (!defined(IDN2_VERSION_NUMBER) || IDN2_VERSION_NUMBER < 0x02000003) /* older libidn2 strips underscores, so don't do IDN processing if the name has an underscore (check_name() returned 2) */ @@ -218,24 +222,24 @@ char *canonicalise(char *in, int *nomem) { if (ret) free(ret); - + if (nomem && (rc == IDNA_MALLOC_ERROR || rc == IDNA_DLOPEN_ERROR)) { my_syslog(LOG_ERR, _("failed to allocate memory")); *nomem = 1; } - + return NULL; } - + return ret; } #endif - + if ((ret = whine_malloc(strlen(in)+1))) strcpy(ret, in); else if (nomem) - *nomem = 1; + *nomem = 1; return ret; } @@ -243,7 +247,7 @@ char *canonicalise(char *in, int *nomem) unsigned char *do_rfc1035_name(unsigned char *p, char *sval, char *limit) { int j; - + while (sval && *sval) { unsigned char *cp = p++; @@ -260,15 +264,15 @@ unsigned char *do_rfc1035_name(unsigned char *p, char *sval, char *limit) if (option_bool(OPT_DNSSEC_VALID) && *sval == NAME_ESCAPE) *p++ = (*(++sval))-1; else -#endif +#endif *p++ = *sval; } - + *cp = j; if (*sval) sval++; } - + return p; } @@ -276,10 +280,10 @@ unsigned char *do_rfc1035_name(unsigned char *p, char *sval, char *limit) void *safe_malloc(size_t size) { void *ret = calloc(1, size); - + if (!ret) die(_("could not get memory"), NULL, EC_NOMEM); - + return ret; } @@ -296,7 +300,7 @@ void safe_strncpy(char *dest, const char *src, size_t size) void safe_pipe(int *fd, int read_noblock) { - if (pipe(fd) == -1 || + if (pipe(fd) == -1 || !fix_fd(fd[1]) || (read_noblock && !fix_fd(fd[0]))) die(_("cannot create pipe: %s"), NULL, EC_MISC); @@ -308,25 +312,24 @@ void *whine_malloc(size_t size) if (!ret) my_syslog(LOG_ERR, _("failed to allocate %d bytes"), (int) size); - + return ret; } int sockaddr_isequal(union mysockaddr *s1, union mysockaddr *s2) { if (s1->sa.sa_family == s2->sa.sa_family) - { + { if (s1->sa.sa_family == AF_INET && s1->in.sin_port == s2->in.sin_port && s1->in.sin_addr.s_addr == s2->in.sin_addr.s_addr) return 1; -#ifdef HAVE_IPV6 + if (s1->sa.sa_family == AF_INET6 && s1->in6.sin6_port == s2->in6.sin6_port && s1->in6.sin6_scope_id == s2->in6.sin6_scope_id && IN6_ARE_ADDR_EQUAL(&s1->in6.sin6_addr, &s2->in6.sin6_addr)) return 1; -#endif } return 0; } @@ -336,12 +339,10 @@ int sa_len(union mysockaddr *addr) #ifdef HAVE_SOCKADDR_SA_LEN return addr->sa.sa_len; #else -#ifdef HAVE_IPV6 if (addr->sa.sa_family == AF_INET6) return sizeof(addr->in6); else -#endif - return sizeof(addr->in); + return sizeof(addr->in); #endif } @@ -349,20 +350,20 @@ int sa_len(union mysockaddr *addr) int hostname_isequal(const char *a, const char *b) { unsigned int c1, c2; - + do { c1 = (unsigned char) *a++; c2 = (unsigned char) *b++; - + if (c1 >= 'A' && c1 <= 'Z') c1 += 'a' - 'A'; if (c2 >= 'A' && c2 <= 'Z') c2 += 'a' - 'A'; - + if (c1 != c2) return 0; } while (c1); - + return 1; } @@ -371,9 +372,9 @@ int hostname_issubdomain(char *a, char *b) { char *ap, *bp; unsigned int c1, c2; - + /* move to the end */ - for (ap = a; *ap; ap++); + for (ap = a; *ap; ap++); for (bp = b; *bp; bp++); /* a shorter than b or a empty. */ @@ -384,7 +385,7 @@ int hostname_issubdomain(char *a, char *b) { c1 = (unsigned char) *(--ap); c2 = (unsigned char) *(--bp); - + if (c1 >= 'A' && c1 <= 'Z') c1 += 'a' - 'A'; if (c2 >= 'A' && c2 <= 'Z') @@ -402,8 +403,8 @@ int hostname_issubdomain(char *a, char *b) return 0; } - - + + time_t dnsmasq_time(void) { #ifdef HAVE_BROKEN_RTC @@ -423,21 +424,20 @@ int netmask_length(struct in_addr mask) { int zero_count = 0; - while (0x0 == (mask.s_addr & 0x1) && zero_count < 32) + while (0x0 == (mask.s_addr & 0x1) && zero_count < 32) { mask.s_addr >>= 1; zero_count++; } - + return 32 - zero_count; } int is_same_net(struct in_addr a, struct in_addr b, struct in_addr mask) { return (a.s_addr & mask.s_addr) == (b.s_addr & mask.s_addr); -} +} -#ifdef HAVE_IPV6 int is_same_net6(struct in6_addr *a, struct in6_addr *b, int prefixlen) { int pfbytes = prefixlen >> 3; @@ -476,15 +476,12 @@ void setaddr6part(struct in6_addr *addr, u64 host) } } -#endif - /* returns port number from address */ int prettyprint_addr(union mysockaddr *addr, char *buf) { int port = 0; - -#ifdef HAVE_IPV6 + if (addr->sa.sa_family == AF_INET) { inet_ntop(AF_INET, &addr->in.sin_addr, buf, ADDRSTRLEN); @@ -503,11 +500,7 @@ int prettyprint_addr(union mysockaddr *addr, char *buf) } port = ntohs(addr->in6.sin6_port); } -#else - strcpy(buf, inet_ntoa(addr->in.sin_addr)); - port = ntohs(addr->in.sin_port); -#endif - + return port; } @@ -530,26 +523,26 @@ void prettyprint_time(char *buf, unsigned int t) } -/* in may equal out, when maxlen may be -1 (No max len). +/* in may equal out, when maxlen may be -1 (No max len). Return -1 for extraneous no-hex chars found. */ -int parse_hex(char *in, unsigned char *out, int maxlen, +int parse_hex(char *in, unsigned char *out, int maxlen, unsigned int *wildcard_mask, int *mac_type) { - int mask = 0, i = 0; + int done = 0, mask = 0, i = 0; char *r; - + if (mac_type) *mac_type = 0; - - while (maxlen == -1 || i < maxlen) + + while (!done && (maxlen == -1 || i < maxlen)) { for (r = in; *r != 0 && *r != ':' && *r != '-' && *r != ' '; r++) if (*r != '*' && !isxdigit((unsigned char)*r)) return -1; - + if (*r == 0) - maxlen = i; - + done = 1; + if (r != in ) { if (*r == '-' && i == 0 && mac_type) @@ -570,7 +563,7 @@ int parse_hex(char *in, unsigned char *out, int maxlen, { int j, bytes = (1 + (r - in))/2; for (j = 0; j < bytes; j++) - { + { char sav = sav; if (j < bytes - 1) { @@ -584,7 +577,7 @@ int parse_hex(char *in, unsigned char *out, int maxlen, out[i] = strtol(&in[j*2], NULL, 16); mask = mask << 1; if (++i == maxlen) - break; + break; if (j < bytes - 1) in[(j+1)*2] = sav; } @@ -593,7 +586,7 @@ int parse_hex(char *in, unsigned char *out, int maxlen, } in = r+1; } - + if (wildcard_mask) *wildcard_mask = mask; @@ -645,13 +638,13 @@ char *print_mac(char *buff, unsigned char *mac, int len) { char *p = buff; int i; - + if (len == 0) sprintf(p, ""); else for (i = 0; i < len; i++) p += sprintf(p, "%.2x%s", mac[i], (i == len - 1) ? "" : ":"); - + return buff; } @@ -662,17 +655,17 @@ int retry_send(ssize_t rc) { static int retries = 0; struct timespec waiter; - + if (rc != -1) { retries = 0; errno = 0; return 0; } - + /* Linux kernels can return EAGAIN in perpetuity when calling sendmsg() and the relevant interface has gone. Here we loop - retrying in EAGAIN for 1 second max, to avoid this hanging + retrying in EAGAIN for 1 second max, to avoid this hanging dnsmasq. */ if (errno == EAGAIN || errno == EWOULDBLOCK) @@ -683,39 +676,80 @@ int retry_send(ssize_t rc) if (retries++ < 1000) return 1; } - + retries = 0; - + if (errno == EINTR) return 1; - + return 0; } int read_write(int fd, unsigned char *packet, int size, int rw) { ssize_t n, done; - + for (done = 0; done < size; done += n) { - do { + do { if (rw) n = read(fd, &packet[done], (size_t)(size - done)); else n = write(fd, &packet[done], (size_t)(size - done)); - + if (n == 0) return 0; - + } while (retry_send(n) || errno == ENOMEM || errno == ENOBUFS); if (errno != 0) return 0; } - + return 1; } +/* close all fds except STDIN, STDOUT and STDERR, spare1, spare2 and spare3 */ +void close_fds(long max_fd, int spare1, int spare2, int spare3) +{ + /* On Linux, use the /proc/ filesystem to find which files + are actually open, rather than iterate over the whole space, + for efficiency reasons. If this fails we drop back to the dumb code. */ +#ifdef HAVE_LINUX_NETWORK + DIR *d; + + if ((d = opendir("/proc/self/fd"))) + { + struct dirent *de; + + while ((de = readdir(d))) + { + long fd; + char *e = NULL; + + errno = 0; + fd = strtol(de->d_name, &e, 10); + + if (errno != 0 || !e || *e || fd == dirfd(d) || + fd == STDOUT_FILENO || fd == STDERR_FILENO || fd == STDIN_FILENO || + fd == spare1 || fd == spare2 || fd == spare3) + continue; + + close(fd); + } + + closedir(d); + return; + } +#endif + + /* fallback, dumb code. */ + for (max_fd--; max_fd >= 0; max_fd--) + if (max_fd != STDOUT_FILENO && max_fd != STDERR_FILENO && max_fd != STDIN_FILENO && + max_fd != spare1 && max_fd != spare2 && max_fd != spare3) + close(max_fd); +} + /* Basically match a string value against a wildcard pattern. */ int wildcard_match(const char* wildcard, const char* match) { @@ -725,7 +759,7 @@ int wildcard_match(const char* wildcard, const char* match) return 1; if (*wildcard != *match) - return 0; + return 0; ++wildcard; ++match; @@ -743,7 +777,7 @@ int wildcard_matchn(const char* wildcard, const char* match, int num) return 1; if (*wildcard != *match) - return 0; + return 0; ++wildcard; ++match; @@ -752,3 +786,22 @@ int wildcard_matchn(const char* wildcard, const char* match, int num) return (!num) || (*wildcard == *match); } + +#ifdef HAVE_LINUX_NETWORK +int kernel_version(void) +{ + struct utsname utsname; + int version; + char *split; + + if (uname(&utsname) < 0) + die(_("failed to find kernel version: %s"), NULL, EC_MISC); + + split = strtok(utsname.release, "."); + version = (split ? atoi(split) : 0); + split = strtok(NULL, "."); + version = version * 256 + (split ? atoi(split) : 0); + split = strtok(NULL, "."); + return version * 256 + (split ? atoi(split) : 0); +} +#endif diff --git a/src/dnsmasq_interface.c b/src/dnsmasq_interface.c new file mode 100644 index 000000000..b619d124b --- /dev/null +++ b/src/dnsmasq_interface.c @@ -0,0 +1,1794 @@ +/* Pi-hole: A black hole for Internet advertisements +* (c) 2017 Pi-hole, LLC (https://pi-hole.net) +* Network-wide ad blocking via your own hardware. +* +* FTL Engine +* dnsmasq interfacing routines +* +* This file is copyright under the latest version of the EUPL. +* Please see LICENSE file for your rights under this license. */ + +#define FTLDNS +#include "dnsmasq/dnsmasq.h" +#undef __USE_XOPEN +#include "FTL.h" +#include "dnsmasq_interface.h" +#include "shmem.h" +#include "overTime.h" +#include "memory.h" +#include "database/common.h" +#include "database/database-thread.h" +#include "datastructure.h" +#include "database/gravity-db.h" +#include "setupVars.h" +#include "daemon.h" +#include "timers.h" +#include "gc.h" +#include "api/socket.h" +#include "regex_r.h" +#include "config.h" +#include "capabilities.h" +#include "resolve.h" +#include "files.h" +#include "log.h" +// Prototype of getCacheInformation() +#include "api/api.h" +// global variable daemonmode +#include "args.h" + +static void print_flags(const unsigned int flags); +static void save_reply_type(const unsigned int flags, queriesData* query, const struct timeval response); +static unsigned long converttimeval(const struct timeval time) __attribute__((const)); +static void detect_blocked_IP(const unsigned short flags, const char* answer, const int queryID); +static void query_externally_blocked(const int queryID, const unsigned char status); +static int findQueryID(const int id); +static void prepare_blocking_metadata(void); +static void query_blocked(queriesData* query, domainsData* domain, clientsData* client, const unsigned char new_status); + +// Static blocking metadata (stored precomputed as time-critical) +static unsigned int blocking_flags = 0; +static union all_addr blocking_addrp_v4 = {{ 0 }}; +static union all_addr blocking_addrp_v6 = {{ 0 }}; +static unsigned char force_next_DNS_reply = 0u; + +// Adds debug information to the regular pihole.log file +char debug_dnsmasq_lines = 0; + +unsigned char* pihole_privacylevel = &config.privacylevel; +const char flagnames[][12] = {"F_IMMORTAL ", "F_NAMEP ", "F_REVERSE ", "F_FORWARD ", "F_DHCP ", "F_NEG ", "F_HOSTS ", "F_IPV4 ", "F_IPV6 ", "F_BIGNAME ", "F_NXDOMAIN ", "F_CNAME ", "F_DNSKEY ", "F_CONFIG ", "F_DS ", "F_DNSSECOK ", "F_UPSTREAM ", "F_RRNAME ", "F_SERVER ", "F_QUERY ", "F_NOERR ", "F_AUTH ", "F_DNSSEC ", "F_KEYTAG ", "F_SECSTAT ", "F_NO_RR ", "F_IPSET ", "F_NOEXTRA ", "F_SERVFAIL", "F_RCODE"}; + +static bool check_domain_blocked(const char *domainString, const int clientID, + clientsData *client, queriesData *query, DNSCacheData *dns_cache, + const char **blockingreason, unsigned char *new_status) +{ + // Check domains against exact blacklist + // Skipped when the domain is whitelisted + bool blockDomain = false; + if(in_blacklist(domainString, clientID, client)) + { + // We block this domain + blockDomain = true; + *new_status = QUERY_BLACKLIST; + *blockingreason = "exactly blacklisted"; + + // Mark domain as exactly blacklisted for this client + dns_cache->blocking_status = BLACKLIST_BLOCKED; + return true; + } + + // Check domains against gravity domains + // Skipped when the domain is whitelisted or blocked by exact blacklist + if(!query->whitelisted && !blockDomain && + in_gravity(domainString, clientID, client)) + { + // We block this domain + blockDomain = true; + *new_status = QUERY_GRAVITY; + *blockingreason = "gravity blocked"; + + // Mark domain as gravity blocked for this client + dns_cache->blocking_status = GRAVITY_BLOCKED; + return true; + } + + // Check domain against blacklist regex filters + // Skipped when the domain is whitelisted or blocked by exact blacklist or gravity + int regex_idx = 0; + if(!query->whitelisted && !blockDomain && + (regex_idx = match_regex(domainString, clientID, REGEX_BLACKLIST)) > -1) + { + // We block this domain + blockDomain = true; + *new_status = QUERY_REGEX; + *blockingreason = "regex blacklisted"; + + // Mark domain as regex matched for this client + dns_cache->blocking_status = REGEX_BLOCKED; + dns_cache->black_regex_idx = regex_idx; + return true; + } + + // Not blocked + return false; +} + +static bool _FTL_check_blocking(int queryID, int domainID, int clientID, const char **blockingreason, + const char* file, const int line) +{ + // Only check blocking conditions when global blocking is enabled + if(blockingstatus == BLOCKING_DISABLED) + { + return false; + } + + // Get query, domain and client pointers + queriesData* query = getQuery(queryID, true); + domainsData* domain = getDomain(domainID, true); + clientsData* client = getClient(clientID, true); + unsigned int cacheID = findCacheID(domainID, clientID); + DNSCacheData *dns_cache = getDNSCache(cacheID, true); + if(query == NULL || domain == NULL || client == NULL || dns_cache == NULL) + { + // Encountered memory error, skip query + return false; + } + + // Skip the entire chain of tests if we already know the answer for this + // particular client + unsigned char blockingStatus = dns_cache->blocking_status; + const char* domainstr = getstr(domain->domainpos); + switch(blockingStatus) + { + case UNKNOWN_BLOCKED: + // New domain/client combination. + // We have to go through all the tests below + if(config.debug & DEBUG_QUERIES) + { + logg("%s is not known", domainstr); + } + + break; + + case BLACKLIST_BLOCKED: + // Known as exactly blacklistes, we + // return this result early, skipping + // all the lengthy tests below + *blockingreason = "exactly blacklisted"; + if(config.debug & DEBUG_QUERIES) + { + logg("%s is known as %s", domainstr, *blockingreason); + } + + // Do not block if the entire query is to be permitted + // as sometving along the CNAME path hit the whitelist + if(!query->whitelisted) + { + query_blocked(query, domain, client, QUERY_BLACKLIST); + force_next_DNS_reply = dns_cache->force_reply; + return true; + } + break; + + case GRAVITY_BLOCKED: + // Known as gravity blocked, we + // return this result early, skipping + // all the lengthy tests below + *blockingreason = "gravity blocked"; + if(config.debug & DEBUG_QUERIES) + { + logg("%s is known as %s", domainstr, *blockingreason); + } + + // Do not block if the entire query is to be permitted + // as sometving along the CNAME path hit the whitelist + if(!query->whitelisted) + { + query_blocked(query, domain, client, QUERY_GRAVITY); + force_next_DNS_reply = dns_cache->force_reply; + return true; + } + break; + + case REGEX_BLOCKED: + // Known as regex blacklisted, we + // return this result early, skipping + // all the lengthy tests below + *blockingreason = "regex blacklisted"; + if(config.debug & DEBUG_QUERIES) + { + logg("%s is known as %s", domainstr, *blockingreason); + force_next_DNS_reply = dns_cache->force_reply; + } + + // Do not block if the entire query is to be permitted + // as sometving along the CNAME path hit the whitelist + if(!query->whitelisted) + { + query_blocked(query, domain, client, QUERY_REGEX); + return true; + } + break; + + case WHITELISTED: + // Known as whitelisted, we + // return this result early, skipping + // all the lengthy tests below + if(config.debug & DEBUG_QUERIES) + { + logg("%s is known as not to be blocked (whitelisted)", domainstr); + } + + query->whitelisted = true; + + return false; + break; + + case NOT_BLOCKED: + // Known as not blocked, we + // return this result early, skipping + // all the lengthy tests below + if(config.debug & DEBUG_QUERIES) + { + logg("%s is known as not to be blocked", domainstr); + } + + return false; + break; + } + + // Skip all checks and continue if we hit already at least one whitelist in the chain + if(query->whitelisted) + { + if(config.debug & DEBUG_QUERIES) + { + logg("Query is permitted as at least one whitelist entry matched"); + } + return false; + } + + // Check whitelist (exact + regex) for match + const char *domainString = getstr(domain->domainpos); + const char *blockedDomain = domainString; + query->whitelisted = in_whitelist(domainString, clientID, client); + + bool blockDomain = false; + unsigned char new_status = QUERY_UNKNOWN; + // Check blacklist (exact + regex) and gravity for queried domain + if(!query->whitelisted) + { + blockDomain = check_domain_blocked(domainString, clientID, client, query, dns_cache, blockingreason, &new_status); + } + + // Check blacklist (exact + regex) and gravity for _esni.domain if enabled (defaulting to true) + if(config.block_esni && !query->whitelisted && !blockDomain && strncasecmp(domainString, "_esni.", 6u) == 0) + { + blockDomain = check_domain_blocked(domainString + 6u, clientID, client, query, dns_cache, blockingreason, &new_status); + + if(blockDomain) + { + // Truncate "_esni." from queried domain if the parenting domain was the reason for blocking this query + blockedDomain = domainString + 6u; + // Force next DNS reply to be NXDOMAIN for _esni.* queries + force_next_DNS_reply = NXDOMAIN; + dns_cache->force_reply = NXDOMAIN; + } + } + + // Common actions regardless what the possible blocking reason is + if(blockDomain) + { + // Adjust counters + query_blocked(query, domain, client, new_status); + + // Debug output + if(config.debug & DEBUG_QUERIES) + logg("Blocking %s as %s is %s", domainString, blockedDomain, *blockingreason); + } + else + { + // Explicitly mark as not blocked to skip the entire + // gravity/blacklist chain when the same client asks + // for the same domain in the future. Explicitly store + // domain as whitelisted if this is the case + dns_cache->blocking_status = query->whitelisted ? WHITELISTED : NOT_BLOCKED; + } + + return blockDomain; +} + + +bool _FTL_CNAME(const char *domain, const struct crec *cpp, const int id, const char* file, const int line) +{ + // Don't analyze anything if in PRIVACY_NOSTATS mode + if(config.privacylevel >= PRIVACY_NOSTATS) + return false; + + // Does the user want to skip deep CNAME inspection? + if(!config.cname_inspection) + { + return false; + } + + // Lock shared memory + lock_shm(); + + // Get CNAME destination and source (if applicable) + const char *src = cpp != NULL ? cpp->flags & F_BIGNAME ? cpp->name.bname->name : cpp->name.sname : NULL; + const char *dst = domain; + + // Save status and upstreamID in corresponding query identified by dnsmasq's ID + const int queryID = findQueryID(id); + if(queryID < 0) + { + // This may happen e.g. if the original query was a PTR query + // or "pi.hole" and we ignored them altogether + unlock_shm(); + return false; + } + + // Get query pointer so we can later extract the client requesting this domain for + // the per-client blocking evaluation + queriesData* query = getQuery(queryID, true); + if(query == NULL) + { + // Nothing to be done here + unlock_shm(); + return false; + } + + // Go through already knows domains and see if it is one of them + // As this domain might have been found in the middle of a CNAME-path, + // it may be not have been seen by FTL_new_query() before + char *domainString = strdup(domain); + strtolower(domainString); + const int domainID = findDomainID(domainString, false); + + // Get client ID from original query + const int clientID = query->clientID; + + // Perform per-client blocking evaluation for this domain. The result for this + // domain-client combination will be cached to be immediately available for later + // queries of the same domain by the same client + const char *blockingreason = NULL; + bool block = FTL_check_blocking(queryID, domainID, clientID, &blockingreason); + + // If we find during a CNAME inspection that we want to block the entire chain, + // the originally queried domain itself was not counted as blocked (but as + // (permitted). Later in the chain, when we find that this is a bad guy, we + // short-circuit it. We need to correct the domain counter of the domain at the + // head of the chain, otherwise, the data for the top lists is misleading. + // For this, we go back the entire path and change the original request to blocked + // by increasing the blocked count of this domain by one. Fortunately, each CNAME + // path can easily be tracked back to the original head in FTL's data so we do not + // need to search it. This makes the change able to happen without causing any delay. + if(block) + { + domainsData* head_domain = getDomain(query->domainID, true); + head_domain->blockedcount++; + + // Store query response as CNAME type + struct timeval response; + gettimeofday(&response, 0); + save_reply_type(F_CNAME, query, response); + + // Store domain that was the reason for blocking the entire chain + query->CNAME_domainID = domainID; + + // Change blocking reason into CNAME-caused blocking + if(query->status == QUERY_GRAVITY) + query->status = QUERY_GRAVITY_CNAME; + else if(query->status == QUERY_REGEX) + { + // Get parent DNS cache entry + unsigned int parent_cacheID = findCacheID(domainID, query->clientID); + DNSCacheData *parent_dns_cache = getDNSCache(parent_cacheID, true); + + // Get child DNS cache entry + unsigned int child_cacheID = findCacheID(query->domainID, query->clientID); + DNSCacheData *child_dns_cache = getDNSCache(child_cacheID, true); + + // Propagate ID of responsible regex up from the child to the parent domain + if(parent_dns_cache != NULL && child_dns_cache != NULL) + child_dns_cache->black_regex_idx = parent_dns_cache->black_regex_idx; + + query->status = QUERY_REGEX_CNAME; + } + else if(query->status == QUERY_BLACKLIST) + query->status = QUERY_BLACKLIST_CNAME; + } + + // Debug logging for deep CNAME inspection (if enabled) + if(config.debug & DEBUG_QUERIES) + { + if(src == NULL) + logg("CNAME %s", dst); + else + logg("CNAME %s ---> %s", src, dst); + } + + // Return result + free(domainString); + unlock_shm(); + return block; +} + + +bool _FTL_new_query(const unsigned int flags, const char *name, + const char **blockingreason, const union all_addr *addr, + const char *types, const int id, const char type, + const char* file, const int line) +{ + // Create new query in data structure + + // Don't analyze anything if in PRIVACY_NOSTATS mode + if(config.privacylevel >= PRIVACY_NOSTATS) + return false; + + // Lock shared memory + lock_shm(); + + // Get timestamp + const time_t querytimestamp = time(NULL); + + // Save request time + struct timeval request; + gettimeofday(&request, 0); + + // Determine query type + unsigned char querytype = 0; + if(strcmp(types,"query[A]") == 0) + querytype = TYPE_A; + else if(strcmp(types,"query[AAAA]") == 0) + querytype = TYPE_AAAA; + else if(strcmp(types,"query[ANY]") == 0) + querytype = TYPE_ANY; + else if(strcmp(types,"query[SRV]") == 0) + querytype = TYPE_SRV; + else if(strcmp(types,"query[SOA]") == 0) + querytype = TYPE_SOA; + else if(strcmp(types,"query[PTR]") == 0) + querytype = TYPE_PTR; + else if(strcmp(types,"query[TXT]") == 0) + querytype = TYPE_TXT; + else if(strcmp(types,"query[NAPTR]") == 0) + querytype = TYPE_NAPTR; + else + { + // Return early to avoid accessing querytypedata out of bounds + if(config.debug & DEBUG_QUERIES) + logg("Notice: Skipping unknown query type: %s (%i)", types, id); + unlock_shm(); + return false; + } + + // Skip AAAA queries if user doesn't want to have them analyzed + if(!config.analyze_AAAA && querytype == TYPE_AAAA) + { + if(config.debug & DEBUG_QUERIES) + logg("Not analyzing AAAA query"); + unlock_shm(); + return false; + } + + // Ensure we have enough space in the queries struct + memory_check(QUERIES); + const int queryID = counters->queries; + + // If domain is "pi.hole" we skip this query + if(strcasecmp(name, "pi.hole") == 0) + { + unlock_shm(); + return false; + } + + // Convert domain to lower case + char *domainString = strdup(name); + strtolower(domainString); + + // Get client IP address + char dest[ADDRSTRLEN]; + inet_ntop((flags & F_IPV4) ? AF_INET : AF_INET6, addr, dest, ADDRSTRLEN); + char *clientIP = strdup(dest); + strtolower(clientIP); + + // Check if user wants to skip queries coming from localhost + if(config.ignore_localhost && + (strcmp(clientIP, "127.0.0.1") == 0 || strcmp(clientIP, "::1") == 0)) + { + free(domainString); + free(clientIP); + unlock_shm(); + return false; + } + + // Log new query if in debug mode + const char *proto = (type == UDP) ? "UDP" : "TCP"; + if(config.debug & DEBUG_QUERIES) + { + logg("**** new %s %s \"%s\" from %s (ID %i, FTL %i, %s:%i)", + proto, types, domainString, clientIP, id, queryID, file, line); + } + + // Update counters + counters->querytype[querytype-1]++; + + // Update overTime + const unsigned int timeidx = getOverTimeID(querytimestamp); + overTime[timeidx].querytypedata[querytype-1]++; + + // Skip rest of the analysis if this query is not of type A or AAAA + // but user wants to see only A and AAAA queries (pre-v4.1 behavior) + if(config.analyze_only_A_AAAA && querytype != TYPE_A && querytype != TYPE_AAAA) + { + // Don't process this query further here, we already counted it + if(config.debug & DEBUG_QUERIES) logg("Notice: Skipping new query: %s (%i)", types, id); + free(domainString); + free(clientIP); + unlock_shm(); + return false; + } + + // Go through already knows domains and see if it is one of them + const int domainID = findDomainID(domainString, true); + + // Go through already knows clients and see if it is one of them + const int clientID = findClientID(clientIP, true); + + // Save everything + queriesData* query = getQuery(queryID, false); + if(query == NULL) + { + // Encountered memory error, skip query + // Free allocated memory + free(domainString); + free(clientIP); + // Release thread lock + unlock_shm(); + return false; + } + + query->magic = MAGICBYTE; + query->timestamp = querytimestamp; + query->type = querytype; + query->status = QUERY_UNKNOWN; + query->domainID = domainID; + query->clientID = clientID; + query->timeidx = timeidx; + // Initialize database rowID with zero, will be set when the query is stored in the long-term DB + query->db = 0; + query->id = id; + query->complete = false; + query->response = converttimeval(request); + // Initialize reply type + query->reply = REPLY_UNKNOWN; + // Store DNSSEC result for this domain + query->dnssec = DNSSEC_UNSPECIFIED; + query->CNAME_domainID = -1; + + // Check and apply possible privacy level rules + // The currently set privacy level (at the time the query is + // generated) is stored in the queries structure + get_privacy_level(NULL); + query->privacylevel = config.privacylevel; + + // Increase DNS queries counter + counters->queries++; + // Count this query as unknown as long as no reply has + // been found and analyzed + counters->unknown++; + + // Update overTime data + overTime[timeidx].total++; + + // Get client pointer + clientsData* client = getClient(clientID, true); + if(client == NULL) + { + // Encountered memory error, skip query + // Free allocated memory + free(domainString); + free(clientIP); + // Release thread lock + unlock_shm(); + return false; + } + + // Update overTime data structure with the new client + client->overTime[timeidx]++; + + // Set lastQuery timer and add one query for network table + client->lastQuery = querytimestamp; + client->numQueriesARP++; + + bool blockDomain = FTL_check_blocking(queryID, domainID, clientID, blockingreason); + + // Free allocated memory + free(domainString); + free(clientIP); + + // Release thread lock + unlock_shm(); + + return blockDomain; +} + +void _FTL_get_blocking_metadata(union all_addr **addrp, unsigned int *flags, const char* file, const int line) +{ + // Check first if we need to force our reply to something different than the + // default/configured blocking mode For instance, we need to force NXDOMAIN + // for intercepted _esni.* queries + if(force_next_DNS_reply == NXDOMAIN) + { + *flags = F_NXDOMAIN; + // Reset DNS reply forcing + force_next_DNS_reply = 0u; + return; + } + + // Add flags according to current blocking mode + // We bit-add here as flags already contains either F_IPV4 or F_IPV6 + *flags |= blocking_flags; + + if(*flags & F_IPV6) + { + // Pass blocking IPv6 address (will be :: in most cases) + *addrp = &blocking_addrp_v6; + } + else + { + // Pass blocking IPv4 address (will be 0.0.0.0 in most cases) + *addrp = &blocking_addrp_v4; + } + + if(config.blockingmode == MODE_NX) + { + // If we block in NXDOMAIN mode, we add the NEGATIVE response + // and the NXDOMAIN flags + *flags = F_NXDOMAIN; + } + else if(config.blockingmode == MODE_NODATA || + (config.blockingmode == MODE_IP_NODATA_AAAA && (*flags & F_IPV6))) + { + // If we block in NODATA mode or NODATA for AAAA queries, we apply + // the NOERROR response flag. This ensures we're sending an empty response + *flags = F_NOERR; + } +} + +static int findQueryID(const int id) +{ + // Loop over all queries - we loop in reverse order (start from the most recent query and + // continuously walk older queries while trying to find a match. Ideally, we should always + // find the correct query with zero iterations, but it may happen that queries are processed + // asynchronously, e.g. for slow upstream relies to a huge amount of requests. + // We iterate from the most recent query down to at most MAXITER queries in the past to avoid + // iterating through the entire array of queries + // MAX(0, a) is used to return 0 in case a is negative (negative array indices are harmful) + const int until = MAX(0, counters->queries-MAXITER); + const int start = MAX(0, counters->queries-1); + + // Check UUIDs of queries + for(int i = start; i >= until; i--) + { + const queriesData* query = getQuery(i, true); + + // Check if the returned pointer is valid before trying to access it + if(query == NULL) + continue; + + if(query->id == id) + return i; + } + + // If not found + return -1; +} + +void _FTL_forwarded(const unsigned int flags, const char *name, const union all_addr *addr, const int id, + const char* file, const int line) +{ + // Save that this query got forwarded to an upstream server + + // Don't analyze anything if in PRIVACY_NOSTATS mode + if(config.privacylevel >= PRIVACY_NOSTATS) + return; + + // Lock shared memory + lock_shm(); + + // Get forward destination IP address + char dest[ADDRSTRLEN]; + // If addr == NULL, we will only duplicate an empty string instead of uninitialized memory + dest[0] = '\0'; + if(addr != NULL) + inet_ntop((flags & F_IPV4) ? AF_INET : AF_INET6, addr, dest, ADDRSTRLEN); + + // Convert upstreamIP to lower case + char *upstreamIP = strdup(dest); + strtolower(upstreamIP); + + // Debug logging + if(config.debug & DEBUG_QUERIES) logg("**** forwarded %s to %s (ID %i, %s:%i)", name, upstreamIP, id, file, line); + + // Save status and upstreamID in corresponding query identified by dnsmasq's ID + const int queryID = findQueryID(id); + if(queryID < 0) + { + // This may happen e.g. if the original query was a PTR query or "pi.hole" + // as we ignore them altogether + free(upstreamIP); + unlock_shm(); + return; + } + + // Get query pointer + queriesData* query = getQuery(queryID, true); + + // Proceed only if + // - current query has not been marked as replied to so far + // (it could be that answers from multiple forward + // destinations are coming in for the same query) + // - the query was formally known as cached but had to be forwarded + // (this is a special case further described below) + // Use short-circuit evaluation to check if query is NULL + if(query == NULL || (query->complete && query->status != QUERY_CACHE)) + { + free(upstreamIP); + unlock_shm(); + return; + } + + // Get ID of upstream destination, create new upstream record + // if not found in current data structure + const int upstreamID = findUpstreamID(upstreamIP, true); + query->upstreamID = upstreamID; + + // Get time index for this query + const unsigned int timeidx = query->timeidx; + + if(query->status == QUERY_CACHE) + { + // Detect if we cached the but need to ask the upstream + // servers for the actual IPs now, we remove this query from the + // counters for cache replied queries as we had to forward a + // request for it. Example: + // Assume a domain a.com is a CNAME which is cached and has a very + // long TTL. It point to another domain server.a.com which has an + // A record but this has a much lower TTL. + // If you now query a.com and then again after some time, you end + // up in a situation where dnsmasq can answer the first level of + // the DNS result (the CNAME) from cache, hence the status of this + // query is marked as "answered from cache" in FTLDNS. However, for + // server.a.com wit the much shorter TTL, we still have to forward + // something and ask the upstream server for the final IP address. + // This code section acknowledges this by removing one entry from + // the cached counters as we will re-brand this query as having been + // forwarded in the following. + counters->cached--; + // Also correct overTime data + overTime[timeidx].cached--; + + // Correct reply timer + struct timeval response; + gettimeofday(&response, 0); + // Reset timer, shift slightly into the past to acknowledge the time + // FTLDNS needed to look up the CNAME in its cache + query->response = converttimeval(response) - query->response; + } + else + { + // Normal forwarded query (status is set below) + // Query is no longer unknown + counters->unknown--; + // Hereby, this query is now fully determined + query->complete = true; + } + + // Set query status to forwarded only after the + // if(query->status == QUERY_CACHE) { ... } + // from above as otherwise this check will always + // be negative + query->status = QUERY_FORWARDED; + + // Update overTime data + overTime[timeidx].forwarded++; + + // Update counter for forwarded queries + counters->forwarded++; + + // Release allocated memory + free(upstreamIP); + + // Unlock shared memory + unlock_shm(); +} + +void FTL_dnsmasq_reload(void) +{ + // This function is called by the dnsmasq code on receive of SIGHUP + // *before* clearing the cache and rereading the lists + // This is the only hook that is not skipped in PRIVACY_NOSTATS mode + + logg("Reloading DNS cache"); + + // Inspect 01-pihole.conf to see if Pi-hole blocking is enabled, + // i.e. if /etc/pihole/gravity.list is sourced as addn-hosts file + check_blocking_status(); + + // Reread pihole-FTL.conf to see which blocking mode the user wants to use + // It is possible to change the blocking mode here as we anyhow clear the + // cache and reread all blocking lists + // Passing NULL to this function means it has to open the config file on + // its own behalf (on initial reading, the config file is already opened) + get_blocking_mode(NULL); + // Update blocking metadata (target IP addresses and DNS header flags) + // as the blocking mode might have changed + prepare_blocking_metadata(); + + // Reread pihole-FTL.conf to see which debugging flags are set + read_debuging_settings(NULL); + + FTL_reload_all_domainlists(); + + // Print current set of capabilities if requested via debug flag + if(config.debug & DEBUG_CAPS) + check_capabilities(); +} + +void _FTL_reply(const unsigned short flags, const char *name, const union all_addr *addr, const int id, + const char* file, const int line) +{ + // Don't analyze anything if in PRIVACY_NOSTATS mode + if(config.privacylevel >= PRIVACY_NOSTATS) + return; + + // Lock shared memory + lock_shm(); + + // Determine returned result if available + char dest[ADDRSTRLEN]; dest[0] = '\0'; + if(addr) + { + inet_ntop((flags & F_IPV4) ? AF_INET : AF_INET6, addr, dest, ADDRSTRLEN); + } + + // Extract answer (used e.g. for detecting if a local config is a user-defined + // wildcard blocking entry in form "server=/tobeblocked.com/") + const char *answer = dest; + if(flags & F_CNAME) + answer = "(CNAME)"; + else if((flags & F_NEG) && (flags & F_NXDOMAIN)) + answer = "(NXDOMAIN)"; + else if(flags & F_NEG) + answer = "(NODATA)"; + + // Possible debugging output + if(config.debug & DEBUG_QUERIES) + { + logg("**** got reply %s is %s (ID %i, %s:%i)", name, answer, id, file, line); + print_flags(flags); + } + + // Get response time + struct timeval response; + gettimeofday(&response, 0); + + // Save status in corresponding query identified by dnsmasq's ID + const int i = findQueryID(id); + if(i < 0) + { + // This may happen e.g. if the original query was "pi.hole" + if(config.debug & DEBUG_QUERIES) logg("FTL_reply(): Query %i has not been found", id); + unlock_shm(); + return; + } + + // Get query pointer + queriesData* query = getQuery(i, true); + + // Check if reply time is still unknown + // We only process the first reply in here + // Use short-circuit evaluation to check if query is NULL + if(query == NULL || query->reply != REPLY_UNKNOWN) + { + // Nothing to be done here + unlock_shm(); + return; + } + + // Determine if this reply is an exact match for the queried domain + const int domainID = query->domainID; + + // Get domain pointer + domainsData* domain = getDomain(domainID, true); + if(domain == NULL) + { + // Memory error, skip reply + unlock_shm(); + return; + } + + // Check if this domain matches exactly + const bool isExactMatch = (name != NULL && strcasecmp(getstr(domain->domainpos), name) == 0); + + if((flags & F_CONFIG) && isExactMatch && !query->complete) + { + // Answered from local configuration, might be a wildcard or user-provided + // This query is no longer unknown + counters->unknown--; + + // Get time index + const unsigned int timeidx = query->timeidx; + + // Check whether this query was blocked + if(strcmp(answer, "(NXDOMAIN)") == 0 || + strcmp(answer, "0.0.0.0") == 0 || + strcmp(answer, "::") == 0) + { + // Mark query as blocked + clientsData* client = getClient(query->clientID, true); + query_blocked(query, domain, client, QUERY_REGEX); + } + else + { + // Answered from a custom (user provided) cache file + counters->cached++; + overTime[timeidx].cached++; + + query->status = QUERY_CACHE; + } + + // Save reply type and update individual reply counters + save_reply_type(flags, query, response); + + // Hereby, this query is now fully determined + query->complete = true; + } + else if((flags & F_FORWARD) && isExactMatch) + { + // Only proceed if query is not already known + // to have been blocked by Quad9 + if(query->reply != QUERY_EXTERNAL_BLOCKED_IP && + query->reply != QUERY_EXTERNAL_BLOCKED_NULL && + query->reply != QUERY_EXTERNAL_BLOCKED_NXRA) + { + // Save reply type and update individual reply counters + save_reply_type(flags, query, response); + + // Detect if returned IP indicates that this query was blocked + detect_blocked_IP(flags, answer, i); + } + } + else if(flags & F_REVERSE) + { + // isExactMatch is not used here as the PTR is special. + // Example: + // Question: PTR 8.8.8.8 + // will lead to: + // domain->domain = 8.8.8.8.in-addr.arpa + // and will return + // name = google-public-dns-a.google.com + // Hence, isExactMatch is always false + + // Save reply type and update individual reply counters + save_reply_type(flags, query, response); + } + else if(isExactMatch && !query->complete) + { + logg("*************************** unknown REPLY ***************************"); + print_flags(flags); + } + + unlock_shm(); +} + +static void detect_blocked_IP(const unsigned short flags, const char* answer, const int queryID) +{ + // Compare returned IP against list of known blocking splash pages + + // First, we check if we want to skip this result even before comparing against the known IPs + if(flags & F_HOSTS || flags & F_REVERSE) + { + // Skip replies which originated locally. Otherwise, we would + // count gravity.list blocked queries as externally blocked. + // Also: Do not mark responses of PTR requests as externally blocked. + if(config.debug & DEBUG_EXTBLOCKED) + { + const char *cause = (flags & F_HOSTS) ? "origin is HOSTS" : "query is PTR"; + logg("Skipping detection of external blocking IP for ID %i as %s", queryID, cause); + } + + // Return early, do not compare against known blocking page IP addresses below + return; + } + + // If received one of the following IPs as reply, OpenDNS + // (Cisco Umbrella) blocked this query + // See https://support.opendns.com/hc/en-us/articles/227986927-What-are-the-Cisco-Umbrella-Block-Page-IP-Addresses- + // for a full list of these IP addresses + if(flags & F_IPV4 && answer != NULL && + (strcmp("146.112.61.104", answer) == 0 || + strcmp("146.112.61.105", answer) == 0 || + strcmp("146.112.61.106", answer) == 0 || + strcmp("146.112.61.107", answer) == 0 || + strcmp("146.112.61.108", answer) == 0 || + strcmp("146.112.61.109", answer) == 0 || + strcmp("146.112.61.110", answer) == 0 )) + { + if(config.debug & DEBUG_EXTBLOCKED) + { + const queriesData* query = getQuery(queryID, true); + if(query != NULL) + { + const domainsData* domain = getDomain(query->domainID, true); + if(domain != NULL) + { + logg("Upstream responded with known blocking page (IPv4), ID %i:\n\t\"%s\" -> \"%s\"", + queryID, getstr(domain->domainpos), answer); + } + } + } + + // Update status + query_externally_blocked(queryID, QUERY_EXTERNAL_BLOCKED_IP); + } + else if(flags & F_IPV6 && answer != NULL && + (strcmp("::ffff:146.112.61.104", answer) == 0 || + strcmp("::ffff:146.112.61.105", answer) == 0 || + strcmp("::ffff:146.112.61.106", answer) == 0 || + strcmp("::ffff:146.112.61.107", answer) == 0 || + strcmp("::ffff:146.112.61.108", answer) == 0 || + strcmp("::ffff:146.112.61.109", answer) == 0 || + strcmp("::ffff:146.112.61.110", answer) == 0 )) + { + if(config.debug & DEBUG_EXTBLOCKED) + { + const queriesData* query = getQuery(queryID, true); + if(query != NULL) + { + const domainsData* domain = getDomain(query->domainID, true); + if(domain != NULL) + { + logg("Upstream responded with known blocking page (IPv6), ID %i:\n\t\"%s\" -> \"%s\"", + queryID, getstr(domain->domainpos), answer); + } + } + } + + // Update status + query_externally_blocked(queryID, QUERY_EXTERNAL_BLOCKED_IP); + } + + // If upstream replied with 0.0.0.0 or ::, + // we assume that it filtered the reply as + // nothing is reachable under these addresses + else if(flags & F_IPV4 && answer != NULL && + strcmp("0.0.0.0", answer) == 0) + { + if(config.debug & DEBUG_EXTBLOCKED) + { + const queriesData* query = getQuery(queryID, true); + if(query != NULL) + { + const domainsData* domain = getDomain(query->domainID, true); + if(domain != NULL) + { + logg("Upstream responded with 0.0.0.0, ID %i:\n\t\"%s\" -> \"%s\"", + queryID, getstr(domain->domainpos), answer); + } + } + } + + // Update status + query_externally_blocked(queryID, QUERY_EXTERNAL_BLOCKED_NULL); + } + else if(flags & F_IPV6 && answer != NULL && + strcmp("::", answer) == 0) + { + if(config.debug & DEBUG_EXTBLOCKED) + { + const queriesData* query = getQuery(queryID, true); + if(query != NULL) + { + const domainsData* domain = getDomain(query->domainID, true); + if(domain != NULL) + { + logg("Upstream responded with ::, ID %i:\n\t\"%s\" -> \"%s\"", + queryID, getstr(domain->domainpos), answer); + } + } + } + + // Update status + query_externally_blocked(queryID, QUERY_EXTERNAL_BLOCKED_NULL); + } +} + +static void query_externally_blocked(const int queryID, const unsigned char status) +{ + // Get query pointer + queriesData* query = getQuery(queryID, true); + if(query == NULL) + { + // Memory error, skip check for this query + return; + } + + // Get time index + const unsigned int timeidx = query->timeidx; + + // If query is already known to be externally blocked, + // then we have nothing to do here + if(query->status == QUERY_EXTERNAL_BLOCKED_IP || + query->status == QUERY_EXTERNAL_BLOCKED_NULL || + query->status == QUERY_EXTERNAL_BLOCKED_NXRA) + return; + + // Correct counters if necessary ... + if(query->status == QUERY_FORWARDED) + { + counters->forwarded--; + overTime[timeidx].forwarded--; + + // Get forward pointer + upstreamsData* upstream = getUpstream(query->upstreamID, true); + if(upstream != NULL) + upstream->count--; + } + + // Mark query as blocked + domainsData* domain = getDomain(query->domainID, true); + clientsData* client = getClient(query->clientID, true); + query_blocked(query, domain, client, status); +} + +void _FTL_cache(const unsigned int flags, const char *name, const union all_addr *addr, + const char *arg, const int id, const char* file, const int line) +{ + // Save that this query got answered from cache + + // Don't analyze anything if in PRIVACY_NOSTATS mode + if(config.privacylevel >= PRIVACY_NOSTATS) + return; + + // Lock shared memory + lock_shm(); + + // Obtain destination IP address if available for this query type + char dest[ADDRSTRLEN]; dest[0] = '\0'; + if(addr) + { + inet_ntop((flags & F_IPV4) ? AF_INET : AF_INET6, addr, dest, ADDRSTRLEN); + } + + // If domain is "pi.hole", we skip this query + // We compare case-insensitive here + if(strcasecmp(name, "pi.hole") == 0) + { + unlock_shm(); + return; + } + + // Debug logging + if(config.debug & DEBUG_QUERIES) + { + logg("**** got cache answer for %s / %s / %s (ID %i, %s:%i)", name, dest, arg, id, file, line); + print_flags(flags); + } + + // Get response time + struct timeval response; + gettimeofday(&response, 0); + + if(((flags & F_HOSTS) && (flags & F_IMMORTAL)) || + ((flags & F_NAMEP) && (flags & F_DHCP)) || + (flags & F_FORWARD) || + (flags & F_REVERSE) || + (flags & F_RRNAME)) + { + // Local list: /etc/hosts, /etc/pihole/local.list, etc. + // or + // DHCP server reply + // or + // cached answer to previously forwarded request + + // Determine requesttype + unsigned char requesttype = 0; + if((flags & F_HOSTS) || // local.list, hostname.list, /etc/hosts and others + ((flags & F_NAMEP) && (flags & F_DHCP)) || // DHCP server reply + (flags & F_FORWARD) || // cached answer to previously forwarded request + (flags & F_REVERSE) || // cached answer to reverse request (PTR) + (flags & F_RRNAME)) // cached answer to TXT query + { + requesttype = QUERY_CACHE; + } + else + { + logg("*************************** unknown CACHE reply (1) ***************************"); + print_flags(flags); + unlock_shm(); + return; + } + + // Search query in FTL's query data + const int queryID = findQueryID(id); + if(queryID < 0) + { + // This may happen e.g. if the original query was a PTR query or "pi.hole" + // as we ignore them altogether + unlock_shm(); + return; + } + + // Get query pointer + queriesData* query = getQuery(queryID, true); + + // Skip this query if already marked as complete + // Use short-circuit evaluation to check query if query is NULL + if(query == NULL || query->complete) + { + unlock_shm(); + return; + } + + // This query is no longer unknown + counters->unknown--; + + // Get time index + const unsigned int timeidx = query->timeidx; + + query->status = requesttype; + + // Detect if returned IP indicates that this query was blocked + detect_blocked_IP(flags, dest, queryID); + + // Re-read requesttype as detect_blocked_IP() might have changed it + requesttype = query->status; + + // Handle counters accordingly + switch(requesttype) + { + case QUERY_CACHE: // cached from one of the lists + counters->cached++; + overTime[timeidx].cached++; + break; + case QUERY_EXTERNAL_BLOCKED_IP: + case QUERY_EXTERNAL_BLOCKED_NULL: + case QUERY_EXTERNAL_BLOCKED_NXRA: + // everything has already been done + // in query_externally_blocked() + break; + } + + // Save reply type and update individual reply counters + save_reply_type(flags, query, response); + + // Hereby, this query is now fully determined + query->complete = true; + } + else + { + logg("*************************** unknown CACHE reply (2) ***************************"); + print_flags(flags); + } + unlock_shm(); +} + +static void query_blocked(queriesData* query, domainsData* domain, clientsData* client, const unsigned char new_status) +{ + // Get response time + struct timeval response; + gettimeofday(&response, 0); + save_reply_type(blocking_flags, query, response); + + // Adjust counters if we recorded a non-blocking status + if(query->status == QUERY_UNKNOWN) + { + counters->unknown--; + } + else if(query->status == QUERY_FORWARDED) + { + counters->forwarded--; + } + else if(query->status == QUERY_CACHE) + { + counters->cached--; + } + else + { + // Already a blocked query, no need to change anything + return; + } + + // Count as blocked query + counters->blocked++; + overTime[query->timeidx].blocked++; + if(domain != NULL) + domain->blockedcount++; + if(client != NULL) + client->blockedcount++; + + // Update status + query->status = new_status; +} + +void _FTL_dnssec(const int status, const int id, const char* file, const int line) +{ + // Process DNSSEC result for a domain + + // Don't analyze anything if in PRIVACY_NOSTATS mode + if(config.privacylevel >= PRIVACY_NOSTATS) + return; + + // Lock shared memory + lock_shm(); + + // Search for corresponding query identified by ID + const int queryID = findQueryID(id); + if(queryID < 0) + { + // This may happen e.g. if the original query was an unhandled query type + unlock_shm(); + return; + } + + // Get query pointer + queriesData* query = getQuery(queryID, true); + if(query == NULL) + { + // Memory error, skip this DNSSEC details + unlock_shm(); + return; + } + + // Debug logging + if(config.debug & DEBUG_QUERIES) + { + // Get domain pointer + const domainsData* domain = getDomain(query->domainID, true); + if(domain != NULL) + { + logg("**** got DNSSEC details for %s: %i (ID %i, %s:%i)", getstr(domain->domainpos), status, id, file, line); + } + } + + // Iterate through possible values + if(status == STAT_SECURE) + query->dnssec = DNSSEC_SECURE; + else if(status == STAT_INSECURE) + query->dnssec = DNSSEC_INSECURE; + else + query->dnssec = DNSSEC_BOGUS; + + // Unlock shared memory + unlock_shm(); +} + +void _FTL_upstream_error(const unsigned int rcode, const int id, const char* file, const int line) +{ + // Process upstream errors + // Queries with error are those where the RCODE + // in the DNS header is neither NOERROR nor NXDOMAIN. + + // Don't analyze anything if in PRIVACY_NOSTATS mode + if(config.privacylevel >= PRIVACY_NOSTATS) + return; + + // Lock shared memory + lock_shm(); + + // Search for corresponding query identified by ID + const int queryID = findQueryID(id); + if(queryID < 0) + { + // This may happen e.g. if the original query was an unhandled query type + unlock_shm(); + return; + } + + // Get query pointer + queriesData* query = getQuery(queryID, true); + if(query == NULL) + { + // Memory error, skip this query + unlock_shm(); + return; + } + + // Translate dnsmasq's rcode into something we can use + const char *rcodestr = NULL; + switch(rcode) + { + case SERVFAIL: + rcodestr = "SERVFAIL"; + query->reply = REPLY_SERVFAIL; + break; + case REFUSED: + rcodestr = "REFUSED"; + query->reply = REPLY_REFUSED; + break; + case NOTIMP: + rcodestr = "NOT IMPLEMENTED"; + query->reply = REPLY_NOTIMP; + break; + default: + rcodestr = "UNKNOWN"; + query->reply = REPLY_OTHER; + break; + } + + // Debug logging + if(config.debug & DEBUG_QUERIES) + { + // Get domain pointer + const domainsData* domain = getDomain(query->domainID, true); + + // Get domain name + const char *domainname; + if(domain != NULL) + domainname = getstr(domain->domainpos); + else + domainname = ""; + + logg("**** got error report for %s: %s (ID %i, %s:%i)", domainname, rcodestr, id, file, line); + + if(query->reply == REPLY_OTHER) + { + logg("Unknown rcode = %i", rcode); + } + } + + // Unlock shared memory + unlock_shm(); +} + +void _FTL_header_analysis(const unsigned char header4, const unsigned int rcode, const int id, const char* file, const int line) +{ + // Analyze DNS header bits + + // Don't analyze anything if in PRIVACY_NOSTATS mode + if(config.privacylevel >= PRIVACY_NOSTATS) + return; + + // Check if RA bit is unset in DNS header and rcode is NXDOMAIN + // If the response code (rcode) is NXDOMAIN, we may be seeing a response from + // an externally blocked query. As they are not always accompany a necessary + // SOA record, they are not getting added to our cache and, therefore, + // FTL_reply() is never getting called from within the cache routines. + // Hence, we have to store the necessary information about the NXDOMAIN + // reply already here. + if((header4 & 0x80) || rcode != NXDOMAIN) + { + // RA bit is set or rcode is not NXDOMAIN + return; + } + + // Lock shared memory + lock_shm(); + + // Search for corresponding query identified by ID + const int queryID = findQueryID(id); + if(queryID < 0) + { + // This may happen e.g. if the original query was an unhandled query type + unlock_shm(); + return; + } + + // Get query pointer + queriesData* query = getQuery(queryID, true); + if(query == NULL) + { + // Memory error, skip this query + unlock_shm(); + return; + } + + // Possible debugging information + if(config.debug & DEBUG_QUERIES) + { + // Get domain pointer + const domainsData* domain = getDomain(query->domainID, true); + + // Get domain name + const char *domainname; + if(domain != NULL) + domainname = getstr(domain->domainpos); + else + domainname = ""; + + logg("**** %s externally blocked (ID %i, FTL %i, %s:%i)", domainname, id, queryID, file, line); + } + + // Get response time + struct timeval response; + gettimeofday(&response, 0); + + // Store query as externally blocked + query_externally_blocked(queryID, QUERY_EXTERNAL_BLOCKED_NXRA); + + // Store reply type as replied with NXDOMAIN + save_reply_type(F_NEG | F_NXDOMAIN, query, response); + + // Unlock shared memory + unlock_shm(); +} + +void print_flags(const unsigned int flags) +{ + // Debug function, listing resolver flags in clear text + // e.g. "Flags: F_FORWARD F_NEG F_IPV6" + + // Only print flags if corresponding debugging flag is set + if(!(config.debug & DEBUG_FLAGS)) + return; + + char *flagstr = calloc(sizeof(flagnames) + 1, sizeof(char)); + for (unsigned int i = 0; i < (sizeof(flagnames) / sizeof(*flagnames)); i++) + if (flags & (1u << i)) + strcat(flagstr, flagnames[i]); + logg(" Flags: %s", flagstr); + free(flagstr); +} + +static void save_reply_type(const unsigned int flags, queriesData* query, const struct timeval response) +{ + // Iterate through possible values + if(flags & F_NEG) + { + if(flags & F_NXDOMAIN) + { + // NXDOMAIN + query->reply = REPLY_NXDOMAIN; + counters->reply_NXDOMAIN++; + } + else + { + // NODATA(-IPv6) + query->reply = REPLY_NODATA; + counters->reply_NODATA++; + } + } + else if(flags & F_CNAME) + { + // + query->reply = REPLY_CNAME; + counters->reply_CNAME++; + } + else if(flags & F_REVERSE) + { + // reserve lookup + query->reply = REPLY_DOMAIN; + counters->reply_domain++; + } + else if(flags & F_RRNAME) + { + // TXT query + query->reply = REPLY_RRNAME; + } + else + { + // Valid IP + query->reply = REPLY_IP; + counters->reply_IP++; + } + + // Save response time (relative time) + query->response = converttimeval(response) - + query->response; +} + +pthread_t telnet_listenthreadv4; +pthread_t telnet_listenthreadv6; +pthread_t socket_listenthread; +pthread_t DBthread; +pthread_t GCthread; +pthread_t DNSclientthread; + +void FTL_fork_and_bind_sockets(struct passwd *ent_pw) +{ + // Going into daemon mode involves storing the + // PID of the generated child process. If FTL + // is asked to stay in foreground, we just save + // the PID of the current process in the PID file + if(daemonmode) + go_daemon(); + else + savepid(); + + // We will use the attributes object later to start all threads in + // detached mode + pthread_attr_t attr; + // Initialize thread attributes object with default attribute values + pthread_attr_init(&attr); + // When a detached thread terminates, its resources are automatically + // released back to the system without the need for another thread to + // join with the terminated thread + pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); + + // Bind to sockets + bind_sockets(); + + // Start TELNET IPv4 thread + if(ipv4telnet && pthread_create( &telnet_listenthreadv4, &attr, telnet_listening_thread_IPv4, NULL ) != 0) + { + logg("Unable to open IPv4 telnet listening thread. Exiting..."); + exit(EXIT_FAILURE); + } + + // Start TELNET IPv6 thread + if(ipv6telnet && pthread_create( &telnet_listenthreadv6, &attr, telnet_listening_thread_IPv6, NULL ) != 0) + { + logg("Unable to open IPv6 telnet listening thread. Exiting..."); + exit(EXIT_FAILURE); + } + + // Start SOCKET thread + if(pthread_create( &socket_listenthread, &attr, socket_listening_thread, NULL ) != 0) + { + logg("Unable to open Unix socket listening thread. Exiting..."); + exit(EXIT_FAILURE); + } + + // Start database thread if database is used + if(database && pthread_create( &DBthread, &attr, DB_thread, NULL ) != 0) + { + logg("Unable to open database thread. Exiting..."); + exit(EXIT_FAILURE); + } + + // Start thread that will stay in the background until garbage + // collection needs to be done + if(pthread_create( &GCthread, &attr, GC_thread, NULL ) != 0) + { + logg("Unable to open GC thread. Exiting..."); + exit(EXIT_FAILURE); + } + + // Start thread that will stay in the background until host names + // needs to be resolved + if(pthread_create( &DNSclientthread, &attr, DNSclient_thread, NULL ) != 0) + { + logg("Unable to open DNS client thread. Exiting..."); + exit(EXIT_FAILURE); + } + + // Chown files if FTL started as user root but a dnsmasq config + // option states to run as a different user/group (e.g. "nobody") + if(ent_pw != NULL && getuid() == 0) + { + if(chown(FTLfiles.log, ent_pw->pw_uid, ent_pw->pw_gid) == -1) + logg("Setting ownership (%i:%i) of %s failed: %s (%i)", + ent_pw->pw_uid, ent_pw->pw_gid, FTLfiles.log, strerror(errno), errno); + if(database && chown(FTLfiles.FTL_db, ent_pw->pw_uid, ent_pw->pw_gid) == -1) + logg("Setting ownership (%i:%i) of %s failed: %s (%i)", + ent_pw->pw_uid, ent_pw->pw_gid, FTLfiles.FTL_db, strerror(errno), errno); + chown_all_shmem(ent_pw); + } + + // Obtain DNS port from dnsmasq daemon + config.dns_port = daemon->port; +} + +// int cache_inserted, cache_live_freed are defined in dnsmasq/cache.c +void getCacheInformation(const int *sock) +{ + ssend(*sock,"cache-size: %i\ncache-live-freed: %i\ncache-inserted: %i\n", + daemon->cachesize, + daemon->metrics[METRIC_DNS_CACHE_LIVE_FREED], + daemon->metrics[METRIC_DNS_CACHE_INSERTED]); + // cache-size is obvious + // It means the resolver handled names lookups that + // needed to be sent to upstream servers and that + // was thrown out of the cache before reaching the end of its + // time-to-live, to make room for a newer name. + // For , smaller is better. New queries are always + // cached. If the cache is full with entries which haven't reached + // the end of their time-to-live, then the entry which hasn't been + // looked up for the longest time is evicted. +} + +void _FTL_forwarding_failed(const struct server *server, const char* file, const int line) +{ + // Forwarding to upstream server failed + + // Don't analyze anything if in PRIVACY_NOSTATS mode + if(config.privacylevel >= PRIVACY_NOSTATS) + return; + + // Lock shared memory + lock_shm(); + + // Try to obtain destination IP address if available + char dest[ADDRSTRLEN]; + if(server->addr.sa.sa_family == AF_INET) + inet_ntop(AF_INET, &server->addr.in.sin_addr, dest, ADDRSTRLEN); + else + inet_ntop(AF_INET6, &server->addr.in6.sin6_addr, dest, ADDRSTRLEN); + + // Convert upstream to lower case + char *upstreamIP = strdup(dest); + strtolower(upstreamIP); + + // Get upstream ID + const int upstreamID = findUpstreamID(upstreamIP, false); + + // Possible debugging information + if(config.debug & DEBUG_QUERIES) logg("**** forwarding to %s (ID %i, %s:%i) FAILED", dest, upstreamID, file, line); + + // Get upstream pointer + upstreamsData* upstream = getUpstream(upstreamID, true); + + // Update counter + if(upstream != NULL) + upstream->failed++; + + // Clean up and unlock shared memory + free(upstreamIP); + unlock_shm(); + return; +} + +static unsigned long __attribute__((const)) converttimeval(const struct timeval time) +{ + // Convert time from struct timeval into units + // of 10*milliseconds + return time.tv_sec*10000 + time.tv_usec/100; +} + +// This subroutine prepares IPv4 and IPv6 addresses for blocking queries depending on the configured blocking mode +static void prepare_blocking_metadata(void) +{ + // Reset all blocking metadata + blocking_flags = 0; + memset(&blocking_addrp_v4, 0, sizeof(blocking_addrp_v4)); + memset(&blocking_addrp_v6, 0, sizeof(blocking_addrp_v6)); + + // Set blocking_flags to F_HOSTS so dnsmasq logs blocked queries being answered from a specific source + // (it would otherwise assume it knew the blocking status from cache which would prevent us from + // printing the blocking source (blacklist, regex, gravity) in dnsmasq's log file, our pihole.log) + blocking_flags = F_HOSTS; + + // Use the blocking IPv4 address from setupVars.conf only if needed for selected blocking mode + char* const IPv4addr = read_setupVarsconf("IPV4_ADDRESS"); + if((config.blockingmode == MODE_IP || config.blockingmode == MODE_IP_NODATA_AAAA) && + IPv4addr != NULL && strlen(IPv4addr) > 0) + { + // Strip off everything at the end of the IP (CIDR might be there) + char* a=IPv4addr; for(;*a;a++) if(*a == '/') *a = 0; + // Prepare IPv4 address for records + if(inet_pton(AF_INET, IPv4addr, &blocking_addrp_v4) != 1) + logg("ERROR: Found invalid IPv4 address in setupVars.conf: %s", IPv4addr); + } + // Free IPv4addr + clearSetupVarsArray(); + + // Use the blocking IPv6 address from setupVars.conf only if needed for selected blocking mode + char* const IPv6addr = read_setupVarsconf("IPV6_ADDRESS"); + if(config.blockingmode == MODE_IP && + IPv6addr != NULL && strlen(IPv6addr) > 0) + { + // Strip off everything at the end of the IP (CIDR might be there) + char* a=IPv6addr; for(;*a;a++) if(*a == '/') *a = 0; + // Prepare IPv6 address for records + if(inet_pton(AF_INET6, IPv6addr, &blocking_addrp_v6) != 1) + logg("ERROR: Found invalid IPv6 address in setupVars.conf: %s", IPv4addr); + } + // Free IPv6addr + clearSetupVarsArray(); +} + +// Called when a (forked) TCP worker is terminated by receiving SIGALRM +// We close the dedicated database connection this client had opened +// to avoid dangling database locks +void FTL_TCP_worker_terminating(void) +{ + if(config.debug & DEBUG_DATABASE) + { + logg("TCP worker terminating, " + "closing gravity database connection"); + } + + // Close dedicated database connection of this fork + gravityDB_close(); +} diff --git a/src/dnsmasq_interface.h b/src/dnsmasq_interface.h new file mode 100644 index 000000000..ab4c77fc0 --- /dev/null +++ b/src/dnsmasq_interface.h @@ -0,0 +1,59 @@ +/* Pi-hole: A black hole for Internet advertisements +* (c) 2017 Pi-hole, LLC (https://pi-hole.net) +* Network-wide ad blocking via your own hardware. +* +* FTL Engine +* dnsmasq server interfacing routines +* +* This file is copyright under the latest version of the EUPL. +* Please see LICENSE file for your rights under this license. */ +#ifndef DNSMASQ_INTERFACE_H +#define DNSMASQ_INTERFACE_H + +// Including stdbool.h here as it is required for defining the boolean prototype of FTL_new_query +#include + +extern int socketfd, telnetfd4, telnetfd6; +extern unsigned char* pihole_privacylevel; +enum { TCP, UDP }; + +#define FTL_new_query(flags, name, blockingreason, addr, types, id, type) _FTL_new_query(flags, name, blockingreason, addr, types, id, type, __FILE__, __LINE__) +bool _FTL_new_query(const unsigned int flags, const char *name, const char** blockingreason, const union all_addr *addr, const char *types, const int id, const char type, const char* file, const int line); + +#define FTL_forwarded(flags, name, addr, id) _FTL_forwarded(flags, name, addr, id, __FILE__, __LINE__) +void _FTL_forwarded(const unsigned int flags, const char *name, const union all_addr *addr, const int id, const char* file, const int line); + +#define FTL_reply(flags, name, addr, id) _FTL_reply(flags, name, addr, id, __FILE__, __LINE__) +void _FTL_reply(const unsigned short flags, const char *name, const union all_addr *addr, const int id, const char* file, const int line); + +#define FTL_cache(flags, name, addr, arg, id) _FTL_cache(flags, name, addr, arg, id, __FILE__, __LINE__) +void _FTL_cache(const unsigned int flags, const char *name, const union all_addr *addr, const char * arg, const int id, const char* file, const int line); + +#define FTL_dnssec(status, id) _FTL_dnssec(status, id, __FILE__, __LINE__) +void _FTL_dnssec(const int status, const int id, const char* file, const int line); + +#define FTL_header_analysis(header4, rcode, id) _FTL_header_analysis(header4, rcode, id, __FILE__, __LINE__) +void _FTL_header_analysis(const unsigned char header4, const unsigned int rcode, const int id, const char* file, const int line); + +#define FTL_forwarding_failed(server) _FTL_forwarding_failed(server, __FILE__, __LINE__) +void _FTL_forwarding_failed(const struct server *server, const char* file, const int line); + +#define FTL_upstream_error(rcode, id) _FTL_upstream_error(rcode, id, __FILE__, __LINE__) +void _FTL_upstream_error(const unsigned int rcode, const int id, const char* file, const int line); + +#define FTL_get_blocking_metadata(addrp, flags) _FTL_get_blocking_metadata(addrp, flags, __FILE__, __LINE__) +void _FTL_get_blocking_metadata(union all_addr **addrp, unsigned int *flags, const char* file, const int line); + +#define FTL_check_blocking(queryID, domainID, clientID, blockingreason) _FTL_check_blocking(queryID, domainID, clientID, blockingreason, __FILE__, __LINE__) + +#define FTL_CNAME(domain, cpp, id) _FTL_CNAME(domain, cpp, id, __FILE__, __LINE__) +bool _FTL_CNAME(const char *domain, const struct crec *cpp, const int id, const char* file, const int line); + +void FTL_dnsmasq_reload(void); +void FTL_fork_and_bind_sockets(struct passwd *ent_pw); +void FTL_TCP_worker_terminating(void); + +void set_debug_dnsmasq_lines(char enabled); +extern char debug_dnsmasq_lines; + +#endif // DNSMASQ_INTERFACE_H diff --git a/src/files.c b/src/files.c new file mode 100644 index 000000000..1d3ff78da --- /dev/null +++ b/src/files.c @@ -0,0 +1,147 @@ +/* Pi-hole: A black hole for Internet advertisements +* (c) 2017 Pi-hole, LLC (https://pi-hole.net) +* Network-wide ad blocking via your own hardware. +* +* FTL Engine +* File operation routines +* +* This file is copyright under the latest version of the EUPL. +* Please see LICENSE file for your rights under this license. */ + +#include "FTL.h" +#include "files.h" +#include "memory.h" +#include "config.h" +#include "setupVars.h" +#include "log.h" + +// opendir(), readdir() +#include +// getpwuid() +#include +// getgrgid() +#include + +// chmod_file() changes the file mode bits of a given file (relative +// to the directory file descriptor) according to mode. mode is an +// octal number representing the bit pattern for the new mode bits +bool chmod_file(const char *filename, const mode_t mode) +{ + if(chmod(filename, mode) < 0) + { + logg("WARNING: chmod(%s, %d): chmod() failed: %s (%d)", filename, mode, strerror(errno), errno); + return false; + } + + struct stat st; + if(stat(filename, &st) < 0) + { + logg("WARNING: chmod(%s, %d): stat() failed: %s (%d)", filename, mode, strerror(errno), errno); + return false; + } + + // We need to apply a bitmask on st.st_mode as the upper bits may contain random data + // 0x1FF = 0b111_111_111 corresponding to the three-digit octal mode number + if((st.st_mode & 0x1FF) != mode) + { + logg("WARNING: chmod(%s, %d): Verification failed, %d != %d", filename, mode, st.st_mode, mode); + return false; + } + + return true; +} + +bool file_exists(const char *filename) +{ + struct stat st; + return stat(filename, &st) == 0; +} + +unsigned long long get_FTL_db_filesize(void) +{ + struct stat st; + if(stat(FTLfiles.FTL_db, &st) != 0) + { + // stat() failed (maybe the DB file does not exist?) + return 0; + } + return st.st_size; +} + +void ls_dir(const char* path) +{ + // Open directory stream + DIR* dirp = opendir(path); + if(dirp == NULL) + { + logg("opendir(\"%s\") failed with %s (%d)", path, strerror(errno), errno); + return; + } + + // Stack space for full path (directory + "/" + filename + terminating \0) + char full_path[strlen(path)+NAME_MAX+2]; + + logg("------ Listing content of directory %s ------", path); + logg("File Mode User:Group Filesize Filename"); + + struct dirent *dircontent = NULL; + // Walk directory file by file + while((dircontent = readdir(dirp)) != NULL) + { + // Get filename + const char *filename = dircontent->d_name; + + // Construct full path + snprintf(full_path, sizeof(full_path), "%s/%s", path, filename); + + struct stat st; + // Use stat to get file size, permissions, and ownership + if(stat(full_path, &st) < 0) + { + logg("%s failed with %s (%d)", filename, strerror(errno), errno); + continue; + } + + // Get owner's name + struct passwd *pwd; + char user[256]; + if ((pwd = getpwuid(st.st_uid)) != NULL) + snprintf(user, sizeof(user), "%s", pwd->pw_name); + else + snprintf(user, sizeof(user), "%d", st.st_uid); + + struct group *grp; + char group[256]; + // Get out group name + if ((grp = getgrgid(st.st_gid)) != NULL) + snprintf(group, sizeof(group), "%s", grp->gr_name); + else + snprintf(group, sizeof(group), "%d", st.st_gid); + + char permissions[10]; + // Get human-readable format of permissions as known from ls + snprintf(permissions, sizeof(permissions), + "%s%s%s%s%s%s%s%s%s", + st.st_mode & S_IRUSR ? "r":"-", + st.st_mode & S_IWUSR ? "w":"-", + st.st_mode & S_IXUSR ? "x":"-", + st.st_mode & S_IRGRP ? "r":"-", + st.st_mode & S_IWGRP ? "w":"-", + st.st_mode & S_IXGRP ? "x":"-", + st.st_mode & S_IROTH ? "r":"-", + st.st_mode & S_IWOTH ? "w":"-", + st.st_mode & S_IXOTH ? "x":"-"); + + char prefix[2] = " "; + double formated = 0.0; + format_memory_size(prefix, (unsigned long long)st.st_size, &formated); + + // Log output for this file + logg("%s %s:%s %.0f%s %s", permissions, user, group, formated, prefix, filename); + } + + logg("---------------------------------------------------"); + + // Close directory stream + closedir(dirp); +} diff --git a/src/files.h b/src/files.h new file mode 100644 index 000000000..4df7b745d --- /dev/null +++ b/src/files.h @@ -0,0 +1,18 @@ +/* Pi-hole: A black hole for Internet advertisements +* (c) 2019 Pi-hole, LLC (https://pi-hole.net) +* Network-wide ad blocking via your own hardware. +* +* FTL Engine +* File prototypes +* +* This file is copyright under the latest version of the EUPL. +* Please see LICENSE file for your rights under this license. */ +#ifndef FILE_H +#define FILE_H + +bool chmod_file(const char *filename, const mode_t mode); +bool file_exists(const char *filename); +unsigned long long get_FTL_db_filesize(void); +void ls_dir(const char* path); + +#endif //FILE_H diff --git a/gc.c b/src/gc.c similarity index 56% rename from gc.c rename to src/gc.c index 79dabbee1..d8435ade5 100644 --- a/gc.c +++ b/src/gc.c @@ -9,7 +9,19 @@ * Please see LICENSE file for your rights under this license. */ #include "FTL.h" +#include "gc.h" #include "shmem.h" +#include "timers.h" +#include "config.h" +#include "overTime.h" +#include "database/common.h" +#include "log.h" +// global variable counters +#include "memory.h" +// global variable killed +#include "signals.h" +// data getter functions +#include "datastructure.h" bool doGC = false; @@ -42,38 +54,47 @@ void *GC_thread(void *val) mintime -= mintime % 3600; mintime += 3600; - if(config.debug & DEBUG_GC) timer_start(GC_TIMER); - - long int i; - int removed = 0; - if(config.debug & DEBUG_GC) logg("GC starting, mintime: %lu %s", mintime, ctime(&mintime)); + if(config.debug & DEBUG_GC) + { + timer_start(GC_TIMER); + char timestring[84] = ""; + get_timestr(timestring, mintime); + logg("GC starting, mintime: %s (%lu)", timestring, mintime); + } // Process all queries - for(i=0; i < counters->queries; i++) + int removed = 0; + for(long int i=0; i < counters->queries; i++) { - validate_access("queries", i, true, __LINE__, __FUNCTION__, __FILE__); + queriesData* query = getQuery(i, true); + if(query == NULL) + continue; + // Test if this query is too new - if(queries[i].timestamp > mintime) + if(query->timestamp > mintime) break; // Adjust client counter - int clientID = queries[i].clientID; - validate_access("clients", clientID, true, __LINE__, __FUNCTION__, __FILE__); - clients[clientID].count--; + clientsData* client = getClient(query->clientID, true); + if(client != NULL) + client->count--; // Adjust total counters and total over time data - int timeidx = queries[i].timeidx; + const int timeidx = query->timeidx; overTime[timeidx].total--; - // Adjust corresponding overTime counters - clients[clientID].overTime[timeidx]--; + if(client != NULL) + client->overTime[timeidx]--; // Adjust domain counter (no overTime information) - int domainID = queries[i].domainID; - validate_access("domains", domainID, true, __LINE__, __FUNCTION__, __FILE__); - domains[domainID].count--; + domainsData* domain = getDomain(query->domainID, true); + if(domain != NULL) + domain->count--; + + // Get upstream pointer + upstreamsData* upstream = getUpstream(query->upstreamID, true); // Change other counters according to status of this query - switch(queries[i].status) + switch(query->status) { case QUERY_UNKNOWN: // Unknown (?) @@ -81,9 +102,10 @@ void *GC_thread(void *val) break; case QUERY_FORWARDED: // Forwarded to an upstream DNS server - counters->forwardedqueries--; - validate_access("forwarded", queries[i].forwardID, true, __LINE__, __FUNCTION__, __FILE__); - forwarded[queries[i].forwardID].count--; + // Adjust counters + counters->forwarded--; + if(upstream != NULL) + upstream->count--; overTime[timeidx].forwarded--; break; case QUERY_CACHE: @@ -93,14 +115,19 @@ void *GC_thread(void *val) break; case QUERY_GRAVITY: // Blocked by Pi-hole's blocking lists (fall through) case QUERY_BLACKLIST: // Exact blocked (fall through) - case QUERY_WILDCARD: // Regex blocked (fall through) + case QUERY_REGEX: // Regex blocked (fall through) case QUERY_EXTERNAL_BLOCKED_IP: // Blocked by upstream provider (fall through) case QUERY_EXTERNAL_BLOCKED_NXRA: // Blocked by upstream provider (fall through) case QUERY_EXTERNAL_BLOCKED_NULL: // Blocked by upstream provider (fall through) + case QUERY_GRAVITY_CNAME: // Gravity domain in CNAME chain (fall through) + case QUERY_BLACKLIST_CNAME: // Exactly blacklisted domain in CNAME chain (fall through) + case QUERY_REGEX_CNAME: // Regex blacklisted domain in CNAME chain (fall through) counters->blocked--; overTime[timeidx].blocked--; - domains[domainID].blockedcount--; - clients[clientID].blockedcount--; + if(domain != NULL) + domain->blockedcount--; + if(client != NULL) + client->blockedcount--; break; default: /* That cannot happen */ @@ -108,7 +135,7 @@ void *GC_thread(void *val) } // Update reply counters - switch(queries[i].reply) + switch(query->reply) { case REPLY_NODATA: // NODATA(-IPv6) counters->reply_NODATA--; @@ -135,10 +162,10 @@ void *GC_thread(void *val) } // Update type counters - if(queries[i].type >= TYPE_A && queries[i].type < TYPE_MAX) + if(query->type >= TYPE_A && query->type < TYPE_MAX) { - counters->querytype[queries[i].type-1]--; - overTime[timeidx].querytypedata[queries[i].type-1]--; + counters->querytype[query->type-1]--; + overTime[timeidx].querytypedata[query->type-1]--; } // Count removed queries @@ -146,25 +173,30 @@ void *GC_thread(void *val) } - // Move memory forward to keep only what we want - // Note: for overlapping memory blocks, memmove() is a safer approach than memcpy() - // Example: (I = now invalid, X = still valid queries, F = free space) - // Before: IIIIIIXXXXFF - // After: XXXXFFFFFFFF - memmove(&queries[0], &queries[removed], (counters->queries - removed)*sizeof(*queries)); - - // Update queries counter - counters->queries -= removed; - // Update DB index as total number of queries reduced - lastdbindex -= removed; - - // Zero out remaining memory (marked as "F" in the above example) - memset(&queries[counters->queries], 0, (counters->queries_MAX - counters->queries)*sizeof(*queries)); + // Only perform memory operations when we actually removed queries + if(removed > 0) + { + // Move memory forward to keep only what we want + // Note: for overlapping memory blocks, memmove() is a safer approach than memcpy() + // Example: (I = now invalid, X = still valid queries, F = free space) + // Before: IIIIIIXXXXFF + // After: XXXXFFFFFFFF + memmove(getQuery(0, true), getQuery(removed, true), (counters->queries - removed)*sizeof(queriesData)); + + // Update queries counter + counters->queries -= removed; + // Update DB index as total number of queries reduced + lastdbindex -= removed; + + // ensure remaining memory is zeroed out (marked as "F" in the above example) + memset(getQuery(counters->queries, true), 0, (counters->queries_MAX - counters->queries)*sizeof(queriesData)); + } // Determine if overTime memory needs to get moved moveOverTimeMemory(mintime); - if(config.debug & DEBUG_GC) logg("Notice: GC removed %i queries (took %.2f ms)", removed, timer_elapsed_msec(GC_TIMER)); + if(config.debug & DEBUG_GC) + logg("Notice: GC removed %i queries (took %.2f ms)", removed, timer_elapsed_msec(GC_TIMER)); // Release thread lock unlock_shm(); diff --git a/src/gc.h b/src/gc.h new file mode 100644 index 000000000..e8cbbab5a --- /dev/null +++ b/src/gc.h @@ -0,0 +1,15 @@ +/* Pi-hole: A black hole for Internet advertisements +* (c) 2019 Pi-hole, LLC (https://pi-hole.net) +* Network-wide ad blocking via your own hardware. +* +* FTL Engine +* Garbage collection prototypes +* +* This file is copyright under the latest version of the EUPL. +* Please see LICENSE file for your rights under this license. */ +#ifndef GC_H +#define GC_H + +void *GC_thread(void *val); + +#endif //GC_H diff --git a/log.c b/src/log.c similarity index 68% rename from log.c rename to src/log.c index fe71c4173..93a8771de 100644 --- a/log.c +++ b/src/log.c @@ -10,9 +10,19 @@ #include "FTL.h" #include "version.h" - -pthread_mutex_t lock; -FILE *logfile = NULL; +#include "memory.h" +#include "daemon.h" +#include "config.h" +#include "log.h" +// global variable username +#include "main.h" +// global variable daemonmode +#include "args.h" +// global counters variable +#include "shmem.h" + +static pthread_mutex_t lock; +static FILE *logfile = NULL; static void close_FTL_log(void) { @@ -20,7 +30,7 @@ static void close_FTL_log(void) fclose(logfile); } -void open_FTL_log(bool test) +void open_FTL_log(const bool test) { if(test) { @@ -50,13 +60,14 @@ void open_FTL_log(bool test) } } -static void get_timestr(char *timestring) +void get_timestr(char *timestring, const time_t timein) { - time_t t = time(NULL); - struct tm tm = *localtime(&t); + struct tm tm; + localtime_r(&timein, &tm); + struct timeval tv; gettimeofday(&tv, NULL); - int millisec = tv.tv_usec/1000; + const int millisec = tv.tv_usec/1000; sprintf(timestring,"%d-%02d-%02d %02d:%02d:%02d.%03i", tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec, millisec); } @@ -68,11 +79,11 @@ void __attribute__ ((format (gnu_printf, 1, 2))) logg(const char *format, ...) pthread_mutex_lock(&lock); - get_timestr(timestring); + get_timestr(timestring, time(NULL)); // Get and log PID of current process to avoid ambiguities when more than one // pihole-FTL instance is logging into the same file - long pid = (long)getpid(); + const long pid = (long)getpid(); // Print to stdout before writing to file if(!daemonmode) @@ -108,7 +119,7 @@ void __attribute__ ((format (gnu_printf, 1, 2))) logg(const char *format, ...) pthread_mutex_unlock(&lock); } -void format_memory_size(char *prefix, unsigned long int bytes, double *formated) +void format_memory_size(char *prefix, const unsigned long long int bytes, double *formated) { int i; *formated = bytes; @@ -124,31 +135,48 @@ void format_memory_size(char *prefix, unsigned long int bytes, double *formated) strcpy(prefix, prefixes[i]); } -void logg_struct_resize(const char* str, int to, int step) -{ - logg("Notice: Increasing %s struct size from %i to %i", str, (to-step), to); -} - void log_counter_info(void) { logg(" -> Total DNS queries: %i", counters->queries); logg(" -> Cached DNS queries: %i", counters->cached); - logg(" -> Forwarded DNS queries: %i", counters->forwardedqueries); - logg(" -> Exactly blocked DNS queries: %i", counters->blocked); + logg(" -> Forwarded DNS queries: %i", counters->forwarded); + logg(" -> Blocked DNS queries: %i", counters->blocked); logg(" -> Unknown DNS queries: %i", counters->unknown); logg(" -> Unique domains: %i", counters->domains); logg(" -> Unique clients: %i", counters->clients); - logg(" -> Known forward destinations: %i", counters->forwarded); + logg(" -> Known forward destinations: %i", counters->upstreams); } -void log_FTL_version(bool crashreport) +void log_FTL_version(const bool crashreport) { logg("FTL branch: %s", GIT_BRANCH); - logg("FTL version: %s", GIT_TAG); + logg("FTL version: %s", get_FTL_version()); logg("FTL commit: %s", GIT_HASH); logg("FTL date: %s", GIT_DATE); if(crashreport) logg("FTL user: started as %s, ended as %s", username, getUserName()); else logg("FTL user: %s", username); + logg("Compiled for %s using %s", FTL_ARCH, FTL_CC); +} + +static char *FTLversion = NULL; +const char __attribute__ ((malloc)) *get_FTL_version(void) +{ + // Obtain FTL version if not already determined + if(FTLversion == NULL) + { + if(strlen(GIT_TAG) > 1) + { + FTLversion = strdup(GIT_VERSION); + } + else + { + FTLversion = calloc(13, sizeof(char)); + // Build version by appending 7 characters of the hash to "vDev-" + snprintf(FTLversion, 13, "vDev-%.7s", GIT_HASH); + } + } + + return FTLversion; } diff --git a/src/log.h b/src/log.h new file mode 100644 index 000000000..fa31574ac --- /dev/null +++ b/src/log.h @@ -0,0 +1,24 @@ +/* Pi-hole: A black hole for Internet advertisements +* (c) 2017 Pi-hole, LLC (https://pi-hole.net) +* Network-wide ad blocking via your own hardware. +* +* FTL Engine +* Logging prototypes +* +* This file is copyright under the latest version of the EUPL. +* Please see LICENSE file for your rights under this license. */ +#ifndef LOG_H +#define LOG_H + +#include +#include + +void open_FTL_log(const bool test); +void logg(const char* format, ...) __attribute__ ((format (gnu_printf, 1, 2))); +void log_counter_info(void); +void format_memory_size(char *prefix, unsigned long long int bytes, double *formated); +const char *get_FTL_version(void) __attribute__ ((malloc)); +void log_FTL_version(bool crashreport); +void get_timestr(char *timestring, const time_t timein); + +#endif //LOG_H diff --git a/main.c b/src/main.c similarity index 69% rename from main.c rename to src/main.c index 7ef8f6c7d..72b69be2d 100644 --- a/main.c +++ b/src/main.c @@ -9,10 +9,26 @@ * Please see LICENSE file for your rights under this license. */ #include "FTL.h" +#include "daemon.h" +#include "log.h" +#include "api/socket.h" +#include "setupVars.h" +#include "args.h" +#include "config.h" +#include "database/common.h" +#include "database/query-table.h" +#include "main.h" +#include "signals.h" +#include "regex_r.h" +#include "shmem.h" +#include "capabilities.h" +#include "database/gravity-db.h" +#include "timers.h" char * username; bool needGC = false; bool needDBGC = false; +bool startup = true; int main (int argc, char* argv[]) { @@ -52,13 +68,12 @@ int main (int argc, char* argv[]) if(strcmp(username, "pihole") != 0) logg("WARNING: Starting pihole-FTL as user %s is not recommended", username); - // Initialize database - if(config.maxDBdays != 0) - db_init(); + // Initialize query database (pihole-FTL.db) + db_init(); // Try to import queries from long-term database if available if(database && config.DBimport) - read_data_from_DB(); + DB_read_queries(); log_counter_info(); check_setupVarsconf(); @@ -67,7 +82,14 @@ int main (int argc, char* argv[]) // immediately before starting the resolver. check_capabilities(); - // Start the resolver + // Start the resolver, delay startup if requested + delay_startup(); + startup = false; + if(config.debug != 0) + { + for(int i = 0; i < argc_dnsmasq; i++) + logg("DEBUG: argv[%i] = \"%s\"", i, argv_dnsmasq[i]); + } main_dnsmasq(argc_dnsmasq, argv_dnsmasq); logg("Shutting down..."); @@ -80,7 +102,7 @@ int main (int argc, char* argv[]) // Save new queries to database if(database) { - save_to_DB(); + DB_save_queries(); logg("Finished final database update"); } @@ -88,14 +110,17 @@ int main (int argc, char* argv[]) close_telnet_socket(); close_unix_socket(); - // Invalidate blocking regex if compiled - free_regex(); + // Close gravity database connection + gravityDB_close(); // Remove shared memory objects + // Important: This invalidated all objects such as + // counters-> ... Do this last when + // terminating in main.c ! destroy_shmem(); //Remove PID file removepid(); - logg("########## FTL terminated after %.1f ms! ##########", timer_elapsed_msec(EXIT_TIMER)); + logg("########## FTL terminated after %e s! ##########", 1e-3*timer_elapsed_msec(EXIT_TIMER)); return EXIT_SUCCESS; } diff --git a/src/main.h b/src/main.h new file mode 100644 index 000000000..53b95e3f5 --- /dev/null +++ b/src/main.h @@ -0,0 +1,18 @@ +/* Pi-hole: A black hole for Internet advertisements +* (c) 2019 Pi-hole, LLC (https://pi-hole.net) +* Network-wide ad blocking via your own hardware. +* +* FTL Engine +* Main prototypes +* +* This file is copyright under the latest version of the EUPL. +* Please see LICENSE file for your rights under this license. */ +#ifndef MAIN_H +#define MAIN_H + +int main_dnsmasq(int argc, const char ** argv); + +extern char * username; +extern bool startup; + +#endif //MAIN_H diff --git a/src/memory.c b/src/memory.c new file mode 100644 index 000000000..81dae6be3 --- /dev/null +++ b/src/memory.c @@ -0,0 +1,95 @@ +/* Pi-hole: A black hole for Internet advertisements +* (c) 2017 Pi-hole, LLC (https://pi-hole.net) +* Network-wide ad blocking via your own hardware. +* +* FTL Engine +* Global variable definitions and memory reallocation handling +* +* This file is copyright under the latest version of the EUPL. +* Please see LICENSE file for your rights under this license. */ + +#include "FTL.h" +#include "shmem.h" +#include "memory.h" +#include "log.h" + +// The special memory handling routines have to be the last ones in this source file +// as we restore the original definition of the strdup, free, calloc, and realloc +// functions in here, i.e. if anything extra would come below these lines, it would +// not be protected by our (error logging) functions! + +#undef strdup +char* __attribute__((malloc)) FTLstrdup(const char *src, const char * file, const char * function, const int line) +{ + // The FTLstrdup() function returns a pointer to a new string which is a + // duplicate of the string s. Memory for the new string is obtained with + // calloc(3), and can be freed with free(3). + if(src == NULL) + { + logg("WARN: Trying to copy a NULL string in %s() (%s:%i)", function, file, line); + return NULL; + } + const size_t len = strlen(src); + char *dest = calloc(len+1, sizeof(char)); + if(dest == NULL) + { + logg("FATAL: Memory allocation failed in %s() (%s:%i)", function, file, line); + return NULL; + } + // Use memcpy as memory areas cannot overlap + memcpy(dest, src, len); + dest[len] = '\0'; + + return dest; +} + +#undef calloc +void* __attribute__((malloc)) __attribute__((alloc_size(1,2))) FTLcalloc(const size_t nmemb, const size_t size, const char * file, const char * function, const int line) +{ + // The FTLcalloc() function allocates memory for an array of nmemb elements + // of size bytes each and returns a pointer to the allocated memory. The + // memory is set to zero. If nmemb or size is 0, then calloc() returns + // either NULL, or a unique pointer value that can later be successfully + // passed to free(). + void *ptr = calloc(nmemb, size); + if(ptr == NULL) + logg("FATAL: Memory allocation (%zu x %zu) failed in %s() (%s:%i)", + nmemb, size, function, file, line); + + return ptr; +} + +#undef realloc +void __attribute__((alloc_size(2))) *FTLrealloc(void *ptr_in, const size_t size, const char * file, const char * function, const int line) +{ + // The FTLrealloc() function changes the size of the memory block pointed to + // by ptr to size bytes. The contents will be unchanged in the range from + // the start of the region up to the minimum of the old and new sizes. If + // the new size is larger than the old size, the added memory will not be + // initialized. If ptr is NULL, then the call is equivalent to malloc(size), + // for all values of size; if size is equal to zero, and ptr is + // not NULL, then the call is equivalent to free(ptr). Unless ptr is + // NULL, it must have been returned by an earlier call to malloc(), cal‐ + // loc() or realloc(). If the area pointed to was moved, a free(ptr) is + // done. + void *ptr_out = realloc(ptr_in, size); + if(ptr_out == NULL) + logg("FATAL: Memory reallocation (%p -> %zu) failed in %s() (%s:%i)", + ptr_in, size, function, file, line); + + return ptr_out; +} + +#undef free +void FTLfree(void *ptr, const char * file, const char * function, const int line) +{ + // The free() function frees the memory space pointed to by ptr, which + // must have been returned by a previous call to malloc(), calloc(), or + // realloc(). Otherwise, or if free(ptr) has already been called before, + // undefined behavior occurs. If ptr is NULL, no operation is performed. + if(ptr == NULL) + logg("FATAL: Trying to free NULL pointer in %s() (%s:%i)", function, file, line); + + // We intentionally run free() nevertheless to see the crash in the debugger + free(ptr); +} diff --git a/src/memory.h b/src/memory.h new file mode 100644 index 000000000..b5648f8b3 --- /dev/null +++ b/src/memory.h @@ -0,0 +1,19 @@ +/* Pi-hole: A black hole for Internet advertisements +* (c) 2019 Pi-hole, LLC (https://pi-hole.net) +* Network-wide ad blocking via your own hardware. +* +* FTL Engine +* Memory prototypes +* +* This file is copyright under the latest version of the EUPL. +* Please see LICENSE file for your rights under this license. */ +#ifndef MEMORY_H +#define MEMORY_H + +void memory_check(const int which); +char *FTLstrdup(const char *src, const char *file, const char *function, const int line) __attribute__((malloc)); +void *FTLcalloc(size_t nmemb, size_t size, const char *file, const char *function, const int line) __attribute__((malloc)) __attribute__((alloc_size(1,2))); +void *FTLrealloc(void *ptr_in, size_t size, const char *file, const char *function, const int line) __attribute__((alloc_size(2))); +void FTLfree(void *ptr, const char* file, const char *function, const int line); + +#endif //MEMORY_H diff --git a/overTime.c b/src/overTime.c similarity index 70% rename from overTime.c rename to src/overTime.c index 207d0964d..191d1d231 100644 --- a/overTime.c +++ b/src/overTime.c @@ -9,6 +9,16 @@ * Please see LICENSE file for your rights under this license. */ #include "FTL.h" +#include "overTime.h" +#include "shmem.h" +#include "config.h" +#include "log.h" +// global variable counters +#include "memory.h" +// data getter functions +#include "datastructure.h" + +overTimeData *overTime = NULL; /** * Initialize the overTime slot @@ -16,12 +26,15 @@ * @param index The overTime slot index * @param timestamp The timestamp of the slot */ -static void initSlot(unsigned int index, time_t timestamp) +static void initSlot(const unsigned int index, const time_t timestamp) { // Possible debug printing if(config.debug & DEBUG_OVERTIME) + { logg("initSlot(%u, %lu): Zeroing overTime slot", index, timestamp); + } + // Initialize overTime entry overTime[index].magic = MAGICBYTE; overTime[index].timestamp = timestamp; overTime[index].total = 0; @@ -31,11 +44,21 @@ static void initSlot(unsigned int index, time_t timestamp) // Zero all query types for(unsigned int queryType = 0; queryType < TYPE_MAX-1; queryType++) + { overTime[index].querytypedata[queryType] = 0; + } // Zero overTime counter for all known clients for(int clientID = 0; clientID < counters->clients; clientID++) - clients[clientID].overTime[index] = 0; + { + // Get client pointer + clientsData* client = getClient(clientID, true); + if(client != NULL) + { + // Set overTime data to zero + client->overTime[index] = 0; + } + } } void initOverTime(void) @@ -50,11 +73,11 @@ void initOverTime(void) if(config.debug & DEBUG_OVERTIME) logg("initOverTime(): Initializing %i slots from %lu to %lu", OVERTIME_SLOTS, timestamp-OVERTIME_SLOTS*OVERTIME_INTERVAL, timestamp); - // Iterate over overTime and initialize it + // Iterate over overTime for(int i = OVERTIME_SLOTS-1; i >= 0 ; i--) { + // Initialize onerTime slot initSlot(i, timestamp); - // Prepare for next iteration timestamp -= OVERTIME_INTERVAL; } @@ -67,10 +90,10 @@ unsigned int getOverTimeID(time_t timestamp) timestamp += OVERTIME_INTERVAL/2; // Get timestamp of first interval - time_t firstTimestamp = overTime[0].timestamp; + const time_t firstTimestamp = overTime[0].timestamp; // Compute overTime ID - int id = (int) ((timestamp - firstTimestamp) / OVERTIME_INTERVAL); + const int id = (int) ((timestamp - firstTimestamp) / OVERTIME_INTERVAL); // Check bounds manually if(id < 0) @@ -87,15 +110,18 @@ unsigned int getOverTimeID(time_t timestamp) } if(config.debug & DEBUG_OVERTIME) + { + // Debug output logg("getOverTimeID(%lu): %i", timestamp, id); + } return (unsigned int) id; } // This routine is called by garbage collection to rearrange the overTime structure for the next hour -void moveOverTimeMemory(time_t mintime) +void moveOverTimeMemory(const time_t mintime) { - time_t oldestOverTimeIS = overTime[0].timestamp; + const time_t oldestOverTimeIS = overTime[0].timestamp; // Shift SHOULD timestemp into the future by the amount GC is running earlier time_t oldestOverTimeSHOULD = mintime; @@ -105,13 +131,16 @@ void moveOverTimeMemory(time_t mintime) // Calculate the number of slots to be garbage collected, which is also the // ID of the slot to move to the zero position - unsigned int moveOverTime = (unsigned int) ((oldestOverTimeSHOULD - oldestOverTimeIS) / OVERTIME_INTERVAL); + const unsigned int moveOverTime = (unsigned int) ((oldestOverTimeSHOULD - oldestOverTimeIS) / OVERTIME_INTERVAL); // The number of slots which will be moved (not garbage collected) - unsigned int remainingSlots = OVERTIME_SLOTS - moveOverTime; + const unsigned int remainingSlots = OVERTIME_SLOTS - moveOverTime; if(config.debug & DEBUG_OVERTIME) - logg("moveOverTimeMemory(): IS: %lu, SHOULD: %lu, MOVING: %u", oldestOverTimeIS, oldestOverTimeSHOULD, moveOverTime); + { + logg("moveOverTimeMemory(): IS: %lu, SHOULD: %lu, MOVING: %u", + oldestOverTimeIS, oldestOverTimeSHOULD, moveOverTime); + } // Check if the move over amount is valid. This prevents errors if the // function is called before GC is necessary. @@ -119,36 +148,51 @@ void moveOverTimeMemory(time_t mintime) { // Move overTime memory if(config.debug & DEBUG_OVERTIME) - logg("moveOverTimeMemory(): Moving overTime %u - %u to 0 - %u", moveOverTime, moveOverTime+remainingSlots, remainingSlots); - memmove(&overTime[0], &overTime[moveOverTime], remainingSlots*sizeof(*overTime)); + { + logg("moveOverTimeMemory(): Moving overTime %u - %u to 0 - %u", + moveOverTime, moveOverTime+remainingSlots, remainingSlots); + } + + // Move overTime memory forward to update data structure + memmove(&overTime[0], + &overTime[moveOverTime], + remainingSlots*sizeof(*overTime)); // Correct time indices of queries. This is necessary because we just moved the slot this index points to for(int queryID = 0; queryID < counters->queries; queryID++) { + // Get query pointer + queriesData* query = getQuery(queryID, true); + if(query == NULL) + continue; + // Check if the index would become negative if we adjusted it - if(((int)queries[queryID].timeidx - (int)moveOverTime) < 0) + if(((int)query->timeidx - (int)moveOverTime) < 0) { // This should never happen, but we print a warning if it still happens // We don't do anything in this case - logg("WARN: moveOverTimeMemory(): overTime time index correction failed (%i: %u / %u)", queryID, queries[queryID].timeidx, moveOverTime); + logg("WARN: moveOverTimeMemory(): overTime time index correction failed (%i: %u / %u)", + queryID, query->timeidx, moveOverTime); } else { - queries[queryID].timeidx -= moveOverTime; + query->timeidx -= moveOverTime; } } // Move client-specific overTime memory for(int clientID = 0; clientID < counters->clients; clientID++) { - memmove(&clients[clientID].overTime[0], &clients[clientID].overTime[moveOverTime], remainingSlots*sizeof(int)); + memmove(&(getClient(clientID, true)->overTime[0]), + &(getClient(clientID, true)->overTime[moveOverTime]), + remainingSlots*sizeof(int)); } // Iterate over new overTime region and initialize it for(unsigned int timeidx = remainingSlots; timeidx < OVERTIME_SLOTS ; timeidx++) { // This slot is OVERTIME_INTERVAL seconds after the previous slot - time_t timestamp = overTime[timeidx-1].timestamp + OVERTIME_INTERVAL; + const time_t timestamp = overTime[timeidx-1].timestamp + OVERTIME_INTERVAL; initSlot(timeidx, timestamp); } } diff --git a/src/overTime.h b/src/overTime.h new file mode 100644 index 000000000..d94ff4333 --- /dev/null +++ b/src/overTime.h @@ -0,0 +1,37 @@ +/* Pi-hole: A black hole for Internet advertisements +* (c) 2018 Pi-hole, LLC (https://pi-hole.net) +* Network-wide ad blocking via your own hardware. +* +* FTL Engine +* Over Time data header +* +* This file is copyright under the latest version of the EUPL. +* Please see LICENSE file for your rights under this license. */ + +#ifndef OVERTIME_H +#define OVERTIME_H + +void initOverTime(void); +unsigned int getOverTimeID(const time_t timestamp); + +/** + * Move the overTime slots so the oldest interval starts with mintime. The time + * given will be aligned to OVERTIME_INTERVAL. + * + * @param mintime The start of the oldest interval + */ +void moveOverTimeMemory(const time_t mintime); + +typedef struct { + unsigned char magic; + time_t timestamp; + int total; + int blocked; + int cached; + int forwarded; + int querytypedata[TYPE_MAX-1]; +} overTimeData; + +extern overTimeData *overTime; + +#endif //OVERTIME_H diff --git a/src/regex.c b/src/regex.c new file mode 100644 index 000000000..da94f2ac0 --- /dev/null +++ b/src/regex.c @@ -0,0 +1,304 @@ +/* Pi-hole: A black hole for Internet advertisements +* (c) 2017 Pi-hole, LLC (https://pi-hole.net) +* Network-wide ad blocking via your own hardware. +* +* FTL Engine +* Regular Expressions +* +* This file is copyright under the latest version of the EUPL. +* Please see LICENSE file for your rights under this license. */ + +#include "FTL.h" +#include "regex_r.h" +#include "timers.h" +#include "memory.h" +#include "log.h" +#include "config.h" +// data getter functions +#include "datastructure.h" +#include +#include "database/gravity-db.h" +// bool startup +#include "main.h" +// add_per_client_regex_client() +#include "shmem.h" + +static regex_t *regex[2] = { NULL }; +static bool *regex_available[2] = { NULL }; +static int *regex_id[2] = { NULL }; +static char **regexbuffer[2] = { NULL }; + +const char *regextype[] = { "blacklist", "whitelist" }; + +// Log Regex failure (most likely a regex syntax error, we include a hint to the error) +static void log_regex_error(const int errcode, const int index, const unsigned char regexid, const char *regexin) +{ + // Get error string and log it + const size_t length = regerror(errcode, ®ex[regexid][index], NULL, 0); + char *buffer = calloc(length,sizeof(char)); + (void) regerror (errcode, ®ex[regexid][index], buffer, length); + logg("Warning: Invalid regex %s filter \"%s\": %s (error code %i)", regextype[regexid], regexin, buffer, errcode); + free(buffer); +} + +/* Compile regular expressions into data structures that can be used with + regexec() to match against a string */ +static bool compile_regex(const char *regexin, const int index, const unsigned char regexid) +{ + // We use the extended RegEx flavor (ERE) and specify that matching should + // always be case INsensitive + const int errcode = regcomp(®ex[regexid][index], regexin, REG_EXTENDED | REG_ICASE); + if(errcode != 0) + { + log_regex_error(errcode, index, regexid, regexin); + return false; + } + + // Store compiled regex string in buffer if in regex debug mode + if(config.debug & DEBUG_REGEX) + { + regexbuffer[regexid][index] = strdup(regexin); + } + + return true; +} + +int match_regex(const char *input, const int clientID, const unsigned char regexid) +{ + int match_idx = -1; + + // Start matching timer + timer_start(REGEX_TIMER); + for(int index = 0; index < counters->num_regex[regexid]; index++) + { + // Only check regex which have been successfully compiled ... + if(!regex_available[regexid][index]) + { + if(config.debug & DEBUG_REGEX) + logg("Regex %s (DB ID %d) \"%s\" is NOT AVAILABLE", + regextype[regexid], regex_id[regexid][index], + regexbuffer[regexid][index]); + + continue; + } + // ... and are enabled for this client + int regexID = index; + if(regexid == REGEX_WHITELIST) + regexID += counters->num_regex[REGEX_BLACKLIST]; + + if(!get_per_client_regex(clientID, regexID)) + { + if(config.debug & DEBUG_REGEX) + { + clientsData* client = getClient(clientID, true); + logg("Regex %s (DB ID %d) \"%s\" NOT ENABLED for client %s", + regextype[regexid], regex_id[regexid][index], + regexbuffer[regexid][index], getstr(client->ippos)); + } + + continue; + } + + // Try to match the compiled regular expression against input + int errcode = regexec(®ex[regexid][index], input, 0, NULL, 0); + // regexec() returns zero for a successful match or REG_NOMATCH for failure. + // We are only interested in the matching case here. + if (errcode == 0) + { + // Match, return true + match_idx = regex_id[regexid][index]; + + // Print match message when in regex debug mode + if(config.debug & DEBUG_REGEX) + { + logg("Regex %s (DB ID %i) >> MATCH: \"%s\" vs. \"%s\"", + regextype[regexid], regex_id[regexid][index], + input, regexbuffer[regexid][index]); + } + break; + } + + // Print no match message when in regex debug mode + if(config.debug & DEBUG_REGEX && match_idx > -1) + { + logg("Regex %s (DB ID %i) NO match: \"%s\" vs. \"%s\"", + regextype[regexid], regex_id[regexid][index], + input, regexbuffer[regexid][index]); + } + } + + double elapsed = timer_elapsed_msec(REGEX_TIMER); + + // Only log evaluation times if they are longer than normal + if(elapsed > 10.0) + logg("WARN: Regex %s evaluation took %.3f msec", regextype[regexid], elapsed); + + // No match, no error, return false + return match_idx; +} + +static void free_regex(void) +{ + // Reset FTL's DNS cache + FTL_reset_per_client_domain_data(); + + // Return early if we don't use any regex filters + if(regex[REGEX_WHITELIST] == NULL && + regex[REGEX_BLACKLIST] == NULL) + return; + + // Reset client configuration + for(int clientID = 0; clientID < counters->clients; clientID++) + { + reset_per_client_regex(clientID); + } + + // Free regex datastructure + for(int regexid = 0; regexid < 2; regexid++) + { + for(int index = 0; index < counters->num_regex[regexid]; index++) + { + if(!regex_available[regexid][index]) + continue; + + regfree(®ex[regexid][index]); + + // Also free buffered regex strings if in regex debug mode + if(config.debug & DEBUG_REGEX && regexbuffer[regexid][index] != NULL) + { + free(regexbuffer[regexid][index]); + regexbuffer[regexid][index] = NULL; + } + } + + // Free array with regex datastructure + if(regex[regexid] != NULL) + { + free(regex[regexid]); + regex[regexid] = NULL; + } + + // Reset counter for number of regex + counters->num_regex[regexid] = 0; + } +} + +void allocate_regex_client_enabled(clientsData *client, const int clientID) +{ + add_per_client_regex(clientID); + + // Only initialize regex associations when dnsmasq is ready (otherwise, we're still in history reading mode) + if(!startup) + { + gravityDB_get_regex_client_groups(client, counters->num_regex[REGEX_BLACKLIST], + regex_id[REGEX_BLACKLIST], REGEX_BLACKLIST, + "vw_regex_blacklist", clientID); + gravityDB_get_regex_client_groups(client, counters->num_regex[REGEX_WHITELIST], + regex_id[REGEX_WHITELIST], REGEX_WHITELIST, + "vw_regex_whitelist", clientID); + } +} + +static void read_regex_table(const unsigned char regexid) +{ + // Get table ID + unsigned char tableID = (regexid == REGEX_BLACKLIST) ? REGEX_BLACKLIST_TABLE : REGEX_WHITELIST_TABLE; + + // Get number of lines in the regex table + counters->num_regex[regexid] = gravityDB_count(tableID); + + if(counters->num_regex[regexid] == 0) + { + logg("INFO: No regex %s entries found", regextype[regexid]); + return; + } + else if(counters->num_regex[regexid] == DB_FAILED) + { + logg("WARN: Database query failed, assuming there are no %s regex entries", regextype[regexid]); + counters->num_regex[regexid] = 0; + return; + } + + // Allocate memory for regex + regex[regexid] = calloc(counters->num_regex[regexid], sizeof(regex_t)); + regex_id[regexid] = calloc(counters->num_regex[regexid], sizeof(int)); + regex_available[regexid] = calloc(counters->num_regex[regexid], sizeof(bool)); + + // Buffer strings if in regex debug mode + if(config.debug & DEBUG_REGEX) + regexbuffer[regexid] = calloc(counters->num_regex[regexid], sizeof(char*)); + + // Connect to regex table + if(!gravityDB_getTable(tableID)) + { + logg("read_regex_from_database(): Error getting %s regex table from database", regextype[regexid]); + return; + } + + // Walk database table + const char *domain = NULL; + int i = 0, rowid = 0; + while((domain = gravityDB_getDomain(&rowid)) != NULL) + { + // Avoid buffer overflow if database table changed + // since we counted its entries + if(i >= counters->num_regex[regexid]) + break; + + // Skip this entry if empty: an empty regex filter would match + // anything anywhere and hence match all incoming domains. A user + // can still achieve this with a filter such as ".*", however empty + // filters in the regex table are probably not expected to have such + // an effect and would immediately lead to "blocking or whitelisting + // the entire Internet" + if(strlen(domain) < 1) + continue; + + // Compile this regex + if(config.debug & DEBUG_REGEX) + { + logg("Compiling %s regex %i (database ID %i): %s", regextype[regexid], i, rowid, domain); + } + regex_available[regexid][i] = compile_regex(domain, i, regexid); + regex_id[regexid][i] = rowid; + + // Increase counter + i++; + } + + // Finalize statement and close gravity database handle + gravityDB_finalizeTable(); +} + +void read_regex_from_database(void) +{ + // Free regex filters + // This routine is safe to be called even when there + // are no regex filters at the moment + free_regex(); + + // Start timer for regex compilation analysis + timer_start(REGEX_TIMER); + + // Read and compile regex blacklist + read_regex_table(REGEX_BLACKLIST); + + // Read and compile regex whitelist + read_regex_table(REGEX_WHITELIST); + + + for(int clientID = 0; clientID < counters->clients; clientID++) + { + // Get client pointer + clientsData *client = getClient(clientID, true); + if(client == NULL) + continue; + + allocate_regex_client_enabled(client, clientID); + } + + // Print message to FTL's log after reloading regex filters + logg("Compiled %i whitelist and %i blacklist regex filters in %.1f msec", + counters->num_regex[REGEX_WHITELIST], counters->num_regex[REGEX_BLACKLIST], + timer_elapsed_msec(REGEX_TIMER)); +} diff --git a/src/regex_r.h b/src/regex_r.h new file mode 100644 index 000000000..aca18f979 --- /dev/null +++ b/src/regex_r.h @@ -0,0 +1,27 @@ +/* Pi-hole: A black hole for Internet advertisements +* (c) 2019 Pi-hole, LLC (https://pi-hole.net) +* Network-wide ad blocking via your own hardware. +* +* FTL Engine +* Regex prototypes +* +* This file is copyright under the latest version of the EUPL. +* Please see LICENSE file for your rights under this license. */ +#ifndef REGEX_H +#define REGEX_H + +// clientsData type +#include "datastructure.h" + +extern const char *regextype[]; + +int match_regex(const char *input, const int clientID, const unsigned char regexid); +void allocate_regex_client_enabled(clientsData *client, const int clientID); +void read_regex_from_database(void); + +// Blocking status constants used by the domain->clientstatus vector +// We explicitly force UNKNOWN_BLOCKED to zero on all platforms as this is the +// default value set initially with calloc +enum { UNKNOWN_BLOCKED = 0, GRAVITY_BLOCKED, BLACKLIST_BLOCKED, REGEX_BLOCKED, WHITELISTED, NOT_BLOCKED }; + +#endif //REGEX_H diff --git a/src/resolve.c b/src/resolve.c new file mode 100644 index 000000000..b02ae411c --- /dev/null +++ b/src/resolve.c @@ -0,0 +1,434 @@ +/* Pi-hole: A black hole for Internet advertisements +* (c) 2017 Pi-hole, LLC (https://pi-hole.net) +* Network-wide ad blocking via your own hardware. +* +* FTL Engine +* DNS Client Implementation +* +* This file is copyright under the latest version of the EUPL. +* Please see LICENSE file for your rights under this license. */ + +#include "FTL.h" +#include "shmem.h" +#include "memory.h" +#include "datastructure.h" +#include "resolve.h" +#include "config.h" +#include "timers.h" +#include "log.h" +// global variable killed +#include "signals.h" +// getDatabaseHostname() +#include "database/network-table.h" +// struct _res +#include + +static bool res_initialized = false; + +// Validate given hostname +static bool valid_hostname(char* name, const char* clientip) +{ + // Check for validity of input + if(name == NULL) + return false; + + // Check for maximum length of hostname + // Truncate if too long (MAXHOSTNAMELEN defaults to 64, see asm-generic/param.h) + if(strlen(name) > MAXHOSTNAMELEN) + { + logg("WARNING: Hostname of client %s too long, truncating to %d chars!", + clientip, MAXHOSTNAMELEN); + // We can modify the string in-place as the target is + // shorter than the source + name[MAXHOSTNAMELEN] = '\0'; + } + + // Iterate over characters in hostname + // to check for legal char: A-Z a-z 0-9 - _ . + for (char c; (c = *name); name++) + { + if ((c >= 'A' && c <= 'Z') || + (c >= 'a' && c <= 'z') || + (c >= '0' && c <= '9') || + c == '-' || + c == '_' || + c == '.' ) + continue; + + // Invalid character found, log and return hostname being invalid + logg("WARN: Hostname of client %s contains invalid character: %c (char code %d)", + clientip, (unsigned char)c, (unsigned char)c); + return false; + } + + // No invalid characters found + return true; +} + +static void print_used_resolvers(const char *message) +{ + logg("%s", message); + for(unsigned int i = 0u; i < MAXNS; i++) + logg(" %u: %s:%d", i, + inet_ntoa(_res.nsaddr_list[i].sin_addr), + ntohs(_res.nsaddr_list[i].sin_port)); +} + +static char *resolveHostname(const char *addr) +{ + // Get host name + struct hostent *he = NULL; + char *hostname = NULL; + bool IPv6 = false; + + if(config.debug & DEBUG_RESOLVER) + logg("Trying to resolve %s", addr); + + // Check if this is a hidden client + // if so, return "hidden" as hostname + if(strcmp(addr, "0.0.0.0") == 0) + { + hostname = strdup("hidden"); + if(config.debug & DEBUG_RESOLVER) + logg("---> \"%s\" (privacy settings)", hostname); + return hostname; + } + + // Test if we want to resolve an IPv6 address + if(strstr(addr,":") != NULL) + { + IPv6 = true; + } + + if( (IPv6 && !config.resolveIPv6) || + (!IPv6 && !config.resolveIPv4)) + { + if(config.debug & DEBUG_RESOLVER) + { + logg(" ---> \"\" (configured to not resolve %s host names)", + IPv6 ? "IPv6" : "IPv4"); + return strdup(""); + } + } + + // Initialize resolver subroutines if trying to resolve for the first time + // res_init() reads resolv.conf to get the default domain name and name server + // address(es). If no server is given, the local host is tried. If no domain + // is given, that associated with the local host is used. + if(!res_initialized) + { + res_init(); + res_initialized = true; + } + + // Step 1: Backup configured name servers and invalidate them + struct in_addr ns_addr_bck[MAXNS]; + in_port_t ns_port_bck[MAXNS]; + for(unsigned int i = 0u; i < MAXNS; i++) + { + ns_addr_bck[i] = _res.nsaddr_list[i].sin_addr; + ns_port_bck[i] = _res.nsaddr_list[i].sin_port; + _res.nsaddr_list[i].sin_addr.s_addr = 0; // 0.0.0.0 + } + // Step 2: Set 127.0.0.1 (FTL) as the only resolver + const char *FTLip = "127.0.0.1"; + // Set resolver address + inet_pton(AF_INET, FTLip, &_res.nsaddr_list[0].sin_addr); + // Set resolver port (have to convert from host to network byte order) + _res.nsaddr_list[0].sin_port = htons(config.dns_port); + + if(config.debug & DEBUG_RESOLVER) + print_used_resolvers("Setting nameservers to:"); + + // Step 3: Try to resolve addresses + if(IPv6) // Resolve IPv6 address + { + struct in6_addr ipaddr; + inet_pton(AF_INET6, addr, &ipaddr); + // Known to leak some tiny amounts of memory under certain conditions + he = gethostbyaddr(&ipaddr, sizeof ipaddr, AF_INET6); + } + else // Resolve IPv4 address + { + struct in_addr ipaddr; + inet_pton(AF_INET, addr, &ipaddr); + // Known to leak some tiny amounts of memory under certain conditions + he = gethostbyaddr(&ipaddr, sizeof ipaddr, AF_INET); + } + + // Step 4: Check if gethostbyaddr() returned a host name + // First check for he not being NULL before trying to dereference it + if(he != NULL) + { + if(valid_hostname(he->h_name, addr)) + { + // Return hostname copied to new memory location + hostname = strdup(he->h_name); + + // Convert hostname to lower case + if(hostname != NULL) + strtolower(hostname); + } + else + { + hostname = strdup("[invalid host name]"); + } + + if(config.debug & DEBUG_RESOLVER) + logg(" ---> \"%s\" (found internally)", hostname); + } + + // Step 5: Restore resolvers (without forced FTL) + for(unsigned int i = 0u; i < MAXNS; i++) + { + _res.nsaddr_list[i].sin_addr = ns_addr_bck[i]; + _res.nsaddr_list[i].sin_port = ns_port_bck[i]; + } + if(config.debug & DEBUG_RESOLVER) + print_used_resolvers("Setting nameservers back to default:"); + + // Step 6: If no host name was found before, try again with system-configured + // resolvers (necessary for docker and friends) + if(hostname == NULL) + { + if(IPv6) // Resolve IPv6 address + { + struct in6_addr ipaddr; + inet_pton(AF_INET6, addr, &ipaddr); + // Known to leak some tiny amounts of memory under certain conditions + he = gethostbyaddr(&ipaddr, sizeof ipaddr, AF_INET6); + } + else // Resolve IPv4 address + { + struct in_addr ipaddr; + inet_pton(AF_INET, addr, &ipaddr); + // Known to leak some tiny amounts of memory under certain conditions + he = gethostbyaddr(&ipaddr, sizeof ipaddr, AF_INET); + } + + // Step 6.1: Check if gethostbyaddr() returned a host name this time + // First check for he not being NULL before trying to dereference it + if(he != NULL) + { + if(valid_hostname(he->h_name, addr)) + { + // Return hostname copied to new memory location + hostname = strdup(he->h_name); + + // Convert hostname to lower case + if(hostname != NULL) + strtolower(hostname); + } + else + { + hostname = strdup("[invalid host name]"); + } + + if(config.debug & DEBUG_RESOLVER) + logg(" ---> \"%s\" (found externally)", hostname); + } + else + { + // No (he == NULL) or invalid (valid_hostname returned false) hostname found + hostname = strdup(""); + + if(config.debug & DEBUG_RESOLVER) + logg(" ---> \"%s\" (%s)", hostname, he != NULL ? he->h_name : "N/A"); + } + } + + // Return result + return hostname; +} + +// Resolve upstream destination host names +static size_t resolveAndAddHostname(size_t ippos, size_t oldnamepos) +{ + // Get IP and host name strings. They are cloned in case shared memory is + // resized before the next lock + lock_shm(); + char* ipaddr = strdup(getstr(ippos)); + char* oldname = strdup(getstr(oldnamepos)); + unlock_shm(); + + // Important: Don't hold a lock while resolving as the main thread + // (dnsmasq) needs to be operable during the call to resolveHostname() + char* newname = resolveHostname(ipaddr); + + // If no hostname was found, try to obtain hostname from the network table + if(strlen(newname) == 0) + { + free(newname); + newname = getDatabaseHostname(ipaddr); + } + + // Only store new newname if it is valid and differs from oldname + // We do not need to check for oldname == NULL as names are + // always initialized with an empty string at position 0 + if(newname != NULL && strcmp(oldname, newname) != 0) + { + lock_shm(); + size_t newnamepos = addstr(newname); + // newname has already been checked against NULL + // so we can safely free it + free(newname); + free(ipaddr); + free(oldname); + unlock_shm(); + return newnamepos; + } + else if(config.debug & DEBUG_SHMEM) + { + // Debugging output + logg("Not adding \"%s\" to buffer (unchanged)", oldname); + } + + free(newname); + free(ipaddr); + free(oldname); + + // Not changed, return old namepos + return oldnamepos; +} + +// Resolve client host names +void resolveClients(const bool onlynew) +{ + // Lock counter access here, we use a copy in the following loop + lock_shm(); + int clientscount = counters->clients; + unlock_shm(); + + int skipped = 0; + for(int clientID = 0; clientID < clientscount; clientID++) + { + // Get client pointer + clientsData* client = getClient(clientID, true); + if(client == NULL) + { + logg("ERROR: Unable to get client pointer with ID %i, skipping...", clientID); + skipped++; + continue; + } + + // Memory access needs to get locked + lock_shm(); + bool newflag = client->new; + size_t ippos = client->ippos; + size_t oldnamepos = client->namepos; + unlock_shm(); + + // If onlynew flag is set, we will only resolve new clients + // If not, we will try to re-resolve all known clients + if(onlynew && !newflag) + { + skipped++; + continue; + } + + // Obtain/update hostname of this client + size_t newnamepos = resolveAndAddHostname(ippos, oldnamepos); + + lock_shm(); + // Store obtained host name (may be unchanged) + client->namepos = newnamepos; + // Mark entry as not new + client->new = false; + unlock_shm(); + } + + if(config.debug & DEBUG_RESOLVER) + { + logg("%i / %i client host names resolved", + clientscount-skipped, clientscount); + } +} + +// Resolve upstream destination host names +void resolveForwardDestinations(const bool onlynew) +{ + // Lock counter access here, we use a copy in the following loop + lock_shm(); + int upstreams = counters->upstreams; + unlock_shm(); + + int skipped = 0; + for(int upstreamID = 0; upstreamID < upstreams; upstreamID++) + { + // Get upstream pointer + upstreamsData* upstream = getUpstream(upstreamID, true); + if(upstream == NULL) + { + logg("ERROR: Unable to get upstream pointer with ID %i, skipping...", upstreamID); + skipped++; + continue; + } + + // Memory access needs to get locked + lock_shm(); + bool newflag = upstream->new; + size_t ippos = upstream->ippos; + size_t oldnamepos = upstream->namepos; + unlock_shm(); + + // If onlynew flag is set, we will only resolve new upstream destinations + // If not, we will try to re-resolve all known upstream destinations + if(onlynew && !newflag) + { + skipped++; + continue; + } + + // Obtain/update hostname of this client + size_t newnamepos = resolveAndAddHostname(ippos, oldnamepos); + + lock_shm(); + // Store obtained host name (may be unchanged) + upstream->namepos = newnamepos; + // Mark entry as not new + upstream->new = false; + unlock_shm(); + } + + if(config.debug & DEBUG_RESOLVER) + { + logg("%i / %i upstream server host names resolved", + upstreams-skipped, upstreams); + } +} + +void *DNSclient_thread(void *val) +{ + // Set thread name + prctl(PR_SET_NAME, "DNS client", 0, 0, 0); + + while(!killed) + { + // Run every minute to resolve only new clients and upstream servers + if(time(NULL) % RESOLVE_INTERVAL == 0) + { + // Try to resolve new client host names (onlynew=true) + resolveClients(true); + // Try to resolve new upstream destination host names (onlynew=true) + resolveForwardDestinations(true); + // Prevent immediate re-run of this routine + sleepms(500); + } + + // Run every hour to update possibly changed client host names + if(time(NULL) % RERESOLVE_INTERVAL == 0) + { + // Try to resolve all client host names (onlynew=false) + resolveClients(false); + // Try to resolve all upstream destination host names (onlynew=false) + resolveForwardDestinations(false); + // Prevent immediate re-run of this routine + sleepms(500); + } + + // Idle for 0.5 sec before checking again the time criteria + sleepms(500); + } + + return NULL; +} diff --git a/src/resolve.h b/src/resolve.h new file mode 100644 index 000000000..b0e01afd0 --- /dev/null +++ b/src/resolve.h @@ -0,0 +1,24 @@ +/* Pi-hole: A black hole for Internet advertisements +* (c) 2019 Pi-hole, LLC (https://pi-hole.net) +* Network-wide ad blocking via your own hardware. +* +* FTL Engine +* Domain name resolution prototypes +* +* This file is copyright under the latest version of the EUPL. +* Please see LICENSE file for your rights under this license. */ +#ifndef RESOLVE_H +#define RESOLVE_H + +void *DNSclient_thread(void *val); +void resolveClients(const bool onlynew); +void resolveForwardDestinations(const bool onlynew); + +// musl does not define MAXHOSTNAMELEN +// If it is not defined, we set the value +// found on a x86_64 glibc instance +#ifndef MAXHOSTNAMELEN +#define MAXHOSTNAMELEN 64 +#endif + +#endif //RESOLVE_H diff --git a/setupVars.c b/src/setupVars.c similarity index 87% rename from setupVars.c rename to src/setupVars.c index 0cc925343..80924ecd3 100644 --- a/setupVars.c +++ b/src/setupVars.c @@ -9,7 +9,10 @@ * Please see LICENSE file for your rights under this license. */ #include "FTL.h" - +#include "log.h" +#include "memory.h" +#include "config.h" +#include "setupVars.h" int setupVarsElements = 0; char ** setupVarsArray = NULL; @@ -17,7 +20,7 @@ char ** setupVarsArray = NULL; void check_setupVarsconf(void) { FILE *setupVarsfp; - if((setupVarsfp = fopen(files.setupVars, "r")) == NULL) + if((setupVarsfp = fopen(FTLfiles.setupVars, "r")) == NULL) { logg("WARN: Opening of setupVars.conf failed!"); logg(" Make sure it exists and is readable"); @@ -70,7 +73,7 @@ size_t linebuffersize = 0; char * read_setupVarsconf(const char * key) { FILE *setupVarsfp; - if((setupVarsfp = fopen(files.setupVars, "r")) == NULL) + if((setupVarsfp = fopen(FTLfiles.setupVars, "r")) == NULL) { logg("WARN: Reading setupVars.conf failed: %s", strerror(errno)); return NULL; @@ -227,3 +230,29 @@ bool __attribute__((pure)) getSetupVarsBool(const char * input) else return false; } + +// Global variable showing current blocking status +unsigned char blockingstatus = BLOCKING_UNKNOWN; + +void check_blocking_status(void) +{ + const char* blocking = read_setupVarsconf("BLOCKING_ENABLED"); + const char* message; + + if(blocking == NULL || getSetupVarsBool(blocking)) + { + // Parameter either not present in setupVars.conf + // or explicitly set to true + blockingstatus = BLOCKING_ENABLED; + message = "enabled"; + clearSetupVarsArray(); + } + else + { + // Disabled + blockingstatus = BLOCKING_DISABLED; + message = "disabled"; + } + + logg("Blocking status is %s", message); +} diff --git a/src/setupVars.h b/src/setupVars.h new file mode 100644 index 000000000..af3279f69 --- /dev/null +++ b/src/setupVars.h @@ -0,0 +1,27 @@ +/* Pi-hole: A black hole for Internet advertisements +* (c) 2019 Pi-hole, LLC (https://pi-hole.net) +* Network-wide ad blocking via your own hardware. +* +* FTL Engine +* pihole-FTL.conf processing prototypes +* +* This file is copyright under the latest version of the EUPL. +* Please see LICENSE file for your rights under this license. */ +#ifndef SETUPVARS_H +#define SETUPVARS_H + +void check_setupVarsconf(void); +char * read_setupVarsconf(const char * key); +void getSetupVarsArray(const char * input); +void clearSetupVarsArray(void); +bool insetupVarsArray(const char * str); +bool getSetupVarsBool(const char * input) __attribute__((pure)); +char* find_equals(const char* s) __attribute__((pure)); +void trim_whitespace(char *string); +void check_blocking_status(void); + +extern unsigned char blockingstatus; + +enum { BLOCKING_DISABLED, BLOCKING_ENABLED, BLOCKING_UNKNOWN }; + +#endif //SETUPVARS_H diff --git a/shmem.c b/src/shmem.c similarity index 52% rename from shmem.c rename to src/shmem.c index dd868bad6..39f8e86d9 100644 --- a/shmem.c +++ b/src/shmem.c @@ -10,9 +10,15 @@ #include "FTL.h" #include "shmem.h" +#include "overTime.h" +#include "log.h" +#include "memory.h" +#include "config.h" +// data getter functions +#include "datastructure.h" /// The version of shared memory used -#define SHARED_MEMORY_VERSION 6 +#define SHARED_MEMORY_VERSION 9 /// The name of the shared memory. Use this when connecting to the shared memory. #define SHARED_LOCK_NAME "/FTL-lock" @@ -21,9 +27,14 @@ #define SHARED_DOMAINS_NAME "/FTL-domains" #define SHARED_CLIENTS_NAME "/FTL-clients" #define SHARED_QUERIES_NAME "/FTL-queries" -#define SHARED_FORWARDED_NAME "/FTL-forwarded" +#define SHARED_UPSTREAMS_NAME "/FTL-upstreams" #define SHARED_OVERTIME_NAME "/FTL-overTime" #define SHARED_SETTINGS_NAME "/FTL-settings" +#define SHARED_DNS_CACHE "/FTL-dns-cache" +#define SHARED_PER_CLIENT_REGEX "/FTL-per-client-regex" + +// Global counters struct +countersStruct *counters = NULL; /// The pointer in shared memory to the shared string buffer static SharedMemory shm_lock = { 0 }; @@ -32,9 +43,18 @@ static SharedMemory shm_counters = { 0 }; static SharedMemory shm_domains = { 0 }; static SharedMemory shm_clients = { 0 }; static SharedMemory shm_queries = { 0 }; -static SharedMemory shm_forwarded = { 0 }; +static SharedMemory shm_upstreams = { 0 }; static SharedMemory shm_overTime = { 0 }; static SharedMemory shm_settings = { 0 }; +static SharedMemory shm_dns_cache = { 0 }; +static SharedMemory shm_per_client_regex = { 0 }; + +// Variable size array structs +static queriesData *queries = NULL; +static clientsData *clients = NULL; +static domainsData *domains = NULL; +static upstreamsData *upstreams = NULL; +static DNSCacheData *dns_cache = NULL; typedef struct { pthread_mutex_t lock; @@ -46,7 +66,47 @@ static ShmSettings *shmSettings = NULL; static int pagesize; static unsigned int local_shm_counter = 0; -static size_t get_optimal_object_size(size_t objsize, size_t minsize); +static size_t get_optimal_object_size(const size_t objsize, const size_t minsize); + +// chown_shmem() changes the file ownership of a given shared memory object +static bool chown_shmem(SharedMemory *sharedMemory, struct passwd *ent_pw) +{ + // Open shared memory object + const int fd = shm_open(sharedMemory->name, O_RDWR, S_IRUSR | S_IWUSR); + if(fd == -1) + { + logg("FATAL: chown_shmem(): Failed to open shared memory object \"%s\": %s", + sharedMemory->name, strerror(errno)); + exit(EXIT_FAILURE); + } + if(fchown(fd, ent_pw->pw_uid, ent_pw->pw_gid) == -1) + { + logg("WARNING: chown_shmem(%d, %d, %d): failed for %s: %s (%d)", + fd, ent_pw->pw_uid, ent_pw->pw_gid, sharedMemory->name, + strerror(errno), errno); + return false; + } + logg("Changing %s (%d) to %d:%d", sharedMemory->name, fd, ent_pw->pw_uid, ent_pw->pw_gid); + // Close shared memory object file descriptor as it is no longer + // needed after having called ftruncate() + close(fd); + return true; +} + +void chown_all_shmem(struct passwd *ent_pw) +{ + chown_shmem(&shm_lock, ent_pw); + chown_shmem(&shm_strings, ent_pw); + chown_shmem(&shm_counters, ent_pw); + chown_shmem(&shm_domains, ent_pw); + chown_shmem(&shm_clients, ent_pw); + chown_shmem(&shm_queries, ent_pw); + chown_shmem(&shm_upstreams, ent_pw); + chown_shmem(&shm_overTime, ent_pw); + chown_shmem(&shm_settings, ent_pw); + chown_shmem(&shm_dns_cache, ent_pw); + chown_shmem(&shm_per_client_regex, ent_pw); +} size_t addstr(const char *str) { @@ -56,22 +116,28 @@ size_t addstr(const char *str) return 0; } - // Get string length - size_t len = strlen(str); + // Get string length, add terminating character + size_t len = strlen(str) + 1; - // If this is an empty string, use the one at position zero - if(len == 0) { + // If this is an empty string (only the terminating character is present), + // use the shared memory string at position zero instead of creating a new + // entry here. We also ensure that the given string is not too long to + // prevent possible memory corruption caused by strncpy() further down + if(len == 1) { return 0; } + else if(len > (size_t)(pagesize-1)) + { + logg("WARN: Shortening too long string (len %zu)", len); + len = pagesize; + } // Debugging output if(config.debug & DEBUG_SHMEM) logg("Adding \"%s\" (len %zu) to buffer. next_str_pos is %u", str, len, shmSettings->next_str_pos); // Reserve additional memory if necessary - size_t required_size = shmSettings->next_str_pos + len + 1; - // Need to cast to long long because size_t calculations cannot be negative - if((long long)required_size-(long long)shm_strings.size > 0 && + if(shmSettings->next_str_pos + len > shm_strings.size && !realloc_shm(&shm_strings, shm_strings.size + pagesize, true)) return 0; @@ -81,16 +147,15 @@ size_t addstr(const char *str) // Copy the C string pointed by str into the shared string buffer strncpy(&((char*)shm_strings.ptr)[shmSettings->next_str_pos], str, len); - ((char*)shm_strings.ptr)[shmSettings->next_str_pos + len] = '\0'; // Increment string length counter - shmSettings->next_str_pos += len+1; + shmSettings->next_str_pos += len; // Return start of stored string - return (shmSettings->next_str_pos - (len + 1)); + return (shmSettings->next_str_pos - len); } -const char *getstr(size_t pos) +const char *getstr(const size_t pos) { // Only access the string memory if this memory region has already been set if(pos < shmSettings->next_str_pos) @@ -128,14 +193,21 @@ static pthread_mutex_t create_mutex(void) { static void remap_shm(void) { // Remap shared object pointers which might have changed - realloc_shm(&shm_queries, counters->queries_MAX*sizeof(queriesDataStruct), false); - queries = (queriesDataStruct*)shm_queries.ptr; - realloc_shm(&shm_domains, counters->domains_MAX*sizeof(domainsDataStruct), false); - domains = (domainsDataStruct*)shm_domains.ptr; - realloc_shm(&shm_clients, counters->clients_MAX*sizeof(clientsDataStruct), false); - clients = (clientsDataStruct*)shm_clients.ptr; - realloc_shm(&shm_forwarded, counters->forwarded_MAX*sizeof(forwardedDataStruct), false); - forwarded = (forwardedDataStruct*)shm_forwarded.ptr; + realloc_shm(&shm_queries, counters->queries_MAX*sizeof(queriesData), false); + queries = (queriesData*)shm_queries.ptr; + + realloc_shm(&shm_domains, counters->domains_MAX*sizeof(domainsData), false); + domains = (domainsData*)shm_domains.ptr; + + realloc_shm(&shm_clients, counters->clients_MAX*sizeof(clientsData), false); + clients = (clientsData*)shm_clients.ptr; + + realloc_shm(&shm_upstreams, counters->upstreams_MAX*sizeof(upstreamsData), false); + upstreams = (upstreamsData*)shm_upstreams.ptr; + + realloc_shm(&shm_dns_cache, counters->dns_cache_MAX*sizeof(DNSCacheData), false); + dns_cache = (DNSCacheData*)shm_dns_cache.ptr; + realloc_shm(&shm_strings, counters->strings_MAX, false); // strings are not exposed by a global pointer @@ -143,17 +215,17 @@ static void remap_shm(void) local_shm_counter = shmSettings->global_shm_counter; } -void _lock_shm(const char* function, const int line, const char * file) { +void _lock_shm(const char* func, const int line, const char * file) { // Signal that FTL is waiting for a lock shmLock->waitingForLock = true; if(config.debug & DEBUG_LOCKS) - logg("Waiting for lock in %s() (%s:%i)", function, file, line); + logg("Waiting for lock in %s() (%s:%i)", func, file, line); int result = pthread_mutex_lock(&shmLock->lock); if(config.debug & DEBUG_LOCKS) - logg("Obtained lock for %s() (%s:%i)", function, file, line); + logg("Obtained lock for %s() (%s:%i)", func, file, line); // Check if this process needs to remap the shared memory objects if(shmSettings != NULL && @@ -179,11 +251,11 @@ void _lock_shm(const char* function, const int line, const char * file) { logg("Failed to obtain SHM lock: %s", strerror(result)); } -void _unlock_shm(const char* function, const int line, const char * file) { +void _unlock_shm(const char* func, const int line, const char * file) { int result = pthread_mutex_unlock(&shmLock->lock); if(config.debug & DEBUG_LOCKS) - logg("Removed lock in %s() (%s:%i)", function, file, line); + logg("Removed lock in %s() (%s:%i)", func, file, line); if(result != 0) logg("Failed to unlock SHM lock: %s", strerror(result)); @@ -224,37 +296,49 @@ bool init_shmem(void) /****************************** shared domains struct ******************************/ // Try to create shared memory object - shm_domains = create_shm(SHARED_DOMAINS_NAME, pagesize*sizeof(domainsDataStruct)); - domains = (domainsDataStruct*)shm_domains.ptr; + shm_domains = create_shm(SHARED_DOMAINS_NAME, pagesize*sizeof(domainsData)); + domains = (domainsData*)shm_domains.ptr; counters->domains_MAX = pagesize; /****************************** shared clients struct ******************************/ - size_t size = get_optimal_object_size(sizeof(clientsDataStruct), 1); + size_t size = get_optimal_object_size(sizeof(clientsData), 1); // Try to create shared memory object - shm_clients = create_shm(SHARED_CLIENTS_NAME, size*sizeof(clientsDataStruct)); - clients = (clientsDataStruct*)shm_clients.ptr; + shm_clients = create_shm(SHARED_CLIENTS_NAME, size*sizeof(clientsData)); + clients = (clientsData*)shm_clients.ptr; counters->clients_MAX = size; - /****************************** shared forwarded struct ******************************/ - size = get_optimal_object_size(sizeof(forwardedDataStruct), 1); + /****************************** shared upstreams struct ******************************/ + size = get_optimal_object_size(sizeof(upstreamsData), 1); // Try to create shared memory object - shm_forwarded = create_shm(SHARED_FORWARDED_NAME, size*sizeof(forwardedDataStruct)); - forwarded = (forwardedDataStruct*)shm_forwarded.ptr; - counters->forwarded_MAX = size; + shm_upstreams = create_shm(SHARED_UPSTREAMS_NAME, size*sizeof(upstreamsData)); + upstreams = (upstreamsData*)shm_upstreams.ptr; + counters->upstreams_MAX = size; /****************************** shared queries struct ******************************/ // Try to create shared memory object - shm_queries = create_shm(SHARED_QUERIES_NAME, pagesize*sizeof(queriesDataStruct)); - queries = (queriesDataStruct*)shm_queries.ptr; + shm_queries = create_shm(SHARED_QUERIES_NAME, pagesize*sizeof(queriesData)); + queries = (queriesData*)shm_queries.ptr; counters->queries_MAX = pagesize; /****************************** shared overTime struct ******************************/ - size = get_optimal_object_size(sizeof(overTimeDataStruct), OVERTIME_SLOTS); + size = get_optimal_object_size(sizeof(overTimeData), OVERTIME_SLOTS); // Try to create shared memory object - shm_overTime = create_shm(SHARED_OVERTIME_NAME, size*sizeof(overTimeDataStruct)); - overTime = (overTimeDataStruct*)shm_overTime.ptr; + shm_overTime = create_shm(SHARED_OVERTIME_NAME, size*sizeof(overTimeData)); + overTime = (overTimeData*)shm_overTime.ptr; initOverTime(); + /****************************** shared DNS cache struct ******************************/ + size = get_optimal_object_size(sizeof(DNSCacheData), 1); + // Try to create shared memory object + shm_dns_cache = create_shm(SHARED_DNS_CACHE, size*sizeof(DNSCacheData)); + dns_cache = (DNSCacheData*)shm_dns_cache.ptr; + counters->dns_cache_MAX = size; + + /****************************** shared per-client regex buffer ******************************/ + size = get_optimal_object_size(1, 2); + // Try to create shared memory object + shm_per_client_regex = create_shm(SHARED_PER_CLIENT_REGEX, size); + return true; } @@ -269,12 +353,14 @@ void destroy_shmem(void) delete_shm(&shm_domains); delete_shm(&shm_clients); delete_shm(&shm_queries); - delete_shm(&shm_forwarded); + delete_shm(&shm_upstreams); delete_shm(&shm_overTime); delete_shm(&shm_settings); + delete_shm(&shm_dns_cache); + delete_shm(&shm_per_client_regex); } -SharedMemory create_shm(const char *name, size_t size) +SharedMemory create_shm(const char *name, const size_t size) { if(config.debug & DEBUG_SHMEM) logg("Creating shared memory with name \"%s\" and size %zu", name, size); @@ -307,10 +393,10 @@ SharedMemory create_shm(const char *name, size_t size) } // Resize shared memory file - int result = ftruncate(fd, size); + ret = ftruncate(fd, size); // Check for `ftruncate` error - if(result == -1) + if(ret == -1) { logg("FATAL: create_shm(): ftruncate(%i, %zu): Failed to resize shared memory object \"%s\": %s", fd, size, sharedMemory.name, strerror(errno)); @@ -336,7 +422,7 @@ SharedMemory create_shm(const char *name, size_t size) return sharedMemory; } -void *enlarge_shmem_struct(char type) +void *enlarge_shmem_struct(const char type) { SharedMemory *sharedMemory = NULL; size_t sizeofobj, allocation_step; @@ -348,29 +434,35 @@ void *enlarge_shmem_struct(char type) case QUERIES: sharedMemory = &shm_queries; allocation_step = pagesize; - sizeofobj = sizeof(queriesDataStruct); + sizeofobj = sizeof(queriesData); counter = &counters->queries_MAX; break; case CLIENTS: sharedMemory = &shm_clients; - allocation_step = get_optimal_object_size(sizeof(clientsDataStruct), 1); - sizeofobj = sizeof(clientsDataStruct); + allocation_step = get_optimal_object_size(sizeof(clientsData), 1); + sizeofobj = sizeof(clientsData); counter = &counters->clients_MAX; break; case DOMAINS: sharedMemory = &shm_domains; allocation_step = pagesize; - sizeofobj = sizeof(domainsDataStruct); + sizeofobj = sizeof(domainsData); counter = &counters->domains_MAX; break; - case FORWARDED: - sharedMemory = &shm_forwarded; - allocation_step = get_optimal_object_size(sizeof(forwardedDataStruct), 1); - sizeofobj = sizeof(forwardedDataStruct); - counter = &counters->forwarded_MAX; + case UPSTREAMS: + sharedMemory = &shm_upstreams; + allocation_step = get_optimal_object_size(sizeof(upstreamsData), 1); + sizeofobj = sizeof(upstreamsData); + counter = &counters->upstreams_MAX; + break; + case DNS_CACHE: + sharedMemory = &shm_dns_cache; + allocation_step = get_optimal_object_size(sizeof(DNSCacheData), 1); + sizeofobj = sizeof(DNSCacheData); + counter = &counters->dns_cache_MAX; break; default: - logg("Invalid argument in enlarge_shmem_struct(): %i", type); + logg("Invalid argument in enlarge_shmem_struct(%i)", type); return 0; } @@ -383,7 +475,7 @@ void *enlarge_shmem_struct(char type) return sharedMemory->ptr; } -bool realloc_shm(SharedMemory *sharedMemory, size_t size, bool resize) +bool realloc_shm(SharedMemory *sharedMemory, const size_t size, const bool resize) { // Check if we can skip this routine as nothing is to be done // when an object is not to be resized and its size didn't @@ -401,7 +493,7 @@ bool realloc_shm(SharedMemory *sharedMemory, size_t size, bool resize) if(resize) { // Open shared memory object - int fd = shm_open(sharedMemory->name, O_RDWR, S_IRUSR | S_IWUSR); + const int fd = shm_open(sharedMemory->name, O_RDWR, S_IRUSR | S_IWUSR); if(fd == -1) { logg("FATAL: realloc_shm(): Failed to open shared memory object \"%s\": %s", @@ -410,7 +502,7 @@ bool realloc_shm(SharedMemory *sharedMemory, size_t size, bool resize) } // Truncate shared memory object to specified size - int result = ftruncate(fd, size); + const int result = ftruncate(fd, size); if(result == -1) { logg("FATAL: realloc_shm(): ftruncate(%i, %zu): Failed to resize \"%s\": %s", fd, size, sharedMemory->name, strerror(errno)); @@ -444,8 +536,7 @@ bool realloc_shm(SharedMemory *sharedMemory, size_t size, bool resize) void delete_shm(SharedMemory *sharedMemory) { // Unmap shared memory - int ret; - ret = munmap(sharedMemory->ptr, sharedMemory->size); + int ret = munmap(sharedMemory->ptr, sharedMemory->size); if(ret != 0) logg("delete_shm(): munmap(%p, %zu) failed: %s", sharedMemory->ptr, sharedMemory->size, strerror(errno)); @@ -472,9 +563,9 @@ static size_t __attribute__((const)) gcd(size_t a, size_t b) // shared memory objects. This routine works by computing the LCM // of two numbers, the pagesize and the size of a single element // in the shared memory object -static size_t get_optimal_object_size(size_t objsize, size_t minsize) +static size_t get_optimal_object_size(const size_t objsize, const size_t minsize) { - size_t optsize = pagesize / gcd(pagesize, objsize); + const size_t optsize = pagesize / gcd(pagesize, objsize); if(optsize < minsize) { if(config.debug & DEBUG_SHMEM) @@ -490,7 +581,7 @@ static size_t get_optimal_object_size(size_t objsize, size_t minsize) // First part: Integer division, may cause clipping, e.g., 5/3 = 1 // Second part: Catch a possibly happened clipping event by adding // one to the number: (5 % 3 != 0) is 1 - size_t multiplier = (minsize/optsize) + ((minsize % optsize != 0) ? 1u : 0u); + const size_t multiplier = (minsize/optsize) + ((minsize % optsize != 0) ? 1u : 0u); if(config.debug & DEBUG_SHMEM) { logg("DEBUG: Using %zu*%zu == %zu >= %zu", @@ -516,3 +607,200 @@ static size_t get_optimal_object_size(size_t objsize, size_t minsize) return optsize; } } + +void memory_check(int which) +{ + switch(which) + { + case QUERIES: + if(counters->queries >= counters->queries_MAX-1) + { + // Have to reallocate shared memory + queries = enlarge_shmem_struct(QUERIES); + if(queries == NULL) + { + logg("FATAL: Memory allocation failed! Exiting"); + exit(EXIT_FAILURE); + } + } + break; + case UPSTREAMS: + if(counters->upstreams >= counters->upstreams_MAX-1) + { + // Have to reallocate shared memory + upstreams = enlarge_shmem_struct(UPSTREAMS); + if(upstreams == NULL) + { + logg("FATAL: Memory allocation failed! Exiting"); + exit(EXIT_FAILURE); + } + } + break; + case CLIENTS: + if(counters->clients >= counters->clients_MAX-1) + { + // Have to reallocate shared memory + clients = enlarge_shmem_struct(CLIENTS); + if(clients == NULL) + { + logg("FATAL: Memory allocation failed! Exiting"); + exit(EXIT_FAILURE); + } + } + break; + case DOMAINS: + if(counters->domains >= counters->domains_MAX-1) + { + // Have to reallocate shared memory + domains = enlarge_shmem_struct(DOMAINS); + if(domains == NULL) + { + logg("FATAL: Memory allocation failed! Exiting"); + exit(EXIT_FAILURE); + } + } + break; + case DNS_CACHE: + if(counters->dns_cache_size >= counters->dns_cache_MAX-1) + { + // Have to reallocate shared memory + dns_cache = enlarge_shmem_struct(DNS_CACHE); + if(dns_cache == NULL) + { + logg("FATAL: Memory allocation failed! Exiting"); + exit(EXIT_FAILURE); + } + } + break; + default: + /* That cannot happen */ + logg("Fatal error in memory_check(%i)", which); + exit(EXIT_FAILURE); + break; + } +} + +void reset_per_client_regex(const int clientID) +{ + const unsigned int num_regex_tot = counters->num_regex[REGEX_BLACKLIST] + + counters->num_regex[REGEX_WHITELIST]; + for(unsigned int i = 0u; i < num_regex_tot; i++) + { + // Zero-initialize/reset (= false) all regex (white + black) + set_per_client_regex(clientID, i, false); + } +} + +void add_per_client_regex(unsigned int clientID) +{ + const unsigned int num_regex_tot = counters->num_regex[REGEX_BLACKLIST] + + counters->num_regex[REGEX_WHITELIST]; + const size_t size = counters->clients * num_regex_tot; + if(size > shm_per_client_regex.size && + realloc_shm(&shm_per_client_regex, size, true)) + { + reset_per_client_regex(clientID); + } +} + +bool get_per_client_regex(const int clientID, const int regexID) +{ + const unsigned int num_regex_tot = counters->num_regex[REGEX_BLACKLIST] + + counters->num_regex[REGEX_WHITELIST]; + const unsigned int id = clientID * num_regex_tot + regexID; + const unsigned int maxval = counters->clients * num_regex_tot; + if(id > maxval) + { + logg("ERROR: get_per_client_regex(%d,%d): Out of bounds (%d > %d * %d == %d)!", + clientID, regexID, id, counters->clients-1, num_regex_tot, maxval); + return false; + } + return ((bool*) shm_per_client_regex.ptr)[id]; +} + +void set_per_client_regex(const int clientID, const int regexID, const bool value) +{ + const unsigned int num_regex_tot = counters->num_regex[REGEX_BLACKLIST] + + counters->num_regex[REGEX_WHITELIST]; + const unsigned int id = clientID * num_regex_tot + regexID; + const unsigned int maxval = counters->clients * num_regex_tot; + if(id > maxval) + { + logg("ERROR: set_per_client_regex(%d,%d,%s): Out of bounds (%d > %d * %d == %d)!", + clientID, regexID, value ? "true" : "false", + id, counters->clients-1, num_regex_tot, maxval); + return; + } + ((bool*) shm_per_client_regex.ptr)[id] = value; +} + +static inline bool check_range(int ID, int MAXID, const char* type, int line, const char * function, const char * file) +{ + if(ID < 0 || ID > MAXID) + { + // Check bounds + logg("FATAL: Trying to access %s ID %i, but maximum is %i", type, ID, MAXID); + logg(" found in %s() (%s:%i)", function, file, line); + return false; + } + // Everything okay + return true; +} + +static inline bool check_magic(int ID, bool checkMagic, unsigned char magic, const char* type, int line, const char * function, const char * file) +{ + if(checkMagic && magic != MAGICBYTE) + { + // Check magic only if requested (skipped for new entries which are uninitialized) + logg("FATAL: Trying to access %s ID %i, but magic byte is %x", type, ID, magic); + logg(" found in %s() (%s:%i)", function, file, line); + return false; + } + // Everything okay + return true; +} + +queriesData* _getQuery(int queryID, bool checkMagic, int line, const char * function, const char * file) +{ + if(check_range(queryID, counters->queries_MAX, "query", line, function, file) && + check_magic(queryID, checkMagic, queries[queryID].magic, "query", line, function, file)) + return &queries[queryID]; + else + return NULL; +} + +clientsData* _getClient(int clientID, bool checkMagic, int line, const char * function, const char * file) +{ + if(check_range(clientID, counters->clients_MAX, "client", line, function, file) && + check_magic(clientID, checkMagic, clients[clientID].magic, "client", line, function, file)) + return &clients[clientID]; + else + return NULL; +} + +domainsData* _getDomain(int domainID, bool checkMagic, int line, const char * function, const char * file) +{ + if(check_range(domainID, counters->domains_MAX, "domain", line, function, file) && + check_magic(domainID, checkMagic, domains[domainID].magic, "domain", line, function, file)) + return &domains[domainID]; + else + return NULL; +} + +upstreamsData* _getUpstream(int upstreamID, bool checkMagic, int line, const char * function, const char * file) +{ + if(check_range(upstreamID, counters->upstreams_MAX, "upstream", line, function, file) && + check_magic(upstreamID, checkMagic, upstreams[upstreamID].magic, "upstream", line, function, file)) + return &upstreams[upstreamID]; + else + return NULL; +} + +DNSCacheData* _getDNSCache(int cacheID, bool checkMagic, int line, const char * function, const char * file) +{ + if(check_range(cacheID, counters->dns_cache_MAX, "dns_cache", line, function, file) && + check_magic(cacheID, checkMagic, dns_cache[cacheID].magic, "dns_cache", line, function, file)) + return &dns_cache[cacheID]; + else + return NULL; +} diff --git a/src/shmem.h b/src/shmem.h new file mode 100644 index 000000000..969436bc5 --- /dev/null +++ b/src/shmem.h @@ -0,0 +1,114 @@ +/* Pi-hole: A black hole for Internet advertisements +* (c) 2018 Pi-hole, LLC (https://pi-hole.net) +* Network-wide ad blocking via your own hardware. +* +* FTL Engine +* Shared memory header +* +* This file is copyright under the latest version of the EUPL. +* Please see LICENSE file for your rights under this license. */ + +#ifndef SHARED_MEMORY_SERVER_H +#define SHARED_MEMORY_SERVER_H +#include /* For shm_* functions */ +#include /* For mode constants */ +#include /* For O_* constants */ +#include + +typedef struct { + const char *name; + size_t size; + void *ptr; +} SharedMemory; + +typedef struct { + int version; + unsigned int global_shm_counter; + unsigned int next_str_pos; +} ShmSettings; + +typedef struct { + int queries; + int blocked; + int forwarded; + int cached; + int unknown; + int upstreams; + int clients; + int domains; + int queries_MAX; + int upstreams_MAX; + int clients_MAX; + int domains_MAX; + int strings_MAX; + int gravity; + int querytype[TYPE_MAX-1]; + int reply_NODATA; + int reply_NXDOMAIN; + int reply_CNAME; + int reply_IP; + int reply_domain; + int dns_cache_size; + int dns_cache_MAX; + int num_regex[2]; +} countersStruct; + +extern countersStruct *counters; + +/// Create shared memory +/// +/// \param name the name of the shared memory +/// \param size the size to allocate +/// \return a structure with a pointer to the mounted shared memory. The pointer +/// will always be valid, because if it failed FTL will have exited. +SharedMemory create_shm(const char *name, const size_t size); + +/// Reallocate shared memory +/// +/// \param sharedMemory the shared memory struct +/// \param size the new size +/// \param resize whether the object should be resized or only remapped +/// \return if reallocation was successful +bool realloc_shm(SharedMemory *sharedMemory, const size_t size, const bool resize); + +/// Disconnect from shared memory. If there are no other connections to shared memory, it will be deleted. +/// +/// \param sharedMemory the shared memory struct +void delete_shm(SharedMemory *sharedMemory); + +/// Block until a lock can be obtained +#define lock_shm() _lock_shm(__FUNCTION__, __LINE__, __FILE__) +void _lock_shm(const char* func, const int line, const char* file); + +/// Unlock the lock. Only call this if there is an active lock. +#define unlock_shm() _unlock_shm(__FUNCTION__, __LINE__, __FILE__) +void _unlock_shm(const char* func, const int line, const char* file); + +bool init_shmem(void); +void destroy_shmem(void); +size_t addstr(const char *str); +const char *getstr(const size_t pos); +void *enlarge_shmem_struct(const char type); + +/** + * Create a new overTime client shared memory block. + * This also updates `overTimeClientData`. + */ +void newOverTimeClient(const int clientID); + +/** + * Add a new overTime slot to each overTime client shared memory block. + * This also updates `overTimeClientData`. + */ +void addOverTimeClientSlot(void); + +// Change ownership of shared memory objects +void chown_all_shmem(struct passwd *ent_pw); + +// Per-client regex buffer storing whether or not a specific regex is enabled for a particular client +void add_per_client_regex(unsigned int clientID); +void reset_per_client_regex(const int clientID); +bool get_per_client_regex(const int clientID, const int regexID); +void set_per_client_regex(const int clientID, const int regexID, const bool value); + +#endif //SHARED_MEMORY_SERVER_H diff --git a/signals.c b/src/signals.c similarity index 57% rename from signals.c rename to src/signals.c index a0e39526e..c118bfdff 100644 --- a/signals.c +++ b/src/signals.c @@ -9,12 +9,21 @@ * Please see LICENSE file for your rights under this license. */ #include "FTL.h" +#if defined(__GLIBC__) #include +#endif +#include "signals.h" +#include "log.h" +#include "memory.h" +// ls_dir() +#include "files.h" +// FTL_reload_all_domainlists() +#include "datastructure.h" volatile sig_atomic_t killed = 0; -time_t FTLstarttime = 0; +static time_t FTLstarttime = 0; -static void SIGSEGV_handler(int sig, siginfo_t *si, void *unused) +static void __attribute__((noreturn)) SIGSEGV_handler(int sig, siginfo_t *si, void *unused) { logg("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"); logg("----------------------------> FTL crashed! <----------------------------"); @@ -29,7 +38,7 @@ static void SIGSEGV_handler(int sig, siginfo_t *si, void *unused) log_FTL_version(true); logg("Received signal: %s", strsignal(sig)); - logg(" at address: %lu", (unsigned long) si->si_addr); + logg(" at address: %p", si->si_addr); switch (si->si_code) { case SEGV_MAPERR: logg(" with code: SEGV_MAPERR (Address not mapped to object)"); break; @@ -37,35 +46,58 @@ static void SIGSEGV_handler(int sig, siginfo_t *si, void *unused) #if defined(SEGV_BNDERR) case SEGV_BNDERR: logg(" with code: SEGV_BNDERR (Failed address bound checks)"); break; #endif - default: logg(" with code: Unknown (%i), ",si->si_code); break; + default: logg(" with code: Unknown (%i)", si->si_code); break; } +// Check GLIBC availability as MUSL does not support live backtrace generation +#if defined(__GLIBC__) // Try to obtain backtrace. This may not always be helpful, but it is better than nothing void *buffer[255]; const int calls = backtrace(buffer, sizeof(buffer)/sizeof(void *)); + logg("Backtrace:"); + char ** bcktrace = backtrace_symbols(buffer, calls); if(bcktrace == NULL) + logg("Unable to obtain backtrace symbols!"); + + for(int j = 0; j < calls; j++) { - logg("Unable to obtain backtrace (%i)!",calls); - } - else - { - logg("Backtrace:"); - int j; - for (j = 0; j < calls; j++) - { - logg("B[%04i]: %s",j,bcktrace[j]); - } + logg("B[%04i]: %p, %s", j, buffer[j], + bcktrace != NULL ? bcktrace[j] : "---"); } - free(bcktrace); + if(bcktrace != NULL) + free(bcktrace); +#else + logg("!!! INFO: pihole-FTL has not been compiled with glibc/backtrace support, not generating one !!!"); +#endif + // Print content of /dev/shm + ls_dir("/dev/shm"); logg("Thank you for helping us to improve our FTL engine!"); // Print message and abort logg("FTL terminated!"); - abort(); + exit(EXIT_FAILURE); } +static void SIGRT_handler(int signum, siginfo_t *si, void *unused) +{ + int rtsig = signum - SIGRTMIN; + logg("Received: %s (%d -> %d)", strsignal(signum), signum, rtsig); + + if(rtsig == 0) + { + // Reload + // - gravity + // - exact whitelist + // - regex whitelist + // - exact blacklist + // - exact blacklist + // WITHOUT wiping the DNS cache itself + FTL_reload_all_domainlists(); + } +} + void handle_signals(void) { struct sigaction old_action; @@ -82,6 +114,17 @@ void handle_signals(void) sigaction(SIGSEGV, &SEGVaction, NULL); } + // Catch first five real-time signals + for(unsigned int i = 0; i < 5; i++) + { + struct sigaction SIGACTION; + memset(&SIGACTION, 0, sizeof(struct sigaction)); + SIGACTION.sa_flags = SA_SIGINFO; + sigemptyset(&SIGACTION.sa_mask); + SIGACTION.sa_sigaction = &SIGRT_handler; + sigaction(SIGRTMIN + i, &SIGACTION, NULL); + } + // Log start time of FTL FTLstarttime = time(NULL); } diff --git a/src/signals.h b/src/signals.h new file mode 100644 index 000000000..e4007ab23 --- /dev/null +++ b/src/signals.h @@ -0,0 +1,17 @@ +/* Pi-hole: A black hole for Internet advertisements +* (c) 2019 Pi-hole, LLC (https://pi-hole.net) +* Network-wide ad blocking via your own hardware. +* +* FTL Engine +* Signal handling prototypes +* +* This file is copyright under the latest version of the EUPL. +* Please see LICENSE file for your rights under this license. */ +#ifndef SIGNALS_H +#define SIGNALS_H + +void handle_signals(void); + +extern volatile sig_atomic_t killed; + +#endif //SIGNALS_H diff --git a/src/timers.c b/src/timers.c new file mode 100644 index 000000000..58ce51d1f --- /dev/null +++ b/src/timers.c @@ -0,0 +1,46 @@ +/* Pi-hole: A black hole for Internet advertisements +* (c) 2017 Pi-hole, LLC (https://pi-hole.net) +* Network-wide ad blocking via your own hardware. +* +* FTL Engine +* Timing routines +* +* This file is copyright under the latest version of the EUPL. +* Please see LICENSE file for your rights under this license. */ + +#include "FTL.h" +#include "timers.h" +#include "memory.h" +#include "log.h" + +struct timeval t0[NUMTIMERS]; + +void timer_start(const int i) +{ + if(i >= NUMTIMERS) + { + logg("Code error: Timer %i not defined in timer_start().", i); + exit(EXIT_FAILURE); + } + gettimeofday(&t0[i], 0); +} + +double timer_elapsed_msec(const int i) +{ + if(i >= NUMTIMERS) + { + logg("Code error: Timer %i not defined in timer_elapsed_msec().", i); + exit(EXIT_FAILURE); + } + struct timeval t1; + gettimeofday(&t1, 0); + return (t1.tv_sec - t0[i].tv_sec) * 1000.0f + (t1.tv_usec - t0[i].tv_usec) / 1000.0f; +} + +void sleepms(const int milliseconds) +{ + struct timeval tv; + tv.tv_sec = milliseconds / 1000; + tv.tv_usec = (milliseconds % 1000) * 1000; + select(0, NULL, NULL, NULL, &tv); +} diff --git a/src/timers.h b/src/timers.h new file mode 100644 index 000000000..e30f98785 --- /dev/null +++ b/src/timers.h @@ -0,0 +1,22 @@ +/* Pi-hole: A black hole for Internet advertisements +* (c) 2019 Pi-hole, LLC (https://pi-hole.net) +* Network-wide ad blocking via your own hardware. +* +* FTL Engine +* Timer prototypes +* +* This file is copyright under the latest version of the EUPL. +* Please see LICENSE file for your rights under this license. */ +#ifndef TIMERS_H +#define TIMERS_H + +// Timer enumeration +enum { DATABASE_WRITE_TIMER, EXIT_TIMER, GC_TIMER, LISTS_TIMER, REGEX_TIMER, ARP_TIMER, LAST_TIMER }; + +#define NUMTIMERS LAST_TIMER + +void timer_start(const int i); +double timer_elapsed_msec(const int i); +void sleepms(const int milliseconds); + +#endif //TIMERS_H diff --git a/src/vector.c b/src/vector.c new file mode 100644 index 000000000..07b7653ac --- /dev/null +++ b/src/vector.c @@ -0,0 +1,123 @@ +/* Pi-hole: A black hole for Internet advertisements +* (c) 2020 Pi-hole, LLC (https://pi-hole.net) +* Network-wide ad blocking via your own hardware. +* +* FTL Engine +* Dynamic vector routines +* +* This file is copyright under the latest version of the EUPL. +* Please see LICENSE file for your rights under this license. */ + +#include "vector.h" +// struct config +#include "config.h" +// logg() +#include "log.h" + +sqlite3_stmt_vec *new_sqlite3_stmt_vec(unsigned int initial_size) +{ + if(config.debug & DEBUG_VECTORS) + logg("Initializing new sqlite3_stmt* vector with size %u", initial_size); + + sqlite3_stmt_vec *v = calloc(1, sizeof(sqlite3_stmt_vec)); + v->capacity = initial_size; + // Calloc ensures they are all set to zero which is the default state + v->items = calloc(initial_size, sizeof(sqlite3_stmt *) * initial_size); + // Set correct subroutine pointers + v->set = set_sqlite3_stmt_vec; + v->get = get_sqlite3_stmt_vec; + v->free = free_sqlite3_stmt_vec; + return v; +} + +static void resize_sqlite3_stmt_vec(sqlite3_stmt_vec *v, unsigned int capacity) +{ + if(config.debug & DEBUG_VECTORS) + logg("Resizing sqlite3_stmt* vector %p from %u to %u", v, v->capacity, capacity); + + // If ptr is NULL, the call to realloc(ptr, size) is equivalent to + // malloc(size) so we can use it also for initializing a vector for the + // first time. + sqlite3_stmt **items = realloc(v->items, sizeof(sqlite3_stmt *) * capacity); + if(!items) + { + logg("ERROR: Memory allocation failed in resize_sqlite3_stmt_vec(%p, %u)", + v, capacity); + return; + } + + // Update items pointer + v->items = items; + + // NULL-initialize newly allocated memory slots + for(unsigned int i = v->capacity; i < capacity; i++) + v->items[i] = NULL; + + // Update capacity + v->capacity = capacity; +} + +void set_sqlite3_stmt_vec(sqlite3_stmt_vec *v, unsigned int index, sqlite3_stmt *item) +{ + if(config.debug & DEBUG_VECTORS) + logg("Setting sqlite3_stmt** %p[%u] <-- %p", v, index, item); + + if(v == NULL) + { + logg("ERROR: Passed NULL vector to set_sqlite3_stmt_vec(%p, %u, %p)", + v, index, item); + return; + } + + if(index >= v->capacity) + { + // Allocate more memory when trying to set a statement vector entry with + // an index larger than the current array size (this makes set an + // equivalent alternative to append) + resize_sqlite3_stmt_vec(v, index + VEC_ALLOC_STEP); + } + + // Set item + v->items[index] = item; +} + +// This function has no effects except to return a value. It can be subject to +// data flow analysis and might be eliminated. Hence, we add the "pure" +// attribute to this function. +sqlite3_stmt * __attribute__((pure)) get_sqlite3_stmt_vec(sqlite3_stmt_vec *v, unsigned int index) +{ + if(v == NULL) + { + logg("ERROR: Passed NULL vector to get_sqlite3_stmt_vec(%p, %u)", + v, index); + return 0; + } + + if(index >= v->capacity) + { + // Silently increase size of vector if trying to read out-of-bounds + resize_sqlite3_stmt_vec(v, index + VEC_ALLOC_STEP); + } + + sqlite3_stmt* item = v->items[index]; + if(config.debug & DEBUG_VECTORS) + logg("Getting sqlite3_stmt** %p[%u] --> %p", v, index, item); + + return item; +} + +void free_sqlite3_stmt_vec(sqlite3_stmt_vec *v) +{ + if(config.debug & DEBUG_VECTORS) + logg("Freeing sqlite3_stmt* vector %p", v); + + // This vector was never allocated, invoking free_sqlite3_stmt_vec() on a + // NULL pointer should be a harmless no-op. + if(v == NULL || v->items == NULL) + return; + + // Free elements of the vector... + free(v->items); + // ...and then the vector itself + free(v); +} diff --git a/src/vector.h b/src/vector.h new file mode 100644 index 000000000..f39aa9875 --- /dev/null +++ b/src/vector.h @@ -0,0 +1,38 @@ +/* Pi-hole: A black hole for Internet advertisements +* (c) 2020 Pi-hole, LLC (https://pi-hole.net) +* Network-wide ad blocking via your own hardware. +* +* FTL Engine +* Dynamic vector prototypes +* +* This file is copyright under the latest version of the EUPL. +* Please see LICENSE file for your rights under this license. */ +#ifndef VECTOR_H +#define VECTOR_H + +#include +#include +// memmove() +#include +// type bool +#include +// type sqlite3_stmt +#include "database/sqlite3.h" + +#define VEC_ALLOC_STEP 10u + +typedef struct sqlite3_stmt_vec { + unsigned int capacity; + sqlite3_stmt **items; + sqlite3_stmt *(*get)(struct sqlite3_stmt_vec *, unsigned int); + void (*set)(struct sqlite3_stmt_vec *, unsigned int, sqlite3_stmt*); + void (*free)(struct sqlite3_stmt_vec *); +} sqlite3_stmt_vec; + +sqlite3_stmt_vec *new_sqlite3_stmt_vec(unsigned int initial_size); +void set_sqlite3_stmt_vec(sqlite3_stmt_vec *v, unsigned int index, sqlite3_stmt* item); +sqlite3_stmt* get_sqlite3_stmt_vec(sqlite3_stmt_vec *v, unsigned int index) __attribute__((pure)); +void del_sqlite3_stmt_vec(sqlite3_stmt_vec *v, unsigned int index); +void free_sqlite3_stmt_vec(sqlite3_stmt_vec *v); + +#endif //VECTOR_H diff --git a/test/gravity.db.sql b/test/gravity.db.sql new file mode 100644 index 000000000..03e902307 --- /dev/null +++ b/test/gravity.db.sql @@ -0,0 +1,196 @@ +PRAGMA FOREIGN_KEYS=ON; +BEGIN TRANSACTION; + +CREATE TABLE "group" +( + id INTEGER PRIMARY KEY AUTOINCREMENT, + enabled BOOLEAN NOT NULL DEFAULT 1, + name TEXT NOT NULL, + date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), + date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), + description TEXT +); + +INSERT OR REPLACE INTO "group" (id,enabled,name) VALUES (0,1,'Unassociated'); +CREATE TRIGGER tr_group_zero AFTER DELETE ON "group" + BEGIN + INSERT OR REPLACE INTO "group" (id,enabled,name) VALUES (0,1,'Unassociated'); + END; +CREATE TRIGGER tr_group_update AFTER UPDATE ON "group" + BEGIN + UPDATE "group" SET date_modified = (cast(strftime('%s', 'now') as int)) WHERE id = NEW.id; + END; + +CREATE TABLE domainlist +( + id INTEGER PRIMARY KEY AUTOINCREMENT, + type INTEGER NOT NULL DEFAULT 0, + domain TEXT UNIQUE NOT NULL, + enabled BOOLEAN NOT NULL DEFAULT 1, + date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), + date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), + comment TEXT +); +CREATE TRIGGER tr_domainlist_update AFTER UPDATE ON domainlist + BEGIN + UPDATE domainlist SET date_modified = (cast(strftime('%s', 'now') as int)) WHERE domain = NEW.domain; + END; +CREATE TRIGGER tr_domainlist_add AFTER INSERT ON domainlist + BEGIN + INSERT INTO domainlist_by_group (domainlist_id, group_id) VALUES (NEW.id, 0); + END; + +CREATE TABLE domainlist_by_group +( + domainlist_id INTEGER NOT NULL REFERENCES domainlist (id), + group_id INTEGER NOT NULL REFERENCES "group" (id), + PRIMARY KEY (domainlist_id, group_id) +); + +CREATE TABLE adlist +( + id INTEGER PRIMARY KEY AUTOINCREMENT, + address TEXT UNIQUE NOT NULL, + enabled BOOLEAN NOT NULL DEFAULT 1, + date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), + date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), + comment TEXT +); +CREATE TRIGGER tr_adlist_update AFTER UPDATE ON adlist + BEGIN + UPDATE adlist SET date_modified = (cast(strftime('%s', 'now') as int)) WHERE address = NEW.address; + END; +CREATE TRIGGER tr_adlist_add AFTER INSERT ON adlist + BEGIN + INSERT INTO adlist_by_group (adlist_id, group_id) VALUES (NEW.id, 0); + END; + +CREATE TABLE adlist_by_group +( + adlist_id INTEGER NOT NULL REFERENCES adlist (id), + group_id INTEGER NOT NULL REFERENCES "group" (id), + PRIMARY KEY (adlist_id, group_id) +); + +CREATE TABLE gravity +( + domain TEXT NOT NULL, + adlist_id INTEGER NOT NULL REFERENCES adlist (id), + PRIMARY KEY(domain, adlist_id) +); + +CREATE TABLE client +( + id INTEGER PRIMARY KEY AUTOINCREMENT, + ip TEXT NOL NULL UNIQUE +); + +CREATE TABLE client_by_group +( + client_id INTEGER NOT NULL REFERENCES client (id), + group_id INTEGER NOT NULL REFERENCES "group" (id), + PRIMARY KEY (client_id, group_id) +); + +CREATE TRIGGER tr_client_add AFTER INSERT ON client + BEGIN + INSERT INTO client_by_group (client_id, group_id) VALUES (NEW.id, 0); + END; + +CREATE TABLE domain_audit +( + id INTEGER PRIMARY KEY AUTOINCREMENT, + domain TEXT UNIQUE NOT NULL, + date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)) +); + +CREATE TABLE info +( + property TEXT PRIMARY KEY, + value TEXT NOT NULL +); + +CREATE VIEW vw_gravity AS SELECT domain, adlist_by_group.group_id AS group_id + FROM gravity + LEFT JOIN adlist_by_group ON adlist_by_group.adlist_id = gravity.adlist_id + LEFT JOIN adlist ON adlist.id = gravity.adlist_id + LEFT JOIN "group" ON "group".id = adlist_by_group.group_id + WHERE adlist.enabled = 1 AND (adlist_by_group.group_id IS NULL OR "group".enabled = 1); + +CREATE VIEW vw_whitelist AS SELECT domain, domainlist.id AS id, domainlist_by_group.group_id AS group_id + FROM domainlist + LEFT JOIN domainlist_by_group ON domainlist_by_group.domainlist_id = domainlist.id + LEFT JOIN "group" ON "group".id = domainlist_by_group.group_id + WHERE domainlist.enabled = 1 AND (domainlist_by_group.group_id IS NULL OR "group".enabled = 1) + AND domainlist.type = 0 + ORDER BY domainlist.id; + +CREATE VIEW vw_blacklist AS SELECT domain, domainlist.id AS id, domainlist_by_group.group_id AS group_id + FROM domainlist + LEFT JOIN domainlist_by_group ON domainlist_by_group.domainlist_id = domainlist.id + LEFT JOIN "group" ON "group".id = domainlist_by_group.group_id + WHERE domainlist.enabled = 1 AND (domainlist_by_group.group_id IS NULL OR "group".enabled = 1) + AND domainlist.type = 1 + ORDER BY domainlist.id; + +CREATE VIEW vw_regex_whitelist AS SELECT domain, domainlist.id AS id, domainlist_by_group.group_id AS group_id + FROM domainlist + LEFT JOIN domainlist_by_group ON domainlist_by_group.domainlist_id = domainlist.id + LEFT JOIN "group" ON "group".id = domainlist_by_group.group_id + WHERE domainlist.enabled = 1 AND (domainlist_by_group.group_id IS NULL OR "group".enabled = 1) + AND domainlist.type = 2 + ORDER BY domainlist.id; + +CREATE VIEW vw_regex_blacklist AS SELECT domain, domainlist.id AS id, domainlist_by_group.group_id AS group_id + FROM domainlist + LEFT JOIN domainlist_by_group ON domainlist_by_group.domainlist_id = domainlist.id + LEFT JOIN "group" ON "group".id = domainlist_by_group.group_id + WHERE domainlist.enabled = 1 AND (domainlist_by_group.group_id IS NULL OR "group".enabled = 1) + AND domainlist.type = 3 + ORDER BY domainlist.id; + +CREATE VIEW vw_adlist AS SELECT DISTINCT address, adlist.id AS id + FROM adlist + LEFT JOIN adlist_by_group ON adlist_by_group.adlist_id = adlist.id + LEFT JOIN "group" ON "group".id = adlist_by_group.group_id + WHERE adlist.enabled = 1 AND (adlist_by_group.group_id IS NULL OR "group".enabled = 1) + ORDER BY adlist.id; + +INSERT INTO domainlist VALUES(1,0,'whitelisted.test.pi-hole.net',1,1559928803,1559928803,'Migrated from /etc/pihole/whitelist.txt'); +INSERT INTO domainlist VALUES(2,0,'regex1.test.pi-hole.net',1,1559928803,1559928803,''); +INSERT INTO domainlist VALUES(3,2,'regex2',1,1559928803,1559928803,''); +INSERT INTO domainlist VALUES(4,2,'discourse',1,1559928803,1559928803,''); + +INSERT INTO domainlist VALUES(5,1,'blacklist-blocked.test.pi-hole.net',1,1559928803,1559928803,'Migrated from /etc/pihole/blacklist.txt'); +INSERT INTO domainlist VALUES(6,3,'regex[0-9].test.pi-hole.net',1,1559928803,1559928803,'Migrated from /etc/pihole/regex.list'); + +INSERT INTO adlist VALUES(1,'https://hosts-file.net/ad_servers.txt',1,1559928803,1559928803,'Migrated from /etc/pihole/adlists.list'); + +INSERT INTO gravity VALUES('whitelisted.test.pi-hole.net',1); +INSERT INTO gravity VALUES('gravity-blocked.test.pi-hole.net',1); +INSERT INTO gravity VALUES('discourse.pi-hole.net',1); +INSERT INTO info VALUES("gravity_count",3); + +INSERT INTO "group" VALUES(1,0,'Test group',1559928803,1559928803,'A disabled test group'); +INSERT INTO domainlist VALUES(7,1,'blacklisted-group-disabled.com',1,1559928803,1559928803,'Entry disabled by a group'); +INSERT INTO domainlist_by_group VALUES(7,1); + +INSERT INTO domain_audit VALUES(1,'google.com',1559928803); + +INSERT INTO client VALUES(1,"127.0.0.1"); + +INSERT INTO client VALUES(2,"127.0.0.2"); +INSERT INTO "group" VALUES(2,1,"Second test group",1559928803,1559928803,"A group associated with client 127.0.0.2"); +DELETE FROM client_by_group WHERE client_id = 2 AND group_id = 0; +INSERT INTO client_by_group VALUES(2,2); +INSERT INTO adlist_by_group VALUES(1,2); +INSERT INTO domainlist_by_group VALUES(6,2); + +INSERT INTO client VALUES(3,"127.0.0.3"); +INSERT INTO "group" VALUES(3,1,"Third test group",1559928803,1559928803,"A group associated with client 127.0.0.3"); +DELETE FROM client_by_group WHERE client_id = 3 AND group_id = 0; +INSERT INTO client_by_group VALUES(3,3); + +INSERT INTO info VALUES("version","9"); + +COMMIT; diff --git a/test/libs/bats b/test/libs/bats deleted file mode 160000 index 03608115d..000000000 --- a/test/libs/bats +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 03608115df2071fff4eaaff1605768c275e5f81f diff --git a/test/libs/bats-support b/test/libs/bats-support deleted file mode 160000 index 004e70763..000000000 --- a/test/libs/bats-support +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 004e707638eedd62e0481e8cdc9223ad471f12ee diff --git a/test/run.sh b/test/run.sh index a8891d210..6d8fc2b3a 100755 --- a/test/run.sh +++ b/test/run.sh @@ -1,78 +1,81 @@ #!/bin/bash -dnsmasq_pre() { - echo -n $(date +"%b %e %H:%M:%S") - echo -n "dnsmasq[123]:" -} - -# Prepare FTL's files -ts="$(dnsmasq_pre)" -cat <> pihole.log -${ts} 1 1270.0.01/1234 query[AAAA] raspberrypi from 127.0.0.1 -${ts} 1 1270.0.01/1234 /etc/pihole/local.list raspberrypi is fda2:2001:5647:0:ba27:ebff:fe37:4205 -${ts} 2 1270.0.01/1234 query[A] ChEcKiP.DyNdNs.OrG from 127.0.0.1 -${ts} 2 1270.0.01/1234 forwarded ChEcKiP.DyNdNs.OrG to 2001:1608:10:25::9249:d69b -${ts} 2 1270.0.01/1234 forwarded ChEcKiP.DyNdNs.OrG to 2001:1608:10:25::1c04:b12f -${ts} 2 1270.0.01/1234 forwarded ChEcKiP.DyNdNs.OrG to 2620:0:ccd::2 -${ts} 2 1270.0.01/1234 forwarded ChEcKiP.DyNdNs.OrG to 2620:0:ccc::2 -${ts} 2 1270.0.01/1234 reply ChEcKiP.DyNdNs.OrG is -${ts} 2 1270.0.01/1234 reply ChEcKiP.DyNdNs.OrG is 216.146.38.70 -${ts} 2 1270.0.01/1234 reply ChEcKiP.DyNdNs.OrG is 216.146.43.71 -${ts} 2 1270.0.01/1234 reply ChEcKiP.DyNdNs.OrG is 91.198.22.70 -${ts} 2 1270.0.01/1234 reply ChEcKiP.DyNdNs.OrG is 216.146.43.70 -${ts} 3 1270.0.01/1234 query[A] pi.hole from 10.8.0.2 -${ts} 3 1270.0.01/1234 /etc/pihole/local.list pi.hole is 192.168.2.10 -${ts} 4 1270.0.01/1234 query[A] example.com from 10.8.0.2 -${ts} 4 1270.0.01/1234 /etc/pihole/local.list example.com is 192.168.2.10 -${ts} 5 1270.0.01/1234 query[A] play.google.com from 192.168.2.208 -${ts} 5 1270.0.01/1234 forwarded play.google.com to 2001:1608:10:25::9249:d69b -${ts} 5 1270.0.01/1234 forwarded play.google.com to 2001:1608:10:25::1c04:b12f -${ts} 5 1270.0.01/1234 forwarded play.google.com to 2620:0:ccd::2 -${ts} 5 1270.0.01/1234 forwarded play.google.com to 2620:0:ccc::2 -${ts} 5 1270.0.01/1234 reply play.google.com is -${ts} 5 1270.0.01/1234 reply play.l.google.com is 216.58.208.110 -${ts} 5 1270.0.01/1234 reply play.l.google.com is 216.58.208.110 -${ts} 5 1270.0.01/1234 reply play.l.google.com is 216.58.208.110 -${ts} 5 1270.0.01/1234 reply play.google.com is -${ts} 6 1270.0.01/1234 query[AAAA] play.google.com from 192.168.2.208 -${ts} 6 1270.0.01/1234 forwarded play.google.com to 2620:0:ccd::2 -${ts} 6 1270.0.01/1234 reply play.l.google.com is 2a00:1450:4017:802::200e -${ts} 7 1270.0.01/1234 query[A] blacklisted.com from 192.168.2.208 -${ts} 7 1270.0.01/1234 /etc/pihole/black.list blacklisted.com is 1.2.3.4 -${ts} 8 1270.0.01/1234 query[A] addomain.com from 192.168.2.208 -${ts} 8 1270.0.01/1234 /etc/pihole/gravity.list addomain.com is 1.2.3.4 -EOT -touch "pihole-FTL.log" - -cat <> pihole-FTL.conf -DBFILE=pihole-FTL.db -LOGFILE=pihole-FTL.log -SOCKETFILE=pihole-FTL.sock -EOT +# Only run tests on x86_64, x86_64-musl, and x86_32 targets +if [[ ${CI} == "true" && "${CIRCLE_JOB}" != "x86_64" && "${CIRCLE_JOB}" != "x86_64-musl" && "${CIRCLE_JOB}" != "x86_32" ]]; then + echo "Skipping tests (CIRCLE_JOB: ${CIRCLE_JOB})!" + exit 0 +fi + +# Create pihole user if it does not exist +if ! id -u pihole &> /dev/null; then + useradd -m -s /usr/sbin/nologin pihole +fi + +# Clean up possible old files from earlier test runs +rm -f /etc/pihole/gravity.db /etc/pihole/pihole-FTL.db /var/log/pihole.log /var/log/pihole-FTL.log + +# Create necessary directories and files +mkdir -p /etc/pihole /var/run/pihole /var/log +touch /var/log/pihole-FTL.log /var/log/pihole.log /var/run/pihole-FTL.pid /var/run/pihole-FTL.port +chown pihole:pihole /etc/pihole /var/run/pihole /var/log/pihole.log /var/log/pihole-FTL.log /var/run/pihole-FTL.pid /var/run/pihole-FTL.port + +# Copy binary into a location the new user pihole can access +cp ./pihole-FTL /home/pihole +chmod +x /home/pihole/pihole-FTL +# Note: We cannot add CAP_NET_RAW and CAP_NET_ADMIN at this point +setcap CAP_NET_BIND_SERVICE+eip /home/pihole/pihole-FTL + +# Prepare gravity database +sqlite3 /etc/pihole/gravity.db < test/gravity.db.sql + +# Prepare setupVars.conf +echo "BLOCKING_ENABLED=true" > /etc/pihole/setupVars.conf + +# Prepare pihole-FTL.conf +echo -e "DEBUG_ALL=true\nRESOLVE_IPV4=no\nRESOLVE_IPV6=no" > /etc/pihole/pihole-FTL.conf + +# Prepare dnsmasq.conf +echo -e "log-queries\nlog-facility=/var/log/pihole.log" > /etc/dnsmasq.conf + +# Set restrictive umask +OLDUMASK=$(umask) +umask 0022 # Start FTL -./pihole-FTL travis-ci +if ! su pihole -s /bin/sh -c /home/pihole/pihole-FTL; then + echo "pihole-FTL failed to start" + exit 1 +fi # Prepare BATS mkdir -p test/libs -git submodule add https://github.com/sstephenson/bats test/libs/bats -git submodule add https://github.com/ztombol/bats-support test/libs/bats-support -# git submodule add https://github.com/ztombol/bats-assert test/libs/bats-assert +git clone --depth=1 --quiet https://github.com/bats-core/bats-core test/libs/bats > /dev/null # Block until FTL is ready, retry once per second for 45 seconds -n=0 -until [ $n -ge 45 ]; do - nc -vv -z -w 30 127.0.0.1 4711 && break - n=$[$n+1] - echo "..." - tail -n2 pihole-FTL.log - echo "..." - sleep 1 -done - -# Print content of pihole-FTL.log -cat pihole-FTL.log +sleep 2 + +# Print versions of pihole-FTL +echo -n "FTL version: " +dig TXT CHAOS version.FTL @127.0.0.1 +short +echo -n "Contained dnsmasq version: " +dig TXT CHAOS version.bind @127.0.0.1 +short + +# Print content of pihole.log and pihole-FTL.log +cat /var/log/pihole.log +cat /var/log/pihole-FTL.log # Run tests -test/libs/bats/bin/bats "test/test_suite.sh" -exit $? +test/libs/bats/bin/bats "test/test_suite.bats" +RET=$? + +# Kill pihole-FTL after having completed tests +kill $(pidof pihole-FTL) + +# Restore umask +umask $OLDUMASK + +# Remove copied file +rm /home/pihole/pihole-FTL + +# Exit with return code of bats tests +exit $RET diff --git a/test/test_suite.bats b/test/test_suite.bats new file mode 100644 index 000000000..ff662ab8a --- /dev/null +++ b/test/test_suite.bats @@ -0,0 +1,453 @@ +#!./test/libs/bats/bin/bats + +@test "Version, Tag, Branch, Hash, Date is reported" { + run bash -c 'echo ">version >quit" | nc -v 127.0.0.1 4711' + printf "%s\n" "${lines[@]}" + [[ ${lines[1]} == "version "* ]] + [[ ${lines[2]} == "tag "* ]] + [[ ${lines[3]} == "branch "* ]] + [[ ${lines[4]} == "hash "* ]] + [[ ${lines[5]} == "date "* ]] + [[ ${lines[6]} == "" ]] +} + +@test "Starting tests without prior history" { + run bash -c 'grep -c "Total DNS queries: 0" /var/log/pihole-FTL.log' + printf "%s\n" "${lines[@]}" + [[ ${lines[0]} == "1" ]] +} + +@test "Initial blocking status is enabled" { + run bash -c 'grep -c "Blocking status is enabled" /var/log/pihole-FTL.log' + printf "%s\n" "${lines[@]}" + [[ ${lines[0]} == "1" ]] +} + +@test "Number of compiled regex filters as expected" { + run bash -c 'grep -c "Compiled 2 whitelist and 1 blacklist regex filters" /var/log/pihole-FTL.log' + printf "%s\n" "${lines[@]}" + [[ ${lines[0]} == "1" ]] +} + +@test "Blacklisted domain is blocked" { + run bash -c "dig blacklist-blocked.test.pi-hole.net @127.0.0.1 +short" + printf "%s\n" "${lines[@]}" + [[ ${lines[0]} == "0.0.0.0" ]] + [[ ${lines[1]} == "" ]] +} + +@test "Gravity domain is blocked" { + run bash -c "dig gravity-blocked.test.pi-hole.net @127.0.0.1 +short" + printf "%s\n" "${lines[@]}" + [[ ${lines[0]} == "0.0.0.0" ]] + [[ ${lines[1]} == "" ]] +} + +@test "Gravity domain is blocked (TCP)" { + run bash -c "dig gravity-blocked.test.pi-hole.net @127.0.0.1 +tcp +short" + printf "%s\n" "${lines[@]}" + [[ ${lines[0]} == "0.0.0.0" ]] + [[ ${lines[1]} == "" ]] +} + +@test "Gravity domain + whitelist exact match is not blocked" { + run bash -c "dig whitelisted.test.pi-hole.net @127.0.0.1 +short" + printf "%s\n" "${lines[@]}" + [[ ${lines[0]} != "0.0.0.0" ]] +} + +@test "Gravity domain + whitelist regex match is not blocked" { + run bash -c "dig discourse.pi-hole.net @127.0.0.1 +short" + printf "%s\n" "${lines[@]}" + [[ ${lines[0]} != "0.0.0.0" ]] +} + +@test "Regex blacklist match is blocked" { + run bash -c "dig regex5.test.pi-hole.net @127.0.0.1 +short" + printf "%s\n" "${lines[@]}" + [[ ${lines[0]} == "0.0.0.0" ]] + [[ ${lines[1]} == "" ]] +} + +@test "Regex blacklist mismatch is not blocked" { + run bash -c "dig regexA.test.pi-hole.net @127.0.0.1 +short" + printf "%s\n" "${lines[@]}" + [[ ${lines[0]} != "0.0.0.0" ]] +} + +@test "Regex blacklist match + whitelist exact match is not blocked" { + run bash -c "dig regex1.test.pi-hole.net @127.0.0.1 +short" + printf "%s\n" "${lines[@]}" + [[ ${lines[0]} != "0.0.0.0" ]] +} + +@test "Regex blacklist match + whitelist regex match is not blocked" { + run bash -c "dig regex2.test.pi-hole.net @127.0.0.1 +short" + printf "%s\n" "${lines[@]}" + [[ ${lines[0]} != "0.0.0.0" ]] +} + +@test "Client 2: Gravity match matching unassociated whitelist is blocked" { + run bash -c "dig whitelisted.test.pi-hole.net -b 127.0.0.2 @127.0.0.1 +short" + printf "%s\n" "${lines[@]}" + [[ ${lines[0]} == "0.0.0.0" ]] +} + +@test "Client 2: Regex blacklist match matching unassociated whitelist is blocked" { + run bash -c "dig regex1.test.pi-hole.net -b 127.0.0.2 @127.0.0.1 +short" + printf "%s\n" "${lines[@]}" + [[ ${lines[0]} == "0.0.0.0" ]] +} + +@test "Same domain is not blocked for client 1 ..." { + run bash -c "dig regex1.test.pi-hole.net @127.0.0.1 +short" + printf "%s\n" "${lines[@]}" + [[ ${lines[0]} != "0.0.0.0" ]] +} + +@test "... or client 3" { + run bash -c "dig regex1.test.pi-hole.net -b 127.0.0.3 @127.0.0.1 +short" + printf "%s\n" "${lines[@]}" + [[ ${lines[0]} != "0.0.0.0" ]] +} + +@test "Client 2: Unassociated blacklist match is not blocked" { + run bash -c "dig blacklist-blocked.test.pi-hole.net -b 127.0.0.2 @127.0.0.1 +short" + printf "%s\n" "${lines[@]}" + [[ ${lines[0]} != "0.0.0.0" ]] +} + +@test "Client 3: Exact blacklist domain is not blocked" { + run bash -c "dig blacklist-blocked.test.pi-hole.net -b 127.0.0.3 @127.0.0.1 +short" + printf "%s\n" "${lines[@]}" + [[ ${lines[0]} != "0.0.0.0" ]] +} + +@test "Client 3: Regex blacklist domain is not blocked" { + run bash -c "dig regex1.test.pi-hole.net -b 127.0.0.3 @127.0.0.1 +short" + printf "%s\n" "${lines[@]}" + [[ ${lines[0]} != "0.0.0.0" ]] +} + +@test "Client 3: Gravity domain is not blocked" { + run bash -c "dig discourse.pi-hole.net -b 127.0.0.3 @127.0.0.1 +short" + printf "%s\n" "${lines[@]}" + [[ ${lines[0]} != "0.0.0.0" ]] +} + +@test "Google.com (A) is not blocked" { + run bash -c "dig A google.com @127.0.0.1 +short" + printf "%s\n" "${lines[@]}" + [[ ${lines[0]} != "0.0.0.0" ]] +} + +@test "Google.com (AAAA) is not blocked (TCP query)" { + run bash -c "dig AAAA google.com @127.0.0.1 +short +tcp" + printf "%s\n" "${lines[@]}" + [[ ${lines[0]} != "::" ]] + [[ ${lines[1]} == "" ]] +} + +@test "Known host is resolved as expected" { + run bash -c "dig ftl.pi-hole.net @127.0.0.1 +short" + printf "%s\n" "${lines[@]}" + [[ ${lines[0]} == "139.59.170.52" ]] + [[ ${lines[1]} == "" ]] +} + +@test "Statistics as expected" { + run bash -c 'echo ">stats >quit" | nc -v 127.0.0.1 4711' + printf "%s\n" "${lines[@]}" + [[ ${lines[1]} == "domains_being_blocked 3" ]] + [[ ${lines[2]} == "dns_queries_today 22" ]] + [[ ${lines[3]} == "ads_blocked_today 6" ]] + [[ ${lines[4]} == "ads_percentage_today 27.272728" ]] + [[ ${lines[5]} == "unique_domains 12" ]] + [[ ${lines[6]} == "queries_forwarded 9" ]] + [[ ${lines[7]} == "queries_cached 7" ]] + # Clients ever seen is commented out as CircleCI may have + # more devices in its ARP cache so testing against a fixed + # number of clients may not work in all cases + #[[ ${lines[8]} == "clients_ever_seen 3" ]] + #[[ ${lines[9]} == "unique_clients 3" ]] + [[ ${lines[10]} == "dns_queries_all_types 22" ]] + [[ ${lines[11]} == "reply_NODATA 0" ]] + [[ ${lines[12]} == "reply_NXDOMAIN 0" ]] + [[ ${lines[13]} == "reply_CNAME 0" ]] + [[ ${lines[14]} == "reply_IP 20" ]] + [[ ${lines[15]} == "privacy_level 0" ]] + [[ ${lines[16]} == "status enabled" ]] + [[ ${lines[17]} == "" ]] +} + +# Here and below: It is not meaningful to assume a particular order +# here as the values are sorted before output. It is unpredictable in +# which order they may come out. While this has always been the same +# when compiling for glibc, the new musl build reveals that another +# library may have a different interpretation here. + +@test "Top Clients" { + run bash -c 'echo ">top-clients >quit" | nc -v 127.0.0.1 4711' + printf "%s\n" "${lines[@]}" + [[ ${lines[1]} == "0 15 127.0.0.1 "* ]] + [[ ${lines[2]} == "1 4 127.0.0.3 "* ]] + [[ ${lines[3]} == "2 3 127.0.0.2 "* ]] + [[ ${lines[4]} == "" ]] +} + +@test "Top Domains" { + run bash -c 'echo ">top-domains (20) >quit" | nc -v 127.0.0.1 4711' + printf "%s\n" "${lines[@]}" + [[ "${lines[@]}" == *" 4 regex1.test.pi-hole.net"* ]] + [[ "${lines[@]}" == *" 2 google.com"* ]] + [[ "${lines[@]}" == *" 2 blacklist-blocked.test.pi-hole.net"* ]] + [[ "${lines[@]}" == *" 2 discourse.pi-hole.net"* ]] + [[ "${lines[@]}" == *" 1 version.ftl"* ]] + [[ "${lines[@]}" == *" 1 version.bind"* ]] + [[ "${lines[@]}" == *" 1 whitelisted.test.pi-hole.net"* ]] + [[ "${lines[@]}" == *" 1 regexa.test.pi-hole.net"* ]] + [[ "${lines[@]}" == *" 1 regex2.test.pi-hole.net"* ]] + [[ "${lines[@]}" == *" 1 ftl.pi-hole.net"* ]] + [[ "${lines[11]}" == "" ]] +} + +@test "Top Ads" { + run bash -c 'echo ">top-ads (20) >quit" | nc -v 127.0.0.1 4711' + printf "%s\n" "${lines[@]}" + [[ "${lines[@]}" == *" 2 gravity-blocked.test.pi-hole.net"* ]] + [[ "${lines[@]}" == *" 1 blacklist-blocked.test.pi-hole.net"* ]] + [[ "${lines[@]}" == *" 1 whitelisted.test.pi-hole.net"* ]] + [[ "${lines[@]}" == *" 1 regex5.test.pi-hole.net"* ]] + [[ "${lines[@]}" == *" 1 regex1.test.pi-hole.net"* ]] + [[ ${lines[6]} == "" ]] +} + +@test "Domain auditing, approved domains are not shown" { + run bash -c 'echo ">top-domains for audit >quit" | nc -v 127.0.0.1 4711' + printf "%s\n" "${lines[@]}" + [[ ${lines[@]} != *"google.com"* ]] +} + +@test "Forward Destinations" { + run bash -c 'echo ">forward-dest >quit" | nc -v 127.0.0.1 4711' + printf "%s\n" "${lines[@]}" + [[ ${lines[1]} == "-2 27.27 blocklist blocklist" ]] + [[ ${lines[2]} == "-1 31.82 cache cache" ]] + [[ ${lines[3]} == "0 40.91 "* ]] + [[ ${lines[4]} == "" ]] +} + +@test "Query Types" { + run bash -c 'echo ">querytypes >quit" | nc -v 127.0.0.1 4711' + printf "%s\n" "${lines[@]}" + [[ ${lines[1]} == "A (IPv4): 86.36" ]] + [[ ${lines[2]} == "AAAA (IPv6): 4.55" ]] + [[ ${lines[3]} == "ANY: 0.00" ]] + [[ ${lines[4]} == "SRV: 0.00" ]] + [[ ${lines[5]} == "SOA: 0.00" ]] + [[ ${lines[6]} == "PTR: 0.00" ]] + [[ ${lines[7]} == "TXT: 9.09" ]] + [[ ${lines[8]} == "NAPTR: 0.00" ]] + [[ ${lines[9]} == "" ]] +} + +# Here and below: Acknowledge that there might be a host name after +# the IP address of the client (..."*"...) + +@test "Get all queries" { + run bash -c 'echo ">getallqueries >quit" | nc -v 127.0.0.1 4711' + printf "%s\n" "${lines[@]}" + [[ ${lines[1]} == *"TXT version.ftl "?*" 3 0 6"* ]] + [[ ${lines[2]} == *"TXT version.bind "?*" 3 0 6"* ]] + [[ ${lines[3]} == *"A blacklist-blocked.test.pi-hole.net "?*" 5 0 4"* ]] + [[ ${lines[4]} == *"A gravity-blocked.test.pi-hole.net "?*" 1 0 4"* ]] + [[ ${lines[5]} == *"A gravity-blocked.test.pi-hole.net "?*" 1 0 4"* ]] + [[ ${lines[6]} == *"A whitelisted.test.pi-hole.net "?*" 2 0 4"* ]] + [[ ${lines[7]} == *"A discourse.pi-hole.net "?*" 2 0 4"* ]] + [[ ${lines[8]} == *"A regex5.test.pi-hole.net "?*" 4 0 4"* ]] + [[ ${lines[9]} == *"A regexa.test.pi-hole.net "?*" 2 0 4"* ]] + [[ ${lines[10]} == *"A regex1.test.pi-hole.net "?*" 2 0 4"* ]] + [[ ${lines[11]} == *"A regex2.test.pi-hole.net "?*" 2 0 4"* ]] + [[ ${lines[12]} == *"A whitelisted.test.pi-hole.net 127.0.0.2 1 0 4"* ]] + [[ ${lines[13]} == *"A regex1.test.pi-hole.net 127.0.0.2 4 0 4"* ]] + [[ ${lines[14]} == *"A regex1.test.pi-hole.net 127.0.0.1 3 0 4"* ]] + [[ ${lines[15]} == *"A regex1.test.pi-hole.net 127.0.0.3 3 0 4"* ]] + [[ ${lines[16]} == *"A blacklist-blocked.test.pi-hole.net 127.0.0.2 2 0 4"* ]] + [[ ${lines[17]} == *"A blacklist-blocked.test.pi-hole.net 127.0.0.3 3 0 4"* ]] + [[ ${lines[18]} == *"A regex1.test.pi-hole.net 127.0.0.3 3 0 4"* ]] + [[ ${lines[19]} == *"A discourse.pi-hole.net 127.0.0.3 3 0 4"* ]] + [[ ${lines[20]} == *"A google.com "?*" 2 0 4"* ]] + [[ ${lines[21]} == *"AAAA google.com "?*" 2 0 4"* ]] + [[ ${lines[22]} == *"A ftl.pi-hole.net "?*" 2 0 4"* ]] + [[ ${lines[23]} == "" ]] +} + +@test "Get all queries (domain filtered)" { + run bash -c 'echo ">getallqueries-domain regexa.test.pi-hole.net >quit" | nc -v 127.0.0.1 4711' + printf "%s\n" "${lines[@]}" + [[ ${lines[1]} == *"A regexa.test.pi-hole.net "?*" 2 0 4"* ]] + [[ ${lines[2]} == "" ]] +} + +@test "Get all queries (domain + number filtered)" { + run bash -c 'echo ">getallqueries-domain regexa.test.pi-hole.net (20) >quit" | nc -v 127.0.0.1 4711' + printf "%s\n" "${lines[@]}" + [[ ${lines[1]} == *"A regexa.test.pi-hole.net "?*" 2 0 4"* ]] + [[ ${lines[2]} == "" ]] +} + +@test "Get all queries (client filtered)" { + run bash -c 'echo ">getallqueries-client 127.0.0.1 >quit" | nc -v 127.0.0.1 4711' + printf "%s\n" "${lines[@]}" + [[ ${lines[1]} == *"TXT version.ftl "?*" 3 0 6"* ]] + [[ ${lines[2]} == *"TXT version.bind "?*" 3 0 6"* ]] + [[ ${lines[3]} == *"A blacklist-blocked.test.pi-hole.net "?*" 5 0 4"* ]] + [[ ${lines[4]} == *"A gravity-blocked.test.pi-hole.net "?*" 1 0 4"* ]] + [[ ${lines[5]} == *"A gravity-blocked.test.pi-hole.net "?*" 1 0 4"* ]] + [[ ${lines[6]} == *"A whitelisted.test.pi-hole.net "?*" 2 0 4"* ]] + [[ ${lines[7]} == *"A discourse.pi-hole.net "?*" 2 0 4"* ]] + [[ ${lines[8]} == *"A regex5.test.pi-hole.net "?*" 4 0 4"* ]] + [[ ${lines[9]} == *"A regexa.test.pi-hole.net "?*" 2 0 4"* ]] + [[ ${lines[10]} == *"A regex1.test.pi-hole.net "?*" 2 0 4"* ]] + [[ ${lines[11]} == *"A regex2.test.pi-hole.net "?*" 2 0 4"* ]] + [[ ${lines[12]} == *"A regex1.test.pi-hole.net "?*" 3 0 4"* ]] + [[ ${lines[13]} == *"A google.com "?*" 2 0 4"* ]] + [[ ${lines[14]} == *"AAAA google.com "?*" 2 0 4"* ]] + [[ ${lines[15]} == *"A ftl.pi-hole.net "?*" 2 0 4"* ]] + [[ ${lines[16]} == "" ]] +} + +@test "Get all queries (client + number filtered)" { + run bash -c 'echo ">getallqueries-client 127.0.0.1 (2) >quit" | nc -v 127.0.0.1 4711' + printf "%s\n" "${lines[@]}" + [[ ${lines[1]} == *"AAAA google.com "?*" 2 0 4"* ]] + [[ ${lines[2]} == *"A ftl.pi-hole.net "?*" 2 0 4"* ]] + [[ ${lines[3]} == "" ]] +} + +@test "Recent blocked" { + run bash -c 'echo ">recentBlocked >quit" | nc -v 127.0.0.1 4711' + printf "%s\n" "${lines[@]}" + [[ ${lines[1]} == "regex1.test.pi-hole.net" ]] + [[ ${lines[2]} == "" ]] +} + +@test "pihole-FTL.db schema as expected" { + run bash -c 'sqlite3 /etc/pihole/pihole-FTL.db .dump' + printf "%s\n" "${lines[@]}" + [[ "${lines[@]}" == *"CREATE TABLE queries ( id INTEGER PRIMARY KEY AUTOINCREMENT, timestamp INTEGER NOT NULL, type INTEGER NOT NULL, status INTEGER NOT NULL, domain TEXT NOT NULL, client TEXT NOT NULL, forward TEXT );"* ]] + [[ "${lines[@]}" == *"CREATE TABLE ftl ( id INTEGER PRIMARY KEY NOT NULL, value BLOB NOT NULL );"* ]] + [[ "${lines[@]}" == *"CREATE TABLE counters ( id INTEGER PRIMARY KEY NOT NULL, value INTEGER NOT NULL );"* ]] + [[ "${lines[@]}" == *"CREATE TABLE IF NOT EXISTS \"network\" ( id INTEGER PRIMARY KEY NOT NULL, hwaddr TEXT UNIQUE NOT NULL, interface TEXT NOT NULL, name TEXT, firstSeen INTEGER NOT NULL, lastQuery INTEGER NOT NULL, numQueries INTEGER NOT NULL, macVendor TEXT);"* ]] + [[ "${lines[@]}" == *"CREATE TABLE network_addresses ( network_id INTEGER NOT NULL, ip TEXT NOT NULL, lastSeen INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), UNIQUE(network_id,ip), FOREIGN KEY(network_id) REFERENCES network(id));"* ]] + [[ "${lines[@]}" == *"CREATE INDEX idx_queries_timestamps ON queries (timestamp);"* ]] +} + +@test "Fail on invalid argument" { + run bash -c '/home/pihole/pihole-FTL abc' + printf "%s\n" "${lines[@]}" + [[ ${lines[0]} == "pihole-FTL: invalid option -- 'abc'" ]] + [[ ${lines[1]} == "Try '/home/pihole/pihole-FTL --help' for more information" ]] +} + +@test "Help argument return help text" { + run bash -c '/home/pihole/pihole-FTL help' + printf "%s\n" "${lines[@]}" + [[ ${lines[0]} == "pihole-FTL - The Pi-hole FTL engine" ]] + [[ ${lines[3]} == "Available arguments:" ]] +} + +@test "No WARNING messages in pihole-FTL.log (besides known capability issues)" { + run bash -c 'grep "WARNING:" /var/log/pihole-FTL.log | grep -c -v -E "CAP_NET_ADMIN|CAP_NET_RAW"' + printf "%s\n" "${lines[@]}" + [[ ${lines[0]} == "0" ]] +} + +@test "No ERROR messages in pihole-FTL.log" { + run bash -c 'grep -c "ERROR:" /var/log/pihole-FTL.log' + printf "%s\n" "${lines[@]}" + [[ ${lines[0]} == "0" ]] +} + +@test "No FATAL messages in pihole-FTL.log" { + run bash -c 'grep -c "FATAL:" /var/log/pihole-FTL.log' + printf "%s\n" "${lines[@]}" + [[ ${lines[0]} == "0" ]] +} + +# x86_64-musl is built on busybox which has a slightly different +# variant of ls displaying three, instead of one, spaces between the +# user and group names. + +@test "Ownership and permissions of pihole-FTL.db correct" { + run bash -c 'ls -l /etc/pihole/pihole-FTL.db' + printf "%s\n" "${lines[@]}" + [[ ${lines[0]} == *"pihole pihole"* || ${lines[0]} == *"pihole pihole"* ]] + [[ ${lines[0]} == "-rw-r--r--"* ]] +} + +# "ldd" prints library dependencies and the used interpreter for a given program +# +# Dependencies on shared libraries are displayed like +# libm.so.6 => /lib/x86_64-linux-gnu/libm.so.6 (0x00007fa7d28be000) +# +# In this test, we use ldd and check for the dependency arrow "=>" to check if +# our generated binary depends on shared libraries in the way we expect it to + +@test "Dependence on shared libraries" { + run bash -c 'ldd ./pihole-FTL' + printf "%s\n" "${lines[@]}" + [[ "${STATIC}" != "true" && "${lines[@]}" == *"=>"* ]] || \ + [[ "${STATIC}" == "true" && "${lines[@]}" != *"=>"* ]] +} + +# "file" determines the file type of our generated binary +# +# We use its ability to test whether a specific interpreter is +# required by the given executable. What the interpreter is, is not +# really well documented in "man elf(5)", however, one can say that +# the interpreter is a program that finds and loads the shared +# libraries needed by a program, prepares the program to run, and then +# runs it. +# +# In this test, we use "file" to confirm the absence of the dependence +# on an interpreter for the static binary. + +@test "Dependence on specific interpreter" { + run bash -c 'file ./pihole-FTL' + printf "%s\n" "${lines[@]}" + [[ "${STATIC}" != "true" && "${lines[@]}" == *"interpreter"* ]] || \ + [[ "${STATIC}" == "true" && "${lines[@]}" != *"interpreter"* ]] +} + +@test "Architecture is correctly reported on startup" { + run bash -c 'grep "Compiled for" /var/log/pihole-FTL.log' + printf "Output: %s\n\$CIRCLE_JOB: %s\nuname -m: %s\n" "${lines[@]:-not set}" "${CIRCLE_JOB:-not set}" "$(uname -m)" + [[ ${lines[0]} == *"Compiled for ${CIRCLE_JOB:-$(uname -m)}"* ]] +} + +@test "Building machine (CI) is reported on startup" { + [[ ${CIRCLE_JOB} != "" ]] && compiled_str="on CI" || compiled_str="locally" && export compiled_str + run bash -c 'grep "Compiled for" /var/log/pihole-FTL.log' + printf "Output: %s\n\$CIRCLE_JOB: %s\n" "${lines[@]:-not set}" "${CIRCLE_JOB:-not set}" + [[ ${lines[0]} == *"(compiled ${compiled_str})"* ]] +} + +@test "Compiler version is correctly reported on startup" { + compiler_version="$(${CC} --version | head -n1)" && export compiler_version + run bash -c 'grep "Compiled for" /var/log/pihole-FTL.log' + printf "Output: %s\n\$CC: %s\nVersion: %s\n" "${lines[@]:-not set}" "${CC:-not set}" "${compiler_version:-not set}" + [[ ${lines[0]} == *"using ${compiler_version}"* ]] +} + +@test "No errors on setting busy handlers for the databases" { + run bash -c 'grep -c "Cannot set busy handler" /var/log/pihole-FTL.log' + printf "%s\n" "${lines[@]}" + [[ ${lines[0]} == "0" ]] +} + +@test "Blocking status is correctly logged in pihole.log" { + run bash -c 'grep -c "gravity blocked gravity-blocked.test.pi-hole.net is 0.0.0.0" /var/log/pihole.log' + printf "%s\n" "${lines[@]}" + [[ ${lines[0]} == "2" ]] +} diff --git a/test/test_suite.sh b/test/test_suite.sh deleted file mode 100644 index 451899864..000000000 --- a/test/test_suite.sh +++ /dev/null @@ -1,253 +0,0 @@ -#!./test/libs/bats/bin/bats - -load 'libs/bats-support/load' -# load 'libs/bats-assert/load' - -@test "Version" { - run bash -c 'echo ">version" | nc -v 127.0.0.1 4711' - echo "output: ${lines[@]}" - [[ ${lines[0]} == "Connection to 127.0.0.1 4711 port [tcp/*] succeeded!" ]] - [[ ${lines[1]} =~ "version" ]] - [[ ${lines[2]} =~ "tag" ]] - [[ ${lines[3]} =~ "branch" ]] - [[ ${lines[4]} =~ "date" ]] - [[ ${lines[5]} == "---EOM---" ]] -} - -@test "Statistics" { - run bash -c 'echo ">stats" | nc -v 127.0.0.1 4711' - echo "output: ${lines[@]}" - [[ ${lines[0]} == "Connection to 127.0.0.1 4711 port [tcp/*] succeeded!" ]] - [[ ${lines[1]} =~ "domains_being_blocked -1" ]] - [[ ${lines[2]} =~ "dns_queries_today 7" ]] - [[ ${lines[3]} =~ "ads_blocked_today 2" ]] - [[ ${lines[4]} =~ "ads_percentage_today 28.571428" ]] - [[ ${lines[5]} =~ "unique_domains 6" ]] - [[ ${lines[6]} =~ "queries_forwarded 3" ]] - [[ ${lines[7]} =~ "queries_cached 2" ]] - [[ ${lines[8]} == "clients_ever_seen 3" ]] - [[ ${lines[9]} == "unique_clients 3" ]] - [[ ${lines[10]} == "status unknown" ]] - [[ ${lines[11]} == "---EOM---" ]] -} - -@test "Top Clients (descending, default)" { - run bash -c 'echo ">top-clients" | nc -v 127.0.0.1 4711' - echo "output: ${lines[@]}" - [[ ${lines[0]} == "Connection to 127.0.0.1 4711 port [tcp/*] succeeded!" ]] - [[ ${lines[1]} =~ "0 4 192.168.2.208" ]] - [[ ${lines[2]} =~ "1 2 127.0.0.1" ]] - [[ ${lines[3]} =~ "2 1 10.8.0.2" ]] - [[ ${lines[4]} == "---EOM---" ]] -} - -@test "Top Clients (ascending)" { - run bash -c 'echo ">top-clients asc" | nc -v 127.0.0.1 4711' - echo "output: ${lines[@]}" - [[ ${lines[0]} == "Connection to 127.0.0.1 4711 port [tcp/*] succeeded!" ]] - [[ ${lines[1]} =~ "0 1 10.8.0.2" ]] - [[ ${lines[2]} =~ "1 2 127.0.0.1" ]] - [[ ${lines[3]} =~ "2 4 192.168.2.208" ]] - [[ ${lines[4]} == "---EOM---" ]] -} - -@test "Top Domains (descending, default)" { - run bash -c 'echo ">top-domains" | nc -v 127.0.0.1 4711' - echo "output: ${lines[@]}" - [[ ${lines[0]} == "Connection to 127.0.0.1 4711 port [tcp/*] succeeded!" ]] - [[ ${lines[1]} == "0 2 play.google.com" ]] - [[ ${lines[2]} == "1 1 raspberrypi" ]] - [[ ${lines[3]} == "2 1 checkip.dyndns.org" ]] - [[ ${lines[4]} == "3 1 example.com" ]] - [[ ${lines[5]} == "---EOM---" ]] -} - -@test "Top Domains (ascending)" { - run bash -c 'echo ">top-domains asc" | nc -v 127.0.0.1 4711' - echo "output: ${lines[@]}" - [[ ${lines[0]} == "Connection to 127.0.0.1 4711 port [tcp/*] succeeded!" ]] - [[ ${lines[1]} == "0 1 raspberrypi" ]] - [[ ${lines[2]} == "1 1 checkip.dyndns.org" ]] - [[ ${lines[3]} == "2 1 example.com" ]] - [[ ${lines[4]} == "3 2 play.google.com" ]] - [[ ${lines[5]} == "---EOM---" ]] -} - -@test "Top Ads (descending, default)" { - run bash -c 'echo ">top-ads" | nc -v 127.0.0.1 4711' - echo "output: ${lines[@]}" - [[ ${lines[0]} == "Connection to 127.0.0.1 4711 port [tcp/*] succeeded!" ]] - [[ ${lines[1]} == "0 1 blacklisted.com" ]] - [[ ${lines[2]} == "1 1 addomain.com" ]] - [[ ${lines[3]} == "---EOM---" ]] -} - -@test "Top Ads (ascending)" { - run bash -c 'echo ">top-ads asc" | nc -v 127.0.0.1 4711' - echo "output: ${lines[@]}" - [[ ${lines[0]} == "Connection to 127.0.0.1 4711 port [tcp/*] succeeded!" ]] - [[ ${lines[1]} == "0 1 blacklisted.com" ]] - [[ ${lines[2]} == "1 1 addomain.com" ]] - [[ ${lines[3]} == "---EOM---" ]] -} - -@test "Over Time" { - run bash -c 'echo ">overTime" | nc -v 127.0.0.1 4711' - echo "output: ${lines[@]}" - [[ ${lines[0]} == "Connection to 127.0.0.1 4711 port [tcp/*] succeeded!" ]] - [[ ${lines[1]} =~ "7 2" ]] - [[ ${lines[2]} == "---EOM---" ]] -} - -@test "Forward Destinations" { - run bash -c 'echo ">forward-dest" | nc -v 127.0.0.1 4711' - echo "output: ${lines[@]}" - [[ ${lines[0]} == "Connection to 127.0.0.1 4711 port [tcp/*] succeeded!" ]] - [[ ${lines[1]} =~ "0 57.14 ::1 local" ]] - [[ ${lines[2]} =~ "1 28.57 2001:1608:10:25::9249:d69b" ]] - [[ ${lines[3]} =~ "2 14.29 2620:0:ccd::2 resolver2.ipv6-sandbox.opendns.com" ]] - [[ ${lines[4]} == "---EOM---" ]] -} - -@test "Forward Destinations (unsorted)" { - run bash -c 'echo ">forward-dest unsorted" | nc -v 127.0.0.1 4711' - echo "output: ${lines[@]}" - [[ ${lines[0]} == "Connection to 127.0.0.1 4711 port [tcp/*] succeeded!" ]] - [[ ${lines[1]} =~ "0 28.57 2001:1608:10:25::9249:d69b" ]] - [[ ${lines[2]} =~ "1 14.29 2620:0:ccd::2 resolver2.ipv6-sandbox.opendns.com" ]] - [[ ${lines[3]} =~ "2 57.14 ::1 local" ]] - [[ ${lines[4]} == "---EOM---" ]] -} - -@test "Query Types" { - run bash -c 'echo ">querytypes" | nc -v 127.0.0.1 4711' - echo "output: ${lines[@]}" - [[ ${lines[0]} == "Connection to 127.0.0.1 4711 port [tcp/*] succeeded!" ]] - [[ ${lines[1]} == "A (IPv4): 71.43" ]] - [[ ${lines[2]} == "AAAA (IPv6): 28.57" ]] - [[ ${lines[3]} == "---EOM---" ]] -} - -@test "Get all queries" { - run bash -c 'echo ">getallqueries" | nc -v 127.0.0.1 4711' - echo "output: ${lines[@]}" - [[ ${lines[0]} == "Connection to 127.0.0.1 4711 port [tcp/*] succeeded!" ]] - [[ ${lines[1]} =~ "IPv6 raspberrypi" ]] - [[ ${lines[2]} =~ "IPv4 checkip.dyndns.org" ]] - [[ ${lines[3]} =~ "IPv4 example.com" ]] - [[ ${lines[4]} =~ "IPv4 play.google.com" ]] - [[ ${lines[5]} =~ "IPv6 play.google.com" ]] - [[ ${lines[6]} =~ "IPv4 blacklisted.com" ]] - [[ ${lines[7]} =~ "IPv4 addomain.com" ]] - [[ ${lines[8]} == "---EOM---" ]] -} - -@test "Get all queries (domain filtered)" { - run bash -c 'echo ">getallqueries-domain play.google.com" | nc -v 127.0.0.1 4711' - echo "output: ${lines[@]}" - [[ ${lines[0]} == "Connection to 127.0.0.1 4711 port [tcp/*] succeeded!" ]] - [[ ${lines[1]} =~ "IPv4 play.google.com" ]] - [[ ${lines[2]} =~ "IPv6 play.google.com" ]] - [[ ${lines[3]} == "---EOM---" ]] -} - -@test "Get all queries (domain + number filtered)" { - run bash -c 'echo ">getallqueries-domain play.google.com (3)" | nc -v 127.0.0.1 4711' - echo "output: ${lines[@]}" - [[ ${lines[0]} == "Connection to 127.0.0.1 4711 port [tcp/*] succeeded!" ]] - [[ ${lines[1]} =~ "IPv6 play.google.com" ]] - [[ ${lines[2]} == "---EOM---" ]] -} - -@test "Get all queries (client filtered)" { - run bash -c 'echo ">getallqueries-client 127.0.0.1" | nc -v 127.0.0.1 4711' - echo "output: ${lines[@]}" - [[ ${lines[0]} == "Connection to 127.0.0.1 4711 port [tcp/*] succeeded!" ]] - [[ ${lines[1]} =~ "IPv6 raspberrypi" ]] - [[ ${lines[2]} =~ "IPv4 checkip.dyndns.org" ]] - [[ ${lines[3]} == "---EOM---" ]] -} - -@test "Get all queries (client + number filtered)" { - run bash -c 'echo ">getallqueries-client 127.0.0.1 (6)" | nc -v 127.0.0.1 4711' - echo "output: ${lines[@]}" - [[ ${lines[0]} == "Connection to 127.0.0.1 4711 port [tcp/*] succeeded!" ]] - [[ ${lines[1]} =~ "IPv4 checkip.dyndns.org" ]] - [[ ${lines[2]} == "---EOM---" ]] -} - -@test "Memory" { - run bash -c 'echo ">memory" | nc -v 127.0.0.1 4711' - echo "output: ${lines[@]}" - [[ ${lines[0]} == "Connection to 127.0.0.1 4711 port [tcp/*] succeeded!" ]] - [[ ${lines[1]} =~ "memory allocated for internal data structure:" ]] - [[ ${lines[2]} =~ "dynamically allocated allocated memory used for strings:" ]] - [[ ${lines[3]} =~ "Sum:" ]] - [[ ${lines[4]} == "---EOM---" ]] -} - -@test "Get client ID" { - run bash -c 'echo ">clientID" | nc -v 127.0.0.1 4711' - echo "output: ${lines[@]}" - [[ ${lines[0]} == "Connection to 127.0.0.1 4711 port [tcp/*] succeeded!" ]] - [[ ${lines[2]} == "---EOM---" ]] -} - -@test "Recent blocked" { - run bash -c 'echo ">recentBlocked" | nc -v 127.0.0.1 4711' - echo "output: ${lines[@]}" - [[ ${lines[0]} == "Connection to 127.0.0.1 4711 port [tcp/*] succeeded!" ]] - [[ ${lines[1]} == "addomain.com" ]] - [[ ${lines[2]} == "---EOM---" ]] -} - -# @test "IPv6 socket connection" { -# run bash -c 'echo ">recentBlocked" | nc -v ::1 4711' -# echo "output: ${lines[@]}" -# [[ ${lines[0]} == "Connection to ::1 4711 port [tcp/*] succeeded!" ]] -# [[ ${lines[1]} == "addomain.com" ]] -# [[ ${lines[2]} == "---EOM---" ]] -# } - -@test "DB test: Tables created and populated?" { - run bash -c 'sqlite3 pihole-FTL.db .dump' - echo "output: ${lines[@]}" - [[ "${lines[@]}" == *"CREATE TABLE queries ( id INTEGER PRIMARY KEY AUTOINCREMENT, timestamp INTEGER NOT NULL, type INTEGER NOT NULL, status INTEGER NOT NULL, domain TEXT NOT NULL, client TEXT NOT NULL, forward TEXT );"* ]] - [[ "${lines[@]}" == *"CREATE TABLE ftl ( id INTEGER PRIMARY KEY NOT NULL, value BLOB NOT NULL );"* ]] - [[ "${lines[@]}" == *"CREATE TABLE counters ( id INTEGER PRIMARY KEY NOT NULL, value INTEGER NOT NULL );"* ]] - [[ "${lines[@]}" == *"INSERT INTO \"counters\" VALUES(0,0);"* ]] - [[ "${lines[@]}" == *"INSERT INTO \"counters\" VALUES(1,0);"* ]] - [[ "${lines[@]}" == *"INSERT INTO \"ftl\" VALUES(0,2);"* ]] - [[ "${lines[@]}" == *"CREATE INDEX idx_queries_timestamps ON queries (timestamp);"* ]] -} - -@test "Arguments check: Invalid option" { - run bash -c './pihole-FTL abc' - echo "output: ${lines[@]}" - [[ ${lines[0]} == "pihole-FTL: invalid option -- 'abc'" ]] - [[ ${lines[1]} == "Try './pihole-FTL --help' for more information" ]] -} - -@test "Help argument return help text" { - run bash -c './pihole-FTL help' - echo "output: ${lines[@]}" - [[ ${lines[0]} == "pihole-FTL - The Pi-hole FTL engine" ]] -} - -@test "Unix socket returning data" { - run bash -c './socket-test travis' - echo "output: ${lines[@]}" - [[ ${lines[0]} == "Socket created" ]] - [[ ${lines[1]} == "Connection established" ]] - [[ ${lines[2]} == "d2 ff ff ff ff d2 00 00 00 07 d2 00 00 00 02 ca 41 e4 92 49 d2 00 00 00 06 d2 00 00 00 03 d2 00 00 00 02 d2 00 00 00 03 d2 00 00 00 03 cc 02 c1 " ]] -} - -@test "Verify no FATAL warnings are present in the generated log" { - run bash -c 'grep -c "FATAL" pihole-FTL.log' - echo "output: ${lines[@]}" - [[ ${lines[0]} == "0" ]] -} - -@test "Final part of the tests: Killing pihole-FTL process" { - run bash -c 'kill $(pidof pihole-FTL)' -} diff --git a/aux/macvendor.py b/tools/macvendor.py similarity index 98% rename from aux/macvendor.py rename to tools/macvendor.py index 344dab3e7..3dc557530 100644 --- a/aux/macvendor.py +++ b/tools/macvendor.py @@ -29,7 +29,7 @@ line = line.strip() # Skip comments and empty lines - if line[1] == "#" or line == "": + if line == "" or line[0] == "#": continue # Remove quotation marks as these might interfere with later INSERT / UPDATE commands diff --git a/socket_client.c b/tools/socket_client.c similarity index 92% rename from socket_client.c rename to tools/socket_client.c index 5f6c7e176..96434dac7 100644 --- a/socket_client.c +++ b/tools/socket_client.c @@ -49,10 +49,6 @@ int main (int argc, char **argv) { command = argv[i]; continue; } - - // Set socket file location (respect special location on the CI system Travis) - if(strcmp(argv[i], "travis") == 0) - strcpy(address.sun_path,"pihole-FTL.sock"); } // Connect to the socket provided by pihole-FTL