diff --git a/CMakeLists.txt b/CMakeLists.txt index e2df948ac6..798ecbde32 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -324,13 +324,13 @@ if(NOT WIN32 AND NOT APPLE) set(CURL_LIBRARIES "${CURL_BUNDLE_DIR}/lib/.libs/libcurl.a") if(NOT USE_BUNDLED_OPENSSL) - set(CURL_SSL_OPTION "") + set(CURL_SSL_OPTION "--with-ssl") else() set(CURL_SSL_OPTION "--with-ssl=${OPENSSL_INSTALL_DIR}") + message(STATUS "Using bundled curl in '${CURL_BUNDLE_DIR}'") + message(STATUS "Using SSL for curl in '${CURL_SSL_OPTION}'") endif() - message(STATUS "Using bundled curl in '${CURL_BUNDLE_DIR}'") - message(STATUS "Using SSL for curl in '${CURL_SSL_OPTION}'") ExternalProject_Add(curl DEPENDS openssl diff --git a/docker/dev/Dockerfile b/docker/dev/Dockerfile index f6a6e43a82..bad1257318 100644 --- a/docker/dev/Dockerfile +++ b/docker/dev/Dockerfile @@ -24,7 +24,7 @@ RUN echo "deb http://httpredir.debian.org/debian jessie main" > /etc/apt/sources gcc \ gcc-5 \ gcc-4.9 \ - libelf1 && rm -rf /var/lib/apt/lists/* + libelf1 less && rm -rf /var/lib/apt/lists/* # Terribly terrible hacks: since our base Debian image ships with GCC 5.0 which breaks older kernels, # revert the default to gcc-4.9. Also, since some customers use some very old distributions whose kernel diff --git a/docker/stable/Dockerfile b/docker/stable/Dockerfile index 4018be8586..4c29e11adc 100644 --- a/docker/stable/Dockerfile +++ b/docker/stable/Dockerfile @@ -24,7 +24,7 @@ RUN echo "deb http://httpredir.debian.org/debian jessie main" > /etc/apt/sources gcc \ gcc-5 \ gcc-4.9 \ - libelf1 && rm -rf /var/lib/apt/lists/* + libelf1 less && rm -rf /var/lib/apt/lists/* # Terribly terrible hacks: since our base Debian image ships with GCC 5.0 which breaks older kernels, # revert the default to gcc-4.9. Also, since some customers use some very old distributions whose kernel diff --git a/driver/main.c b/driver/main.c index a9229ac812..c274b8ff78 100644 --- a/driver/main.c +++ b/driver/main.c @@ -1627,7 +1627,11 @@ TRACEPOINT_PROBE(syscall_enter_probe, struct pt_regs *regs, long id) * If this is a 32bit process running on a 64bit kernel (see the CONFIG_IA32_EMULATION * kernel flag), we switch to the ia32 syscall table. */ +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 9, 0) + if(in_ia32_syscall()) { +#else if (unlikely(task_thread_info(current)->status & TS_COMPAT)) { +#endif cur_g_syscall_table = g_syscall_ia32_table; cur_g_syscall_code_routing_table = g_syscall_ia32_code_routing_table; socketcall_syscall = __NR_ia32_socketcall; @@ -1689,7 +1693,11 @@ TRACEPOINT_PROBE(syscall_exit_probe, struct pt_regs *regs, long ret) * use 64bit syscall table. On 32bit __NR_execve is equal to __NR_ia32_oldolduname * which is a very old syscall, not used anymore by most applications */ +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 9, 0) + if(in_ia32_syscall() && id != __NR_execve) { +#else if (unlikely((task_thread_info(current)->status & TS_COMPAT) && id != __NR_execve)) { +#endif cur_g_syscall_table = g_syscall_ia32_table; cur_g_syscall_code_routing_table = g_syscall_ia32_code_routing_table; socketcall_syscall = __NR_ia32_socketcall; diff --git a/driver/ppm.h b/driver/ppm.h index ba446adc60..b9cd626db6 100644 --- a/driver/ppm.h +++ b/driver/ppm.h @@ -122,6 +122,8 @@ long ppm_strncpy_from_user(char *to, const char __user *from, unsigned long n); #define SYSCALL_TABLE_ID0 0 #elif defined CONFIG_PPC64 #define SYSCALL_TABLE_ID0 0 +#elif defined CONFIG_S390 + #define SYSCALL_TABLE_ID0 0 #endif #define SYSCALL_TABLE_SIZE 512 diff --git a/driver/ppm_events.c b/driver/ppm_events.c index f648258cef..715ca29ef7 100644 --- a/driver/ppm_events.c +++ b/driver/ppm_events.c @@ -705,61 +705,6 @@ int val_to_ring(struct event_filler_arguments *args, uint64_t val, u16 val_len, return PPM_SUCCESS; } -/* - * Get the current working directory for the current process. - * Returns the pointer to the string, which is NOT going to be at the beginning - * of buf. - * Buf must be at least 1 page in size. - */ -#if LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 20) -char *npm_getcwd(char *buf, unsigned long bufsize) -{ - struct path pwd; - char *res; - - ASSERT(bufsize >= PAGE_SIZE - 1); - -#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 36) || defined CONFIG_VE - get_fs_pwd(current->fs, &pwd); -#else - read_lock(¤t->fs->lock); - pwd = current->fs->pwd; - path_get(&pwd); - read_unlock(¤t->fs->lock); -#endif - - res = d_path(&pwd, buf, bufsize); - - if (IS_ERR(res)) - res = NULL; - - path_put(&pwd); - - return res; -} -#else /* LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 20) */ -char *npm_getcwd(char *buf, unsigned long bufsize) -{ - struct dentry *dentry; - struct vfsmount *mnt; - char *res; - - ASSERT(bufsize >= PAGE_SIZE - 1); - - read_lock(¤t->fs->lock); - mnt = mntget(current->fs->pwdmnt); - dentry = dget(current->fs->pwd); - read_unlock(¤t->fs->lock); - - res = d_path(dentry, mnt, buf, bufsize); - - if (IS_ERR(res)) - res = NULL; - - return res; -} -#endif - static inline u8 socket_family_to_scap(u8 family) { if (family == AF_INET) diff --git a/driver/ppm_events.h b/driver/ppm_events.h index e902cae256..d1f4494a08 100644 --- a/driver/ppm_events.h +++ b/driver/ppm_events.h @@ -129,7 +129,6 @@ extern const struct ppm_event_entry g_ppm_events[]; int32_t dpi_lookahead_init(void); int32_t f_sys_autofill(struct event_filler_arguments *args, const struct ppm_event_entry *evinfo); int32_t val_to_ring(struct event_filler_arguments *args, u64 val, u16 val_len, bool fromuser, u8 dyn_idx); -char *npm_getcwd(char *buf, unsigned long bufsize); u16 pack_addr(struct sockaddr *usrsockaddr, int ulen, char *targetbuf, u16 targetbufsize); u16 fd_to_socktuple(int fd, struct sockaddr *usrsockaddr, int ulen, bool use_userdata, bool is_inbound, char *targetbuf, u16 targetbufsize); int addr_to_kernel(void __user *uaddr, int ulen, struct sockaddr *kaddr); diff --git a/driver/ppm_fillers.c b/driver/ppm_fillers.c index ef673c718d..5bf28db920 100644 --- a/driver/ppm_fillers.c +++ b/driver/ppm_fillers.c @@ -812,7 +812,7 @@ static int append_cgroup(const char *subsys_name, int subsys_id, char *buf, int int subsys_len; char *path; -#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 15, 0) +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 15, 0) || LINUX_VERSION_CODE >= KERNEL_VERSION(4, 9, 0) int res; #endif @@ -832,7 +832,17 @@ static int append_cgroup(const char *subsys_name, int subsys_id, char *buf, int return 1; } -#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 15, 0) +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 9, 0) + // According to https://github.com/torvalds/linux/commit/4c737b41de7f4eef2a593803bad1b918dd718b10 + // cgroup_path now returns an int again + res = cgroup_path(css->cgroup, buf, *available); + if (res < 0) { + ASSERT(false); + path = "NA"; + } else { + path = buf; + } +#elif LINUX_VERSION_CODE >= KERNEL_VERSION(3, 15, 0) path = cgroup_path(css->cgroup, buf, *available); if (!path) { ASSERT(false); @@ -1006,7 +1016,7 @@ static int f_proc_startupdate(struct event_filler_arguments *args) struct mm_struct *mm = current->mm; int64_t retval; int ptid; - char *spwd; + char *spwd = ""; long total_vm = 0; long total_rss = 0; long swap = 0; @@ -1149,14 +1159,9 @@ static int f_proc_startupdate(struct event_filler_arguments *args) return res; /* - * cwd + * cwd, pushed empty to avoid breaking compatibility + * with the older event format */ - spwd = npm_getcwd(args->str_storage, STR_STORAGE_SIZE - 1); - if (spwd == NULL) - spwd = ""; - - args->str_storage[STR_STORAGE_SIZE - 1] = '\0'; - res = val_to_ring(args, (uint64_t)(long)spwd, 0, false, 0); if (unlikely(res != PPM_SUCCESS)) return res; @@ -3890,14 +3895,22 @@ static inline u16 ptrace_requests_to_scap(unsigned long req) case PTRACE_GETFPXREGS: return PPM_PTRACE_GETFPXREGS; #endif +#ifdef PTRACE_SETFPREGS case PTRACE_SETFPREGS: return PPM_PTRACE_SETFPREGS; +#endif +#ifdef PTRACE_GETFPREGS case PTRACE_GETFPREGS: return PPM_PTRACE_GETFPREGS; +#endif +#ifdef PTRACE_SETREGS case PTRACE_SETREGS: return PPM_PTRACE_SETREGS; +#endif +#ifdef PTRACE_GETREGS case PTRACE_GETREGS: return PPM_PTRACE_GETREGS; +#endif #ifdef PTRACE_SETSIGMASK case PTRACE_SETSIGMASK: return PPM_PTRACE_SETSIGMASK; diff --git a/driver/syscall_table.c b/driver/syscall_table.c index d354fed0f8..7231c7f62a 100644 --- a/driver/syscall_table.c +++ b/driver/syscall_table.c @@ -288,7 +288,9 @@ const enum ppm_syscall_code g_syscall_code_routing_table[SYSCALL_TABLE_SIZE] = { [__NR_link - SYSCALL_TABLE_ID0] = PPM_SC_LINK, [__NR_unlink - SYSCALL_TABLE_ID0] = PPM_SC_UNLINK, [__NR_chdir - SYSCALL_TABLE_ID0] = PPM_SC_CHDIR, +#ifdef __NR_time [__NR_time - SYSCALL_TABLE_ID0] = PPM_SC_TIME, +#endif [__NR_mknod - SYSCALL_TABLE_ID0] = PPM_SC_MKNOD, [__NR_chmod - SYSCALL_TABLE_ID0] = PPM_SC_CHMOD, /* [__NR_lchown16 - SYSCALL_TABLE_ID0] = PPM_SC_NR_LCHOWN16, */ diff --git a/scripts/build-probe-binaries b/scripts/build-probe-binaries index 82c31717fd..28ba887e6c 100755 --- a/scripts/build-probe-binaries +++ b/scripts/build-probe-binaries @@ -17,6 +17,8 @@ PROBE_VERSION=$2 REPOSITORY_NAME=$3 BASEDIR=$(pwd) ARCH=$(uname -m) +URL_TIMEOUT=300 +RETRY=10 if [ ! -d $BASEDIR/output ]; then mkdir $BASEDIR/output @@ -140,7 +142,7 @@ function coreos_build_old { cd $COREOS_DIR if [ ! -f config_orig ]; then - wget ${VERSION_URL}coreos_developer_container.bin.bz2 + wget --timeout=${URL_TIMEOUT} --tries=${RETRY} ${VERSION_URL}coreos_developer_container.bin.bz2 bunzip2 coreos_developer_container.bin.bz2 sudo kpartx -asv coreos_developer_container.bin LOOPDEV=$(sudo kpartx -asv coreos_developer_container.bin | cut -d\ -f 3) @@ -162,7 +164,7 @@ function coreos_build_old { KERNEL_URL=https://www.kernel.org/pub/linux/kernel/v${MAJOR}.x/$TGZ_NAME if [ ! -f $TGZ_NAME ]; then - wget $KERNEL_URL + wget --timeout=${URL_TIMEOUT} --tries=${RETRY} $KERNEL_URL fi if [ ! -d $DIR_NAME ]; then @@ -197,7 +199,7 @@ function coreos_build_new { cd $COREOS_DIR if [ ! -f coreos_developer_container.bin ]; then - wget ${VERSION_URL}coreos_developer_container.bin.bz2 + wget --timeout=${URL_TIMEOUT} --tries=${RETRY} ${VERSION_URL}coreos_developer_container.bin.bz2 bunzip2 coreos_developer_container.bin.bz2 fi sudo kpartx -asv coreos_developer_container.bin @@ -245,7 +247,7 @@ function boot2docker_build { if [ ! -f $TGZ_NAME ]; then echo Downloading $TGZ_NAME [Boot2Docker] - wget $KERNEL_URL + wget --timeout=${URL_TIMEOUT} --tries=${RETRY} $KERNEL_URL fi if [ ! -d $DIR_NAME ]; then @@ -268,7 +270,7 @@ function boot2docker_build { ; do \ patch -p1 < "$patch"; \ done - wget -O .config $KERNEL_CONFIG + wget --timeout=${URL_TIMEOUT} --tries=${RETRY} -O .config $KERNEL_CONFIG cp .config ../config-orig make olddefconfig make modules_prepare @@ -307,7 +309,7 @@ function ubuntu_build { if [ ! -f $DEB ]; then echo Downloading $DEB [Ubuntu] - wget $URL + wget --timeout=${URL_TIMEOUT} --tries=${RETRY} $URL dpkg -x $DEB ./ fi @@ -349,7 +351,7 @@ function rhel_build { if [ ! -f $RPM ]; then echo Downloading $RPM [RHEL and CentOS] - wget $URL + wget --timeout=${URL_TIMEOUT} --tries=${RETRY} $URL rpm2cpio $RPM | cpio -idm fi @@ -387,7 +389,7 @@ function debian_build { fi if [ ! -f ${BASEDIR}/common-dependencies/debian/kbuild/${DEB} ]; then echo Downloading ${DEB} [Ubuntu] - wget -P ${BASEDIR}/common-dependencies/debian/kbuild ${URL} + wget --timeout=${URL_TIMEOUT} --tries=${RETRY} -P ${BASEDIR}/common-dependencies/debian/kbuild ${URL} fi return else @@ -409,7 +411,7 @@ function debian_build { if [ ! -f ${DEB} ]; then echo Downloading ${DEB} [Ubuntu] - wget ${URL} + wget --timeout=${URL_TIMEOUT} --tries=${RETRY} ${URL} dpkg -x ${DEB} ./ fi fi diff --git a/scripts/install-sysdig.in b/scripts/install-sysdig.in index 1d95336aac..66010740fa 100644 --- a/scripts/install-sysdig.in +++ b/scripts/install-sysdig.in @@ -27,18 +27,18 @@ function install_rpm { if ! yum -q list dkms > /dev/null 2>&1; then echo "* Installing EPEL repository (for DKMS)" if [ $VERSION -eq 7 ] && [ $DISTRO = "centos" ]; then - rpm --quiet -i http://mirrors.kernel.org/centos/7/extras/x86_64/Packages/epel-release-7-5.noarch.rpm + rpm --quiet -i https://mirrors.kernel.org/centos/7/extras/x86_64/Packages/epel-release-7-5.noarch.rpm elif [ $VERSION -eq 7 ]; then - rpm --quiet -i http://mirrors.kernel.org/fedora-epel/7/x86_64/e/epel-release-7-8.noarch.rpm + rpm --quiet -i https://mirrors.kernel.org/fedora-epel/7/x86_64/e/epel-release-7-8.noarch.rpm else - rpm --quiet -i http://mirrors.kernel.org/fedora-epel/6/i386/epel-release-6-8.noarch.rpm + rpm --quiet -i https://mirrors.kernel.org/fedora-epel/6/i386/epel-release-6-8.noarch.rpm fi fi echo "* Installing _COMPONENT_ public key" rpm --quiet --import https://s3.amazonaws.com/download.draios.com/DRAIOS-GPG-KEY.public echo "* Installing _COMPONENT_ repository" - curl -s -o /etc/yum.repos.d/draios.repo http://download.draios.com/_REPOSITORY_NAME_/rpm/draios.repo + curl -s -o /etc/yum.repos.d/draios.repo https://s3.amazonaws.com/download.draios.com/_REPOSITORY_NAME_/rpm/draios.repo echo "* Installing kernel headers" KERNEL_VERSION=$(uname -r) if [[ $KERNEL_VERSION == *PAE* ]]; then @@ -66,7 +66,7 @@ function install_deb { echo "* Installing Sysdig public key" curl -s https://s3.amazonaws.com/download.draios.com/DRAIOS-GPG-KEY.public | apt-key add - echo "* Installing _COMPONENT_ repository" - curl -s -o /etc/apt/sources.list.d/draios.list http://download.draios.com/_REPOSITORY_NAME_/deb/draios.list + curl -s -o /etc/apt/sources.list.d/draios.list https://s3.amazonaws.com/download.draios.com/_REPOSITORY_NAME_/deb/draios.list apt-get -qq update < /dev/null echo "* Installing kernel headers" apt-get -qq -y install linux-headers-$(uname -r) < /dev/null || kernel_warning diff --git a/scripts/kernel-crawler.py b/scripts/kernel-crawler.py index 0aa7501269..a5350d0335 100755 --- a/scripts/kernel-crawler.py +++ b/scripts/kernel-crawler.py @@ -182,6 +182,7 @@ # code. # urls = set() +URL_TIMEOUT=30 if len(sys.argv) < 2 or not sys.argv[1] in repos: sys.stderr.write("Usage: " + sys.argv[0] + " \n") @@ -192,28 +193,21 @@ # patterns given. Save the result in `packages`. # for repo in repos[sys.argv[1]]: - - root = urllib2.urlopen(repo["root"]).read() - versions = html.fromstring(root).xpath(repo["discovery_pattern"], namespaces = {"regex": "http://exslt.org/regular-expressions"}) - - for version in versions: - for subdir in repo["subdirs"]: - - # The try - except block is used because 404 errors and similar - # might happen (and actually happen because not all repos have - # packages we need) - try: - source = repo["root"] + version + subdir - page = urllib2.urlopen(source).read() - rpms = html.fromstring(page).xpath(repo["page_pattern"], namespaces = {"regex": "http://exslt.org/regular-expressions"}) - - for rpm in rpms: - if "exclude_patterns" in repo and any(x in rpm for x in repo["exclude_patterns"]): - continue - else: - urls.add(source + str(urllib2.unquote(rpm))) - except: - continue + try: + root = urllib2.urlopen(repo["root"],timeout=URL_TIMEOUT).read() + versions = html.fromstring(root).xpath(repo["discovery_pattern"], namespaces = {"regex": "http://exslt.org/regular-expressions"}) + for version in versions: + for subdir in repo["subdirs"]: + source = repo["root"] + version + subdir + page = urllib2.urlopen(source,timeout=URL_TIMEOUT).read() + rpms = html.fromstring(page).xpath(repo["page_pattern"], namespaces = {"regex": "http://exslt.org/regular-expressions"}) + for rpm in rpms: + if "exclude_patterns" in repo and any(x in rpm for x in repo["exclude_patterns"]): + continue + else: + urls.add(source + str(urllib2.unquote(rpm))) + except: + continue # # Print URLs to stdout diff --git a/test/sysdig_trace_regression.sh b/test/sysdig_trace_regression.sh index 05c2d27677..9bb0fc0db7 100755 --- a/test/sysdig_trace_regression.sh +++ b/test/sysdig_trace_regression.sh @@ -104,6 +104,8 @@ $BASEDIR/sysdig_batch_parser.sh $SYSDIG $CHISELS "-cps" $TRACEDIR $RESULTDIR/ps $BASEDIR/sysdig_batch_parser.sh $SYSDIG $CHISELS "-j -n 10000" $TRACEDIR $RESULTDIR/fd_fields_json $BASELINEDIR/fd_fields_json || ret=1 # Sessions $BASEDIR/sysdig_batch_parser.sh $SYSDIG $CHISELS "-p '*%evt.num %evt.outputtime %evt.cpu %proc.name (%thread.tid) %evt.dir %evt.type %evt.info sid=%proc.sid sname=%proc.sname'" $TRACEDIR $RESULTDIR/sessions $BASELINEDIR/sessions || ret=1 +# Cwd +$BASEDIR/sysdig_batch_parser.sh $SYSDIG $CHISELS "-pc -p\"*%evt.num %evt.outputtime %evt.cpu %container.name (%container.id) %proc.name (%thread.tid:%thread.vtid) %evt.dir %evt.type %evt.info %proc.cwd\"" $TRACEDIR $RESULTDIR/cwd $BASELINEDIR/cwd || ret=1 rm -rf "${TMPBASE}" exit $ret diff --git a/userspace/libscap/scap-int.h b/userspace/libscap/scap-int.h index e12e8df540..f2dd4d1099 100644 --- a/userspace/libscap/scap-int.h +++ b/userspace/libscap/scap-int.h @@ -97,6 +97,7 @@ struct scap void* m_proc_callback_context; struct ppm_proclist_info* m_driver_procinfo; bool refresh_proc_table_when_saving; + uint64_t m_unexpected_block_readsize; }; struct scap_ns_socket_list diff --git a/userspace/libscap/scap.c b/userspace/libscap/scap.c index 84f5344a06..b62f426195 100644 --- a/userspace/libscap/scap.c +++ b/userspace/libscap/scap.c @@ -284,7 +284,8 @@ scap_t* scap_open_offline_int(const char* fname, char *error, proc_entry_callback proc_callback, void* proc_callback_context, - bool import_users) + bool import_users, + uint64_t start_offset) { scap_t* handle = NULL; @@ -334,6 +335,14 @@ scap_t* scap_open_offline_int(const char* fname, return NULL; } + // + // If this is a merged file, we might have to move the read offset to the next section + // + if(start_offset != 0) + { + scap_fseek(handle, start_offset); + } + // // Validate the file and load the non-event blocks // @@ -368,7 +377,7 @@ scap_t* scap_open_offline_int(const char* fname, scap_t* scap_open_offline(const char* fname, char *error) { - return scap_open_offline_int(fname, error, NULL, NULL, true); + return scap_open_offline_int(fname, error, NULL, NULL, true, 0); } scap_t* scap_open_live(char *error) @@ -382,7 +391,8 @@ scap_t* scap_open(scap_open_args args, char *error) { return scap_open_offline_int(args.fname, error, args.proc_callback, args.proc_callback_context, - args.import_users); + args.import_users, + args.start_offset); } else { @@ -1264,3 +1274,8 @@ void scap_set_refresh_proc_table_when_saving(scap_t* handle, bool refresh) { handle->refresh_proc_table_when_saving = refresh; } + +uint64_t scap_get_unexpected_block_readsize(scap_t* handle) +{ + return handle->m_unexpected_block_readsize; +} diff --git a/userspace/libscap/scap.h b/userspace/libscap/scap.h index cb50257eb4..067dc2e360 100644 --- a/userspace/libscap/scap.h +++ b/userspace/libscap/scap.h @@ -69,6 +69,7 @@ typedef struct ppm_evt_hdr scap_evt; #define SCAP_NOTFOUND 4 #define SCAP_INPUT_TOO_SMALL 5 #define SCAP_EOF 6 +#define SCAP_UNEXPECTED_BLOCK 7 // // Last error string size for scap_open_live() @@ -236,6 +237,7 @@ typedef struct scap_open_args proc_entry_callback proc_callback; ///< Callback to be invoked for each thread/fd that is extracted from /proc, or NULL if no callback is needed. void* proc_callback_context; ///< Opaque pointer that will be included in the calls to proc_callback. Ignored if proc_callback is NULL. bool import_users; ///< true if the user list should be created when opening the capture. + uint64_t start_offset; ///< Used to start reading a capture file from an arbitrary offset. This is leveraged when opening merged files. }scap_open_args; @@ -871,6 +873,8 @@ void scap_set_refresh_proc_table_when_saving(scap_t* handle, bool refresh); uint64_t scap_ftell(scap_t *handle); void scap_fseek(scap_t *handle, uint64_t off); int32_t scap_enable_tracers_capture(scap_t* handle); +uint64_t scap_get_unexpected_block_readsize(scap_t* handle); +int32_t scap_proc_add(scap_t* handle, uint64_t tid, scap_threadinfo* tinfo); #ifdef __cplusplus } diff --git a/userspace/libscap/scap_procs.c b/userspace/libscap/scap_procs.c index 0683e5161c..647554133b 100644 --- a/userspace/libscap/scap_procs.c +++ b/userspace/libscap/scap_procs.c @@ -973,6 +973,21 @@ void scap_proc_free(scap_t* handle, struct scap_threadinfo* proc) free(proc); } +int32_t scap_proc_add(scap_t* handle, uint64_t tid, scap_threadinfo* tinfo) +{ + int32_t uth_status = SCAP_SUCCESS; + + HASH_ADD_INT64(handle->m_proclist, tid, tinfo); + if(uth_status == SCAP_SUCCESS) + { + return SCAP_SUCCESS; + } + else + { + return SCAP_FAILURE; + } +} + // // Internal helper functions to output the process table to screen // diff --git a/userspace/libscap/scap_savefile.c b/userspace/libscap/scap_savefile.c index 4d992768c7..c9a17d2225 100755 --- a/userspace/libscap/scap_savefile.c +++ b/userspace/libscap/scap_savefile.c @@ -2087,7 +2087,8 @@ int32_t scap_next_offline(scap_t *handle, OUT scap_evt **pevent, OUT uint16_t *p bh.block_type != EVF_BLOCK_TYPE) { snprintf(handle->m_lasterr, SCAP_LASTERR_SIZE, "unexpected block type %u", (uint32_t)bh.block_type); - return SCAP_FAILURE; + handle->m_unexpected_block_readsize = readsize; + return SCAP_UNEXPECTED_BLOCK; } if(bh.block_total_length < sizeof(bh) + sizeof(struct ppm_evt_hdr) + 4) diff --git a/userspace/libsinsp/CMakeLists.txt b/userspace/libsinsp/CMakeLists.txt index 9530922f6b..a777310a14 100644 --- a/userspace/libsinsp/CMakeLists.txt +++ b/userspace/libsinsp/CMakeLists.txt @@ -29,6 +29,7 @@ add_library(sinsp STATIC fdinfo.cpp filter.cpp filterchecks.cpp + http_parser.c ifinfo.cpp json_query.cpp k8s.cpp @@ -73,6 +74,7 @@ add_library(sinsp STATIC table.cpp sinsp_auth.cpp sinsp_curl.cpp + stopwatch.cpp uri_parser.c uri.cpp utils.cpp @@ -94,12 +96,19 @@ if(NOT WIN32) "${JQ_LIB}" "${B64_LIB}" "${CURL_LIBRARIES}" - "${OPENSSL_LIBRARY_SSL}" - "${OPENSSL_LIBRARY_CRYPTO}" rt anl) endif() + if(USE_BUNDLED_OPENSSL) + target_link_libraries(sinsp + "${OPENSSL_LIBRARY_SSL}" + "${OPENSSL_LIBRARY_CRYPTO}") + else() + target_link_libraries(sinsp + "${OPENSSL_LIBRARIES}") + endif() + target_link_libraries(sinsp "${LUAJIT_LIB}" dl diff --git a/userspace/libsinsp/chisel_api.cpp b/userspace/libsinsp/chisel_api.cpp index 902454222f..3de0c83e31 100644 --- a/userspace/libsinsp/chisel_api.cpp +++ b/userspace/libsinsp/chisel_api.cpp @@ -284,7 +284,7 @@ int lua_cbacks::request_field(lua_State *ls) throw sinsp_exception("chisel error"); } - chk->parse_field_name(fld, true); + chk->parse_field_name(fld, true, false); lua_pushlightuserdata(ls, chk); diff --git a/userspace/libsinsp/container.cpp b/userspace/libsinsp/container.cpp index f4aa158d36..01eff82ee0 100644 --- a/userspace/libsinsp/container.cpp +++ b/userspace/libsinsp/container.cpp @@ -204,7 +204,7 @@ bool sinsp_container_manager::set_mesos_task_id(sinsp_container_info* container, else { g_logger.log("Mesos task ID not found for Mesos container [" + container->m_id + "]," - "thread [" + std::to_string(tinfo->m_tid) + ']', sinsp_logger::SEV_WARNING); + "thread [" + std::to_string(tinfo->m_tid) + ']', sinsp_logger::SEV_DEBUG); } } } @@ -660,14 +660,14 @@ bool sinsp_container_manager::parse_docker(sinsp_container_info* container) } const Json::Value& env_vars = config_obj["Env"]; - string mesos_task_id = get_mesos_task_id(env_vars, "MESOS_TASK_ID"); + string mesos_task_id = get_docker_env(env_vars, "MESOS_TASK_ID"); if(mesos_task_id.empty()) { - mesos_task_id = get_mesos_task_id(env_vars, "mesos_task_id"); + mesos_task_id = get_docker_env(env_vars, "mesos_task_id"); } if(mesos_task_id.empty()) { - mesos_task_id = get_mesos_task_id(env_vars, "MESOS_EXECUTOR_ID"); + mesos_task_id = get_docker_env(env_vars, "MESOS_EXECUTOR_ID"); } if(!mesos_task_id.empty()) { @@ -697,20 +697,23 @@ bool sinsp_container_manager::parse_docker(sinsp_container_info* container) sinsp_container_info::parse_json_mounts(root["Mounts"], container->m_mounts); +#ifdef HAS_ANALYZER + container->m_sysdig_agent_conf = get_docker_env(env_vars, "SYSDIG_AGENT_CONF"); +#endif return true; } -string sinsp_container_manager::get_mesos_task_id(const Json::Value& env_vars, const string& mti) +string sinsp_container_manager::get_docker_env(const Json::Value &env_vars, const string &mti) { - string mesos_task_id; + string ret; for(const auto& env_var : env_vars) { if(env_var.isString()) { - mesos_task_id = env_var.asString(); - if((mesos_task_id.length() > (mti.length() + 1)) && (mesos_task_id.substr(0, mti.length()) == mti)) + ret = env_var.asString(); + if((ret.length() > (mti.length() + 1)) && (ret.substr(0, mti.length()) == mti)) { - return mesos_task_id.substr(mti.length() + 1); + return ret.substr(mti.length() + 1); } } } diff --git a/userspace/libsinsp/container.h b/userspace/libsinsp/container.h index 0e181c9567..5ad343aef8 100644 --- a/userspace/libsinsp/container.h +++ b/userspace/libsinsp/container.h @@ -48,8 +48,8 @@ class sinsp_container_info class container_mount_info { public: - container_mount_info(): - m_source(""), + container_mount_info(): + m_source(""), m_dest(""), m_mode(""), m_rdwr(false), @@ -129,6 +129,9 @@ class sinsp_container_info int64_t m_cpu_shares; int64_t m_cpu_quota; int64_t m_cpu_period; +#ifdef HAS_ANALYZER + string m_sysdig_agent_conf; +#endif }; class sinsp_container_manager @@ -151,7 +154,7 @@ class sinsp_container_manager string container_to_json(const sinsp_container_info& container_info); bool container_to_sinsp_event(const string& json, sinsp_evt* evt); bool parse_docker(sinsp_container_info* container); - string get_mesos_task_id(const Json::Value& env_vars, const string& mti); + string get_docker_env(const Json::Value &env_vars, const string &mti); bool parse_rkt(sinsp_container_info* container, const string& podid, const string& appname); sinsp_container_info* get_container(const string& id); diff --git a/userspace/libsinsp/docker.cpp b/userspace/libsinsp/docker.cpp index 1ad2a4d7a6..6f87f28039 100644 --- a/userspace/libsinsp/docker.cpp +++ b/userspace/libsinsp/docker.cpp @@ -124,7 +124,6 @@ docker::docker(std::string url, } m_event_http = std::make_shared(*this, "docker", url, path, http_version, timeout_ms); m_event_http->set_json_callback(&docker::set_event_json); - m_event_http->set_json_end("}\n"); m_event_http->add_json_filter("."); m_collector.add(m_event_http); m_collector.set_steady_state(true); diff --git a/userspace/libsinsp/dumper.cpp b/userspace/libsinsp/dumper.cpp index 69875521df..2e7c71ebca 100644 --- a/userspace/libsinsp/dumper.cpp +++ b/userspace/libsinsp/dumper.cpp @@ -35,13 +35,20 @@ sinsp_dumper::~sinsp_dumper() } } -void sinsp_dumper::open(const string& filename, bool compress) +void sinsp_dumper::open(const string& filename, bool compress, bool create_scap_table) { if(m_inspector->m_h == NULL) { throw sinsp_exception("can't start event dump, inspector not opened yet"); } + /* + if(create_scap_table) + { + m_inspector->m_thread_manager->to_scap(); + } + */ + if(compress) { m_dumper = scap_dump_open(m_inspector->m_h, filename.c_str(), SCAP_COMPRESSION_GZIP); diff --git a/userspace/libsinsp/dumper.h b/userspace/libsinsp/dumper.h index feb2cd8752..741b032aba 100644 --- a/userspace/libsinsp/dumper.h +++ b/userspace/libsinsp/dumper.h @@ -52,7 +52,7 @@ class SINSP_PUBLIC sinsp_dumper \note There's no close() because the file is closed when the dumper is destroyed. */ - void open(const string& filename, bool compress); + void open(const string& filename, bool compress, bool create_scap_table=false); /*! \brief Return the current size of a tracefile. diff --git a/userspace/libsinsp/eventformatter.cpp b/userspace/libsinsp/eventformatter.cpp index 7edd822cf2..70ed74be13 100644 --- a/userspace/libsinsp/eventformatter.cpp +++ b/userspace/libsinsp/eventformatter.cpp @@ -128,8 +128,8 @@ void sinsp_evt_formatter::set_format(const string& fmt) } } - sinsp_filter_check* chk = g_filterlist.new_filter_check_from_fldname(string(cfmt + j + 1), - m_inspector, + sinsp_filter_check* chk = g_filterlist.new_filter_check_from_fldname(string(cfmt + j + 1), + m_inspector, false); if(chk == NULL) @@ -139,7 +139,7 @@ void sinsp_evt_formatter::set_format(const string& fmt) m_chks_to_free.push_back(chk); - j += chk->parse_field_name(cfmt + j + 1, true); + j += chk->parse_field_name(cfmt + j + 1, true, false); ASSERT(j <= lfmt.length()); m_tokens.push_back(chk); @@ -151,7 +151,9 @@ void sinsp_evt_formatter::set_format(const string& fmt) if(last_nontoken_str_start != j) { - m_tokens.push_back(new rawstring_check(lfmt.substr(last_nontoken_str_start, j - last_nontoken_str_start))); + sinsp_filter_check * chk = new rawstring_check(lfmt.substr(last_nontoken_str_start, j - last_nontoken_str_start)); + m_tokens.push_back(chk); + m_chks_to_free.push_back(chk); m_tokenlens.push_back(0); } } @@ -206,12 +208,12 @@ bool sinsp_evt_formatter::tostring(sinsp_evt* evt, OUT string* res) fi = m_tokens[j]->get_field_info(); - if(fi) + if(fi) { m_root[fi->m_name] = m_tokens[j]->tojson(evt); - } - } - else + } + } + else { char* str = m_tokens[j]->tostring(evt); @@ -220,14 +222,14 @@ bool sinsp_evt_formatter::tostring(sinsp_evt* evt, OUT string* res) continue; } - if(str == NULL) + if(str == NULL) { if(m_require_all_values) { retval = false; continue; } - else + else { str = (char*)""; } @@ -254,13 +256,13 @@ bool sinsp_evt_formatter::tostring(sinsp_evt* evt, OUT string* res) || m_inspector->get_buffer_format() == sinsp_evt::PF_JSONHEXASCII || m_inspector->get_buffer_format() == sinsp_evt::PF_JSONBASE64) { - if(m_first) + if(m_first) { // Give it the opening stanza of a JSON array (*res) = '['; m_first = false; - } - else + } + else { // Otherwise say this is another object in an // existing JSON array diff --git a/userspace/libsinsp/filter.cpp b/userspace/libsinsp/filter.cpp index 5644216a86..2eae5086e2 100644 --- a/userspace/libsinsp/filter.cpp +++ b/userspace/libsinsp/filter.cpp @@ -78,7 +78,9 @@ sinsp_filter_check_list::sinsp_filter_check_list() add_filter_check(new sinsp_filter_check_container()); add_filter_check(new sinsp_filter_check_utils()); add_filter_check(new sinsp_filter_check_fdlist()); +#ifndef HAS_ANALYZER add_filter_check(new sinsp_filter_check_k8s()); +#endif // HAS_ANALYZER add_filter_check(new sinsp_filter_check_mesos()); add_filter_check(new sinsp_filter_check_tracer()); add_filter_check(new sinsp_filter_check_evtin()); @@ -119,7 +121,7 @@ sinsp_filter_check* sinsp_filter_check_list::new_filter_check_from_fldname(const { m_check_list[j]->m_inspector = inspector; - int32_t fldnamelen = m_check_list[j]->parse_field_name(name.c_str(), false); + int32_t fldnamelen = m_check_list[j]->parse_field_name(name.c_str(), false, true); if(fldnamelen != -1) { @@ -970,10 +972,11 @@ Json::Value sinsp_filter_check::tojson(sinsp_evt* evt) return jsonval; } -int32_t sinsp_filter_check::parse_field_name(const char* str, bool alloc_state) +int32_t sinsp_filter_check::parse_field_name(const char* str, bool alloc_state, bool needed_for_filtering) { int32_t j; int32_t max_fldlen = -1; + uint32_t max_flags = 0; ASSERT(m_info.m_fields != NULL); ASSERT(m_info.m_nfields != -1); @@ -994,10 +997,19 @@ int32_t sinsp_filter_check::parse_field_name(const char* str, bool alloc_state) m_field_id = j; m_field = &m_info.m_fields[j]; max_fldlen = fldlen; + max_flags = (m_info.m_fields[j]).m_flags; } } } + if(!needed_for_filtering) + { + if(max_flags & EPF_FILTER_ONLY) + { + throw sinsp_exception(string(str) + " is filter only and cannot be used as a display field"); + } + } + return max_fldlen; } @@ -1012,7 +1024,7 @@ int32_t sinsp_filter_check::get_check_id() } -void sinsp_filter_check::add_filter_value(const char* str, uint32_t len, uint16_t i) +void sinsp_filter_check::add_filter_value(const char* str, uint32_t len, uint32_t i) { if (i >= m_val_storages.size()) @@ -1664,7 +1676,7 @@ void sinsp_filter_compiler::parse_check() chk->m_boolop = op; chk->m_cmpop = co; - chk->parse_field_name((char *)&operand1[0], true); + chk->parse_field_name((char *)&operand1[0], true, true); if(co == CO_IN || co == CO_PMATCH) { @@ -1697,7 +1709,7 @@ void sinsp_filter_compiler::parse_check() // // Create the 'or' sequence // - uint64_t num_values = 0; + uint32_t num_values = 0; while(true) { // 'in' clause aware @@ -1750,7 +1762,7 @@ void sinsp_filter_compiler::parse_check() // // Create the 'or' sequence // - uint64_t num_values = 0; + uint32_t num_values = 0; while(true) { // 'in' clause aware diff --git a/userspace/libsinsp/filter_value.h b/userspace/libsinsp/filter_value.h index f718ca867b..67c780b0e7 100644 --- a/userspace/libsinsp/filter_value.h +++ b/userspace/libsinsp/filter_value.h @@ -34,14 +34,13 @@ struct g_hash_membuf { size_t operator()(filter_value_t val) const { -#ifdef __GNUC__ +#if defined(__GNUC__) && !defined(__clang__) return std::_Hash_impl::hash(val.first, val.second); #else size_t hash = 5381; - for(uint8_t *p = val.first; p-val.first < val.second; p++) + for(uint8_t *p = val.first; (uint32_t)(p-val.first) < val.second; p++) { int c = *p; - hash = ((hash << 5) + hash) + c; /* hash * 33 + c */ } return hash; diff --git a/userspace/libsinsp/filterchecks.cpp b/userspace/libsinsp/filterchecks.cpp index f4b477a212..29b5b89d91 100644 --- a/userspace/libsinsp/filterchecks.cpp +++ b/userspace/libsinsp/filterchecks.cpp @@ -1396,7 +1396,7 @@ int32_t sinsp_filter_check_thread::extract_arg(string fldname, string val, OUT c return parsed_len; } -int32_t sinsp_filter_check_thread::parse_field_name(const char* str, bool alloc_state) +int32_t sinsp_filter_check_thread::parse_field_name(const char* str, bool alloc_state, bool needed_for_filtering) { string val(str); @@ -1461,7 +1461,7 @@ int32_t sinsp_filter_check_thread::parse_field_name(const char* str, bool alloc_ m_th_state_id = m_inspector->reserve_thread_memory(sizeof(uint64_t)); } - return sinsp_filter_check::parse_field_name(str, alloc_state); + return sinsp_filter_check::parse_field_name(str, alloc_state, needed_for_filtering); } else if(string(val, 0, sizeof("thread.cgroup") - 1) == "thread.cgroup" && string(val, 0, sizeof("thread.cgroups") - 1) != "thread.cgroups") @@ -1478,11 +1478,11 @@ int32_t sinsp_filter_check_thread::parse_field_name(const char* str, bool alloc_ m_th_state_id = m_inspector->reserve_thread_memory(sizeof(uint64_t)); } - return sinsp_filter_check::parse_field_name(str, alloc_state); + return sinsp_filter_check::parse_field_name(str, alloc_state, needed_for_filtering); } else { - return sinsp_filter_check::parse_field_name(str, alloc_state); + return sinsp_filter_check::parse_field_name(str, alloc_state, needed_for_filtering); } } @@ -2428,7 +2428,7 @@ int32_t sinsp_filter_check_event::extract_type(string fldname, string val, OUT c return parsed_len; } -int32_t sinsp_filter_check_event::parse_field_name(const char* str, bool alloc_state) +int32_t sinsp_filter_check_event::parse_field_name(const char* str, bool alloc_state, bool needed_for_filtering) { string val(str); int32_t res = 0; @@ -2475,7 +2475,7 @@ int32_t sinsp_filter_check_event::parse_field_name(const char* str, bool alloc_s m_th_state_id = m_inspector->reserve_thread_memory(sizeof(uint16_t)); } - res = sinsp_filter_check::parse_field_name(str, alloc_state); + res = sinsp_filter_check::parse_field_name(str, alloc_state, needed_for_filtering); } else if(string(val, 0, sizeof("evt.abspath") - 1) == "evt.abspath") { @@ -2510,7 +2510,7 @@ int32_t sinsp_filter_check_event::parse_field_name(const char* str, bool alloc_s } else { - res = sinsp_filter_check::parse_field_name(str, alloc_state); + res = sinsp_filter_check::parse_field_name(str, alloc_state, needed_for_filtering); } return res; @@ -4157,7 +4157,7 @@ int32_t sinsp_filter_check_tracer::extract_arg(string fldname, string val, OUT c return parsed_len; } -int32_t sinsp_filter_check_tracer::parse_field_name(const char* str, bool alloc_state) +int32_t sinsp_filter_check_tracer::parse_field_name(const char* str, bool alloc_state, bool needed_for_filtering) { int32_t res; string val(str); @@ -4219,7 +4219,7 @@ int32_t sinsp_filter_check_tracer::parse_field_name(const char* str, bool alloc_ } else { - res = sinsp_filter_check::parse_field_name(str, alloc_state); + res = sinsp_filter_check::parse_field_name(str, alloc_state, needed_for_filtering); } if(m_field_id == TYPE_DURATION || @@ -4650,34 +4650,34 @@ uint8_t* sinsp_filter_check_tracer::extract(sinsp_evt *evt, OUT uint32_t* len, b /////////////////////////////////////////////////////////////////////////////// const filtercheck_field_info sinsp_filter_check_evtin_fields[] = { - { PT_INT64, EPF_FILTER_ONLY, PF_ID, "evtin.span.id", "accepts all the events that are between the enter and exit tracers of the spans with the given ID and are generated by the same thread that generated the tracers." }, - { PT_UINT32, EPF_FILTER_ONLY, PF_DEC, "evtin.span.ntags", "accepts all the events that are between the enter and exit tracers of the spans with the given number of tags and are generated by the same thread that generated the tracers." }, - { PT_UINT32, EPF_FILTER_ONLY, PF_DEC, "evtin.span.nargs", "accepts all the events that are between the enter and exit tracers of the spans with the given number of arguments and are generated by the same thread that generated the tracers." }, - { PT_CHARBUF, EPF_FILTER_ONLY, PF_NA, "evtin.span.tags", "accepts all the events that are between the enter and exit tracers of the spans with the given tags and are generated by the same thread that generated the tracers." }, - { PT_CHARBUF, EPF_FILTER_ONLY, PF_NA, "evtin.span.tag", "accepts all the events that are between the enter and exit tracers of the spans with the given tag and are generated by the same thread that generated the tracers. See the description of span.tag for information about the syntax accepted by this field." }, - { PT_CHARBUF, EPF_FILTER_ONLY, PF_NA, "evtin.span.args", "accepts all the events that are between the enter and exit tracers of the spans with the given arguments and are generated by the same thread that generated the tracers." }, - { PT_CHARBUF, EPF_FILTER_ONLY, PF_NA, "evtin.span.arg", "accepts all the events that are between the enter and exit tracers of the spans with the given argument and are generated by the same thread that generated the tracers. See the description of span.arg for information about the syntax accepted by this field." }, - { PT_INT64, EPF_FILTER_ONLY, PF_ID, "evtin.span.p.id", "same as evtin.span.id, but also accepts events generated by other threads in the same process that produced the span." }, - { PT_UINT32, EPF_FILTER_ONLY, PF_DEC, "evtin.span.p.ntags", "same as evtin.span.ntags, but also accepts events generated by other threads in the same process that produced the span." }, - { PT_UINT32, EPF_FILTER_ONLY, PF_DEC, "evtin.span.p.nargs", "same as evtin.span.nargs, but also accepts events generated by other threads in the same process that produced the span." }, - { PT_CHARBUF, EPF_FILTER_ONLY, PF_NA, "evtin.span.p.tags", "same as evtin.span.tags, but also accepts events generated by other threads in the same process that produced the span." }, - { PT_CHARBUF, EPF_FILTER_ONLY, PF_NA, "evtin.span.p.tag", "same as evtin.span.tag, but also accepts events generated by other threads in the same process that produced the span." }, - { PT_CHARBUF, EPF_FILTER_ONLY, PF_NA, "evtin.span.p.args", "same as evtin.span.args, but also accepts events generated by other threads in the same process that produced the span." }, - { PT_CHARBUF, EPF_FILTER_ONLY, PF_NA, "evtin.span.p.arg", "same as evtin.span.arg, but also accepts events generated by other threads in the same process that produced the span." }, - { PT_INT64, EPF_FILTER_ONLY, PF_ID, "evtin.span.s.id", "same as evtin.span.id, but also accepts events generated by the script that produced the span, i.e. by the processes whose parent PID is the same as the one of the process generating the span." }, - { PT_UINT32, EPF_FILTER_ONLY, PF_DEC, "evtin.span.s.ntags", "same as evtin.span.id, but also accepts events generated by the script that produced the span, i.e. by the processes whose parent PID is the same as the one of the process generating the span." }, - { PT_UINT32, EPF_FILTER_ONLY, PF_DEC, "evtin.span.s.nargs", "same as evtin.span.id, but also accepts events generated by the script that produced the span, i.e. by the processes whose parent PID is the same as the one of the process generating the span." }, - { PT_CHARBUF, EPF_FILTER_ONLY, PF_NA, "evtin.span.s.tags", "same as evtin.span.id, but also accepts events generated by the script that produced the span, i.e. by the processes whose parent PID is the same as the one of the process generating the span." }, - { PT_CHARBUF, EPF_FILTER_ONLY, PF_NA, "evtin.span.s.tag", "same as evtin.span.id, but also accepts events generated by the script that produced the span, i.e. by the processes whose parent PID is the same as the one of the process generating the span." }, - { PT_CHARBUF, EPF_FILTER_ONLY, PF_NA, "evtin.span.s.args", "same as evtin.span.id, but also accepts events generated by the script that produced the span, i.e. by the processes whose parent PID is the same as the one of the process generating the span." }, - { PT_CHARBUF, EPF_FILTER_ONLY, PF_NA, "evtin.span.s.arg", "same as evtin.span.id, but also accepts events generated by the script that produced the span, i.e. by the processes whose parent PID is the same as the one of the process generating the span." }, - { PT_INT64, EPF_FILTER_ONLY, PF_ID, "evtin.span.m.id", "same as evtin.span.id, but accepts all the events generated on the machine during the span, including other threads and other processes." }, - { PT_UINT32, EPF_FILTER_ONLY, PF_DEC, "evtin.span.m.ntags", "same as evtin.span.id, but accepts all the events generated on the machine during the span, including other threads and other processes." }, - { PT_UINT32, EPF_FILTER_ONLY, PF_DEC, "evtin.span.m.nargs", "same as evtin.span.id, but accepts all the events generated on the machine during the span, including other threads and other processes." }, - { PT_CHARBUF, EPF_FILTER_ONLY, PF_NA, "evtin.span.m.tags", "same as evtin.span.id, but accepts all the events generated on the machine during the span, including other threads and other processes." }, - { PT_CHARBUF, EPF_FILTER_ONLY, PF_NA, "evtin.span.m.tag", "same as evtin.span.id, but accepts all the events generated on the machine during the span, including other threads and other processes." }, - { PT_CHARBUF, EPF_FILTER_ONLY, PF_NA, "evtin.span.m.args", "same as evtin.span.id, but accepts all the events generated on the machine during the span, including other threads and other processes." }, - { PT_CHARBUF, EPF_FILTER_ONLY, PF_NA, "evtin.span.m.arg", "same as evtin.span.id, but accepts all the events generated on the machine during the span, including other threads and other processes." }, + { PT_INT64, EPF_NONE, PF_ID, "evtin.span.id", "accepts all the events that are between the enter and exit tracers of the spans with the given ID and are generated by the same thread that generated the tracers." }, + { PT_UINT32, EPF_NONE, PF_DEC, "evtin.span.ntags", "accepts all the events that are between the enter and exit tracers of the spans with the given number of tags and are generated by the same thread that generated the tracers." }, + { PT_UINT32, EPF_NONE, PF_DEC, "evtin.span.nargs", "accepts all the events that are between the enter and exit tracers of the spans with the given number of arguments and are generated by the same thread that generated the tracers." }, + { PT_CHARBUF, EPF_NONE, PF_NA, "evtin.span.tags", "accepts all the events that are between the enter and exit tracers of the spans with the given tags and are generated by the same thread that generated the tracers." }, + { PT_CHARBUF, EPF_NONE, PF_NA, "evtin.span.tag", "accepts all the events that are between the enter and exit tracers of the spans with the given tag and are generated by the same thread that generated the tracers. See the description of span.tag for information about the syntax accepted by this field." }, + { PT_CHARBUF, EPF_NONE, PF_NA, "evtin.span.args", "accepts all the events that are between the enter and exit tracers of the spans with the given arguments and are generated by the same thread that generated the tracers." }, + { PT_CHARBUF, EPF_NONE, PF_NA, "evtin.span.arg", "accepts all the events that are between the enter and exit tracers of the spans with the given argument and are generated by the same thread that generated the tracers. See the description of span.arg for information about the syntax accepted by this field." }, + { PT_INT64, EPF_NONE, PF_ID, "evtin.span.p.id", "same as evtin.span.id, but also accepts events generated by other threads in the same process that produced the span." }, + { PT_UINT32, EPF_NONE, PF_DEC, "evtin.span.p.ntags", "same as evtin.span.ntags, but also accepts events generated by other threads in the same process that produced the span." }, + { PT_UINT32, EPF_NONE, PF_DEC, "evtin.span.p.nargs", "same as evtin.span.nargs, but also accepts events generated by other threads in the same process that produced the span." }, + { PT_CHARBUF, EPF_NONE, PF_NA, "evtin.span.p.tags", "same as evtin.span.tags, but also accepts events generated by other threads in the same process that produced the span." }, + { PT_CHARBUF, EPF_NONE, PF_NA, "evtin.span.p.tag", "same as evtin.span.tag, but also accepts events generated by other threads in the same process that produced the span." }, + { PT_CHARBUF, EPF_NONE, PF_NA, "evtin.span.p.args", "same as evtin.span.args, but also accepts events generated by other threads in the same process that produced the span." }, + { PT_CHARBUF, EPF_NONE, PF_NA, "evtin.span.p.arg", "same as evtin.span.arg, but also accepts events generated by other threads in the same process that produced the span." }, + { PT_INT64, EPF_NONE, PF_ID, "evtin.span.s.id", "same as evtin.span.id, but also accepts events generated by the script that produced the span, i.e. by the processes whose parent PID is the same as the one of the process generating the span." }, + { PT_UINT32, EPF_NONE, PF_DEC, "evtin.span.s.ntags", "same as evtin.span.id, but also accepts events generated by the script that produced the span, i.e. by the processes whose parent PID is the same as the one of the process generating the span." }, + { PT_UINT32, EPF_NONE, PF_DEC, "evtin.span.s.nargs", "same as evtin.span.id, but also accepts events generated by the script that produced the span, i.e. by the processes whose parent PID is the same as the one of the process generating the span." }, + { PT_CHARBUF, EPF_NONE, PF_NA, "evtin.span.s.tags", "same as evtin.span.id, but also accepts events generated by the script that produced the span, i.e. by the processes whose parent PID is the same as the one of the process generating the span." }, + { PT_CHARBUF, EPF_NONE, PF_NA, "evtin.span.s.tag", "same as evtin.span.id, but also accepts events generated by the script that produced the span, i.e. by the processes whose parent PID is the same as the one of the process generating the span." }, + { PT_CHARBUF, EPF_NONE, PF_NA, "evtin.span.s.args", "same as evtin.span.id, but also accepts events generated by the script that produced the span, i.e. by the processes whose parent PID is the same as the one of the process generating the span." }, + { PT_CHARBUF, EPF_NONE, PF_NA, "evtin.span.s.arg", "same as evtin.span.id, but also accepts events generated by the script that produced the span, i.e. by the processes whose parent PID is the same as the one of the process generating the span." }, + { PT_INT64, EPF_NONE, PF_ID, "evtin.span.m.id", "same as evtin.span.id, but accepts all the events generated on the machine during the span, including other threads and other processes." }, + { PT_UINT32, EPF_NONE, PF_DEC, "evtin.span.m.ntags", "same as evtin.span.id, but accepts all the events generated on the machine during the span, including other threads and other processes." }, + { PT_UINT32, EPF_NONE, PF_DEC, "evtin.span.m.nargs", "same as evtin.span.id, but accepts all the events generated on the machine during the span, including other threads and other processes." }, + { PT_CHARBUF, EPF_NONE, PF_NA, "evtin.span.m.tags", "same as evtin.span.id, but accepts all the events generated on the machine during the span, including other threads and other processes." }, + { PT_CHARBUF, EPF_NONE, PF_NA, "evtin.span.m.tag", "same as evtin.span.id, but accepts all the events generated on the machine during the span, including other threads and other processes." }, + { PT_CHARBUF, EPF_NONE, PF_NA, "evtin.span.m.args", "same as evtin.span.id, but accepts all the events generated on the machine during the span, including other threads and other processes." }, + { PT_CHARBUF, EPF_NONE, PF_NA, "evtin.span.m.arg", "same as evtin.span.id, but accepts all the events generated on the machine during the span, including other threads and other processes." }, }; sinsp_filter_check_evtin::sinsp_filter_check_evtin() @@ -4750,7 +4750,7 @@ int32_t sinsp_filter_check_evtin::extract_arg(string fldname, string val) return parsed_len; } -int32_t sinsp_filter_check_evtin::parse_field_name(const char* str, bool alloc_state) +int32_t sinsp_filter_check_evtin::parse_field_name(const char* str, bool alloc_state, bool needed_for_filtering) { int32_t res; string val(str); @@ -4829,7 +4829,7 @@ int32_t sinsp_filter_check_evtin::parse_field_name(const char* str, bool alloc_s } else { - res = sinsp_filter_check::parse_field_name(str, alloc_state); + res = sinsp_filter_check::parse_field_name(str, alloc_state, needed_for_filtering); } return res; @@ -4840,12 +4840,7 @@ sinsp_filter_check* sinsp_filter_check_evtin::allocate_new() return (sinsp_filter_check*) new sinsp_filter_check_evtin(); } -uint8_t* sinsp_filter_check_evtin::extract(sinsp_evt *evt, OUT uint32_t* len, bool sanitize_strings) -{ - return NULL; -} - -inline bool sinsp_filter_check_evtin::compare_tracer(sinsp_evt *evt, sinsp_partial_tracer* pae) +inline uint8_t* sinsp_filter_check_evtin::extract_tracer(sinsp_evt *evt, sinsp_partial_tracer* pae) { ASSERT(pae); uint32_t field_id = m_field_id; @@ -4857,7 +4852,7 @@ inline bool sinsp_filter_check_evtin::compare_tracer(sinsp_evt *evt, sinsp_parti // if(static_cast(pae->m_tid) != evt->get_thread_info()->m_tid) { - return false; + return NULL; } } else if(field_id >= TYPE_P_ID && field_id <= TYPE_P_ARG) @@ -4871,12 +4866,12 @@ inline bool sinsp_filter_check_evtin::compare_tracer(sinsp_evt *evt, sinsp_parti { if(tinfo->m_tid != evt->get_thread_info()->m_tid) { - return false; + return NULL; } } else { - return false; + return NULL; } field_id -= TYPE_P_ID; @@ -4892,12 +4887,12 @@ inline bool sinsp_filter_check_evtin::compare_tracer(sinsp_evt *evt, sinsp_parti { if(tinfo->m_pid != evt->get_thread_info()->m_ptid) { - return false; + return NULL; } } else { - return false; + return NULL; } field_id -= TYPE_S_ID; @@ -4910,39 +4905,13 @@ inline bool sinsp_filter_check_evtin::compare_tracer(sinsp_evt *evt, sinsp_parti switch(field_id) { case TYPE_ID: - if(flt_compare(m_cmpop, PT_UINT64, - &pae->m_id) == true) - { - return true; - } - else - { - return false; - } + return (uint8_t*)&pae->m_id; case TYPE_NTAGS: m_u32val = (uint32_t)pae->m_tags.size(); - - if(flt_compare(m_cmpop, PT_UINT32, - &m_u32val) == true) - { - return true; - } - else - { - return false; - } + return (uint8_t*)&m_u32val; case TYPE_NARGS: m_u32val = (uint32_t)pae->m_argvals.size(); - - if(flt_compare(m_cmpop, PT_UINT32, - &m_u32val) == true) - { - return true; - } - else - { - return false; - } + return (uint8_t*)&m_u32val; case TYPE_TAGS: { vector::iterator it; @@ -4963,7 +4932,7 @@ inline bool sinsp_filter_check_evtin::compare_tracer(sinsp_evt *evt, sinsp_parti { memcpy(p, *it, (*sit)); p += (*sit); - *p++ = ','; + *p++ = '.'; } if(p != m_storage) @@ -4975,15 +4944,7 @@ inline bool sinsp_filter_check_evtin::compare_tracer(sinsp_evt *evt, sinsp_parti *p = 0; } - if(flt_compare(m_cmpop, PT_CHARBUF, - m_storage) == true) - { - return true; - } - else - { - return false; - } + return (uint8_t*)m_storage; } case TYPE_TAG: { @@ -5006,20 +4967,7 @@ inline bool sinsp_filter_check_evtin::compare_tracer(sinsp_evt *evt, sinsp_parti } } - if(val == NULL) - { - return false; - } - - if(flt_compare(m_cmpop, PT_CHARBUF, - val) == true) - { - return true; - } - else - { - return false; - } + return (uint8_t*) val; } case TYPE_ARGS: { @@ -5063,15 +5011,7 @@ inline bool sinsp_filter_check_evtin::compare_tracer(sinsp_evt *evt, sinsp_parti *p = 0; } - if(flt_compare(m_cmpop, PT_CHARBUF, - m_storage) == true) - { - return true; - } - else - { - return false; - } + return (uint8_t*)m_storage; } case TYPE_ARG: { @@ -5118,27 +5058,72 @@ inline bool sinsp_filter_check_evtin::compare_tracer(sinsp_evt *evt, sinsp_parti } } - if(val == NULL) - { - return false; - } - - if(flt_compare(m_cmpop, PT_CHARBUF, - val) == true) - { - return true; - } - else - { - return false; - } + return (uint8_t*) val; } default: ASSERT(false); break; } - return false; + return NULL; +} + +uint8_t* sinsp_filter_check_evtin::extract(sinsp_evt *evt, OUT uint32_t* len, bool sanitize_strings) +{ + list* partial_tracers_list = &m_inspector->m_partial_tracers_list; + list::iterator it; + uint16_t etype = evt->get_type(); + + // + // Tracer events are excluded + // + if(etype == PPME_TRACER_E || etype == PPME_TRACER_X) + { + return NULL; + } + + // + // Events without thread information are excluded + // + sinsp_threadinfo* tinfo = evt->get_thread_info(); + if(tinfo == NULL || tinfo->m_tracer_parser == NULL) + { + return NULL; + } + + // + // Scan the list and see if there's a match + // + for(it = partial_tracers_list->begin(); it != partial_tracers_list->end(); ++it) + { + uint8_t* res = extract_tracer(evt, *it); + if(res != NULL) + { + return res; + } + } + + return NULL; +} + +inline bool sinsp_filter_check_evtin::compare_tracer(sinsp_evt *evt, sinsp_partial_tracer* pae) +{ + uint8_t* res = extract_tracer(evt, pae); + + if(res == NULL) + { + return false; + } + + if(flt_compare(m_cmpop, m_info.m_fields[m_field_id].m_type, + res) == true) + { + return true; + } + else + { + return false; + } } bool sinsp_filter_check_evtin::compare(sinsp_evt *evt) @@ -5239,7 +5224,7 @@ void rawstring_check::set_text(string text) m_text = text; } -int32_t rawstring_check::parse_field_name(const char* str, bool alloc_state) +int32_t rawstring_check::parse_field_name(const char* str, bool alloc_state, bool needed_for_filtering) { ASSERT(false); return -1; @@ -5276,9 +5261,9 @@ sinsp_filter_check* sinsp_filter_check_syslog::allocate_new() return (sinsp_filter_check*) new sinsp_filter_check_syslog(); } -int32_t sinsp_filter_check_syslog::parse_field_name(const char* str, bool alloc_state) +int32_t sinsp_filter_check_syslog::parse_field_name(const char* str, bool alloc_state, bool needed_for_filtering) { - int32_t res = sinsp_filter_check::parse_field_name(str, alloc_state); + int32_t res = sinsp_filter_check::parse_field_name(str, alloc_state, needed_for_filtering); if(res != -1) { m_decoder = (sinsp_decoder_syslog*)m_inspector->require_protodecoder("syslog"); @@ -5379,7 +5364,7 @@ int32_t sinsp_filter_check_container::extract_arg(const string &val, size_t base return end+1; } -int32_t sinsp_filter_check_container::parse_field_name(const char* str, bool alloc_state) +int32_t sinsp_filter_check_container::parse_field_name(const char* str, bool alloc_state, bool needed_for_filtering) { string val(str); int32_t res = 0; @@ -5429,7 +5414,7 @@ int32_t sinsp_filter_check_container::parse_field_name(const char* str, bool all } else { - res = sinsp_filter_check::parse_field_name(str, alloc_state); + res = sinsp_filter_check::parse_field_name(str, alloc_state, needed_for_filtering); } return res; @@ -5762,7 +5747,7 @@ sinsp_filter_check* sinsp_filter_check_reference::allocate_new() return NULL; } -int32_t sinsp_filter_check_reference::parse_field_name(const char* str, bool alloc_state) +int32_t sinsp_filter_check_reference::parse_field_name(const char* str, bool alloc_state, bool needed_for_filtering) { ASSERT(false); return -1; @@ -6316,6 +6301,8 @@ uint8_t* sinsp_filter_check_fdlist::extract(sinsp_evt *evt, OUT uint32_t* len, b } } +#ifndef HAS_ANALYZER + /////////////////////////////////////////////////////////////////////////////// // sinsp_filter_check_k8s implementation /////////////////////////////////////////////////////////////////////////////// @@ -6360,7 +6347,7 @@ sinsp_filter_check* sinsp_filter_check_k8s::allocate_new() return (sinsp_filter_check*) new sinsp_filter_check_k8s(); } -int32_t sinsp_filter_check_k8s::parse_field_name(const char* str, bool alloc_state) +int32_t sinsp_filter_check_k8s::parse_field_name(const char* str, bool alloc_state, bool needed_for_filtering) { string val(str); @@ -6414,7 +6401,7 @@ int32_t sinsp_filter_check_k8s::parse_field_name(const char* str, bool alloc_sta } else { - return sinsp_filter_check::parse_field_name(str, alloc_state); + return sinsp_filter_check::parse_field_name(str, alloc_state, needed_for_filtering); } } @@ -6883,6 +6870,8 @@ uint8_t* sinsp_filter_check_k8s::extract(sinsp_evt *evt, OUT uint32_t* len, bool return NULL; } +#endif // HAS_ANALYZER + /////////////////////////////////////////////////////////////////////////////// // sinsp_filter_check_mesos implementation /////////////////////////////////////////////////////////////////////////////// @@ -6915,7 +6904,7 @@ sinsp_filter_check* sinsp_filter_check_mesos::allocate_new() return (sinsp_filter_check*) new sinsp_filter_check_mesos(); } -int32_t sinsp_filter_check_mesos::parse_field_name(const char* str, bool alloc_state) +int32_t sinsp_filter_check_mesos::parse_field_name(const char* str, bool alloc_state, bool needed_for_filtering) { string val(str); @@ -6937,7 +6926,7 @@ int32_t sinsp_filter_check_mesos::parse_field_name(const char* str, bool alloc_s } else { - return sinsp_filter_check::parse_field_name(str, alloc_state); + return sinsp_filter_check::parse_field_name(str, alloc_state, needed_for_filtering); } } diff --git a/userspace/libsinsp/filterchecks.h b/userspace/libsinsp/filterchecks.h index f96ba70b84..be2d7f1c6b 100644 --- a/userspace/libsinsp/filterchecks.h +++ b/userspace/libsinsp/filterchecks.h @@ -78,13 +78,13 @@ class sinsp_filter_check // Returns the length of the parsed field if successful, an exception in // case of error. // - virtual int32_t parse_field_name(const char* str, bool alloc_state); + virtual int32_t parse_field_name(const char* str, bool alloc_state, bool needed_for_filtering); // // If this check is used by a filter, extract the constant to compare it to // Doesn't return the field length because the filtering engine can calculate it. // - void add_filter_value(const char* str, uint32_t len, uint16_t i = 0 ); + void add_filter_value(const char* str, uint32_t len, uint32_t i = 0 ); virtual void parse_filter_value(const char* str, uint32_t len, uint8_t *storage, uint32_t storage_len); // @@ -213,7 +213,7 @@ class sinsp_filter_expression : public sinsp_filter_check // The following methods are part of the filter check interface but are irrelevant // for this class, because they are used only for the leaves of the filtering tree. // - int32_t parse_field_name(const char* str, bool alloc_state) + int32_t parse_field_name(const char* str, bool alloc_state, bool needed_for_filtering) { ASSERT(false); return 0; @@ -373,7 +373,7 @@ class sinsp_filter_check_thread : public sinsp_filter_check sinsp_filter_check_thread(); sinsp_filter_check* allocate_new(); - int32_t parse_field_name(const char* str, bool alloc_state); + int32_t parse_field_name(const char* str, bool alloc_state, bool needed_for_filtering); uint8_t* extract(sinsp_evt *evt, OUT uint32_t* len, bool sanitize_strings = true); bool compare(sinsp_evt *evt); @@ -471,7 +471,7 @@ class sinsp_filter_check_event : public sinsp_filter_check sinsp_filter_check_event(); ~sinsp_filter_check_event(); sinsp_filter_check* allocate_new(); - int32_t parse_field_name(const char* str, bool alloc_state); + int32_t parse_field_name(const char* str, bool alloc_state, bool needed_for_filtering); void parse_filter_value(const char* str, uint32_t len, uint8_t *storage, uint32_t storage_len); void validate_filter_value(const char* str, uint32_t len); const filtercheck_field_info* get_field_info(); @@ -586,7 +586,7 @@ class sinsp_filter_check_tracer : public sinsp_filter_check sinsp_filter_check_tracer(); ~sinsp_filter_check_tracer(); sinsp_filter_check* allocate_new(); - int32_t parse_field_name(const char* str, bool alloc_state); + int32_t parse_field_name(const char* str, bool alloc_state, bool needed_for_filtering); uint8_t* extract(sinsp_evt *evt, OUT uint32_t* len, bool sanitize_strings = true); private: @@ -646,7 +646,7 @@ class sinsp_filter_check_evtin : public sinsp_filter_check sinsp_filter_check_evtin(); ~sinsp_filter_check_evtin(); - int32_t parse_field_name(const char* str, bool alloc_state); + int32_t parse_field_name(const char* str, bool alloc_state, bool needed_for_filtering); sinsp_filter_check* allocate_new(); uint8_t* extract(sinsp_evt *evt, OUT uint32_t* len, bool sanitize_strings = true); bool compare(sinsp_evt *evt); @@ -669,6 +669,7 @@ class sinsp_filter_check_evtin : public sinsp_filter_check private: int32_t extract_arg(string fldname, string val); + inline uint8_t* extract_tracer(sinsp_evt *evt, sinsp_partial_tracer* pae); inline bool compare_tracer(sinsp_evt *evt, sinsp_partial_tracer* pae); bool m_is_compare; @@ -687,7 +688,7 @@ class rawstring_check : public sinsp_filter_check rawstring_check(string text); sinsp_filter_check* allocate_new(); void set_text(string text); - int32_t parse_field_name(const char* str, bool alloc_state); + int32_t parse_field_name(const char* str, bool alloc_state, bool needed_for_filtering); uint8_t* extract(sinsp_evt *evt, OUT uint32_t* len, bool sanitize_strings = true); // XXX this is overkill and wasted for most of the fields. @@ -717,7 +718,7 @@ class sinsp_filter_check_syslog : public sinsp_filter_check sinsp_filter_check_syslog(); sinsp_filter_check* allocate_new(); - int32_t parse_field_name(const char* str, bool alloc_state); + int32_t parse_field_name(const char* str, bool alloc_state, bool needed_for_filtering); uint8_t* extract(sinsp_evt *evt, OUT uint32_t* len, bool sanitize_strings = true); sinsp_decoder_syslog* m_decoder; @@ -750,7 +751,7 @@ class sinsp_filter_check_container : public sinsp_filter_check uint8_t* extract(sinsp_evt *evt, OUT uint32_t* len, bool sanitize_strings = true); private: - int32_t parse_field_name(const char* str, bool alloc_state); + int32_t parse_field_name(const char* str, bool alloc_state, bool needed_for_filtering); int32_t extract_arg(const string& val, size_t basename); string m_tstr; @@ -783,7 +784,7 @@ class sinsp_filter_check_reference : public sinsp_filter_check m_cnt = cnt; m_print_format = print_format; } - int32_t parse_field_name(const char* str, bool alloc_state); + int32_t parse_field_name(const char* str, bool alloc_state, bool needed_for_filtering); uint8_t* extract(sinsp_evt *evt, OUT uint32_t* len, bool sanitize_strings = true); char* tostring_nice(sinsp_evt* evt, uint32_t str_len, uint64_t time_delta); @@ -844,6 +845,8 @@ class sinsp_filter_check_fdlist : public sinsp_filter_check char m_addrbuff[100]; }; +#ifndef HAS_ANALYZER + class sinsp_filter_check_k8s : public sinsp_filter_check { public: @@ -877,7 +880,7 @@ class sinsp_filter_check_k8s : public sinsp_filter_check sinsp_filter_check_k8s(); sinsp_filter_check* allocate_new(); - int32_t parse_field_name(const char* str, bool alloc_state); + int32_t parse_field_name(const char* str, bool alloc_state, bool needed_for_filtering); uint8_t* extract(sinsp_evt *evt, OUT uint32_t* len, bool sanitize_strings = true); private: @@ -895,6 +898,8 @@ class sinsp_filter_check_k8s : public sinsp_filter_check string m_tstr; }; +#endif // HAS_ANALYZER + class sinsp_filter_check_mesos : public sinsp_filter_check { public: @@ -916,7 +921,7 @@ class sinsp_filter_check_mesos : public sinsp_filter_check sinsp_filter_check_mesos(); sinsp_filter_check* allocate_new(); - int32_t parse_field_name(const char* str, bool alloc_state); + int32_t parse_field_name(const char* str, bool alloc_state, bool needed_for_filtering); uint8_t* extract(sinsp_evt *evt, OUT uint32_t* len, bool sanitize_strings = true); private: diff --git a/userspace/libsinsp/http_parser.c b/userspace/libsinsp/http_parser.c new file mode 100644 index 0000000000..895bf0c737 --- /dev/null +++ b/userspace/libsinsp/http_parser.c @@ -0,0 +1,2470 @@ +/* Based on src/http/ngx_http_parse.c from NGINX copyright Igor Sysoev + * + * Additional changes are licensed under the same terms as NGINX and + * copyright Joyent, Inc. and other Node contributors. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ +#include "http_parser.h" +#include +#include +#include +#include +#include +#include + +#ifndef ULLONG_MAX +# define ULLONG_MAX ((uint64_t) -1) /* 2^64-1 */ +#endif + +#ifndef MIN +# define MIN(a,b) ((a) < (b) ? (a) : (b)) +#endif + +#ifndef ARRAY_SIZE +# define ARRAY_SIZE(a) (sizeof(a) / sizeof((a)[0])) +#endif + +#ifndef BIT_AT +# define BIT_AT(a, i) \ + (!!((unsigned int) (a)[(unsigned int) (i) >> 3] & \ + (1 << ((unsigned int) (i) & 7)))) +#endif + +#ifndef ELEM_AT +# define ELEM_AT(a, i, v) ((unsigned int) (i) < ARRAY_SIZE(a) ? (a)[(i)] : (v)) +#endif + +#define SET_ERRNO(e) \ +do { \ + parser->http_errno = (e); \ +} while(0) + +#define CURRENT_STATE() p_state +#define UPDATE_STATE(V) p_state = (enum state) (V); +#define RETURN(V) \ +do { \ + parser->state = CURRENT_STATE(); \ + return (V); \ +} while (0); +#define REEXECUTE() \ + goto reexecute; \ + + +#ifdef __GNUC__ +# define LIKELY(X) __builtin_expect(!!(X), 1) +# define UNLIKELY(X) __builtin_expect(!!(X), 0) +#else +# define LIKELY(X) (X) +# define UNLIKELY(X) (X) +#endif + + +/* Run the notify callback FOR, returning ER if it fails */ +#define CALLBACK_NOTIFY_(FOR, ER) \ +do { \ + assert(HTTP_PARSER_ERRNO(parser) == HPE_OK); \ + \ + if (LIKELY(settings->on_##FOR)) { \ + parser->state = CURRENT_STATE(); \ + if (UNLIKELY(0 != settings->on_##FOR(parser))) { \ + SET_ERRNO(HPE_CB_##FOR); \ + } \ + UPDATE_STATE(parser->state); \ + \ + /* We either errored above or got paused; get out */ \ + if (UNLIKELY(HTTP_PARSER_ERRNO(parser) != HPE_OK)) { \ + return (ER); \ + } \ + } \ +} while (0) + +/* Run the notify callback FOR and consume the current byte */ +#define CALLBACK_NOTIFY(FOR) CALLBACK_NOTIFY_(FOR, p - data + 1) + +/* Run the notify callback FOR and don't consume the current byte */ +#define CALLBACK_NOTIFY_NOADVANCE(FOR) CALLBACK_NOTIFY_(FOR, p - data) + +/* Run data callback FOR with LEN bytes, returning ER if it fails */ +#define CALLBACK_DATA_(FOR, LEN, ER) \ +do { \ + assert(HTTP_PARSER_ERRNO(parser) == HPE_OK); \ + \ + if (FOR##_mark) { \ + if (LIKELY(settings->on_##FOR)) { \ + parser->state = CURRENT_STATE(); \ + if (UNLIKELY(0 != \ + settings->on_##FOR(parser, FOR##_mark, (LEN)))) { \ + SET_ERRNO(HPE_CB_##FOR); \ + } \ + UPDATE_STATE(parser->state); \ + \ + /* We either errored above or got paused; get out */ \ + if (UNLIKELY(HTTP_PARSER_ERRNO(parser) != HPE_OK)) { \ + return (ER); \ + } \ + } \ + FOR##_mark = NULL; \ + } \ +} while (0) + +/* Run the data callback FOR and consume the current byte */ +#define CALLBACK_DATA(FOR) \ + CALLBACK_DATA_(FOR, p - FOR##_mark, p - data + 1) + +/* Run the data callback FOR and don't consume the current byte */ +#define CALLBACK_DATA_NOADVANCE(FOR) \ + CALLBACK_DATA_(FOR, p - FOR##_mark, p - data) + +/* Set the mark FOR; non-destructive if mark is already set */ +#define MARK(FOR) \ +do { \ + if (!FOR##_mark) { \ + FOR##_mark = p; \ + } \ +} while (0) + +/* Don't allow the total size of the HTTP headers (including the status + * line) to exceed HTTP_MAX_HEADER_SIZE. This check is here to protect + * embedders against denial-of-service attacks where the attacker feeds + * us a never-ending header that the embedder keeps buffering. + * + * This check is arguably the responsibility of embedders but we're doing + * it on the embedder's behalf because most won't bother and this way we + * make the web a little safer. HTTP_MAX_HEADER_SIZE is still far bigger + * than any reasonable request or response so this should never affect + * day-to-day operation. + */ +#define COUNT_HEADER_SIZE(V) \ +do { \ + parser->nread += (V); \ + if (UNLIKELY(parser->nread > (HTTP_MAX_HEADER_SIZE))) { \ + SET_ERRNO(HPE_HEADER_OVERFLOW); \ + goto error; \ + } \ +} while (0) + + +#define PROXY_CONNECTION "proxy-connection" +#define CONNECTION "connection" +#define CONTENT_LENGTH "content-length" +#define TRANSFER_ENCODING "transfer-encoding" +#define UPGRADE "upgrade" +#define CHUNKED "chunked" +#define KEEP_ALIVE "keep-alive" +#define CLOSE "close" + + +static const char *method_strings[] = + { +#define XX(num, name, string) #string, + HTTP_METHOD_MAP(XX) +#undef XX + }; + + +/* Tokens as defined by rfc 2616. Also lowercases them. + * token = 1* + * separators = "(" | ")" | "<" | ">" | "@" + * | "," | ";" | ":" | "\" | <"> + * | "/" | "[" | "]" | "?" | "=" + * | "{" | "}" | SP | HT + */ +static const char tokens[256] = { +/* 0 nul 1 soh 2 stx 3 etx 4 eot 5 enq 6 ack 7 bel */ + 0, 0, 0, 0, 0, 0, 0, 0, +/* 8 bs 9 ht 10 nl 11 vt 12 np 13 cr 14 so 15 si */ + 0, 0, 0, 0, 0, 0, 0, 0, +/* 16 dle 17 dc1 18 dc2 19 dc3 20 dc4 21 nak 22 syn 23 etb */ + 0, 0, 0, 0, 0, 0, 0, 0, +/* 24 can 25 em 26 sub 27 esc 28 fs 29 gs 30 rs 31 us */ + 0, 0, 0, 0, 0, 0, 0, 0, +/* 32 sp 33 ! 34 " 35 # 36 $ 37 % 38 & 39 ' */ + 0, '!', 0, '#', '$', '%', '&', '\'', +/* 40 ( 41 ) 42 * 43 + 44 , 45 - 46 . 47 / */ + 0, 0, '*', '+', 0, '-', '.', 0, +/* 48 0 49 1 50 2 51 3 52 4 53 5 54 6 55 7 */ + '0', '1', '2', '3', '4', '5', '6', '7', +/* 56 8 57 9 58 : 59 ; 60 < 61 = 62 > 63 ? */ + '8', '9', 0, 0, 0, 0, 0, 0, +/* 64 @ 65 A 66 B 67 C 68 D 69 E 70 F 71 G */ + 0, 'a', 'b', 'c', 'd', 'e', 'f', 'g', +/* 72 H 73 I 74 J 75 K 76 L 77 M 78 N 79 O */ + 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', +/* 80 P 81 Q 82 R 83 S 84 T 85 U 86 V 87 W */ + 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', +/* 88 X 89 Y 90 Z 91 [ 92 \ 93 ] 94 ^ 95 _ */ + 'x', 'y', 'z', 0, 0, 0, '^', '_', +/* 96 ` 97 a 98 b 99 c 100 d 101 e 102 f 103 g */ + '`', 'a', 'b', 'c', 'd', 'e', 'f', 'g', +/* 104 h 105 i 106 j 107 k 108 l 109 m 110 n 111 o */ + 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', +/* 112 p 113 q 114 r 115 s 116 t 117 u 118 v 119 w */ + 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', +/* 120 x 121 y 122 z 123 { 124 | 125 } 126 ~ 127 del */ + 'x', 'y', 'z', 0, '|', 0, '~', 0 }; + + +static const int8_t unhex[256] = + {-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1 + ,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1 + ,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1 + , 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,-1,-1,-1,-1,-1,-1 + ,-1,10,11,12,13,14,15,-1,-1,-1,-1,-1,-1,-1,-1,-1 + ,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1 + ,-1,10,11,12,13,14,15,-1,-1,-1,-1,-1,-1,-1,-1,-1 + ,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1 + }; + + +#if HTTP_PARSER_STRICT +# define T(v) 0 +#else +# define T(v) v +#endif + + +static const uint8_t normal_url_char[32] = { +/* 0 nul 1 soh 2 stx 3 etx 4 eot 5 enq 6 ack 7 bel */ + 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0, +/* 8 bs 9 ht 10 nl 11 vt 12 np 13 cr 14 so 15 si */ + 0 | T(2) | 0 | 0 | T(16) | 0 | 0 | 0, +/* 16 dle 17 dc1 18 dc2 19 dc3 20 dc4 21 nak 22 syn 23 etb */ + 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0, +/* 24 can 25 em 26 sub 27 esc 28 fs 29 gs 30 rs 31 us */ + 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0, +/* 32 sp 33 ! 34 " 35 # 36 $ 37 % 38 & 39 ' */ + 0 | 2 | 4 | 0 | 16 | 32 | 64 | 128, +/* 40 ( 41 ) 42 * 43 + 44 , 45 - 46 . 47 / */ + 1 | 2 | 4 | 8 | 16 | 32 | 64 | 128, +/* 48 0 49 1 50 2 51 3 52 4 53 5 54 6 55 7 */ + 1 | 2 | 4 | 8 | 16 | 32 | 64 | 128, +/* 56 8 57 9 58 : 59 ; 60 < 61 = 62 > 63 ? */ + 1 | 2 | 4 | 8 | 16 | 32 | 64 | 0, +/* 64 @ 65 A 66 B 67 C 68 D 69 E 70 F 71 G */ + 1 | 2 | 4 | 8 | 16 | 32 | 64 | 128, +/* 72 H 73 I 74 J 75 K 76 L 77 M 78 N 79 O */ + 1 | 2 | 4 | 8 | 16 | 32 | 64 | 128, +/* 80 P 81 Q 82 R 83 S 84 T 85 U 86 V 87 W */ + 1 | 2 | 4 | 8 | 16 | 32 | 64 | 128, +/* 88 X 89 Y 90 Z 91 [ 92 \ 93 ] 94 ^ 95 _ */ + 1 | 2 | 4 | 8 | 16 | 32 | 64 | 128, +/* 96 ` 97 a 98 b 99 c 100 d 101 e 102 f 103 g */ + 1 | 2 | 4 | 8 | 16 | 32 | 64 | 128, +/* 104 h 105 i 106 j 107 k 108 l 109 m 110 n 111 o */ + 1 | 2 | 4 | 8 | 16 | 32 | 64 | 128, +/* 112 p 113 q 114 r 115 s 116 t 117 u 118 v 119 w */ + 1 | 2 | 4 | 8 | 16 | 32 | 64 | 128, +/* 120 x 121 y 122 z 123 { 124 | 125 } 126 ~ 127 del */ + 1 | 2 | 4 | 8 | 16 | 32 | 64 | 0, }; + +#undef T + +enum state + { s_dead = 1 /* important that this is > 0 */ + + , s_start_req_or_res + , s_res_or_resp_H + , s_start_res + , s_res_H + , s_res_HT + , s_res_HTT + , s_res_HTTP + , s_res_first_http_major + , s_res_http_major + , s_res_first_http_minor + , s_res_http_minor + , s_res_first_status_code + , s_res_status_code + , s_res_status_start + , s_res_status + , s_res_line_almost_done + + , s_start_req + + , s_req_method + , s_req_spaces_before_url + , s_req_schema + , s_req_schema_slash + , s_req_schema_slash_slash + , s_req_server_start + , s_req_server + , s_req_server_with_at + , s_req_path + , s_req_query_string_start + , s_req_query_string + , s_req_fragment_start + , s_req_fragment + , s_req_http_start + , s_req_http_H + , s_req_http_HT + , s_req_http_HTT + , s_req_http_HTTP + , s_req_first_http_major + , s_req_http_major + , s_req_first_http_minor + , s_req_http_minor + , s_req_line_almost_done + + , s_header_field_start + , s_header_field + , s_header_value_discard_ws + , s_header_value_discard_ws_almost_done + , s_header_value_discard_lws + , s_header_value_start + , s_header_value + , s_header_value_lws + + , s_header_almost_done + + , s_chunk_size_start + , s_chunk_size + , s_chunk_parameters + , s_chunk_size_almost_done + + , s_headers_almost_done + , s_headers_done + + /* Important: 's_headers_done' must be the last 'header' state. All + * states beyond this must be 'body' states. It is used for overflow + * checking. See the PARSING_HEADER() macro. + */ + + , s_chunk_data + , s_chunk_data_almost_done + , s_chunk_data_done + + , s_body_identity + , s_body_identity_eof + + , s_message_done + }; + + +#define PARSING_HEADER(state) (state <= s_headers_done) + + +enum header_states + { h_general = 0 + , h_C + , h_CO + , h_CON + + , h_matching_connection + , h_matching_proxy_connection + , h_matching_content_length + , h_matching_transfer_encoding + , h_matching_upgrade + + , h_connection + , h_content_length + , h_transfer_encoding + , h_upgrade + + , h_matching_transfer_encoding_chunked + , h_matching_connection_token_start + , h_matching_connection_keep_alive + , h_matching_connection_close + , h_matching_connection_upgrade + , h_matching_connection_token + + , h_transfer_encoding_chunked + , h_connection_keep_alive + , h_connection_close + , h_connection_upgrade + }; + +enum http_host_state + { + s_http_host_dead = 1 + , s_http_userinfo_start + , s_http_userinfo + , s_http_host_start + , s_http_host_v6_start + , s_http_host + , s_http_host_v6 + , s_http_host_v6_end + , s_http_host_v6_zone_start + , s_http_host_v6_zone + , s_http_host_port_start + , s_http_host_port +}; + +/* Macros for character classes; depends on strict-mode */ +#define CR '\r' +#define LF '\n' +#define LOWER(c) (unsigned char)(c | 0x20) +#define IS_ALPHA(c) (LOWER(c) >= 'a' && LOWER(c) <= 'z') +#define IS_NUM(c) ((c) >= '0' && (c) <= '9') +#define IS_ALPHANUM(c) (IS_ALPHA(c) || IS_NUM(c)) +#define IS_HEX(c) (IS_NUM(c) || (LOWER(c) >= 'a' && LOWER(c) <= 'f')) +#define IS_MARK(c) ((c) == '-' || (c) == '_' || (c) == '.' || \ + (c) == '!' || (c) == '~' || (c) == '*' || (c) == '\'' || (c) == '(' || \ + (c) == ')') +#define IS_USERINFO_CHAR(c) (IS_ALPHANUM(c) || IS_MARK(c) || (c) == '%' || \ + (c) == ';' || (c) == ':' || (c) == '&' || (c) == '=' || (c) == '+' || \ + (c) == '$' || (c) == ',') + +#define STRICT_TOKEN(c) (tokens[(unsigned char)c]) + +#if HTTP_PARSER_STRICT +#define TOKEN(c) (tokens[(unsigned char)c]) +#define IS_URL_CHAR(c) (BIT_AT(normal_url_char, (unsigned char)c)) +#define IS_HOST_CHAR(c) (IS_ALPHANUM(c) || (c) == '.' || (c) == '-') +#else +#define TOKEN(c) ((c == ' ') ? ' ' : tokens[(unsigned char)c]) +#define IS_URL_CHAR(c) \ + (BIT_AT(normal_url_char, (unsigned char)c) || ((c) & 0x80)) +#define IS_HOST_CHAR(c) \ + (IS_ALPHANUM(c) || (c) == '.' || (c) == '-' || (c) == '_') +#endif + +/** + * Verify that a char is a valid visible (printable) US-ASCII + * character or %x80-FF + **/ +#define IS_HEADER_CHAR(ch) \ + (ch == CR || ch == LF || ch == 9 || ((unsigned char)ch > 31 && ch != 127)) + +#define start_state (parser->type == HTTP_REQUEST ? s_start_req : s_start_res) + + +#if HTTP_PARSER_STRICT +# define STRICT_CHECK(cond) \ +do { \ + if (cond) { \ + SET_ERRNO(HPE_STRICT); \ + goto error; \ + } \ +} while (0) +# define NEW_MESSAGE() (http_should_keep_alive(parser) ? start_state : s_dead) +#else +# define STRICT_CHECK(cond) +# define NEW_MESSAGE() start_state +#endif + + +/* Map errno values to strings for human-readable output */ +#define HTTP_STRERROR_GEN(n, s) { "HPE_" #n, s }, +static struct { + const char *name; + const char *description; +} http_strerror_tab[] = { + HTTP_ERRNO_MAP(HTTP_STRERROR_GEN) +}; +#undef HTTP_STRERROR_GEN + +int http_message_needs_eof(const http_parser *parser); + +/* Our URL parser. + * + * This is designed to be shared by http_parser_execute() for URL validation, + * hence it has a state transition + byte-for-byte interface. In addition, it + * is meant to be embedded in http_parser_parse_url(), which does the dirty + * work of turning state transitions URL components for its API. + * + * This function should only be invoked with non-space characters. It is + * assumed that the caller cares about (and can detect) the transition between + * URL and non-URL states by looking for these. + */ +static enum state +parse_url_char(enum state s, const char ch) +{ + if (ch == ' ' || ch == '\r' || ch == '\n') { + return s_dead; + } + +#if HTTP_PARSER_STRICT + if (ch == '\t' || ch == '\f') { + return s_dead; + } +#endif + + switch (s) { + case s_req_spaces_before_url: + /* Proxied requests are followed by scheme of an absolute URI (alpha). + * All methods except CONNECT are followed by '/' or '*'. + */ + + if (ch == '/' || ch == '*') { + return s_req_path; + } + + if (IS_ALPHA(ch)) { + return s_req_schema; + } + + break; + + case s_req_schema: + if (IS_ALPHA(ch)) { + return s; + } + + if (ch == ':') { + return s_req_schema_slash; + } + + break; + + case s_req_schema_slash: + if (ch == '/') { + return s_req_schema_slash_slash; + } + + break; + + case s_req_schema_slash_slash: + if (ch == '/') { + return s_req_server_start; + } + + break; + + case s_req_server_with_at: + if (ch == '@') { + return s_dead; + } + + /* FALLTHROUGH */ + case s_req_server_start: + case s_req_server: + if (ch == '/') { + return s_req_path; + } + + if (ch == '?') { + return s_req_query_string_start; + } + + if (ch == '@') { + return s_req_server_with_at; + } + + if (IS_USERINFO_CHAR(ch) || ch == '[' || ch == ']') { + return s_req_server; + } + + break; + + case s_req_path: + if (IS_URL_CHAR(ch)) { + return s; + } + + switch (ch) { + case '?': + return s_req_query_string_start; + + case '#': + return s_req_fragment_start; + } + + break; + + case s_req_query_string_start: + case s_req_query_string: + if (IS_URL_CHAR(ch)) { + return s_req_query_string; + } + + switch (ch) { + case '?': + /* allow extra '?' in query string */ + return s_req_query_string; + + case '#': + return s_req_fragment_start; + } + + break; + + case s_req_fragment_start: + if (IS_URL_CHAR(ch)) { + return s_req_fragment; + } + + switch (ch) { + case '?': + return s_req_fragment; + + case '#': + return s; + } + + break; + + case s_req_fragment: + if (IS_URL_CHAR(ch)) { + return s; + } + + switch (ch) { + case '?': + case '#': + return s; + } + + break; + + default: + break; + } + + /* We should never fall out of the switch above unless there's an error */ + return s_dead; +} + +size_t http_parser_execute (http_parser *parser, + const http_parser_settings *settings, + const char *data, + size_t len) +{ + char c, ch; + int8_t unhex_val; + const char *p = data; + const char *header_field_mark = 0; + const char *header_value_mark = 0; + const char *url_mark = 0; + const char *body_mark = 0; + const char *status_mark = 0; + enum state p_state = (enum state) parser->state; + const unsigned int lenient = parser->lenient_http_headers; + + /* We're in an error state. Don't bother doing anything. */ + if (HTTP_PARSER_ERRNO(parser) != HPE_OK) { + return 0; + } + + if (len == 0) { + switch (CURRENT_STATE()) { + case s_body_identity_eof: + /* Use of CALLBACK_NOTIFY() here would erroneously return 1 byte read if + * we got paused. + */ + CALLBACK_NOTIFY_NOADVANCE(message_complete); + return 0; + + case s_dead: + case s_start_req_or_res: + case s_start_res: + case s_start_req: + return 0; + + default: + SET_ERRNO(HPE_INVALID_EOF_STATE); + return 1; + } + } + + + if (CURRENT_STATE() == s_header_field) + header_field_mark = data; + if (CURRENT_STATE() == s_header_value) + header_value_mark = data; + switch (CURRENT_STATE()) { + case s_req_path: + case s_req_schema: + case s_req_schema_slash: + case s_req_schema_slash_slash: + case s_req_server_start: + case s_req_server: + case s_req_server_with_at: + case s_req_query_string_start: + case s_req_query_string: + case s_req_fragment_start: + case s_req_fragment: + url_mark = data; + break; + case s_res_status: + status_mark = data; + break; + default: + break; + } + + for (p=data; p != data + len; p++) { + ch = *p; + + if (PARSING_HEADER(CURRENT_STATE())) + COUNT_HEADER_SIZE(1); + +reexecute: + switch (CURRENT_STATE()) { + + case s_dead: + /* this state is used after a 'Connection: close' message + * the parser will error out if it reads another message + */ + if (LIKELY(ch == CR || ch == LF)) + break; + + SET_ERRNO(HPE_CLOSED_CONNECTION); + goto error; + + case s_start_req_or_res: + { + if (ch == CR || ch == LF) + break; + parser->flags = 0; + parser->content_length = ULLONG_MAX; + + if (ch == 'H') { + UPDATE_STATE(s_res_or_resp_H); + + CALLBACK_NOTIFY(message_begin); + } else { + parser->type = HTTP_REQUEST; + UPDATE_STATE(s_start_req); + REEXECUTE(); + } + + break; + } + + case s_res_or_resp_H: + if (ch == 'T') { + parser->type = HTTP_RESPONSE; + UPDATE_STATE(s_res_HT); + } else { + if (UNLIKELY(ch != 'E')) { + SET_ERRNO(HPE_INVALID_CONSTANT); + goto error; + } + + parser->type = HTTP_REQUEST; + parser->method = HTTP_HEAD; + parser->index = 2; + UPDATE_STATE(s_req_method); + } + break; + + case s_start_res: + { + parser->flags = 0; + parser->content_length = ULLONG_MAX; + + switch (ch) { + case 'H': + UPDATE_STATE(s_res_H); + break; + + case CR: + case LF: + break; + + default: + SET_ERRNO(HPE_INVALID_CONSTANT); + goto error; + } + + CALLBACK_NOTIFY(message_begin); + break; + } + + case s_res_H: + STRICT_CHECK(ch != 'T'); + UPDATE_STATE(s_res_HT); + break; + + case s_res_HT: + STRICT_CHECK(ch != 'T'); + UPDATE_STATE(s_res_HTT); + break; + + case s_res_HTT: + STRICT_CHECK(ch != 'P'); + UPDATE_STATE(s_res_HTTP); + break; + + case s_res_HTTP: + STRICT_CHECK(ch != '/'); + UPDATE_STATE(s_res_first_http_major); + break; + + case s_res_first_http_major: + if (UNLIKELY(ch < '0' || ch > '9')) { + SET_ERRNO(HPE_INVALID_VERSION); + goto error; + } + + parser->http_major = ch - '0'; + UPDATE_STATE(s_res_http_major); + break; + + /* major HTTP version or dot */ + case s_res_http_major: + { + if (ch == '.') { + UPDATE_STATE(s_res_first_http_minor); + break; + } + + if (!IS_NUM(ch)) { + SET_ERRNO(HPE_INVALID_VERSION); + goto error; + } + + parser->http_major *= 10; + parser->http_major += ch - '0'; + + if (UNLIKELY(parser->http_major > 999)) { + SET_ERRNO(HPE_INVALID_VERSION); + goto error; + } + + break; + } + + /* first digit of minor HTTP version */ + case s_res_first_http_minor: + if (UNLIKELY(!IS_NUM(ch))) { + SET_ERRNO(HPE_INVALID_VERSION); + goto error; + } + + parser->http_minor = ch - '0'; + UPDATE_STATE(s_res_http_minor); + break; + + /* minor HTTP version or end of request line */ + case s_res_http_minor: + { + if (ch == ' ') { + UPDATE_STATE(s_res_first_status_code); + break; + } + + if (UNLIKELY(!IS_NUM(ch))) { + SET_ERRNO(HPE_INVALID_VERSION); + goto error; + } + + parser->http_minor *= 10; + parser->http_minor += ch - '0'; + + if (UNLIKELY(parser->http_minor > 999)) { + SET_ERRNO(HPE_INVALID_VERSION); + goto error; + } + + break; + } + + case s_res_first_status_code: + { + if (!IS_NUM(ch)) { + if (ch == ' ') { + break; + } + + SET_ERRNO(HPE_INVALID_STATUS); + goto error; + } + parser->status_code = ch - '0'; + UPDATE_STATE(s_res_status_code); + break; + } + + case s_res_status_code: + { + if (!IS_NUM(ch)) { + switch (ch) { + case ' ': + UPDATE_STATE(s_res_status_start); + break; + case CR: + UPDATE_STATE(s_res_line_almost_done); + break; + case LF: + UPDATE_STATE(s_header_field_start); + break; + default: + SET_ERRNO(HPE_INVALID_STATUS); + goto error; + } + break; + } + + parser->status_code *= 10; + parser->status_code += ch - '0'; + + if (UNLIKELY(parser->status_code > 999)) { + SET_ERRNO(HPE_INVALID_STATUS); + goto error; + } + + break; + } + + case s_res_status_start: + { + if (ch == CR) { + UPDATE_STATE(s_res_line_almost_done); + break; + } + + if (ch == LF) { + UPDATE_STATE(s_header_field_start); + break; + } + + MARK(status); + UPDATE_STATE(s_res_status); + parser->index = 0; + break; + } + + case s_res_status: + if (ch == CR) { + UPDATE_STATE(s_res_line_almost_done); + CALLBACK_DATA(status); + break; + } + + if (ch == LF) { + UPDATE_STATE(s_header_field_start); + CALLBACK_DATA(status); + break; + } + + break; + + case s_res_line_almost_done: + STRICT_CHECK(ch != LF); + UPDATE_STATE(s_header_field_start); + break; + + case s_start_req: + { + if (ch == CR || ch == LF) + break; + parser->flags = 0; + parser->content_length = ULLONG_MAX; + + if (UNLIKELY(!IS_ALPHA(ch))) { + SET_ERRNO(HPE_INVALID_METHOD); + goto error; + } + + parser->method = (enum http_method) 0; + parser->index = 1; + switch (ch) { + case 'A': parser->method = HTTP_ACL; break; + case 'B': parser->method = HTTP_BIND; break; + case 'C': parser->method = HTTP_CONNECT; /* or COPY, CHECKOUT */ break; + case 'D': parser->method = HTTP_DELETE; break; + case 'G': parser->method = HTTP_GET; break; + case 'H': parser->method = HTTP_HEAD; break; + case 'L': parser->method = HTTP_LOCK; /* or LINK */ break; + case 'M': parser->method = HTTP_MKCOL; /* or MOVE, MKACTIVITY, MERGE, M-SEARCH, MKCALENDAR */ break; + case 'N': parser->method = HTTP_NOTIFY; break; + case 'O': parser->method = HTTP_OPTIONS; break; + case 'P': parser->method = HTTP_POST; + /* or PROPFIND|PROPPATCH|PUT|PATCH|PURGE */ + break; + case 'R': parser->method = HTTP_REPORT; /* or REBIND */ break; + case 'S': parser->method = HTTP_SUBSCRIBE; /* or SEARCH */ break; + case 'T': parser->method = HTTP_TRACE; break; + case 'U': parser->method = HTTP_UNLOCK; /* or UNSUBSCRIBE, UNBIND, UNLINK */ break; + default: + SET_ERRNO(HPE_INVALID_METHOD); + goto error; + } + UPDATE_STATE(s_req_method); + + CALLBACK_NOTIFY(message_begin); + + break; + } + + case s_req_method: + { + const char *matcher; + if (UNLIKELY(ch == '\0')) { + SET_ERRNO(HPE_INVALID_METHOD); + goto error; + } + + matcher = method_strings[parser->method]; + if (ch == ' ' && matcher[parser->index] == '\0') { + UPDATE_STATE(s_req_spaces_before_url); + } else if (ch == matcher[parser->index]) { + ; /* nada */ + } else if (IS_ALPHA(ch)) { + + switch (parser->method << 16 | parser->index << 8 | ch) { +#define XX(meth, pos, ch, new_meth) \ + case (HTTP_##meth << 16 | pos << 8 | ch): \ + parser->method = HTTP_##new_meth; break; + + XX(POST, 1, 'U', PUT) + XX(POST, 1, 'A', PATCH) + XX(CONNECT, 1, 'H', CHECKOUT) + XX(CONNECT, 2, 'P', COPY) + XX(MKCOL, 1, 'O', MOVE) + XX(MKCOL, 1, 'E', MERGE) + XX(MKCOL, 2, 'A', MKACTIVITY) + XX(MKCOL, 3, 'A', MKCALENDAR) + XX(SUBSCRIBE, 1, 'E', SEARCH) + XX(REPORT, 2, 'B', REBIND) + XX(POST, 1, 'R', PROPFIND) + XX(PROPFIND, 4, 'P', PROPPATCH) + XX(PUT, 2, 'R', PURGE) + XX(LOCK, 1, 'I', LINK) + XX(UNLOCK, 2, 'S', UNSUBSCRIBE) + XX(UNLOCK, 2, 'B', UNBIND) + XX(UNLOCK, 3, 'I', UNLINK) +#undef XX + + default: + SET_ERRNO(HPE_INVALID_METHOD); + goto error; + } + } else if (ch == '-' && + parser->index == 1 && + parser->method == HTTP_MKCOL) { + parser->method = HTTP_MSEARCH; + } else { + SET_ERRNO(HPE_INVALID_METHOD); + goto error; + } + + ++parser->index; + break; + } + + case s_req_spaces_before_url: + { + if (ch == ' ') break; + + MARK(url); + if (parser->method == HTTP_CONNECT) { + UPDATE_STATE(s_req_server_start); + } + + UPDATE_STATE(parse_url_char(CURRENT_STATE(), ch)); + if (UNLIKELY(CURRENT_STATE() == s_dead)) { + SET_ERRNO(HPE_INVALID_URL); + goto error; + } + + break; + } + + case s_req_schema: + case s_req_schema_slash: + case s_req_schema_slash_slash: + case s_req_server_start: + { + switch (ch) { + /* No whitespace allowed here */ + case ' ': + case CR: + case LF: + SET_ERRNO(HPE_INVALID_URL); + goto error; + default: + UPDATE_STATE(parse_url_char(CURRENT_STATE(), ch)); + if (UNLIKELY(CURRENT_STATE() == s_dead)) { + SET_ERRNO(HPE_INVALID_URL); + goto error; + } + } + + break; + } + + case s_req_server: + case s_req_server_with_at: + case s_req_path: + case s_req_query_string_start: + case s_req_query_string: + case s_req_fragment_start: + case s_req_fragment: + { + switch (ch) { + case ' ': + UPDATE_STATE(s_req_http_start); + CALLBACK_DATA(url); + break; + case CR: + case LF: + parser->http_major = 0; + parser->http_minor = 9; + UPDATE_STATE((ch == CR) ? + s_req_line_almost_done : + s_header_field_start); + CALLBACK_DATA(url); + break; + default: + UPDATE_STATE(parse_url_char(CURRENT_STATE(), ch)); + if (UNLIKELY(CURRENT_STATE() == s_dead)) { + SET_ERRNO(HPE_INVALID_URL); + goto error; + } + } + break; + } + + case s_req_http_start: + switch (ch) { + case 'H': + UPDATE_STATE(s_req_http_H); + break; + case ' ': + break; + default: + SET_ERRNO(HPE_INVALID_CONSTANT); + goto error; + } + break; + + case s_req_http_H: + STRICT_CHECK(ch != 'T'); + UPDATE_STATE(s_req_http_HT); + break; + + case s_req_http_HT: + STRICT_CHECK(ch != 'T'); + UPDATE_STATE(s_req_http_HTT); + break; + + case s_req_http_HTT: + STRICT_CHECK(ch != 'P'); + UPDATE_STATE(s_req_http_HTTP); + break; + + case s_req_http_HTTP: + STRICT_CHECK(ch != '/'); + UPDATE_STATE(s_req_first_http_major); + break; + + /* first digit of major HTTP version */ + case s_req_first_http_major: + if (UNLIKELY(ch < '1' || ch > '9')) { + SET_ERRNO(HPE_INVALID_VERSION); + goto error; + } + + parser->http_major = ch - '0'; + UPDATE_STATE(s_req_http_major); + break; + + /* major HTTP version or dot */ + case s_req_http_major: + { + if (ch == '.') { + UPDATE_STATE(s_req_first_http_minor); + break; + } + + if (UNLIKELY(!IS_NUM(ch))) { + SET_ERRNO(HPE_INVALID_VERSION); + goto error; + } + + parser->http_major *= 10; + parser->http_major += ch - '0'; + + if (UNLIKELY(parser->http_major > 999)) { + SET_ERRNO(HPE_INVALID_VERSION); + goto error; + } + + break; + } + + /* first digit of minor HTTP version */ + case s_req_first_http_minor: + if (UNLIKELY(!IS_NUM(ch))) { + SET_ERRNO(HPE_INVALID_VERSION); + goto error; + } + + parser->http_minor = ch - '0'; + UPDATE_STATE(s_req_http_minor); + break; + + /* minor HTTP version or end of request line */ + case s_req_http_minor: + { + if (ch == CR) { + UPDATE_STATE(s_req_line_almost_done); + break; + } + + if (ch == LF) { + UPDATE_STATE(s_header_field_start); + break; + } + + /* XXX allow spaces after digit? */ + + if (UNLIKELY(!IS_NUM(ch))) { + SET_ERRNO(HPE_INVALID_VERSION); + goto error; + } + + parser->http_minor *= 10; + parser->http_minor += ch - '0'; + + if (UNLIKELY(parser->http_minor > 999)) { + SET_ERRNO(HPE_INVALID_VERSION); + goto error; + } + + break; + } + + /* end of request line */ + case s_req_line_almost_done: + { + if (UNLIKELY(ch != LF)) { + SET_ERRNO(HPE_LF_EXPECTED); + goto error; + } + + UPDATE_STATE(s_header_field_start); + break; + } + + case s_header_field_start: + { + if (ch == CR) { + UPDATE_STATE(s_headers_almost_done); + break; + } + + if (ch == LF) { + /* they might be just sending \n instead of \r\n so this would be + * the second \n to denote the end of headers*/ + UPDATE_STATE(s_headers_almost_done); + REEXECUTE(); + } + + c = TOKEN(ch); + + if (UNLIKELY(!c)) { + SET_ERRNO(HPE_INVALID_HEADER_TOKEN); + goto error; + } + + MARK(header_field); + + parser->index = 0; + UPDATE_STATE(s_header_field); + + switch (c) { + case 'c': + parser->header_state = h_C; + break; + + case 'p': + parser->header_state = h_matching_proxy_connection; + break; + + case 't': + parser->header_state = h_matching_transfer_encoding; + break; + + case 'u': + parser->header_state = h_matching_upgrade; + break; + + default: + parser->header_state = h_general; + break; + } + break; + } + + case s_header_field: + { + const char* start = p; + for (; p != data + len; p++) { + ch = *p; + c = TOKEN(ch); + + if (!c) + break; + + switch (parser->header_state) { + case h_general: + break; + + case h_C: + parser->index++; + parser->header_state = (c == 'o' ? h_CO : h_general); + break; + + case h_CO: + parser->index++; + parser->header_state = (c == 'n' ? h_CON : h_general); + break; + + case h_CON: + parser->index++; + switch (c) { + case 'n': + parser->header_state = h_matching_connection; + break; + case 't': + parser->header_state = h_matching_content_length; + break; + default: + parser->header_state = h_general; + break; + } + break; + + /* connection */ + + case h_matching_connection: + parser->index++; + if (parser->index > sizeof(CONNECTION)-1 + || c != CONNECTION[parser->index]) { + parser->header_state = h_general; + } else if (parser->index == sizeof(CONNECTION)-2) { + parser->header_state = h_connection; + } + break; + + /* proxy-connection */ + + case h_matching_proxy_connection: + parser->index++; + if (parser->index > sizeof(PROXY_CONNECTION)-1 + || c != PROXY_CONNECTION[parser->index]) { + parser->header_state = h_general; + } else if (parser->index == sizeof(PROXY_CONNECTION)-2) { + parser->header_state = h_connection; + } + break; + + /* content-length */ + + case h_matching_content_length: + parser->index++; + if (parser->index > sizeof(CONTENT_LENGTH)-1 + || c != CONTENT_LENGTH[parser->index]) { + parser->header_state = h_general; + } else if (parser->index == sizeof(CONTENT_LENGTH)-2) { + parser->header_state = h_content_length; + } + break; + + /* transfer-encoding */ + + case h_matching_transfer_encoding: + parser->index++; + if (parser->index > sizeof(TRANSFER_ENCODING)-1 + || c != TRANSFER_ENCODING[parser->index]) { + parser->header_state = h_general; + } else if (parser->index == sizeof(TRANSFER_ENCODING)-2) { + parser->header_state = h_transfer_encoding; + } + break; + + /* upgrade */ + + case h_matching_upgrade: + parser->index++; + if (parser->index > sizeof(UPGRADE)-1 + || c != UPGRADE[parser->index]) { + parser->header_state = h_general; + } else if (parser->index == sizeof(UPGRADE)-2) { + parser->header_state = h_upgrade; + } + break; + + case h_connection: + case h_content_length: + case h_transfer_encoding: + case h_upgrade: + if (ch != ' ') parser->header_state = h_general; + break; + + default: + assert(0 && "Unknown header_state"); + break; + } + } + + COUNT_HEADER_SIZE(p - start); + + if (p == data + len) { + --p; + break; + } + + if (ch == ':') { + UPDATE_STATE(s_header_value_discard_ws); + CALLBACK_DATA(header_field); + break; + } + + SET_ERRNO(HPE_INVALID_HEADER_TOKEN); + goto error; + } + + case s_header_value_discard_ws: + if (ch == ' ' || ch == '\t') break; + + if (ch == CR) { + UPDATE_STATE(s_header_value_discard_ws_almost_done); + break; + } + + if (ch == LF) { + UPDATE_STATE(s_header_value_discard_lws); + break; + } + + /* FALLTHROUGH */ + + case s_header_value_start: + { + MARK(header_value); + + UPDATE_STATE(s_header_value); + parser->index = 0; + + c = LOWER(ch); + + switch (parser->header_state) { + case h_upgrade: + parser->flags |= F_UPGRADE; + parser->header_state = h_general; + break; + + case h_transfer_encoding: + /* looking for 'Transfer-Encoding: chunked' */ + if ('c' == c) { + parser->header_state = h_matching_transfer_encoding_chunked; + } else { + parser->header_state = h_general; + } + break; + + case h_content_length: + if (UNLIKELY(!IS_NUM(ch))) { + SET_ERRNO(HPE_INVALID_CONTENT_LENGTH); + goto error; + } + + if (parser->flags & F_CONTENTLENGTH) { + SET_ERRNO(HPE_UNEXPECTED_CONTENT_LENGTH); + goto error; + } + + parser->flags |= F_CONTENTLENGTH; + parser->content_length = ch - '0'; + break; + + case h_connection: + /* looking for 'Connection: keep-alive' */ + if (c == 'k') { + parser->header_state = h_matching_connection_keep_alive; + /* looking for 'Connection: close' */ + } else if (c == 'c') { + parser->header_state = h_matching_connection_close; + } else if (c == 'u') { + parser->header_state = h_matching_connection_upgrade; + } else { + parser->header_state = h_matching_connection_token; + } + break; + + /* Multi-value `Connection` header */ + case h_matching_connection_token_start: + break; + + default: + parser->header_state = h_general; + break; + } + break; + } + + case s_header_value: + { + const char* start = p; + enum header_states h_state = (enum header_states) parser->header_state; + for (; p != data + len; p++) { + ch = *p; + if (ch == CR) { + UPDATE_STATE(s_header_almost_done); + parser->header_state = h_state; + CALLBACK_DATA(header_value); + break; + } + + if (ch == LF) { + UPDATE_STATE(s_header_almost_done); + COUNT_HEADER_SIZE(p - start); + parser->header_state = h_state; + CALLBACK_DATA_NOADVANCE(header_value); + REEXECUTE(); + } + + if (!lenient && !IS_HEADER_CHAR(ch)) { + SET_ERRNO(HPE_INVALID_HEADER_TOKEN); + goto error; + } + + c = LOWER(ch); + + switch (h_state) { + case h_general: + { + const char* p_cr; + const char* p_lf; + size_t limit = data + len - p; + + limit = MIN(limit, HTTP_MAX_HEADER_SIZE); + + p_cr = (const char*) memchr(p, CR, limit); + p_lf = (const char*) memchr(p, LF, limit); + if (p_cr != NULL) { + if (p_lf != NULL && p_cr >= p_lf) + p = p_lf; + else + p = p_cr; + } else if (UNLIKELY(p_lf != NULL)) { + p = p_lf; + } else { + p = data + len; + } + --p; + + break; + } + + case h_connection: + case h_transfer_encoding: + assert(0 && "Shouldn't get here."); + break; + + case h_content_length: + { + uint64_t t; + + if (ch == ' ') break; + + if (UNLIKELY(!IS_NUM(ch))) { + SET_ERRNO(HPE_INVALID_CONTENT_LENGTH); + parser->header_state = h_state; + goto error; + } + + t = parser->content_length; + t *= 10; + t += ch - '0'; + + /* Overflow? Test against a conservative limit for simplicity. */ + if (UNLIKELY((ULLONG_MAX - 10) / 10 < parser->content_length)) { + SET_ERRNO(HPE_INVALID_CONTENT_LENGTH); + parser->header_state = h_state; + goto error; + } + + parser->content_length = t; + break; + } + + /* Transfer-Encoding: chunked */ + case h_matching_transfer_encoding_chunked: + parser->index++; + if (parser->index > sizeof(CHUNKED)-1 + || c != CHUNKED[parser->index]) { + h_state = h_general; + } else if (parser->index == sizeof(CHUNKED)-2) { + h_state = h_transfer_encoding_chunked; + } + break; + + case h_matching_connection_token_start: + /* looking for 'Connection: keep-alive' */ + if (c == 'k') { + h_state = h_matching_connection_keep_alive; + /* looking for 'Connection: close' */ + } else if (c == 'c') { + h_state = h_matching_connection_close; + } else if (c == 'u') { + h_state = h_matching_connection_upgrade; + } else if (STRICT_TOKEN(c)) { + h_state = h_matching_connection_token; + } else if (c == ' ' || c == '\t') { + /* Skip lws */ + } else { + h_state = h_general; + } + break; + + /* looking for 'Connection: keep-alive' */ + case h_matching_connection_keep_alive: + parser->index++; + if (parser->index > sizeof(KEEP_ALIVE)-1 + || c != KEEP_ALIVE[parser->index]) { + h_state = h_matching_connection_token; + } else if (parser->index == sizeof(KEEP_ALIVE)-2) { + h_state = h_connection_keep_alive; + } + break; + + /* looking for 'Connection: close' */ + case h_matching_connection_close: + parser->index++; + if (parser->index > sizeof(CLOSE)-1 || c != CLOSE[parser->index]) { + h_state = h_matching_connection_token; + } else if (parser->index == sizeof(CLOSE)-2) { + h_state = h_connection_close; + } + break; + + /* looking for 'Connection: upgrade' */ + case h_matching_connection_upgrade: + parser->index++; + if (parser->index > sizeof(UPGRADE) - 1 || + c != UPGRADE[parser->index]) { + h_state = h_matching_connection_token; + } else if (parser->index == sizeof(UPGRADE)-2) { + h_state = h_connection_upgrade; + } + break; + + case h_matching_connection_token: + if (ch == ',') { + h_state = h_matching_connection_token_start; + parser->index = 0; + } + break; + + case h_transfer_encoding_chunked: + if (ch != ' ') h_state = h_general; + break; + + case h_connection_keep_alive: + case h_connection_close: + case h_connection_upgrade: + if (ch == ',') { + if (h_state == h_connection_keep_alive) { + parser->flags |= F_CONNECTION_KEEP_ALIVE; + } else if (h_state == h_connection_close) { + parser->flags |= F_CONNECTION_CLOSE; + } else if (h_state == h_connection_upgrade) { + parser->flags |= F_CONNECTION_UPGRADE; + } + h_state = h_matching_connection_token_start; + parser->index = 0; + } else if (ch != ' ') { + h_state = h_matching_connection_token; + } + break; + + default: + UPDATE_STATE(s_header_value); + h_state = h_general; + break; + } + } + parser->header_state = h_state; + + COUNT_HEADER_SIZE(p - start); + + if (p == data + len) + --p; + break; + } + + case s_header_almost_done: + { + if (UNLIKELY(ch != LF)) { + SET_ERRNO(HPE_LF_EXPECTED); + goto error; + } + + UPDATE_STATE(s_header_value_lws); + break; + } + + case s_header_value_lws: + { + if (ch == ' ' || ch == '\t') { + UPDATE_STATE(s_header_value_start); + REEXECUTE(); + } + + /* finished the header */ + switch (parser->header_state) { + case h_connection_keep_alive: + parser->flags |= F_CONNECTION_KEEP_ALIVE; + break; + case h_connection_close: + parser->flags |= F_CONNECTION_CLOSE; + break; + case h_transfer_encoding_chunked: + parser->flags |= F_CHUNKED; + break; + case h_connection_upgrade: + parser->flags |= F_CONNECTION_UPGRADE; + break; + default: + break; + } + + UPDATE_STATE(s_header_field_start); + REEXECUTE(); + } + + case s_header_value_discard_ws_almost_done: + { + STRICT_CHECK(ch != LF); + UPDATE_STATE(s_header_value_discard_lws); + break; + } + + case s_header_value_discard_lws: + { + if (ch == ' ' || ch == '\t') { + UPDATE_STATE(s_header_value_discard_ws); + break; + } else { + switch (parser->header_state) { + case h_connection_keep_alive: + parser->flags |= F_CONNECTION_KEEP_ALIVE; + break; + case h_connection_close: + parser->flags |= F_CONNECTION_CLOSE; + break; + case h_connection_upgrade: + parser->flags |= F_CONNECTION_UPGRADE; + break; + case h_transfer_encoding_chunked: + parser->flags |= F_CHUNKED; + break; + default: + break; + } + + /* header value was empty */ + MARK(header_value); + UPDATE_STATE(s_header_field_start); + CALLBACK_DATA_NOADVANCE(header_value); + REEXECUTE(); + } + } + + case s_headers_almost_done: + { + STRICT_CHECK(ch != LF); + + if (parser->flags & F_TRAILING) { + /* End of a chunked request */ + UPDATE_STATE(s_message_done); + CALLBACK_NOTIFY_NOADVANCE(chunk_complete); + REEXECUTE(); + } + + /* Cannot use chunked encoding and a content-length header together + per the HTTP specification. */ + if ((parser->flags & F_CHUNKED) && + (parser->flags & F_CONTENTLENGTH)) { + SET_ERRNO(HPE_UNEXPECTED_CONTENT_LENGTH); + goto error; + } + + UPDATE_STATE(s_headers_done); + + /* Set this here so that on_headers_complete() callbacks can see it */ + parser->upgrade = + ((parser->flags & (F_UPGRADE | F_CONNECTION_UPGRADE)) == + (F_UPGRADE | F_CONNECTION_UPGRADE) || + parser->method == HTTP_CONNECT); + + /* Here we call the headers_complete callback. This is somewhat + * different than other callbacks because if the user returns 1, we + * will interpret that as saying that this message has no body. This + * is needed for the annoying case of recieving a response to a HEAD + * request. + * + * We'd like to use CALLBACK_NOTIFY_NOADVANCE() here but we cannot, so + * we have to simulate it by handling a change in errno below. + */ + if (settings->on_headers_complete) { + switch (settings->on_headers_complete(parser)) { + case 0: + break; + + case 2: + parser->upgrade = 1; + + case 1: + parser->flags |= F_SKIPBODY; + break; + + default: + SET_ERRNO(HPE_CB_headers_complete); + RETURN(p - data); /* Error */ + } + } + + if (HTTP_PARSER_ERRNO(parser) != HPE_OK) { + RETURN(p - data); + } + + REEXECUTE(); + } + + case s_headers_done: + { + int hasBody; + STRICT_CHECK(ch != LF); + + parser->nread = 0; + + hasBody = parser->flags & F_CHUNKED || + (parser->content_length > 0 && parser->content_length != ULLONG_MAX); + if (parser->upgrade && (parser->method == HTTP_CONNECT || + (parser->flags & F_SKIPBODY) || !hasBody)) { + /* Exit, the rest of the message is in a different protocol. */ + UPDATE_STATE(NEW_MESSAGE()); + CALLBACK_NOTIFY(message_complete); + RETURN((p - data) + 1); + } + + if (parser->flags & F_SKIPBODY) { + UPDATE_STATE(NEW_MESSAGE()); + CALLBACK_NOTIFY(message_complete); + } else if (parser->flags & F_CHUNKED) { + /* chunked encoding - ignore Content-Length header */ + UPDATE_STATE(s_chunk_size_start); + } else { + if (parser->content_length == 0) { + /* Content-Length header given but zero: Content-Length: 0\r\n */ + UPDATE_STATE(NEW_MESSAGE()); + CALLBACK_NOTIFY(message_complete); + } else if (parser->content_length != ULLONG_MAX) { + /* Content-Length header given and non-zero */ + UPDATE_STATE(s_body_identity); + } else { + if (!http_message_needs_eof(parser)) { + /* Assume content-length 0 - read the next */ + UPDATE_STATE(NEW_MESSAGE()); + CALLBACK_NOTIFY(message_complete); + } else { + /* Read body until EOF */ + UPDATE_STATE(s_body_identity_eof); + } + } + } + + break; + } + + case s_body_identity: + { + uint64_t to_read = MIN(parser->content_length, + (uint64_t) ((data + len) - p)); + + assert(parser->content_length != 0 + && parser->content_length != ULLONG_MAX); + + /* The difference between advancing content_length and p is because + * the latter will automaticaly advance on the next loop iteration. + * Further, if content_length ends up at 0, we want to see the last + * byte again for our message complete callback. + */ + MARK(body); + parser->content_length -= to_read; + p += to_read - 1; + + if (parser->content_length == 0) { + UPDATE_STATE(s_message_done); + + /* Mimic CALLBACK_DATA_NOADVANCE() but with one extra byte. + * + * The alternative to doing this is to wait for the next byte to + * trigger the data callback, just as in every other case. The + * problem with this is that this makes it difficult for the test + * harness to distinguish between complete-on-EOF and + * complete-on-length. It's not clear that this distinction is + * important for applications, but let's keep it for now. + */ + CALLBACK_DATA_(body, p - body_mark + 1, p - data); + REEXECUTE(); + } + + break; + } + + /* read until EOF */ + case s_body_identity_eof: + MARK(body); + p = data + len - 1; + + break; + + case s_message_done: + UPDATE_STATE(NEW_MESSAGE()); + CALLBACK_NOTIFY(message_complete); + if (parser->upgrade) { + /* Exit, the rest of the message is in a different protocol. */ + RETURN((p - data) + 1); + } + break; + + case s_chunk_size_start: + { + assert(parser->nread == 1); + assert(parser->flags & F_CHUNKED); + + unhex_val = unhex[(unsigned char)ch]; + if (UNLIKELY(unhex_val == -1)) { + SET_ERRNO(HPE_INVALID_CHUNK_SIZE); + goto error; + } + + parser->content_length = unhex_val; + UPDATE_STATE(s_chunk_size); + break; + } + + case s_chunk_size: + { + uint64_t t; + + assert(parser->flags & F_CHUNKED); + + if (ch == CR) { + UPDATE_STATE(s_chunk_size_almost_done); + break; + } + + unhex_val = unhex[(unsigned char)ch]; + + if (unhex_val == -1) { + if (ch == ';' || ch == ' ') { + UPDATE_STATE(s_chunk_parameters); + break; + } + + SET_ERRNO(HPE_INVALID_CHUNK_SIZE); + goto error; + } + + t = parser->content_length; + t *= 16; + t += unhex_val; + + /* Overflow? Test against a conservative limit for simplicity. */ + if (UNLIKELY((ULLONG_MAX - 16) / 16 < parser->content_length)) { + SET_ERRNO(HPE_INVALID_CONTENT_LENGTH); + goto error; + } + + parser->content_length = t; + break; + } + + case s_chunk_parameters: + { + assert(parser->flags & F_CHUNKED); + /* just ignore this shit. TODO check for overflow */ + if (ch == CR) { + UPDATE_STATE(s_chunk_size_almost_done); + break; + } + break; + } + + case s_chunk_size_almost_done: + { + assert(parser->flags & F_CHUNKED); + STRICT_CHECK(ch != LF); + + parser->nread = 0; + + if (parser->content_length == 0) { + parser->flags |= F_TRAILING; + UPDATE_STATE(s_header_field_start); + } else { + UPDATE_STATE(s_chunk_data); + } + CALLBACK_NOTIFY(chunk_header); + break; + } + + case s_chunk_data: + { + uint64_t to_read = MIN(parser->content_length, + (uint64_t) ((data + len) - p)); + + assert(parser->flags & F_CHUNKED); + assert(parser->content_length != 0 + && parser->content_length != ULLONG_MAX); + + /* See the explanation in s_body_identity for why the content + * length and data pointers are managed this way. + */ + MARK(body); + parser->content_length -= to_read; + p += to_read - 1; + + if (parser->content_length == 0) { + UPDATE_STATE(s_chunk_data_almost_done); + } + + break; + } + + case s_chunk_data_almost_done: + assert(parser->flags & F_CHUNKED); + assert(parser->content_length == 0); + STRICT_CHECK(ch != CR); + UPDATE_STATE(s_chunk_data_done); + CALLBACK_DATA(body); + break; + + case s_chunk_data_done: + assert(parser->flags & F_CHUNKED); + STRICT_CHECK(ch != LF); + parser->nread = 0; + UPDATE_STATE(s_chunk_size_start); + CALLBACK_NOTIFY(chunk_complete); + break; + + default: + assert(0 && "unhandled state"); + SET_ERRNO(HPE_INVALID_INTERNAL_STATE); + goto error; + } + } + + /* Run callbacks for any marks that we have leftover after we ran our of + * bytes. There should be at most one of these set, so it's OK to invoke + * them in series (unset marks will not result in callbacks). + * + * We use the NOADVANCE() variety of callbacks here because 'p' has already + * overflowed 'data' and this allows us to correct for the off-by-one that + * we'd otherwise have (since CALLBACK_DATA() is meant to be run with a 'p' + * value that's in-bounds). + */ + + assert(((header_field_mark ? 1 : 0) + + (header_value_mark ? 1 : 0) + + (url_mark ? 1 : 0) + + (body_mark ? 1 : 0) + + (status_mark ? 1 : 0)) <= 1); + + CALLBACK_DATA_NOADVANCE(header_field); + CALLBACK_DATA_NOADVANCE(header_value); + CALLBACK_DATA_NOADVANCE(url); + CALLBACK_DATA_NOADVANCE(body); + CALLBACK_DATA_NOADVANCE(status); + + RETURN(len); + +error: + if (HTTP_PARSER_ERRNO(parser) == HPE_OK) { + SET_ERRNO(HPE_UNKNOWN); + } + + RETURN(p - data); +} + + +/* Does the parser need to see an EOF to find the end of the message? */ +int +http_message_needs_eof (const http_parser *parser) +{ + if (parser->type == HTTP_REQUEST) { + return 0; + } + + /* See RFC 2616 section 4.4 */ + if (parser->status_code / 100 == 1 || /* 1xx e.g. Continue */ + parser->status_code == 204 || /* No Content */ + parser->status_code == 304 || /* Not Modified */ + parser->flags & F_SKIPBODY) { /* response to a HEAD request */ + return 0; + } + + if ((parser->flags & F_CHUNKED) || parser->content_length != ULLONG_MAX) { + return 0; + } + + return 1; +} + + +int +http_should_keep_alive (const http_parser *parser) +{ + if (parser->http_major > 0 && parser->http_minor > 0) { + /* HTTP/1.1 */ + if (parser->flags & F_CONNECTION_CLOSE) { + return 0; + } + } else { + /* HTTP/1.0 or earlier */ + if (!(parser->flags & F_CONNECTION_KEEP_ALIVE)) { + return 0; + } + } + + return !http_message_needs_eof(parser); +} + + +const char * +http_method_str (enum http_method m) +{ + return ELEM_AT(method_strings, m, ""); +} + + +void +http_parser_init (http_parser *parser, enum http_parser_type t) +{ + void *data = parser->data; /* preserve application data */ + memset(parser, 0, sizeof(*parser)); + parser->data = data; + parser->type = t; + parser->state = (t == HTTP_REQUEST ? s_start_req : (t == HTTP_RESPONSE ? s_start_res : s_start_req_or_res)); + parser->http_errno = HPE_OK; +} + +void +http_parser_settings_init(http_parser_settings *settings) +{ + memset(settings, 0, sizeof(*settings)); +} + +const char * +http_errno_name(enum http_errno err) { + assert(((size_t) err) < ARRAY_SIZE(http_strerror_tab)); + return http_strerror_tab[err].name; +} + +const char * +http_errno_description(enum http_errno err) { + assert(((size_t) err) < ARRAY_SIZE(http_strerror_tab)); + return http_strerror_tab[err].description; +} + +static enum http_host_state +http_parse_host_char(enum http_host_state s, const char ch) { + switch(s) { + case s_http_userinfo: + case s_http_userinfo_start: + if (ch == '@') { + return s_http_host_start; + } + + if (IS_USERINFO_CHAR(ch)) { + return s_http_userinfo; + } + break; + + case s_http_host_start: + if (ch == '[') { + return s_http_host_v6_start; + } + + if (IS_HOST_CHAR(ch)) { + return s_http_host; + } + + break; + + case s_http_host: + if (IS_HOST_CHAR(ch)) { + return s_http_host; + } + + /* FALLTHROUGH */ + case s_http_host_v6_end: + if (ch == ':') { + return s_http_host_port_start; + } + + break; + + case s_http_host_v6: + if (ch == ']') { + return s_http_host_v6_end; + } + + /* FALLTHROUGH */ + case s_http_host_v6_start: + if (IS_HEX(ch) || ch == ':' || ch == '.') { + return s_http_host_v6; + } + + if (s == s_http_host_v6 && ch == '%') { + return s_http_host_v6_zone_start; + } + break; + + case s_http_host_v6_zone: + if (ch == ']') { + return s_http_host_v6_end; + } + + /* FALLTHROUGH */ + case s_http_host_v6_zone_start: + /* RFC 6874 Zone ID consists of 1*( unreserved / pct-encoded) */ + if (IS_ALPHANUM(ch) || ch == '%' || ch == '.' || ch == '-' || ch == '_' || + ch == '~') { + return s_http_host_v6_zone; + } + break; + + case s_http_host_port: + case s_http_host_port_start: + if (IS_NUM(ch)) { + return s_http_host_port; + } + + break; + + default: + break; + } + return s_http_host_dead; +} + +static int +http_parse_host(const char * buf, struct http_parser_url *u, int found_at) { + enum http_host_state s; + + const char *p; + size_t buflen = u->field_data[UF_HOST].off + u->field_data[UF_HOST].len; + + assert(u->field_set & (1 << UF_HOST)); + + u->field_data[UF_HOST].len = 0; + + s = found_at ? s_http_userinfo_start : s_http_host_start; + + for (p = buf + u->field_data[UF_HOST].off; p < buf + buflen; p++) { + enum http_host_state new_s = http_parse_host_char(s, *p); + + if (new_s == s_http_host_dead) { + return 1; + } + + switch(new_s) { + case s_http_host: + if (s != s_http_host) { + u->field_data[UF_HOST].off = p - buf; + } + u->field_data[UF_HOST].len++; + break; + + case s_http_host_v6: + if (s != s_http_host_v6) { + u->field_data[UF_HOST].off = p - buf; + } + u->field_data[UF_HOST].len++; + break; + + case s_http_host_v6_zone_start: + case s_http_host_v6_zone: + u->field_data[UF_HOST].len++; + break; + + case s_http_host_port: + if (s != s_http_host_port) { + u->field_data[UF_PORT].off = p - buf; + u->field_data[UF_PORT].len = 0; + u->field_set |= (1 << UF_PORT); + } + u->field_data[UF_PORT].len++; + break; + + case s_http_userinfo: + if (s != s_http_userinfo) { + u->field_data[UF_USERINFO].off = p - buf ; + u->field_data[UF_USERINFO].len = 0; + u->field_set |= (1 << UF_USERINFO); + } + u->field_data[UF_USERINFO].len++; + break; + + default: + break; + } + s = new_s; + } + + /* Make sure we don't end somewhere unexpected */ + switch (s) { + case s_http_host_start: + case s_http_host_v6_start: + case s_http_host_v6: + case s_http_host_v6_zone_start: + case s_http_host_v6_zone: + case s_http_host_port_start: + case s_http_userinfo: + case s_http_userinfo_start: + return 1; + default: + break; + } + + return 0; +} + +void +http_parser_url_init(struct http_parser_url *u) { + memset(u, 0, sizeof(*u)); +} + +int +http_parser_parse_url(const char *buf, size_t buflen, int is_connect, + struct http_parser_url *u) +{ + enum state s; + const char *p; + enum http_parser_url_fields uf, old_uf; + int found_at = 0; + + u->port = u->field_set = 0; + s = is_connect ? s_req_server_start : s_req_spaces_before_url; + old_uf = UF_MAX; + + for (p = buf; p < buf + buflen; p++) { + s = parse_url_char(s, *p); + + /* Figure out the next field that we're operating on */ + switch (s) { + case s_dead: + return 1; + + /* Skip delimeters */ + case s_req_schema_slash: + case s_req_schema_slash_slash: + case s_req_server_start: + case s_req_query_string_start: + case s_req_fragment_start: + continue; + + case s_req_schema: + uf = UF_SCHEMA; + break; + + case s_req_server_with_at: + found_at = 1; + + /* FALLTROUGH */ + case s_req_server: + uf = UF_HOST; + break; + + case s_req_path: + uf = UF_PATH; + break; + + case s_req_query_string: + uf = UF_QUERY; + break; + + case s_req_fragment: + uf = UF_FRAGMENT; + break; + + default: + assert(!"Unexpected state"); + return 1; + } + + /* Nothing's changed; soldier on */ + if (uf == old_uf) { + u->field_data[uf].len++; + continue; + } + + u->field_data[uf].off = p - buf; + u->field_data[uf].len = 1; + + u->field_set |= (1 << uf); + old_uf = uf; + } + + /* host must be present if there is a schema */ + /* parsing http:///toto will fail */ + if ((u->field_set & (1 << UF_SCHEMA)) && + (u->field_set & (1 << UF_HOST)) == 0) { + return 1; + } + + if (u->field_set & (1 << UF_HOST)) { + if (http_parse_host(buf, u, found_at) != 0) { + return 1; + } + } + + /* CONNECT requests can only contain "hostname:port" */ + if (is_connect && u->field_set != ((1 << UF_HOST)|(1 << UF_PORT))) { + return 1; + } + + if (u->field_set & (1 << UF_PORT)) { + /* Don't bother with endp; we've already validated the string */ + unsigned long v = strtoul(buf + u->field_data[UF_PORT].off, NULL, 10); + + /* Ports have a max value of 2^16 */ + if (v > 0xffff) { + return 1; + } + + u->port = (uint16_t) v; + } + + return 0; +} + +void +http_parser_pause(http_parser *parser, int paused) { + /* Users should only be pausing/unpausing a parser that is not in an error + * state. In non-debug builds, there's not much that we can do about this + * other than ignore it. + */ + if (HTTP_PARSER_ERRNO(parser) == HPE_OK || + HTTP_PARSER_ERRNO(parser) == HPE_PAUSED) { + SET_ERRNO((paused) ? HPE_PAUSED : HPE_OK); + } else { + assert(0 && "Attempting to pause parser in error state"); + } +} + +int +http_body_is_final(const struct http_parser *parser) { + return parser->state == s_message_done; +} + +unsigned long +http_parser_version(void) { + return HTTP_PARSER_VERSION_MAJOR * 0x10000 | + HTTP_PARSER_VERSION_MINOR * 0x00100 | + HTTP_PARSER_VERSION_PATCH * 0x00001; +} diff --git a/userspace/libsinsp/http_parser.h b/userspace/libsinsp/http_parser.h new file mode 100644 index 0000000000..ea26394824 --- /dev/null +++ b/userspace/libsinsp/http_parser.h @@ -0,0 +1,362 @@ +/* Copyright Joyent, Inc. and other Node contributors. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ +#ifndef http_parser_h +#define http_parser_h +#ifdef __cplusplus +extern "C" { +#endif + +/* Also update SONAME in the Makefile whenever you change these. */ +#define HTTP_PARSER_VERSION_MAJOR 2 +#define HTTP_PARSER_VERSION_MINOR 7 +#define HTTP_PARSER_VERSION_PATCH 1 + +#include +#if defined(_WIN32) && !defined(__MINGW32__) && \ + (!defined(_MSC_VER) || _MSC_VER<1600) && !defined(__WINE__) +#include +#include +typedef __int8 int8_t; +typedef unsigned __int8 uint8_t; +typedef __int16 int16_t; +typedef unsigned __int16 uint16_t; +typedef __int32 int32_t; +typedef unsigned __int32 uint32_t; +typedef __int64 int64_t; +typedef unsigned __int64 uint64_t; +#else +#include +#endif + +/* Compile with -DHTTP_PARSER_STRICT=0 to make less checks, but run + * faster + */ +#ifndef HTTP_PARSER_STRICT +# define HTTP_PARSER_STRICT 1 +#endif + +/* Maximium header size allowed. If the macro is not defined + * before including this header then the default is used. To + * change the maximum header size, define the macro in the build + * environment (e.g. -DHTTP_MAX_HEADER_SIZE=). To remove + * the effective limit on the size of the header, define the macro + * to a very large number (e.g. -DHTTP_MAX_HEADER_SIZE=0x7fffffff) + */ +#ifndef HTTP_MAX_HEADER_SIZE +# define HTTP_MAX_HEADER_SIZE (80*1024) +#endif + +typedef struct http_parser http_parser; +typedef struct http_parser_settings http_parser_settings; + + +/* Callbacks should return non-zero to indicate an error. The parser will + * then halt execution. + * + * The one exception is on_headers_complete. In a HTTP_RESPONSE parser + * returning '1' from on_headers_complete will tell the parser that it + * should not expect a body. This is used when receiving a response to a + * HEAD request which may contain 'Content-Length' or 'Transfer-Encoding: + * chunked' headers that indicate the presence of a body. + * + * Returning `2` from on_headers_complete will tell parser that it should not + * expect neither a body nor any futher responses on this connection. This is + * useful for handling responses to a CONNECT request which may not contain + * `Upgrade` or `Connection: upgrade` headers. + * + * http_data_cb does not return data chunks. It will be called arbitrarily + * many times for each string. E.G. you might get 10 callbacks for "on_url" + * each providing just a few characters more data. + */ +typedef int (*http_data_cb) (http_parser*, const char *at, size_t length); +typedef int (*http_cb) (http_parser*); + + +/* Request Methods */ +#define HTTP_METHOD_MAP(XX) \ + XX(0, DELETE, DELETE) \ + XX(1, GET, GET) \ + XX(2, HEAD, HEAD) \ + XX(3, POST, POST) \ + XX(4, PUT, PUT) \ + /* pathological */ \ + XX(5, CONNECT, CONNECT) \ + XX(6, OPTIONS, OPTIONS) \ + XX(7, TRACE, TRACE) \ + /* WebDAV */ \ + XX(8, COPY, COPY) \ + XX(9, LOCK, LOCK) \ + XX(10, MKCOL, MKCOL) \ + XX(11, MOVE, MOVE) \ + XX(12, PROPFIND, PROPFIND) \ + XX(13, PROPPATCH, PROPPATCH) \ + XX(14, SEARCH, SEARCH) \ + XX(15, UNLOCK, UNLOCK) \ + XX(16, BIND, BIND) \ + XX(17, REBIND, REBIND) \ + XX(18, UNBIND, UNBIND) \ + XX(19, ACL, ACL) \ + /* subversion */ \ + XX(20, REPORT, REPORT) \ + XX(21, MKACTIVITY, MKACTIVITY) \ + XX(22, CHECKOUT, CHECKOUT) \ + XX(23, MERGE, MERGE) \ + /* upnp */ \ + XX(24, MSEARCH, M-SEARCH) \ + XX(25, NOTIFY, NOTIFY) \ + XX(26, SUBSCRIBE, SUBSCRIBE) \ + XX(27, UNSUBSCRIBE, UNSUBSCRIBE) \ + /* RFC-5789 */ \ + XX(28, PATCH, PATCH) \ + XX(29, PURGE, PURGE) \ + /* CalDAV */ \ + XX(30, MKCALENDAR, MKCALENDAR) \ + /* RFC-2068, section 19.6.1.2 */ \ + XX(31, LINK, LINK) \ + XX(32, UNLINK, UNLINK) \ + +enum http_method + { +#define XX(num, name, string) HTTP_##name = num, + HTTP_METHOD_MAP(XX) +#undef XX + }; + + +enum http_parser_type { HTTP_REQUEST, HTTP_RESPONSE, HTTP_BOTH }; + + +/* Flag values for http_parser.flags field */ +enum flags + { F_CHUNKED = 1 << 0 + , F_CONNECTION_KEEP_ALIVE = 1 << 1 + , F_CONNECTION_CLOSE = 1 << 2 + , F_CONNECTION_UPGRADE = 1 << 3 + , F_TRAILING = 1 << 4 + , F_UPGRADE = 1 << 5 + , F_SKIPBODY = 1 << 6 + , F_CONTENTLENGTH = 1 << 7 + }; + + +/* Map for errno-related constants + * + * The provided argument should be a macro that takes 2 arguments. + */ +#define HTTP_ERRNO_MAP(XX) \ + /* No error */ \ + XX(OK, "success") \ + \ + /* Callback-related errors */ \ + XX(CB_message_begin, "the on_message_begin callback failed") \ + XX(CB_url, "the on_url callback failed") \ + XX(CB_header_field, "the on_header_field callback failed") \ + XX(CB_header_value, "the on_header_value callback failed") \ + XX(CB_headers_complete, "the on_headers_complete callback failed") \ + XX(CB_body, "the on_body callback failed") \ + XX(CB_message_complete, "the on_message_complete callback failed") \ + XX(CB_status, "the on_status callback failed") \ + XX(CB_chunk_header, "the on_chunk_header callback failed") \ + XX(CB_chunk_complete, "the on_chunk_complete callback failed") \ + \ + /* Parsing-related errors */ \ + XX(INVALID_EOF_STATE, "stream ended at an unexpected time") \ + XX(HEADER_OVERFLOW, \ + "too many header bytes seen; overflow detected") \ + XX(CLOSED_CONNECTION, \ + "data received after completed connection: close message") \ + XX(INVALID_VERSION, "invalid HTTP version") \ + XX(INVALID_STATUS, "invalid HTTP status code") \ + XX(INVALID_METHOD, "invalid HTTP method") \ + XX(INVALID_URL, "invalid URL") \ + XX(INVALID_HOST, "invalid host") \ + XX(INVALID_PORT, "invalid port") \ + XX(INVALID_PATH, "invalid path") \ + XX(INVALID_QUERY_STRING, "invalid query string") \ + XX(INVALID_FRAGMENT, "invalid fragment") \ + XX(LF_EXPECTED, "LF character expected") \ + XX(INVALID_HEADER_TOKEN, "invalid character in header") \ + XX(INVALID_CONTENT_LENGTH, \ + "invalid character in content-length header") \ + XX(UNEXPECTED_CONTENT_LENGTH, \ + "unexpected content-length header") \ + XX(INVALID_CHUNK_SIZE, \ + "invalid character in chunk size header") \ + XX(INVALID_CONSTANT, "invalid constant string") \ + XX(INVALID_INTERNAL_STATE, "encountered unexpected internal state")\ + XX(STRICT, "strict mode assertion failed") \ + XX(PAUSED, "parser is paused") \ + XX(UNKNOWN, "an unknown error occurred") + + +/* Define HPE_* values for each errno value above */ +#define HTTP_ERRNO_GEN(n, s) HPE_##n, +enum http_errno { + HTTP_ERRNO_MAP(HTTP_ERRNO_GEN) +}; +#undef HTTP_ERRNO_GEN + + +/* Get an http_errno value from an http_parser */ +#define HTTP_PARSER_ERRNO(p) ((enum http_errno) (p)->http_errno) + + +struct http_parser { + /** PRIVATE **/ + unsigned int type : 2; /* enum http_parser_type */ + unsigned int flags : 8; /* F_* values from 'flags' enum; semi-public */ + unsigned int state : 7; /* enum state from http_parser.c */ + unsigned int header_state : 7; /* enum header_state from http_parser.c */ + unsigned int index : 7; /* index into current matcher */ + unsigned int lenient_http_headers : 1; + + uint32_t nread; /* # bytes read in various scenarios */ + uint64_t content_length; /* # bytes in body (0 if no Content-Length header) */ + + /** READ-ONLY **/ + unsigned short http_major; + unsigned short http_minor; + unsigned int status_code : 16; /* responses only */ + unsigned int method : 8; /* requests only */ + unsigned int http_errno : 7; + + /* 1 = Upgrade header was present and the parser has exited because of that. + * 0 = No upgrade header present. + * Should be checked when http_parser_execute() returns in addition to + * error checking. + */ + unsigned int upgrade : 1; + + /** PUBLIC **/ + void *data; /* A pointer to get hook to the "connection" or "socket" object */ +}; + + +struct http_parser_settings { + http_cb on_message_begin; + http_data_cb on_url; + http_data_cb on_status; + http_data_cb on_header_field; + http_data_cb on_header_value; + http_cb on_headers_complete; + http_data_cb on_body; + http_cb on_message_complete; + /* When on_chunk_header is called, the current chunk length is stored + * in parser->content_length. + */ + http_cb on_chunk_header; + http_cb on_chunk_complete; +}; + + +enum http_parser_url_fields + { UF_SCHEMA = 0 + , UF_HOST = 1 + , UF_PORT = 2 + , UF_PATH = 3 + , UF_QUERY = 4 + , UF_FRAGMENT = 5 + , UF_USERINFO = 6 + , UF_MAX = 7 + }; + + +/* Result structure for http_parser_parse_url(). + * + * Callers should index into field_data[] with UF_* values iff field_set + * has the relevant (1 << UF_*) bit set. As a courtesy to clients (and + * because we probably have padding left over), we convert any port to + * a uint16_t. + */ +struct http_parser_url { + uint16_t field_set; /* Bitmask of (1 << UF_*) values */ + uint16_t port; /* Converted UF_PORT string */ + + struct { + uint16_t off; /* Offset into buffer in which field starts */ + uint16_t len; /* Length of run in buffer */ + } field_data[UF_MAX]; +}; + + +/* Returns the library version. Bits 16-23 contain the major version number, + * bits 8-15 the minor version number and bits 0-7 the patch level. + * Usage example: + * + * unsigned long version = http_parser_version(); + * unsigned major = (version >> 16) & 255; + * unsigned minor = (version >> 8) & 255; + * unsigned patch = version & 255; + * printf("http_parser v%u.%u.%u\n", major, minor, patch); + */ +unsigned long http_parser_version(void); + +void http_parser_init(http_parser *parser, enum http_parser_type type); + + +/* Initialize http_parser_settings members to 0 + */ +void http_parser_settings_init(http_parser_settings *settings); + + +/* Executes the parser. Returns number of parsed bytes. Sets + * `parser->http_errno` on error. */ +size_t http_parser_execute(http_parser *parser, + const http_parser_settings *settings, + const char *data, + size_t len); + + +/* If http_should_keep_alive() in the on_headers_complete or + * on_message_complete callback returns 0, then this should be + * the last message on the connection. + * If you are the server, respond with the "Connection: close" header. + * If you are the client, close the connection. + */ +int http_should_keep_alive(const http_parser *parser); + +/* Returns a string version of the HTTP method. */ +const char *http_method_str(enum http_method m); + +/* Return a string name of the given error */ +const char *http_errno_name(enum http_errno err); + +/* Return a string description of the given error */ +const char *http_errno_description(enum http_errno err); + +/* Initialize all http_parser_url members to 0 */ +void http_parser_url_init(struct http_parser_url *u); + +/* Parse a URL; return nonzero on failure */ +int http_parser_parse_url(const char *buf, size_t buflen, + int is_connect, + struct http_parser_url *u); + +/* Pause or un-pause the parser; a nonzero value pauses */ +void http_parser_pause(http_parser *parser, int paused); + +/* Checks if this is the final chunk of the body. */ +int http_body_is_final(const http_parser *parser); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/userspace/libsinsp/k8s.cpp b/userspace/libsinsp/k8s.cpp index c649fcc0fe..08d878ef61 100644 --- a/userspace/libsinsp/k8s.cpp +++ b/userspace/libsinsp/k8s.cpp @@ -17,7 +17,9 @@ k8s_component::type_map k8s::m_components; k8s::k8s(const std::string& uri, bool is_captured, #ifdef HAS_CAPTURE - ssl_ptr_t ssl, bt_ptr_t bt, + ssl_ptr_t ssl, + bt_ptr_t bt, + bool block, #endif // HAS_CAPTURE filter_ptr_t event_filter, ext_list_ptr_t extensions) : @@ -25,7 +27,7 @@ k8s::k8s(const std::string& uri, bool is_captured, m_event_filter(event_filter) #ifdef HAS_CAPTURE ,m_net(uri.empty() ? - nullptr : new k8s_net(*this, m_state, uri, ssl, bt, extensions, event_filter)) + nullptr : new k8s_net(*this, m_state, uri, ssl, bt, extensions, event_filter, block)) #endif { g_logger.log(std::string("Creating K8s object for [" + diff --git a/userspace/libsinsp/k8s.h b/userspace/libsinsp/k8s.h index 56dcee83e5..08f9589cd5 100644 --- a/userspace/libsinsp/k8s.h +++ b/userspace/libsinsp/k8s.h @@ -34,6 +34,7 @@ class k8s #ifdef HAS_CAPTURE ssl_ptr_t ssl = 0, bt_ptr_t bt = 0, + bool block = false, #endif // HAS_CAPTURE filter_ptr_t event_filter = nullptr, ext_list_ptr_t extensions = nullptr); @@ -72,12 +73,14 @@ class k8s k8s_state_t m_state; filter_ptr_t m_event_filter; -#ifdef HAS_CAPTURE + typedef std::map> dispatch_map_t; typedef std::map> handler_map_t; - k8s_net* m_net = nullptr; dispatch_map_t m_dispatch_map; handler_map_t m_handler_map; + +#ifdef HAS_CAPTURE + k8s_net* m_net = nullptr; #endif static k8s_component::type_map m_components; @@ -100,15 +103,18 @@ inline void k8s::clear_events() inline std::string k8s::get_machine_id() const { +#ifdef HAS_CAPTURE if(m_net) { return m_net->get_machine_id(); } +#endif // HAS_CAPTURE return ""; } inline void k8s::set_machine_id(const std::string& machine_id) { +#ifdef HAS_CAPTURE if(m_net) { m_net->set_machine_id(machine_id); @@ -119,4 +125,5 @@ inline void k8s::set_machine_id(const std::string& machine_id) "scope may not be available for events.", sinsp_logger::SEV_WARNING); } +#endif // HAS_CAPTURE } \ No newline at end of file diff --git a/userspace/libsinsp/k8s_api_handler.cpp b/userspace/libsinsp/k8s_api_handler.cpp index 206d66cdc5..045c906e34 100644 --- a/userspace/libsinsp/k8s_api_handler.cpp +++ b/userspace/libsinsp/k8s_api_handler.cpp @@ -2,6 +2,8 @@ // k8s_api_handler.cpp // +#ifdef HAS_CAPTURE + #include "k8s_api_handler.h" #include "sinsp.h" #include "sinsp_int.h" @@ -13,12 +15,19 @@ k8s_api_handler::k8s_api_handler(collector_ptr_t collector, const std::string& url, const std::string& path, const std::string& filter, - const std::string& http_version, - ssl_ptr_t ssl, - bt_ptr_t bt): - k8s_handler("k8s_api_handler", false, url, path, - filter, ".", std::make_shared(), - collector, http_version, 1000L, ssl, bt, nullptr, false) + const std::string& http_version +#ifdef HAS_CAPTURE + ,ssl_ptr_t ssl + ,bt_ptr_t bt + ,bool blocking_socket +#endif // HAS_CAPTURE +): + k8s_handler("k8s_api_handler", false, +#ifdef HAS_CAPTURE + url, path, filter, ".", collector, http_version, 1000L, ssl, bt, + false, true, std::make_shared(), blocking_socket, +#endif // HAS_CAPTURE + ~0, nullptr) { } @@ -41,8 +50,8 @@ bool k8s_api_handler::handle_component(const Json::Value& json, const msg_data* } else { - g_logger.log("K8s API handler error: could not extract versions from JSON.", - sinsp_logger::SEV_ERROR); + g_logger.log("K8s API handler error: could not extract API versions or extensions from JSON.", + sinsp_logger::SEV_ERROR); m_error = true; return false; } @@ -54,7 +63,7 @@ bool k8s_api_handler::handle_component(const Json::Value& json, const msg_data* } else { - g_logger.log("K8s API handler error: could not extract version from JSON.", + g_logger.log("K8s API handler error: could not extract API versions or extensions from JSON.", sinsp_logger::SEV_ERROR); m_error = true; return false; @@ -74,7 +83,8 @@ void k8s_api_handler::handle_json(Json::Value&& root) { if(g_logger.get_severity() >= sinsp_logger::SEV_TRACE) { - g_logger.log("K8S API handler: \n" + json_as_string(root), sinsp_logger::SEV_TRACE); + g_logger.log("K8S API handler [" + json_as_string(root) + "] reply:\n", + sinsp_logger::SEV_TRACE); } handle_component(root); @@ -91,4 +101,6 @@ bool k8s_api_handler::has(const std::string& version) const } return false; } + + #endif // HAS_CAPTURE \ No newline at end of file diff --git a/userspace/libsinsp/k8s_api_handler.h b/userspace/libsinsp/k8s_api_handler.h index 578c025cab..a63a527fea 100644 --- a/userspace/libsinsp/k8s_api_handler.h +++ b/userspace/libsinsp/k8s_api_handler.h @@ -2,6 +2,8 @@ // k8s_api_handler.h // +#ifdef HAS_CAPTURE + #pragma once #include "json/json.h" @@ -16,13 +18,13 @@ class k8s_api_handler : public k8s_handler const std::string& url, const std::string& path, const std::string& filter, - const std::string& http_version = "1.0", + const std::string& http_version = "1.1", ssl_ptr_t ssl = 0, - bt_ptr_t bt = 0); + bt_ptr_t bt = 0, + bool blocking_socket = false); ~k8s_api_handler(); - bool ready() const; bool error() const; const api_list_t& extensions() const; bool has(const std::string& version) const; @@ -33,15 +35,9 @@ class k8s_api_handler : public k8s_handler api_list_t m_extensions; - bool m_data_received = false; bool m_error = false; }; -inline bool k8s_api_handler::ready() const -{ - return m_data_received; -} - inline bool k8s_api_handler::error() const { return m_error; @@ -51,3 +47,5 @@ inline const k8s_api_handler::api_list_t& k8s_api_handler::extensions() const { return m_extensions; } + +#endif // HAS_CAPTURE diff --git a/userspace/libsinsp/k8s_component.cpp b/userspace/libsinsp/k8s_component.cpp index 3b618b9dc7..968b51e02e 100644 --- a/userspace/libsinsp/k8s_component.cpp +++ b/userspace/libsinsp/k8s_component.cpp @@ -923,10 +923,11 @@ bool k8s_event_t::update(const Json::Value& item, k8s_state_t& state) tags["source"] = "kubernetes"; g_logger.log(sinsp_user_event::to_string(epoch_time_evt_s, std::move(event_name), std::move(description), std::move(scope), std::move(tags)), severity); - return true; // TODO: sysdig capture? #endif // _WIN32 + + return true; } void k8s_event_t::make_scope_impl(const Json::Value& obj, std::string comp, std::string& scope, bool ns) diff --git a/userspace/libsinsp/k8s_daemonset_handler.cpp b/userspace/libsinsp/k8s_daemonset_handler.cpp index a233eba8a5..16cda62eb8 100644 --- a/userspace/libsinsp/k8s_daemonset_handler.cpp +++ b/userspace/libsinsp/k8s_daemonset_handler.cpp @@ -51,18 +51,28 @@ std::string k8s_daemonset_handler::STATE_FILTER = " ]" "}"; -k8s_daemonset_handler::k8s_daemonset_handler(k8s_state_t& state, - ptr_t dependency_handler, - collector_ptr_t collector, - std::string url, - const std::string& http_version, - ssl_ptr_t ssl, - bt_ptr_t bt, - bool connect): - k8s_handler("k8s_daemonset_handler", true, url, +k8s_daemonset_handler::k8s_daemonset_handler(k8s_state_t& state +#ifdef HAS_CAPTURE + ,ptr_t dependency_handler + ,collector_ptr_t collector + ,std::string url + ,const std::string& http_version + ,ssl_ptr_t ssl + ,bt_ptr_t bt + ,bool connect + ,bool blocking_socket +#endif // HAS_CAPTURE + ): + k8s_handler("k8s_daemonset_handler", true, +#ifdef HAS_CAPTURE + url, "/apis/extensions/v1beta1/daemonsets", - STATE_FILTER, EVENT_FILTER, dependency_handler, collector, - http_version, 1000L, ssl, bt, &state, true, connect) + STATE_FILTER, EVENT_FILTER, collector, + http_version, 1000L, ssl, bt, true, + connect, dependency_handler, blocking_socket, +#endif // HAS_CAPTURE + 100, // max msgs + &state) { } diff --git a/userspace/libsinsp/k8s_daemonset_handler.h b/userspace/libsinsp/k8s_daemonset_handler.h index 9dc2383e72..19c940f9a5 100644 --- a/userspace/libsinsp/k8s_daemonset_handler.h +++ b/userspace/libsinsp/k8s_daemonset_handler.h @@ -14,14 +14,18 @@ class sinsp; class k8s_daemonset_handler : public k8s_handler { public: - k8s_daemonset_handler(k8s_state_t& state, - ptr_t dependency_handler, - collector_ptr_t collector = nullptr, - std::string url = "", - const std::string& http_version = "1.0", - ssl_ptr_t ssl = 0, - bt_ptr_t bt = 0, - bool connect = true); + k8s_daemonset_handler(k8s_state_t& state +#ifdef HAS_CAPTURE + ,ptr_t dependency_handler + ,collector_ptr_t collector = nullptr + ,std::string url = "" + ,const std::string& http_version = "1.1" + ,ssl_ptr_t ssl = 0 + ,bt_ptr_t bt = 0 + ,bool connect = true + ,bool blocking_socket = false +#endif // HAS_CAPTURE + ); ~k8s_daemonset_handler(); diff --git a/userspace/libsinsp/k8s_deployment_handler.cpp b/userspace/libsinsp/k8s_deployment_handler.cpp index f534a131ae..804cf73e47 100644 --- a/userspace/libsinsp/k8s_deployment_handler.cpp +++ b/userspace/libsinsp/k8s_deployment_handler.cpp @@ -51,18 +51,27 @@ std::string k8s_deployment_handler::STATE_FILTER = " ]" "}"; -k8s_deployment_handler::k8s_deployment_handler(k8s_state_t& state, - ptr_t dependency_handler, - collector_ptr_t collector, - std::string url, - const std::string& http_version, - ssl_ptr_t ssl, - bt_ptr_t bt, - bool connect): - k8s_handler("k8s_deployment_handler", true, url, - "/apis/extensions/v1beta1/deployments", - STATE_FILTER, EVENT_FILTER, dependency_handler, collector, - http_version, 1000L, ssl, bt, &state, true, connect) +k8s_deployment_handler::k8s_deployment_handler(k8s_state_t& state +#ifdef HAS_CAPTURE + ,ptr_t dependency_handler + ,collector_ptr_t collector + ,std::string url + ,const std::string& http_version + ,ssl_ptr_t ssl + ,bt_ptr_t bt + ,bool connect + ,bool blocking_socket +#endif // HAS_CAPTURE + ): + k8s_handler("k8s_deployment_handler", true, +#ifdef HAS_CAPTURE + url, "/apis/extensions/v1beta1/deployments", + STATE_FILTER, EVENT_FILTER, collector, + http_version, 1000L, ssl, bt, true, + connect, dependency_handler, blocking_socket, +#endif // HAS_CAPTURE + 100, // max msgs + &state) { } diff --git a/userspace/libsinsp/k8s_deployment_handler.h b/userspace/libsinsp/k8s_deployment_handler.h index f6bd625d4f..e627cb12ee 100644 --- a/userspace/libsinsp/k8s_deployment_handler.h +++ b/userspace/libsinsp/k8s_deployment_handler.h @@ -14,14 +14,18 @@ class sinsp; class k8s_deployment_handler : public k8s_handler { public: - k8s_deployment_handler(k8s_state_t& state, - ptr_t dependency_handler, - collector_ptr_t collector = nullptr, - std::string url = "", - const std::string& http_version = "1.0", - ssl_ptr_t ssl = 0, - bt_ptr_t bt = 0, - bool connect = true); + k8s_deployment_handler(k8s_state_t& state +#ifdef HAS_CAPTURE + ,ptr_t dependency_handler + ,collector_ptr_t collector = nullptr + ,std::string url = "" + ,const std::string& http_version = "1.1" + ,ssl_ptr_t ssl = 0 + ,bt_ptr_t bt = 0 + ,bool connect = true + ,bool blocking_socket = false +#endif // HAS_CAPTURE + ); ~k8s_deployment_handler(); diff --git a/userspace/libsinsp/k8s_event_handler.cpp b/userspace/libsinsp/k8s_event_handler.cpp index 866c335bcd..c1f1529b45 100644 --- a/userspace/libsinsp/k8s_event_handler.cpp +++ b/userspace/libsinsp/k8s_event_handler.cpp @@ -51,19 +51,26 @@ std::string k8s_event_handler::STATE_FILTER = " ]" "}"; -k8s_event_handler::k8s_event_handler(k8s_state_t& state, - ptr_t dependency_handler, - collector_ptr_t collector, - std::string url, - const std::string& http_version, - ssl_ptr_t ssl, - bt_ptr_t bt, - bool connect, - filter_ptr_t event_filter): +k8s_event_handler::k8s_event_handler(k8s_state_t& state +#ifdef HAS_CAPTURE + ,ptr_t dependency_handler + ,collector_ptr_t collector + ,std::string url + ,const std::string& http_version + ,ssl_ptr_t ssl + ,bt_ptr_t bt + ,bool connect + ,bool blocking_socket +#endif // HAS_CAPTURE + ,filter_ptr_t event_filter): k8s_handler("k8s_event_handler", true, +#ifdef HAS_CAPTURE url, "/api/v1/events", - STATE_FILTER, EVENT_FILTER, dependency_handler, collector, - http_version, 1000L, ssl, bt, &state, true, connect), + STATE_FILTER, EVENT_FILTER, collector, + http_version, 1000L, ssl, bt, true, + connect, dependency_handler, blocking_socket, +#endif // HAS_CAPTURE + ~0, &state), m_event_filter(event_filter) { } diff --git a/userspace/libsinsp/k8s_event_handler.h b/userspace/libsinsp/k8s_event_handler.h index 09d08bc84c..39c770f43e 100644 --- a/userspace/libsinsp/k8s_event_handler.h +++ b/userspace/libsinsp/k8s_event_handler.h @@ -16,15 +16,18 @@ class k8s_event_handler : public k8s_handler public: typedef user_event_filter_t::ptr_t filter_ptr_t; - k8s_event_handler(k8s_state_t& state, - ptr_t dependency_handler, - collector_ptr_t collector = nullptr, - std::string url = "", - const std::string& http_version = "1.0", - ssl_ptr_t ssl = 0, - bt_ptr_t bt = 0, - bool connect = true, - filter_ptr_t event_filter = 0); + k8s_event_handler(k8s_state_t& state +#ifdef HAS_CAPTURE + ,ptr_t dependency_handler + ,collector_ptr_t collector = nullptr + ,std::string url = "" + ,const std::string& http_version = "1.1" + ,ssl_ptr_t ssl = 0 + ,bt_ptr_t bt = 0 + ,bool connect = true + ,bool blocking_socket = false +#endif // HAS_CAPTURE + ,filter_ptr_t event_filter = 0); ~k8s_event_handler(); diff --git a/userspace/libsinsp/k8s_handler.cpp b/userspace/libsinsp/k8s_handler.cpp index b67ba4add9..1f88ad2fb4 100644 --- a/userspace/libsinsp/k8s_handler.cpp +++ b/userspace/libsinsp/k8s_handler.cpp @@ -9,7 +9,8 @@ // to match regular K8s API message format, // error is wrapped into a single-entry array std::string k8s_handler::ERROR_FILTER = - "{ type: \"ERROR\"," + "{" + " type: \"ERROR\"," " apiVersion: .apiVersion," " kind: .kind," " items:" @@ -28,25 +29,30 @@ std::string k8s_handler::ERROR_FILTER = k8s_handler::k8s_handler(const std::string& id, bool is_captured, +#ifdef HAS_CAPTURE std::string url, const std::string& path, const std::string& state_filter, const std::string& event_filter, - ptr_t dependency_handler, collector_ptr_t collector, const std::string& http_version, int timeout_ms, ssl_ptr_t ssl, bt_ptr_t bt, - k8s_state_t* state, bool watch, - bool connect): m_state(state), - m_collector(collector), + bool connect, + ptr_t dependency_handler, + bool blocking_socket, +#endif // HAS_CAPTURE + unsigned max_messages, + k8s_state_t* state): m_state(state), m_id(id + "_state"), +#ifdef HAS_CAPTURE + m_collector(collector), m_path(path), m_state_filter(state_filter), m_event_filter(event_filter), - m_filter(m_state_filter), + m_filter(&m_state_filter), m_timeout_ms(timeout_ms), m_url(url), m_http_version(http_version), @@ -54,13 +60,30 @@ k8s_handler::k8s_handler(const std::string& id, m_bt(bt), m_watch(watch), m_connect(connect), - m_is_captured(is_captured), - m_dependency_handler(dependency_handler) + m_dependency_handler(dependency_handler), + m_blocking_socket(blocking_socket), +#endif // HAS_CAPTURE + m_max_messages(max_messages), + m_is_captured(is_captured) { +#ifdef HAS_CAPTURE g_logger.log("Creating K8s " + name() + " (" + m_id + ") " "handler object for [" + uri(m_url).to_string(false) + m_path + ']', sinsp_logger::SEV_DEBUG); - make_http(); + if(m_connect) + { + g_logger.log(std::string("K8s (" + m_id + ") creating handler for " + + uri(m_url).to_string(false) + m_path), sinsp_logger::SEV_DEBUG); + m_handler = std::make_shared(*this, m_id, m_url, m_path, m_http_version, + m_timeout_ms, m_ssl, m_bt, !m_blocking_socket, m_blocking_socket); + m_handler->set_json_callback(&k8s_handler::set_event_json); + m_handler->add_json_filter(*m_filter); + m_handler->add_json_filter(ERROR_FILTER); + m_handler->close_on_chunked_end(false); + m_handler->set_check_chunked(false); + this->connect(); + } +#endif // HAS_CAPTURE } k8s_handler::~k8s_handler() @@ -69,59 +92,83 @@ k8s_handler::~k8s_handler() void k8s_handler::make_http() { - if(m_connect) +#ifdef HAS_CAPTURE + if(m_connect && m_collector) { - g_logger.log(std::string("K8s (" + m_id + ") creating handler for " + + if(!m_handler) + { + g_logger.log(std::string("K8s (" + m_id + ") creating handler for " + uri(m_url).to_string(false) + m_path), sinsp_logger::SEV_INFO); - bool keep_alive = (m_id.find("_state") == std::string::npos); - m_http = std::make_shared(*this, m_id, - m_url, m_path, m_http_version, - m_timeout_ms, m_ssl, m_bt, keep_alive); - m_http->set_json_callback(&k8s_handler::set_event_json); - m_http->set_json_end("}\n"); - m_http->add_json_filter(m_filter); - m_http->add_json_filter(ERROR_FILTER); + m_handler = std::make_shared(*this, m_id, m_url, m_path, m_http_version, + m_timeout_ms, m_ssl, m_bt, true, m_blocking_socket); + m_handler->set_json_callback(&k8s_handler::set_event_json); + } + else if(m_collector->has(m_handler)) + { + m_collector->remove(m_handler); + } + m_handler->remove_json_filter(m_state_filter); + m_filter = &m_event_filter; + if(!m_handler->has_json_filter(ERROR_FILTER)) + { + m_handler->add_json_filter(ERROR_FILTER); + } + // good event filter must always be before error event filter + m_handler->add_json_filter(*m_filter, ERROR_FILTER); + m_handler->set_path(m_path); + m_handler->set_id(m_id); + m_collector->set_steady_state(true); + m_watching = true; + m_blocking_socket = false; + m_handler->close_on_chunked_end(false); + m_handler->set_check_chunked(true); + m_req_sent = false; m_resp_recvd = false; connect(); + m_handler->set_socket_option(SOCK_NONBLOCK); } +#endif // HAS_CAPTURE } void k8s_handler::check_enabled() { - if(!m_http->is_enabled()) +#ifdef HAS_CAPTURE + if(!m_handler->is_enabled()) { g_logger.log("k8s_handler (" + m_id + - ") connect() enabling socket in collector", sinsp_logger::SEV_TRACE); - m_http->enable(); + ") check_enabled() enabling socket in collector", sinsp_logger::SEV_TRACE); + m_handler->enable(); } else { g_logger.log("k8s_handler (" + m_id + - ") connect() socket in collector is enabled, " + ") check_enabled() socket in collector is enabled, " "checking collector status.", sinsp_logger::SEV_TRACE); check_collector_status(); } +#endif // HAS_CAPTURE } bool k8s_handler::connect() { - if(m_collector && m_http) +#ifdef HAS_CAPTURE + if(m_collector && m_handler) { - if(!m_collector->has(m_http)) + if(!m_collector->has(m_handler)) { g_logger.log(std::string("k8s_handler (" + m_id + - ") k8s_handler::connect() adding socket to collector"), sinsp_logger::SEV_TRACE); - m_collector->add(m_http); + ") k8s_handler::connect() adding handler to collector"), sinsp_logger::SEV_TRACE); + m_collector->add(m_handler); return false; } - if(m_http->is_connecting()) + if(m_handler->is_connecting()) { g_logger.log(std::string("k8s_handler (" + m_id + - "), k8s_handler::connect() connecting to " + m_http->get_url().to_string(false)), sinsp_logger::SEV_TRACE); + "), k8s_handler::connect() connecting to " + m_handler->get_url().to_string(false)), sinsp_logger::SEV_TRACE); return false; } - if(m_http->is_connected()) + if(m_handler->is_connected()) { g_logger.log("k8s_handler (" + m_id + ") k8s_handler::connect() socket is connected.", sinsp_logger::SEV_TRACE); @@ -136,51 +183,96 @@ bool k8s_handler::connect() sinsp_logger::SEV_WARNING); make_http(); } +#endif // HAS_CAPTURE return false; } void k8s_handler::send_data_request() { - if(m_http && !m_req_sent) +#ifdef HAS_CAPTURE + if(m_handler) { - if(m_http->is_connected()) + if(!m_req_sent) { - g_logger.log("k8s_handler (" + m_id + ") sending request to " + - m_http->get_url().to_string(false) + m_path, - sinsp_logger::SEV_DEBUG); - m_http->send_request(); - m_req_sent = true; + if(m_handler->is_connected()) + { + g_logger.log("k8s_handler (" + m_id + ") sending request to " + + m_handler->get_url().to_string(false) + m_path, + sinsp_logger::SEV_DEBUG); + m_handler->send_request(); + m_req_sent = true; + } + else if(m_handler->is_connecting()) + { + g_logger.log("k8s_handler (" + m_id + ") is connecting to " + + m_handler->get_url().to_string(false), + sinsp_logger::SEV_DEBUG); + } } - else if(m_http->is_connecting()) + } + else + { + throw sinsp_exception("k8s_handler (" + m_id + ") HTTP client (" + uri(m_url).to_string(false) + ") is null."); + } +#endif // HAS_CAPTURE +} + +void k8s_handler::receive_response() +{ +#ifdef HAS_CAPTURE + if(m_handler) + { + if(m_req_sent) { - g_logger.log("k8s_handler (" + m_id + ") is connecting to " + - m_http->get_url().to_string(false), - sinsp_logger::SEV_DEBUG); + if(!m_watching) + { + if(m_handler->get_all_data()) + { + m_data_received = true; + } + else + { + throw sinsp_exception("K8s k8s_handler::receive_response(): no data received."); + } + } + else + { + throw sinsp_exception("K8s k8s_handler::receive_response(): invalid call (in watch mode)."); + } + } + else + { + throw sinsp_exception("K8s k8s_handler::receive_response(): invalid call (request not sent)."); } } else { - throw sinsp_exception("k8s_handler (" + m_id + ") HTTP client (" + uri(m_url).to_string(false) + ") is null."); + throw sinsp_exception("K8s k8s_handler::receive_response(): handler is null."); } +#endif // HAS_CAPTURE } bool k8s_handler::is_alive() const { - if(m_http && !m_http->is_connecting() && !m_http->is_connected()) +#ifdef HAS_CAPTURE + if(m_handler && !m_handler->is_connecting() && !m_handler->is_connected()) { - g_logger.log("k8s_handler (" + m_id + ") connection (" + m_http->get_url().to_string(false) + ") loss.", + g_logger.log("k8s_handler (" + m_id + ") connection (" + m_handler->get_url().to_string(false) + ") loss.", sinsp_logger::SEV_WARNING); return false; } +#endif // HAS_CAPTURE return true; } void k8s_handler::check_collector_status() { +#ifdef HAS_CAPTURE if(m_collector) { - if(!m_collector->has(m_http)) + if(!m_collector->has(m_handler)) { + m_handler.reset(); make_http(); } } @@ -188,85 +280,105 @@ void k8s_handler::check_collector_status() { throw sinsp_exception("k8s_handler (" + m_id + ") collector is null."); } +#endif // HAS_CAPTURE } void k8s_handler::check_state() { - if(m_collector && m_resp_recvd && m_watch && !m_watching) - { - g_logger.log("k8s_handler (" + m_id + ") switching to watch connection for " + - uri(m_url).to_string(false) + m_path, - sinsp_logger::SEV_DEBUG); - // done with initial state handling, switch to events - m_collector->remove(m_http); - m_http.reset(); - std::string::size_type pos = m_id.find("_state"); - if(pos != std::string::npos) - { - m_id = m_id.substr(0, pos).append("_event"); - } - pos = m_path.find("/watch"); - if(pos == std::string::npos) +#ifdef HAS_CAPTURE + if(m_collector && m_handler) + { + if(m_resp_recvd && m_watch && !m_watching) { - pos = m_path.rfind('/'); + g_logger.log("k8s_handler (" + m_id + ") switching to watch connection for " + + uri(m_url).to_string(false) + m_path, + sinsp_logger::SEV_DEBUG); + std::string::size_type pos = m_id.find("_state"); if(pos != std::string::npos) { - m_path.insert(pos, "/watch"); + m_id = m_id.substr(0, pos).append("_event"); } - else + pos = m_path.find("/watch"); + if(pos == std::string::npos) { - throw sinsp_exception("k8s_handler (" + m_id + "), invalid URL path: " + m_path); + pos = m_path.rfind('/'); + if(pos != std::string::npos) + { + m_path.insert(pos, "/watch"); + } + else + { + throw sinsp_exception("k8s_handler (" + m_id + "), invalid URL path: " + m_path); + } } + m_handler->set_socket_option(SOCK_NONBLOCK); + make_http(); + } + if(m_watching && m_id.find("_state") == std::string::npos && m_handler->wants_send()) + { + m_req_sent = false; + m_resp_recvd = false; } - m_filter = m_event_filter; - make_http(); - m_collector->set_steady_state(true); - m_watching = true; } +#endif // HAS_CAPTURE } bool k8s_handler::connection_error() const { - if(m_http) +#ifdef HAS_CAPTURE + if(m_handler) { - return m_http->connection_error(); + return m_handler->connection_error(); } +#endif // HAS_CAPTURE return false; } void k8s_handler::collect_data() { - if(m_collector && m_http) +#ifdef HAS_CAPTURE + if(m_collector && m_handler) { process_events(); // there may be leftovers from state connection closed by collector check_state(); // switch to events, if needed - g_logger.log("k8s_handler (" + m_id + ") collect_data(), checking connection to " + uri(m_url).to_string(false), sinsp_logger::SEV_DEBUG); - if(m_http->is_connecting()) + g_logger.log("k8s_handler (" + m_id + ")::collect_data(), checking connection to " + uri(m_url).to_string(false), sinsp_logger::SEV_DEBUG); + if(m_handler->is_connecting()) { - g_logger.log("k8s_handler (" + m_id + ") collect_data(), connecting to " + uri(m_url).to_string(false), sinsp_logger::SEV_DEBUG); + g_logger.log("k8s_handler (" + m_id + ")::collect_data(), connecting to " + uri(m_url).to_string(false), sinsp_logger::SEV_DEBUG); return; } - else if(m_http->is_connected()) + else if(m_handler->is_connected()) { + if(!m_connect_logged) + { + g_logger.log("k8s_handler (" + m_id + ")::collect_data(), connected to " + uri(m_url).to_string(false) + m_path, sinsp_logger::SEV_DEBUG); + m_connect_logged = true; + } check_enabled(); if(!m_req_sent) { - g_logger.log("k8s_handler (" + m_id + ") collect_data(), connected to " + uri(m_url).to_string(false) + ", requesting data " - "from " + m_path + "...", sinsp_logger::SEV_DEBUG); + g_logger.log("k8s_handler (" + m_id + ")::collect_data() [" + uri(m_url).to_string(false) + "], requesting data " + "from " + m_path + "... m_blocking_socket=" + std::to_string(m_blocking_socket) + ", m_watching=" + std::to_string(m_watching), + sinsp_logger::SEV_DEBUG); send_data_request(); + if(m_blocking_socket && !m_watching) + { + receive_response(); + process_events(); + return; + } } if(m_collector->subscription_count()) { - g_logger.log("k8s_handler (" + m_id + ") collect_data(), connected to " + uri(m_url).to_string(false) + ", getting data " + g_logger.log("k8s_handler (" + m_id + ")::collect_data() [" + uri(m_url).to_string(false) + "], getting data " "from " + m_path + "...", sinsp_logger::SEV_DEBUG); m_collector->get_data(); - g_logger.log("k8s_handler (" + m_id + ") collect_data(), " + std::to_string(m_events.size()) + + g_logger.log("k8s_handler (" + m_id + ")::collect_data(), " + std::to_string(m_events.size()) + " events from " + uri(m_url).to_string(false) + m_path, sinsp_logger::SEV_DEBUG); if(m_events.size()) { - g_logger.log("k8s_handler (" + m_id + ") collect_data(), data from " + uri(m_url).to_string(false) + m_path + - ", event count=" + std::to_string(m_events.size()), - sinsp_logger::SEV_DEBUG); + g_logger.log("k8s_handler (" + m_id + ")::collect_data(), data from " + uri(m_url).to_string(false) + m_path + + ", event count=" + std::to_string(m_events.size()), sinsp_logger::SEV_DEBUG); process_events(); check_state(); } @@ -294,6 +406,7 @@ void k8s_handler::collect_data() g_logger.log("k8s_handler (" + m_id + "), http interface not (yet?) created for " + uri(m_url).to_string(false) + ").", sinsp_logger::SEV_TRACE); } +#endif // HAS_CAPTURE } k8s_handler::msg_data k8s_handler::get_msg_data(const std::string& type, const std::string& kind, const Json::Value& json) @@ -341,7 +454,11 @@ void k8s_handler::handle_json(Json::Value&& root) if(!m_state) { +#ifdef HAS_CAPTURE throw sinsp_exception("k8s_handler (" + m_id + "), state is null for " + uri(m_url).to_string(false) + ")."); +#else + throw sinsp_exception("k8s_handler (" + m_id + "), state is null."); +#endif // HAS_CAPTURE } const Json::Value& type = root["type"]; if(!type.isNull()) @@ -385,9 +502,11 @@ void k8s_handler::handle_json(Json::Value&& root) if(m_state->has(data.m_uid)) { std::ostringstream os; - os << "K8s " + reason_type << " message received by " << m_id + " " - "[" << uri(m_url).to_string(false) << "] for existing " << data.m_kind << - " [" << data.m_uid << "], updating only."; + os << "K8s " + reason_type << " message received by " << m_id << +#ifdef HAS_CAPTURE + " [" << uri(m_url).to_string(false) << "]" +#endif // HAS_CAPTURE + "for existing " << data.m_kind << " [" << data.m_uid << "], updating only."; g_logger.log(os.str(), sinsp_logger::SEV_DEBUG); } } @@ -396,9 +515,11 @@ void k8s_handler::handle_json(Json::Value&& root) if(!m_state->has(data.m_uid)) { std::ostringstream os; - os << "K8s " << reason_type << " message received by " << m_id + " " - "[" << uri(m_url).to_string(false) << "] for non-existing " << data.m_kind << - " [" << data.m_uid << "], giving up."; + os << "K8s " << reason_type << " message received by " << m_id << +#ifdef HAS_CAPTURE + " [" << uri(m_url).to_string(false) << "]" +#endif // HAS_CAPTURE + " for non-existing " << data.m_kind << " [" << data.m_uid << "], giving up."; g_logger.log(os.str(), sinsp_logger::SEV_WARNING); continue; } @@ -408,9 +529,11 @@ void k8s_handler::handle_json(Json::Value&& root) if(!m_state->has(data.m_uid)) { std::ostringstream os; - os << "K8s " + reason_type + " message received by " << m_id + " " - "[" << uri(m_url).to_string(false) << "] for non-existing " << data.m_kind << - " [" << data.m_uid << "], giving up."; + os << "K8s " + reason_type + " message received by " << m_id << +#ifdef HAS_CAPTURE + " [" << uri(m_url).to_string(false) << "]" +#endif // HAS_CAPTURE + " for non-existing " << data.m_kind << " [" << data.m_uid << "], giving up."; g_logger.log(os.str(), sinsp_logger::SEV_WARNING); continue; } @@ -459,6 +582,8 @@ void k8s_handler::handle_json(Json::Value&& root) } } +#ifdef HAS_CAPTURE + bool k8s_handler::is_ip_address(const std::string& addr) { struct sockaddr_in serv_addr = {0}; @@ -490,44 +615,78 @@ k8s_handler::ip_addr_list_t k8s_handler::hostname_to_ip(const std::string& hostn return ip_addrs; } +#endif // HAS_CAPTURE + +bool k8s_handler::dependency_ready() const +{ +#ifdef HAS_CAPTURE + g_logger.log("k8s_handler (" + m_id + ") dependency " + "(" + m_dependency_handler->get_id() + ") ready: " + + std::to_string(m_dependency_handler->is_state_built()), + sinsp_logger::SEV_TRACE); + return m_dependency_handler->is_state_built(); +#else + return true; +#endif // HAS_CAPTURE +} + void k8s_handler::process_events() { - if(m_dependency_handler->is_state_built()) + if(dependency_ready()) { - for(auto evt : m_events) + unsigned counter = 0; + for(auto evt = m_events.begin(); evt != m_events.end();) { - if(evt && !evt->isNull()) + m_state_processing_started = true; + if(++counter >= get_max_messages()) { break; } + if(*evt && !(*evt)->isNull()) { - g_logger.log("k8s_handler (" + m_id + ") data:\n" + json_as_string(*evt), - sinsp_logger::SEV_TRACE); + if(g_logger.get_severity() >= sinsp_logger::SEV_TRACE) + { + g_logger.log("k8s_handler (" + m_id + ") processing event data:\n" + json_as_string(*(*evt)), + sinsp_logger::SEV_TRACE); + } +#ifdef HAS_CAPTURE if(m_is_captured) { - m_state->enqueue_capture_event(*evt); + m_state->enqueue_capture_event(**evt); } - handle_json(std::move(*evt)); +#endif // HAS_CAPTURE + handle_json(std::move(**evt)); } else { - g_logger.log("k8s_handler (" + m_id + ") error (" + uri(m_url).to_string(false) + ") " + - (!evt ? "data is null." : (evt->isNull() ? "JSON is null." : "Unknown")), + g_logger.log("k8s_handler (" + m_id + ") error " + +#ifdef HAS_CAPTURE + "(" + uri(m_url).to_string(false) + ") " + +#endif // HAS_CAPTURE + (!(*evt) ? "data is null." : ((*evt)->isNull() ? "JSON is null." : "Unknown")), sinsp_logger::SEV_ERROR); } + evt = m_events.erase(evt); } - m_events.clear(); - if(!m_state_built) { m_state_built = true; } + if(!m_state_built && m_state_processing_started && !m_events.size()) { m_state_built = true; } } } void k8s_handler::set_event_json(json_ptr_t json, const std::string&) { - g_logger.log("k8s_handler adding event, (" + m_id + ") has " + std::to_string(m_events.size()) + - " events from " + uri(m_url).to_string(false), sinsp_logger::SEV_TRACE); + g_logger.log("k8s_handler adding event, (" + m_id + ") has " + std::to_string(m_events.size()) +#ifdef HAS_CAPTURE + + " events from " + uri(m_url).to_string(false) +#endif // HAS_CAPTURE + , sinsp_logger::SEV_TRACE); // empty JSON is fine here; if there are no entities, state and first watch will pass nothing in here // null is checked when processing m_events.emplace_back(json); - g_logger.log("k8s_handler added event, (" + m_id + ") has " + std::to_string(m_events.size()) + - " events from " + uri(m_url).to_string(false), sinsp_logger::SEV_TRACE); + g_logger.log("k8s_handler added event, (" + m_id + ") has " + std::to_string(m_events.size()) +#ifdef HAS_CAPTURE + + " events from " + uri(m_url).to_string(false) +#endif // HAS_CAPTURE + , sinsp_logger::SEV_TRACE); +#ifdef HAS_CAPTURE if(!m_resp_recvd) { m_resp_recvd = true; } +#endif // HAS_CAPTURE } k8s_pair_list k8s_handler::extract_object(const Json::Value& object) @@ -551,6 +710,7 @@ k8s_pair_list k8s_handler::extract_object(const Json::Value& object) std::string k8s_handler::name() const { std::string n; +#ifdef HAS_CAPTURE std::string::size_type slash_pos = m_path.rfind('/'); std::string::size_type qm_pos = m_path.rfind('?'); std::string::size_type length = @@ -561,7 +721,7 @@ std::string k8s_handler::name() const { n = m_path.substr(slash_pos, length); } - +#endif // HAS_CAPTURE return n; } @@ -575,6 +735,7 @@ void k8s_handler::handle_error(const msg_data& data, const Json::Value& root, bo void k8s_handler::log_error(const msg_data& data, const Json::Value& json) { +#ifdef HAS_CAPTURE std::string unk_err = "Unknown."; std::ostringstream os;; os << "K8S server reported " << name() << " error for [" + uri(m_url).to_string(false) + m_path + "]: "; @@ -586,4 +747,5 @@ void k8s_handler::log_error(const msg_data& data, const Json::Value& json) } os << unk_err; g_logger.log(os.str(), sinsp_logger::SEV_ERROR); +#endif // HAS_CAPTURE } diff --git a/userspace/libsinsp/k8s_handler.h b/userspace/libsinsp/k8s_handler.h index 0662bf1e9c..3538ea07a5 100644 --- a/userspace/libsinsp/k8s_handler.h +++ b/userspace/libsinsp/k8s_handler.h @@ -5,6 +5,7 @@ #pragma once #include "json/json.h" +#include "sinsp_auth.h" #include "socket_collector.h" #include "k8s_state.h" #include "k8s_api_error.h" @@ -18,63 +19,73 @@ class k8s_handler typedef k8s_component::msg_reason msg_reason; typedef k8s_component::msg_data msg_data; - typedef std::shared_ptr ptr_t; - typedef std::vector uri_list_t; - typedef std::shared_ptr json_ptr_t; - typedef sinsp_ssl::ptr_t ssl_ptr_t; - typedef sinsp_bearer_token::ptr_t bt_ptr_t; - typedef socket_data_handler handler_t; - typedef handler_t::ptr_t handler_ptr_t; - typedef socket_collector collector_t; - typedef std::shared_ptr> collector_ptr_t; - typedef std::shared_ptr api_error_ptr; + typedef std::shared_ptr ptr_t; + typedef std::vector uri_list_t; + typedef std::shared_ptr json_ptr_t; + typedef std::shared_ptr api_error_ptr; +#ifdef HAS_CAPTURE + typedef sinsp_ssl::ptr_t ssl_ptr_t; + typedef sinsp_bearer_token::ptr_t bt_ptr_t; + typedef socket_data_handler handler_t; + typedef handler_t::ptr_t handler_ptr_t; + typedef socket_collector collector_t; + typedef std::shared_ptr> collector_ptr_t; +#endif // HAS_CAPTURE static const int default_timeout_ms = 1000L; k8s_handler(const std::string& id, bool is_captured, +#ifdef HAS_CAPTURE std::string url, const std::string& path, const std::string& state_filter, const std::string& event_filter, - ptr_t dependency_handler, collector_ptr_t collector = nullptr, - const std::string& http_version = "1.0", + const std::string& http_version = "1.1", int timeout_ms = default_timeout_ms, ssl_ptr_t ssl = nullptr, bt_ptr_t bt = nullptr, - k8s_state_t* state = nullptr, bool watch = true, - bool connect = true); + bool connect = true, + ptr_t dependency_handler = nullptr, + bool blocking_socket = false, +#endif // HAS_CAPTURE + unsigned max_messages = ~0, + k8s_state_t* state = nullptr); virtual ~k8s_handler(); bool connection_error() const; bool is_alive() const; + bool ready() const; void set_event_json(json_ptr_t json, const std::string&); const std::string& get_id() const; - const std::string& get_url() const; +#ifdef HAS_CAPTURE + handler_ptr_t handler(); +#endif // HAS_CAPTURE + std::string get_url() const; void collect_data(); void set_machine_id(const std::string& machine_id); const std::string& get_machine_id() const; bool is_state_built() const; - - handler_ptr_t handler(); - std::string name() const; - api_error_ptr error() const; - virtual void handle_json(Json::Value&& root); + unsigned get_max_messages() const; + void set_max_messages(unsigned max_msgs); + protected: typedef std::unordered_set ip_addr_list_t; virtual bool handle_component(const Json::Value& json, const msg_data* data = 0) = 0; msg_data get_msg_data(const std::string& evt, const std::string& type, const Json::Value& root); +#ifdef HAS_CAPTURE static bool is_ip_address(const std::string& addr); +#endif // HAS_CAPTURE k8s_pair_list extract_object(const Json::Value& object); @@ -98,7 +109,7 @@ class k8s_handler k8s_state_t* m_state = nullptr; bool m_state_built = false; - + bool m_data_received = false; static std::string ERROR_FILTER; private: @@ -106,36 +117,40 @@ class k8s_handler typedef std::vector event_list_t; +#ifdef HAS_CAPTURE static ip_addr_list_t hostname_to_ip(const std::string& hostname); +#endif // HAS_CAPTURE bool connect(); void make_http(); void send_data_request(); + void receive_response(); void check_enabled(); void check_state(); void check_collector_status(); void process_events(); const std::string& translate_name(const std::string& event_name); + bool dependency_ready() const; - collector_ptr_t m_collector; - handler_ptr_t m_http; std::string m_id; + std::string m_machine_id; +#ifdef HAS_CAPTURE + collector_ptr_t m_collector; + handler_ptr_t m_handler; std::string m_path; std::string m_state_filter; std::string m_event_filter; - std::string& m_filter; + std::string* m_filter; std::string m_event_uri; - event_list_t m_events; long m_timeout_ms; - std::string m_machine_id; - json_query m_jq; std::string m_url; + bool m_req_sent = false; + bool m_resp_recvd = false; + json_query m_jq; std::string m_http_version; ssl_ptr_t m_ssl; bt_ptr_t m_bt; - bool m_req_sent = false; - bool m_resp_recvd = false; // some handlers only fetch state and die by design (eg. api or extensions handlers // have no need to continuously watch for updates) @@ -148,6 +163,20 @@ class k8s_handler // replaying capture) bool m_connect; + // k8s_handler on which this handler depends; the dependency handler must not be null and + // it must have its state fully built before this handler can begin building its own state + ptr_t m_dependency_handler; + + bool m_blocking_socket = false; + +#endif // HAS_CAPTURE + + // limits the number of messages handled in single cycle + unsigned m_max_messages = ~0; + bool m_state_processing_started = false; + + event_list_t m_events; + // error indicating something went wrong with the K8s component handled by this handler // this error is later examined by k8s::check_components() and if it is // HTTP status > 400, one of the following actions is taken: @@ -165,24 +194,38 @@ class k8s_handler // global capture flag is checked in the k8s state call bool m_is_captured = false; - // k8s_handler on which this handler depends; the dependency handler must not be null and - // it must have its state fully built before this handler can begin building its own state - ptr_t m_dependency_handler; + bool m_connect_logged = false; }; -inline k8s_handler::handler_ptr_t k8s_handler::handler() +inline unsigned k8s_handler::get_max_messages() const { - return m_http; + return m_max_messages; } -inline const std::string& k8s_handler::get_id() const +inline void k8s_handler::set_max_messages(unsigned max_msgs) { - return m_id; + m_max_messages = max_msgs; +} + +#ifdef HAS_CAPTURE +inline k8s_handler::handler_ptr_t k8s_handler::handler() +{ + return m_handler; } +#endif // HAS_CAPTURE -inline const std::string& k8s_handler::get_url() const +inline std::string k8s_handler::get_url() const { +#ifdef HAS_CAPTURE return m_url; +#else + return ""; +#endif +} + +inline const std::string& k8s_handler::get_id() const +{ + return m_id; } inline void k8s_handler::set_machine_id(const std::string& machine_id) @@ -195,6 +238,11 @@ inline const std::string& k8s_handler::get_machine_id() const return m_machine_id; } +inline bool k8s_handler::ready() const +{ + return m_data_received; +} + inline bool k8s_handler::is_state_built() const { return m_state_built; @@ -219,17 +267,23 @@ inline k8s_handler::api_error_ptr k8s_handler::error() const return m_error; } -// This dummy class serves only as a dependency stand-in for nodes, -// which have no dependencies but the logic requires a pointer to -// handler to determine whether dependency is ready; to avoid -// special casing nodes handler all over the place, we have this dummy -// liar which is always returning true for its state being built +// This dummy class serves only as a dependency stand-in for handlers +// which have no dependencies (eg. nodes handler, which is first populated +// into the state and has no dependency; or special-purpose handlers, +// such as delegator, api handler etc), but the logic requires a non-null +// pointer to handler to determine whether dependency is ready; to avoid +// special-casing eg. nodes handler all over the place, we use this dummy +// liar, always returning true for its state being built, as the dependency class k8s_dummy_handler : public k8s_handler { public: - k8s_dummy_handler(): k8s_handler("k8s_dummy_handler", false, "", "", - "", "", nullptr, nullptr, "", 0, - nullptr, nullptr, nullptr, false, false) + k8s_dummy_handler(): k8s_handler("k8s_dummy_handler", false, +#ifdef HAS_CAPTURE + "", "", "", "", nullptr, + "", 0, nullptr, nullptr, + false, false, nullptr, false, +#endif // HAS_CAPTURE + ~0, nullptr) { m_state_built = true; } diff --git a/userspace/libsinsp/k8s_namespace_handler.cpp b/userspace/libsinsp/k8s_namespace_handler.cpp index b31868f85c..498538dbd4 100644 --- a/userspace/libsinsp/k8s_namespace_handler.cpp +++ b/userspace/libsinsp/k8s_namespace_handler.cpp @@ -42,18 +42,26 @@ std::string k8s_namespace_handler::STATE_FILTER = " ]" "}"; -k8s_namespace_handler::k8s_namespace_handler(k8s_state_t& state, - ptr_t dependency_handler, - collector_ptr_t collector, - std::string url, - const std::string& http_version, - ssl_ptr_t ssl, - bt_ptr_t bt, - bool connect): +k8s_namespace_handler::k8s_namespace_handler(k8s_state_t& state +#ifdef HAS_CAPTURE + ,ptr_t dependency_handler + ,collector_ptr_t collector + ,std::string url + ,const std::string& http_version + ,ssl_ptr_t ssl + ,bt_ptr_t bt + ,bool connect + ,bool blocking_socket +#endif // HAS_CAPTURE + ): k8s_handler("k8s_namespace_handler", true, +#ifdef HAS_CAPTURE url, "/api/v1/namespaces", - STATE_FILTER, EVENT_FILTER, dependency_handler, collector, - http_version, 1000L, ssl, bt, &state, true, connect) + STATE_FILTER, EVENT_FILTER, collector, + http_version, 1000L, ssl, bt, true, + connect, dependency_handler, blocking_socket, +#endif // HAS_CAPTURE + ~0, &state) { } diff --git a/userspace/libsinsp/k8s_namespace_handler.h b/userspace/libsinsp/k8s_namespace_handler.h index ace19559e1..df9870379e 100644 --- a/userspace/libsinsp/k8s_namespace_handler.h +++ b/userspace/libsinsp/k8s_namespace_handler.h @@ -13,14 +13,18 @@ class sinsp; class k8s_namespace_handler : public k8s_handler { public: - k8s_namespace_handler(k8s_state_t& state, - ptr_t dependency_handler, - collector_ptr_t collector = nullptr, - std::string url = "", - const std::string& http_version = "1.0", - ssl_ptr_t ssl = 0, - bt_ptr_t bt = 0, - bool connect = true); + k8s_namespace_handler(k8s_state_t& state +#ifdef HAS_CAPTURE + ,ptr_t dependency_handler + ,collector_ptr_t collector = nullptr + ,std::string url = "" + ,const std::string& http_version = "1.1" + ,ssl_ptr_t ssl = 0 + ,bt_ptr_t bt = 0 + ,bool connect = true + ,bool blocking_socket = false +#endif // HAS_CAPTURE + ); ~k8s_namespace_handler(); diff --git a/userspace/libsinsp/k8s_net.cpp b/userspace/libsinsp/k8s_net.cpp index c3795ed47c..f078862dfc 100644 --- a/userspace/libsinsp/k8s_net.cpp +++ b/userspace/libsinsp/k8s_net.cpp @@ -27,12 +27,14 @@ k8s_net::k8s_net(k8s& kube, k8s_state_t& state, const std::string& uri, ssl_ptr_t ssl, bt_ptr_t bt, ext_list_ptr_t extensions, - filter_ptr_t event_filter) : m_k8s(kube), m_state(state), + filter_ptr_t event_filter, + bool blocking_sockets) : m_k8s(kube), m_state(state), m_collector(std::make_shared()), m_uri(uri), m_ssl(ssl), m_bt(bt), m_stopped(true), + m_blocking_sockets(blocking_sockets), m_extensions(extensions), m_event_filter(event_filter) { @@ -112,15 +114,15 @@ k8s_net::handler_ptr_t k8s_net::get_dependency_handler(const handler_map_t& hand case k8s_component::K8S_REPLICATIONCONTROLLERS: return get_handler(handlers, k8s_component::K8S_PODS); case k8s_component::K8S_SERVICES: - return get_handler(handlers, k8s_component::K8S_REPLICATIONCONTROLLERS); + return get_handler(handlers, k8s_component::K8S_PODS); case k8s_component::K8S_REPLICASETS: - return get_handler(handlers, k8s_component::K8S_SERVICES); + return get_handler(handlers, k8s_component::K8S_PODS); case k8s_component::K8S_DAEMONSETS: - return get_handler(handlers, k8s_component::K8S_REPLICASETS); + return get_handler(handlers, k8s_component::K8S_PODS); case k8s_component::K8S_DEPLOYMENTS: - return get_handler(handlers, k8s_component::K8S_DAEMONSETS); + return get_handler(handlers, k8s_component::K8S_PODS); case k8s_component::K8S_EVENTS: - return get_handler(handlers, k8s_component::K8S_SERVICES); + return std::make_shared(); case k8s_component::K8S_COMPONENT_COUNT: default: break; } @@ -140,7 +142,7 @@ bool k8s_net::has_dependency(const k8s_component::type_map::value_type& componen k8s_net::handler_ptr_t k8s_net::make_handler(k8s_state_t& state, const k8s_component::type component, bool connect, handler_ptr_t dep, collector_ptr_t collector, const std::string& urlstr, - ssl_ptr_t ssl, bt_ptr_t bt, filter_ptr_t event_filter) + ssl_ptr_t ssl, bt_ptr_t bt, bool blocking, filter_ptr_t event_filter) { std::ostringstream os; if(!urlstr.empty()) @@ -154,23 +156,23 @@ k8s_net::handler_ptr_t k8s_net::make_handler(k8s_state_t& state, const k8s_compo switch(component) { case k8s_component::K8S_NODES: - return std::make_shared(state, dep, collector, os.str(), "1.0", ssl, bt, connect); + return std::make_shared(state, dep, collector, os.str(), "1.1", ssl, bt, connect, blocking); case k8s_component::K8S_NAMESPACES: - return std::make_shared(state, dep, collector, os.str(), "1.0", ssl, bt, connect); + return std::make_shared(state, dep, collector, os.str(), "1.1", ssl, bt, connect, blocking); case k8s_component::K8S_PODS: - return std::make_shared(state, dep, collector, os.str(), "1.0", ssl, bt, connect); + return std::make_shared(state, dep, collector, os.str(), "1.1", ssl, bt, connect, blocking); case k8s_component::K8S_REPLICATIONCONTROLLERS: - return std::make_shared(state, dep, collector, os.str(), "1.0", ssl, bt, connect); + return std::make_shared(state, dep, collector, os.str(), "1.1", ssl, bt, connect, blocking); case k8s_component::K8S_REPLICASETS: - return std::make_shared(state, dep, collector, os.str(), "1.0", ssl, bt, connect); + return std::make_shared(state, dep, collector, os.str(), "1.1", ssl, bt, connect, blocking); case k8s_component::K8S_SERVICES: - return std::make_shared(state, dep, collector, os.str(), "1.0", ssl, bt, connect); + return std::make_shared(state, dep, collector, os.str(), "1.1", ssl, bt, connect, blocking); case k8s_component::K8S_DAEMONSETS: - return std::make_shared(state, dep, collector, os.str(), "1.0", ssl, bt, connect); + return std::make_shared(state, dep, collector, os.str(), "1.1", ssl, bt, connect, blocking); case k8s_component::K8S_DEPLOYMENTS: - return std::make_shared(state, dep, collector, os.str(), "1.0", ssl, bt, connect); + return std::make_shared(state, dep, collector, os.str(), "1.1", ssl, bt, connect, blocking); case k8s_component::K8S_EVENTS: - return std::make_shared(state, dep, collector, os.str(), "1.0", ssl, bt, connect, event_filter); + return std::make_shared(state, dep, collector, os.str(), "1.1", ssl, bt, connect, blocking, event_filter); case k8s_component::K8S_COMPONENT_COUNT: default: return nullptr; @@ -184,7 +186,8 @@ void k8s_net::add_handler(const k8s_component::type_map::value_type& component) if(!has_handler(component)) { handler_ptr_t handler = - make_handler(m_state, component.first, true, get_dependency_handler(m_handlers, component), m_collector, m_uri.to_string(), m_ssl, m_bt, m_event_filter); + make_handler(m_state, component.first, true, get_dependency_handler(m_handlers, component), + m_collector, m_uri.to_string(), m_ssl, m_bt, m_blocking_sockets, m_event_filter); if(handler) { if(!m_machine_id.empty()) @@ -221,4 +224,49 @@ void k8s_net::add_handler(const k8s_component::type_map::value_type& component) sinsp_logger::SEV_TRACE); } } +#else // !HAS_CAPTURE + +#include "k8s_component.h" +#include "k8s_node_handler.h" +#include "k8s_namespace_handler.h" +#include "k8s_pod_handler.h" +#include "k8s_replicationcontroller_handler.h" +#include "k8s_replicaset_handler.h" +#include "k8s_service_handler.h" +#include "k8s_daemonset_handler.h" +#include "k8s_deployment_handler.h" +#include "k8s_event_handler.h" + +namespace k8s_net +{ + k8s_handler::ptr_t make_handler(k8s_state_t& state, const k8s_component::type component, bool /*connect*/) + { + switch(component) + { + case k8s_component::K8S_NODES: + return std::make_shared(state); + case k8s_component::K8S_NAMESPACES: + return std::make_shared(state); + case k8s_component::K8S_PODS: + return std::make_shared(state); + case k8s_component::K8S_REPLICATIONCONTROLLERS: + return std::make_shared(state); + case k8s_component::K8S_REPLICASETS: + return std::make_shared(state); + case k8s_component::K8S_SERVICES: + return std::make_shared(state); + case k8s_component::K8S_DAEMONSETS: + return std::make_shared(state); + case k8s_component::K8S_DEPLOYMENTS: + return std::make_shared(state); + case k8s_component::K8S_EVENTS: + return std::make_shared(state); + case k8s_component::K8S_COMPONENT_COUNT: + default: + return nullptr; + } + return nullptr; + } +} + #endif // HAS_CAPTURE diff --git a/userspace/libsinsp/k8s_net.h b/userspace/libsinsp/k8s_net.h index 7417ac2f2b..58ed2a2563 100644 --- a/userspace/libsinsp/k8s_net.h +++ b/userspace/libsinsp/k8s_net.h @@ -33,14 +33,15 @@ class k8s_net ssl_ptr_t ssl = nullptr, bt_ptr_t bt = nullptr, ext_list_ptr_t extensions = nullptr, - filter_ptr_t event_filter = nullptr); + filter_ptr_t event_filter = nullptr, + bool blocking_sockets = false); ~k8s_net(); static handler_ptr_t make_handler(k8s_state_t& state, const k8s_component::type component, bool connect = true, handler_ptr_t dep = std::make_shared(), collector_ptr_t collector = nullptr, const std::string& urlstr = "", - ssl_ptr_t ssl = nullptr, bt_ptr_t bt = nullptr, + ssl_ptr_t ssl = nullptr, bt_ptr_t bt = nullptr, bool blocking = false, filter_ptr_t event_filter = nullptr); void add_handler(const k8s_component::type_map::value_type& component); bool has_handler(const k8s_component::type_map::value_type& component); @@ -77,6 +78,7 @@ class k8s_net bt_ptr_t m_bt; bool m_stopped; handler_map_t m_handlers; + bool m_blocking_sockets = false; ext_list_ptr_t m_extensions; filter_ptr_t m_event_filter; std::string m_machine_id; @@ -152,4 +154,14 @@ inline const k8s_net::handler_map_t& k8s_net::handlers() const return m_handlers; } +#else // !HAS_CAPTURE + +#include "k8s_component.h" +#include "k8s_handler.h" + +namespace k8s_net +{ + k8s_handler::ptr_t make_handler(k8s_state_t& state, const k8s_component::type component, bool /*connect*/); +} + #endif // HAS_CAPTURE diff --git a/userspace/libsinsp/k8s_node_handler.cpp b/userspace/libsinsp/k8s_node_handler.cpp index 0b0eb1f943..262ea23e9b 100644 --- a/userspace/libsinsp/k8s_node_handler.cpp +++ b/userspace/libsinsp/k8s_node_handler.cpp @@ -45,18 +45,26 @@ std::string k8s_node_handler::STATE_FILTER = " ]" "}"; -k8s_node_handler::k8s_node_handler(k8s_state_t& state, - ptr_t dependency_handler, - collector_ptr_t collector, - std::string url, - const std::string& http_version, - ssl_ptr_t ssl, - bt_ptr_t bt, - bool connect): +k8s_node_handler::k8s_node_handler(k8s_state_t& state +#ifdef HAS_CAPTURE + ,ptr_t dependency_handler + ,collector_ptr_t collector + ,std::string url + ,const std::string& http_version + ,ssl_ptr_t ssl + ,bt_ptr_t bt + ,bool connect + ,bool blocking_socket +#endif // HAS_CAPTURE + ): k8s_handler("k8s_node_handler", true, +#ifdef HAS_CAPTURE url, "/api/v1/nodes", - STATE_FILTER, EVENT_FILTER, dependency_handler, collector, - http_version, 1000L, ssl, bt, &state, true, connect) + STATE_FILTER, EVENT_FILTER, collector, + http_version, 1000L, ssl, bt, true, + connect, dependency_handler, blocking_socket, +#endif // HAS_CAPTURE + ~0, &state) { } diff --git a/userspace/libsinsp/k8s_node_handler.h b/userspace/libsinsp/k8s_node_handler.h index 0de3ab8d0f..126479bf90 100644 --- a/userspace/libsinsp/k8s_node_handler.h +++ b/userspace/libsinsp/k8s_node_handler.h @@ -14,14 +14,18 @@ class sinsp; class k8s_node_handler : public k8s_handler { public: - k8s_node_handler(k8s_state_t& state, - ptr_t dependency_handler, - collector_ptr_t collector = nullptr, - std::string url = "", - const std::string& http_version = "1.0", - ssl_ptr_t ssl = 0, - bt_ptr_t bt = 0, - bool connect = true); + k8s_node_handler(k8s_state_t& state +#ifdef HAS_CAPTURE + ,ptr_t dependency_handler + ,collector_ptr_t collector = nullptr + ,std::string url = "" + ,const std::string& http_version = "1.1" + ,ssl_ptr_t ssl = 0 + ,bt_ptr_t bt = 0 + ,bool connect = true + ,bool blocking_socket = false +#endif // HAS_CAPTURE + ); ~k8s_node_handler(); diff --git a/userspace/libsinsp/k8s_pod_handler.cpp b/userspace/libsinsp/k8s_pod_handler.cpp index b5d9de195a..74f145423d 100644 --- a/userspace/libsinsp/k8s_pod_handler.cpp +++ b/userspace/libsinsp/k8s_pod_handler.cpp @@ -57,18 +57,26 @@ std::string k8s_pod_handler::STATE_FILTER = " ]" "}"; -k8s_pod_handler::k8s_pod_handler(k8s_state_t& state, - ptr_t dependency_handler, - collector_ptr_t collector, - std::string url, - const std::string& http_version, - ssl_ptr_t ssl, - bt_ptr_t bt, - bool connect): - k8s_handler("k8s_pod_handler", true, url, - "/api/v1/pods?fieldSelector=status.phase%3DRunning", - STATE_FILTER, EVENT_FILTER, dependency_handler, collector, - http_version, 1000L, ssl, bt, &state, true, connect) +k8s_pod_handler::k8s_pod_handler(k8s_state_t& state +#ifdef HAS_CAPTURE + ,ptr_t dependency_handler + ,collector_ptr_t collector + ,std::string url + ,const std::string& http_version + ,ssl_ptr_t ssl + ,bt_ptr_t bt + ,bool connect + ,bool blocking_socket +#endif // HAS_CAPTURE + ): + k8s_handler("k8s_pod_handler", true, +#ifdef HAS_CAPTURE + url, "/api/v1/pods?fieldSelector=status.phase%3DRunning", + STATE_FILTER, EVENT_FILTER, collector, + http_version, 1000L, ssl, bt, true, + connect, dependency_handler, blocking_socket, +#endif // HAS_CAPTURE + ~0, &state) { } diff --git a/userspace/libsinsp/k8s_pod_handler.h b/userspace/libsinsp/k8s_pod_handler.h index 2cb8554903..fbe94aeeb5 100644 --- a/userspace/libsinsp/k8s_pod_handler.h +++ b/userspace/libsinsp/k8s_pod_handler.h @@ -14,14 +14,18 @@ class sinsp; class k8s_pod_handler : public k8s_handler { public: - k8s_pod_handler(k8s_state_t& state, - ptr_t dependency_handler, - collector_ptr_t collector = nullptr, - std::string url = "", - const std::string& http_version = "1.0", - ssl_ptr_t ssl = 0, - bt_ptr_t bt = 0, - bool connect = true); + k8s_pod_handler(k8s_state_t& state +#ifdef HAS_CAPTURE + ,ptr_t dependency_handler + ,collector_ptr_t collector = nullptr + ,std::string url = "" + ,const std::string& http_version = "1.1" + ,ssl_ptr_t ssl = 0 + ,bt_ptr_t bt = 0 + ,bool connect = true + ,bool blocking_socket = false +#endif // HAS_CAPTURE + ); ~k8s_pod_handler(); diff --git a/userspace/libsinsp/k8s_replicaset_handler.cpp b/userspace/libsinsp/k8s_replicaset_handler.cpp index cc04fef474..a9388d92d0 100644 --- a/userspace/libsinsp/k8s_replicaset_handler.cpp +++ b/userspace/libsinsp/k8s_replicaset_handler.cpp @@ -51,18 +51,27 @@ std::string k8s_replicaset_handler::STATE_FILTER = " ]" "}"; -k8s_replicaset_handler::k8s_replicaset_handler(k8s_state_t& state, - ptr_t dependency_handler, - collector_ptr_t collector, - std::string url, - const std::string& http_version, - ssl_ptr_t ssl, - bt_ptr_t bt, - bool connect): +k8s_replicaset_handler::k8s_replicaset_handler(k8s_state_t& state +#ifdef HAS_CAPTURE + ,ptr_t dependency_handler + ,collector_ptr_t collector + ,std::string url + ,const std::string& http_version + ,ssl_ptr_t ssl + ,bt_ptr_t bt + ,bool connect + ,bool blocking_socket +#endif // HAS_CAPTURE + ): k8s_handler("k8s_replicaset_handler", true, +#ifdef HAS_CAPTURE url, "/apis/extensions/v1beta1/replicasets", - STATE_FILTER, EVENT_FILTER, dependency_handler, collector, - http_version, 1000L, ssl, bt, &state, true, connect) + STATE_FILTER, EVENT_FILTER, collector, + http_version, 1000L, ssl, bt, true, + connect, dependency_handler, blocking_socket, +#endif // HAS_CAPTURE + 100, // max msgs + &state) { } diff --git a/userspace/libsinsp/k8s_replicaset_handler.h b/userspace/libsinsp/k8s_replicaset_handler.h index dc9f03363f..361fc11d06 100644 --- a/userspace/libsinsp/k8s_replicaset_handler.h +++ b/userspace/libsinsp/k8s_replicaset_handler.h @@ -14,14 +14,18 @@ class sinsp; class k8s_replicaset_handler : public k8s_handler { public: - k8s_replicaset_handler(k8s_state_t& state, - ptr_t dependency_handler, - collector_ptr_t collector = nullptr, - std::string url = "", - const std::string& http_version = "1.0", - ssl_ptr_t ssl = 0, - bt_ptr_t bt = 0, - bool connect = true); + k8s_replicaset_handler(k8s_state_t& state +#ifdef HAS_CAPTURE + ,ptr_t dependency_handler + ,collector_ptr_t collector = nullptr + ,std::string url = "" + ,const std::string& http_version = "1.1" + ,ssl_ptr_t ssl = 0 + ,bt_ptr_t bt = 0 + ,bool connect = true + ,bool blocking_socket = false +#endif // HAS_CAPTURE + ); ~k8s_replicaset_handler(); diff --git a/userspace/libsinsp/k8s_replicationcontroller_handler.cpp b/userspace/libsinsp/k8s_replicationcontroller_handler.cpp index 28d3182fa0..2d2f6b588b 100644 --- a/userspace/libsinsp/k8s_replicationcontroller_handler.cpp +++ b/userspace/libsinsp/k8s_replicationcontroller_handler.cpp @@ -51,18 +51,27 @@ std::string k8s_replicationcontroller_handler::STATE_FILTER = " ]" "}"; -k8s_replicationcontroller_handler::k8s_replicationcontroller_handler(k8s_state_t& state, - ptr_t dependency_handler, - collector_ptr_t collector, - std::string url, - const std::string& http_version, - ssl_ptr_t ssl, - bt_ptr_t bt, - bool connect): +k8s_replicationcontroller_handler::k8s_replicationcontroller_handler(k8s_state_t& state +#ifdef HAS_CAPTURE + ,ptr_t dependency_handler + ,collector_ptr_t collector + ,std::string url + ,const std::string& http_version + ,ssl_ptr_t ssl + ,bt_ptr_t bt + ,bool connect + ,bool blocking_socket +#endif // HAS_CAPTURE + ): k8s_handler("k8s_replicationcontroller_handler", true, +#ifdef HAS_CAPTURE url, "/api/v1/replicationcontrollers", - STATE_FILTER, EVENT_FILTER, dependency_handler, collector, - http_version, 1000L, ssl, bt, &state, true, connect) + STATE_FILTER, EVENT_FILTER, collector, + http_version, 1000L, ssl, bt, true, + connect, dependency_handler, blocking_socket, +#endif // HAS_CAPTURE + 100, // max msgs + &state) { } diff --git a/userspace/libsinsp/k8s_replicationcontroller_handler.h b/userspace/libsinsp/k8s_replicationcontroller_handler.h index c2828a54b3..32aa027e64 100644 --- a/userspace/libsinsp/k8s_replicationcontroller_handler.h +++ b/userspace/libsinsp/k8s_replicationcontroller_handler.h @@ -14,14 +14,18 @@ class sinsp; class k8s_replicationcontroller_handler : public k8s_handler { public: - k8s_replicationcontroller_handler(k8s_state_t& state, - ptr_t dependency_handler, - collector_ptr_t collector = nullptr, - std::string url = "", - const std::string& http_version = "1.0", - ssl_ptr_t ssl = 0, - bt_ptr_t bt = 0, - bool connect = true); + k8s_replicationcontroller_handler(k8s_state_t& state +#ifdef HAS_CAPTURE + ,ptr_t dependency_handler + ,collector_ptr_t collector = nullptr + ,std::string url = "" + ,const std::string& http_version = "1.1" + ,ssl_ptr_t ssl = 0 + ,bt_ptr_t bt = 0 + ,bool connect = true + ,bool blocking_socket = false +#endif // HAS_CAPTURE + ); ~k8s_replicationcontroller_handler(); diff --git a/userspace/libsinsp/k8s_service_handler.cpp b/userspace/libsinsp/k8s_service_handler.cpp index b6f48d4fe4..19f8f1eeb1 100644 --- a/userspace/libsinsp/k8s_service_handler.cpp +++ b/userspace/libsinsp/k8s_service_handler.cpp @@ -51,18 +51,27 @@ std::string k8s_service_handler::STATE_FILTER = " ]" "}"; -k8s_service_handler::k8s_service_handler(k8s_state_t& state, - ptr_t dependency_handler, - collector_ptr_t collector, - std::string url, - const std::string& http_version, - ssl_ptr_t ssl, - bt_ptr_t bt, - bool connect): +k8s_service_handler::k8s_service_handler(k8s_state_t& state +#ifdef HAS_CAPTURE + ,ptr_t dependency_handler + ,collector_ptr_t collector + ,std::string url + ,const std::string& http_version + ,ssl_ptr_t ssl + ,bt_ptr_t bt + ,bool connect + ,bool blocking_socket +#endif // HAS_CAPTURE + ): k8s_handler("k8s_service_handler", true, +#ifdef HAS_CAPTURE url, "/api/v1/services", - STATE_FILTER, EVENT_FILTER, dependency_handler, collector, - http_version, 1000L, ssl, bt, &state, true, connect) + STATE_FILTER, EVENT_FILTER, collector, + http_version, 1000L, ssl, bt, true, + connect, dependency_handler, blocking_socket, +#endif // HAS_CAPTURE + 100, // max msgs + &state) { } diff --git a/userspace/libsinsp/k8s_service_handler.h b/userspace/libsinsp/k8s_service_handler.h index 91b4e27028..d3490b371a 100644 --- a/userspace/libsinsp/k8s_service_handler.h +++ b/userspace/libsinsp/k8s_service_handler.h @@ -14,14 +14,18 @@ class sinsp; class k8s_service_handler : public k8s_handler { public: - k8s_service_handler(k8s_state_t& state, - ptr_t dependency_handler, - collector_ptr_t collector = nullptr, - std::string url = "", - const std::string& http_version = "1.0", - ssl_ptr_t ssl = 0, - bt_ptr_t bt = 0, - bool connect = true); + k8s_service_handler(k8s_state_t& state +#ifdef HAS_CAPTURE + ,ptr_t dependency_handler + ,collector_ptr_t collector = nullptr + ,std::string url = "" + ,const std::string& http_version = "1.1" + ,ssl_ptr_t ssl = 0 + ,bt_ptr_t bt = 0 + ,bool connect = true + ,bool blocking_socket = false +#endif // HAS_CAPTURE + ); ~k8s_service_handler(); diff --git a/userspace/libsinsp/k8s_state.cpp b/userspace/libsinsp/k8s_state.cpp index 694c9c9c72..5fe74315df 100644 --- a/userspace/libsinsp/k8s_state.cpp +++ b/userspace/libsinsp/k8s_state.cpp @@ -293,6 +293,7 @@ void k8s_state_t::clear(k8s_component::type type) void k8s_state_t::update_cache(const k8s_component::type_map::key_type& component) { +#ifndef HAS_ANALYZER switch (component) { case k8s_component::K8S_NAMESPACES: @@ -416,19 +417,19 @@ void k8s_state_t::update_cache(const k8s_component::type_map::key_type& componen case k8s_component::K8S_DAEMONSETS: { // TODO - /*const k8s_services& services = get_services(); + /*const k8s_daemonsets& daemonsets = get_daemonsets(); const k8s_pods& pods = get_pods(); - k8s_state_t::pod_service_map& pod_svc_map = get_pod_service_map(); + k8s_state_t::pod_daemonset_map& pod_svc_map = get_pod_daemonset_map(); pod_svc_map.clear(); - for(const auto& service : services) + for(const auto& daemonset : daemonsets) { - std::vector pod_subset = service.get_selected_pods(pods); + std::vector pod_subset = daemonset.get_selected_pods(pods); for(auto& pod : pod_subset) { const std::string& pod_uid = pod->get_uid(); - if(!is_component_cached(pod_svc_map, pod_uid, &service)) + if(!is_component_cached(pod_svc_map, pod_uid, &daemonset)) { - cache_component(pod_svc_map, pod_uid, &service); + cache_component(pod_svc_map, pod_uid, &daemonset); } else { @@ -466,6 +467,7 @@ void k8s_state_t::update_cache(const k8s_component::type_map::key_type& componen default: return; } +#endif // HAS_ANALYZER } k8s_component::type k8s_state_t::component_from_json(const Json::Value& item) diff --git a/userspace/libsinsp/k8s_state.h b/userspace/libsinsp/k8s_state.h index 656b77308c..a2d7b40093 100644 --- a/userspace/libsinsp/k8s_state.h +++ b/userspace/libsinsp/k8s_state.h @@ -232,6 +232,8 @@ class k8s_state_t // any component by uid const k8s_component* get_component(const std::string& uid, std::string* t = 0) const; +#ifndef HAS_ANALYZER + // pod by container; const k8s_pod_t* get_pod(const std::string& container) const { @@ -250,9 +252,12 @@ class k8s_state_t const pod_rs_map& get_pod_rs_map() const { return m_pod_rss; } const pod_deployment_map& get_pod_deployment_map() const { return m_pod_deployments; } -#ifdef HAS_CAPTURE +#endif // HAS_ANALYZER + void set_capture_version(int version); int get_capture_version() const; + +#ifdef HAS_CAPTURE typedef std::deque event_list_t; const event_list_t& get_capture_events() const { return m_capture_events; } void enqueue_capture_event(const Json::Value& item); @@ -317,6 +322,8 @@ class k8s_state_t } } +#ifndef HAS_ANALYZER + namespace_map& get_namespace_map() { return m_namespace_map; } container_pod_map& get_container_pod_map() { return m_container_pods; } pod_service_map& get_pod_service_map() { return m_pod_services; } @@ -324,15 +331,23 @@ class k8s_state_t pod_rs_map& get_pod_rs_map() { return m_pod_rss; } pod_deployment_map& get_pod_deployment_map() { return m_pod_deployments; } +#endif // HAS_ANALYZER + static const std::string m_docker_prefix; // "docker://" static const std::string m_rkt_prefix; // "rkt://" static const unsigned m_id_length; // portion of the ID to be cached (=12) + +#ifndef HAS_ANALYZER + namespace_map m_namespace_map; container_pod_map m_container_pods; pod_service_map m_pod_services; pod_rc_map m_pod_rcs; pod_rs_map m_pod_rss; pod_deployment_map m_pod_deployments; + +#endif // HAS_ANALYZER + #ifdef HAS_CAPTURE event_list_t m_capture_events; #endif // HAS_CAPTURE diff --git a/userspace/libsinsp/lua_parser_api.cpp b/userspace/libsinsp/lua_parser_api.cpp index 8975c53d41..7511814e38 100644 --- a/userspace/libsinsp/lua_parser_api.cpp +++ b/userspace/libsinsp/lua_parser_api.cpp @@ -212,7 +212,7 @@ int lua_parser_cbacks::rel_expr(lua_State *ls) chk->m_boolop = parser->m_last_boolop; parser->m_last_boolop = BO_NONE; - chk->parse_field_name(fld, true); + chk->parse_field_name(fld, true, true); const char* cmpop = luaL_checkstring(ls, 2); chk->m_cmpop = string_to_cmpop(cmpop); diff --git a/userspace/libsinsp/marathon_http.cpp b/userspace/libsinsp/marathon_http.cpp index 5a845bf78c..f7937b681f 100644 --- a/userspace/libsinsp/marathon_http.cpp +++ b/userspace/libsinsp/marathon_http.cpp @@ -18,8 +18,8 @@ #include #include -marathon_http::marathon_http(mesos& m, const uri& url, bool discover_marathon, int timeout_ms): - mesos_http(m, url, false, discover_marathon, timeout_ms) +marathon_http::marathon_http(mesos& m, const uri& url, bool discover_marathon, int timeout_ms, const string& token): + mesos_http(m, url, false, discover_marathon, timeout_ms, token) { g_logger.log("Creating Marathon HTTP object for [" + url.to_string(false) + "] ...", sinsp_logger::SEV_DEBUG); if(refresh_data()) diff --git a/userspace/libsinsp/marathon_http.h b/userspace/libsinsp/marathon_http.h index 75af42fa4b..03ae5a702e 100644 --- a/userspace/libsinsp/marathon_http.h +++ b/userspace/libsinsp/marathon_http.h @@ -16,7 +16,7 @@ class marathon_http : public mesos_http public: typedef std::shared_ptr ptr_t; - marathon_http(mesos& m, const uri& url, bool discover_marathon, int timeout_ms = 5000L); + marathon_http(mesos& m, const uri& url, bool discover_marathon, int timeout_ms = 5000L, const string& token = ""); ~marathon_http(); diff --git a/userspace/libsinsp/mesos.cpp b/userspace/libsinsp/mesos.cpp index 713c132a1d..242c258eee 100644 --- a/userspace/libsinsp/mesos.cpp +++ b/userspace/libsinsp/mesos.cpp @@ -20,6 +20,7 @@ const std::string mesos::default_marathon_uri = "http://localhost:8080"; const std::string mesos::default_groups_api = "/v2/groups"; const std::string mesos::default_apps_api = "/v2/apps?embed=apps.tasks"; const std::string mesos::default_watch_api = "/v2/events"; +const std::string mesos::default_version_api = "/version"; const int mesos::default_timeout_ms = 5000; mesos::mesos(const std::string& mesos_state_json, @@ -132,8 +133,53 @@ mesos::mesos(const std::string& state_uri, init(); } +mesos::mesos(const std::string& state_uri, + const uri_list_t& marathon_uris, + bool discover_mesos_leader, + bool discover_marathon_leader, + const credentials_t& dcos_enterprise_credentials, + int timeout_ms, + bool is_captured, + bool verbose): +#ifdef HAS_CAPTURE + m_collector(false), + m_mesos_uri(state_uri), + m_marathon_uris(marathon_uris), +#endif // HAS_CAPTURE + m_state(is_captured, verbose), + m_discover_mesos_leader(discover_mesos_leader), + m_discover_marathon_uris(discover_marathon_leader || marathon_uris.empty()), + m_timeout_ms(timeout_ms), + m_verbose(verbose), + m_testing(false), + m_dcos_enterprise_credentials(dcos_enterprise_credentials) +{ +#ifdef HAS_CAPTURE + g_logger.log(std::string("Creating Mesos object for [" + + (m_mesos_uri.empty() ? std::string("capture replay") : m_mesos_uri) + + "], failover autodiscovery set to ") + + (m_discover_mesos_leader ? "true" : "false"), + sinsp_logger::SEV_DEBUG); + + if(m_marathon_uris.size() > 1) + { + std::string marathon_uri = m_marathon_uris[0]; + m_marathon_uris.clear(); + m_marathon_uris.emplace_back(marathon_uri); + g_logger.log("Multiple root marathon URIs configured; only the first one (" + marathon_uri + ") will have effect;" + " others will be treated as generic frameworks (user Marathon frameworks will be discovered).", sinsp_logger::SEV_WARNING); + } + + authenticate(); +#endif + init(); +} + mesos::~mesos() { +#ifdef HAS_CAPTURE + curl_global_cleanup(); +#endif // HAS_CAPTURE } void mesos::init() @@ -141,15 +187,19 @@ void mesos::init() #ifdef HAS_CAPTURE if(!m_mesos_uri.empty()) { + curl_global_init(CURL_GLOBAL_DEFAULT); m_collector.remove_all(); if((m_state_http) && (!m_state_http.unique())) { throw sinsp_exception("Invalid access to Mesos initializer: mesos state http client for [" + m_mesos_uri + "] not unique."); } - m_state_http = std::make_shared(*this, m_mesos_uri + default_state_api, m_discover_mesos_leader, m_marathon_uris.empty(), m_timeout_ms); + m_state_http = std::make_shared(*this, m_mesos_uri + default_state_api, m_discover_mesos_leader, m_marathon_uris.empty(), m_timeout_ms, m_token); rebuild_mesos_state(true); - init_marathon(); + if(!has_marathon()) + { + init_marathon(); + } } #endif // HAS_CAPTURE } @@ -162,16 +212,15 @@ void mesos::init_marathon() m_marathon_groups_http.clear(); m_marathon_apps_http.clear(); - bool discover_marathon = m_marathon_uris.size() == 0; - const uri_list_t& marathons = discover_marathon ? m_state_http->get_marathon_uris() : m_marathon_uris; + const uri_list_t& marathons = m_discover_marathon_uris ? m_state_http->get_marathon_uris() : m_marathon_uris; if(marathons.size()) { g_logger.log("Found " + std::to_string(marathons.size()) + " Marathon URIs", sinsp_logger::SEV_DEBUG); for(const auto& muri : marathons) { g_logger.log("Creating Marathon http objects: " + uri(muri).to_string(false), sinsp_logger::SEV_DEBUG); - m_marathon_groups_http[muri] = std::make_shared(*this, muri + default_groups_api, discover_marathon, m_timeout_ms); - m_marathon_apps_http[muri] = std::make_shared(*this, muri + default_apps_api, discover_marathon, m_timeout_ms); + m_marathon_groups_http[muri] = std::make_shared(*this, muri + default_groups_api, m_discover_marathon_uris, m_timeout_ms, m_token); + m_marathon_apps_http[muri] = std::make_shared(*this, muri + default_apps_api, m_discover_marathon_uris, m_timeout_ms, m_token); } if(has_marathon()) @@ -183,6 +232,74 @@ void mesos::init_marathon() #endif // HAS_CAPTURE } +void mesos::refresh_token() +{ +#ifdef HAS_CAPTURE + authenticate(); + m_state_http->set_token(m_token); + if(has_marathon()) + { + for(auto& group_http : m_marathon_groups_http) + { + if(group_http.second) + { + group_http.second->set_token(m_token); + } + else + { + throw sinsp_exception("Marathon groups HTTP client is null."); + } + } + for(auto& app_http : m_marathon_apps_http) + { + if(app_http.second) + { + app_http.second->set_token(m_token); + } + else + { + throw sinsp_exception("Marathon apps HTTP client is null."); + } + } + } +#endif // HAS_CAPTURE +} + +void mesos::authenticate() +{ +#ifdef HAS_CAPTURE + sinsp_curl auth_request(uri("https://localhost/acs/api/v1/auth/login"), "", ""); + Json::FastWriter json_writer; + Json::Value auth_obj; + auth_obj["uid"] = m_dcos_enterprise_credentials.first; + auth_obj["password"] = m_dcos_enterprise_credentials.second; + auth_request.add_header("Content-Type: application/json"); + auth_request.setopt(CURLOPT_POST, 1); + auth_request.set_body(json_writer.write(auth_obj)); + //auth_request.enable_debug(); + auto response = auth_request.get_data(); + + if(auth_request.get_response_code() == 200) + { + Json::Reader json_reader; + Json::Value response_obj; + auto parse_ok = json_reader.parse(response, response_obj, false); + if(parse_ok && response_obj.isMember("token")) + { + m_token = response_obj["token"].asString(); + g_logger.format(sinsp_logger::SEV_DEBUG, "Mesos authenticated with token=%s", m_token.c_str()); + } + else + { + throw sinsp_exception(string("Cannot authenticate on Mesos master, response=") + response); + } + } else + { + throw sinsp_exception(string("Cannot authenticate on Mesos master, response_code=") + to_string(auth_request.get_response_code())); + } +#endif // HAS_CAPTURE +} + void mesos::refresh() { rebuild_mesos_state(); @@ -419,8 +536,65 @@ void mesos::send_data_request(bool collect) if(collect) { collect_data(); } } +void mesos::capture_frameworks(const Json::Value& root, Json::Value& capture) +{ + const Json::Value& frameworks = root["frameworks"]; + if(!frameworks.isNull()) + { + if(frameworks.isArray()) + { + if(frameworks.size()) + { + capture["frameworks"] = Json::arrayValue; + for(const auto& framework : frameworks) + { + Json::Value c_framework; + c_framework["active"] = framework["active"]; + c_framework["id"] = framework["id"]; + c_framework["name"] = framework["name"]; + c_framework["hostname"] = framework["hostname"]; + c_framework["webui_url"] = framework["webui_url"]; + c_framework["tasks"] = Json::arrayValue; + Json::Value& c_tasks = c_framework["tasks"]; + for(const auto& task : framework["tasks"]) + { + Json::Value& c_task = c_tasks.append(Json::Value()); + c_task["id"] = task["id"]; + c_task["name"] = task["name"]; + c_task["framework_id"] = task["framework_id"]; + c_task["executor_id"] = task["executor_id"]; + c_task["slave_id"] = task["slave_id"]; + c_task["state"] = task["state"]; + //? TODO: statuses + c_task["labels"] = task["labels"]; + } + capture["frameworks"].append(c_framework); + } + } + } + } +} + +void mesos::capture_slaves(const Json::Value& root, Json::Value& capture) +{ + const Json::Value& slaves = root["slaves"]; + if(!slaves.isNull()) + { + capture["slaves"] = Json::arrayValue; + for(const auto& slave : slaves) + { + Json::Value c_slave; + c_slave["hostname"] = slave["hostname"]; + c_slave["id"] = slave["id"]; + capture["slaves"].append(c_slave); + } + } +} +#endif // HAS_CAPTURE + bool mesos::collect_data() { +#ifdef HAS_CAPTURE const int tout_s = 30; //TODO: see if we can do better here - instead of timing out, depending on @@ -499,7 +673,7 @@ bool mesos::collect_data() { g_logger.log("Detected null Marathon app (" + app_it->first + "), resetting current state.", sinsp_logger::SEV_WARNING); m_mesos_state_json.reset(); - json_map_type_t::iterator app_it = m_marathon_apps_json.find(group.first); + group.second.reset(); app_it->second.reset(); m_json_error = false; } @@ -541,63 +715,10 @@ bool mesos::collect_data() } return ret; -} - -void mesos::capture_frameworks(const Json::Value& root, Json::Value& capture) -{ - const Json::Value& frameworks = root["frameworks"]; - if(!frameworks.isNull()) - { - if(frameworks.isArray()) - { - if(frameworks.size()) - { - capture["frameworks"] = Json::arrayValue; - for(const auto& framework : frameworks) - { - Json::Value c_framework; - c_framework["active"] = framework["active"]; - c_framework["id"] = framework["id"]; - c_framework["name"] = framework["name"]; - c_framework["hostname"] = framework["hostname"]; - c_framework["webui_url"] = framework["webui_url"]; - c_framework["tasks"] = Json::arrayValue; - Json::Value& c_tasks = c_framework["tasks"]; - for(const auto& task : framework["tasks"]) - { - Json::Value& c_task = c_tasks.append(Json::Value()); - c_task["id"] = task["id"]; - c_task["name"] = task["name"]; - c_task["framework_id"] = task["framework_id"]; - c_task["executor_id"] = task["executor_id"]; - c_task["slave_id"] = task["slave_id"]; - c_task["state"] = task["state"]; - //? TODO: statuses - c_task["labels"] = task["labels"]; - } - capture["frameworks"].append(c_framework); - } - } - } - } -} - -void mesos::capture_slaves(const Json::Value& root, Json::Value& capture) -{ - const Json::Value& slaves = root["slaves"]; - if(!slaves.isNull()) - { - capture["slaves"] = Json::arrayValue; - for(const auto& slave : slaves) - { - Json::Value c_slave; - c_slave["hostname"] = slave["hostname"]; - c_slave["id"] = slave["id"]; - capture["slaves"].append(c_slave); - } - } -} +#else + return true; #endif // HAS_CAPTURE +} void mesos::handle_frameworks(const Json::Value& root) { @@ -643,10 +764,16 @@ void mesos::handle_frameworks(const Json::Value& root) { g_logger.log("New or activated Mesos framework detected: " + name + " [" + uid.asString() + ']', sinsp_logger::SEV_INFO); m_activated_frameworks.insert(uid.asString()); - if(mesos_framework::is_root_marathon(name)) +#ifdef HAS_CAPTURE + if(mesos_framework::is_root_marathon(name) && + find_if(m_marathon_groups_http.begin(), m_marathon_groups_http.end(), [uid](const decltype(m_marathon_groups_http)::value_type& item) + { + return item.second->get_framework_id() == uid.asString(); + }) == m_marathon_groups_http.end()) { init_marathon(); } +#endif } } } @@ -838,16 +965,6 @@ void mesos::set_state_json(json_ptr_t json, const std::string&) void mesos::parse_state(Json::Value&& root) { clear_mesos(); -#ifdef HAS_CAPTURE - if(m_discover_marathon_uris && !has_marathon()) - { - m_state_http->discover_framework_uris(root["frameworks"]); - if(has_marathon()) - { - init_marathon(); - } - } -#endif // HAS_CAPTURE handle_frameworks(root); handle_slaves(root); #ifdef HAS_CAPTURE diff --git a/userspace/libsinsp/mesos.h b/userspace/libsinsp/mesos.h index 543f431f2b..01fb11c4a3 100644 --- a/userspace/libsinsp/mesos.h +++ b/userspace/libsinsp/mesos.h @@ -35,6 +35,7 @@ class mesos static const std::string default_groups_api; static const std::string default_apps_api; static const std::string default_watch_api; + static const std::string default_version_api; static const int default_timeout_ms; // constructor for testing only, not to be used in production @@ -52,6 +53,15 @@ class mesos bool is_captured = false, bool verbose = false); + mesos(const std::string& state_uri, + const uri_list_t& marathon_uris = uri_list_t(), + bool discover_mesos_leader = false, + bool discover_marathon_leader = false, + const credentials_t& dcos_enterprise_credentials = credentials_t(), + int timeout_ms = default_timeout_ms, + bool is_captured = false, + bool verbose = false); + ~mesos(); const mesos_state_t& get_state() const; @@ -63,10 +73,11 @@ class mesos void clear_marathon(); void simulate_event(const std::string& json); - + bool collect_data(); + void refresh_token(); + #ifdef HAS_CAPTURE void send_data_request(bool collect = true); - bool collect_data(); const mesos_state_t::capture_list& get_capture_events() const; std::string dequeue_capture_event(); @@ -118,6 +129,7 @@ class mesos private: void init(); void init_marathon(); + void authenticate(); void rebuild_mesos_state(bool full = false); void rebuild_marathon_state(bool full = false); @@ -157,7 +169,10 @@ class mesos bool m_testing = false; uri::credentials_t m_mesos_credentials; uri::credentials_t m_marathon_credentials; - + uri::credentials_t m_dcos_enterprise_credentials; + string m_token; + bool m_token_authentication; + typedef std::unordered_set framework_list_t; framework_list_t m_inactive_frameworks; framework_list_t m_activated_frameworks; diff --git a/userspace/libsinsp/mesos_http.cpp b/userspace/libsinsp/mesos_http.cpp index a252455479..18ac9f5b5d 100644 --- a/userspace/libsinsp/mesos_http.cpp +++ b/userspace/libsinsp/mesos_http.cpp @@ -20,7 +20,10 @@ #include #include -mesos_http::mesos_http(mesos& m, const uri& url, bool discover_mesos_lead_master, bool discover_marathon, int timeout_ms): +mesos_http::mesos_http(mesos& m, const uri& url, + bool discover_mesos_lead_master, + bool discover_marathon, + int timeout_ms, const string& token): m_sync_curl(curl_easy_init()), m_select_curl(curl_easy_init()), m_mesos(m), @@ -30,10 +33,10 @@ mesos_http::mesos_http(mesos& m, const uri& url, bool discover_mesos_lead_master m_timeout_ms(timeout_ms), m_callback_func(0), m_curl_version(curl_version_info(CURLVERSION_NOW)), - m_request(make_request(url, m_curl_version)), m_is_mesos_state(url.to_string().find(mesos::default_state_api) != std::string::npos), m_discover_lead_master(discover_mesos_lead_master), - m_discover_marathon(discover_marathon) + m_discover_marathon(discover_marathon), + m_token(token) { if(!m_sync_curl || !m_select_curl) { @@ -41,17 +44,25 @@ mesos_http::mesos_http(mesos& m, const uri& url, bool discover_mesos_lead_master } ASSERT(m_curl_version); - if((m_url.get_scheme() == "https") && (m_curl_version && !(m_curl_version->features | CURL_VERSION_SSL))) + + m_request = make_request(url, m_curl_version); + if(!m_token.empty()) { - throw sinsp_exception("mesos_http: HTTPS NOT supported"); + m_sync_curl_headers.add(string("Authorization: token=") + m_token); + check_error(curl_easy_setopt(m_sync_curl, CURLOPT_HTTPHEADER, m_sync_curl_headers.ptr())); + } + if(m_url.is_secure()) + { + check_error(curl_easy_setopt(m_sync_curl, CURLOPT_SSL_VERIFYPEER, 0)); + check_error(curl_easy_setopt(m_sync_curl, CURLOPT_SSL_VERIFYHOST, 0)); + check_error(curl_easy_setopt(m_select_curl, CURLOPT_SSL_VERIFYPEER, 0)); + check_error(curl_easy_setopt(m_select_curl, CURLOPT_SSL_VERIFYHOST, 0)); } - check_error(curl_easy_setopt(m_sync_curl, CURLOPT_FORBID_REUSE, 1L)); check_error(curl_easy_setopt(m_sync_curl, CURLOPT_CONNECTTIMEOUT_MS, m_timeout_ms)); check_error(curl_easy_setopt(m_sync_curl, CURLOPT_TIMEOUT_MS, m_timeout_ms)); check_error(curl_easy_setopt(m_select_curl, CURLOPT_CONNECTTIMEOUT_MS, m_timeout_ms)); - discover_mesos_leader(); } @@ -76,6 +87,12 @@ void mesos_http::cleanup(CURL** curl) m_connected = false; } +void mesos_http::set_token(const string& token) +{ + m_token = token; + m_request = make_request(m_url, m_curl_version); +} + Json::Value mesos_http::get_state_frameworks() { Json::Value frameworks; @@ -116,6 +133,16 @@ void mesos_http::discover_mesos_leader() CURLcode res = get_data(m_url.to_string(), os); if(res == CURLE_OK) { + long http_response_code = 0; + check_error(curl_easy_getinfo(m_sync_curl, CURLINFO_RESPONSE_CODE, &http_response_code)); + if(sinsp_curl::is_redirect(http_response_code)) + { + uri newurl(m_redirect); + m_url.set_host(newurl.get_host()); + g_logger.log("mesos_http: Detected Mesos master leader HTTP redirect: [" + m_url.to_string(false) + ']', sinsp_logger::SEV_INFO); + discover_mesos_leader(); + return; + } Json::Value root; Json::Reader reader; if(reader.parse(os.str(), root)) @@ -145,7 +172,7 @@ void mesos_http::discover_mesos_leader() std::string::size_type pos = leader_address.find('@'); if(pos != std::string::npos && (pos + 1) < leader_address.size()) { - std::string address = "http://"; + std::string address = m_url.get_scheme() + "://"; if(!m_mesos.m_mesos_credentials.first.empty()) { address.append(m_mesos.m_mesos_credentials.first).append(1, ':').append(m_mesos.m_mesos_credentials.second).append(1, '@'); @@ -352,6 +379,10 @@ std::string mesos_http::make_request(uri url, curl_version_info_data* curl_versi base64::encoder().encode(is, os); request << "Authorization: Basic " << os.str() << "\r\n"; } + if(!m_token.empty()) + { + request << "Authorization: token=" << m_token << "\r\n"; + } request << "\r\n"; return request.str(); @@ -509,12 +540,14 @@ void mesos_http::send_request() throw sinsp_exception("mesos_http: Mesos send invalid socket."); } - size_t iolen = send(m_watch_socket, m_request.c_str(), m_request.size(), 0); + //size_t iolen = send(m_watch_socket, m_request.c_str(), m_request.size(), 0); + size_t iolen; + check_error(curl_easy_send(m_select_curl, m_request.c_str(), m_request.size(), &iolen)); if((iolen <= 0) || (m_request.size() != iolen)) { throw sinsp_exception("mesos_http: Mesos send socket connection error."); } - else if(!wait(1)) + else if(!wait(0)) { throw sinsp_exception("mesos_http: Mesos send timeout."); } @@ -640,41 +673,20 @@ bool mesos_http::on_data() } size_t iolen = 0; - std::vector buf; + char buf[1024]; std::string data; - + CURLcode ret; try { - int loop_counter = 0; do { - ssize_t iolen = 0; - int count = 0; - int ioret = 0; - ioret = ioctl(m_watch_socket, FIONREAD, &count); - if(ioret >= 0 && count > 0) - { - if(count > static_cast(buf.size())) - { - buf.resize(count); - } - iolen = recv(m_watch_socket, &buf[0], count, 0); - if(iolen > 0) - { - ssize_t buf_size = static_cast(buf.size()); - data.append(&buf[0], iolen <= buf_size ? iolen : buf_size); - } - else if(iolen == 0) { goto connection_closed; } - else if(iolen < 0) { goto connection_error; } - } - else + check_error(ret = curl_easy_recv(m_select_curl, buf, sizeof(buf), &iolen)); + if(iolen > 0) { - if(ioret < 0) { goto connection_error; } - else if(loop_counter == 0 && count == 0) { goto connection_closed; } - break; + data.append(buf, iolen); } - ++loop_counter; - } while(iolen && errno != CURLE_AGAIN); + else if(ret != CURLE_AGAIN) { goto connection_closed; } + } while(iolen && ret != CURLE_AGAIN); if(data.size()) { extract_data(data); @@ -687,13 +699,6 @@ bool mesos_http::on_data() } return true; -connection_error: -{ - std::string err = strerror(errno); - g_logger.log("mesos_http: Mesos or Marathon API connection [" + m_url.to_string() + "] error : " + err, sinsp_logger::SEV_ERROR); - return false; -} - connection_closed: g_logger.log("mesos_http: Mesos or Marathon API connection [" + m_url.to_string() + "] closed.", sinsp_logger::SEV_ERROR); m_connected = false; diff --git a/userspace/libsinsp/mesos_http.h b/userspace/libsinsp/mesos_http.h index 73c3c32fb3..635f0e1c11 100644 --- a/userspace/libsinsp/mesos_http.h +++ b/userspace/libsinsp/mesos_http.h @@ -13,6 +13,7 @@ #include #include #include +#include "sinsp_curl.h" class mesos; @@ -24,7 +25,11 @@ class mesos_http typedef void (mesos::*callback_func_t)(json_ptr_t, const std::string&); typedef std::vector marathon_uri_t; - mesos_http(mesos& m, const uri& url, bool discover_mesos_lead_master = false, bool discover_marathon = false, int timeout_ms = 5000L); + mesos_http(mesos& m, const uri& url, + bool discover_mesos_lead_master = false, + bool discover_marathon = false, + int timeout_ms = 5000L, + const string& token = ""); virtual ~mesos_http(); @@ -55,6 +60,7 @@ class mesos_http void set_framework_version(const std::string& id); const marathon_uri_t& get_marathon_uris() const; + void set_token(const string& token); protected: CURL* get_sync_curl(); @@ -67,7 +73,7 @@ class mesos_http int wait(int for_recv); callback_func_t get_parse_func(); - static std::string make_request(uri url, curl_version_info_data* m_curl_version = 0); + std::string make_request(uri url, curl_version_info_data* m_curl_version = 0); static json_ptr_t try_parse(const std::string& json); static bool is_framework_active(const Json::Value& framework); std::string get_framework_url(const Json::Value& framework); @@ -101,6 +107,8 @@ class mesos_http //bool m_redirect = false; std::string::size_type m_content_length = std::string::npos; char m_redirect[CURL_MAX_HTTP_HEADER] = {0}; + string m_token; + sinsp_curl_http_headers m_sync_curl_headers; friend class mesos; @@ -199,4 +207,27 @@ inline const mesos_http::marathon_uri_t& mesos_http::get_marathon_uris() const return m_marathon_uris; } +#else // !HAS_CAPTURE + +#include "json/json.h" + +class mesos_http +{ +public: + typedef std::shared_ptr json_ptr_t; + static json_ptr_t try_parse(const std::string& json) + { + json_ptr_t root(new Json::Value()); + try + { + if(Json::Reader().parse(json, *root)) + { + return root; + } + } + catch(...) { } + return nullptr; + } +}; + #endif // HAS_CAPTURE diff --git a/userspace/libsinsp/parsers.cpp b/userspace/libsinsp/parsers.cpp index 53e93812c9..bf28735cd5 100644 --- a/userspace/libsinsp/parsers.cpp +++ b/userspace/libsinsp/parsers.cpp @@ -197,7 +197,7 @@ void sinsp_parser::process_event(sinsp_evt *evt) if(etype == PPME_SYSCALL_WRITE_X) { // - // Check if this is a user event + // Check if this is a tracer // sinsp_fdinfo_t* fdinfo = evt->m_fdinfo; @@ -947,6 +947,58 @@ void sinsp_parser::parse_clone_exit(sinsp_evt *evt) { // // clone() returns 0 in the child. + // + + int64_t parenttid; + + // + // Check if this is a process or a new thread + // + if(flags & PPM_CL_CLONE_THREAD) + { + // + // This is a thread, the parent tid is the pid + // + parinfo = evt->get_param(4); + ASSERT(parinfo->m_len == sizeof(int64_t)); + parenttid = *(int64_t *)parinfo->m_val; + } + else + { + // + // This is not a thread, the parent tid is ptid + // + parinfo = evt->get_param(5); + ASSERT(parinfo->m_len == sizeof(int64_t)); + parenttid = *(int64_t *)parinfo->m_val; + } + + // + // If the threadinfo in the event exists, and we're in + // a container, the threadinfo in the event must be + // stale (e.g. from a prior process with the same + // tid), because only the child side of a clone + // creates the threadinfo for the child. Clear and + // remove the old threadinfo. + // + if(evt->m_tinfo && in_container) + { + // See if the parent thread is in a + // container. If it is, the parent thread + // did *not* create the thread for this child, + // and any existing thread state must be + // stale. + + sinsp_threadinfo* ptinfo = m_inspector->get_thread(parenttid, false, true); + + + if(ptinfo && ptinfo->m_tid != ptinfo->m_vtid) + { + m_inspector->remove_thread(tid, true); + evt->m_tinfo = NULL; + } + } + // Validate that the child thread info has actually been created. // if(!evt->m_tinfo) @@ -969,27 +1021,7 @@ void sinsp_parser::parse_clone_exit(sinsp_evt *evt) // childtid = tid; - // - // Check if this is a process or a new thread - // - if(flags & PPM_CL_CLONE_THREAD) - { - // - // This is a thread, the parent tid is the pid - // - parinfo = evt->get_param(4); - ASSERT(parinfo->m_len == sizeof(int64_t)); - tid = *(int64_t *)parinfo->m_val; - } - else - { - // - // This is not a thread, the parent tid is ptid - // - parinfo = evt->get_param(5); - ASSERT(parinfo->m_len == sizeof(int64_t)); - tid = *(int64_t *)parinfo->m_val; - } + tid = parenttid; // // Keep going and add the event with the standard code below @@ -1189,15 +1221,15 @@ void sinsp_parser::parse_clone_exit(sinsp_evt *evt) tinfo.m_pid = childtid; } - // - // Copy the fd list - // XXX this is a gross oversimplification that will need to be fixed. - // What we do is: if the child is NOT a thread, we copy all the parent fds. - // The right thing to do is looking at PPM_CL_CLONE_FILES, but there are - // syscalls like open and pipe2 that can override PPM_CL_CLONE_FILES with the O_CLOEXEC flag - // if(!(tinfo.m_flags & PPM_CL_CLONE_THREAD)) { + // + // Copy the fd list + // XXX this is a gross oversimplification that will need to be fixed. + // What we do is: if the child is NOT a thread, we copy all the parent fds. + // The right thing to do is looking at PPM_CL_CLONE_FILES, but there are + // syscalls like open and pipe2 that can override PPM_CL_CLONE_FILES with the O_CLOEXEC flag + // tinfo.m_fdtable = *(ptinfo->get_fd_table()); // @@ -1205,6 +1237,11 @@ void sinsp_parser::parse_clone_exit(sinsp_evt *evt) // referring to an element in the parent's table. // tinfo.m_fdtable.reset_cache(); + + // + // Not a thread, copy cwd + // + tinfo.m_cwd = ptinfo->m_cwd; } //if((tinfo.m_flags & (PPM_CL_CLONE_FILES))) //{ @@ -1245,10 +1282,6 @@ void sinsp_parser::parse_clone_exit(sinsp_evt *evt) parinfo = evt->get_param(2); tinfo.set_args(parinfo->m_val, parinfo->m_len); - // Copy the working directory - parinfo = evt->get_param(6); - tinfo.set_cwd(parinfo->m_val, parinfo->m_len); - // Copy the fdlimit parinfo = evt->get_param(7); ASSERT(parinfo->m_len == sizeof(int64_t)); @@ -1485,10 +1518,6 @@ void sinsp_parser::parse_execve_exit(sinsp_evt *evt) ASSERT(parinfo->m_len == sizeof(uint64_t)); evt->m_tinfo->m_pid = *(uint64_t *)parinfo->m_val; - // Get the working directory - parinfo = evt->get_param(6); - evt->m_tinfo->set_cwd(parinfo->m_val, parinfo->m_len); - // Get the fdlimit parinfo = evt->get_param(7); ASSERT(parinfo->m_len == sizeof(int64_t)); diff --git a/userspace/libsinsp/prefix_search.cpp b/userspace/libsinsp/prefix_search.cpp index 27149b84b2..2eb5f17983 100644 --- a/userspace/libsinsp/prefix_search.cpp +++ b/userspace/libsinsp/prefix_search.cpp @@ -60,7 +60,22 @@ void path_prefix_search::split_path(const filter_value_t &path, filter_value_t & start++; } - void *pos = memmem(path.first+start, path.second, "/", 1); + uint8_t* pos = path.first + start; + uint32_t counter = 0; + while(counter < path.second) + { + if (*pos == 0x2F) // '/' + { + break; + } + ++pos; + if(++counter >= path.second) + { + pos = NULL; + break; + } + } + if(pos == NULL || pos >= (path.first + length)) { dirent.first = path.first + start; diff --git a/userspace/libsinsp/sinsp.cpp b/userspace/libsinsp/sinsp.cpp index 699e437bf8..c36737a92b 100644 --- a/userspace/libsinsp/sinsp.cpp +++ b/userspace/libsinsp/sinsp.cpp @@ -109,6 +109,7 @@ sinsp::sinsp() : m_last_procrequest_tod = 0; m_get_procs_cpu_from_driver = false; m_is_tracers_capture_enabled = false; + m_file_start_offset = 0; // Unless the cmd line arg "-pc" or "-pcontainer" is supplied this is false m_print_container_data = false; @@ -509,6 +510,14 @@ void sinsp::open(string filename) oargs.proc_callback = NULL; oargs.proc_callback_context = NULL; oargs.import_users = m_import_users; + if(m_file_start_offset != 0) + { + oargs.start_offset = m_file_start_offset; + } + else + { + oargs.start_offset = 0; + } m_h = scap_open(oargs, error); @@ -876,6 +885,14 @@ int32_t sinsp::next(OUT sinsp_evt **puevt) } #endif } + else if(res == SCAP_UNEXPECTED_BLOCK) + { + uint64_t filepos = scap_ftell(m_h) - scap_get_unexpected_block_readsize(m_h); + m_file_start_offset = filepos; + close(); + open(m_input_filename); + return SCAP_TIMEOUT; + } else { m_lasterr = scap_getlasterr(m_h); @@ -1729,9 +1746,14 @@ void sinsp::make_k8s_client() #ifdef HAS_CAPTURE ,m_k8s_ssl ,m_k8s_bt + ,true // blocking #endif // HAS_CAPTURE ,nullptr +#ifdef HAS_CAPTURE ,m_ext_list_ptr +#else + ,nullptr +#endif // HAS_CAPTURE ); } @@ -1742,7 +1764,10 @@ void sinsp::init_k8s_client(string* api_server, string* ssl_cert, bool verbose) m_k8s_api_server = api_server; m_k8s_api_cert = ssl_cert; + +#ifdef HAS_CAPTURE if(m_k8s_api_detected && m_k8s_ext_detect_done) +#endif // HAS_CAPTURE { if(m_k8s_client) { @@ -1756,41 +1781,42 @@ void sinsp::init_k8s_client(string* api_server, string* ssl_cert, bool verbose) void sinsp::collect_k8s() { - if(m_lastevent_ts > m_k8s_last_watch_time_ns + ONE_SECOND_IN_NS) + if(m_parser) { - m_k8s_last_watch_time_ns = m_lastevent_ts; - - if(m_parser) + if(m_k8s_api_server) + { + if(!m_k8s_client) { - if(m_k8s_api_server) + init_k8s_client(m_k8s_api_server, m_k8s_api_cert, m_verbose_json); + if(m_k8s_client) { - if(m_k8s_client) - { - uint64_t delta = sinsp_utils::get_current_time_ns(); - m_k8s_client->watch(); - m_parser->schedule_k8s_events(&m_meta_evt); - delta = sinsp_utils::get_current_time_ns() - delta; - g_logger.format(sinsp_logger::SEV_DEBUG, "Updating Kubernetes state took %" PRIu64 " ms", delta / 1000000LL); - } - else - { - init_k8s_client(m_k8s_api_server, m_k8s_api_cert, m_verbose_json); - if(m_k8s_client) - { - g_logger.log("K8s client created.", sinsp_logger::SEV_DEBUG); - } - else - { - g_logger.log("K8s client NOT created.", sinsp_logger::SEV_DEBUG); - } - } + g_logger.log("K8s client created.", sinsp_logger::SEV_DEBUG); + } + else + { + g_logger.log("K8s client NOT created.", sinsp_logger::SEV_DEBUG); } } + if(m_k8s_client) + { + if(m_lastevent_ts > m_k8s_last_watch_time_ns + ONE_SECOND_IN_NS) + { + m_k8s_last_watch_time_ns = m_lastevent_ts; + g_logger.log("K8s updating state ...", sinsp_logger::SEV_DEBUG); + uint64_t delta = sinsp_utils::get_current_time_ns(); + m_k8s_client->watch(); + m_parser->schedule_k8s_events(&m_meta_evt); + delta = sinsp_utils::get_current_time_ns() - delta; + g_logger.format(sinsp_logger::SEV_DEBUG, "Updating Kubernetes state took %" PRIu64 " ms", delta / 1000000LL); + } + } + } } } void sinsp::k8s_discover_ext() { +#ifdef HAS_CAPTURE try { if(m_k8s_api_server && !m_k8s_api_server->empty() && !m_k8s_ext_detect_done) @@ -1805,7 +1831,7 @@ void sinsp::k8s_discover_ext() if(uri(*m_k8s_api_server).is_secure()) { init_k8s_ssl(m_k8s_api_server, m_k8s_api_cert); } m_k8s_ext_handler.reset(new k8s_api_handler(m_k8s_collector, *m_k8s_api_server, "/apis/extensions/v1beta1", "[.resources[].name]", - "1.0", m_k8s_ssl, m_k8s_bt)); + "1.1", m_k8s_ssl, m_k8s_bt, true)); g_logger.log("K8s API extensions handler: collector created.", sinsp_logger::SEV_TRACE); } else @@ -1855,10 +1881,12 @@ void sinsp::k8s_discover_ext() m_k8s_ext_handler.reset(); } g_logger.log("K8s API extensions handler: detection done.", sinsp_logger::SEV_TRACE); +#endif // HAS_CAPTURE } void sinsp::update_k8s_state() { +#ifdef HAS_CAPTURE try { if(m_k8s_api_server && !m_k8s_api_server->empty()) @@ -1876,8 +1904,8 @@ void sinsp::update_k8s_state() init_k8s_ssl(m_k8s_api_server, m_k8s_api_cert); } m_k8s_api_handler.reset(new k8s_api_handler(m_k8s_collector, *m_k8s_api_server, - "/api", ".versions", "1.0", - m_k8s_ssl, m_k8s_bt)); + "/api", ".versions", "1.1", + m_k8s_ssl, m_k8s_bt, true)); } else { @@ -1922,6 +1950,7 @@ void sinsp::update_k8s_state() g_logger.log(std::string("Error fetching K8s data: ").append(e.what()), sinsp_logger::SEV_ERROR); throw; } +#endif // HAS_CAPTURE } bool sinsp::get_mesos_data() diff --git a/userspace/libsinsp/sinsp.h b/userspace/libsinsp/sinsp.h index 0afc7f2fe1..cc60b91050 100644 --- a/userspace/libsinsp/sinsp.h +++ b/userspace/libsinsp/sinsp.h @@ -111,12 +111,15 @@ class sinsp_protodecoder; class k8s; class sinsp_partial_tracer; class mesos; + +#ifdef HAS_CAPTURE class sinsp_ssl; class sinsp_bearer_token; template class socket_data_handler; template class socket_collector; class k8s_handler; class k8s_api_handler; +#endif // HAS_CAPTURE vector sinsp_split(const string &s, char delim); @@ -831,6 +834,9 @@ VISIBILITY_PRIVATE uint32_t m_num_cpus; sinsp_thread_privatestate_manager m_thread_privatestate_manager; bool m_is_tracers_capture_enabled; + // This is used to support reading merged files, where the capture needs to + // restart in the middle of the file. + uint64_t m_file_start_offset; sinsp_network_interfaces* m_network_interfaces; @@ -843,16 +849,18 @@ VISIBILITY_PRIVATE // string* m_k8s_api_server; string* m_k8s_api_cert; +#ifdef HAS_CAPTURE std::shared_ptr m_k8s_ssl; std::shared_ptr m_k8s_bt; - k8s* m_k8s_client; - uint64_t m_k8s_last_watch_time_ns; unique_ptr m_k8s_api_handler; shared_ptr>> m_k8s_collector; bool m_k8s_api_detected = false; unique_ptr m_k8s_ext_handler; k8s_ext_list_ptr_t m_ext_list_ptr; bool m_k8s_ext_detect_done = false; +#endif // HAS_CAPTURE + k8s* m_k8s_client; + uint64_t m_k8s_last_watch_time_ns; // // Mesos/Marathon @@ -1002,6 +1010,9 @@ VISIBILITY_PRIVATE friend class sinsp_network_interfaces; friend class k8s_delegator; +#ifdef HAS_ANALYZER + friend class thread_analyzer_info; +#endif template friend class sinsp_connection_manager; }; diff --git a/userspace/libsinsp/sinsp_curl.cpp b/userspace/libsinsp/sinsp_curl.cpp index 6447211c45..e47128914d 100644 --- a/userspace/libsinsp/sinsp_curl.cpp +++ b/userspace/libsinsp/sinsp_curl.cpp @@ -15,6 +15,25 @@ #include #include +sinsp_curl_http_headers::sinsp_curl_http_headers(): + m_curl_header_list(NULL) +{ + +} + +sinsp_curl_http_headers::~sinsp_curl_http_headers() +{ + if(m_curl_header_list) + { + curl_slist_free_all(m_curl_header_list); + } +} + +void sinsp_curl_http_headers::add(const string& header) +{ + m_curl_header_list = curl_slist_append(m_curl_header_list, header.c_str()); +} + sinsp_curl::data sinsp_curl::m_config; sinsp_curl::sinsp_curl(const uri& url, long timeout_ms, bool debug): @@ -73,6 +92,7 @@ void sinsp_curl::init() } enable_debug(m_curl, m_debug); + m_response_code = -1; } sinsp_curl::~sinsp_curl() @@ -171,6 +191,7 @@ size_t sinsp_curl::header_callback(char *buffer, size_t size, size_t nitems, voi { size_t sz = nitems * size; std::string buf(buffer, sz); + const std::string loc = "Location:"; const std::string nl = "\r\n"; std::string::size_type loc_pos = buf.find(loc); @@ -201,8 +222,8 @@ size_t sinsp_curl::header_callback(char *buffer, size_t size, size_t nitems, voi if(sz < CURL_MAX_HTTP_HEADER) { g_logger.log("HTTP redirect Location: (" + buf + ')', sinsp_logger::SEV_TRACE); - std::strncpy((char*)userdata, buf.data(), sz); - ((char*)userdata)[sz] = 0; + std::strncpy((char*) userdata, buf.data(), sz); + ((char*) userdata)[sz] = 0; } } return nitems * size; @@ -248,6 +269,13 @@ bool sinsp_curl::handle_redirect(uri& url, std::string&& loc, std::ostream& os) return false; } +size_t read_data(void* buffer, size_t size, size_t nmemb, void* instream) +{ + auto body = (stringstream*) instream; + body->read((char*) buffer, size*nmemb); + return body->gcount(); +} + bool sinsp_curl::get_data(std::ostream& os) { CURLcode res = CURLE_OK; @@ -260,7 +288,12 @@ bool sinsp_curl::get_data(std::ostream& os) check_error(curl_easy_setopt(m_curl, CURLOPT_ACCEPT_ENCODING, "deflate")); check_error(curl_easy_setopt(m_curl, CURLOPT_WRITEFUNCTION, &sinsp_curl::write_data)); check_error(curl_easy_setopt(m_curl, CURLOPT_WRITEDATA, &os)); - + check_error(curl_easy_setopt(m_curl, CURLOPT_READFUNCTION, &read_data)); + check_error(curl_easy_setopt(m_curl, CURLOPT_READDATA, &m_body)); + if(m_headers.ptr() != NULL) + { + setopt(CURLOPT_HTTPHEADER, m_headers.ptr()); + } res = curl_easy_perform(m_curl); if(res != CURLE_OK) { @@ -270,16 +303,15 @@ bool sinsp_curl::get_data(std::ostream& os) { // HTTP errors are not returned by curl API // error will be in the response stream - long http_code = 0; - check_error(curl_easy_getinfo(m_curl, CURLINFO_RESPONSE_CODE, &http_code)); - if(http_code >= 400) + check_error(curl_easy_getinfo(m_curl, CURLINFO_RESPONSE_CODE, &m_response_code)); + if(m_response_code >= 400) { - g_logger.log("CURL HTTP error: " + std::to_string(http_code), sinsp_logger::SEV_ERROR); + g_logger.log("CURL HTTP error: " + std::to_string(m_response_code), sinsp_logger::SEV_ERROR); return false; } - else if(is_redirect(http_code)) + else if(is_redirect(m_response_code)) { - g_logger.log("HTTP redirect (" + std::to_string(http_code) + ')', sinsp_logger::SEV_DEBUG); + g_logger.log("HTTP redirect (" + std::to_string(m_response_code) + ')', sinsp_logger::SEV_DEBUG); if(handle_redirect(m_uri, std::string(m_redirect), os)) { std::ostringstream* pos = dynamic_cast(&os); @@ -417,5 +449,12 @@ int sinsp_curl::trace(CURL *handle, curl_infotype type, char *data, size_t size, return 0; } +void sinsp_curl::set_body(const string& data) +{ + m_body.clear(); + m_body << data; + add_header(string("Content-Length: ") + to_string(data.size())); +} + #endif // __linux__ diff --git a/userspace/libsinsp/sinsp_curl.h b/userspace/libsinsp/sinsp_curl.h index 7670548027..1fcce0d9e5 100644 --- a/userspace/libsinsp/sinsp_curl.h +++ b/userspace/libsinsp/sinsp_curl.h @@ -16,6 +16,23 @@ #include #include +class sinsp_curl_http_headers +{ +public: + sinsp_curl_http_headers(); + ~sinsp_curl_http_headers(); + + void add(const string& header); + + struct curl_slist* ptr() + { + return m_curl_header_list; + } + +private: + struct curl_slist* m_curl_header_list; +}; + class sinsp_curl { public: @@ -45,9 +62,28 @@ class sinsp_curl void set_url(const std::string& url); std::string get_url(bool show_creds = true) const; - + void set_body(const string& data); + bool is_secure() const; ssl::ptr_t get_ssl(); + + template + void setopt(Opt opt, Arg arg) + { + check_error(curl_easy_setopt(m_curl, opt, arg)); + } + + void enable_debug() + { + sinsp_curl::enable_debug(m_curl); + } + + template + void add_header(T body) + { + m_headers.add(forward(body)); + } + static void init_ssl(CURL* curl, ssl::ptr_t ssl_data); bearer_token::ptr_t get_bt(); static void init_bt(CURL* curl, bearer_token::ptr_t bt); @@ -59,6 +95,16 @@ class sinsp_curl static bool handle_redirect(uri& url, std::string&& loc, std::ostream& os); static size_t write_data(void *ptr, size_t size, size_t nmemb, void *cb); + const vector& response_headers() + { + return m_response_headers; + } + + const long get_response_code() const + { + return m_response_code; + } + private: struct data { @@ -77,6 +123,10 @@ class sinsp_curl bearer_token::ptr_t m_bt; bool m_debug; char m_redirect[CURL_MAX_HTTP_HEADER] = {0}; + stringstream m_body; + sinsp_curl_http_headers m_headers; + vector m_response_headers; + long m_response_code; }; inline void sinsp_curl::set_timeout(long milliseconds) diff --git a/userspace/libsinsp/socket_collector.h b/userspace/libsinsp/socket_collector.h index dd8e706c12..9c48aeeb09 100644 --- a/userspace/libsinsp/socket_collector.h +++ b/userspace/libsinsp/socket_collector.h @@ -257,7 +257,7 @@ class socket_collector else { g_logger.log("Socket collector: connection close detected while handling data" - ", removing handler [" + id + ']', sinsp_logger::SEV_ERROR); + ", removing handler [" + id + ']', sinsp_logger::SEV_DEBUG); } remove(it); continue; diff --git a/userspace/libsinsp/socket_handler.h b/userspace/libsinsp/socket_handler.h index d4b0d53ab9..c63d0c6e24 100644 --- a/userspace/libsinsp/socket_handler.h +++ b/userspace/libsinsp/socket_handler.h @@ -1,11 +1,12 @@ // -// socket_collector.h +// socket_handler.h // #pragma once #ifdef HAS_CAPTURE +#include "http_parser.h" #include "uri.h" #include "json/json.h" #define BUFFERSIZE 512 // b64 needs this macro @@ -15,6 +16,7 @@ #include "sinsp_auth.h" #include "json_query.h" #include +#include #include #ifndef _GNU_SOURCE #define _GNU_SOURCE @@ -53,22 +55,25 @@ class socket_data_handler int timeout_ms = 1000L, ssl_ptr_t ssl = 0, bt_ptr_t bt = 0, - bool keep_alive = true): m_obj(obj), + bool keep_alive = true, + bool blocking = false, + unsigned data_limit = 524288): m_obj(obj), m_id(id), m_url(url), m_keep_alive(keep_alive ? std::string("Connection: keep-alive\r\n") : std::string()), m_path(path.empty() ? m_url.get_path() : path), + m_blocking(blocking), m_ssl(ssl), m_bt(bt), m_timeout_ms(timeout_ms), - m_request(make_request(m_url, http_version)), + m_request(make_request(url, http_version)), m_http_version(http_version), - m_json_begin("\r\n{"), - m_json_end(m_http_version == HTTP_VERSION_10 ? "}\r\n" : "}\r\n0") + m_data_limit(data_limit) { g_logger.log(std::string("Creating Socket handler object for (" + id + ") " "[" + uri(url).to_string(false) + ']'), sinsp_logger::SEV_DEBUG); m_buf.resize(1024); + init_http_parser(); } virtual ~socket_data_handler() @@ -109,11 +114,27 @@ class socket_data_handler return m_connecting; } + void close_on_chunked_end(bool close = true) + { + m_close_on_chunked_end = close; + } + const uri& get_url() const { return m_url; } + void set_path(const std::string& path) + { + m_path = path; + m_request = make_request(m_url, m_http_version); + } + + void set_check_chunked(bool check = true) + { + m_check_chunked = check; + } + std::string make_request(uri url, const std::string& http_version) { std::ostringstream request; @@ -156,6 +177,11 @@ class socket_data_handler return request.str(); } + void set_id(const std::string& id) + { + m_id = id; + } + const std::string& get_id() const { return m_id; @@ -171,9 +197,14 @@ class socket_data_handler return m_ssl_connection; } + bool wants_send() const + { + return m_wants_send; + } + void send_request() { - g_logger.log("Socket handler (" + m_id + ") send:\n" + m_request, sinsp_logger::SEV_TRACE); + m_wants_send = false; // no matter what happens, this is a one-shot if(m_request.empty()) { throw sinsp_exception("Socket handler (" + m_id + ") send: request (empty)."); @@ -278,6 +309,179 @@ class socket_data_handler } } + void set_socket_option(int opt) + { + int flags = fcntl(m_socket, F_GETFL, 0); + if(flags != -1) + { + fcntl(m_socket, F_SETFL, flags | opt); + } + else + { + throw sinsp_exception("Socket handler (" + m_id + + ") error while setting socket option (" + + std::to_string(opt) + "): " + strerror(errno)); + } + } + + int get_all_data() + { + g_logger.log("Socket handler (" + m_id + ") Retrieving all data in blocking mode ...", + sinsp_logger::SEV_TRACE); + ssize_t rec = 0; + std::vector buf(1024, 0); + std::string data; + int counter = 0; + int processed = 0; + m_msg_completed = false; + do + { + int count = 0; + int ioret = ioctl(m_socket, FIONREAD, &count); + if(ioret >= 0 && count > 0) + { + buf.resize(count); + if(m_url.is_secure()) + { + rec = SSL_read(m_ssl_connection, &buf[0], buf.size()); + } + else + { + rec = recv(m_socket, &buf[0], buf.size(), 0); + } + if(rec > 0) + { + data.append(buf.begin(), buf.begin() + rec); + } + else if(rec == 0) + { + throw sinsp_exception("Socket handler (" + m_id + "): Connection closed."); + } + else if(rec < 0) + { + throw sinsp_exception("Socket handler (" + m_id + "): " + strerror(errno)); + } + //g_logger.log("Socket handler (" + m_id + ") received=" + std::to_string(rec) + + // "\n\n" + data + "\n\n", sinsp_logger::SEV_TRACE); + } + if(data.size()) + { + process(data); + processed += data.size(); + data.clear(); + } + if(++counter > 10000) + { + throw sinsp_exception("Socket handler (" + m_id + "): " + "unable to retrieve data from " + m_url.to_string(false) + m_path + + " (" + std::to_string(counter) + " attempts)"); + } + else { usleep(10000); } + } while(!m_msg_completed); + + return processed; + } + + bool is_chunked_end_char(char c) + { + return c == '0' || c == '\r' || c == '\n'; + } + + void check_chunked_end(const std::string& data) + { + if(!m_check_chunked) { return; } + if(m_chunked_end.size()) + { + for(auto c : data) + { + if(!is_chunked_end_char(c)) + { + m_chunked_end.clear(); + break; + } + else + { + m_chunked_end.append(1, c); + } + } + } + + if(!m_chunked_end.size()) + { + auto it = data.crbegin(); + for(; it != data.crend(); ++it) + { + if(!is_chunked_end_char(*it)) { return; } + else { m_chunked_end.insert(0, 1, *it); } + } + } + } + + void data_handling_error(const std::string& data, size_t nparsed) + { + std::ostringstream os; + os << "Socket handler (" << m_id + ") an error occurred during http parsing. " + "processed=" << nparsed << ", expected=" << data.size() << ", status_code=" << + std::to_string(m_http_parser->status_code) << ", http_errno=" << + std::to_string(m_http_parser->http_errno) << "data:" << std::endl << data; + throw sinsp_exception(os.str()); + } + + void parse_http(const std::string& data) + { + size_t nparsed = http_parser_execute(m_http_parser, &m_http_parser_settings, data.c_str(), data.length()); + if(nparsed != data.size()) { data_handling_error(data, nparsed); } + } + + void process_json() + { + if(m_json_filters.empty()) { add_json_filter("."); } + bool handled = false; + for(auto js = m_json.begin(); js != m_json.end();) + { + handled = false; + for(auto it = m_json_filters.cbegin(); it != m_json_filters.cend(); ++it) + { + json_ptr_t pjson = try_parse(m_jq, *js, *it, m_id, m_url.to_string(false)); + if(pjson) + { + (m_obj.*m_json_callback)(pjson, m_id); + handled = true; + break; + } + } + if(!handled) + { + g_logger.log("Socket handler: (" + m_id + ") JSON not handled, " + "discarding:\n" + *js, sinsp_logger::SEV_ERROR); + } + js = m_json.erase(js); + } + } + + int process(const std::string& data) + { + if(data.size()) + { + check_chunked_end(data); + parse_http(data); + if(m_chunked_end.find("0\r\n\r\n") != std::string::npos) + { + m_data_buf.clear(); + // In HTTP 1.1 connnections with chunked transfer, this socket may not be closed by server, + // (K8s API server is an example of such behavior), in which case the chunked data will just + // stop flowing. We can keep the good socket and resend the request instead of severing the + // connection. The m_wants_send flag has to be checked by the caller and request re-sent, otherwise + // this pipeline will remain idle. To force client-initiated socket close on chunked transfer end, + // set the m_close_on_chunked_end flag to true (default). + if(m_close_on_chunked_end) { return CONNECTION_CLOSED; } + else { m_wants_send = true; } + } + else { process_json(); } + } + return 0; + } + int on_data() { bool is_error = false; @@ -287,87 +491,95 @@ class socket_data_handler throw sinsp_exception("Socket handler (" + m_id + "): cannot parse data (callback is null)."); } - size_t iolen = 0; + ssize_t iolen = 0; + size_t len_to_read = m_buf.size(); std::string data; - try { do { - size_t iolen = 0; + if(data.size() >= m_data_limit) { break; } + else if((data.size() + m_buf.size()) > m_data_limit) + { + len_to_read = m_data_limit - data.size(); + } + errno = 0; + if(m_url.is_secure()) + { + iolen = static_cast(SSL_read(m_ssl_connection, &m_buf[0], len_to_read)); + } + else + { + iolen = recv(m_socket, &m_buf[0], len_to_read, 0); + } + m_sock_err = errno; + g_logger.log(m_id + ' ' + m_url.to_string(false) + ", iolen=" + + std::to_string(iolen) + ", data=" + std::to_string(data.size()) + " bytes, " + "errno=" + std::to_string(m_sock_err) + " (" + strerror(m_sock_err) + ')', + sinsp_logger::SEV_TRACE); + if(iolen > 0) + { + data.append(&m_buf[0], iolen <= static_cast(m_buf.size()) ? + static_cast(iolen) : m_buf.size()); + } + else if(iolen == 0 || m_sock_err == ENOTCONN || m_sock_err == EPIPE) { - errno = 0; if(m_url.is_secure()) { - iolen = SSL_read(m_ssl_connection, &m_buf[0], m_buf.size()); - } - else - { - iolen = recv(m_socket, &m_buf[0], m_buf.size(), 0); - } - m_sock_err = errno; - g_logger.log(m_id + ' ' + m_url.to_string(false) + //" loop_counter=" + std::to_string(loop_counter) + - ", iolen=" + std::to_string(iolen), sinsp_logger::SEV_TRACE); - if(iolen > 0) - { - data.append(&m_buf[0], iolen <= m_buf.size() ? iolen : m_buf.size()); - } - else if(iolen == 0 || errno == ENOTCONN || errno == EPIPE) - { - if(m_url.is_secure()) + if(m_ssl_connection) { - if(m_ssl_connection) + int sd = SSL_get_shutdown(m_ssl_connection); + if(sd == 0) { - int sd = SSL_get_shutdown(m_ssl_connection); - if(sd == 0) - { - g_logger.log("Socket handler (" + m_id + "): SSL zero bytes received, " - "but no shutdown state set for [" + m_url.to_string(false) + "]: ", - sinsp_logger::SEV_WARNING); - } - if(sd & SSL_RECEIVED_SHUTDOWN) - { - g_logger.log("Socket handler(" + m_id + "): SSL shutdown from [" + - m_url.to_string(false) + "]: ", sinsp_logger::SEV_TRACE); - } - if(sd & SSL_SENT_SHUTDOWN) - { - g_logger.log("Socket handler(" + m_id + "): SSL shutdown sent to [" + - m_url.to_string(false) + "]: ", sinsp_logger::SEV_TRACE); - } + g_logger.log("Socket handler (" + m_id + "): SSL zero bytes received, " + "but no shutdown state set for [" + m_url.to_string(false) + "]: ", + sinsp_logger::SEV_WARNING); } - else + if(sd & SSL_RECEIVED_SHUTDOWN) { - g_logger.log("Socket handler(" + m_id + "): SSL connection is null", - sinsp_logger::SEV_WARNING); + g_logger.log("Socket handler(" + m_id + "): SSL shutdown from [" + + m_url.to_string(false) + "]: ", sinsp_logger::SEV_TRACE); } + if(sd & SSL_SENT_SHUTDOWN) + { + g_logger.log("Socket handler(" + m_id + "): SSL shutdown sent to [" + + m_url.to_string(false) + "]: ", sinsp_logger::SEV_TRACE); + } + } + else + { + g_logger.log("Socket handler(" + m_id + "): SSL connection is null", + sinsp_logger::SEV_WARNING); } + } + goto connection_closed; + } + else if(iolen < 0) + { + if(m_sock_err == ENOTCONN || m_sock_err == EPIPE) + { goto connection_closed; } - else if(iolen < 0) + else if(m_sock_err != EAGAIN && m_sock_err != EWOULDBLOCK) { - if(errno == ENOTCONN || errno == EPIPE) - { - goto connection_closed; - } - else if(errno != EAGAIN && errno != EWOULDBLOCK) + goto connection_error; + } + if(m_url.is_secure()) + { + int err = SSL_get_error(m_ssl_connection, iolen); + if(err != SSL_ERROR_WANT_READ && err != SSL_ERROR_WANT_WRITE) { goto connection_error; } - if(m_url.is_secure()) - { - int err = SSL_get_error(m_ssl_connection, iolen); - if(err != SSL_ERROR_WANT_READ && err != SSL_ERROR_WANT_WRITE) - { - goto connection_error; - } - } } } - } while(iolen && errno != EAGAIN); - if(data.size()) + } while(iolen && (m_sock_err != EAGAIN) && (data.size() < m_data_limit)); + g_logger.log("Socket handler (" + m_id + ") " + + std::to_string(data.size()) + " bytes of data received", + sinsp_logger::SEV_TRACE); + if(CONNECTION_CLOSED == process(data)) { - extract_data(data); + return CONNECTION_CLOSED; } } catch(sinsp_exception& ex) @@ -403,29 +615,50 @@ class socket_data_handler { } - void set_json_begin(const std::string& b) - { - m_json_begin = b; - } - - const std::string& get_json_begin() const + bool has_json_filter(const std::string& filter) { - return m_json_begin; - } - - void set_json_end(const std::string& e) - { - m_json_end = e; - } - - const std::string& get_json_end() const - { - return m_json_end; + for(auto flt : m_json_filters) + { + if(flt == filter) + { + return true; + } + } + return false; } - void add_json_filter(const std::string& filter) + void add_json_filter(const std::string& filter, const std::string& before_filter = "") { - m_json_filters.push_back(filter); + if(filter.empty()) + { + throw sinsp_exception(std::string("Socket handler (") + m_id + "), " + "[" + m_url.to_string(false) + "] " + "attempt to add empty filter"); + } + remove_json_filter(filter); + if(before_filter.empty()) + { + m_json_filters.push_back(filter); + return; + } + else + { + auto it = m_json_filters.begin(); + for(; it != m_json_filters.end(); ++it) + { + if(*it == before_filter) { break; } + } + if(it != m_json_filters.end()) + { + m_json_filters.insert(it, filter); + } + else + { + throw sinsp_exception(std::string("Socket handler (") + m_id + "), " + "[" + m_url.to_string(false) + "] " + "attempt to insert before non-existing filter"); + } + } } void remove_json_filter(const std::string& filter) @@ -442,7 +675,7 @@ class socket_data_handler void replace_json_filter(const std::string& from, const std::string& to) { - for(auto it = m_json_filters.cbegin(); it != m_json_filters.cend(); ++it) + for(auto it = m_json_filters.begin(); it != m_json_filters.end(); ++it) { if(*it == from) { @@ -455,6 +688,17 @@ class socket_data_handler "attempt to replace non-existing filter"); } + void print_filters(sinsp_logger::severity sev = sinsp_logger::SEV_DEBUG) + { + std::ostringstream filters; + filters << std::endl << "Filters:" << std::endl; + for(auto filter : m_json_filters) + { + filters << filter << std::endl; + } + g_logger.log("Socket handler (" + m_id + "), [" + m_url.to_string(false) + "]" + filters.str(), sev); + } + static json_ptr_t try_parse(json_query& jq, const std::string& json, const std::string& filter, const std::string& id, const std::string& url) { @@ -494,7 +738,7 @@ class socket_data_handler return nullptr; } - // connection is non-blocking and a socket + // when connection is non-blocking and a socket // should not be polled until it is connected // this flag indicates readiness to be polled bool is_enabled() const @@ -530,163 +774,6 @@ class socket_data_handler typedef std::vector password_vec_t; - bool purge_chunked_markers(std::string& data) - { - std::string::size_type pos = data.find("}\r\n0"); - if(pos != std::string::npos) - { - data = data.substr(0, pos); - } - - const std::string nl = "\r\n"; - std::string::size_type begin, end; - while((begin = data.find(nl)) != std::string::npos) - { - end = data.find(nl, begin + 2); - if(end != std::string::npos) - { - data.erase(begin, end + 2 - begin); - } - else // newlines must come in pairs - { - return false; - } - } - return true; - } - - void handle_json(std::string::size_type end_pos, bool chunked) - { - if(end_pos != std::string::npos) - { - if(m_data_buf.length() >= end_pos + 1) - { - std::string json = m_data_buf.substr(0, end_pos + 1); - if(m_data_buf.length() > end_pos + 1) - { - m_data_buf = m_data_buf.substr(end_pos + 2); - } - else - { - m_data_buf.clear(); - m_content_length = std::string::npos; - } - if(json.size()) - { - if(chunked && !purge_chunked_markers(m_data_buf)) - { - g_logger.log("Socket handler (" + m_id + "): Invalid JSON data detected " - "(chunked transfer).", sinsp_logger::SEV_ERROR); - (m_obj.*m_json_callback)(nullptr, m_id); - } - else - { - g_logger.log("Socket handler (" + m_id + "): invoking callback(s).", - sinsp_logger::SEV_TRACE); - if(m_json_filters.empty()) - { - // if no filters provided and we got here, just try to do the whole JSON as-is - add_json_filter("."); - } - for(auto it = m_json_filters.cbegin(); it != m_json_filters.cend(); ++it) - { - json_ptr_t pjson = try_parse(m_jq, json, *it, m_id, m_url.to_string(false)); - if(pjson) - { - (m_obj.*m_json_callback)(pjson, m_id); - return; - } - } - } - g_logger.log("Socket handler (" + m_id + ") " + m_url.to_string(false) + ": " - "An error occurred while handling JSON.", - sinsp_logger::SEV_ERROR); - g_logger.log(json, sinsp_logger::SEV_TRACE); - } - } - } - } - - bool detect_chunked_transfer(const std::string& data) - { - if(m_content_length == std::string::npos) - { - std::string::size_type cl_pos = data.find("Content-Length:"); - if(cl_pos != std::string::npos) - { - std::string::size_type nl_pos = data.find("\r\n", cl_pos); - if(nl_pos != std::string::npos) - { - cl_pos += std::string("Content-Length:").length(); - std::string cl = data.substr(cl_pos, nl_pos - cl_pos); - long len = strtol(cl.c_str(), NULL, 10); - if(len == 0L || len == LONG_MAX || len == LONG_MIN || errno == ERANGE) - { - (m_obj.*m_json_callback)(nullptr, m_id); - m_data_buf.clear(); - g_logger.log("Socket handler (" + m_id + "): Invalid HTTP content length from " - "[: " + m_url.to_string(false) + ']' + - std::to_string(len), sinsp_logger::SEV_ERROR); - return false; - } - else - { - m_content_length = static_cast(len); - } - } - } - } - return true; - } - - void extract_data(std::string& data) - { - if(data.empty()) - { - g_logger.log(m_id + ' ' + m_url.to_string(false) + m_path + ": no data received, giving up extraction ...", - sinsp_logger::SEV_TRACE); - return; - } - g_logger.log(m_id + ' ' + m_url.to_string(false) + m_path + ":\n\n" + data + "\n\n", sinsp_logger::SEV_TRACE); - if(!detect_chunked_transfer(data)) - { - g_logger.log("Socket handler (" + m_id + ") " + m_url.to_string(false) + ": " - "An error occurred while detecting chunked transfer.", - sinsp_logger::SEV_ERROR); - return; - } - - if(m_data_buf.empty()) { m_data_buf = data; } - else { m_data_buf.append(data); } - std::string::size_type pos = m_data_buf.find(m_json_begin); - if(pos != std::string::npos) // JSON begin - { - m_data_buf = m_data_buf.substr(pos + 2); - } - else if(m_data_buf[0] == '{') // docker HTTP stream does this - { - pos = 0; - } - bool chunked = (m_content_length == std::string::npos); - if(chunked) - { - std::string::size_type end = std::string::npos; - while(true) - { - end = m_data_buf.find(m_json_end); - if(end == std::string::npos) { break; } - g_logger.log("Socket handler (" + m_id + ") " + m_url.to_string(false) + ": " - "found JSON end, handling JSON", sinsp_logger::SEV_TRACE); - handle_json(end, true); - } - } - else if (m_data_buf.length() >= m_content_length) - { - handle_json(m_data_buf.length() - 1, false); - } - return; - } - int wait(bool for_recv, long tout = 1000L) { struct timeval tv; @@ -959,15 +1046,20 @@ class socket_data_handler void create_socket() { + int sock_type = SOCK_STREAM; + if(!m_blocking) + { + sock_type |= SOCK_NONBLOCK; + } if(m_socket < 0) { if(m_url.is_file()) { - m_socket = socket(PF_UNIX, SOCK_STREAM | SOCK_NONBLOCK, 0); + m_socket = socket(PF_UNIX, sock_type, 0); } else { - m_socket = socket(PF_INET, SOCK_STREAM | SOCK_NONBLOCK, 0); + m_socket = socket(PF_INET, sock_type, 0); } if(m_socket < 0) { @@ -1416,11 +1508,79 @@ class socket_data_handler void cleanup() { + free(m_http_parser); close_socket(); dns_cleanup(); ssl_cleanup(); } + struct http_parser_data + { + std::string* m_data_buf = nullptr; + std::vector* m_json = nullptr; + bool* m_msg_completed = nullptr; + }; + + static int http_body_callback(http_parser* parser, const char* data, size_t len) + { + if(data && len) + { + if(parser && parser->data) + { + http_parser_data* parser_data = (http_parser_data*) parser->data; + if(parser_data->m_data_buf && parser_data->m_json) + { + parser_data->m_data_buf->append(data, len); + std::string::size_type pos = parser_data->m_data_buf->find('\n'); + while(pos != std::string::npos) + { + parser_data->m_json->push_back(parser_data->m_data_buf->substr(0, pos)); + parser_data->m_data_buf->erase(0, pos + 1); + pos = parser_data->m_data_buf->find('\n'); + } + } + else { throw sinsp_exception("Socket handler: http or json buffer null."); } + } + else { throw sinsp_exception("Socket handler: parser or data null."); } + } + return 0; + } + + static int http_msg_completed_callback(http_parser* parser) + { + if(parser && parser->data) + { + http_parser_data* parser_data = (http_parser_data*) parser->data; + if(parser_data->m_msg_completed) + { + *(parser_data->m_msg_completed) = true; + } + else { throw sinsp_exception("Socket handler: parser msg complete null."); } + } + else { throw sinsp_exception("Socket handler: parser or data null."); } + return 0; + } + + void init_http_parser() + { + http_parser_settings_init(&m_http_parser_settings); + m_http_parser_settings.on_body = http_body_callback; + m_http_parser_settings.on_message_complete = http_msg_completed_callback; + m_http_parser = (http_parser *)std::malloc(sizeof(http_parser)); + if(m_http_parser) + { + m_http_parser_data.m_data_buf = &m_data_buf; + m_http_parser_data.m_json = &m_json; + m_http_parser_data.m_msg_completed = &m_msg_completed; + http_parser_init(m_http_parser, HTTP_RESPONSE); + m_http_parser->data = &m_http_parser_data; + } + else + { + throw sinsp_exception("Socket handler: cannot create http parser."); + } + } + typedef std::deque dns_list_t; T& m_obj; @@ -1435,6 +1595,7 @@ class socket_data_handler bool m_connection_error = false; bool m_enabled = false; int m_socket = -1; + bool m_blocking = false; std::vector m_buf; int m_sock_err = 0; struct gaicb** m_dns_reqs = nullptr; @@ -1446,9 +1607,8 @@ class socket_data_handler std::string m_data_buf; std::string m_request; std::string m_http_version; - std::string m_json_begin; - std::string m_json_end; std::vector m_json_filters; + std::vector m_json; json_query m_jq; bool m_ssl_init_complete = false; SSL_CTX* m_ssl_context = nullptr; @@ -1459,6 +1619,15 @@ class socket_data_handler struct sockaddr* m_sa = 0; socklen_t m_sa_len = 0; std::string::size_type m_content_length = std::string::npos; + std::string m_chunked_end; + bool m_check_chunked = false; + bool m_close_on_chunked_end = true; + bool m_wants_send = false; + bool m_msg_completed = false; + http_parser_settings m_http_parser_settings; + http_parser* m_http_parser = nullptr; + http_parser_data m_http_parser_data; + unsigned m_data_limit = 524288; // bytes }; template diff --git a/userspace/libsinsp/stopwatch.cpp b/userspace/libsinsp/stopwatch.cpp new file mode 100644 index 0000000000..5862dbccfa --- /dev/null +++ b/userspace/libsinsp/stopwatch.cpp @@ -0,0 +1,12 @@ +// +// stopwatch.cpp +// +// stopwatch utility +// + +#include "stopwatch.h" + +sinsp_stopwatch::sinsp_stopwatch() +{ + start(); +} diff --git a/userspace/libsinsp/stopwatch.h b/userspace/libsinsp/stopwatch.h new file mode 100644 index 0000000000..5a8bafeec5 --- /dev/null +++ b/userspace/libsinsp/stopwatch.h @@ -0,0 +1,52 @@ +// +// stopwatch.h +// +// stopwatch utility +// + +#pragma once + +#include + +class sinsp_stopwatch +{ +public: + sinsp_stopwatch(); + + void stop(); + void start(); + void reset(); + + template + typename T::rep elapsed() const + { + return std::chrono::duration_cast(m_stop - m_start).count(); + } + +private: + void record(std::chrono::high_resolution_clock::time_point& tp); + + std::chrono::high_resolution_clock::time_point m_start; + std::chrono::high_resolution_clock::time_point m_stop; +}; + +inline void sinsp_stopwatch::sinsp_stopwatch::reset() +{ + m_start = std::chrono::high_resolution_clock::time_point::min(); + m_start = std::chrono::high_resolution_clock::time_point::max(); +} + +inline void sinsp_stopwatch::sinsp_stopwatch::start() +{ + record(m_start); +} + +inline void sinsp_stopwatch::sinsp_stopwatch::stop() +{ + record(m_stop); +} + +inline void sinsp_stopwatch::record(std::chrono::high_resolution_clock::time_point& tp) +{ + tp = std::chrono::high_resolution_clock::now(); +} diff --git a/userspace/libsinsp/table.cpp b/userspace/libsinsp/table.cpp index 44c6329006..bce7db8ce0 100644 --- a/userspace/libsinsp/table.cpp +++ b/userspace/libsinsp/table.cpp @@ -177,7 +177,7 @@ void sinsp_table::configure(vector* entries, const strin chk->m_aggregation = (sinsp_field_aggregation)vit.m_aggregation; m_chks_to_free.push_back(chk); - chk->parse_field_name(vit.get_field(m_view_depth).c_str(), true); + chk->parse_field_name(vit.get_field(m_view_depth).c_str(), true, false); if((vit.m_flags & TEF_IS_KEY) != 0) { @@ -219,7 +219,7 @@ void sinsp_table::configure(vector* entries, const strin chk->m_aggregation = A_NONE; m_chks_to_free.push_back(chk); - chk->parse_field_name("util.cnt", true); + chk->parse_field_name("util.cnt", true, false); if(m_is_key_present) { diff --git a/userspace/libsinsp/threadinfo.cpp b/userspace/libsinsp/threadinfo.cpp index 8d02da35c4..8ca4326286 100644 --- a/userspace/libsinsp/threadinfo.cpp +++ b/userspace/libsinsp/threadinfo.cpp @@ -535,6 +535,13 @@ void sinsp_threadinfo::set_cgroups(const char* cgroups, size_t len) { subsys = "memory"; } + else if(subsys == "io") + { + // blkio has been renamed just `io` + // in kernel space: + // https://github.com/torvalds/linux/commit/c165b3e3c7bb68c2ed55a5ac2623f030d01d9567 + subsys = "blkio"; + } m_cgroups.push_back(std::make_pair(subsys, cgroup)); offset += subsys_length + 1 + cgroup.length() + 1; @@ -630,7 +637,12 @@ sinsp_threadinfo* sinsp_threadinfo::get_cwd_root() string sinsp_threadinfo::get_cwd() { - sinsp_threadinfo* tinfo = get_cwd_root(); + // Ideally we should use get_cwd_root() + // but scap does not read CLONE_FS from /proc + // Also glibc and muslc use always + // CLONE_THREAD|CLONE_FS so let's use + // get_main_thread() for now + sinsp_threadinfo* tinfo = get_main_thread(); if(tinfo) { @@ -646,7 +658,7 @@ string sinsp_threadinfo::get_cwd() void sinsp_threadinfo::set_cwd(const char* cwd, uint32_t cwdlen) { char tpath[SCAP_MAX_PATH_SIZE]; - sinsp_threadinfo* tinfo = get_cwd_root(); + sinsp_threadinfo* tinfo = get_main_thread(); if(tinfo) { @@ -1066,3 +1078,58 @@ void sinsp_thread_manager::update_statistics() } #endif } + +void sinsp_thread_manager::to_scap() +{ +/* + scap_proc_free_table(m_inspector->m_h); + + for(auto it = m_threadtable.begin(); it != m_threadtable.end(); ++it) + { + sinsp_threadinfo& tinfo = it->second; + + // + // Allocate the scap thread info + // + scap_threadinfo* sctinfo = (scap_threadinfo*)malloc(sizeof(scap_threadinfo)); + if(sctinfo == NULL) + { + throw sinsp_exception("memory allocation error in sinsp_thread_manager::to_scap"); + } + + // + // Fill in the data + // +sctinfo->tid = tinfo.m_tid; +sctinfo->pid = tinfo.m_pid; +sctinfo->ptid = tinfo.m_ptid; +sctinfo->sid = tinfo.m_sid; + +strncpy(sctinfo->comm, tinfo.m_comm.c_str(), SCAP_MAX_PATH_SIZE); +strncpy(sctinfo->exe, tinfo.m_exe.c_str(), SCAP_MAX_PATH_SIZE); +tinfo.set_args(sctinfo->args, sctinfo->args_len); +tinfo.set_env(sctinfo->env, sctinfo->env_len); +tinfo.set_cwd(sctinfo->cwd, (uint32_t)strlen(sctinfo->cwd)); +sctinfo->flags = tinfo.m_flags ; +sctinfo->fdlimit = tinfo.m_fdlimit; +sctinfo->uid = tinfo.m_uid; +sctinfo->gid = tinfo.m_gid; +sctinfo->vmsize_kb = tinfo.m_vmsize_kb; +sctinfo->vmrss_kb = tinfo.m_vmrss_kb; +sctinfo->vmswap_kb = tinfo.m_vmswap_kb; +sctinfo->pfmajor = tinfo.m_pfmajor; +sctinfo->pfminor = tinfo.m_pfminor; +sctinfo->vtid = tinfo.m_vtid; +sctinfo->vpid = tinfo.m_vpid; + +tinfo.set_cgroups(sctinfo->cgroups, sctinfo->cgroups_len); +sctinfo->root = tinfo.m_root; + + + // + // Add the created info to scap + // + scap_proc_add(m_inspector->m_h, it->second.m_tid, sctinfo); + } +*/ +} diff --git a/userspace/libsinsp/threadinfo.h b/userspace/libsinsp/threadinfo.h index 7027f69274..309102e19e 100644 --- a/userspace/libsinsp/threadinfo.h +++ b/userspace/libsinsp/threadinfo.h @@ -387,6 +387,8 @@ class SINSP_PUBLIC sinsp_thread_manager void create_child_dependencies(); void recreate_child_dependencies(); + void to_scap(); + uint32_t get_thread_count() { return (uint32_t)m_threadtable.size(); diff --git a/userspace/libsinsp/uri.h b/userspace/libsinsp/uri.h index a5034cadd1..24db329944 100644 --- a/userspace/libsinsp/uri.h +++ b/userspace/libsinsp/uri.h @@ -35,6 +35,9 @@ class uri const std::string& get_query() const; int get_port() const; + void set_scheme(std::string scheme); + void set_host(std::string host); + bool is(const std::string& proto); bool is_file() const; bool is_secure() const; @@ -70,6 +73,11 @@ inline const std::string& uri::get_scheme() const return m_scheme; } +inline void uri::set_scheme(std::string scheme) +{ + m_scheme = move(scheme); +} + inline const std::string& uri::get_user() const { return m_user; @@ -85,6 +93,11 @@ inline const std::string& uri::get_host() const return m_host; } +inline void uri::set_host(std::string host) +{ + m_host = move(host); +} + inline const std::string& uri::get_path() const { return m_path; diff --git a/userspace/libsinsp/uri_parser.c b/userspace/libsinsp/uri_parser.c index 363bc7c0ea..a634c5132f 100644 --- a/userspace/libsinsp/uri_parser.c +++ b/userspace/libsinsp/uri_parser.c @@ -154,7 +154,7 @@ do { \ * * This is designed to be shared by http_parser_execute() for URL validation, * hence it has a state transition + byte-for-byte interface. In addition, it - * is meant to be embedded in http_parser_parse_url(), which does the dirty + * is meant to be embedded in http_parser_parse_uri(), which does the dirty * work of turning state transitions URL components for its API. * * This function should only be invoked with non-space characters. It is @@ -392,18 +392,18 @@ http_parse_host_char(enum http_host_state s, const char ch) { } static int -http_parse_host(const char * buf, struct http_parser_url *u, int found_at) { - assert(u->field_set & (1 << UF_HOST)); +http_parse_host(const char * buf, struct http_parser_uri *u, int found_at) { + assert(u->field_set & (1 << URI_FLD_HOST)); enum http_host_state s; const char *p; - size_t buflen = u->field_data[UF_HOST].off + u->field_data[UF_HOST].len; + size_t buflen = u->field_data[URI_FLD_HOST].off + u->field_data[URI_FLD_HOST].len; - u->field_data[UF_HOST].len = 0; + u->field_data[URI_FLD_HOST].len = 0; s = found_at ? s_http_userinfo_start : s_http_host_start; - for (p = buf + u->field_data[UF_HOST].off; p < buf + buflen; p++) { + for (p = buf + u->field_data[URI_FLD_HOST].off; p < buf + buflen; p++) { enum http_host_state new_s = http_parse_host_char(s, *p); if (new_s == s_http_host_dead) { @@ -413,39 +413,39 @@ http_parse_host(const char * buf, struct http_parser_url *u, int found_at) { switch(new_s) { case s_http_host: if (s != s_http_host) { - u->field_data[UF_HOST].off = p - buf; + u->field_data[URI_FLD_HOST].off = p - buf; } - u->field_data[UF_HOST].len++; + u->field_data[URI_FLD_HOST].len++; break; case s_http_host_v6: if (s != s_http_host_v6) { - u->field_data[UF_HOST].off = p - buf; + u->field_data[URI_FLD_HOST].off = p - buf; } - u->field_data[UF_HOST].len++; + u->field_data[URI_FLD_HOST].len++; break; case s_http_host_v6_zone_start: case s_http_host_v6_zone: - u->field_data[UF_HOST].len++; + u->field_data[URI_FLD_HOST].len++; break; case s_http_host_port: if (s != s_http_host_port) { - u->field_data[UF_PORT].off = p - buf; - u->field_data[UF_PORT].len = 0; - u->field_set |= (1 << UF_PORT); + u->field_data[URI_FLD_PORT].off = p - buf; + u->field_data[URI_FLD_PORT].len = 0; + u->field_set |= (1 << URI_FLD_PORT); } - u->field_data[UF_PORT].len++; + u->field_data[URI_FLD_PORT].len++; break; case s_http_userinfo: if (s != s_http_userinfo) { - u->field_data[UF_USERINFO].off = p - buf ; - u->field_data[UF_USERINFO].len = 0; - u->field_set |= (1 << UF_USERINFO); + u->field_data[URI_FLD_USERINFO].off = p - buf ; + u->field_data[URI_FLD_USERINFO].len = 0; + u->field_set |= (1 << URI_FLD_USERINFO); } - u->field_data[UF_USERINFO].len++; + u->field_data[URI_FLD_USERINFO].len++; break; default: @@ -473,22 +473,22 @@ http_parse_host(const char * buf, struct http_parser_url *u, int found_at) { } void -http_parser_url_init(struct http_parser_url *u) { +http_parser_uri_init(struct http_parser_uri *u) { memset(u, 0, sizeof(*u)); } int -http_parser_parse_url(const char *buf, size_t buflen, int is_connect, - struct http_parser_url *u) +http_parser_parse_uri(const char *buf, size_t buflen, int is_connect, + struct http_parser_uri *u) { enum state s; const char *p; - enum http_parser_url_fields uf, old_uf; + enum http_parser_uri_fields uf, old_uf; int found_at = 0; u->port = u->field_set = 0; s = is_connect ? s_req_server_start : s_req_spaces_before_url; - old_uf = UF_MAX; + old_uf = URI_FLD_MAX; for (p = buf; p < buf + buflen; p++) { s = parse_url_char(s, *p); @@ -507,7 +507,7 @@ http_parser_parse_url(const char *buf, size_t buflen, int is_connect, continue; case s_req_schema: - uf = UF_SCHEMA; + uf = URI_FLD_SCHEMA; break; case s_req_server_with_at: @@ -515,19 +515,19 @@ http_parser_parse_url(const char *buf, size_t buflen, int is_connect, /* FALLTROUGH */ case s_req_server: - uf = UF_HOST; + uf = URI_FLD_HOST; break; case s_req_path: - uf = UF_PATH; + uf = URI_FLD_PATH; break; case s_req_query_string: - uf = UF_QUERY; + uf = URI_FLD_QUERY; break; case s_req_fragment: - uf = UF_FRAGMENT; + uf = URI_FLD_FRAGMENT; break; default: @@ -550,25 +550,25 @@ http_parser_parse_url(const char *buf, size_t buflen, int is_connect, /* host must be present if there is a schema */ /* parsing http:///toto will fail */ - if ((u->field_set & (1 << UF_SCHEMA)) && - (u->field_set & (1 << UF_HOST)) == 0) { + if ((u->field_set & (1 << URI_FLD_SCHEMA)) && + (u->field_set & (1 << URI_FLD_HOST)) == 0) { return 1; } - if (u->field_set & (1 << UF_HOST)) { + if (u->field_set & (1 << URI_FLD_HOST)) { if (http_parse_host(buf, u, found_at) != 0) { return 1; } } /* CONNECT requests can only contain "hostname:port" */ - if (is_connect && u->field_set != ((1 << UF_HOST)|(1 << UF_PORT))) { + if (is_connect && u->field_set != ((1 << URI_FLD_HOST)|(1 << URI_FLD_PORT))) { return 1; } - if (u->field_set & (1 << UF_PORT)) { + if (u->field_set & (1 << URI_FLD_PORT)) { /* Don't bother with endp; we've already validated the string */ - unsigned long v = strtoul(buf + u->field_data[UF_PORT].off, NULL, 10); + unsigned long v = strtoul(buf + u->field_data[URI_FLD_PORT].off, NULL, 10); /* Ports have a max value of 2^16 */ if (v > 0xffff) { @@ -582,10 +582,10 @@ http_parser_parse_url(const char *buf, size_t buflen, int is_connect, } struct parsed_uri parse_uri(const char *uri_string) { - struct http_parser_url u; - http_parser_url_init(&u); + struct http_parser_uri u; + http_parser_uri_init(&u); - int rc = http_parser_parse_url(uri_string, strlen(uri_string), 0, &u); + int rc = http_parser_parse_uri(uri_string, strlen(uri_string), 0, &u); if (rc) { struct parsed_uri uri = {1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,}; diff --git a/userspace/libsinsp/uri_parser.h b/userspace/libsinsp/uri_parser.h index b2e4621369..502c1fc6d8 100644 --- a/userspace/libsinsp/uri_parser.h +++ b/userspace/libsinsp/uri_parser.h @@ -18,8 +18,8 @@ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS * IN THE SOFTWARE. */ -#ifndef http_parser_h -#define http_parser_h +#ifndef uri_parser_h +#define uri_parser_h #ifdef __cplusplus extern "C" { #endif @@ -51,43 +51,43 @@ typedef unsigned __int64 uint64_t; /* Get an http_errno value from an http_parser */ #define HTTP_PARSER_ERRNO(p) ((enum http_errno) (p)->http_errno) -enum http_parser_url_fields - { UF_SCHEMA = 0 - , UF_HOST = 1 - , UF_PORT = 2 - , UF_PATH = 3 - , UF_QUERY = 4 - , UF_FRAGMENT = 5 - , UF_USERINFO = 6 - , UF_MAX = 7 +enum http_parser_uri_fields + { URI_FLD_SCHEMA = 0 + , URI_FLD_HOST = 1 + , URI_FLD_PORT = 2 + , URI_FLD_PATH = 3 + , URI_FLD_QUERY = 4 + , URI_FLD_FRAGMENT = 5 + , URI_FLD_USERINFO = 6 + , URI_FLD_MAX = 7 }; -/* Result structure for http_parser_parse_url(). +/* Result structure for http_parser_parse_uri(). * * Callers should index into field_data[] with UF_* values iff field_set * has the relevant (1 << UF_*) bit set. As a courtesy to clients (and * because we probably have padding left over), we convert any port to * a uint16_t. */ -struct http_parser_url { +struct http_parser_uri { uint16_t field_set; /* Bitmask of (1 << UF_*) values */ - uint16_t port; /* Converted UF_PORT string */ + uint16_t port; /* Converted URI_FLD_PORT string */ struct { uint16_t off; /* Offset into buffer in which field starts */ uint16_t len; /* Length of run in buffer */ - } field_data[UF_MAX]; + } field_data[URI_FLD_MAX]; }; -/* Initialize all http_parser_url members to 0 */ -void http_parser_url_init(struct http_parser_url *u); +/* Initialize all http_parser_uri members to 0 */ +void http_parser_uri_init(struct http_parser_uri *u); /* Parse a URL; return nonzero on failure */ -int http_parser_parse_url(const char *buf, size_t buflen, +int http_parser_parse_uri(const char *buf, size_t buflen, int is_connect, - struct http_parser_url *u); + struct http_parser_uri *u); struct parsed_uri { const uint8_t error; diff --git a/userspace/libsinsp/utils.cpp b/userspace/libsinsp/utils.cpp index 639f5cdb77..08179b7554 100644 --- a/userspace/libsinsp/utils.cpp +++ b/userspace/libsinsp/utils.cpp @@ -46,7 +46,6 @@ along with sysdig. If not, see . #include "filterchecks.h" #include "chisel.h" #include "protodecoder.h" -#include "json/json.h" #include "uri.h" #ifndef _WIN32 #include "curl/curl.h" @@ -698,7 +697,11 @@ bool sinsp_utils::is_ipv4_mapped_ipv6(uint8_t* paddr) { if(paddr[0] == 0 && paddr[1] == 0 && paddr[2] == 0 && paddr[3] == 0 && paddr[4] == 0 && paddr[5] == 0 && paddr[6] == 0 && paddr[7] == 0 && paddr[8] == 0 && paddr[9] == 0 && - paddr[10] == 0xff && paddr[11] == 0xff) + ( + ( paddr[10] == 0xff && paddr[11] == 0xff) || // A real IPv4 address + (paddr[10] == 0 && paddr[11] == 0 && paddr[12] == 0 && paddr[13] == 0 && paddr[14] == 0 && paddr[15] == 0) // all zero address, assume IPv4 as well + ) + ) { return true; } diff --git a/userspace/libsinsp/utils.h b/userspace/libsinsp/utils.h index fc6552d542..73584d3f6b 100644 --- a/userspace/libsinsp/utils.h +++ b/userspace/libsinsp/utils.h @@ -18,10 +18,17 @@ along with sysdig. If not, see . #pragma once +#include +#include +#include +#include #include #include #include +#include +#include "json/json.h" + class sinsp_evttables; typedef union _sinsp_sockinfo sinsp_sockinfo; typedef union _ipv4tuple ipv4tuple; @@ -80,12 +87,12 @@ class sinsp_utils // // Given a string, scan the event list and find the longest argument that the input string contains // - static const struct ppm_param_info* find_longest_matching_evt_param(string name); + static const struct ppm_param_info* find_longest_matching_evt_param(std::string name); // // Get the list of filtercheck fields // - static void get_filtercheck_fields_info(vector* list); + static void get_filtercheck_fields_info(std::vector* list); static uint64_t get_current_time_ns(); @@ -116,7 +123,7 @@ struct g_invalidchar } }; -inline void sanitize_string(string &str) +inline void sanitize_string(std::string &str) { // It turns out with -O3 (release flags) using erase and // remove_if is slighly faster than the inline version that @@ -149,7 +156,7 @@ SINSP_PUBLIC int gettimeofday(struct timeval *tv, struct timezone2 *tz); /////////////////////////////////////////////////////////////////////////////// // gethostname wrapper /////////////////////////////////////////////////////////////////////////////// -string sinsp_gethostname(); +std::string sinsp_gethostname(); /////////////////////////////////////////////////////////////////////////////// // tuples to string @@ -157,25 +164,25 @@ string sinsp_gethostname(); // each of these functions uses values in network byte order -string ipv4tuple_to_string(ipv4tuple* tuple, bool resolve); -string ipv6tuple_to_string(_ipv6tuple* tuple, bool resolve); -string ipv4serveraddr_to_string(ipv4serverinfo* addr, bool resolve); -string ipv6serveraddr_to_string(ipv6serverinfo* addr, bool resolve); +std::string ipv4tuple_to_string(ipv4tuple* tuple, bool resolve); +std::string ipv6tuple_to_string(_ipv6tuple* tuple, bool resolve); +std::string ipv4serveraddr_to_string(ipv4serverinfo* addr, bool resolve); +std::string ipv6serveraddr_to_string(ipv6serverinfo* addr, bool resolve); // `l4proto` should be of type scap_l4_proto, but since it's an enum sometimes // is used as int and we would have to cast // `port` must be saved with network byte order // `l4proto` could be neither TCP nor UDP, in this case any protocol will be // matched -string port_to_string(uint16_t port, uint8_t l4proto, bool resolve); +std::string port_to_string(uint16_t port, uint8_t l4proto, bool resolve); /////////////////////////////////////////////////////////////////////////////// // String helpers /////////////////////////////////////////////////////////////////////////////// -vector sinsp_split(const string& s, char delim); +std::vector sinsp_split(const std::string& s, char delim); template -string sinsp_join(It begin, It end, char delim) +std::string sinsp_join(It begin, It end, char delim) { if(begin == end) { @@ -191,11 +198,11 @@ string sinsp_join(It begin, It end, char delim) return ss.str(); } -string& ltrim(string& s); -string& rtrim(string& s); -string& trim(string& s); -string& replace_in_place(string& s, const string& search, const string& replacement); -string replace(const string& str, const string& search, const string& replacement); +std::string& ltrim(std::string& s); +std::string& rtrim(std::string& s); +std::string& trim(std::string& s); +std::string& replace_in_place(std::string& s, const std::string& search, const std::string& replacement); +std::string replace(const std::string& str, const std::string& search, const std::string& replacement); /////////////////////////////////////////////////////////////////////////////// // number parser @@ -203,19 +210,19 @@ string replace(const string& str, const string& search, const string& replacemen class sinsp_numparser { public: - static uint8_t parseu8(const string& str); - static int8_t parsed8(const string& str); - static uint16_t parseu16(const string& str); - static int16_t parsed16(const string& str); - static uint32_t parseu32(const string& str); - static int32_t parsed32(const string& str); - static uint64_t parseu64(const string& str); - static int64_t parsed64(const string& str); - - static bool tryparseu32(const string& str, uint32_t* res); - static bool tryparsed32(const string& str, int32_t* res); - static bool tryparseu64(const string& str, uint64_t* res); - static bool tryparsed64(const string& str, int64_t* res); + static uint8_t parseu8(const std::string& str); + static int8_t parsed8(const std::string& str); + static uint16_t parseu16(const std::string& str); + static int16_t parsed16(const std::string& str); + static uint32_t parseu32(const std::string& str); + static int32_t parsed32(const std::string& str); + static uint64_t parseu64(const std::string& str); + static int64_t parsed64(const std::string& str); + + static bool tryparseu32(const std::string& str, uint32_t* res); + static bool tryparsed32(const std::string& str, int32_t* res); + static bool tryparseu64(const std::string& str, uint64_t* res); + static bool tryparsed64(const std::string& str, int64_t* res); static bool tryparseu32_fast(const char* str, uint32_t strlen, uint32_t* res); static bool tryparsed32_fast(const char* str, uint32_t strlen, int32_t* res); @@ -285,8 +292,8 @@ class simple_lifo_queue } private: - list m_avail_list; - list m_full_list; + std::list m_avail_list; + std::list m_full_list; }; /////////////////////////////////////////////////////////////////////////////// diff --git a/userspace/sysdig/chisels/flame.lua b/userspace/sysdig/chisels/flame.lua index c96b354e6a..232e9e7c3e 100644 --- a/userspace/sysdig/chisels/flame.lua +++ b/userspace/sysdig/chisels/flame.lua @@ -1,486 +1,521 @@ ---[[ -Copyright (C) 2013-2014 Draios inc. - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License version 2 as -published by the Free Software Foundation. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see . ---]] - --- Chisel description -disabled_description = "Flame graph generator"; -short_description = "Sysdig trace flame graph builder"; -category = "Performance"; - --- Chisel argument list -args = -{ -} - -require "common" -json = require ("dkjson") - -local CAPTURE_LOGS = true - -local spans = {} -local fid -local flatency -local fcontname -local fexe -local fbuf -local fdir -local ftime -local MAX_DEPTH = 256 -local avg_tree = {} -local full_tree = {} -local max_tree = {} -local min_tree = {} -local logs_tree = {} -local next = next -- make next faster - --- Argument notification callback -function on_set_arg(name, val) - return true -end - --- Initialization callback -function on_init() - -- Request the fields needed for this chisel - for j = 0, MAX_DEPTH do - local fname = "span.tag[" .. j .. "]" - local minfo = chisel.request_field(fname) - spans[j] = minfo - end - - fid = chisel.request_field("span.id") - flatency = chisel.request_field("span.duration") - fcontname = chisel.request_field("container.name") - fexe = chisel.request_field("proc.exeline") - fbuf = chisel.request_field("evt.buffer") - fdir = chisel.request_field("evt.dir") - ftid = chisel.request_field("thread.tid") - ftime = chisel.request_field("evt.time") - - -- set the filter - if CAPTURE_LOGS then - chisel.set_filter("(evt.type=tracer) or (evt.is_io_write=true and evt.dir=< and (fd.num=1 or fd.num=2 or fd.name contains log))") - else - chisel.set_filter("evt.type=tracer and evt.dir=<") - end - - return true -end - --- Add a log entry into the proper place(s) in the log table -function collect_log(tid_tree) - for k,entry in pairs(tid_tree) do - while true do - local lastv = v - k,v = next(entry) - if v == nil then - if lastv.l == nil then - lastv.l = {} - end - - local etime = evt.field(ftime) - local buf = evt.field(fbuf) - local tid = evt.field(ftid) - local hi, low = evt.get_ts() - - local linedata = {t=etime, th=hi, tl=low, tid=tid, b=buf} - - table.insert(lastv.l, linedata) ---print("*** " .. evt.get_num() .. " " .. linedata) ---print(st(logs_tree)) ---print("***************************") - return - end - - entry = v.ch - end - end -end - --- Parse a tracer enter event and update the logs_tree table -function parse_tracer_enter(logtable_cur, hr) - for j = 1, #hr do - local mv = hr[j] - - if mv == nil then - break - end - - if logtable_cur[mv] == nil then - logtable_cur[mv] = {ch={}} - end - - if j == #hr then - logtable_cur[mv].r=true - end - - logtable_cur = logtable_cur[mv].ch - end -end - --- Parse a tracer exit event and update the given transaction entry -function parse_tracer_exit(mrk_cur, logtable_cur, hr, latency, contname, exe, id) - local res = false - local parent_has_logs = false; - - for j = 1, #hr do - local mv = hr[j] - if mv == nil or mrk_cur == nil then - break - end - - local has_logtable_entry = (logtable_cur ~= nil and logtable_cur[mv] ~= nil) - ---print("! " .. evt.get_num() .. " " .. j) ---print(parent_has_logs) ---print(logtable_cur[mv].r) - if j == #hr then - local llogs - - if has_logtable_entry and logtable_cur[mv].l ~= nil then - llogs = logtable_cur[mv].l - else - llogs = nil - end - ---print("################ " .. evt.get_num() .. " " .. st(logs_tree)) - if mrk_cur[mv] == nil then - mrk_cur[mv] = {t=latency, tt=latency, cont=contname, exe=exe, c=1, logs=llogs} - if j == 1 then - mrk_cur[mv].n = 0 - end - else - mrk_cur[mv]["tt"] = mrk_cur[mv]["tt"] + latency - mrk_cur[mv]["cont"] = contname - mrk_cur[mv]["exe"] = exe - mrk_cur[mv]["c"] = 1 - mrk_cur[mv]["logs"] = llogs - end - ---print("################ " .. evt.get_num()) ---print(st(logs_tree)) ---print("## " .. evt.get_num()) ---print(st(logtable_cur[mv].r)) - - if has_logtable_entry and parent_has_logs == false then - res = true - else - logtable_cur[mv] = nil - has_logtable_entry = false - logtable_cur = nil - end - elseif j == (#hr - 1) then - if mrk_cur[mv] == nil then - mrk_cur[mv] = {tt=0} - if j == 1 then - mrk_cur[mv].n = 0 - end - end - else - if mrk_cur[mv] == nil then - mrk_cur[mv] = {tt=0} - if j == 1 then - mrk_cur[mv].n = 0 - mrk_cur[mv]["id"] = id - end - end - end - - if mrk_cur[mv]["ch"] == nil then - mrk_cur[mv]["ch"] = {} - end - - if #hr == 1 then - mrk_cur[mv].n = mrk_cur[mv].n + 1 - end - - -- end of node parsing, update pointers to movo to the child - if has_logtable_entry then - parent_has_logs = (logtable_cur[mv].r ~= nil) - end - - mrk_cur = mrk_cur[mv].ch - - if logtable_cur ~= nil then - logtable_cur = logtable_cur[mv].ch - end - end - - return res -end - --- Event parsing callback -function on_event() - local etype = evt.get_type() - - if etype ~= "tracer" then - local tid = evt.field(ftid) - - if logs_tree[tid] == nil then - return - else - collect_log(logs_tree[tid]) - end - - return - end - - local latency = evt.field(flatency) - local contname = evt.field(fcontname) - local id = evt.field(fid) - local exe = evt.field(fexe) - local hr = {} - local full_trs = nil - local dir = evt.field(fdir) - local tid = evt.field(ftid) - - for j = 0, MAX_DEPTH do - hr[j + 1] = evt.field(spans[j]) - end - - if dir == ">" then - if logs_tree[tid] == nil then - logs_tree[tid] = {} - end - - local idt = logs_tree[tid][id] - - if idt == nil then - logs_tree[tid][id] = {} - idt = logs_tree[tid][id] - end - - parse_tracer_enter(idt, hr) - return true - else - if latency == nil then - return true - end - - if full_tree[id] == nil then - full_tree[id] = {} - end - - -- find the logs for this transaction span - local logs - - if logs_tree[tid] == nil then - logs = nil - else - if logs_tree[tid][id] == nil then - logs = nil - else - logs = logs_tree[tid][id] - end - end - - if parse_tracer_exit(full_tree[id], logs, hr, latency, contname, exe, id) then ---print(st(logs_tree)) ---print("------------ " .. evt.get_num()) ---print(st(full_tree)) ---print("---------------------------------------------------") - - logs_tree[tid][id] = nil - - if next(logs_tree[tid]) == nil then - logs_tree[tid] = nil - end - - end - - return true - end -end - -function calculate_t_in_node(node) - local totchtime = 0 - local maxchtime = 0 - local nconc = 0 - local ch_to_keep - - if node.ch then - for k,d in pairs(node.ch) do - local nv = calculate_t_in_node(d) - - totchtime = totchtime + nv - - if nv > maxchtime then - maxchtime = nv - ch_to_keep = d - end - - nconc = nconc + 1 - end - end - - if node.tt >= totchtime then - node.t = node.tt - totchtime - else - node.t = node.tt - maxchtime - node.nconc = nconc - - for k,d in pairs(node.ch) do - if d ~= ch_to_keep then - node.ch[k] = nil - end - end - - end - - return node.tt -end - -function normalize(node, factor) - node.t = node.t / factor - node.tt = node.tt / factor - if node.ch then - for k,d in pairs(node.ch) do - normalize(d, factor) - end - end -end - -function is_transaction_complete(node) - if node.c ~= 1 then - return false - end - - if node.ch then - for k,d in pairs(node.ch) do - if is_transaction_complete(d) == false then - return false - end - end - end - - return true -end - -function update_avg_tree(dsttree, key, val) - if dsttree[key] == nil then - dsttree[key] = copytable(val) - return - else - dsttree[key].tt = dsttree[key].tt + val.tt - - if dsttree[key].n then - dsttree[key].n = dsttree[key].n + 1 - end - - if val.logs then - if dsttree[key].logs == nil then - dsttree[key].logs = {} - end - - concattable(dsttree[key].logs, val.logs) - end - end - - if val.ch then - if dsttree[key].ch == nil then - dsttree[key].ch = {} - end - - for k,d in pairs(val.ch) do - update_avg_tree(dsttree[key].ch, k, d) - end - end -end - -function update_max_tree(dsttree, key, val) - if dsttree[key] == nil then - dsttree[key] = val - return - else - if val.tt > dsttree[key].tt then - dsttree[key] = val - end - end -end - -function update_min_tree(dsttree, key, val) - if dsttree[key] == nil then - dsttree[key] = val - return - else - if val.tt < dsttree[key].tt then - dsttree[key] = val - end - end -end - --- This processes the transaction list to extract and aggregate the transactions to emit -function collapse_tree() - -- scan the transaction list - for i,v in pairs(full_tree) do - local ttt = 0 - for key,val in pairs(v) do - ttt = ttt + val.tt - if is_transaction_complete(val) then - update_avg_tree(avg_tree, key, val) - update_max_tree(max_tree, key, val) - update_min_tree(min_tree, key, val) - end - end - end -end - --- Called by the engine at the end of the capture (Ctrl-C) -function on_capture_end() ---print(st(full_tree)) - -- Process the list and create the required transactions - collapse_tree() - - -- calculate the unique time spent in each node - for i,v in pairs(avg_tree) do - calculate_t_in_node(v) - end - - -- normalize each root span tree - for i,v in pairs(avg_tree) do - normalize(v, v.n) - end - - print "var FlameData = {" - - -- emit the average transaction - local AvgData = {} - AvgData[""] = {ch=avg_tree, t=0, tt=0} - local str = json.encode(AvgData, { indent = true }) - print('"AvgData": ' .. str .. ",") - - -- normalize the best transaction - for i,v in pairs(min_tree) do - calculate_t_in_node(v) - end - - -- emit the best transaction - local tdata = {} - tdata[""] = {ch=min_tree, t=0, tt=0} - local str = json.encode(tdata, { indent = true }) - print('"MinData": ' .. str .. ",") - - -- normalize the worst transaction - for i,v in pairs(max_tree) do - calculate_t_in_node(v) - end - - -- emit the worst transaction - local tdata = {} - tdata[""] = {ch=max_tree, t=0, tt=0} - local str = json.encode(tdata, { indent = true }) - print('"MaxData": ' .. str .. ",") - - print "};" -end +--[[ +Copyright (C) 2013-2014 Draios inc. + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License version 2 as +published by the Free Software Foundation. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +--]] + +-- Chisel description +disabled_description = "Flame graph generator"; +short_description = "Sysdig trace flame graph builder"; +category = "Performance"; + +-- Chisel argument list +args = +{ +} + +require "common" +json = require ("dkjson") + +local CAPTURE_LOGS = true + +local spans = {} +local fid +local flatency +local fcontname +local fexe +local fbuf +local fdir +local ftime +local MAX_DEPTH = 256 +local avg_tree = {} +local full_tree = {} +local max_tree = {} +local min_tree = {} +local logs_tree = {} +local next = next -- make next faster +local PAGE_HEADER = [[ + + + + + Flame UI + + + + + + + + + + + + + + + + + + + + + + + +]] + +-- Argument notification callback +function on_set_arg(name, val) + return true +end + +-- Initialization callback +function on_init() + -- Request the fields needed for this chisel + for j = 0, MAX_DEPTH do + local fname = "span.tag[" .. j .. "]" + local minfo = chisel.request_field(fname) + spans[j] = minfo + end + + fid = chisel.request_field("span.id") + flatency = chisel.request_field("span.duration") + fcontname = chisel.request_field("container.name") + fexe = chisel.request_field("proc.exeline") + fbuf = chisel.request_field("evt.buffer") + fdir = chisel.request_field("evt.dir") + ftid = chisel.request_field("thread.tid") + ftime = chisel.request_field("evt.time") + + -- set the filter + if CAPTURE_LOGS then + chisel.set_filter("(evt.type=tracer) or (evt.is_io_write=true and evt.dir=< and (fd.num=1 or fd.num=2 or fd.name contains log))") + else + chisel.set_filter("evt.type=tracer and evt.dir=<") + end + + return true +end + +-- Add a log entry into the proper place(s) in the log table +function collect_log(tid_tree) + for k,entry in pairs(tid_tree) do + while true do + local lastv = v + k,v = next(entry) + if v == nil then + if lastv.l == nil then + lastv.l = {} + end + + local etime = evt.field(ftime) + local buf = evt.field(fbuf) + local tid = evt.field(ftid) + local hi, low = evt.get_ts() + + local linedata = {t=etime, th=hi, tl=low, tid=tid, b=buf} + + table.insert(lastv.l, linedata) +--print("*** " .. evt.get_num() .. " " .. linedata) +--print(st(logs_tree)) +--print("***************************") + return + end + + entry = v.ch + end + end +end + +-- Parse a tracer enter event and update the logs_tree table +function parse_tracer_enter(logtable_cur, hr) + for j = 1, #hr do + local mv = hr[j] + + if mv == nil then + break + end + + if logtable_cur[mv] == nil then + logtable_cur[mv] = {ch={}} + end + + if j == #hr then + logtable_cur[mv].r=true + end + + logtable_cur = logtable_cur[mv].ch + end +end + +-- Parse a tracer exit event and update the given transaction entry +function parse_tracer_exit(mrk_cur, logtable_cur, hr, latency, contname, exe, id) + local res = false + local parent_has_logs = false; + + for j = 1, #hr do + local mv = hr[j] + if mv == nil or mrk_cur == nil then + break + end + + local has_logtable_entry = (logtable_cur ~= nil and logtable_cur[mv] ~= nil) + +--print("! " .. evt.get_num() .. " " .. j) +--print(parent_has_logs) +--print(logtable_cur[mv].r) + if j == #hr then + local llogs + + if has_logtable_entry and logtable_cur[mv].l ~= nil then + llogs = logtable_cur[mv].l + else + llogs = nil + end + +--print("################ " .. evt.get_num() .. " " .. st(logs_tree)) + if mrk_cur[mv] == nil then + mrk_cur[mv] = {t=latency, tt=latency, cont=contname, exe=exe, c=1, logs=llogs} + if j == 1 then + mrk_cur[mv].n = 0 + end + else + mrk_cur[mv]["tt"] = mrk_cur[mv]["tt"] + latency + mrk_cur[mv]["cont"] = contname + mrk_cur[mv]["exe"] = exe + mrk_cur[mv]["c"] = 1 + mrk_cur[mv]["logs"] = llogs + end + +--print("################ " .. evt.get_num()) +--print(st(logs_tree)) +--print("## " .. evt.get_num()) +--print(st(logtable_cur[mv].r)) + + if has_logtable_entry and parent_has_logs == false then + res = true + else + logtable_cur[mv] = nil + has_logtable_entry = false + logtable_cur = nil + end + elseif j == (#hr - 1) then + if mrk_cur[mv] == nil then + mrk_cur[mv] = {tt=0} + if j == 1 then + mrk_cur[mv].n = 0 + end + end + else + if mrk_cur[mv] == nil then + mrk_cur[mv] = {tt=0} + if j == 1 then + mrk_cur[mv].n = 0 + mrk_cur[mv]["id"] = id + end + end + end + + if mrk_cur[mv]["ch"] == nil then + mrk_cur[mv]["ch"] = {} + end + + if #hr == 1 then + mrk_cur[mv].n = mrk_cur[mv].n + 1 + end + + -- end of node parsing, update pointers to movo to the child + if has_logtable_entry then + parent_has_logs = (logtable_cur[mv].r ~= nil) + end + + mrk_cur = mrk_cur[mv].ch + + if logtable_cur ~= nil then + logtable_cur = logtable_cur[mv].ch + end + end + + return res +end + +-- Event parsing callback +function on_event() + local etype = evt.get_type() + + if etype ~= "tracer" then + local tid = evt.field(ftid) + + if logs_tree[tid] == nil then + return + else + collect_log(logs_tree[tid]) + end + + return + end + + local latency = evt.field(flatency) + local contname = evt.field(fcontname) + local id = evt.field(fid) + local exe = evt.field(fexe) + local hr = {} + local full_trs = nil + local dir = evt.field(fdir) + local tid = evt.field(ftid) + + for j = 0, MAX_DEPTH do + hr[j + 1] = evt.field(spans[j]) + end + + if dir == ">" then + if logs_tree[tid] == nil then + logs_tree[tid] = {} + end + + local idt = logs_tree[tid][id] + + if idt == nil then + logs_tree[tid][id] = {} + idt = logs_tree[tid][id] + end + + parse_tracer_enter(idt, hr) + return true + else + if latency == nil then + return true + end + + if full_tree[id] == nil then + full_tree[id] = {} + end + + -- find the logs for this transaction span + local logs + + if logs_tree[tid] == nil then + logs = nil + else + if logs_tree[tid][id] == nil then + logs = nil + else + logs = logs_tree[tid][id] + end + end + + if parse_tracer_exit(full_tree[id], logs, hr, latency, contname, exe, id) then +--print(st(logs_tree)) +--print("------------ " .. evt.get_num()) +--print(st(full_tree)) +--print("---------------------------------------------------") + + logs_tree[tid][id] = nil + + if next(logs_tree[tid]) == nil then + logs_tree[tid] = nil + end + + end + + return true + end +end + +function calculate_t_in_node(node) + local totchtime = 0 + local maxchtime = 0 + local nconc = 0 + local ch_to_keep + + if node.ch then + for k,d in pairs(node.ch) do + local nv = calculate_t_in_node(d) + + totchtime = totchtime + nv + + if nv > maxchtime then + maxchtime = nv + ch_to_keep = d + end + + nconc = nconc + 1 + end + end + + if node.tt >= totchtime then + node.t = node.tt - totchtime + else + node.t = node.tt - maxchtime + node.nconc = nconc + + for k,d in pairs(node.ch) do + if d ~= ch_to_keep then + node.ch[k] = nil + end + end + + end + + return node.tt +end + +function normalize(node, factor) + node.t = node.t / factor + node.tt = node.tt / factor + if node.ch then + for k,d in pairs(node.ch) do + normalize(d, factor) + end + end +end + +function is_transaction_complete(node) + if node.c ~= 1 then + return false + end + + if node.ch then + for k,d in pairs(node.ch) do + if is_transaction_complete(d) == false then + return false + end + end + end + + return true +end + +function update_avg_tree(dsttree, key, val) + if dsttree[key] == nil then + dsttree[key] = copytable(val) + return + else + dsttree[key].tt = dsttree[key].tt + val.tt + + if dsttree[key].n then + dsttree[key].n = dsttree[key].n + 1 + end + + if val.logs then + if dsttree[key].logs == nil then + dsttree[key].logs = {} + end + + concattable(dsttree[key].logs, val.logs) + end + end + + if val.ch then + if dsttree[key].ch == nil then + dsttree[key].ch = {} + end + + for k,d in pairs(val.ch) do + update_avg_tree(dsttree[key].ch, k, d) + end + end +end + +function update_max_tree(dsttree, key, val) + if dsttree[key] == nil then + dsttree[key] = val + return + else + if val.tt > dsttree[key].tt then + dsttree[key] = val + end + end +end + +function update_min_tree(dsttree, key, val) + if dsttree[key] == nil then + dsttree[key] = val + return + else + if val.tt < dsttree[key].tt then + dsttree[key] = val + end + end +end + +-- This processes the transaction list to extract and aggregate the transactions to emit +function collapse_tree() + -- scan the transaction list + for i,v in pairs(full_tree) do + local ttt = 0 + for key,val in pairs(v) do + ttt = ttt + val.tt + if is_transaction_complete(val) then + update_avg_tree(avg_tree, key, val) + update_max_tree(max_tree, key, val) + update_min_tree(min_tree, key, val) + end + end + end +end + +-- Called by the engine at the end of the capture (Ctrl-C) +function on_capture_end() +--print(st(full_tree)) + -- Process the list and create the required transactions + collapse_tree() + + -- calculate the unique time spent in each node + for i,v in pairs(avg_tree) do + calculate_t_in_node(v) + end + + -- normalize each root span tree + for i,v in pairs(avg_tree) do + normalize(v, v.n) + end + + print(PAGE_HEADER) + + -- emit the average transaction + local AvgData = {} + AvgData[""] = {ch=avg_tree, t=0, tt=0} + local str = json.encode(AvgData, { indent = true }) + print('"avg": ' .. str .. ",") + + -- normalize the best transaction + for i,v in pairs(min_tree) do + calculate_t_in_node(v) + end + + -- emit the best transaction + local tdata = {} + tdata[""] = {ch=min_tree, t=0, tt=0} + local str = json.encode(tdata, { indent = true }) + print('"min": ' .. str .. ",") + + -- normalize the worst transaction + for i,v in pairs(max_tree) do + calculate_t_in_node(v) + end + + -- emit the worst transaction + local tdata = {} + tdata[""] = {ch=max_tree, t=0, tt=0} + local str = json.encode(tdata, { indent = true }) + print('"max": ' .. str .. ",") + + print(PAGE_TRAILER) +end diff --git a/userspace/sysdig/chisels/v_threads.lua b/userspace/sysdig/chisels/v_threads.lua index 2c76d5961c..620d49c6ba 100644 --- a/userspace/sysdig/chisels/v_threads.lua +++ b/userspace/sysdig/chisels/v_threads.lua @@ -56,7 +56,7 @@ view_info = }, { tags = {"containers"}, - name = "VTPID", + name = "VTID", field = "thread.vtid", description = "TID that the tread has inside the container.", colsize = 8,