diff --git a/CMakeLists.txt b/CMakeLists.txt index 9245c30839c16c..2109bc851728b9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -389,6 +389,8 @@ set(LIBNETDATA_FILES ${CONFIG_H} libnetdata/popen/popen.h libnetdata/procfile/procfile.c libnetdata/procfile/procfile.h + libnetdata/query_progress/progress.c + libnetdata/query_progress/progress.h libnetdata/required_dummies.h libnetdata/socket/security.c libnetdata/socket/security.h @@ -411,6 +413,9 @@ set(LIBNETDATA_FILES ${CONFIG_H} libnetdata/string/utf8.h libnetdata/worker_utilization/worker_utilization.c libnetdata/worker_utilization/worker_utilization.h + libnetdata/http/http_access.c + libnetdata/http/http_access.h + libnetdata/http/http_defs.c libnetdata/http/http_defs.h libnetdata/dyn_conf/dyn_conf.c libnetdata/dyn_conf/dyn_conf.h) @@ -644,8 +649,16 @@ endif() set(PLUGINSD_PLUGIN_FILES collectors/plugins.d/plugins_d.c collectors/plugins.d/plugins_d.h + collectors/plugins.d/pluginsd_dyncfg.c + collectors/plugins.d/pluginsd_dyncfg.h + collectors/plugins.d/pluginsd_functions.c + collectors/plugins.d/pluginsd_functions.h + collectors/plugins.d/pluginsd_internals.c + collectors/plugins.d/pluginsd_internals.h collectors/plugins.d/pluginsd_parser.c - collectors/plugins.d/pluginsd_parser.h) + collectors/plugins.d/pluginsd_parser.h + collectors/plugins.d/pluginsd_replication.c + collectors/plugins.d/pluginsd_replication.h) set(RRD_PLUGIN_FILES database/contexts/api_v1.c database/contexts/api_v2.c @@ -662,6 +675,8 @@ set(RRD_PLUGIN_FILES database/contexts/api_v1.c database/rrdcalc.h database/rrdcalctemplate.c database/rrdcalctemplate.h + database/rrdcollector.c + database/rrdcollector.h database/rrddim.c database/rrddimvar.c database/rrddimvar.h diff --git a/aclk/aclk_query.c b/aclk/aclk_query.c index da5385fdb83523..ee053d9e676bca 100644 --- a/aclk/aclk_query.c +++ b/aclk/aclk_query.c @@ -106,8 +106,8 @@ static int http_api_v2(struct aclk_query_thread *query_thr, aclk_query_t query) char *start, *end; struct web_client *w = web_client_get_from_cache(); - w->acl = WEB_CLIENT_ACL_ACLK; - w->mode = WEB_CLIENT_MODE_GET; + w->acl = HTTP_ACL_ACLK; + w->mode = HTTP_REQUEST_MODE_GET; w->timings.tv_in = query->created_tv; w->interrupt.callback = aclk_web_client_interrupt_cb; diff --git a/collectors/apps.plugin/apps_plugin.c b/collectors/apps.plugin/apps_plugin.c index eff4aee04d90dc..ae37b48f07fdd8 100644 --- a/collectors/apps.plugin/apps_plugin.c +++ b/collectors/apps.plugin/apps_plugin.c @@ -13,11 +13,15 @@ #define APPS_PLUGIN_PROCESSES_FUNCTION_DESCRIPTION "Detailed information on the currently running processes." #define APPS_PLUGIN_FUNCTIONS() do { \ - fprintf(stdout, PLUGINSD_KEYWORD_FUNCTION " \"processes\" %d \"%s\"\n", PLUGINS_FUNCTIONS_TIMEOUT_DEFAULT, APPS_PLUGIN_PROCESSES_FUNCTION_DESCRIPTION); \ + fprintf(stdout, PLUGINSD_KEYWORD_FUNCTION " \"processes\" %d \"%s\" \"top\" \"members\" %d\n", \ + PLUGINS_FUNCTIONS_TIMEOUT_DEFAULT, APPS_PLUGIN_PROCESSES_FUNCTION_DESCRIPTION, \ + RRDFUNCTIONS_PRIORITY_DEFAULT / 10); \ } while(0) #define APPS_PLUGIN_GLOBAL_FUNCTIONS() do { \ - fprintf(stdout, PLUGINSD_KEYWORD_FUNCTION " GLOBAL \"processes\" %d \"%s\"\n", PLUGINS_FUNCTIONS_TIMEOUT_DEFAULT, APPS_PLUGIN_PROCESSES_FUNCTION_DESCRIPTION); \ + fprintf(stdout, PLUGINSD_KEYWORD_FUNCTION " GLOBAL \"processes\" %d \"%s\" \"top\" \"members\" %d\n", \ + PLUGINS_FUNCTIONS_TIMEOUT_DEFAULT, APPS_PLUGIN_PROCESSES_FUNCTION_DESCRIPTION, \ + RRDFUNCTIONS_PRIORITY_DEFAULT / 10); \ } while(0) // ---------------------------------------------------------------------------- @@ -4393,7 +4397,7 @@ static void apps_plugin_function_processes_help(const char *transaction) { buffer_json_add_array_item_double(wb, _tmp); \ } while(0) -static void function_processes(const char *transaction, char *function __maybe_unused, int timeout __maybe_unused, bool *cancelled __maybe_unused) { +static void function_processes(const char *transaction, char *function __maybe_unused, usec_t *stop_monotonic_ut __maybe_unused, bool *cancelled __maybe_unused) { struct pid_stat *p; char *words[PLUGINSD_MAX_WORDS] = { NULL }; diff --git a/collectors/cgroups.plugin/cgroup-internals.h b/collectors/cgroups.plugin/cgroup-internals.h index a6980224066b43..2cd0673ae17755 100644 --- a/collectors/cgroups.plugin/cgroup-internals.h +++ b/collectors/cgroups.plugin/cgroup-internals.h @@ -452,15 +452,25 @@ static inline char *cgroup_chart_type(char *buffer, struct cgroup *cg) { } #define RRDFUNCTIONS_CGTOP_HELP "View running containers" - -int cgroup_function_cgroup_top(BUFFER *wb, int timeout, const char *function, void *collector_data, - rrd_function_result_callback_t result_cb, void *result_cb_data, - rrd_function_is_cancelled_cb_t is_cancelled_cb, void *is_cancelled_cb_data, - rrd_function_register_canceller_cb_t register_canceller_cb, void *register_canceller_cb_data); -int cgroup_function_systemd_top(BUFFER *wb, int timeout, const char *function, void *collector_data, - rrd_function_result_callback_t result_cb, void *result_cb_data, - rrd_function_is_cancelled_cb_t is_cancelled_cb, void *is_cancelled_cb_data, - rrd_function_register_canceller_cb_t register_canceller_cb, void *register_canceller_cb_data); +#define RRDFUNCTIONS_SYSTEMD_SERVICES_HELP "View systemd services" + +int cgroup_function_cgroup_top(uuid_t *transaction, BUFFER *wb, + usec_t *stop_monotonic_ut, const char *function, void *collector_data, + rrd_function_result_callback_t result_cb, void *result_cb_data, + rrd_function_progress_cb_t progress_cb, void *progress_cb_data, + rrd_function_is_cancelled_cb_t is_cancelled_cb, void *is_cancelled_cb_data, + rrd_function_register_canceller_cb_t register_canceller_cb, void *register_canceller_cb_data, + rrd_function_register_progresser_cb_t register_progresser_cb, + void *register_progresser_cb_data); + +int cgroup_function_systemd_top(uuid_t *transaction, BUFFER *wb, + usec_t *stop_monotonic_ut, const char *function, void *collector_data, + rrd_function_result_callback_t result_cb, void *result_cb_data, + rrd_function_progress_cb_t progress_cb, void *progress_cb_data, + rrd_function_is_cancelled_cb_t is_cancelled_cb, void *is_cancelled_cb_data, + rrd_function_register_canceller_cb_t register_canceller_cb, void *register_canceller_cb_data, + rrd_function_register_progresser_cb_t register_progresser_cb, + void *register_progresser_cb_data); void cgroup_netdev_link_init(void); const DICTIONARY_ITEM *cgroup_netdev_get(struct cgroup *cg); diff --git a/collectors/cgroups.plugin/cgroup-top.c b/collectors/cgroups.plugin/cgroup-top.c index 0e64b908d83340..6daa9ce5052b5d 100644 --- a/collectors/cgroups.plugin/cgroup-top.c +++ b/collectors/cgroups.plugin/cgroup-top.c @@ -97,12 +97,16 @@ void cgroup_netdev_get_bandwidth(struct cgroup *cg, NETDATA_DOUBLE *received, NE *sent = t->sent[slot]; } -int cgroup_function_cgroup_top(BUFFER *wb, int timeout __maybe_unused, const char *function __maybe_unused, - void *collector_data __maybe_unused, - rrd_function_result_callback_t result_cb, void *result_cb_data, - rrd_function_is_cancelled_cb_t is_cancelled_cb, void *is_cancelled_cb_data, - rrd_function_register_canceller_cb_t register_canceller_cb __maybe_unused, - void *register_canceller_cb_data __maybe_unused) { +int cgroup_function_cgroup_top(uuid_t *transaction __maybe_unused, BUFFER *wb, + usec_t *stop_monotonic_ut __maybe_unused, const char *function __maybe_unused, + void *collector_data __maybe_unused, + rrd_function_result_callback_t result_cb, void *result_cb_data, + rrd_function_progress_cb_t progress_cb, void *progress_cb_data, + rrd_function_is_cancelled_cb_t is_cancelled_cb, void *is_cancelled_cb_data, + rrd_function_register_canceller_cb_t register_canceller_cb __maybe_unused, + void *register_canceller_cb_data __maybe_unused, + rrd_function_register_progresser_cb_t register_progresser_cb __maybe_unused, + void *register_progresser_cb_data __maybe_unused) { buffer_flush(wb); wb->content_type = CT_APPLICATION_JSON; @@ -342,12 +346,16 @@ int cgroup_function_cgroup_top(BUFFER *wb, int timeout __maybe_unused, const cha return response; } -int cgroup_function_systemd_top(BUFFER *wb, int timeout __maybe_unused, const char *function __maybe_unused, - void *collector_data __maybe_unused, - rrd_function_result_callback_t result_cb, void *result_cb_data, - rrd_function_is_cancelled_cb_t is_cancelled_cb, void *is_cancelled_cb_data, - rrd_function_register_canceller_cb_t register_canceller_cb __maybe_unused, - void *register_canceller_cb_data __maybe_unused) { +int cgroup_function_systemd_top(uuid_t *transaction __maybe_unused, BUFFER *wb, + usec_t *stop_monotonic_ut __maybe_unused, const char *function __maybe_unused, + void *collector_data __maybe_unused, + rrd_function_result_callback_t result_cb, void *result_cb_data, + rrd_function_progress_cb_t progress_cb __maybe_unused, void *progress_cb_data __maybe_unused, + rrd_function_is_cancelled_cb_t is_cancelled_cb, void *is_cancelled_cb_data, + rrd_function_register_canceller_cb_t register_canceller_cb __maybe_unused, + void *register_canceller_cb_data __maybe_unused, + rrd_function_register_progresser_cb_t register_progresser_cb __maybe_unused, + void *register_progresser_cb_data __maybe_unused) { buffer_flush(wb); wb->content_type = CT_APPLICATION_JSON; diff --git a/collectors/cgroups.plugin/sys_fs_cgroup.c b/collectors/cgroups.plugin/sys_fs_cgroup.c index 705edf6f748fc2..2dbeaee2fbf2b6 100644 --- a/collectors/cgroups.plugin/sys_fs_cgroup.c +++ b/collectors/cgroups.plugin/sys_fs_cgroup.c @@ -1671,8 +1671,14 @@ void *cgroups_main(void *ptr) { // for the other nodes, the origin server should register it rrd_collector_started(); // this creates a collector that runs for as long as netdata runs cgroup_netdev_link_init(); - rrd_function_add(localhost, NULL, "containers-vms", 10, RRDFUNCTIONS_CGTOP_HELP, true, cgroup_function_cgroup_top, NULL); - rrd_function_add(localhost, NULL, "systemd-services", 10, RRDFUNCTIONS_CGTOP_HELP, true, cgroup_function_systemd_top, NULL); + + rrd_function_add(localhost, NULL, "containers-vms", 10, RRDFUNCTIONS_PRIORITY_DEFAULT / 2, + RRDFUNCTIONS_CGTOP_HELP, "top", HTTP_ACCESS_ANY, + true, cgroup_function_cgroup_top, NULL); + + rrd_function_add(localhost, NULL, "systemd-services", 10, RRDFUNCTIONS_PRIORITY_DEFAULT / 3, + RRDFUNCTIONS_SYSTEMD_SERVICES_HELP, "top", HTTP_ACCESS_ANY, + true, cgroup_function_systemd_top, NULL); heartbeat_t hb; heartbeat_init(&hb); diff --git a/collectors/diskspace.plugin/plugin_diskspace.c b/collectors/diskspace.plugin/plugin_diskspace.c index af1d93ba761acc..78d7d5af2be812 100644 --- a/collectors/diskspace.plugin/plugin_diskspace.c +++ b/collectors/diskspace.plugin/plugin_diskspace.c @@ -636,12 +636,16 @@ static void diskspace_main_cleanup(void *ptr) { #error WORKER_UTILIZATION_MAX_JOB_TYPES has to be at least 3 #endif -int diskspace_function_mount_points(BUFFER *wb, int timeout __maybe_unused, const char *function __maybe_unused, - void *collector_data __maybe_unused, - rrd_function_result_callback_t result_cb, void *result_cb_data, - rrd_function_is_cancelled_cb_t is_cancelled_cb, void *is_cancelled_cb_data, - rrd_function_register_canceller_cb_t register_canceller_cb __maybe_unused, - void *register_canceller_cb_data __maybe_unused) { +int diskspace_function_mount_points(uuid_t *transaction __maybe_unused, BUFFER *wb, + usec_t *stop_monotonic_ut __maybe_unused, const char *function __maybe_unused, + void *collector_data __maybe_unused, + rrd_function_result_callback_t result_cb, void *result_cb_data, + rrd_function_progress_cb_t progress_cb __maybe_unused, void *progress_cb_data __maybe_unused, + rrd_function_is_cancelled_cb_t is_cancelled_cb, void *is_cancelled_cb_data, + rrd_function_register_canceller_cb_t register_canceller_cb __maybe_unused, + void *register_canceller_cb_data __maybe_unused, + rrd_function_register_progresser_cb_t register_progresser_cb __maybe_unused, + void *register_progresser_cb_data __maybe_unused) { buffer_flush(wb); wb->content_type = CT_APPLICATION_JSON; @@ -865,7 +869,9 @@ void *diskspace_main(void *ptr) { worker_register_job_name(WORKER_JOB_CLEANUP, "cleanup"); rrd_collector_started(); - rrd_function_add(localhost, NULL, "mount-points", 10, RRDFUNCTIONS_DISKSPACE_HELP, true, diskspace_function_mount_points, NULL); + rrd_function_add(localhost, NULL, "mount-points", 10, RRDFUNCTIONS_PRIORITY_DEFAULT, RRDFUNCTIONS_DISKSPACE_HELP, + "top", HTTP_ACCESS_ANY, + true, diskspace_function_mount_points, NULL); netdata_thread_cleanup_push(diskspace_main_cleanup, ptr); diff --git a/collectors/ebpf.plugin/ebpf_functions.c b/collectors/ebpf.plugin/ebpf_functions.c index 6a481ad64f2839..8d161c4e71e7c2 100644 --- a/collectors/ebpf.plugin/ebpf_functions.c +++ b/collectors/ebpf.plugin/ebpf_functions.c @@ -639,10 +639,9 @@ void ebpf_socket_read_open_connections(BUFFER *buf, struct ebpf_module *em) */ static void ebpf_function_socket_manipulation(const char *transaction, char *function __maybe_unused, - int timeout __maybe_unused, + usec_t *stop_monotonic_ut __maybe_unused, bool *cancelled __maybe_unused) { - UNUSED(timeout); ebpf_module_t *em = &ebpf_modules[EBPF_MODULE_SOCKET_IDX]; char *words[PLUGINSD_MAX_WORDS] = {NULL}; diff --git a/collectors/ebpf.plugin/ebpf_functions.h b/collectors/ebpf.plugin/ebpf_functions.h index 795703b428748d..330c402d877d7f 100644 --- a/collectors/ebpf.plugin/ebpf_functions.h +++ b/collectors/ebpf.plugin/ebpf_functions.h @@ -6,7 +6,8 @@ #ifdef NETDATA_DEV_MODE // Common static inline void EBPF_PLUGIN_FUNCTIONS(const char *NAME, const char *DESC) { - fprintf(stdout, "%s \"%s\" 10 \"%s\"\n", PLUGINSD_KEYWORD_FUNCTION, NAME, DESC); + fprintf(stdout, "%s \"%s\" 10 \"%s\" \"top\" \"any\" %d\n", + PLUGINSD_KEYWORD_FUNCTION, NAME, DESC, RRDFUNCTIONS_PRIORITY_DEFAULT); } #endif diff --git a/collectors/freeipmi.plugin/freeipmi_plugin.c b/collectors/freeipmi.plugin/freeipmi_plugin.c index 6ec9b698bf924a..72bf6dd755c05e 100644 --- a/collectors/freeipmi.plugin/freeipmi_plugin.c +++ b/collectors/freeipmi.plugin/freeipmi_plugin.c @@ -23,7 +23,8 @@ #include "libnetdata/required_dummies.h" #define FREEIPMI_GLOBAL_FUNCTION_SENSORS() do { \ - fprintf(stdout, PLUGINSD_KEYWORD_FUNCTION " GLOBAL \"ipmi-sensors\" %d \"%s\"\n", 5, "Displays current sensor state and readings"); \ + fprintf(stdout, PLUGINSD_KEYWORD_FUNCTION " GLOBAL \"ipmi-sensors\" %d \"%s\" \"top\" \"any\" %d\n", \ + 5, "Displays current sensor state and readings", 100); \ } while(0) // component names, based on our patterns @@ -1470,7 +1471,7 @@ static const char *get_sensor_function_priority(struct sensor *sn) { } } -static void freeimi_function_sensors(const char *transaction, char *function __maybe_unused, int timeout __maybe_unused, bool *cancelled __maybe_unused) { +static void freeimi_function_sensors(const char *transaction, char *function __maybe_unused, usec_t *stop_monotonic_ut __maybe_unused, bool *cancelled __maybe_unused) { time_t expires = now_realtime_sec() + update_every; BUFFER *wb = buffer_create(PLUGINSD_LINE_MAX, NULL); diff --git a/collectors/log2journal/log2journal.c b/collectors/log2journal/log2journal.c index c3204939cda9c5..e4de6cd2486e72 100644 --- a/collectors/log2journal/log2journal.c +++ b/collectors/log2journal/log2journal.c @@ -67,7 +67,7 @@ static inline HASHED_KEY *get_key_from_hashtable(LOG_JOB *jb, HASHED_KEY *k) { if(!k->hashtable_ptr) { HASHED_KEY *ht_key; - SIMPLE_HASHTABLE_SLOT_KEY *slot = simple_hashtable_get_slot_KEY(&jb->hashtable, k->hash, true); + SIMPLE_HASHTABLE_SLOT_KEY *slot = simple_hashtable_get_slot_KEY(&jb->hashtable, k->hash, NULL, true); if((ht_key = SIMPLE_HASHTABLE_SLOT_DATA(slot))) { if(!(ht_key->flags & HK_COLLISION_CHECKED)) { ht_key->flags |= HK_COLLISION_CHECKED; diff --git a/collectors/log2journal/log2journal.h b/collectors/log2journal/log2journal.h index 834a5b135d8a3c..58485cf110d3eb 100644 --- a/collectors/log2journal/log2journal.h +++ b/collectors/log2journal/log2journal.h @@ -99,10 +99,7 @@ static inline void freez(void *ptr) { // hashtable for HASHED_KEY // cleanup hashtable defines -#undef SIMPLE_HASHTABLE_SORT_FUNCTION -#undef SIMPLE_HASHTABLE_VALUE_TYPE -#undef SIMPLE_HASHTABLE_NAME -#undef NETDATA_SIMPLE_HASHTABLE_H +#include "../../libnetdata/simple_hashtable_undef.h" struct hashed_key; static inline int compare_keys(struct hashed_key *k1, struct hashed_key *k2); diff --git a/collectors/plugins.d/README.md b/collectors/plugins.d/README.md index c991171c55a9fb..c2d8e1457939a6 100644 --- a/collectors/plugins.d/README.md +++ b/collectors/plugins.d/README.md @@ -134,6 +134,7 @@ Netdata parses lines starting with: - `FLUSH` - ignore the last collected values - `DISABLE` - disable this plugin - `FUNCTION` - define functions +- `FUNCTION_PROGRESS` - report the progress of a function execution - `FUNCTION_RESULT_BEGIN` - to initiate the transmission of function results - `FUNCTION_RESULT_END` - to end the transmission of function results @@ -146,6 +147,7 @@ Netdata may send the following commands to the plugin's `stdin`: - `FUNCTION` - to call a specific function, with all parameters inline - `FUNCTION_PAYLOAD` - to call a specific function, with a payload of parameters - `FUNCTION_PAYLOAD_END` - to end the payload of parameters +- `FUNCTION_CANCEL` - cancel a running function transaction ### Command line parameters @@ -466,7 +468,10 @@ The `source` is an integer field that can have the following values: The plugin can register functions to Netdata, like this: -> FUNCTION [GLOBAL] "name and parameters of the function" timeout "help string for users" +> FUNCTION [GLOBAL] "name and parameters of the function" timeout "help string for users" "tags" "access" + +- Tags currently recognized are either `top` or `logs` (or both, space separated). +- Access is one of `any`, `members`, or `admins`. A function can be used by users to ask for more information from the collector. Netdata maintains a registry of functions in 2 levels: diff --git a/collectors/plugins.d/gperf-config.txt b/collectors/plugins.d/gperf-config.txt index bad51367ce1abb..e1b975dda37d65 100644 --- a/collectors/plugins.d/gperf-config.txt +++ b/collectors/plugins.d/gperf-config.txt @@ -30,29 +30,31 @@ DIMENSION, 31, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING|PARSER_R END, 13, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 13 FUNCTION, 41, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 14 FUNCTION_RESULT_BEGIN, 42, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 15 -LABEL, 51, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 16 -OVERWRITE, 52, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 17 -SET, 11, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 18 -VARIABLE, 53, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 19 -DYNCFG_ENABLE, 101, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 20 -DYNCFG_REGISTER_MODULE, 102, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 21 -DYNCFG_REGISTER_JOB, 103, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 22 -DYNCFG_RESET, 104, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 23 -REPORT_JOB_STATUS, 110, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 24 -DELETE_JOB, 111, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 25 +FUNCTION_PROGRESS, 43, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 16 +# +LABEL, 51, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 17 +OVERWRITE, 52, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 18 +SET, 11, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 19 +VARIABLE, 53, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 20 +DYNCFG_ENABLE, 101, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 21 +DYNCFG_REGISTER_MODULE, 102, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 22 +DYNCFG_REGISTER_JOB, 103, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 23 +DYNCFG_RESET, 104, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 24 +REPORT_JOB_STATUS, 110, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 25 +DELETE_JOB, 111, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 26 # # Streaming only keywords # -CLAIMED_ID, 61, PARSER_INIT_STREAMING|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 26 -BEGIN2, 2, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 27 -SET2, 1, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 28 -END2, 3, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 29 +CLAIMED_ID, 61, PARSER_INIT_STREAMING|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 27 +BEGIN2, 2, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 28 +SET2, 1, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 29 +END2, 3, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 30 # # Streaming Replication keywords # -CHART_DEFINITION_END, 33, PARSER_INIT_STREAMING|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 30 -RBEGIN, 22, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 31 -RDSTATE, 23, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 32 -REND, 25, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 33 -RSET, 21, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 34 -RSSTATE, 24, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 35 +CHART_DEFINITION_END, 33, PARSER_INIT_STREAMING|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 31 +RBEGIN, 22, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 32 +RDSTATE, 23, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 33 +REND, 25, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 34 +RSET, 21, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 35 +RSSTATE, 24, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 36 diff --git a/collectors/plugins.d/gperf-hashtable.h b/collectors/plugins.d/gperf-hashtable.h index b327d8d6d3404a..05360790e4022a 100644 --- a/collectors/plugins.d/gperf-hashtable.h +++ b/collectors/plugins.d/gperf-hashtable.h @@ -30,12 +30,12 @@ #endif -#define GPERF_PARSER_TOTAL_KEYWORDS 35 +#define GPERF_PARSER_TOTAL_KEYWORDS 36 #define GPERF_PARSER_MIN_WORD_LENGTH 3 #define GPERF_PARSER_MAX_WORD_LENGTH 22 #define GPERF_PARSER_MIN_HASH_VALUE 3 -#define GPERF_PARSER_MAX_HASH_VALUE 47 -/* maximum key range = 45, duplicates = 0 */ +#define GPERF_PARSER_MAX_HASH_VALUE 48 +/* maximum key range = 46, duplicates = 0 */ #ifdef __GNUC__ __inline @@ -49,32 +49,32 @@ gperf_keyword_hash_function (register const char *str, register size_t len) { static unsigned char asso_values[] = { - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 11, 18, 0, 0, 0, - 6, 48, 9, 0, 48, 48, 20, 48, 0, 8, - 48, 48, 1, 12, 48, 20, 18, 48, 2, 0, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48 + 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, + 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, + 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, + 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, + 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, + 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, + 49, 49, 49, 49, 49, 23, 29, 0, 0, 0, + 0, 49, 9, 0, 49, 49, 20, 49, 0, 8, + 49, 49, 1, 12, 49, 23, 6, 49, 2, 0, + 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, + 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, + 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, + 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, + 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, + 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, + 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, + 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, + 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, + 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, + 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, + 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, + 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, + 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, + 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, + 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, + 49, 49, 49, 49, 49, 49 }; return len + asso_values[(unsigned char)str[1]] + asso_values[(unsigned char)str[0]]; } @@ -84,78 +84,81 @@ static PARSER_KEYWORD gperf_keywords[] = {(char*)0}, {(char*)0}, {(char*)0}, #line 30 "gperf-config.txt" {"END", 13, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 13}, -#line 49 "gperf-config.txt" - {"END2", 3, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 29}, -#line 56 "gperf-config.txt" - {"REND", 25, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 33}, +#line 51 "gperf-config.txt" + {"END2", 3, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 30}, +#line 58 "gperf-config.txt" + {"REND", 25, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 34}, #line 17 "gperf-config.txt" {"EXIT", 99, PARSER_INIT_PLUGINSD, WORKER_PARSER_FIRST_JOB + 3}, #line 16 "gperf-config.txt" {"DISABLE", 98, PARSER_INIT_PLUGINSD, WORKER_PARSER_FIRST_JOB + 2}, -#line 55 "gperf-config.txt" - {"RDSTATE", 23, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 32}, +#line 57 "gperf-config.txt" + {"RDSTATE", 23, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 33}, #line 29 "gperf-config.txt" {"DIMENSION", 31, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 12}, -#line 42 "gperf-config.txt" - {"DELETE_JOB", 111, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 25}, +#line 44 "gperf-config.txt" + {"DELETE_JOB", 111, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 26}, {(char*)0}, -#line 40 "gperf-config.txt" - {"DYNCFG_RESET", 104, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 23}, -#line 37 "gperf-config.txt" - {"DYNCFG_ENABLE", 101, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 20}, +#line 42 "gperf-config.txt" + {"DYNCFG_RESET", 104, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 24}, +#line 39 "gperf-config.txt" + {"DYNCFG_ENABLE", 101, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 21}, #line 26 "gperf-config.txt" {"CHART", 32, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 9}, -#line 35 "gperf-config.txt" - {"SET", 11, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 18}, -#line 48 "gperf-config.txt" - {"SET2", 1, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 28}, -#line 57 "gperf-config.txt" - {"RSET", 21, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 34}, +#line 37 "gperf-config.txt" + {"SET", 11, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 19}, +#line 50 "gperf-config.txt" + {"SET2", 1, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 29}, +#line 59 "gperf-config.txt" + {"RSET", 21, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 35}, +#line 43 "gperf-config.txt" + {"REPORT_JOB_STATUS", 110, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 25}, #line 41 "gperf-config.txt" - {"REPORT_JOB_STATUS", 110, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 24}, -#line 39 "gperf-config.txt" - {"DYNCFG_REGISTER_JOB", 103, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 22}, -#line 58 "gperf-config.txt" - {"RSSTATE", 24, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 35}, + {"DYNCFG_REGISTER_JOB", 103, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 23}, +#line 60 "gperf-config.txt" + {"RSSTATE", 24, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 36}, #line 18 "gperf-config.txt" {"HOST", 71, PARSER_INIT_PLUGINSD|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 4}, -#line 38 "gperf-config.txt" - {"DYNCFG_REGISTER_MODULE", 102, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 21}, -#line 25 "gperf-config.txt" - {"BEGIN", 12, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 8}, -#line 47 "gperf-config.txt" - {"BEGIN2", 2, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 27}, -#line 54 "gperf-config.txt" - {"RBEGIN", 22, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 31}, +#line 40 "gperf-config.txt" + {"DYNCFG_REGISTER_MODULE", 102, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 22}, +#line 36 "gperf-config.txt" + {"OVERWRITE", 52, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 18}, + {(char*)0}, +#line 15 "gperf-config.txt" + {"FLUSH", 97, PARSER_INIT_PLUGINSD, WORKER_PARSER_FIRST_JOB + 1}, #line 27 "gperf-config.txt" {"CLABEL", 34, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 10}, #line 21 "gperf-config.txt" {"HOST_LABEL", 74, PARSER_INIT_PLUGINSD|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 7}, #line 19 "gperf-config.txt" {"HOST_DEFINE", 72, PARSER_INIT_PLUGINSD|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 5}, -#line 53 "gperf-config.txt" - {"CHART_DEFINITION_END", 33, PARSER_INIT_STREAMING|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 30}, -#line 46 "gperf-config.txt" - {"CLAIMED_ID", 61, PARSER_INIT_STREAMING|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 26}, -#line 15 "gperf-config.txt" - {"FLUSH", 97, PARSER_INIT_PLUGINSD, WORKER_PARSER_FIRST_JOB + 1}, +#line 55 "gperf-config.txt" + {"CHART_DEFINITION_END", 33, PARSER_INIT_STREAMING|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 31}, +#line 48 "gperf-config.txt" + {"CLAIMED_ID", 61, PARSER_INIT_STREAMING|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 27}, +#line 31 "gperf-config.txt" + {"FUNCTION", 41, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 14}, #line 20 "gperf-config.txt" {"HOST_DEFINE_END", 73, PARSER_INIT_PLUGINSD|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 6}, #line 28 "gperf-config.txt" {"CLABEL_COMMIT", 35, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 11}, -#line 31 "gperf-config.txt" - {"FUNCTION", 41, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 14}, -#line 34 "gperf-config.txt" - {"OVERWRITE", 52, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 17}, +#line 25 "gperf-config.txt" + {"BEGIN", 12, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 8}, +#line 49 "gperf-config.txt" + {"BEGIN2", 2, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 28}, +#line 56 "gperf-config.txt" + {"RBEGIN", 22, PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 32}, +#line 38 "gperf-config.txt" + {"VARIABLE", 53, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 20}, + {(char*)0}, {(char*)0}, #line 33 "gperf-config.txt" - {"LABEL", 51, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 16}, -#line 36 "gperf-config.txt" - {"VARIABLE", 53, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 19}, - {(char*)0}, {(char*)0}, {(char*)0}, {(char*)0}, - {(char*)0}, {(char*)0}, {(char*)0}, {(char*)0}, - {(char*)0}, + {"FUNCTION_PROGRESS", 43, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 16}, + {(char*)0}, {(char*)0}, {(char*)0}, #line 32 "gperf-config.txt" - {"FUNCTION_RESULT_BEGIN", 42, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 15} + {"FUNCTION_RESULT_BEGIN", 42, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 15}, + {(char*)0}, {(char*)0}, {(char*)0}, +#line 35 "gperf-config.txt" + {"LABEL", 51, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 17} }; PARSER_KEYWORD * diff --git a/collectors/plugins.d/plugins_d.h b/collectors/plugins.d/plugins_d.h index 37c70f7e39c0c4..f58300b23fa691 100644 --- a/collectors/plugins.d/plugins_d.h +++ b/collectors/plugins.d/plugins_d.h @@ -10,9 +10,6 @@ #define PLUGINSD_CMD_MAX (FILENAME_MAX*2) #define PLUGINSD_STOCK_PLUGINS_DIRECTORY_PATH 0 -#define PLUGINSD_KEYWORD_FUNCTION_PAYLOAD "FUNCTION_PAYLOAD" -#define PLUGINSD_KEYWORD_FUNCTION_PAYLOAD_END "FUNCTION_PAYLOAD_END" - #define PLUGINSD_KEYWORD_DYNCFG_ENABLE "DYNCFG_ENABLE" #define PLUGINSD_KEYWORD_DYNCFG_REGISTER_MODULE "DYNCFG_REGISTER_MODULE" #define PLUGINSD_KEYWORD_DYNCFG_REGISTER_JOB "DYNCFG_REGISTER_JOB" diff --git a/collectors/plugins.d/pluginsd_dyncfg.c b/collectors/plugins.d/pluginsd_dyncfg.c new file mode 100644 index 00000000000000..b0b1945980cae8 --- /dev/null +++ b/collectors/plugins.d/pluginsd_dyncfg.c @@ -0,0 +1,585 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "pluginsd_dyncfg.h" + +struct mutex_cond { + pthread_mutex_t lock; + pthread_cond_t cond; + int rc; +}; + +static void virt_fnc_got_data_cb(BUFFER *wb __maybe_unused, int code, void *callback_data) +{ + struct mutex_cond *ctx = callback_data; + pthread_mutex_lock(&ctx->lock); + ctx->rc = code; + pthread_cond_broadcast(&ctx->cond); + pthread_mutex_unlock(&ctx->lock); +} + +#define VIRT_FNC_TIMEOUT_S 10 +#define VIRT_FNC_BUF_SIZE (4096) +void call_virtual_function_async(BUFFER *wb, RRDHOST *host, const char *name, const char *payload, rrd_function_result_callback_t callback, void *callback_data) { + PARSER *parser = NULL; + + //TODO simplify (as we really need only first parameter to get plugin name maybe we can avoid parsing all) + char *words[PLUGINSD_MAX_WORDS]; + char *function_with_params = strdupz(name); + size_t num_words = quoted_strings_splitter(function_with_params, words, PLUGINSD_MAX_WORDS, isspace_map_pluginsd); + + if (num_words < 2) { + netdata_log_error("PLUGINSD: virtual function name is empty."); + freez(function_with_params); + return; + } + + const DICTIONARY_ITEM *cpi = dictionary_get_and_acquire_item(host->configurable_plugins, get_word(words, num_words, 1)); + if (unlikely(cpi == NULL)) { + netdata_log_error("PLUGINSD: virtual function plugin '%s' not found.", name); + freez(function_with_params); + return; + } + struct configurable_plugin *cp = dictionary_acquired_item_value(cpi); + parser = (PARSER *)cp->cb_usr_ctx; + + BUFFER *function_out = buffer_create(VIRT_FNC_BUF_SIZE, NULL); + // if we are forwarding this to a plugin (as opposed to streaming/child) we have to remove the first parameter (plugin_name) + buffer_strcat(function_out, get_word(words, num_words, 0)); + for (size_t i = 1; i < num_words; i++) { + if (i == 1 && SERVING_PLUGINSD(parser)) + continue; + buffer_sprintf(function_out, " %s", get_word(words, num_words, i)); + } + freez(function_with_params); + + usec_t now_ut = now_monotonic_usec(); + + struct inflight_function tmp = { + .started_monotonic_ut = now_ut, + .result_body_wb = wb, + .timeout_s = VIRT_FNC_TIMEOUT_S, + .function = string_strdupz(buffer_tostring(function_out)), + .payload = payload != NULL ? strdupz(payload) : NULL, + .virtual = true, + + .result = { + .cb = callback, + .data = callback_data, + }, + .dyncfg = { + .stop_monotonic_ut = now_ut + VIRT_FNC_TIMEOUT_S * USEC_PER_SEC, + } + }; + tmp.stop_monotonic_ut = &tmp.dyncfg.stop_monotonic_ut; + buffer_free(function_out); + + uuid_generate_time(tmp.transaction); + char key[UUID_COMPACT_STR_LEN]; + uuid_unparse_lower_compact(tmp.transaction, key); + + dictionary_write_lock(parser->inflight.functions); + + // if there is any error, our dictionary callbacks will call the caller callback to notify + // the caller about the error - no need for error handling here. + dictionary_set(parser->inflight.functions, key, &tmp, sizeof(struct inflight_function)); + + if(!parser->inflight.smaller_monotonic_timeout_ut || *tmp.stop_monotonic_ut + RRDFUNCTIONS_TIMEOUT_EXTENSION_UT < parser->inflight.smaller_monotonic_timeout_ut) + parser->inflight.smaller_monotonic_timeout_ut = *tmp.stop_monotonic_ut + RRDFUNCTIONS_TIMEOUT_EXTENSION_UT; + + // garbage collect stale inflight functions + if(parser->inflight.smaller_monotonic_timeout_ut < now_ut) + pluginsd_inflight_functions_garbage_collect(parser, now_ut); + + dictionary_write_unlock(parser->inflight.functions); +} + + +dyncfg_config_t call_virtual_function_blocking(PARSER *parser, const char *name, int *rc, const char *payload) { + usec_t now_ut = now_monotonic_usec(); + BUFFER *wb = buffer_create(VIRT_FNC_BUF_SIZE, NULL); + + struct mutex_cond cond = { + .lock = PTHREAD_MUTEX_INITIALIZER, + .cond = PTHREAD_COND_INITIALIZER + }; + + struct inflight_function tmp = { + .started_monotonic_ut = now_ut, + .result_body_wb = wb, + .timeout_s = VIRT_FNC_TIMEOUT_S, + .function = string_strdupz(name), + .payload = payload != NULL ? strdupz(payload) : NULL, + .virtual = true, + + .result = { + .cb = virt_fnc_got_data_cb, + .data = &cond, + }, + .dyncfg = { + .stop_monotonic_ut = now_ut + VIRT_FNC_TIMEOUT_S * USEC_PER_SEC, + } + }; + tmp.stop_monotonic_ut = &tmp.dyncfg.stop_monotonic_ut; + + uuid_generate_time(tmp.transaction); + + char key[UUID_COMPACT_STR_LEN]; + uuid_unparse_lower_compact(tmp.transaction, key); + + dictionary_write_lock(parser->inflight.functions); + + // if there is any error, our dictionary callbacks will call the caller callback to notify + // the caller about the error - no need for error handling here. + dictionary_set(parser->inflight.functions, key, &tmp, sizeof(struct inflight_function)); + + if(!parser->inflight.smaller_monotonic_timeout_ut || *tmp.stop_monotonic_ut + RRDFUNCTIONS_TIMEOUT_EXTENSION_UT < parser->inflight.smaller_monotonic_timeout_ut) + parser->inflight.smaller_monotonic_timeout_ut = *tmp.stop_monotonic_ut + RRDFUNCTIONS_TIMEOUT_EXTENSION_UT; + + // garbage collect stale inflight functions + if(parser->inflight.smaller_monotonic_timeout_ut < now_ut) + pluginsd_inflight_functions_garbage_collect(parser, now_ut); + + dictionary_write_unlock(parser->inflight.functions); + + struct timespec tp; + clock_gettime(CLOCK_REALTIME, &tp); + tp.tv_sec += (time_t)VIRT_FNC_TIMEOUT_S; + + pthread_mutex_lock(&cond.lock); + + int ret = pthread_cond_timedwait(&cond.cond, &cond.lock, &tp); + if (ret == ETIMEDOUT) + netdata_log_error("PLUGINSD: DYNCFG virtual function %s timed out", name); + + pthread_mutex_unlock(&cond.lock); + + dyncfg_config_t cfg; + cfg.data = strdupz(buffer_tostring(wb)); + cfg.data_size = buffer_strlen(wb); + + if (rc != NULL) + *rc = cond.rc; + + buffer_free(wb); + return cfg; +} + +#define CVF_MAX_LEN (1024) +static dyncfg_config_t get_plugin_config_cb(void *usr_ctx, const char *plugin_name) +{ + PARSER *parser = usr_ctx; + + if (SERVING_STREAMING(parser)) { + char buf[CVF_MAX_LEN + 1]; + snprintfz(buf, CVF_MAX_LEN, FUNCTION_NAME_GET_PLUGIN_CONFIG " %s", plugin_name); + return call_virtual_function_blocking(parser, buf, NULL, NULL); + } + + return call_virtual_function_blocking(parser, FUNCTION_NAME_GET_PLUGIN_CONFIG, NULL, NULL); +} + +static dyncfg_config_t get_plugin_config_schema_cb(void *usr_ctx, const char *plugin_name) +{ + PARSER *parser = usr_ctx; + + if (SERVING_STREAMING(parser)) { + char buf[CVF_MAX_LEN + 1]; + snprintfz(buf, CVF_MAX_LEN, FUNCTION_NAME_GET_PLUGIN_CONFIG_SCHEMA " %s", plugin_name); + return call_virtual_function_blocking(parser, buf, NULL, NULL); + } + + return call_virtual_function_blocking(parser, "get_plugin_config_schema", NULL, NULL); +} + +static dyncfg_config_t get_module_config_cb(void *usr_ctx, const char *plugin_name, const char *module_name) +{ + PARSER *parser = usr_ctx; + BUFFER *wb = buffer_create(CVF_MAX_LEN, NULL); + + buffer_strcat(wb, FUNCTION_NAME_GET_MODULE_CONFIG); + if (SERVING_STREAMING(parser)) + buffer_sprintf(wb, " %s", plugin_name); + + buffer_sprintf(wb, " %s", module_name); + + dyncfg_config_t ret = call_virtual_function_blocking(parser, buffer_tostring(wb), NULL, NULL); + + buffer_free(wb); + + return ret; +} + +static dyncfg_config_t get_module_config_schema_cb(void *usr_ctx, const char *plugin_name, const char *module_name) +{ + PARSER *parser = usr_ctx; + BUFFER *wb = buffer_create(CVF_MAX_LEN, NULL); + + buffer_strcat(wb, FUNCTION_NAME_GET_MODULE_CONFIG_SCHEMA); + if (SERVING_STREAMING(parser)) + buffer_sprintf(wb, " %s", plugin_name); + + buffer_sprintf(wb, " %s", module_name); + + dyncfg_config_t ret = call_virtual_function_blocking(parser, buffer_tostring(wb), NULL, NULL); + + buffer_free(wb); + + return ret; +} + +static dyncfg_config_t get_job_config_schema_cb(void *usr_ctx, const char *plugin_name, const char *module_name) +{ + PARSER *parser = usr_ctx; + BUFFER *wb = buffer_create(CVF_MAX_LEN, NULL); + + buffer_strcat(wb, FUNCTION_NAME_GET_JOB_CONFIG_SCHEMA); + + if (SERVING_STREAMING(parser)) + buffer_sprintf(wb, " %s", plugin_name); + + buffer_sprintf(wb, " %s", module_name); + + dyncfg_config_t ret = call_virtual_function_blocking(parser, buffer_tostring(wb), NULL, NULL); + + buffer_free(wb); + + return ret; +} + +static dyncfg_config_t get_job_config_cb(void *usr_ctx, const char *plugin_name, const char *module_name, const char* job_name) +{ + PARSER *parser = usr_ctx; + BUFFER *wb = buffer_create(CVF_MAX_LEN, NULL); + + buffer_strcat(wb, FUNCTION_NAME_GET_JOB_CONFIG); + + if (SERVING_STREAMING(parser)) + buffer_sprintf(wb, " %s", plugin_name); + + buffer_sprintf(wb, " %s %s", module_name, job_name); + + dyncfg_config_t ret = call_virtual_function_blocking(parser, buffer_tostring(wb), NULL, NULL); + + buffer_free(wb); + + return ret; +} + +enum set_config_result set_plugin_config_cb(void *usr_ctx, const char *plugin_name, dyncfg_config_t *cfg) +{ + PARSER *parser = usr_ctx; + BUFFER *wb = buffer_create(CVF_MAX_LEN, NULL); + + buffer_strcat(wb, FUNCTION_NAME_SET_PLUGIN_CONFIG); + + if (SERVING_STREAMING(parser)) + buffer_sprintf(wb, " %s", plugin_name); + + int rc; + call_virtual_function_blocking(parser, buffer_tostring(wb), &rc, cfg->data); + + buffer_free(wb); + if(rc != DYNCFG_VFNC_RET_CFG_ACCEPTED) + return SET_CONFIG_REJECTED; + return SET_CONFIG_ACCEPTED; +} + +enum set_config_result set_module_config_cb(void *usr_ctx, const char *plugin_name, const char *module_name, dyncfg_config_t *cfg) +{ + PARSER *parser = usr_ctx; + BUFFER *wb = buffer_create(CVF_MAX_LEN, NULL); + + buffer_strcat(wb, FUNCTION_NAME_SET_MODULE_CONFIG); + + if (SERVING_STREAMING(parser)) + buffer_sprintf(wb, " %s", plugin_name); + + buffer_sprintf(wb, " %s", module_name); + + int rc; + call_virtual_function_blocking(parser, buffer_tostring(wb), &rc, cfg->data); + + buffer_free(wb); + + if(rc != DYNCFG_VFNC_RET_CFG_ACCEPTED) + return SET_CONFIG_REJECTED; + return SET_CONFIG_ACCEPTED; +} + +enum set_config_result set_job_config_cb(void *usr_ctx, const char *plugin_name, const char *module_name, const char *job_name, dyncfg_config_t *cfg) +{ + PARSER *parser = usr_ctx; + BUFFER *wb = buffer_create(CVF_MAX_LEN, NULL); + + buffer_strcat(wb, FUNCTION_NAME_SET_JOB_CONFIG); + + if (SERVING_STREAMING(parser)) + buffer_sprintf(wb, " %s", plugin_name); + + buffer_sprintf(wb, " %s %s", module_name, job_name); + + int rc; + call_virtual_function_blocking(parser, buffer_tostring(wb), &rc, cfg->data); + + buffer_free(wb); + + if(rc != DYNCFG_VFNC_RET_CFG_ACCEPTED) + return SET_CONFIG_REJECTED; + return SET_CONFIG_ACCEPTED; +} + +enum set_config_result delete_job_cb(void *usr_ctx, const char *plugin_name ,const char *module_name, const char *job_name) +{ + PARSER *parser = usr_ctx; + BUFFER *wb = buffer_create(CVF_MAX_LEN, NULL); + + buffer_strcat(wb, FUNCTION_NAME_DELETE_JOB); + + if (SERVING_STREAMING(parser)) + buffer_sprintf(wb, " %s", plugin_name); + + buffer_sprintf(wb, " %s %s", module_name, job_name); + + int rc; + call_virtual_function_blocking(parser, buffer_tostring(wb), &rc, NULL); + + buffer_free(wb); + + if(rc != DYNCFG_VFNC_RET_CFG_ACCEPTED) + return SET_CONFIG_REJECTED; + return SET_CONFIG_ACCEPTED; +} + + +PARSER_RC pluginsd_register_plugin(char **words __maybe_unused, size_t num_words __maybe_unused, PARSER *parser __maybe_unused) { + netdata_log_info("PLUGINSD: DYNCFG_ENABLE"); + + if (unlikely (num_words != 2)) + return PLUGINSD_DISABLE_PLUGIN(parser, PLUGINSD_KEYWORD_DYNCFG_ENABLE, "missing name parameter"); + + struct configurable_plugin *cfg = callocz(1, sizeof(struct configurable_plugin)); + + cfg->name = strdupz(words[1]); + cfg->set_config_cb = set_plugin_config_cb; + cfg->get_config_cb = get_plugin_config_cb; + cfg->get_config_schema_cb = get_plugin_config_schema_cb; + cfg->cb_usr_ctx = parser; + + const DICTIONARY_ITEM *di = register_plugin(parser->user.host->configurable_plugins, cfg, SERVING_PLUGINSD(parser)); + if (unlikely(di == NULL)) { + freez(cfg->name); + freez(cfg); + return PLUGINSD_DISABLE_PLUGIN(parser, PLUGINSD_KEYWORD_DYNCFG_ENABLE, "error registering plugin"); + } + + if (SERVING_PLUGINSD(parser)) { + // this is optimization for pluginsd to avoid extra dictionary lookup + // as we know which plugin is comunicating with us + parser->user.cd->cfg_dict_item = di; + parser->user.cd->configuration = cfg; + } else { + // register_plugin keeps the item acquired, so we need to release it + dictionary_acquired_item_release(parser->user.host->configurable_plugins, di); + } + + rrdpush_send_dyncfg_enable(parser->user.host, cfg->name); + + return PARSER_RC_OK; +} + +#define LOG_MSG_SIZE (1024) +#define MODULE_NAME_IDX (SERVING_PLUGINSD(parser) ? 1 : 2) +#define MODULE_TYPE_IDX (SERVING_PLUGINSD(parser) ? 2 : 3) + +PARSER_RC pluginsd_register_module(char **words __maybe_unused, size_t num_words __maybe_unused, PARSER *parser __maybe_unused) { + netdata_log_info("PLUGINSD: DYNCFG_REG_MODULE"); + + size_t expected_num_words = SERVING_PLUGINSD(parser) ? 3 : 4; + + if (unlikely(num_words != expected_num_words)) { + char log[LOG_MSG_SIZE + 1]; + snprintfz(log, LOG_MSG_SIZE, "expected %zu (got %zu) parameters: %smodule_name module_type", expected_num_words - 1, num_words - 1, SERVING_PLUGINSD(parser) ? "" : "plugin_name "); + return PLUGINSD_DISABLE_PLUGIN(parser, PLUGINSD_KEYWORD_DYNCFG_REGISTER_MODULE, log); + } + + struct configurable_plugin *plug_cfg; + const DICTIONARY_ITEM *di = NULL; + if (SERVING_PLUGINSD(parser)) { + plug_cfg = parser->user.cd->configuration; + if (unlikely(plug_cfg == NULL)) + return PLUGINSD_DISABLE_PLUGIN(parser, PLUGINSD_KEYWORD_DYNCFG_REGISTER_MODULE, "you have to enable dynamic configuration first using " PLUGINSD_KEYWORD_DYNCFG_ENABLE); + } else { + di = dictionary_get_and_acquire_item(parser->user.host->configurable_plugins, words[1]); + if (unlikely(di == NULL)) + return PLUGINSD_DISABLE_PLUGIN(parser, PLUGINSD_KEYWORD_DYNCFG_REGISTER_MODULE, "plugin not found"); + + plug_cfg = (struct configurable_plugin *)dictionary_acquired_item_value(di); + } + + struct module *mod = callocz(1, sizeof(struct module)); + + mod->type = str2_module_type(words[MODULE_TYPE_IDX]); + if (unlikely(mod->type == MOD_TYPE_UNKNOWN)) { + freez(mod); + return PLUGINSD_DISABLE_PLUGIN(parser, PLUGINSD_KEYWORD_DYNCFG_REGISTER_MODULE, "unknown module type (allowed: job_array, single)"); + } + + mod->name = strdupz(words[MODULE_NAME_IDX]); + + mod->set_config_cb = set_module_config_cb; + mod->get_config_cb = get_module_config_cb; + mod->get_config_schema_cb = get_module_config_schema_cb; + mod->config_cb_usr_ctx = parser; + + mod->get_job_config_cb = get_job_config_cb; + mod->get_job_config_schema_cb = get_job_config_schema_cb; + mod->set_job_config_cb = set_job_config_cb; + mod->delete_job_cb = delete_job_cb; + mod->job_config_cb_usr_ctx = parser; + + register_module(parser->user.host->configurable_plugins, plug_cfg, mod, SERVING_PLUGINSD(parser)); + + if (di != NULL) + dictionary_acquired_item_release(parser->user.host->configurable_plugins, di); + + rrdpush_send_dyncfg_reg_module(parser->user.host, plug_cfg->name, mod->name, mod->type); + + return PARSER_RC_OK; +} + +static inline PARSER_RC pluginsd_register_job_common(char **words __maybe_unused, size_t num_words __maybe_unused, PARSER *parser __maybe_unused, const char *plugin_name) { + const char *module_name = words[0]; + const char *job_name = words[1]; + const char *job_type_str = words[2]; + const char *flags_str = words[3]; + + long f = str2l(flags_str); + + if (f < 0) + return PLUGINSD_DISABLE_PLUGIN(parser, PLUGINSD_KEYWORD_DYNCFG_REGISTER_JOB, "invalid flags received"); + + dyncfg_job_flg_t flags = f; + + if (SERVING_PLUGINSD(parser)) + flags |= JOB_FLG_PLUGIN_PUSHED; + else + flags |= JOB_FLG_STREAMING_PUSHED; + + enum job_type job_type = dyncfg_str2job_type(job_type_str); + if (job_type == JOB_TYPE_UNKNOWN) + return PLUGINSD_DISABLE_PLUGIN(parser, PLUGINSD_KEYWORD_DYNCFG_REGISTER_JOB, "unknown job type"); + + if (SERVING_PLUGINSD(parser) && job_type == JOB_TYPE_USER) + return PLUGINSD_DISABLE_PLUGIN(parser, PLUGINSD_KEYWORD_DYNCFG_REGISTER_JOB, "plugins cannot push jobs of type \"user\" (this is allowed only in streaming)"); + + if (register_job(parser->user.host->configurable_plugins, plugin_name, module_name, job_name, job_type, flags, 0)) // ignore existing is off as this is explicitly called register job + return PLUGINSD_DISABLE_PLUGIN(parser, PLUGINSD_KEYWORD_DYNCFG_REGISTER_JOB, "error registering job"); + + rrdpush_send_dyncfg_reg_job(parser->user.host, plugin_name, module_name, job_name, job_type, flags); + + return PARSER_RC_OK; +} + +PARSER_RC pluginsd_register_job(char **words __maybe_unused, size_t num_words __maybe_unused, PARSER *parser __maybe_unused) { + size_t expected_num_words = SERVING_PLUGINSD(parser) ? 5 : 6; + + if (unlikely(num_words != expected_num_words)) { + char log[LOG_MSG_SIZE + 1]; + snprintfz(log, LOG_MSG_SIZE, "expected %zu (got %zu) parameters: %smodule_name job_name job_type", expected_num_words - 1, num_words - 1, SERVING_PLUGINSD(parser) ? "" : "plugin_name "); + return PLUGINSD_DISABLE_PLUGIN(parser, PLUGINSD_KEYWORD_DYNCFG_REGISTER_JOB, log); + } + + if (SERVING_PLUGINSD(parser)) { + return pluginsd_register_job_common(&words[1], num_words - 1, parser, parser->user.cd->configuration->name); + } + return pluginsd_register_job_common(&words[2], num_words - 2, parser, words[1]); +} + +PARSER_RC pluginsd_dyncfg_reset(char **words __maybe_unused, size_t num_words __maybe_unused, PARSER *parser __maybe_unused) { + if (unlikely(num_words != (SERVING_PLUGINSD(parser) ? 1 : 2))) + return PLUGINSD_DISABLE_PLUGIN(parser, PLUGINSD_KEYWORD_DYNCFG_RESET, SERVING_PLUGINSD(parser) ? "expected 0 parameters" : "expected 1 parameter: plugin_name"); + + if (SERVING_PLUGINSD(parser)) { + unregister_plugin(parser->user.host->configurable_plugins, parser->user.cd->cfg_dict_item); + rrdpush_send_dyncfg_reset(parser->user.host, parser->user.cd->configuration->name); + parser->user.cd->configuration = NULL; + } else { + const DICTIONARY_ITEM *di = dictionary_get_and_acquire_item(parser->user.host->configurable_plugins, words[1]); + if (unlikely(di == NULL)) + return PLUGINSD_DISABLE_PLUGIN(parser, PLUGINSD_KEYWORD_DYNCFG_RESET, "plugin not found"); + unregister_plugin(parser->user.host->configurable_plugins, di); + rrdpush_send_dyncfg_reset(parser->user.host, words[1]); + } + + return PARSER_RC_OK; +} + +static inline PARSER_RC pluginsd_job_status_common(char **words, size_t num_words, PARSER *parser, const char *plugin_name) { + int state = str2i(words[3]); + + enum job_status status = str2job_state(words[2]); + if (unlikely(SERVING_PLUGINSD(parser) && status == JOB_STATUS_UNKNOWN)) + return PLUGINSD_DISABLE_PLUGIN(parser, PLUGINSD_KEYWORD_REPORT_JOB_STATUS, "unknown job status"); + + char *message = NULL; + if (num_words == 5 && strlen(words[4]) > 0) + message = words[4]; + + const DICTIONARY_ITEM *plugin_item; + DICTIONARY *job_dict; + const DICTIONARY_ITEM *job_item = report_job_status_acq_lock(parser->user.host->configurable_plugins, &plugin_item, &job_dict, plugin_name, words[0], words[1], status, state, message); + + if (job_item != NULL) { + struct job *job = dictionary_acquired_item_value(job_item); + rrdpush_send_job_status_update(parser->user.host, plugin_name, words[0], job); + + pthread_mutex_unlock(&job->lock); + dictionary_acquired_item_release(job_dict, job_item); + dictionary_acquired_item_release(parser->user.host->configurable_plugins, plugin_item); + } + + return PARSER_RC_OK; +} + +// job_status [plugin_name if streaming] [message] +PARSER_RC pluginsd_job_status(char **words, size_t num_words, PARSER *parser) { + if (SERVING_PLUGINSD(parser)) { + if (unlikely(num_words != 5 && num_words != 6)) + return PLUGINSD_DISABLE_PLUGIN(parser, PLUGINSD_KEYWORD_REPORT_JOB_STATUS, "expected 4 or 5 parameters: module_name, job_name, status_code, state, [optional: message]"); + } else { + if (unlikely(num_words != 6 && num_words != 7)) + return PLUGINSD_DISABLE_PLUGIN(parser, PLUGINSD_KEYWORD_REPORT_JOB_STATUS, "expected 5 or 6 parameters: plugin_name, module_name, job_name, status_code, state, [optional: message]"); + } + + if (SERVING_PLUGINSD(parser)) { + return pluginsd_job_status_common(&words[1], num_words - 1, parser, parser->user.cd->configuration->name); + } + return pluginsd_job_status_common(&words[2], num_words - 2, parser, words[1]); +} + +PARSER_RC pluginsd_delete_job(char **words, size_t num_words, PARSER *parser) { + // this can confuse a bit but there is a diference between KEYWORD_DELETE_JOB and actual delete_job function + // they are of opossite direction + if (num_words != 4) + return PLUGINSD_DISABLE_PLUGIN(parser, PLUGINSD_KEYWORD_DELETE_JOB, "expected 2 parameters: plugin_name, module_name, job_name"); + + const char *plugin_name = get_word(words, num_words, 1); + const char *module_name = get_word(words, num_words, 2); + const char *job_name = get_word(words, num_words, 3); + + if (SERVING_STREAMING(parser)) + delete_job_pname(parser->user.host->configurable_plugins, plugin_name, module_name, job_name); + + // forward to parent if any + rrdpush_send_job_deleted(parser->user.host, plugin_name, module_name, job_name); + return PARSER_RC_OK; +} + +void pluginsd_dyncfg_cleanup(PARSER *parser) { + if (parser->user.cd != NULL && parser->user.cd->configuration != NULL) { + unregister_plugin(parser->user.host->configurable_plugins, parser->user.cd->cfg_dict_item); + parser->user.cd->configuration = NULL; + } else if (parser->user.host != NULL && SERVING_STREAMING(parser) && parser->user.host != localhost){ + dictionary_flush(parser->user.host->configurable_plugins); + } +} diff --git a/collectors/plugins.d/pluginsd_dyncfg.h b/collectors/plugins.d/pluginsd_dyncfg.h new file mode 100644 index 00000000000000..172fd43c241f49 --- /dev/null +++ b/collectors/plugins.d/pluginsd_dyncfg.h @@ -0,0 +1,17 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef NETDATA_PLUGINSD_DYNCFG_H +#define NETDATA_PLUGINSD_DYNCFG_H + +#include "pluginsd_internals.h" + +PARSER_RC pluginsd_register_plugin(char **words, size_t num_words, PARSER *parser); +PARSER_RC pluginsd_register_module(char **words, size_t num_words, PARSER *parser); +PARSER_RC pluginsd_register_job(char **words, size_t num_words, PARSER *parser); +PARSER_RC pluginsd_dyncfg_reset(char **words, size_t num_words, PARSER *parser); +PARSER_RC pluginsd_job_status(char **words, size_t num_words, PARSER *parser); +PARSER_RC pluginsd_delete_job(char **words, size_t num_words, PARSER *parser); + +void pluginsd_dyncfg_cleanup(PARSER *parser); + +#endif //NETDATA_PLUGINSD_DYNCFG_H diff --git a/collectors/plugins.d/pluginsd_functions.c b/collectors/plugins.d/pluginsd_functions.c new file mode 100644 index 00000000000000..5e7a48e5e87271 --- /dev/null +++ b/collectors/plugins.d/pluginsd_functions.c @@ -0,0 +1,461 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "pluginsd_functions.h" + +#define LOG_FUNCTIONS false + +// ---------------------------------------------------------------------------- +// execution of functions + +static void inflight_functions_insert_callback(const DICTIONARY_ITEM *item, void *func, void *parser_ptr) { + struct inflight_function *pf = func; + + PARSER *parser = parser_ptr; + + // leave this code as default, so that when the dictionary is destroyed this will be sent back to the caller + pf->code = HTTP_RESP_GATEWAY_TIMEOUT; + + const char *transaction = dictionary_acquired_item_name(item); + + int rc = uuid_parse_flexi(transaction, pf->transaction); + if(rc != 0) + netdata_log_error("FUNCTION: '%s': cannot parse transaction UUID", string2str(pf->function)); + + char buffer[2048 + 1]; + snprintfz(buffer, sizeof(buffer) - 1, "%s %s %d \"%s\"\n", + pf->payload ? PLUGINSD_KEYWORD_FUNCTION_PAYLOAD : PLUGINSD_KEYWORD_FUNCTION, + transaction, + pf->timeout_s, + string2str(pf->function)); + + // send the command to the plugin + ssize_t ret = send_to_plugin(buffer, parser); + + pf->sent_monotonic_ut = now_monotonic_usec(); + + if(ret < 0) { + netdata_log_error("FUNCTION '%s': failed to send it to the plugin, error %zd", string2str(pf->function), ret); + rrd_call_function_error(pf->result_body_wb, "Failed to communicate with collector", HTTP_RESP_SERVICE_UNAVAILABLE); + } + else { + internal_error(LOG_FUNCTIONS, + "FUNCTION '%s' with transaction '%s' sent to collector (%zd bytes, in %"PRIu64" usec)", + string2str(pf->function), dictionary_acquired_item_name(item), ret, + pf->sent_monotonic_ut - pf->started_monotonic_ut); + } + + if (!pf->payload) + return; + + // send the payload to the plugin + ret = send_to_plugin(pf->payload, parser); + + if(ret < 0) { + netdata_log_error("FUNCTION_PAYLOAD '%s': failed to send function to plugin, error %zd", string2str(pf->function), ret); + rrd_call_function_error(pf->result_body_wb, "Failed to communicate with collector", HTTP_RESP_SERVICE_UNAVAILABLE); + } + else { + internal_error(LOG_FUNCTIONS, + "FUNCTION_PAYLOAD '%s' with transaction '%s' sent to collector (%zd bytes, in %"PRIu64" usec)", + string2str(pf->function), dictionary_acquired_item_name(item), ret, + pf->sent_monotonic_ut - pf->started_monotonic_ut); + } + + send_to_plugin("\nFUNCTION_PAYLOAD_END\n", parser); +} + +static bool inflight_functions_conflict_callback(const DICTIONARY_ITEM *item __maybe_unused, void *func __maybe_unused, void *new_func, void *parser_ptr __maybe_unused) { + struct inflight_function *pf = new_func; + + netdata_log_error("PLUGINSD_PARSER: duplicate UUID on pending function '%s' detected. Ignoring the second one.", string2str(pf->function)); + pf->code = rrd_call_function_error(pf->result_body_wb, "This request is already in progress", HTTP_RESP_BAD_REQUEST); + pf->result.cb(pf->result_body_wb, pf->code, pf->result.data); + string_freez(pf->function); + + return false; +} + +static void delete_job_finalize(struct parser *parser __maybe_unused, struct configurable_plugin *plug, const char *fnc_sig, int code) { + if (code != DYNCFG_VFNC_RET_CFG_ACCEPTED) + return; + + char *params_local = strdupz(fnc_sig); + char *words[DYNCFG_MAX_WORDS]; + size_t words_c = quoted_strings_splitter(params_local, words, DYNCFG_MAX_WORDS, isspace_map_pluginsd); + + if (words_c != 3) { + netdata_log_error("PLUGINSD_PARSER: invalid number of parameters for delete_job"); + freez(params_local); + return; + } + + const char *module = words[1]; + const char *job = words[2]; + + delete_job(plug, module, job); + + unlink_job(plug->name, module, job); + + rrdpush_send_job_deleted(localhost, plug->name, module, job); + + freez(params_local); +} + +static void set_job_finalize(struct parser *parser __maybe_unused, struct configurable_plugin *plug __maybe_unused, const char *fnc_sig, int code) { + if (code != DYNCFG_VFNC_RET_CFG_ACCEPTED) + return; + + char *params_local = strdupz(fnc_sig); + char *words[DYNCFG_MAX_WORDS]; + size_t words_c = quoted_strings_splitter(params_local, words, DYNCFG_MAX_WORDS, isspace_map_pluginsd); + + if (words_c != 3) { + netdata_log_error("PLUGINSD_PARSER: invalid number of parameters for set_job_config"); + freez(params_local); + return; + } + + const char *module_name = get_word(words, words_c, 1); + const char *job_name = get_word(words, words_c, 2); + + if (register_job(parser->user.host->configurable_plugins, parser->user.cd->configuration->name, module_name, job_name, JOB_TYPE_USER, JOB_FLG_USER_CREATED, 1)) { + freez(params_local); + return; + } + + // only send this if it is not existing already (register_job cares for that) + rrdpush_send_dyncfg_reg_job(localhost, parser->user.cd->configuration->name, module_name, job_name, JOB_TYPE_USER, JOB_FLG_USER_CREATED); + + freez(params_local); +} + +static void inflight_functions_delete_callback(const DICTIONARY_ITEM *item __maybe_unused, void *func, void *parser_ptr) { + struct inflight_function *pf = func; + struct parser *parser = (struct parser *)parser_ptr; + + internal_error(LOG_FUNCTIONS, + "FUNCTION '%s' result of transaction '%s' received from collector (%zu bytes, request %"PRIu64" usec, response %"PRIu64" usec)", + string2str(pf->function), dictionary_acquired_item_name(item), + buffer_strlen(pf->result_body_wb), pf->sent_monotonic_ut - pf->started_monotonic_ut, now_realtime_usec() - pf->sent_monotonic_ut); + + if (pf->virtual && SERVING_PLUGINSD(parser)) { + if (pf->payload) { + if (strncmp(string2str(pf->function), FUNCTION_NAME_SET_JOB_CONFIG, strlen(FUNCTION_NAME_SET_JOB_CONFIG)) == 0) + set_job_finalize(parser, parser->user.cd->configuration, string2str(pf->function), pf->code); + dyn_conf_store_config(string2str(pf->function), pf->payload, parser->user.cd->configuration); + } else if (strncmp(string2str(pf->function), FUNCTION_NAME_DELETE_JOB, strlen(FUNCTION_NAME_DELETE_JOB)) == 0) { + delete_job_finalize(parser, parser->user.cd->configuration, string2str(pf->function), pf->code); + } + } + + pf->result.cb(pf->result_body_wb, pf->code, pf->result.data); + + string_freez(pf->function); + freez((void *)pf->payload); +} + +void pluginsd_inflight_functions_init(PARSER *parser) { + parser->inflight.functions = dictionary_create_advanced(DICT_OPTION_DONT_OVERWRITE_VALUE, &dictionary_stats_category_functions, 0); + dictionary_register_insert_callback(parser->inflight.functions, inflight_functions_insert_callback, parser); + dictionary_register_delete_callback(parser->inflight.functions, inflight_functions_delete_callback, parser); + dictionary_register_conflict_callback(parser->inflight.functions, inflight_functions_conflict_callback, parser); +} + +void pluginsd_inflight_functions_cleanup(PARSER *parser) { + dictionary_destroy(parser->inflight.functions); +} + +// ---------------------------------------------------------------------------- + +void pluginsd_inflight_functions_garbage_collect(PARSER *parser, usec_t now_ut) { + parser->inflight.smaller_monotonic_timeout_ut = 0; + struct inflight_function *pf; + dfe_start_write(parser->inflight.functions, pf) { + if (*pf->stop_monotonic_ut + RRDFUNCTIONS_TIMEOUT_EXTENSION_UT < now_ut) { + internal_error(true, + "FUNCTION '%s' removing expired transaction '%s', after %"PRIu64" usec.", + string2str(pf->function), pf_dfe.name, now_ut - pf->started_monotonic_ut); + + if(!buffer_strlen(pf->result_body_wb) || pf->code == HTTP_RESP_OK) + pf->code = rrd_call_function_error(pf->result_body_wb, + "Timeout waiting for collector response.", + HTTP_RESP_GATEWAY_TIMEOUT); + + dictionary_del(parser->inflight.functions, pf_dfe.name); + } + + else if(!parser->inflight.smaller_monotonic_timeout_ut || *pf->stop_monotonic_ut + RRDFUNCTIONS_TIMEOUT_EXTENSION_UT < parser->inflight.smaller_monotonic_timeout_ut) + parser->inflight.smaller_monotonic_timeout_ut = *pf->stop_monotonic_ut + RRDFUNCTIONS_TIMEOUT_EXTENSION_UT; + } + dfe_done(pf); +} + +// ---------------------------------------------------------------------------- + +static void pluginsd_function_cancel(void *data) { + struct inflight_function *look_for = data, *t; + + bool sent = false; + dfe_start_read(look_for->parser->inflight.functions, t) { + if(look_for == t) { + const char *transaction = t_dfe.name; + + internal_error(true, "PLUGINSD: sending function cancellation to plugin for transaction '%s'", transaction); + + char buffer[2048 + 1]; + snprintfz(buffer, sizeof(buffer) - 1, "%s %s\n", + PLUGINSD_KEYWORD_FUNCTION_CANCEL, + transaction); + + // send the command to the plugin + ssize_t ret = send_to_plugin(buffer, t->parser); + if(ret < 0) + sent = true; + + break; + } + } + dfe_done(t); + + if(sent <= 0) + nd_log(NDLS_DAEMON, NDLP_DEBUG, + "PLUGINSD: FUNCTION_CANCEL request didn't match any pending function requests in pluginsd.d."); +} + +static void pluginsd_function_progress_to_plugin(void *data) { + struct inflight_function *look_for = data, *t; + + bool sent = false; + dfe_start_read(look_for->parser->inflight.functions, t) { + if(look_for == t) { + const char *transaction = t_dfe.name; + + internal_error(true, "PLUGINSD: sending function progress to plugin for transaction '%s'", transaction); + + char buffer[2048 + 1]; + snprintfz(buffer, sizeof(buffer) - 1, "%s %s\n", + PLUGINSD_KEYWORD_FUNCTION_PROGRESS, + transaction); + + // send the command to the plugin + ssize_t ret = send_to_plugin(buffer, t->parser); + if(ret < 0) + sent = true; + + break; + } + } + dfe_done(t); + + if(sent <= 0) + nd_log(NDLS_DAEMON, NDLP_DEBUG, + "PLUGINSD: FUNCTION_PROGRESS request didn't match any pending function requests in pluginsd.d."); +} + +// this is the function called from +// rrd_call_function_and_wait() and rrd_call_function_async() +static int pluginsd_function_execute_cb(uuid_t *transaction, BUFFER *result_body_wb, + usec_t *stop_monotonic_ut, const char *function, + void *execute_cb_data, + rrd_function_result_callback_t result_cb, void *result_cb_data, + rrd_function_progress_cb_t progress_cb, void *progress_cb_data, + rrd_function_is_cancelled_cb_t is_cancelled_cb __maybe_unused, + void *is_cancelled_cb_data __maybe_unused, + rrd_function_register_canceller_cb_t register_canceller_cb, + void *register_canceller_cb_data, + rrd_function_register_progresser_cb_t register_progresser_cb, + void *register_progresser_cb_data) { + PARSER *parser = execute_cb_data; + + usec_t now_ut = now_monotonic_usec(); + + int timeout_s = (*stop_monotonic_ut - now_ut + USEC_PER_SEC / 2) / USEC_PER_SEC; + + struct inflight_function tmp = { + .started_monotonic_ut = now_ut, + .stop_monotonic_ut = stop_monotonic_ut, + .result_body_wb = result_body_wb, + .timeout_s = timeout_s, + .function = string_strdupz(function), + .payload = NULL, + .parser = parser, + + .result = { + .cb = result_cb, + .data = result_cb_data, + }, + .progress = { + .cb = progress_cb, + .data = progress_cb_data, + }, + }; + uuid_copy(tmp.transaction, *transaction); + + char transaction_str[UUID_COMPACT_STR_LEN]; + uuid_unparse_lower_compact(tmp.transaction, transaction_str); + + dictionary_write_lock(parser->inflight.functions); + + // if there is any error, our dictionary callbacks will call the caller callback to notify + // the caller about the error - no need for error handling here. + void *t = dictionary_set(parser->inflight.functions, transaction_str, &tmp, sizeof(struct inflight_function)); + if(register_canceller_cb) + register_canceller_cb(register_canceller_cb_data, pluginsd_function_cancel, t); + + if(register_progresser_cb && (parser->repertoire == PARSER_INIT_PLUGINSD || + (parser->repertoire == PARSER_INIT_STREAMING && stream_has_capability(&parser->user, STREAM_CAP_PROGRESS)))) + register_progresser_cb(register_progresser_cb_data, pluginsd_function_progress_to_plugin, t); + + if(!parser->inflight.smaller_monotonic_timeout_ut || *tmp.stop_monotonic_ut + RRDFUNCTIONS_TIMEOUT_EXTENSION_UT < parser->inflight.smaller_monotonic_timeout_ut) + parser->inflight.smaller_monotonic_timeout_ut = *tmp.stop_monotonic_ut + RRDFUNCTIONS_TIMEOUT_EXTENSION_UT; + + // garbage collect stale inflight functions + if(parser->inflight.smaller_monotonic_timeout_ut < now_ut) + pluginsd_inflight_functions_garbage_collect(parser, now_ut); + + dictionary_write_unlock(parser->inflight.functions); + + return HTTP_RESP_OK; +} + +PARSER_RC pluginsd_function(char **words, size_t num_words, PARSER *parser) { + // a plugin or a child is registering a function + + bool global = false; + size_t i = 1; + if(num_words >= 2 && strcmp(get_word(words, num_words, 1), "GLOBAL") == 0) { + i++; + global = true; + } + + char *name = get_word(words, num_words, i++); + char *timeout_str = get_word(words, num_words, i++); + char *help = get_word(words, num_words, i++); + char *tags = get_word(words, num_words, i++); + char *access_str = get_word(words, num_words, i++); + char *priority_str = get_word(words, num_words, i++); + + RRDHOST *host = pluginsd_require_scope_host(parser, PLUGINSD_KEYWORD_FUNCTION); + if(!host) return PARSER_RC_ERROR; + + RRDSET *st = (global)? NULL: pluginsd_require_scope_chart(parser, PLUGINSD_KEYWORD_FUNCTION, PLUGINSD_KEYWORD_CHART); + if(!st) global = true; + + if (unlikely(!timeout_str || !name || !help || (!global && !st))) { + netdata_log_error("PLUGINSD: 'host:%s/chart:%s' got a FUNCTION, without providing the required data (global = '%s', name = '%s', timeout = '%s', help = '%s'). Ignoring it.", + rrdhost_hostname(host), + st?rrdset_id(st):"(unset)", + global?"yes":"no", + name?name:"(unset)", + timeout_str ? timeout_str : "(unset)", + help?help:"(unset)" + ); + return PARSER_RC_ERROR; + } + + int timeout_s = PLUGINS_FUNCTIONS_TIMEOUT_DEFAULT; + if (timeout_str && *timeout_str) { + timeout_s = str2i(timeout_str); + if (unlikely(timeout_s <= 0)) + timeout_s = PLUGINS_FUNCTIONS_TIMEOUT_DEFAULT; + } + + int priority = RRDFUNCTIONS_PRIORITY_DEFAULT; + if(priority_str && *priority_str) { + priority = str2i(priority_str); + if(priority <= 0) + priority = RRDFUNCTIONS_PRIORITY_DEFAULT; + } + + rrd_function_add(host, st, name, timeout_s, priority, help, tags, + http_access2id(access_str), false, + pluginsd_function_execute_cb, parser); + + parser->user.data_collections_count++; + + return PARSER_RC_OK; +} + +static void pluginsd_function_result_end(struct parser *parser, void *action_data) { + STRING *key = action_data; + if(key) + dictionary_del(parser->inflight.functions, string2str(key)); + string_freez(key); + + parser->user.data_collections_count++; +} + +static inline struct inflight_function *inflight_function_find(PARSER *parser, const char *transaction) { + struct inflight_function *pf = NULL; + + if(transaction && *transaction) + pf = (struct inflight_function *)dictionary_get(parser->inflight.functions, transaction); + + if(!pf) + netdata_log_error("got a " PLUGINSD_KEYWORD_FUNCTION_RESULT_BEGIN " for transaction '%s', but the transaction is not found.", transaction ? transaction : "(unset)"); + + return pf; +} + +PARSER_RC pluginsd_function_result_begin(char **words, size_t num_words, PARSER *parser) { + char *transaction = get_word(words, num_words, 1); + char *status = get_word(words, num_words, 2); + char *format = get_word(words, num_words, 3); + char *expires = get_word(words, num_words, 4); + + if (unlikely(!transaction || !*transaction || !status || !*status || !format || !*format || !expires || !*expires)) { + netdata_log_error("got a " PLUGINSD_KEYWORD_FUNCTION_RESULT_BEGIN " without providing the required data (key = '%s', status = '%s', format = '%s', expires = '%s')." + , transaction ? transaction : "(unset)" + , status ? status : "(unset)" + , format ? format : "(unset)" + , expires ? expires : "(unset)" + ); + } + + int code = (status && *status) ? str2i(status) : 0; + if (code <= 0) + code = HTTP_RESP_BACKEND_RESPONSE_INVALID; + + time_t expiration = (expires && *expires) ? str2l(expires) : 0; + + struct inflight_function *pf = inflight_function_find(parser, transaction); + if(pf) { + if(format && *format) + pf->result_body_wb->content_type = functions_format_to_content_type(format); + + pf->code = code; + + pf->result_body_wb->expires = expiration; + if(expiration <= now_realtime_sec()) + buffer_no_cacheable(pf->result_body_wb); + else + buffer_cacheable(pf->result_body_wb); + } + + parser->defer.response = (pf) ? pf->result_body_wb : NULL; + parser->defer.end_keyword = PLUGINSD_KEYWORD_FUNCTION_RESULT_END; + parser->defer.action = pluginsd_function_result_end; + parser->defer.action_data = string_strdupz(transaction); // it is ok is key is NULL + parser->flags |= PARSER_DEFER_UNTIL_KEYWORD; + + return PARSER_RC_OK; +} + +PARSER_RC pluginsd_function_progress(char **words, size_t num_words, PARSER *parser) { + size_t i = 1; + + char *transaction = get_word(words, num_words, i++); + char *done_str = get_word(words, num_words, i++); + char *all_str = get_word(words, num_words, i++); + + struct inflight_function *pf = inflight_function_find(parser, transaction); + if(pf) { + size_t done = done_str && *done_str ? str2u(done_str) : 0; + size_t all = all_str && *all_str ? str2u(all_str) : 0; + + if(pf->progress.cb) + pf->progress.cb(pf->progress.data, done, all); + } + + return PARSER_RC_OK; +} diff --git a/collectors/plugins.d/pluginsd_functions.h b/collectors/plugins.d/pluginsd_functions.h new file mode 100644 index 00000000000000..f866efdae89346 --- /dev/null +++ b/collectors/plugins.d/pluginsd_functions.h @@ -0,0 +1,45 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef NETDATA_PLUGINSD_FUNCTIONS_H +#define NETDATA_PLUGINSD_FUNCTIONS_H + +#include "pluginsd_internals.h" + +struct inflight_function { + uuid_t transaction; + + int code; + int timeout_s; + STRING *function; + BUFFER *result_body_wb; + usec_t *stop_monotonic_ut; // pointer to caller data + usec_t started_monotonic_ut; + usec_t sent_monotonic_ut; + const char *payload; + PARSER *parser; + bool virtual; + + struct { + rrd_function_result_callback_t cb; + void *data; + } result; + + struct { + rrd_function_progress_cb_t cb; + void *data; + } progress; + + struct { + usec_t stop_monotonic_ut; + } dyncfg; +}; + +PARSER_RC pluginsd_function(char **words, size_t num_words, PARSER *parser); +PARSER_RC pluginsd_function_result_begin(char **words, size_t num_words, PARSER *parser); +PARSER_RC pluginsd_function_progress(char **words, size_t num_words, PARSER *parser); + +void pluginsd_inflight_functions_init(PARSER *parser); +void pluginsd_inflight_functions_cleanup(PARSER *parser); +void pluginsd_inflight_functions_garbage_collect(PARSER *parser, usec_t now_ut); + +#endif //NETDATA_PLUGINSD_FUNCTIONS_H diff --git a/collectors/plugins.d/pluginsd_internals.c b/collectors/plugins.d/pluginsd_internals.c new file mode 100644 index 00000000000000..f08428d8fdbd93 --- /dev/null +++ b/collectors/plugins.d/pluginsd_internals.c @@ -0,0 +1,121 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "pluginsd_internals.h" + +ssize_t send_to_plugin(const char *txt, void *data) { + PARSER *parser = data; + + if(!txt || !*txt) + return 0; + +#ifdef ENABLE_H2O + if(parser->h2o_ctx) + return h2o_stream_write(parser->h2o_ctx, txt, strlen(txt)); +#endif + + errno = 0; + spinlock_lock(&parser->writer.spinlock); + ssize_t bytes = -1; + +#ifdef ENABLE_HTTPS + NETDATA_SSL *ssl = parser->ssl_output; + if(ssl) { + + if(SSL_connection(ssl)) + bytes = netdata_ssl_write(ssl, (void *) txt, strlen(txt)); + + else + netdata_log_error("PLUGINSD: cannot send command (SSL)"); + + spinlock_unlock(&parser->writer.spinlock); + return bytes; + } +#endif + + if(parser->fp_output) { + + bytes = fprintf(parser->fp_output, "%s", txt); + if(bytes <= 0) { + netdata_log_error("PLUGINSD: cannot send command (FILE)"); + bytes = -2; + } + else + fflush(parser->fp_output); + + spinlock_unlock(&parser->writer.spinlock); + return bytes; + } + + if(parser->fd != -1) { + bytes = 0; + ssize_t total = (ssize_t)strlen(txt); + ssize_t sent; + + do { + sent = write(parser->fd, &txt[bytes], total - bytes); + if(sent <= 0) { + netdata_log_error("PLUGINSD: cannot send command (fd)"); + spinlock_unlock(&parser->writer.spinlock); + return -3; + } + bytes += sent; + } + while(bytes < total); + + spinlock_unlock(&parser->writer.spinlock); + return (int)bytes; + } + + spinlock_unlock(&parser->writer.spinlock); + netdata_log_error("PLUGINSD: cannot send command (no output socket/pipe/file given to plugins.d parser)"); + return -4; +} + +PARSER_RC PLUGINSD_DISABLE_PLUGIN(PARSER *parser, const char *keyword, const char *msg) { + parser->user.enabled = 0; + + if(keyword && msg) { + nd_log_limit_static_global_var(erl, 1, 0); + nd_log_limit(&erl, NDLS_COLLECTORS, NDLP_INFO, + "PLUGINSD: keyword %s: %s", keyword, msg); + } + + return PARSER_RC_ERROR; +} + +void pluginsd_keywords_init(PARSER *parser, PARSER_REPERTOIRE repertoire) { + parser_init_repertoire(parser, repertoire); + + if (repertoire & (PARSER_INIT_PLUGINSD | PARSER_INIT_STREAMING)) + pluginsd_inflight_functions_init(parser); +} + +void parser_destroy(PARSER *parser) { + if (unlikely(!parser)) + return; + + pluginsd_dyncfg_cleanup(parser); + pluginsd_inflight_functions_cleanup(parser); + + freez(parser); +} + + +PARSER *parser_init(struct parser_user_object *user, FILE *fp_input, FILE *fp_output, int fd, + PARSER_INPUT_TYPE flags, void *ssl __maybe_unused) { + PARSER *parser; + + parser = callocz(1, sizeof(*parser)); + if(user) + parser->user = *user; + parser->fd = fd; + parser->fp_input = fp_input; + parser->fp_output = fp_output; +#ifdef ENABLE_HTTPS + parser->ssl_output = ssl; +#endif + parser->flags = flags; + + spinlock_init(&parser->writer.spinlock); + return parser; +} diff --git a/collectors/plugins.d/pluginsd_internals.h b/collectors/plugins.d/pluginsd_internals.h new file mode 100644 index 00000000000000..31db02544cd0a4 --- /dev/null +++ b/collectors/plugins.d/pluginsd_internals.h @@ -0,0 +1,355 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef NETDATA_PLUGINSD_INTERNALS_H +#define NETDATA_PLUGINSD_INTERNALS_H + +#include "pluginsd_parser.h" +#include "pluginsd_functions.h" +#include "pluginsd_dyncfg.h" +#include "pluginsd_replication.h" + +#define SERVING_STREAMING(parser) ((parser)->repertoire == PARSER_INIT_STREAMING) +#define SERVING_PLUGINSD(parser) ((parser)->repertoire == PARSER_INIT_PLUGINSD) + +PARSER_RC PLUGINSD_DISABLE_PLUGIN(PARSER *parser, const char *keyword, const char *msg); + +ssize_t send_to_plugin(const char *txt, void *data); + +static inline RRDHOST *pluginsd_require_scope_host(PARSER *parser, const char *cmd) { + RRDHOST *host = parser->user.host; + + if(unlikely(!host)) + netdata_log_error("PLUGINSD: command %s requires a host, but is not set.", cmd); + + return host; +} + +static inline RRDSET *pluginsd_require_scope_chart(PARSER *parser, const char *cmd, const char *parent_cmd) { + RRDSET *st = parser->user.st; + + if(unlikely(!st)) + netdata_log_error("PLUGINSD: command %s requires a chart defined via command %s, but is not set.", cmd, parent_cmd); + + return st; +} + +static inline RRDSET *pluginsd_get_scope_chart(PARSER *parser) { + return parser->user.st; +} + +static inline void pluginsd_lock_rrdset_data_collection(PARSER *parser) { + if(parser->user.st && !parser->user.v2.locked_data_collection) { + spinlock_lock(&parser->user.st->data_collection_lock); + parser->user.v2.locked_data_collection = true; + } +} + +static inline bool pluginsd_unlock_rrdset_data_collection(PARSER *parser) { + if(parser->user.st && parser->user.v2.locked_data_collection) { + spinlock_unlock(&parser->user.st->data_collection_lock); + parser->user.v2.locked_data_collection = false; + return true; + } + + return false; +} + +static inline void pluginsd_unlock_previous_scope_chart(PARSER *parser, const char *keyword, bool stale) { + if(unlikely(pluginsd_unlock_rrdset_data_collection(parser))) { + if(stale) + netdata_log_error("PLUGINSD: 'host:%s/chart:%s/' stale data collection lock found during %s; it has been unlocked", + rrdhost_hostname(parser->user.st->rrdhost), + rrdset_id(parser->user.st), + keyword); + } + + if(unlikely(parser->user.v2.ml_locked)) { + ml_chart_update_end(parser->user.st); + parser->user.v2.ml_locked = false; + + if(stale) + netdata_log_error("PLUGINSD: 'host:%s/chart:%s/' stale ML lock found during %s, it has been unlocked", + rrdhost_hostname(parser->user.st->rrdhost), + rrdset_id(parser->user.st), + keyword); + } +} + +static inline void pluginsd_clear_scope_chart(PARSER *parser, const char *keyword) { + pluginsd_unlock_previous_scope_chart(parser, keyword, true); + + if(parser->user.cleanup_slots && parser->user.st) + rrdset_pluginsd_receive_unslot(parser->user.st); + + parser->user.st = NULL; + parser->user.cleanup_slots = false; +} + +static inline bool pluginsd_set_scope_chart(PARSER *parser, RRDSET *st, const char *keyword) { + RRDSET *old_st = parser->user.st; + pid_t old_collector_tid = (old_st) ? old_st->pluginsd.collector_tid : 0; + pid_t my_collector_tid = gettid(); + + if(unlikely(old_collector_tid)) { + if(old_collector_tid != my_collector_tid) { + nd_log_limit_static_global_var(erl, 1, 0); + nd_log_limit(&erl, NDLS_COLLECTORS, NDLP_WARNING, + "PLUGINSD: keyword %s: 'host:%s/chart:%s' is collected twice (my tid %d, other collector tid %d)", + keyword ? keyword : "UNKNOWN", + rrdhost_hostname(st->rrdhost), rrdset_id(st), + my_collector_tid, old_collector_tid); + + return false; + } + + old_st->pluginsd.collector_tid = 0; + } + + st->pluginsd.collector_tid = my_collector_tid; + + pluginsd_clear_scope_chart(parser, keyword); + + st->pluginsd.pos = 0; + parser->user.st = st; + parser->user.cleanup_slots = false; + + return true; +} + +static inline void pluginsd_rrddim_put_to_slot(PARSER *parser, RRDSET *st, RRDDIM *rd, ssize_t slot, bool obsolete) { + size_t wanted_size = st->pluginsd.size; + + if(slot >= 1) { + st->pluginsd.dims_with_slots = true; + wanted_size = slot; + } + else { + st->pluginsd.dims_with_slots = false; + wanted_size = dictionary_entries(st->rrddim_root_index); + } + + if(wanted_size > st->pluginsd.size) { + st->pluginsd.prd_array = reallocz(st->pluginsd.prd_array, wanted_size * sizeof(struct pluginsd_rrddim)); + + // initialize the empty slots + for(ssize_t i = (ssize_t) wanted_size - 1; i >= (ssize_t) st->pluginsd.size; i--) { + st->pluginsd.prd_array[i].rda = NULL; + st->pluginsd.prd_array[i].rd = NULL; + st->pluginsd.prd_array[i].id = NULL; + } + + st->pluginsd.size = wanted_size; + } + + if(st->pluginsd.dims_with_slots) { + struct pluginsd_rrddim *prd = &st->pluginsd.prd_array[slot - 1]; + + if(prd->rd != rd) { + prd->rda = rrddim_find_and_acquire(st, string2str(rd->id)); + prd->rd = rrddim_acquired_to_rrddim(prd->rda); + prd->id = string2str(prd->rd->id); + } + + if(obsolete) + parser->user.cleanup_slots = true; + } +} + +static inline RRDDIM *pluginsd_acquire_dimension(RRDHOST *host, RRDSET *st, const char *dimension, ssize_t slot, const char *cmd) { + if (unlikely(!dimension || !*dimension)) { + netdata_log_error("PLUGINSD: 'host:%s/chart:%s' got a %s, without a dimension.", + rrdhost_hostname(host), rrdset_id(st), cmd); + return NULL; + } + + if (unlikely(!st->pluginsd.size)) { + netdata_log_error("PLUGINSD: 'host:%s/chart:%s' got a %s, but the chart has no dimensions.", + rrdhost_hostname(host), rrdset_id(st), cmd); + return NULL; + } + + struct pluginsd_rrddim *prd; + RRDDIM *rd; + + if(likely(st->pluginsd.dims_with_slots)) { + // caching with slots + + if(unlikely(slot < 1 || slot > st->pluginsd.size)) { + netdata_log_error("PLUGINSD: 'host:%s/chart:%s' got a %s with slot %zd, but slots in the range [1 - %u] are expected.", + rrdhost_hostname(host), rrdset_id(st), cmd, slot, st->pluginsd.size); + return NULL; + } + + prd = &st->pluginsd.prd_array[slot - 1]; + + rd = prd->rd; + if(likely(rd)) { +#ifdef NETDATA_INTERNAL_CHECKS + if(strcmp(prd->id, dimension) != 0) { + ssize_t t; + for(t = 0; t < st->pluginsd.size ;t++) { + if (strcmp(st->pluginsd.prd_array[t].id, dimension) == 0) + break; + } + if(t >= st->pluginsd.size) + t = -1; + + internal_fatal(true, + "PLUGINSD: expected to find dimension '%s' on slot %zd, but found '%s', " + "the right slot is %zd", + dimension, slot, prd->id, t); + } +#endif + return rd; + } + } + else { + // caching without slots + + if(unlikely(st->pluginsd.pos >= st->pluginsd.size)) + st->pluginsd.pos = 0; + + prd = &st->pluginsd.prd_array[st->pluginsd.pos++]; + + rd = prd->rd; + if(likely(rd)) { + const char *id = prd->id; + + if(strcmp(id, dimension) == 0) { + // we found it cached + return rd; + } + else { + // the cached one is not good for us + rrddim_acquired_release(prd->rda); + prd->rda = NULL; + prd->rd = NULL; + prd->id = NULL; + } + } + } + + // we need to find the dimension and set it to prd + + RRDDIM_ACQUIRED *rda = rrddim_find_and_acquire(st, dimension); + if (unlikely(!rda)) { + netdata_log_error("PLUGINSD: 'host:%s/chart:%s/dim:%s' got a %s but dimension does not exist.", + rrdhost_hostname(host), rrdset_id(st), dimension, cmd); + + return NULL; + } + + prd->rda = rda; + prd->rd = rd = rrddim_acquired_to_rrddim(rda); + prd->id = string2str(rd->id); + + return rd; +} + +static inline RRDSET *pluginsd_find_chart(RRDHOST *host, const char *chart, const char *cmd) { + if (unlikely(!chart || !*chart)) { + netdata_log_error("PLUGINSD: 'host:%s' got a %s without a chart id.", + rrdhost_hostname(host), cmd); + return NULL; + } + + RRDSET *st = rrdset_find(host, chart); + if (unlikely(!st)) + netdata_log_error("PLUGINSD: 'host:%s/chart:%s' got a %s but chart does not exist.", + rrdhost_hostname(host), chart, cmd); + + return st; +} + +static inline ssize_t pluginsd_parse_rrd_slot(char **words, size_t num_words) { + ssize_t slot = -1; + char *id = get_word(words, num_words, 1); + if(id && id[0] == PLUGINSD_KEYWORD_SLOT[0] && id[1] == PLUGINSD_KEYWORD_SLOT[1] && + id[2] == PLUGINSD_KEYWORD_SLOT[2] && id[3] == PLUGINSD_KEYWORD_SLOT[3] && id[4] == ':') { + slot = (ssize_t) str2ull_encoded(&id[5]); + if(slot < 0) slot = 0; // to make the caller increment its idx of the words + } + + return slot; +} + +static inline void pluginsd_rrdset_cache_put_to_slot(PARSER *parser, RRDSET *st, ssize_t slot, bool obsolete) { + // clean possible old cached data + rrdset_pluginsd_receive_unslot(st); + + if(unlikely(slot < 1 || slot >= INT32_MAX)) + return; + + RRDHOST *host = st->rrdhost; + + if(unlikely((size_t)slot > host->rrdpush.receive.pluginsd_chart_slots.size)) { + spinlock_lock(&host->rrdpush.receive.pluginsd_chart_slots.spinlock); + size_t old_slots = host->rrdpush.receive.pluginsd_chart_slots.size; + size_t new_slots = (old_slots < PLUGINSD_MIN_RRDSET_POINTERS_CACHE) ? PLUGINSD_MIN_RRDSET_POINTERS_CACHE : old_slots * 2; + + if(new_slots < (size_t)slot) + new_slots = slot; + + host->rrdpush.receive.pluginsd_chart_slots.array = + reallocz(host->rrdpush.receive.pluginsd_chart_slots.array, new_slots * sizeof(RRDSET *)); + + for(size_t i = old_slots; i < new_slots ;i++) + host->rrdpush.receive.pluginsd_chart_slots.array[i] = NULL; + + host->rrdpush.receive.pluginsd_chart_slots.size = new_slots; + spinlock_unlock(&host->rrdpush.receive.pluginsd_chart_slots.spinlock); + } + + host->rrdpush.receive.pluginsd_chart_slots.array[slot - 1] = st; + st->pluginsd.last_slot = (int32_t)slot - 1; + parser->user.cleanup_slots = obsolete; +} + +static inline RRDSET *pluginsd_rrdset_cache_get_from_slot(PARSER *parser, RRDHOST *host, const char *id, ssize_t slot, const char *keyword) { + if(unlikely(slot < 1 || (size_t)slot > host->rrdpush.receive.pluginsd_chart_slots.size)) + return pluginsd_find_chart(host, id, keyword); + + RRDSET *st = host->rrdpush.receive.pluginsd_chart_slots.array[slot - 1]; + + if(!st) { + st = pluginsd_find_chart(host, id, keyword); + if(st) + pluginsd_rrdset_cache_put_to_slot(parser, st, slot, rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE)); + } + else { + internal_fatal(string_strcmp(st->id, id) != 0, + "PLUGINSD: wrong chart in slot %zd, expected '%s', found '%s'", + slot - 1, id, string2str(st->id)); + } + + return st; +} + +static inline SN_FLAGS pluginsd_parse_storage_number_flags(const char *flags_str) { + SN_FLAGS flags = SN_FLAG_NONE; + + char c; + while ((c = *flags_str++)) { + switch (c) { + case 'A': + flags |= SN_FLAG_NOT_ANOMALOUS; + break; + + case 'R': + flags |= SN_FLAG_RESET; + break; + + case 'E': + flags = SN_EMPTY_SLOT; + return flags; + + default: + internal_error(true, "Unknown SN_FLAGS flag '%c'", c); + break; + } + } + + return flags; +} + +#endif //NETDATA_PLUGINSD_INTERNALS_H diff --git a/collectors/plugins.d/pluginsd_parser.c b/collectors/plugins.d/pluginsd_parser.c index 0620b69cf8ee5d..38660bcdb773c7 100644 --- a/collectors/plugins.d/pluginsd_parser.c +++ b/collectors/plugins.d/pluginsd_parser.c @@ -1,402 +1,6 @@ // SPDX-License-Identifier: GPL-3.0-or-later -#include "pluginsd_parser.h" - -#define LOG_FUNCTIONS false - -#define SERVING_STREAMING(parser) ((parser)->repertoire == PARSER_INIT_STREAMING) -#define SERVING_PLUGINSD(parser) ((parser)->repertoire == PARSER_INIT_PLUGINSD) - -static ssize_t send_to_plugin(const char *txt, void *data) { - PARSER *parser = data; - - if(!txt || !*txt) - return 0; - -#ifdef ENABLE_H2O - if(parser->h2o_ctx) - return h2o_stream_write(parser->h2o_ctx, txt, strlen(txt)); -#endif - - errno = 0; - spinlock_lock(&parser->writer.spinlock); - ssize_t bytes = -1; - -#ifdef ENABLE_HTTPS - NETDATA_SSL *ssl = parser->ssl_output; - if(ssl) { - - if(SSL_connection(ssl)) - bytes = netdata_ssl_write(ssl, (void *) txt, strlen(txt)); - - else - netdata_log_error("PLUGINSD: cannot send command (SSL)"); - - spinlock_unlock(&parser->writer.spinlock); - return bytes; - } -#endif - - if(parser->fp_output) { - - bytes = fprintf(parser->fp_output, "%s", txt); - if(bytes <= 0) { - netdata_log_error("PLUGINSD: cannot send command (FILE)"); - bytes = -2; - } - else - fflush(parser->fp_output); - - spinlock_unlock(&parser->writer.spinlock); - return bytes; - } - - if(parser->fd != -1) { - bytes = 0; - ssize_t total = (ssize_t)strlen(txt); - ssize_t sent; - - do { - sent = write(parser->fd, &txt[bytes], total - bytes); - if(sent <= 0) { - netdata_log_error("PLUGINSD: cannot send command (fd)"); - spinlock_unlock(&parser->writer.spinlock); - return -3; - } - bytes += sent; - } - while(bytes < total); - - spinlock_unlock(&parser->writer.spinlock); - return (int)bytes; - } - - spinlock_unlock(&parser->writer.spinlock); - netdata_log_error("PLUGINSD: cannot send command (no output socket/pipe/file given to plugins.d parser)"); - return -4; -} - -static inline RRDHOST *pluginsd_require_scope_host(PARSER *parser, const char *cmd) { - RRDHOST *host = parser->user.host; - - if(unlikely(!host)) - netdata_log_error("PLUGINSD: command %s requires a host, but is not set.", cmd); - - return host; -} - -static inline RRDSET *pluginsd_require_scope_chart(PARSER *parser, const char *cmd, const char *parent_cmd) { - RRDSET *st = parser->user.st; - - if(unlikely(!st)) - netdata_log_error("PLUGINSD: command %s requires a chart defined via command %s, but is not set.", cmd, parent_cmd); - - return st; -} - -static inline RRDSET *pluginsd_get_scope_chart(PARSER *parser) { - return parser->user.st; -} - -static inline void pluginsd_lock_rrdset_data_collection(PARSER *parser) { - if(parser->user.st && !parser->user.v2.locked_data_collection) { - spinlock_lock(&parser->user.st->data_collection_lock); - parser->user.v2.locked_data_collection = true; - } -} - -static inline bool pluginsd_unlock_rrdset_data_collection(PARSER *parser) { - if(parser->user.st && parser->user.v2.locked_data_collection) { - spinlock_unlock(&parser->user.st->data_collection_lock); - parser->user.v2.locked_data_collection = false; - return true; - } - - return false; -} - -static inline void pluginsd_unlock_previous_scope_chart(PARSER *parser, const char *keyword, bool stale) { - if(unlikely(pluginsd_unlock_rrdset_data_collection(parser))) { - if(stale) - netdata_log_error("PLUGINSD: 'host:%s/chart:%s/' stale data collection lock found during %s; it has been unlocked", - rrdhost_hostname(parser->user.st->rrdhost), - rrdset_id(parser->user.st), - keyword); - } - - if(unlikely(parser->user.v2.ml_locked)) { - ml_chart_update_end(parser->user.st); - parser->user.v2.ml_locked = false; - - if(stale) - netdata_log_error("PLUGINSD: 'host:%s/chart:%s/' stale ML lock found during %s, it has been unlocked", - rrdhost_hostname(parser->user.st->rrdhost), - rrdset_id(parser->user.st), - keyword); - } -} - -static inline void pluginsd_clear_scope_chart(PARSER *parser, const char *keyword) { - pluginsd_unlock_previous_scope_chart(parser, keyword, true); - - if(parser->user.cleanup_slots && parser->user.st) - rrdset_pluginsd_receive_unslot(parser->user.st); - - parser->user.st = NULL; - parser->user.cleanup_slots = false; -} - -static inline bool pluginsd_set_scope_chart(PARSER *parser, RRDSET *st, const char *keyword) { - RRDSET *old_st = parser->user.st; - pid_t old_collector_tid = (old_st) ? old_st->pluginsd.collector_tid : 0; - pid_t my_collector_tid = gettid(); - - if(unlikely(old_collector_tid)) { - if(old_collector_tid != my_collector_tid) { - nd_log_limit_static_global_var(erl, 1, 0); - nd_log_limit(&erl, NDLS_COLLECTORS, NDLP_WARNING, - "PLUGINSD: keyword %s: 'host:%s/chart:%s' is collected twice (my tid %d, other collector tid %d)", - keyword ? keyword : "UNKNOWN", - rrdhost_hostname(st->rrdhost), rrdset_id(st), - my_collector_tid, old_collector_tid); - - return false; - } - - old_st->pluginsd.collector_tid = 0; - } - - st->pluginsd.collector_tid = my_collector_tid; - - pluginsd_clear_scope_chart(parser, keyword); - - st->pluginsd.pos = 0; - parser->user.st = st; - parser->user.cleanup_slots = false; - - return true; -} - -static inline void pluginsd_rrddim_put_to_slot(PARSER *parser, RRDSET *st, RRDDIM *rd, ssize_t slot, bool obsolete) { - size_t wanted_size = st->pluginsd.size; - - if(slot >= 1) { - st->pluginsd.dims_with_slots = true; - wanted_size = slot; - } - else { - st->pluginsd.dims_with_slots = false; - wanted_size = dictionary_entries(st->rrddim_root_index); - } - - if(wanted_size > st->pluginsd.size) { - st->pluginsd.prd_array = reallocz(st->pluginsd.prd_array, wanted_size * sizeof(struct pluginsd_rrddim)); - - // initialize the empty slots - for(ssize_t i = (ssize_t) wanted_size - 1; i >= (ssize_t) st->pluginsd.size; i--) { - st->pluginsd.prd_array[i].rda = NULL; - st->pluginsd.prd_array[i].rd = NULL; - st->pluginsd.prd_array[i].id = NULL; - } - - st->pluginsd.size = wanted_size; - } - - if(st->pluginsd.dims_with_slots) { - struct pluginsd_rrddim *prd = &st->pluginsd.prd_array[slot - 1]; - - if(prd->rd != rd) { - prd->rda = rrddim_find_and_acquire(st, string2str(rd->id)); - prd->rd = rrddim_acquired_to_rrddim(prd->rda); - prd->id = string2str(prd->rd->id); - } - - if(obsolete) - parser->user.cleanup_slots = true; - } -} - -static inline RRDDIM *pluginsd_acquire_dimension(RRDHOST *host, RRDSET *st, const char *dimension, ssize_t slot, const char *cmd) { - if (unlikely(!dimension || !*dimension)) { - netdata_log_error("PLUGINSD: 'host:%s/chart:%s' got a %s, without a dimension.", - rrdhost_hostname(host), rrdset_id(st), cmd); - return NULL; - } - - if (unlikely(!st->pluginsd.size)) { - netdata_log_error("PLUGINSD: 'host:%s/chart:%s' got a %s, but the chart has no dimensions.", - rrdhost_hostname(host), rrdset_id(st), cmd); - return NULL; - } - - struct pluginsd_rrddim *prd; - RRDDIM *rd; - - if(likely(st->pluginsd.dims_with_slots)) { - // caching with slots - - if(unlikely(slot < 1 || slot > st->pluginsd.size)) { - netdata_log_error("PLUGINSD: 'host:%s/chart:%s' got a %s with slot %zd, but slots in the range [1 - %u] are expected.", - rrdhost_hostname(host), rrdset_id(st), cmd, slot, st->pluginsd.size); - return NULL; - } - - prd = &st->pluginsd.prd_array[slot - 1]; - - rd = prd->rd; - if(likely(rd)) { -#ifdef NETDATA_INTERNAL_CHECKS - if(strcmp(prd->id, dimension) != 0) { - ssize_t t; - for(t = 0; t < st->pluginsd.size ;t++) { - if (strcmp(st->pluginsd.prd_array[t].id, dimension) == 0) - break; - } - if(t >= st->pluginsd.size) - t = -1; - - internal_fatal(true, - "PLUGINSD: expected to find dimension '%s' on slot %zd, but found '%s', " - "the right slot is %zd", - dimension, slot, prd->id, t); - } -#endif - return rd; - } - } - else { - // caching without slots - - if(unlikely(st->pluginsd.pos >= st->pluginsd.size)) - st->pluginsd.pos = 0; - - prd = &st->pluginsd.prd_array[st->pluginsd.pos++]; - - rd = prd->rd; - if(likely(rd)) { - const char *id = prd->id; - - if(strcmp(id, dimension) == 0) { - // we found it cached - return rd; - } - else { - // the cached one is not good for us - rrddim_acquired_release(prd->rda); - prd->rda = NULL; - prd->rd = NULL; - prd->id = NULL; - } - } - } - - // we need to find the dimension and set it to prd - - RRDDIM_ACQUIRED *rda = rrddim_find_and_acquire(st, dimension); - if (unlikely(!rda)) { - netdata_log_error("PLUGINSD: 'host:%s/chart:%s/dim:%s' got a %s but dimension does not exist.", - rrdhost_hostname(host), rrdset_id(st), dimension, cmd); - - return NULL; - } - - prd->rda = rda; - prd->rd = rd = rrddim_acquired_to_rrddim(rda); - prd->id = string2str(rd->id); - - return rd; -} - -static inline RRDSET *pluginsd_find_chart(RRDHOST *host, const char *chart, const char *cmd) { - if (unlikely(!chart || !*chart)) { - netdata_log_error("PLUGINSD: 'host:%s' got a %s without a chart id.", - rrdhost_hostname(host), cmd); - return NULL; - } - - RRDSET *st = rrdset_find(host, chart); - if (unlikely(!st)) - netdata_log_error("PLUGINSD: 'host:%s/chart:%s' got a %s but chart does not exist.", - rrdhost_hostname(host), chart, cmd); - - return st; -} - -static inline ssize_t pluginsd_parse_rrd_slot(char **words, size_t num_words) { - ssize_t slot = -1; - char *id = get_word(words, num_words, 1); - if(id && id[0] == PLUGINSD_KEYWORD_SLOT[0] && id[1] == PLUGINSD_KEYWORD_SLOT[1] && - id[2] == PLUGINSD_KEYWORD_SLOT[2] && id[3] == PLUGINSD_KEYWORD_SLOT[3] && id[4] == ':') { - slot = (ssize_t) str2ull_encoded(&id[5]); - if(slot < 0) slot = 0; // to make the caller increment its idx of the words - } - - return slot; -} - -static inline void pluginsd_rrdset_cache_put_to_slot(PARSER *parser, RRDSET *st, ssize_t slot, bool obsolete) { - // clean possible old cached data - rrdset_pluginsd_receive_unslot(st); - - if(unlikely(slot < 1 || slot >= INT32_MAX)) - return; - - RRDHOST *host = st->rrdhost; - - if(unlikely((size_t)slot > host->rrdpush.receive.pluginsd_chart_slots.size)) { - spinlock_lock(&host->rrdpush.receive.pluginsd_chart_slots.spinlock); - size_t old_slots = host->rrdpush.receive.pluginsd_chart_slots.size; - size_t new_slots = (old_slots < PLUGINSD_MIN_RRDSET_POINTERS_CACHE) ? PLUGINSD_MIN_RRDSET_POINTERS_CACHE : old_slots * 2; - - if(new_slots < (size_t)slot) - new_slots = slot; - - host->rrdpush.receive.pluginsd_chart_slots.array = - reallocz(host->rrdpush.receive.pluginsd_chart_slots.array, new_slots * sizeof(RRDSET *)); - - for(size_t i = old_slots; i < new_slots ;i++) - host->rrdpush.receive.pluginsd_chart_slots.array[i] = NULL; - - host->rrdpush.receive.pluginsd_chart_slots.size = new_slots; - spinlock_unlock(&host->rrdpush.receive.pluginsd_chart_slots.spinlock); - } - - host->rrdpush.receive.pluginsd_chart_slots.array[slot - 1] = st; - st->pluginsd.last_slot = (int32_t)slot - 1; - parser->user.cleanup_slots = obsolete; -} - -static inline RRDSET *pluginsd_rrdset_cache_get_from_slot(PARSER *parser, RRDHOST *host, const char *id, ssize_t slot, const char *keyword) { - if(unlikely(slot < 1 || (size_t)slot > host->rrdpush.receive.pluginsd_chart_slots.size)) - return pluginsd_find_chart(host, id, keyword); - - RRDSET *st = host->rrdpush.receive.pluginsd_chart_slots.array[slot - 1]; - - if(!st) { - st = pluginsd_find_chart(host, id, keyword); - if(st) - pluginsd_rrdset_cache_put_to_slot(parser, st, slot, rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE)); - } - else { - internal_fatal(string_strcmp(st->id, id) != 0, - "PLUGINSD: wrong chart in slot %zd, expected '%s', found '%s'", - slot - 1, id, string2str(st->id)); - } - - return st; -} - -static inline PARSER_RC PLUGINSD_DISABLE_PLUGIN(PARSER *parser, const char *keyword, const char *msg) { - parser->user.enabled = 0; - - if(keyword && msg) { - nd_log_limit_static_global_var(erl, 1, 0); - nd_log_limit(&erl, NDLS_COLLECTORS, NDLP_INFO, - "PLUGINSD: keyword %s: %s", keyword, msg); - } - - return PARSER_RC_ERROR; -} +#include "pluginsd_internals.h" static inline PARSER_RC pluginsd_set(char **words, size_t num_words, PARSER *parser) { int idx = 1; @@ -900,384 +504,6 @@ static inline PARSER_RC pluginsd_dimension(char **words, size_t num_words, PARSE return PARSER_RC_OK; } -// ---------------------------------------------------------------------------- -// execution of functions - -struct inflight_function { - int code; - int timeout; - STRING *function; - BUFFER *result_body_wb; - rrd_function_result_callback_t result_cb; - void *result_cb_data; - usec_t timeout_ut; - usec_t started_ut; - usec_t sent_ut; - const char *payload; - PARSER *parser; - bool virtual; -}; - -static void inflight_functions_insert_callback(const DICTIONARY_ITEM *item, void *func, void *parser_ptr) { - struct inflight_function *pf = func; - - PARSER *parser = parser_ptr; - - // leave this code as default, so that when the dictionary is destroyed this will be sent back to the caller - pf->code = HTTP_RESP_GATEWAY_TIMEOUT; - - const char *transaction = dictionary_acquired_item_name(item); - - char buffer[2048 + 1]; - snprintfz(buffer, sizeof(buffer) - 1, "%s %s %d \"%s\"\n", - pf->payload ? "FUNCTION_PAYLOAD" : "FUNCTION", - transaction, - pf->timeout, - string2str(pf->function)); - - // send the command to the plugin - ssize_t ret = send_to_plugin(buffer, parser); - - pf->sent_ut = now_realtime_usec(); - - if(ret < 0) { - netdata_log_error("FUNCTION '%s': failed to send it to the plugin, error %zd", string2str(pf->function), ret); - rrd_call_function_error(pf->result_body_wb, "Failed to communicate with collector", HTTP_RESP_SERVICE_UNAVAILABLE); - } - else { - internal_error(LOG_FUNCTIONS, - "FUNCTION '%s' with transaction '%s' sent to collector (%zd bytes, in %"PRIu64" usec)", - string2str(pf->function), dictionary_acquired_item_name(item), ret, - pf->sent_ut - pf->started_ut); - } - - if (!pf->payload) - return; - - // send the payload to the plugin - ret = send_to_plugin(pf->payload, parser); - - if(ret < 0) { - netdata_log_error("FUNCTION_PAYLOAD '%s': failed to send function to plugin, error %zd", string2str(pf->function), ret); - rrd_call_function_error(pf->result_body_wb, "Failed to communicate with collector", HTTP_RESP_SERVICE_UNAVAILABLE); - } - else { - internal_error(LOG_FUNCTIONS, - "FUNCTION_PAYLOAD '%s' with transaction '%s' sent to collector (%zd bytes, in %"PRIu64" usec)", - string2str(pf->function), dictionary_acquired_item_name(item), ret, - pf->sent_ut - pf->started_ut); - } - - send_to_plugin("\nFUNCTION_PAYLOAD_END\n", parser); -} - -static bool inflight_functions_conflict_callback(const DICTIONARY_ITEM *item __maybe_unused, void *func __maybe_unused, void *new_func, void *parser_ptr __maybe_unused) { - struct inflight_function *pf = new_func; - - netdata_log_error("PLUGINSD_PARSER: duplicate UUID on pending function '%s' detected. Ignoring the second one.", string2str(pf->function)); - pf->code = rrd_call_function_error(pf->result_body_wb, "This request is already in progress", HTTP_RESP_BAD_REQUEST); - pf->result_cb(pf->result_body_wb, pf->code, pf->result_cb_data); - string_freez(pf->function); - - return false; -} - -void delete_job_finalize(struct parser *parser __maybe_unused, struct configurable_plugin *plug, const char *fnc_sig, int code) { - if (code != DYNCFG_VFNC_RET_CFG_ACCEPTED) - return; - - char *params_local = strdupz(fnc_sig); - char *words[DYNCFG_MAX_WORDS]; - size_t words_c = quoted_strings_splitter(params_local, words, DYNCFG_MAX_WORDS, isspace_map_pluginsd); - - if (words_c != 3) { - netdata_log_error("PLUGINSD_PARSER: invalid number of parameters for delete_job"); - freez(params_local); - return; - } - - const char *module = words[1]; - const char *job = words[2]; - - delete_job(plug, module, job); - - unlink_job(plug->name, module, job); - - rrdpush_send_job_deleted(localhost, plug->name, module, job); - - freez(params_local); -} - -void set_job_finalize(struct parser *parser __maybe_unused, struct configurable_plugin *plug __maybe_unused, const char *fnc_sig, int code) { - if (code != DYNCFG_VFNC_RET_CFG_ACCEPTED) - return; - - char *params_local = strdupz(fnc_sig); - char *words[DYNCFG_MAX_WORDS]; - size_t words_c = quoted_strings_splitter(params_local, words, DYNCFG_MAX_WORDS, isspace_map_pluginsd); - - if (words_c != 3) { - netdata_log_error("PLUGINSD_PARSER: invalid number of parameters for set_job_config"); - freez(params_local); - return; - } - - const char *module_name = get_word(words, words_c, 1); - const char *job_name = get_word(words, words_c, 2); - - if (register_job(parser->user.host->configurable_plugins, parser->user.cd->configuration->name, module_name, job_name, JOB_TYPE_USER, JOB_FLG_USER_CREATED, 1)) { - freez(params_local); - return; - } - - // only send this if it is not existing already (register_job cares for that) - rrdpush_send_dyncfg_reg_job(localhost, parser->user.cd->configuration->name, module_name, job_name, JOB_TYPE_USER, JOB_FLG_USER_CREATED); - - freez(params_local); -} - -static void inflight_functions_delete_callback(const DICTIONARY_ITEM *item __maybe_unused, void *func, void *parser_ptr) { - struct inflight_function *pf = func; - struct parser *parser = (struct parser *)parser_ptr; - - internal_error(LOG_FUNCTIONS, - "FUNCTION '%s' result of transaction '%s' received from collector (%zu bytes, request %"PRIu64" usec, response %"PRIu64" usec)", - string2str(pf->function), dictionary_acquired_item_name(item), - buffer_strlen(pf->result_body_wb), pf->sent_ut - pf->started_ut, now_realtime_usec() - pf->sent_ut); - - if (pf->virtual && SERVING_PLUGINSD(parser)) { - if (pf->payload) { - if (strncmp(string2str(pf->function), FUNCTION_NAME_SET_JOB_CONFIG, strlen(FUNCTION_NAME_SET_JOB_CONFIG)) == 0) - set_job_finalize(parser, parser->user.cd->configuration, string2str(pf->function), pf->code); - dyn_conf_store_config(string2str(pf->function), pf->payload, parser->user.cd->configuration); - } else if (strncmp(string2str(pf->function), FUNCTION_NAME_DELETE_JOB, strlen(FUNCTION_NAME_DELETE_JOB)) == 0) { - delete_job_finalize(parser, parser->user.cd->configuration, string2str(pf->function), pf->code); - } - } - - pf->result_cb(pf->result_body_wb, pf->code, pf->result_cb_data); - - string_freez(pf->function); - freez((void *)pf->payload); -} - -void inflight_functions_init(PARSER *parser) { - parser->inflight.functions = dictionary_create_advanced(DICT_OPTION_DONT_OVERWRITE_VALUE, &dictionary_stats_category_functions, 0); - dictionary_register_insert_callback(parser->inflight.functions, inflight_functions_insert_callback, parser); - dictionary_register_delete_callback(parser->inflight.functions, inflight_functions_delete_callback, parser); - dictionary_register_conflict_callback(parser->inflight.functions, inflight_functions_conflict_callback, parser); -} - -static void inflight_functions_garbage_collect(PARSER *parser, usec_t now) { - parser->inflight.smaller_timeout = 0; - struct inflight_function *pf; - dfe_start_write(parser->inflight.functions, pf) { - if (pf->timeout_ut < now) { - internal_error(true, - "FUNCTION '%s' removing expired transaction '%s', after %"PRIu64" usec.", - string2str(pf->function), pf_dfe.name, now - pf->started_ut); - - if(!buffer_strlen(pf->result_body_wb) || pf->code == HTTP_RESP_OK) - pf->code = rrd_call_function_error(pf->result_body_wb, - "Timeout waiting for collector response.", - HTTP_RESP_GATEWAY_TIMEOUT); - - dictionary_del(parser->inflight.functions, pf_dfe.name); - } - - else if(!parser->inflight.smaller_timeout || pf->timeout_ut < parser->inflight.smaller_timeout) - parser->inflight.smaller_timeout = pf->timeout_ut; - } - dfe_done(pf); -} - -void pluginsd_function_cancel(void *data) { - struct inflight_function *look_for = data, *t; - - bool sent = false; - dfe_start_read(look_for->parser->inflight.functions, t) { - if(look_for == t) { - const char *transaction = t_dfe.name; - - internal_error(true, "PLUGINSD: sending function cancellation to plugin for transaction '%s'", transaction); - - char buffer[2048 + 1]; - snprintfz(buffer, sizeof(buffer) - 1, "%s %s\n", - PLUGINSD_KEYWORD_FUNCTION_CANCEL, - transaction); - - // send the command to the plugin - ssize_t ret = send_to_plugin(buffer, t->parser); - if(ret < 0) - sent = true; - - break; - } - } - dfe_done(t); - - if(sent <= 0) - nd_log(NDLS_DAEMON, NDLP_DEBUG, - "PLUGINSD: FUNCTION_CANCEL request didn't match any pending function requests in pluginsd.d."); -} - -// this is the function that is called from -// rrd_call_function_and_wait() and rrd_call_function_async() -static int pluginsd_function_execute_cb(BUFFER *result_body_wb, int timeout, const char *function, - void *execute_cb_data, - rrd_function_result_callback_t result_cb, void *result_cb_data, - rrd_function_is_cancelled_cb_t is_cancelled_cb __maybe_unused, - void *is_cancelled_cb_data __maybe_unused, - rrd_function_register_canceller_cb_t register_canceller_cb, - void *register_canceller_db_data) { - PARSER *parser = execute_cb_data; - - usec_t now = now_realtime_usec(); - - struct inflight_function tmp = { - .started_ut = now, - .timeout_ut = now + timeout * USEC_PER_SEC + RRDFUNCTIONS_TIMEOUT_EXTENSION_UT, - .result_body_wb = result_body_wb, - .timeout = timeout, - .function = string_strdupz(function), - .result_cb = result_cb, - .result_cb_data = result_cb_data, - .payload = NULL, - .parser = parser, - }; - - uuid_t uuid; - uuid_generate_random(uuid); - - char transaction[UUID_STR_LEN]; - uuid_unparse_lower(uuid, transaction); - - dictionary_write_lock(parser->inflight.functions); - - // if there is any error, our dictionary callbacks will call the caller callback to notify - // the caller about the error - no need for error handling here. - void *t = dictionary_set(parser->inflight.functions, transaction, &tmp, sizeof(struct inflight_function)); - if(register_canceller_cb) - register_canceller_cb(register_canceller_db_data, pluginsd_function_cancel, t); - - if(!parser->inflight.smaller_timeout || tmp.timeout_ut < parser->inflight.smaller_timeout) - parser->inflight.smaller_timeout = tmp.timeout_ut; - - // garbage collect stale inflight functions - if(parser->inflight.smaller_timeout < now) - inflight_functions_garbage_collect(parser, now); - - dictionary_write_unlock(parser->inflight.functions); - - return HTTP_RESP_OK; -} - -static inline PARSER_RC pluginsd_function(char **words, size_t num_words, PARSER *parser) { - // a plugin or a child is registering a function - - bool global = false; - size_t i = 1; - if(num_words >= 2 && strcmp(get_word(words, num_words, 1), "GLOBAL") == 0) { - i++; - global = true; - } - - char *name = get_word(words, num_words, i++); - char *timeout_s = get_word(words, num_words, i++); - char *help = get_word(words, num_words, i++); - - RRDHOST *host = pluginsd_require_scope_host(parser, PLUGINSD_KEYWORD_FUNCTION); - if(!host) return PARSER_RC_ERROR; - - RRDSET *st = (global)? NULL: pluginsd_require_scope_chart(parser, PLUGINSD_KEYWORD_FUNCTION, PLUGINSD_KEYWORD_CHART); - if(!st) global = true; - - if (unlikely(!timeout_s || !name || !help || (!global && !st))) { - netdata_log_error("PLUGINSD: 'host:%s/chart:%s' got a FUNCTION, without providing the required data (global = '%s', name = '%s', timeout = '%s', help = '%s'). Ignoring it.", - rrdhost_hostname(host), - st?rrdset_id(st):"(unset)", - global?"yes":"no", - name?name:"(unset)", - timeout_s?timeout_s:"(unset)", - help?help:"(unset)" - ); - return PARSER_RC_ERROR; - } - - int timeout = PLUGINS_FUNCTIONS_TIMEOUT_DEFAULT; - if (timeout_s && *timeout_s) { - timeout = str2i(timeout_s); - if (unlikely(timeout <= 0)) - timeout = PLUGINS_FUNCTIONS_TIMEOUT_DEFAULT; - } - - rrd_function_add(host, st, name, timeout, help, false, pluginsd_function_execute_cb, parser); - - parser->user.data_collections_count++; - - return PARSER_RC_OK; -} - -static void pluginsd_function_result_end(struct parser *parser, void *action_data) { - STRING *key = action_data; - if(key) - dictionary_del(parser->inflight.functions, string2str(key)); - string_freez(key); - - parser->user.data_collections_count++; -} - -static inline PARSER_RC pluginsd_function_result_begin(char **words, size_t num_words, PARSER *parser) { - char *key = get_word(words, num_words, 1); - char *status = get_word(words, num_words, 2); - char *format = get_word(words, num_words, 3); - char *expires = get_word(words, num_words, 4); - - if (unlikely(!key || !*key || !status || !*status || !format || !*format || !expires || !*expires)) { - netdata_log_error("got a " PLUGINSD_KEYWORD_FUNCTION_RESULT_BEGIN " without providing the required data (key = '%s', status = '%s', format = '%s', expires = '%s')." - , key ? key : "(unset)" - , status ? status : "(unset)" - , format ? format : "(unset)" - , expires ? expires : "(unset)" - ); - } - - int code = (status && *status) ? str2i(status) : 0; - if (code <= 0) - code = HTTP_RESP_BACKEND_RESPONSE_INVALID; - - time_t expiration = (expires && *expires) ? str2l(expires) : 0; - - struct inflight_function *pf = NULL; - - if(key && *key) - pf = (struct inflight_function *)dictionary_get(parser->inflight.functions, key); - - if(!pf) { - netdata_log_error("got a " PLUGINSD_KEYWORD_FUNCTION_RESULT_BEGIN " for transaction '%s', but the transaction is not found.", key?key:"(unset)"); - } - else { - if(format && *format) - pf->result_body_wb->content_type = functions_format_to_content_type(format); - - pf->code = code; - - pf->result_body_wb->expires = expiration; - if(expiration <= now_realtime_sec()) - buffer_no_cacheable(pf->result_body_wb); - else - buffer_cacheable(pf->result_body_wb); - } - - parser->defer.response = (pf) ? pf->result_body_wb : NULL; - parser->defer.end_keyword = PLUGINSD_KEYWORD_FUNCTION_RESULT_END; - parser->defer.action = pluginsd_function_result_end; - parser->defer.action_data = string_strdupz(key); // it is ok is key is NULL - parser->flags |= PARSER_DEFER_UNTIL_KEYWORD; - - return PARSER_RC_OK; -} - // ---------------------------------------------------------------------------- static inline PARSER_RC pluginsd_variable(char **words, size_t num_words, PARSER *parser) { @@ -1472,426 +698,31 @@ static inline PARSER_RC pluginsd_clabel(char **words, size_t num_words, PARSER * rrdlabels_add(parser->user.chart_rrdlabels_linked_temporarily, name, value, str2l(label_source)); - return PARSER_RC_OK; -} - -static inline PARSER_RC pluginsd_clabel_commit(char **words __maybe_unused, size_t num_words __maybe_unused, PARSER *parser) { - RRDHOST *host = pluginsd_require_scope_host(parser, PLUGINSD_KEYWORD_CLABEL_COMMIT); - if(!host) return PLUGINSD_DISABLE_PLUGIN(parser, NULL, NULL); - - RRDSET *st = pluginsd_require_scope_chart(parser, PLUGINSD_KEYWORD_CLABEL_COMMIT, PLUGINSD_KEYWORD_BEGIN); - if(!st) return PLUGINSD_DISABLE_PLUGIN(parser, NULL, NULL); - - netdata_log_debug(D_PLUGINSD, "requested to commit chart labels"); - - if(!parser->user.chart_rrdlabels_linked_temporarily) { - netdata_log_error("PLUGINSD: 'host:%s' got CLABEL_COMMIT, without a CHART or BEGIN. Ignoring it.", rrdhost_hostname(host)); - return PLUGINSD_DISABLE_PLUGIN(parser, NULL, NULL); - } - - rrdlabels_remove_all_unmarked(parser->user.chart_rrdlabels_linked_temporarily); - - rrdset_flag_set(st, RRDSET_FLAG_METADATA_UPDATE); - rrdhost_flag_set(st->rrdhost, RRDHOST_FLAG_METADATA_UPDATE); - rrdset_metadata_updated(st); - - parser->user.chart_rrdlabels_linked_temporarily = NULL; - return PARSER_RC_OK; -} - -static inline PARSER_RC pluginsd_replay_begin(char **words, size_t num_words, PARSER *parser) { - int idx = 1; - ssize_t slot = pluginsd_parse_rrd_slot(words, num_words); - if(slot >= 0) idx++; - - char *id = get_word(words, num_words, idx++); - char *start_time_str = get_word(words, num_words, idx++); - char *end_time_str = get_word(words, num_words, idx++); - char *child_now_str = get_word(words, num_words, idx++); - - RRDHOST *host = pluginsd_require_scope_host(parser, PLUGINSD_KEYWORD_REPLAY_BEGIN); - if(!host) return PLUGINSD_DISABLE_PLUGIN(parser, NULL, NULL); - - RRDSET *st; - if (likely(!id || !*id)) - st = pluginsd_require_scope_chart(parser, PLUGINSD_KEYWORD_REPLAY_BEGIN, PLUGINSD_KEYWORD_REPLAY_BEGIN); - else - st = pluginsd_rrdset_cache_get_from_slot(parser, host, id, slot, PLUGINSD_KEYWORD_REPLAY_BEGIN); - - if(!st) return PLUGINSD_DISABLE_PLUGIN(parser, NULL, NULL); - - if(!pluginsd_set_scope_chart(parser, st, PLUGINSD_KEYWORD_REPLAY_BEGIN)) - return PLUGINSD_DISABLE_PLUGIN(parser, NULL, NULL); - - if(start_time_str && end_time_str) { - time_t start_time = (time_t) str2ull_encoded(start_time_str); - time_t end_time = (time_t) str2ull_encoded(end_time_str); - - time_t wall_clock_time = 0, tolerance; - bool wall_clock_comes_from_child; (void)wall_clock_comes_from_child; - if(child_now_str) { - wall_clock_time = (time_t) str2ull_encoded(child_now_str); - tolerance = st->update_every + 1; - wall_clock_comes_from_child = true; - } - - if(wall_clock_time <= 0) { - wall_clock_time = now_realtime_sec(); - tolerance = st->update_every + 5; - wall_clock_comes_from_child = false; - } - -#ifdef NETDATA_LOG_REPLICATION_REQUESTS - internal_error( - (!st->replay.start_streaming && (end_time < st->replay.after || start_time > st->replay.before)), - "REPLAY ERROR: 'host:%s/chart:%s' got a " PLUGINSD_KEYWORD_REPLAY_BEGIN " from %ld to %ld, which does not match our request (%ld to %ld).", - rrdhost_hostname(st->rrdhost), rrdset_id(st), start_time, end_time, st->replay.after, st->replay.before); - - internal_error( - true, - "REPLAY: 'host:%s/chart:%s' got a " PLUGINSD_KEYWORD_REPLAY_BEGIN " from %ld to %ld, child wall clock is %ld (%s), had requested %ld to %ld", - rrdhost_hostname(st->rrdhost), rrdset_id(st), - start_time, end_time, wall_clock_time, wall_clock_comes_from_child ? "from child" : "parent time", - st->replay.after, st->replay.before); -#endif - - if(start_time && end_time && start_time < wall_clock_time + tolerance && end_time < wall_clock_time + tolerance && start_time < end_time) { - if (unlikely(end_time - start_time != st->update_every)) - rrdset_set_update_every_s(st, end_time - start_time); - - st->last_collected_time.tv_sec = end_time; - st->last_collected_time.tv_usec = 0; - - st->last_updated.tv_sec = end_time; - st->last_updated.tv_usec = 0; - - st->counter++; - st->counter_done++; - - // these are only needed for db mode RAM, SAVE, MAP, ALLOC - st->db.current_entry++; - if(st->db.current_entry >= st->db.entries) - st->db.current_entry -= st->db.entries; - - parser->user.replay.start_time = start_time; - parser->user.replay.end_time = end_time; - parser->user.replay.start_time_ut = (usec_t) start_time * USEC_PER_SEC; - parser->user.replay.end_time_ut = (usec_t) end_time * USEC_PER_SEC; - parser->user.replay.wall_clock_time = wall_clock_time; - parser->user.replay.rset_enabled = true; - - return PARSER_RC_OK; - } - - netdata_log_error("PLUGINSD REPLAY ERROR: 'host:%s/chart:%s' got a " PLUGINSD_KEYWORD_REPLAY_BEGIN - " from %ld to %ld, but timestamps are invalid " - "(now is %ld [%s], tolerance %ld). Ignoring " PLUGINSD_KEYWORD_REPLAY_SET, - rrdhost_hostname(st->rrdhost), rrdset_id(st), start_time, end_time, - wall_clock_time, wall_clock_comes_from_child ? "child wall clock" : "parent wall clock", - tolerance); - } - - // the child sends an RBEGIN without any parameters initially - // setting rset_enabled to false, means the RSET should not store any metrics - // to store metrics, the RBEGIN needs to have timestamps - parser->user.replay.start_time = 0; - parser->user.replay.end_time = 0; - parser->user.replay.start_time_ut = 0; - parser->user.replay.end_time_ut = 0; - parser->user.replay.wall_clock_time = 0; - parser->user.replay.rset_enabled = false; - return PARSER_RC_OK; -} - -static inline SN_FLAGS pluginsd_parse_storage_number_flags(const char *flags_str) { - SN_FLAGS flags = SN_FLAG_NONE; - - char c; - while ((c = *flags_str++)) { - switch (c) { - case 'A': - flags |= SN_FLAG_NOT_ANOMALOUS; - break; - - case 'R': - flags |= SN_FLAG_RESET; - break; - - case 'E': - flags = SN_EMPTY_SLOT; - return flags; - - default: - internal_error(true, "Unknown SN_FLAGS flag '%c'", c); - break; - } - } - - return flags; -} - -static inline PARSER_RC pluginsd_replay_set(char **words, size_t num_words, PARSER *parser) { - int idx = 1; - ssize_t slot = pluginsd_parse_rrd_slot(words, num_words); - if(slot >= 0) idx++; - - char *dimension = get_word(words, num_words, idx++); - char *value_str = get_word(words, num_words, idx++); - char *flags_str = get_word(words, num_words, idx++); - - RRDHOST *host = pluginsd_require_scope_host(parser, PLUGINSD_KEYWORD_REPLAY_SET); - if(!host) return PLUGINSD_DISABLE_PLUGIN(parser, NULL, NULL); - - RRDSET *st = pluginsd_require_scope_chart(parser, PLUGINSD_KEYWORD_REPLAY_SET, PLUGINSD_KEYWORD_REPLAY_BEGIN); - if(!st) return PLUGINSD_DISABLE_PLUGIN(parser, NULL, NULL); - - if(!parser->user.replay.rset_enabled) { - nd_log_limit_static_thread_var(erl, 1, 0); - nd_log_limit(&erl, NDLS_COLLECTORS, NDLP_ERR, - "PLUGINSD: 'host:%s/chart:%s' got a %s but it is disabled by %s errors", - rrdhost_hostname(host), rrdset_id(st), PLUGINSD_KEYWORD_REPLAY_SET, PLUGINSD_KEYWORD_REPLAY_BEGIN); - - // we have to return OK here - return PARSER_RC_OK; - } - - RRDDIM *rd = pluginsd_acquire_dimension(host, st, dimension, slot, PLUGINSD_KEYWORD_REPLAY_SET); - if(!rd) return PLUGINSD_DISABLE_PLUGIN(parser, NULL, NULL); - - st->pluginsd.set = true; - - if (unlikely(!parser->user.replay.start_time || !parser->user.replay.end_time)) { - netdata_log_error("PLUGINSD: 'host:%s/chart:%s/dim:%s' got a %s with invalid timestamps %ld to %ld from a %s. Disabling it.", - rrdhost_hostname(host), - rrdset_id(st), - dimension, - PLUGINSD_KEYWORD_REPLAY_SET, - parser->user.replay.start_time, - parser->user.replay.end_time, - PLUGINSD_KEYWORD_REPLAY_BEGIN); - return PLUGINSD_DISABLE_PLUGIN(parser, NULL, NULL); - } - - if (unlikely(!value_str || !*value_str)) - value_str = "NAN"; - - if(unlikely(!flags_str)) - flags_str = ""; - - if (likely(value_str)) { - RRDDIM_FLAGS rd_flags = rrddim_flag_check(rd, RRDDIM_FLAG_OBSOLETE | RRDDIM_FLAG_ARCHIVED); - - if(!(rd_flags & RRDDIM_FLAG_ARCHIVED)) { - NETDATA_DOUBLE value = str2ndd_encoded(value_str, NULL); - SN_FLAGS flags = pluginsd_parse_storage_number_flags(flags_str); - - if (!netdata_double_isnumber(value) || (flags == SN_EMPTY_SLOT)) { - value = NAN; - flags = SN_EMPTY_SLOT; - } - - rrddim_store_metric(rd, parser->user.replay.end_time_ut, value, flags); - rd->collector.last_collected_time.tv_sec = parser->user.replay.end_time; - rd->collector.last_collected_time.tv_usec = 0; - rd->collector.counter++; - } - else { - nd_log_limit_static_global_var(erl, 1, 0); - nd_log_limit(&erl, NDLS_COLLECTORS, NDLP_WARNING, - "PLUGINSD: 'host:%s/chart:%s/dim:%s' has the ARCHIVED flag set, but it is replicated. " - "Ignoring data.", - rrdhost_hostname(st->rrdhost), rrdset_id(st), rrddim_name(rd)); - } - } - - return PARSER_RC_OK; -} - -static inline PARSER_RC pluginsd_replay_rrddim_collection_state(char **words, size_t num_words, PARSER *parser) { - if(parser->user.replay.rset_enabled == false) - return PARSER_RC_OK; - - int idx = 1; - ssize_t slot = pluginsd_parse_rrd_slot(words, num_words); - if(slot >= 0) idx++; - - char *dimension = get_word(words, num_words, idx++); - char *last_collected_ut_str = get_word(words, num_words, idx++); - char *last_collected_value_str = get_word(words, num_words, idx++); - char *last_calculated_value_str = get_word(words, num_words, idx++); - char *last_stored_value_str = get_word(words, num_words, idx++); - - RRDHOST *host = pluginsd_require_scope_host(parser, PLUGINSD_KEYWORD_REPLAY_RRDDIM_STATE); - if(!host) return PLUGINSD_DISABLE_PLUGIN(parser, NULL, NULL); - - RRDSET *st = pluginsd_require_scope_chart(parser, PLUGINSD_KEYWORD_REPLAY_RRDDIM_STATE, PLUGINSD_KEYWORD_REPLAY_BEGIN); - if(!st) return PLUGINSD_DISABLE_PLUGIN(parser, NULL, NULL); - - if(st->pluginsd.set) { - // reset pos to reuse the same RDAs - st->pluginsd.pos = 0; - st->pluginsd.set = false; - } - - RRDDIM *rd = pluginsd_acquire_dimension(host, st, dimension, slot, PLUGINSD_KEYWORD_REPLAY_RRDDIM_STATE); - if(!rd) return PLUGINSD_DISABLE_PLUGIN(parser, NULL, NULL); - - usec_t dim_last_collected_ut = (usec_t)rd->collector.last_collected_time.tv_sec * USEC_PER_SEC + (usec_t)rd->collector.last_collected_time.tv_usec; - usec_t last_collected_ut = last_collected_ut_str ? str2ull_encoded(last_collected_ut_str) : 0; - if(last_collected_ut > dim_last_collected_ut) { - rd->collector.last_collected_time.tv_sec = (time_t)(last_collected_ut / USEC_PER_SEC); - rd->collector.last_collected_time.tv_usec = (last_collected_ut % USEC_PER_SEC); - } - - rd->collector.last_collected_value = last_collected_value_str ? str2ll_encoded(last_collected_value_str) : 0; - rd->collector.last_calculated_value = last_calculated_value_str ? str2ndd_encoded(last_calculated_value_str, NULL) : 0; - rd->collector.last_stored_value = last_stored_value_str ? str2ndd_encoded(last_stored_value_str, NULL) : 0.0; - - return PARSER_RC_OK; -} - -static inline PARSER_RC pluginsd_replay_rrdset_collection_state(char **words, size_t num_words, PARSER *parser) { - if(parser->user.replay.rset_enabled == false) - return PARSER_RC_OK; - - char *last_collected_ut_str = get_word(words, num_words, 1); - char *last_updated_ut_str = get_word(words, num_words, 2); - - RRDHOST *host = pluginsd_require_scope_host(parser, PLUGINSD_KEYWORD_REPLAY_RRDSET_STATE); - if(!host) return PLUGINSD_DISABLE_PLUGIN(parser, NULL, NULL); - - RRDSET *st = pluginsd_require_scope_chart(parser, PLUGINSD_KEYWORD_REPLAY_RRDSET_STATE, - PLUGINSD_KEYWORD_REPLAY_BEGIN); - if(!st) return PLUGINSD_DISABLE_PLUGIN(parser, NULL, NULL); - - usec_t chart_last_collected_ut = (usec_t)st->last_collected_time.tv_sec * USEC_PER_SEC + (usec_t)st->last_collected_time.tv_usec; - usec_t last_collected_ut = last_collected_ut_str ? str2ull_encoded(last_collected_ut_str) : 0; - if(last_collected_ut > chart_last_collected_ut) { - st->last_collected_time.tv_sec = (time_t)(last_collected_ut / USEC_PER_SEC); - st->last_collected_time.tv_usec = (last_collected_ut % USEC_PER_SEC); - } - - usec_t chart_last_updated_ut = (usec_t)st->last_updated.tv_sec * USEC_PER_SEC + (usec_t)st->last_updated.tv_usec; - usec_t last_updated_ut = last_updated_ut_str ? str2ull_encoded(last_updated_ut_str) : 0; - if(last_updated_ut > chart_last_updated_ut) { - st->last_updated.tv_sec = (time_t)(last_updated_ut / USEC_PER_SEC); - st->last_updated.tv_usec = (last_updated_ut % USEC_PER_SEC); - } - - st->counter++; - st->counter_done++; - - return PARSER_RC_OK; -} - -static inline PARSER_RC pluginsd_replay_end(char **words, size_t num_words, PARSER *parser) { - if (num_words < 7) { // accepts 7, but the 7th is optional - netdata_log_error("REPLAY: malformed " PLUGINSD_KEYWORD_REPLAY_END " command"); - return PARSER_RC_ERROR; - } - - const char *update_every_child_txt = get_word(words, num_words, 1); - const char *first_entry_child_txt = get_word(words, num_words, 2); - const char *last_entry_child_txt = get_word(words, num_words, 3); - const char *start_streaming_txt = get_word(words, num_words, 4); - const char *first_entry_requested_txt = get_word(words, num_words, 5); - const char *last_entry_requested_txt = get_word(words, num_words, 6); - const char *child_world_time_txt = get_word(words, num_words, 7); // optional - - time_t update_every_child = (time_t) str2ull_encoded(update_every_child_txt); - time_t first_entry_child = (time_t) str2ull_encoded(first_entry_child_txt); - time_t last_entry_child = (time_t) str2ull_encoded(last_entry_child_txt); - - bool start_streaming = (strcmp(start_streaming_txt, "true") == 0); - time_t first_entry_requested = (time_t) str2ull_encoded(first_entry_requested_txt); - time_t last_entry_requested = (time_t) str2ull_encoded(last_entry_requested_txt); - - // the optional child world time - time_t child_world_time = (child_world_time_txt && *child_world_time_txt) ? (time_t) str2ull_encoded( - child_world_time_txt) : now_realtime_sec(); - - RRDHOST *host = pluginsd_require_scope_host(parser, PLUGINSD_KEYWORD_REPLAY_END); - if(!host) return PLUGINSD_DISABLE_PLUGIN(parser, NULL, NULL); - - RRDSET *st = pluginsd_require_scope_chart(parser, PLUGINSD_KEYWORD_REPLAY_END, PLUGINSD_KEYWORD_REPLAY_BEGIN); - if(!st) return PLUGINSD_DISABLE_PLUGIN(parser, NULL, NULL); - -#ifdef NETDATA_LOG_REPLICATION_REQUESTS - internal_error(true, - "PLUGINSD REPLAY: 'host:%s/chart:%s': got a " PLUGINSD_KEYWORD_REPLAY_END " child db from %llu to %llu, start_streaming %s, had requested from %llu to %llu, wall clock %llu", - rrdhost_hostname(host), rrdset_id(st), - (unsigned long long)first_entry_child, (unsigned long long)last_entry_child, - start_streaming?"true":"false", - (unsigned long long)first_entry_requested, (unsigned long long)last_entry_requested, - (unsigned long long)child_world_time - ); -#endif - - parser->user.data_collections_count++; - - if(parser->user.replay.rset_enabled && st->rrdhost->receiver) { - time_t now = now_realtime_sec(); - time_t started = st->rrdhost->receiver->replication_first_time_t; - time_t current = parser->user.replay.end_time; - - if(started && current > started) { - host->rrdpush_receiver_replication_percent = (NETDATA_DOUBLE) (current - started) * 100.0 / (NETDATA_DOUBLE) (now - started); - worker_set_metric(WORKER_RECEIVER_JOB_REPLICATION_COMPLETION, - host->rrdpush_receiver_replication_percent); - } - } - - parser->user.replay.start_time = 0; - parser->user.replay.end_time = 0; - parser->user.replay.start_time_ut = 0; - parser->user.replay.end_time_ut = 0; - parser->user.replay.wall_clock_time = 0; - parser->user.replay.rset_enabled = false; - - st->counter++; - st->counter_done++; - store_metric_collection_completed(); - -#ifdef NETDATA_LOG_REPLICATION_REQUESTS - st->replay.start_streaming = false; - st->replay.after = 0; - st->replay.before = 0; - if(start_streaming) - st->replay.log_next_data_collection = true; -#endif - - if (start_streaming) { - if (st->update_every != update_every_child) - rrdset_set_update_every_s(st, update_every_child); + return PARSER_RC_OK; +} - if(rrdset_flag_check(st, RRDSET_FLAG_RECEIVER_REPLICATION_IN_PROGRESS)) { - rrdset_flag_set(st, RRDSET_FLAG_RECEIVER_REPLICATION_FINISHED); - rrdset_flag_clear(st, RRDSET_FLAG_RECEIVER_REPLICATION_IN_PROGRESS); - rrdset_flag_clear(st, RRDSET_FLAG_SYNC_CLOCK); - rrdhost_receiver_replicating_charts_minus_one(st->rrdhost); - } -#ifdef NETDATA_LOG_REPLICATION_REQUESTS - else - internal_error(true, "REPLAY ERROR: 'host:%s/chart:%s' got a " PLUGINSD_KEYWORD_REPLAY_END " with enable_streaming = true, but there is no replication in progress for this chart.", - rrdhost_hostname(host), rrdset_id(st)); -#endif +static inline PARSER_RC pluginsd_clabel_commit(char **words __maybe_unused, size_t num_words __maybe_unused, PARSER *parser) { + RRDHOST *host = pluginsd_require_scope_host(parser, PLUGINSD_KEYWORD_CLABEL_COMMIT); + if(!host) return PLUGINSD_DISABLE_PLUGIN(parser, NULL, NULL); - pluginsd_clear_scope_chart(parser, PLUGINSD_KEYWORD_REPLAY_END); + RRDSET *st = pluginsd_require_scope_chart(parser, PLUGINSD_KEYWORD_CLABEL_COMMIT, PLUGINSD_KEYWORD_BEGIN); + if(!st) return PLUGINSD_DISABLE_PLUGIN(parser, NULL, NULL); - host->rrdpush_receiver_replication_percent = 100.0; - worker_set_metric(WORKER_RECEIVER_JOB_REPLICATION_COMPLETION, host->rrdpush_receiver_replication_percent); + netdata_log_debug(D_PLUGINSD, "requested to commit chart labels"); - return PARSER_RC_OK; + if(!parser->user.chart_rrdlabels_linked_temporarily) { + netdata_log_error("PLUGINSD: 'host:%s' got CLABEL_COMMIT, without a CHART or BEGIN. Ignoring it.", rrdhost_hostname(host)); + return PLUGINSD_DISABLE_PLUGIN(parser, NULL, NULL); } - pluginsd_clear_scope_chart(parser, PLUGINSD_KEYWORD_REPLAY_END); + rrdlabels_remove_all_unmarked(parser->user.chart_rrdlabels_linked_temporarily); - rrdcontext_updated_retention_rrdset(st); + rrdset_flag_set(st, RRDSET_FLAG_METADATA_UPDATE); + rrdhost_flag_set(st->rrdhost, RRDHOST_FLAG_METADATA_UPDATE); + rrdset_metadata_updated(st); - bool ok = replicate_chart_request(send_to_plugin, parser, host, st, - first_entry_child, last_entry_child, child_world_time, - first_entry_requested, last_entry_requested); - return ok ? PARSER_RC_OK : PARSER_RC_ERROR; + parser->user.chart_rrdlabels_linked_temporarily = NULL; + return PARSER_RC_OK; } static inline PARSER_RC pluginsd_begin_v2(char **words, size_t num_words, PARSER *parser) { @@ -2160,11 +991,6 @@ static inline PARSER_RC pluginsd_set_v2(char **words, size_t num_words, PARSER * return PARSER_RC_OK; } -void pluginsd_cleanup_v2(PARSER *parser) { - // this is called when the thread is stopped while processing - pluginsd_clear_scope_chart(parser, "THREAD CLEANUP"); -} - static inline PARSER_RC pluginsd_end_v2(char **words __maybe_unused, size_t num_words __maybe_unused, PARSER *parser) { timing_init(); @@ -2243,569 +1069,6 @@ static inline PARSER_RC pluginsd_exit(char **words __maybe_unused, size_t num_wo return PARSER_RC_STOP; } -struct mutex_cond { - pthread_mutex_t lock; - pthread_cond_t cond; - int rc; -}; - -static void virt_fnc_got_data_cb(BUFFER *wb __maybe_unused, int code, void *callback_data) -{ - struct mutex_cond *ctx = callback_data; - pthread_mutex_lock(&ctx->lock); - ctx->rc = code; - pthread_cond_broadcast(&ctx->cond); - pthread_mutex_unlock(&ctx->lock); -} - -#define VIRT_FNC_TIMEOUT 1 -#define VIRT_FNC_BUF_SIZE (4096) -void call_virtual_function_async(BUFFER *wb, RRDHOST *host, const char *name, const char *payload, rrd_function_result_callback_t callback, void *callback_data) { - PARSER *parser = NULL; - - //TODO simplify (as we really need only first parameter to get plugin name maybe we can avoid parsing all) - char *words[PLUGINSD_MAX_WORDS]; - char *function_with_params = strdupz(name); - size_t num_words = quoted_strings_splitter(function_with_params, words, PLUGINSD_MAX_WORDS, isspace_map_pluginsd); - - if (num_words < 2) { - netdata_log_error("PLUGINSD: virtual function name is empty."); - freez(function_with_params); - return; - } - - const DICTIONARY_ITEM *cpi = dictionary_get_and_acquire_item(host->configurable_plugins, get_word(words, num_words, 1)); - if (unlikely(cpi == NULL)) { - netdata_log_error("PLUGINSD: virtual function plugin '%s' not found.", name); - freez(function_with_params); - return; - } - struct configurable_plugin *cp = dictionary_acquired_item_value(cpi); - parser = (PARSER *)cp->cb_usr_ctx; - - BUFFER *function_out = buffer_create(VIRT_FNC_BUF_SIZE, NULL); - // if we are forwarding this to a plugin (as opposed to streaming/child) we have to remove the first parameter (plugin_name) - buffer_strcat(function_out, get_word(words, num_words, 0)); - for (size_t i = 1; i < num_words; i++) { - if (i == 1 && SERVING_PLUGINSD(parser)) - continue; - buffer_sprintf(function_out, " %s", get_word(words, num_words, i)); - } - freez(function_with_params); - - usec_t now = now_realtime_usec(); - - struct inflight_function tmp = { - .started_ut = now, - .timeout_ut = now + VIRT_FNC_TIMEOUT + USEC_PER_SEC, - .result_body_wb = wb, - .timeout = VIRT_FNC_TIMEOUT * 10, - .function = string_strdupz(buffer_tostring(function_out)), - .result_cb = callback, - .result_cb_data = callback_data, - .payload = payload != NULL ? strdupz(payload) : NULL, - .virtual = true, - }; - buffer_free(function_out); - - uuid_t uuid; - uuid_generate_time(uuid); - - char key[UUID_STR_LEN]; - uuid_unparse_lower(uuid, key); - - dictionary_write_lock(parser->inflight.functions); - - // if there is any error, our dictionary callbacks will call the caller callback to notify - // the caller about the error - no need for error handling here. - dictionary_set(parser->inflight.functions, key, &tmp, sizeof(struct inflight_function)); - - if(!parser->inflight.smaller_timeout || tmp.timeout_ut < parser->inflight.smaller_timeout) - parser->inflight.smaller_timeout = tmp.timeout_ut; - - // garbage collect stale inflight functions - if(parser->inflight.smaller_timeout < now) - inflight_functions_garbage_collect(parser, now); - - dictionary_write_unlock(parser->inflight.functions); -} - - -dyncfg_config_t call_virtual_function_blocking(PARSER *parser, const char *name, int *rc, const char *payload) { - usec_t now = now_realtime_usec(); - BUFFER *wb = buffer_create(VIRT_FNC_BUF_SIZE, NULL); - - struct mutex_cond cond = { - .lock = PTHREAD_MUTEX_INITIALIZER, - .cond = PTHREAD_COND_INITIALIZER - }; - - struct inflight_function tmp = { - .started_ut = now, - .timeout_ut = now + VIRT_FNC_TIMEOUT + USEC_PER_SEC, - .result_body_wb = wb, - .timeout = VIRT_FNC_TIMEOUT, - .function = string_strdupz(name), - .result_cb = virt_fnc_got_data_cb, - .result_cb_data = &cond, - .payload = payload != NULL ? strdupz(payload) : NULL, - .virtual = true, - }; - - uuid_t uuid; - uuid_generate_time(uuid); - - char key[UUID_STR_LEN]; - uuid_unparse_lower(uuid, key); - - dictionary_write_lock(parser->inflight.functions); - - // if there is any error, our dictionary callbacks will call the caller callback to notify - // the caller about the error - no need for error handling here. - dictionary_set(parser->inflight.functions, key, &tmp, sizeof(struct inflight_function)); - - if(!parser->inflight.smaller_timeout || tmp.timeout_ut < parser->inflight.smaller_timeout) - parser->inflight.smaller_timeout = tmp.timeout_ut; - - // garbage collect stale inflight functions - if(parser->inflight.smaller_timeout < now) - inflight_functions_garbage_collect(parser, now); - - dictionary_write_unlock(parser->inflight.functions); - - struct timespec tp; - clock_gettime(CLOCK_REALTIME, &tp); - tp.tv_sec += (time_t)VIRT_FNC_TIMEOUT; - - pthread_mutex_lock(&cond.lock); - - int ret = pthread_cond_timedwait(&cond.cond, &cond.lock, &tp); - if (ret == ETIMEDOUT) - netdata_log_error("PLUGINSD: DYNCFG virtual function %s timed out", name); - - pthread_mutex_unlock(&cond.lock); - - dyncfg_config_t cfg; - cfg.data = strdupz(buffer_tostring(wb)); - cfg.data_size = buffer_strlen(wb); - - if (rc != NULL) - *rc = cond.rc; - - buffer_free(wb); - return cfg; -} - -#define CVF_MAX_LEN (1024) -static dyncfg_config_t get_plugin_config_cb(void *usr_ctx, const char *plugin_name) -{ - PARSER *parser = usr_ctx; - - if (SERVING_STREAMING(parser)) { - char buf[CVF_MAX_LEN + 1]; - snprintfz(buf, CVF_MAX_LEN, FUNCTION_NAME_GET_PLUGIN_CONFIG " %s", plugin_name); - return call_virtual_function_blocking(parser, buf, NULL, NULL); - } - - return call_virtual_function_blocking(parser, FUNCTION_NAME_GET_PLUGIN_CONFIG, NULL, NULL); -} - -static dyncfg_config_t get_plugin_config_schema_cb(void *usr_ctx, const char *plugin_name) -{ - PARSER *parser = usr_ctx; - - if (SERVING_STREAMING(parser)) { - char buf[CVF_MAX_LEN + 1]; - snprintfz(buf, CVF_MAX_LEN, FUNCTION_NAME_GET_PLUGIN_CONFIG_SCHEMA " %s", plugin_name); - return call_virtual_function_blocking(parser, buf, NULL, NULL); - } - - return call_virtual_function_blocking(parser, "get_plugin_config_schema", NULL, NULL); -} - -static dyncfg_config_t get_module_config_cb(void *usr_ctx, const char *plugin_name, const char *module_name) -{ - PARSER *parser = usr_ctx; - BUFFER *wb = buffer_create(CVF_MAX_LEN, NULL); - - buffer_strcat(wb, FUNCTION_NAME_GET_MODULE_CONFIG); - if (SERVING_STREAMING(parser)) - buffer_sprintf(wb, " %s", plugin_name); - - buffer_sprintf(wb, " %s", module_name); - - dyncfg_config_t ret = call_virtual_function_blocking(parser, buffer_tostring(wb), NULL, NULL); - - buffer_free(wb); - - return ret; -} - -static dyncfg_config_t get_module_config_schema_cb(void *usr_ctx, const char *plugin_name, const char *module_name) -{ - PARSER *parser = usr_ctx; - BUFFER *wb = buffer_create(CVF_MAX_LEN, NULL); - - buffer_strcat(wb, FUNCTION_NAME_GET_MODULE_CONFIG_SCHEMA); - if (SERVING_STREAMING(parser)) - buffer_sprintf(wb, " %s", plugin_name); - - buffer_sprintf(wb, " %s", module_name); - - dyncfg_config_t ret = call_virtual_function_blocking(parser, buffer_tostring(wb), NULL, NULL); - - buffer_free(wb); - - return ret; -} - -static dyncfg_config_t get_job_config_schema_cb(void *usr_ctx, const char *plugin_name, const char *module_name) -{ - PARSER *parser = usr_ctx; - BUFFER *wb = buffer_create(CVF_MAX_LEN, NULL); - - buffer_strcat(wb, FUNCTION_NAME_GET_JOB_CONFIG_SCHEMA); - - if (SERVING_STREAMING(parser)) - buffer_sprintf(wb, " %s", plugin_name); - - buffer_sprintf(wb, " %s", module_name); - - dyncfg_config_t ret = call_virtual_function_blocking(parser, buffer_tostring(wb), NULL, NULL); - - buffer_free(wb); - - return ret; -} - -static dyncfg_config_t get_job_config_cb(void *usr_ctx, const char *plugin_name, const char *module_name, const char* job_name) -{ - PARSER *parser = usr_ctx; - BUFFER *wb = buffer_create(CVF_MAX_LEN, NULL); - - buffer_strcat(wb, FUNCTION_NAME_GET_JOB_CONFIG); - - if (SERVING_STREAMING(parser)) - buffer_sprintf(wb, " %s", plugin_name); - - buffer_sprintf(wb, " %s %s", module_name, job_name); - - dyncfg_config_t ret = call_virtual_function_blocking(parser, buffer_tostring(wb), NULL, NULL); - - buffer_free(wb); - - return ret; -} - -enum set_config_result set_plugin_config_cb(void *usr_ctx, const char *plugin_name, dyncfg_config_t *cfg) -{ - PARSER *parser = usr_ctx; - BUFFER *wb = buffer_create(CVF_MAX_LEN, NULL); - - buffer_strcat(wb, FUNCTION_NAME_SET_PLUGIN_CONFIG); - - if (SERVING_STREAMING(parser)) - buffer_sprintf(wb, " %s", plugin_name); - - int rc; - call_virtual_function_blocking(parser, buffer_tostring(wb), &rc, cfg->data); - - buffer_free(wb); - if(rc != DYNCFG_VFNC_RET_CFG_ACCEPTED) - return SET_CONFIG_REJECTED; - return SET_CONFIG_ACCEPTED; -} - -enum set_config_result set_module_config_cb(void *usr_ctx, const char *plugin_name, const char *module_name, dyncfg_config_t *cfg) -{ - PARSER *parser = usr_ctx; - BUFFER *wb = buffer_create(CVF_MAX_LEN, NULL); - - buffer_strcat(wb, FUNCTION_NAME_SET_MODULE_CONFIG); - - if (SERVING_STREAMING(parser)) - buffer_sprintf(wb, " %s", plugin_name); - - buffer_sprintf(wb, " %s", module_name); - - int rc; - call_virtual_function_blocking(parser, buffer_tostring(wb), &rc, cfg->data); - - buffer_free(wb); - - if(rc != DYNCFG_VFNC_RET_CFG_ACCEPTED) - return SET_CONFIG_REJECTED; - return SET_CONFIG_ACCEPTED; -} - -enum set_config_result set_job_config_cb(void *usr_ctx, const char *plugin_name, const char *module_name, const char *job_name, dyncfg_config_t *cfg) -{ - PARSER *parser = usr_ctx; - BUFFER *wb = buffer_create(CVF_MAX_LEN, NULL); - - buffer_strcat(wb, FUNCTION_NAME_SET_JOB_CONFIG); - - if (SERVING_STREAMING(parser)) - buffer_sprintf(wb, " %s", plugin_name); - - buffer_sprintf(wb, " %s %s", module_name, job_name); - - int rc; - call_virtual_function_blocking(parser, buffer_tostring(wb), &rc, cfg->data); - - buffer_free(wb); - - if(rc != DYNCFG_VFNC_RET_CFG_ACCEPTED) - return SET_CONFIG_REJECTED; - return SET_CONFIG_ACCEPTED; -} - -enum set_config_result delete_job_cb(void *usr_ctx, const char *plugin_name ,const char *module_name, const char *job_name) -{ - PARSER *parser = usr_ctx; - BUFFER *wb = buffer_create(CVF_MAX_LEN, NULL); - - buffer_strcat(wb, FUNCTION_NAME_DELETE_JOB); - - if (SERVING_STREAMING(parser)) - buffer_sprintf(wb, " %s", plugin_name); - - buffer_sprintf(wb, " %s %s", module_name, job_name); - - int rc; - call_virtual_function_blocking(parser, buffer_tostring(wb), &rc, NULL); - - buffer_free(wb); - - if(rc != DYNCFG_VFNC_RET_CFG_ACCEPTED) - return SET_CONFIG_REJECTED; - return SET_CONFIG_ACCEPTED; -} - - -static inline PARSER_RC pluginsd_register_plugin(char **words __maybe_unused, size_t num_words __maybe_unused, PARSER *parser __maybe_unused) { - netdata_log_info("PLUGINSD: DYNCFG_ENABLE"); - - if (unlikely (num_words != 2)) - return PLUGINSD_DISABLE_PLUGIN(parser, PLUGINSD_KEYWORD_DYNCFG_ENABLE, "missing name parameter"); - - struct configurable_plugin *cfg = callocz(1, sizeof(struct configurable_plugin)); - - cfg->name = strdupz(words[1]); - cfg->set_config_cb = set_plugin_config_cb; - cfg->get_config_cb = get_plugin_config_cb; - cfg->get_config_schema_cb = get_plugin_config_schema_cb; - cfg->cb_usr_ctx = parser; - - const DICTIONARY_ITEM *di = register_plugin(parser->user.host->configurable_plugins, cfg, SERVING_PLUGINSD(parser)); - if (unlikely(di == NULL)) { - freez(cfg->name); - freez(cfg); - return PLUGINSD_DISABLE_PLUGIN(parser, PLUGINSD_KEYWORD_DYNCFG_ENABLE, "error registering plugin"); - } - - if (SERVING_PLUGINSD(parser)) { - // this is optimization for pluginsd to avoid extra dictionary lookup - // as we know which plugin is comunicating with us - parser->user.cd->cfg_dict_item = di; - parser->user.cd->configuration = cfg; - } else { - // register_plugin keeps the item acquired, so we need to release it - dictionary_acquired_item_release(parser->user.host->configurable_plugins, di); - } - - rrdpush_send_dyncfg_enable(parser->user.host, cfg->name); - - return PARSER_RC_OK; -} - -#define LOG_MSG_SIZE (1024) -#define MODULE_NAME_IDX (SERVING_PLUGINSD(parser) ? 1 : 2) -#define MODULE_TYPE_IDX (SERVING_PLUGINSD(parser) ? 2 : 3) -static inline PARSER_RC pluginsd_register_module(char **words __maybe_unused, size_t num_words __maybe_unused, PARSER *parser __maybe_unused) { - netdata_log_info("PLUGINSD: DYNCFG_REG_MODULE"); - - size_t expected_num_words = SERVING_PLUGINSD(parser) ? 3 : 4; - - if (unlikely(num_words != expected_num_words)) { - char log[LOG_MSG_SIZE + 1]; - snprintfz(log, LOG_MSG_SIZE, "expected %zu (got %zu) parameters: %smodule_name module_type", expected_num_words - 1, num_words - 1, SERVING_PLUGINSD(parser) ? "" : "plugin_name "); - return PLUGINSD_DISABLE_PLUGIN(parser, PLUGINSD_KEYWORD_DYNCFG_REGISTER_MODULE, log); - } - - struct configurable_plugin *plug_cfg; - const DICTIONARY_ITEM *di = NULL; - if (SERVING_PLUGINSD(parser)) { - plug_cfg = parser->user.cd->configuration; - if (unlikely(plug_cfg == NULL)) - return PLUGINSD_DISABLE_PLUGIN(parser, PLUGINSD_KEYWORD_DYNCFG_REGISTER_MODULE, "you have to enable dynamic configuration first using " PLUGINSD_KEYWORD_DYNCFG_ENABLE); - } else { - di = dictionary_get_and_acquire_item(parser->user.host->configurable_plugins, words[1]); - if (unlikely(di == NULL)) - return PLUGINSD_DISABLE_PLUGIN(parser, PLUGINSD_KEYWORD_DYNCFG_REGISTER_MODULE, "plugin not found"); - - plug_cfg = (struct configurable_plugin *)dictionary_acquired_item_value(di); - } - - struct module *mod = callocz(1, sizeof(struct module)); - - mod->type = str2_module_type(words[MODULE_TYPE_IDX]); - if (unlikely(mod->type == MOD_TYPE_UNKNOWN)) { - freez(mod); - return PLUGINSD_DISABLE_PLUGIN(parser, PLUGINSD_KEYWORD_DYNCFG_REGISTER_MODULE, "unknown module type (allowed: job_array, single)"); - } - - mod->name = strdupz(words[MODULE_NAME_IDX]); - - mod->set_config_cb = set_module_config_cb; - mod->get_config_cb = get_module_config_cb; - mod->get_config_schema_cb = get_module_config_schema_cb; - mod->config_cb_usr_ctx = parser; - - mod->get_job_config_cb = get_job_config_cb; - mod->get_job_config_schema_cb = get_job_config_schema_cb; - mod->set_job_config_cb = set_job_config_cb; - mod->delete_job_cb = delete_job_cb; - mod->job_config_cb_usr_ctx = parser; - - register_module(parser->user.host->configurable_plugins, plug_cfg, mod, SERVING_PLUGINSD(parser)); - - if (di != NULL) - dictionary_acquired_item_release(parser->user.host->configurable_plugins, di); - - rrdpush_send_dyncfg_reg_module(parser->user.host, plug_cfg->name, mod->name, mod->type); - - return PARSER_RC_OK; -} - -static inline PARSER_RC pluginsd_register_job_common(char **words __maybe_unused, size_t num_words __maybe_unused, PARSER *parser __maybe_unused, const char *plugin_name) { - const char *module_name = words[0]; - const char *job_name = words[1]; - const char *job_type_str = words[2]; - const char *flags_str = words[3]; - - long f = str2l(flags_str); - - if (f < 0) - return PLUGINSD_DISABLE_PLUGIN(parser, PLUGINSD_KEYWORD_DYNCFG_REGISTER_JOB, "invalid flags received"); - - dyncfg_job_flg_t flags = f; - - if (SERVING_PLUGINSD(parser)) - flags |= JOB_FLG_PLUGIN_PUSHED; - else - flags |= JOB_FLG_STREAMING_PUSHED; - - enum job_type job_type = dyncfg_str2job_type(job_type_str); - if (job_type == JOB_TYPE_UNKNOWN) - return PLUGINSD_DISABLE_PLUGIN(parser, PLUGINSD_KEYWORD_DYNCFG_REGISTER_JOB, "unknown job type"); - - if (SERVING_PLUGINSD(parser) && job_type == JOB_TYPE_USER) - return PLUGINSD_DISABLE_PLUGIN(parser, PLUGINSD_KEYWORD_DYNCFG_REGISTER_JOB, "plugins cannot push jobs of type \"user\" (this is allowed only in streaming)"); - - if (register_job(parser->user.host->configurable_plugins, plugin_name, module_name, job_name, job_type, flags, 0)) // ignore existing is off as this is explicitly called register job - return PLUGINSD_DISABLE_PLUGIN(parser, PLUGINSD_KEYWORD_DYNCFG_REGISTER_JOB, "error registering job"); - - rrdpush_send_dyncfg_reg_job(parser->user.host, plugin_name, module_name, job_name, job_type, flags); - - return PARSER_RC_OK; -} - -static inline PARSER_RC pluginsd_register_job(char **words __maybe_unused, size_t num_words __maybe_unused, PARSER *parser __maybe_unused) { - size_t expected_num_words = SERVING_PLUGINSD(parser) ? 5 : 6; - - if (unlikely(num_words != expected_num_words)) { - char log[LOG_MSG_SIZE + 1]; - snprintfz(log, LOG_MSG_SIZE, "expected %zu (got %zu) parameters: %smodule_name job_name job_type", expected_num_words - 1, num_words - 1, SERVING_PLUGINSD(parser) ? "" : "plugin_name "); - return PLUGINSD_DISABLE_PLUGIN(parser, PLUGINSD_KEYWORD_DYNCFG_REGISTER_JOB, log); - } - - if (SERVING_PLUGINSD(parser)) { - return pluginsd_register_job_common(&words[1], num_words - 1, parser, parser->user.cd->configuration->name); - } - return pluginsd_register_job_common(&words[2], num_words - 2, parser, words[1]); -} - -static inline PARSER_RC pluginsd_dyncfg_reset(char **words __maybe_unused, size_t num_words __maybe_unused, PARSER *parser __maybe_unused) { - if (unlikely(num_words != (SERVING_PLUGINSD(parser) ? 1 : 2))) - return PLUGINSD_DISABLE_PLUGIN(parser, PLUGINSD_KEYWORD_DYNCFG_RESET, SERVING_PLUGINSD(parser) ? "expected 0 parameters" : "expected 1 parameter: plugin_name"); - - if (SERVING_PLUGINSD(parser)) { - unregister_plugin(parser->user.host->configurable_plugins, parser->user.cd->cfg_dict_item); - rrdpush_send_dyncfg_reset(parser->user.host, parser->user.cd->configuration->name); - parser->user.cd->configuration = NULL; - } else { - const DICTIONARY_ITEM *di = dictionary_get_and_acquire_item(parser->user.host->configurable_plugins, words[1]); - if (unlikely(di == NULL)) - return PLUGINSD_DISABLE_PLUGIN(parser, PLUGINSD_KEYWORD_DYNCFG_RESET, "plugin not found"); - unregister_plugin(parser->user.host->configurable_plugins, di); - rrdpush_send_dyncfg_reset(parser->user.host, words[1]); - } - - return PARSER_RC_OK; -} - -static inline PARSER_RC pluginsd_job_status_common(char **words, size_t num_words, PARSER *parser, const char *plugin_name) { - int state = str2i(words[3]); - - enum job_status status = str2job_state(words[2]); - if (unlikely(SERVING_PLUGINSD(parser) && status == JOB_STATUS_UNKNOWN)) - return PLUGINSD_DISABLE_PLUGIN(parser, PLUGINSD_KEYWORD_REPORT_JOB_STATUS, "unknown job status"); - - char *message = NULL; - if (num_words == 5 && strlen(words[4]) > 0) - message = words[4]; - - const DICTIONARY_ITEM *plugin_item; - DICTIONARY *job_dict; - const DICTIONARY_ITEM *job_item = report_job_status_acq_lock(parser->user.host->configurable_plugins, &plugin_item, &job_dict, plugin_name, words[0], words[1], status, state, message); - - if (job_item != NULL) { - struct job *job = dictionary_acquired_item_value(job_item); - rrdpush_send_job_status_update(parser->user.host, plugin_name, words[0], job); - - pthread_mutex_unlock(&job->lock); - dictionary_acquired_item_release(job_dict, job_item); - dictionary_acquired_item_release(parser->user.host->configurable_plugins, plugin_item); - } - - return PARSER_RC_OK; -} - -// job_status [plugin_name if streaming] [message] -static PARSER_RC pluginsd_job_status(char **words, size_t num_words, PARSER *parser) { - if (SERVING_PLUGINSD(parser)) { - if (unlikely(num_words != 5 && num_words != 6)) - return PLUGINSD_DISABLE_PLUGIN(parser, PLUGINSD_KEYWORD_REPORT_JOB_STATUS, "expected 4 or 5 parameters: module_name, job_name, status_code, state, [optional: message]"); - } else { - if (unlikely(num_words != 6 && num_words != 7)) - return PLUGINSD_DISABLE_PLUGIN(parser, PLUGINSD_KEYWORD_REPORT_JOB_STATUS, "expected 5 or 6 parameters: plugin_name, module_name, job_name, status_code, state, [optional: message]"); - } - - if (SERVING_PLUGINSD(parser)) { - return pluginsd_job_status_common(&words[1], num_words - 1, parser, parser->user.cd->configuration->name); - } - return pluginsd_job_status_common(&words[2], num_words - 2, parser, words[1]); -} - -static PARSER_RC pluginsd_delete_job(char **words, size_t num_words, PARSER *parser) { - // this can confuse a bit but there is a diference between KEYWORD_DELETE_JOB and actual delete_job function - // they are of opossite direction - if (num_words != 4) - return PLUGINSD_DISABLE_PLUGIN(parser, PLUGINSD_KEYWORD_DELETE_JOB, "expected 2 parameters: plugin_name, module_name, job_name"); - - const char *plugin_name = get_word(words, num_words, 1); - const char *module_name = get_word(words, num_words, 2); - const char *job_name = get_word(words, num_words, 3); - - if (SERVING_STREAMING(parser)) - delete_job_pname(parser->user.host->configurable_plugins, plugin_name, module_name, job_name); - - // forward to parent if any - rrdpush_send_job_deleted(parser->user.host, plugin_name, module_name, job_name); - return PARSER_RC_OK; -} - static inline PARSER_RC streaming_claimed_id(char **words, size_t num_words, PARSER *parser) { const char *host_uuid_str = get_word(words, num_words, 1); @@ -2855,6 +1118,11 @@ static inline PARSER_RC streaming_claimed_id(char **words, size_t num_words, PAR // ---------------------------------------------------------------------------- +void pluginsd_cleanup_v2(PARSER *parser) { + // this is called when the thread is stopped while processing + pluginsd_clear_scope_chart(parser, "THREAD CLEANUP"); +} + void pluginsd_process_thread_cleanup(void *ptr) { PARSER *parser = (PARSER *)ptr; @@ -2996,32 +1264,6 @@ inline size_t pluginsd_process(RRDHOST *host, struct plugind *cd, FILE *fp_plugi return count; } -void pluginsd_keywords_init(PARSER *parser, PARSER_REPERTOIRE repertoire) { - parser_init_repertoire(parser, repertoire); - - if (repertoire & (PARSER_INIT_PLUGINSD | PARSER_INIT_STREAMING)) - inflight_functions_init(parser); -} - -PARSER *parser_init(struct parser_user_object *user, FILE *fp_input, FILE *fp_output, int fd, - PARSER_INPUT_TYPE flags, void *ssl __maybe_unused) { - PARSER *parser; - - parser = callocz(1, sizeof(*parser)); - if(user) - parser->user = *user; - parser->fd = fd; - parser->fp_input = fp_input; - parser->fp_output = fp_output; -#ifdef ENABLE_HTTPS - parser->ssl_output = ssl; -#endif - parser->flags = flags; - - spinlock_init(&parser->writer.spinlock); - return parser; -} - PARSER_RC parser_execute(PARSER *parser, PARSER_KEYWORD *keyword, char **words, size_t num_words) { switch(keyword->id) { case 1: @@ -3078,6 +1320,9 @@ PARSER_RC parser_execute(PARSER *parser, PARSER_KEYWORD *keyword, char **words, case 42: return pluginsd_function_result_begin(words, num_words, parser); + case 43: + return pluginsd_function_progress(words, num_words, parser); + case 51: return pluginsd_label(words, num_words, parser); @@ -3145,25 +1390,6 @@ void parser_init_repertoire(PARSER *parser, PARSER_REPERTOIRE repertoire) { } } -static void parser_destroy_dyncfg(PARSER *parser) { - if (parser->user.cd != NULL && parser->user.cd->configuration != NULL) { - unregister_plugin(parser->user.host->configurable_plugins, parser->user.cd->cfg_dict_item); - parser->user.cd->configuration = NULL; - } else if (parser->user.host != NULL && SERVING_STREAMING(parser) && parser->user.host != localhost){ - dictionary_flush(parser->user.host->configurable_plugins); - } -} - -void parser_destroy(PARSER *parser) { - if (unlikely(!parser)) - return; - - parser_destroy_dyncfg(parser); - - dictionary_destroy(parser->inflight.functions); - freez(parser); -} - int pluginsd_parser_unittest(void) { PARSER *p = parser_init(NULL, NULL, NULL, -1, PARSER_INPUT_SPLIT, NULL); pluginsd_keywords_init(p, PARSER_INIT_PLUGINSD | PARSER_INIT_STREAMING); diff --git a/collectors/plugins.d/pluginsd_parser.h b/collectors/plugins.d/pluginsd_parser.h index 35474642935748..cae7560fed40f8 100644 --- a/collectors/plugins.d/pluginsd_parser.h +++ b/collectors/plugins.d/pluginsd_parser.h @@ -123,7 +123,7 @@ typedef struct parser { struct { DICTIONARY *functions; - usec_t smaller_timeout; + usec_t smaller_monotonic_timeout_ut; } inflight; struct { @@ -136,7 +136,6 @@ PARSER *parser_init(struct parser_user_object *user, FILE *fp_input, FILE *fp_ou void parser_init_repertoire(PARSER *parser, PARSER_REPERTOIRE repertoire); void parser_destroy(PARSER *working_parser); void pluginsd_cleanup_v2(PARSER *parser); -void inflight_functions_init(PARSER *parser); void pluginsd_keywords_init(PARSER *parser, PARSER_REPERTOIRE repertoire); PARSER_RC parser_execute(PARSER *parser, PARSER_KEYWORD *keyword, char **words, size_t num_words); @@ -186,7 +185,7 @@ static inline int parser_action(PARSER *parser, char *input) { if(buffer_strlen(parser->defer.response) > PLUGINSD_MAX_DEFERRED_SIZE) { // more than PLUGINSD_MAX_DEFERRED_SIZE of data, // or a bad plugin that did not send the end_keyword - internal_error(true, "PLUGINSD: deferred response is too big (%zu bytes). Stopping this plugin.", buffer_strlen(parser->defer.response)); + nd_log(NDLS_DAEMON, NDLP_ERR, "PLUGINSD: deferred response is too big (%zu bytes). Stopping this plugin.", buffer_strlen(parser->defer.response)); return 1; } } diff --git a/collectors/plugins.d/pluginsd_replication.c b/collectors/plugins.d/pluginsd_replication.c new file mode 100644 index 00000000000000..8b58a286d4de50 --- /dev/null +++ b/collectors/plugins.d/pluginsd_replication.c @@ -0,0 +1,371 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "pluginsd_replication.h" + +PARSER_RC pluginsd_replay_begin(char **words, size_t num_words, PARSER *parser) { + int idx = 1; + ssize_t slot = pluginsd_parse_rrd_slot(words, num_words); + if(slot >= 0) idx++; + + char *id = get_word(words, num_words, idx++); + char *start_time_str = get_word(words, num_words, idx++); + char *end_time_str = get_word(words, num_words, idx++); + char *child_now_str = get_word(words, num_words, idx++); + + RRDHOST *host = pluginsd_require_scope_host(parser, PLUGINSD_KEYWORD_REPLAY_BEGIN); + if(!host) return PLUGINSD_DISABLE_PLUGIN(parser, NULL, NULL); + + RRDSET *st; + if (likely(!id || !*id)) + st = pluginsd_require_scope_chart(parser, PLUGINSD_KEYWORD_REPLAY_BEGIN, PLUGINSD_KEYWORD_REPLAY_BEGIN); + else + st = pluginsd_rrdset_cache_get_from_slot(parser, host, id, slot, PLUGINSD_KEYWORD_REPLAY_BEGIN); + + if(!st) return PLUGINSD_DISABLE_PLUGIN(parser, NULL, NULL); + + if(!pluginsd_set_scope_chart(parser, st, PLUGINSD_KEYWORD_REPLAY_BEGIN)) + return PLUGINSD_DISABLE_PLUGIN(parser, NULL, NULL); + + if(start_time_str && end_time_str) { + time_t start_time = (time_t) str2ull_encoded(start_time_str); + time_t end_time = (time_t) str2ull_encoded(end_time_str); + + time_t wall_clock_time = 0, tolerance; + bool wall_clock_comes_from_child; (void)wall_clock_comes_from_child; + if(child_now_str) { + wall_clock_time = (time_t) str2ull_encoded(child_now_str); + tolerance = st->update_every + 1; + wall_clock_comes_from_child = true; + } + + if(wall_clock_time <= 0) { + wall_clock_time = now_realtime_sec(); + tolerance = st->update_every + 5; + wall_clock_comes_from_child = false; + } + +#ifdef NETDATA_LOG_REPLICATION_REQUESTS + internal_error( + (!st->replay.start_streaming && (end_time < st->replay.after || start_time > st->replay.before)), + "REPLAY ERROR: 'host:%s/chart:%s' got a " PLUGINSD_KEYWORD_REPLAY_BEGIN " from %ld to %ld, which does not match our request (%ld to %ld).", + rrdhost_hostname(st->rrdhost), rrdset_id(st), start_time, end_time, st->replay.after, st->replay.before); + + internal_error( + true, + "REPLAY: 'host:%s/chart:%s' got a " PLUGINSD_KEYWORD_REPLAY_BEGIN " from %ld to %ld, child wall clock is %ld (%s), had requested %ld to %ld", + rrdhost_hostname(st->rrdhost), rrdset_id(st), + start_time, end_time, wall_clock_time, wall_clock_comes_from_child ? "from child" : "parent time", + st->replay.after, st->replay.before); +#endif + + if(start_time && end_time && start_time < wall_clock_time + tolerance && end_time < wall_clock_time + tolerance && start_time < end_time) { + if (unlikely(end_time - start_time != st->update_every)) + rrdset_set_update_every_s(st, end_time - start_time); + + st->last_collected_time.tv_sec = end_time; + st->last_collected_time.tv_usec = 0; + + st->last_updated.tv_sec = end_time; + st->last_updated.tv_usec = 0; + + st->counter++; + st->counter_done++; + + // these are only needed for db mode RAM, SAVE, MAP, ALLOC + st->db.current_entry++; + if(st->db.current_entry >= st->db.entries) + st->db.current_entry -= st->db.entries; + + parser->user.replay.start_time = start_time; + parser->user.replay.end_time = end_time; + parser->user.replay.start_time_ut = (usec_t) start_time * USEC_PER_SEC; + parser->user.replay.end_time_ut = (usec_t) end_time * USEC_PER_SEC; + parser->user.replay.wall_clock_time = wall_clock_time; + parser->user.replay.rset_enabled = true; + + return PARSER_RC_OK; + } + + netdata_log_error("PLUGINSD REPLAY ERROR: 'host:%s/chart:%s' got a " PLUGINSD_KEYWORD_REPLAY_BEGIN + " from %ld to %ld, but timestamps are invalid " + "(now is %ld [%s], tolerance %ld). Ignoring " PLUGINSD_KEYWORD_REPLAY_SET, + rrdhost_hostname(st->rrdhost), rrdset_id(st), start_time, end_time, + wall_clock_time, wall_clock_comes_from_child ? "child wall clock" : "parent wall clock", + tolerance); + } + + // the child sends an RBEGIN without any parameters initially + // setting rset_enabled to false, means the RSET should not store any metrics + // to store metrics, the RBEGIN needs to have timestamps + parser->user.replay.start_time = 0; + parser->user.replay.end_time = 0; + parser->user.replay.start_time_ut = 0; + parser->user.replay.end_time_ut = 0; + parser->user.replay.wall_clock_time = 0; + parser->user.replay.rset_enabled = false; + return PARSER_RC_OK; +} + +PARSER_RC pluginsd_replay_set(char **words, size_t num_words, PARSER *parser) { + int idx = 1; + ssize_t slot = pluginsd_parse_rrd_slot(words, num_words); + if(slot >= 0) idx++; + + char *dimension = get_word(words, num_words, idx++); + char *value_str = get_word(words, num_words, idx++); + char *flags_str = get_word(words, num_words, idx++); + + RRDHOST *host = pluginsd_require_scope_host(parser, PLUGINSD_KEYWORD_REPLAY_SET); + if(!host) return PLUGINSD_DISABLE_PLUGIN(parser, NULL, NULL); + + RRDSET *st = pluginsd_require_scope_chart(parser, PLUGINSD_KEYWORD_REPLAY_SET, PLUGINSD_KEYWORD_REPLAY_BEGIN); + if(!st) return PLUGINSD_DISABLE_PLUGIN(parser, NULL, NULL); + + if(!parser->user.replay.rset_enabled) { + nd_log_limit_static_thread_var(erl, 1, 0); + nd_log_limit(&erl, NDLS_COLLECTORS, NDLP_ERR, + "PLUGINSD: 'host:%s/chart:%s' got a %s but it is disabled by %s errors", + rrdhost_hostname(host), rrdset_id(st), PLUGINSD_KEYWORD_REPLAY_SET, PLUGINSD_KEYWORD_REPLAY_BEGIN); + + // we have to return OK here + return PARSER_RC_OK; + } + + RRDDIM *rd = pluginsd_acquire_dimension(host, st, dimension, slot, PLUGINSD_KEYWORD_REPLAY_SET); + if(!rd) return PLUGINSD_DISABLE_PLUGIN(parser, NULL, NULL); + + st->pluginsd.set = true; + + if (unlikely(!parser->user.replay.start_time || !parser->user.replay.end_time)) { + netdata_log_error("PLUGINSD: 'host:%s/chart:%s/dim:%s' got a %s with invalid timestamps %ld to %ld from a %s. Disabling it.", + rrdhost_hostname(host), + rrdset_id(st), + dimension, + PLUGINSD_KEYWORD_REPLAY_SET, + parser->user.replay.start_time, + parser->user.replay.end_time, + PLUGINSD_KEYWORD_REPLAY_BEGIN); + return PLUGINSD_DISABLE_PLUGIN(parser, NULL, NULL); + } + + if (unlikely(!value_str || !*value_str)) + value_str = "NAN"; + + if(unlikely(!flags_str)) + flags_str = ""; + + if (likely(value_str)) { + RRDDIM_FLAGS rd_flags = rrddim_flag_check(rd, RRDDIM_FLAG_OBSOLETE | RRDDIM_FLAG_ARCHIVED); + + if(!(rd_flags & RRDDIM_FLAG_ARCHIVED)) { + NETDATA_DOUBLE value = str2ndd_encoded(value_str, NULL); + SN_FLAGS flags = pluginsd_parse_storage_number_flags(flags_str); + + if (!netdata_double_isnumber(value) || (flags == SN_EMPTY_SLOT)) { + value = NAN; + flags = SN_EMPTY_SLOT; + } + + rrddim_store_metric(rd, parser->user.replay.end_time_ut, value, flags); + rd->collector.last_collected_time.tv_sec = parser->user.replay.end_time; + rd->collector.last_collected_time.tv_usec = 0; + rd->collector.counter++; + } + else { + nd_log_limit_static_global_var(erl, 1, 0); + nd_log_limit(&erl, NDLS_COLLECTORS, NDLP_WARNING, + "PLUGINSD: 'host:%s/chart:%s/dim:%s' has the ARCHIVED flag set, but it is replicated. " + "Ignoring data.", + rrdhost_hostname(st->rrdhost), rrdset_id(st), rrddim_name(rd)); + } + } + + return PARSER_RC_OK; +} + +PARSER_RC pluginsd_replay_rrddim_collection_state(char **words, size_t num_words, PARSER *parser) { + if(parser->user.replay.rset_enabled == false) + return PARSER_RC_OK; + + int idx = 1; + ssize_t slot = pluginsd_parse_rrd_slot(words, num_words); + if(slot >= 0) idx++; + + char *dimension = get_word(words, num_words, idx++); + char *last_collected_ut_str = get_word(words, num_words, idx++); + char *last_collected_value_str = get_word(words, num_words, idx++); + char *last_calculated_value_str = get_word(words, num_words, idx++); + char *last_stored_value_str = get_word(words, num_words, idx++); + + RRDHOST *host = pluginsd_require_scope_host(parser, PLUGINSD_KEYWORD_REPLAY_RRDDIM_STATE); + if(!host) return PLUGINSD_DISABLE_PLUGIN(parser, NULL, NULL); + + RRDSET *st = pluginsd_require_scope_chart(parser, PLUGINSD_KEYWORD_REPLAY_RRDDIM_STATE, PLUGINSD_KEYWORD_REPLAY_BEGIN); + if(!st) return PLUGINSD_DISABLE_PLUGIN(parser, NULL, NULL); + + if(st->pluginsd.set) { + // reset pos to reuse the same RDAs + st->pluginsd.pos = 0; + st->pluginsd.set = false; + } + + RRDDIM *rd = pluginsd_acquire_dimension(host, st, dimension, slot, PLUGINSD_KEYWORD_REPLAY_RRDDIM_STATE); + if(!rd) return PLUGINSD_DISABLE_PLUGIN(parser, NULL, NULL); + + usec_t dim_last_collected_ut = (usec_t)rd->collector.last_collected_time.tv_sec * USEC_PER_SEC + (usec_t)rd->collector.last_collected_time.tv_usec; + usec_t last_collected_ut = last_collected_ut_str ? str2ull_encoded(last_collected_ut_str) : 0; + if(last_collected_ut > dim_last_collected_ut) { + rd->collector.last_collected_time.tv_sec = (time_t)(last_collected_ut / USEC_PER_SEC); + rd->collector.last_collected_time.tv_usec = (last_collected_ut % USEC_PER_SEC); + } + + rd->collector.last_collected_value = last_collected_value_str ? str2ll_encoded(last_collected_value_str) : 0; + rd->collector.last_calculated_value = last_calculated_value_str ? str2ndd_encoded(last_calculated_value_str, NULL) : 0; + rd->collector.last_stored_value = last_stored_value_str ? str2ndd_encoded(last_stored_value_str, NULL) : 0.0; + + return PARSER_RC_OK; +} + +PARSER_RC pluginsd_replay_rrdset_collection_state(char **words, size_t num_words, PARSER *parser) { + if(parser->user.replay.rset_enabled == false) + return PARSER_RC_OK; + + char *last_collected_ut_str = get_word(words, num_words, 1); + char *last_updated_ut_str = get_word(words, num_words, 2); + + RRDHOST *host = pluginsd_require_scope_host(parser, PLUGINSD_KEYWORD_REPLAY_RRDSET_STATE); + if(!host) return PLUGINSD_DISABLE_PLUGIN(parser, NULL, NULL); + + RRDSET *st = pluginsd_require_scope_chart(parser, PLUGINSD_KEYWORD_REPLAY_RRDSET_STATE, + PLUGINSD_KEYWORD_REPLAY_BEGIN); + if(!st) return PLUGINSD_DISABLE_PLUGIN(parser, NULL, NULL); + + usec_t chart_last_collected_ut = (usec_t)st->last_collected_time.tv_sec * USEC_PER_SEC + (usec_t)st->last_collected_time.tv_usec; + usec_t last_collected_ut = last_collected_ut_str ? str2ull_encoded(last_collected_ut_str) : 0; + if(last_collected_ut > chart_last_collected_ut) { + st->last_collected_time.tv_sec = (time_t)(last_collected_ut / USEC_PER_SEC); + st->last_collected_time.tv_usec = (last_collected_ut % USEC_PER_SEC); + } + + usec_t chart_last_updated_ut = (usec_t)st->last_updated.tv_sec * USEC_PER_SEC + (usec_t)st->last_updated.tv_usec; + usec_t last_updated_ut = last_updated_ut_str ? str2ull_encoded(last_updated_ut_str) : 0; + if(last_updated_ut > chart_last_updated_ut) { + st->last_updated.tv_sec = (time_t)(last_updated_ut / USEC_PER_SEC); + st->last_updated.tv_usec = (last_updated_ut % USEC_PER_SEC); + } + + st->counter++; + st->counter_done++; + + return PARSER_RC_OK; +} + +PARSER_RC pluginsd_replay_end(char **words, size_t num_words, PARSER *parser) { + if (num_words < 7) { // accepts 7, but the 7th is optional + netdata_log_error("REPLAY: malformed " PLUGINSD_KEYWORD_REPLAY_END " command"); + return PARSER_RC_ERROR; + } + + const char *update_every_child_txt = get_word(words, num_words, 1); + const char *first_entry_child_txt = get_word(words, num_words, 2); + const char *last_entry_child_txt = get_word(words, num_words, 3); + const char *start_streaming_txt = get_word(words, num_words, 4); + const char *first_entry_requested_txt = get_word(words, num_words, 5); + const char *last_entry_requested_txt = get_word(words, num_words, 6); + const char *child_world_time_txt = get_word(words, num_words, 7); // optional + + time_t update_every_child = (time_t) str2ull_encoded(update_every_child_txt); + time_t first_entry_child = (time_t) str2ull_encoded(first_entry_child_txt); + time_t last_entry_child = (time_t) str2ull_encoded(last_entry_child_txt); + + bool start_streaming = (strcmp(start_streaming_txt, "true") == 0); + time_t first_entry_requested = (time_t) str2ull_encoded(first_entry_requested_txt); + time_t last_entry_requested = (time_t) str2ull_encoded(last_entry_requested_txt); + + // the optional child world time + time_t child_world_time = (child_world_time_txt && *child_world_time_txt) ? (time_t) str2ull_encoded( + child_world_time_txt) : now_realtime_sec(); + + RRDHOST *host = pluginsd_require_scope_host(parser, PLUGINSD_KEYWORD_REPLAY_END); + if(!host) return PLUGINSD_DISABLE_PLUGIN(parser, NULL, NULL); + + RRDSET *st = pluginsd_require_scope_chart(parser, PLUGINSD_KEYWORD_REPLAY_END, PLUGINSD_KEYWORD_REPLAY_BEGIN); + if(!st) return PLUGINSD_DISABLE_PLUGIN(parser, NULL, NULL); + +#ifdef NETDATA_LOG_REPLICATION_REQUESTS + internal_error(true, + "PLUGINSD REPLAY: 'host:%s/chart:%s': got a " PLUGINSD_KEYWORD_REPLAY_END " child db from %llu to %llu, start_streaming %s, had requested from %llu to %llu, wall clock %llu", + rrdhost_hostname(host), rrdset_id(st), + (unsigned long long)first_entry_child, (unsigned long long)last_entry_child, + start_streaming?"true":"false", + (unsigned long long)first_entry_requested, (unsigned long long)last_entry_requested, + (unsigned long long)child_world_time + ); +#endif + + parser->user.data_collections_count++; + + if(parser->user.replay.rset_enabled && st->rrdhost->receiver) { + time_t now = now_realtime_sec(); + time_t started = st->rrdhost->receiver->replication_first_time_t; + time_t current = parser->user.replay.end_time; + + if(started && current > started) { + host->rrdpush_receiver_replication_percent = (NETDATA_DOUBLE) (current - started) * 100.0 / (NETDATA_DOUBLE) (now - started); + worker_set_metric(WORKER_RECEIVER_JOB_REPLICATION_COMPLETION, + host->rrdpush_receiver_replication_percent); + } + } + + parser->user.replay.start_time = 0; + parser->user.replay.end_time = 0; + parser->user.replay.start_time_ut = 0; + parser->user.replay.end_time_ut = 0; + parser->user.replay.wall_clock_time = 0; + parser->user.replay.rset_enabled = false; + + st->counter++; + st->counter_done++; + store_metric_collection_completed(); + +#ifdef NETDATA_LOG_REPLICATION_REQUESTS + st->replay.start_streaming = false; + st->replay.after = 0; + st->replay.before = 0; + if(start_streaming) + st->replay.log_next_data_collection = true; +#endif + + if (start_streaming) { + if (st->update_every != update_every_child) + rrdset_set_update_every_s(st, update_every_child); + + if(rrdset_flag_check(st, RRDSET_FLAG_RECEIVER_REPLICATION_IN_PROGRESS)) { + rrdset_flag_set(st, RRDSET_FLAG_RECEIVER_REPLICATION_FINISHED); + rrdset_flag_clear(st, RRDSET_FLAG_RECEIVER_REPLICATION_IN_PROGRESS); + rrdset_flag_clear(st, RRDSET_FLAG_SYNC_CLOCK); + rrdhost_receiver_replicating_charts_minus_one(st->rrdhost); + } +#ifdef NETDATA_LOG_REPLICATION_REQUESTS + else + internal_error(true, "REPLAY ERROR: 'host:%s/chart:%s' got a " PLUGINSD_KEYWORD_REPLAY_END " with enable_streaming = true, but there is no replication in progress for this chart.", + rrdhost_hostname(host), rrdset_id(st)); +#endif + + pluginsd_clear_scope_chart(parser, PLUGINSD_KEYWORD_REPLAY_END); + + host->rrdpush_receiver_replication_percent = 100.0; + worker_set_metric(WORKER_RECEIVER_JOB_REPLICATION_COMPLETION, host->rrdpush_receiver_replication_percent); + + return PARSER_RC_OK; + } + + pluginsd_clear_scope_chart(parser, PLUGINSD_KEYWORD_REPLAY_END); + + rrdcontext_updated_retention_rrdset(st); + + bool ok = replicate_chart_request(send_to_plugin, parser, host, st, + first_entry_child, last_entry_child, child_world_time, + first_entry_requested, last_entry_requested); + return ok ? PARSER_RC_OK : PARSER_RC_ERROR; +} diff --git a/collectors/plugins.d/pluginsd_replication.h b/collectors/plugins.d/pluginsd_replication.h new file mode 100644 index 00000000000000..1c6f617e69fe4d --- /dev/null +++ b/collectors/plugins.d/pluginsd_replication.h @@ -0,0 +1,14 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef NETDATA_PLUGINSD_REPLICATION_H +#define NETDATA_PLUGINSD_REPLICATION_H + +#include "pluginsd_internals.h" + +PARSER_RC pluginsd_replay_begin(char **words, size_t num_words, PARSER *parser); +PARSER_RC pluginsd_replay_set(char **words, size_t num_words, PARSER *parser); +PARSER_RC pluginsd_replay_rrddim_collection_state(char **words, size_t num_words, PARSER *parser); +PARSER_RC pluginsd_replay_rrdset_collection_state(char **words, size_t num_words, PARSER *parser); +PARSER_RC pluginsd_replay_end(char **words, size_t num_words, PARSER *parser); + +#endif //NETDATA_PLUGINSD_REPLICATION_H diff --git a/collectors/proc.plugin/proc_diskstats.c b/collectors/proc.plugin/proc_diskstats.c index 475d90835f5a02..dd35d70401255b 100644 --- a/collectors/proc.plugin/proc_diskstats.c +++ b/collectors/proc.plugin/proc_diskstats.c @@ -1033,12 +1033,16 @@ static void add_labels_to_disk(struct disk *d, RRDSET *st) { rrdlabels_add(st->rrdlabels, "device_type", get_disk_type_string(d->type), RRDLABEL_SRC_AUTO); } -static int diskstats_function_block_devices(BUFFER *wb, int timeout __maybe_unused, const char *function __maybe_unused, - void *collector_data __maybe_unused, - rrd_function_result_callback_t result_cb, void *result_cb_data, - rrd_function_is_cancelled_cb_t is_cancelled_cb, void *is_cancelled_cb_data, - rrd_function_register_canceller_cb_t register_canceller_cb __maybe_unused, - void *register_canceller_cb_data __maybe_unused) { +static int diskstats_function_block_devices(uuid_t *transaction __maybe_unused, BUFFER *wb, + usec_t *stop_monotonic_ut __maybe_unused, const char *function __maybe_unused, + void *collector_data __maybe_unused, + rrd_function_result_callback_t result_cb, void *result_cb_data, + rrd_function_progress_cb_t progress_cb __maybe_unused, void *progress_cb_data __maybe_unused, + rrd_function_is_cancelled_cb_t is_cancelled_cb, void *is_cancelled_cb_data, + rrd_function_register_canceller_cb_t register_canceller_cb __maybe_unused, + void *register_canceller_cb_data __maybe_unused, + rrd_function_register_progresser_cb_t register_progresser_cb __maybe_unused, + void *register_progresser_cb_data __maybe_unused) { buffer_flush(wb); wb->content_type = CT_APPLICATION_JSON; @@ -1490,7 +1494,9 @@ int do_proc_diskstats(int update_every, usec_t dt) { static bool add_func = true; if (add_func) { - rrd_function_add(localhost, NULL, "block-devices", 10, RRDFUNCTIONS_DISKSTATS_HELP, true, diskstats_function_block_devices, NULL); + rrd_function_add(localhost, NULL, "block-devices", 10, RRDFUNCTIONS_PRIORITY_DEFAULT, RRDFUNCTIONS_DISKSTATS_HELP, + "top", HTTP_ACCESS_ANY, true, + diskstats_function_block_devices, NULL); add_func = false; } diff --git a/collectors/proc.plugin/proc_net_dev.c b/collectors/proc.plugin/proc_net_dev.c index 6c7364f1c0a668..55432701f29565 100644 --- a/collectors/proc.plugin/proc_net_dev.c +++ b/collectors/proc.plugin/proc_net_dev.c @@ -610,12 +610,16 @@ static inline void netdev_rename_all_lock(void) { // ---------------------------------------------------------------------------- -int netdev_function_net_interfaces(BUFFER *wb, int timeout __maybe_unused, const char *function __maybe_unused, - void *collector_data __maybe_unused, - rrd_function_result_callback_t result_cb, void *result_cb_data, - rrd_function_is_cancelled_cb_t is_cancelled_cb, void *is_cancelled_cb_data, - rrd_function_register_canceller_cb_t register_canceller_cb __maybe_unused, - void *register_canceller_cb_data __maybe_unused) { +int netdev_function_net_interfaces(uuid_t *transaction __maybe_unused, BUFFER *wb, + usec_t *stop_monotonic_ut __maybe_unused, const char *function __maybe_unused, + void *collector_data __maybe_unused, + rrd_function_result_callback_t result_cb, void *result_cb_data, + rrd_function_progress_cb_t progress_cb __maybe_unused, void *progress_cb_data __maybe_unused, + rrd_function_is_cancelled_cb_t is_cancelled_cb, void *is_cancelled_cb_data, + rrd_function_register_canceller_cb_t register_canceller_cb __maybe_unused, + void *register_canceller_cb_data __maybe_unused, + rrd_function_register_progresser_cb_t register_progresser_cb __maybe_unused, + void *register_progresser_cb_data __maybe_unused) { buffer_flush(wb); wb->content_type = CT_APPLICATION_JSON; @@ -1926,8 +1930,9 @@ void *netdev_main(void *ptr) netdata_thread_cleanup_push(netdev_main_cleanup, ptr) { rrd_collector_started(); - rrd_function_add(localhost, NULL, "network-interfaces", 10, RRDFUNCTIONS_NETDEV_HELP, true - , netdev_function_net_interfaces, NULL); + rrd_function_add(localhost, NULL, "network-interfaces", 10, RRDFUNCTIONS_PRIORITY_DEFAULT, RRDFUNCTIONS_NETDEV_HELP, + "top", HTTP_ACCESS_ANY, + true, netdev_function_net_interfaces, NULL); usec_t step = localhost->rrd_update_every * USEC_PER_SEC; heartbeat_t hb; diff --git a/collectors/systemd-journal.plugin/systemd-internals.h b/collectors/systemd-journal.plugin/systemd-internals.h index e1ae44d4f1b058..80731e6b9e2da5 100644 --- a/collectors/systemd-journal.plugin/systemd-internals.h +++ b/collectors/systemd-journal.plugin/systemd-internals.h @@ -88,7 +88,6 @@ struct journal_file { extern DICTIONARY *journal_files_registry; extern DICTIONARY *used_hashes_registry; -extern DICTIONARY *function_query_status_dict; extern DICTIONARY *boot_ids_to_first_ut; int journal_file_dict_items_backward_compar(const void *a, const void *b); @@ -120,8 +119,7 @@ struct journal_directory { extern struct journal_directory journal_directories[MAX_JOURNAL_DIRECTORIES]; void journal_init_files_and_directories(void); -void journal_init_query_status(void); -void function_systemd_journal(const char *transaction, char *function, int timeout, bool *cancelled); +void function_systemd_journal(const char *transaction, char *function, usec_t *stop_monotonic_ut, bool *cancelled); void journal_file_update_header(const char *filename, struct journal_file *jf); void netdata_systemd_journal_message_ids_init(void); @@ -130,7 +128,7 @@ void netdata_systemd_journal_transform_message_id(FACETS *facets __maybe_unused, void *journal_watcher_main(void *arg); #ifdef ENABLE_SYSTEMD_DBUS -void function_systemd_units(const char *transaction, char *function, int timeout, bool *cancelled); +void function_systemd_units(const char *transaction, char *function, usec_t *stop_monotonic_ut, bool *cancelled); #endif static inline void send_newline_and_flush(void) { diff --git a/collectors/systemd-journal.plugin/systemd-journal.c b/collectors/systemd-journal.plugin/systemd-journal.c index 9ffa1abf7156c0..9addceb2ae2b32 100644 --- a/collectors/systemd-journal.plugin/systemd-journal.c +++ b/collectors/systemd-journal.plugin/systemd-journal.c @@ -26,6 +26,8 @@ #define SYSTEMD_JOURNAL_SAMPLING_SLOTS 1000 #define SYSTEMD_JOURNAL_SAMPLING_RECALIBRATE 10000 +#define SYSTEMD_JOURNAL_PROGRESS_EVERY_UT (250 * USEC_PER_MS) + #define JOURNAL_PARAMETER_HELP "help" #define JOURNAL_PARAMETER_AFTER "after" #define JOURNAL_PARAMETER_BEFORE "before" @@ -39,8 +41,6 @@ #define JOURNAL_PARAMETER_DATA_ONLY "data_only" #define JOURNAL_PARAMETER_SOURCE "source" #define JOURNAL_PARAMETER_INFO "info" -#define JOURNAL_PARAMETER_ID "id" -#define JOURNAL_PARAMETER_PROGRESS "progress" #define JOURNAL_PARAMETER_SLICE "slice" #define JOURNAL_PARAMETER_DELTA "delta" #define JOURNAL_PARAMETER_TAIL "tail" @@ -183,11 +183,11 @@ typedef struct function_query_status { bool *cancelled; // a pointer to the cancelling boolean - usec_t stop_monotonic_ut; - - usec_t started_monotonic_ut; + usec_t *stop_monotonic_ut; // request + const char *transaction; + SD_JOURNAL_FILE_SOURCE_TYPE source_type; SIMPLE_PATTERN *sources; usec_t after_ut; @@ -807,7 +807,7 @@ ND_SD_JOURNAL_STATUS netdata_systemd_journal_query_backward( FUNCTION_PROGRESS_UPDATE_BYTES(fqs->bytes_read, bytes - last_bytes); last_bytes = bytes; - status = check_stop(fqs->cancelled, &fqs->stop_monotonic_ut); + status = check_stop(fqs->cancelled, fqs->stop_monotonic_ut); } } else if(sample == SAMPLING_SKIP_FIELDS) @@ -914,7 +914,7 @@ ND_SD_JOURNAL_STATUS netdata_systemd_journal_query_forward( FUNCTION_PROGRESS_UPDATE_BYTES(fqs->bytes_read, bytes - last_bytes); last_bytes = bytes; - status = check_stop(fqs->cancelled, &fqs->stop_monotonic_ut); + status = check_stop(fqs->cancelled, fqs->stop_monotonic_ut); } } else if(sample == SAMPLING_SKIP_FIELDS) @@ -1150,6 +1150,7 @@ static int netdata_systemd_journal_query(BUFFER *wb, FACETS *facets, FUNCTION_QU usec_t started_ut = query_started_ut; usec_t ended_ut = started_ut; usec_t duration_ut = 0, max_duration_ut = 0; + usec_t progress_duration_ut = 0; sampling_query_init(fqs, facets); @@ -1164,9 +1165,7 @@ static int netdata_systemd_journal_query(BUFFER *wb, FACETS *facets, FUNCTION_QU started_ut = ended_ut; // do not even try to do the query if we expect it to pass the timeout - if(ended_ut > (query_started_ut + (fqs->stop_monotonic_ut - query_started_ut) * 3 / 4) && - ended_ut + max_duration_ut * 2 >= fqs->stop_monotonic_ut) { - + if(ended_ut + max_duration_ut * 3 >= *fqs->stop_monotonic_ut) { partial = true; status = ND_SD_JOURNAL_TIMED_OUT; break; @@ -1211,6 +1210,14 @@ static int netdata_systemd_journal_query(BUFFER *wb, FACETS *facets, FUNCTION_QU if(duration_ut > max_duration_ut) max_duration_ut = duration_ut; + progress_duration_ut += duration_ut; + if(progress_duration_ut >= SYSTEMD_JOURNAL_PROGRESS_EVERY_UT) { + progress_duration_ut = 0; + netdata_mutex_lock(&stdout_mutex); + pluginsd_function_progress_to_stdout(fqs->transaction, f + 1, files_used); + netdata_mutex_unlock(&stdout_mutex); + } + buffer_json_add_array_item_object(wb); // journal file { // information about the file @@ -1422,15 +1429,6 @@ static void netdata_systemd_journal_function_help(const char *transaction) { " all the available systemd journal sources.\n" " When `"JOURNAL_PARAMETER_INFO"` is requested, all other parameters are ignored.\n" "\n" - " "JOURNAL_PARAMETER_ID":STRING\n" - " Caller supplied unique ID of the request.\n" - " This can be used later to request a progress report of the query.\n" - " Optional, but if omitted no `"JOURNAL_PARAMETER_PROGRESS"` can be requested.\n" - "\n" - " "JOURNAL_PARAMETER_PROGRESS"\n" - " Request a progress report (the `id` of a running query is required).\n" - " When `"JOURNAL_PARAMETER_PROGRESS"` is requested, only parameter `"JOURNAL_PARAMETER_ID"` is used.\n" - "\n" " "JOURNAL_PARAMETER_DATA_ONLY":true or "JOURNAL_PARAMETER_DATA_ONLY":false\n" " Quickly respond with data requested, without generating a\n" " `histogram`, `facets` counters and `items`.\n" @@ -1522,67 +1520,7 @@ static void netdata_systemd_journal_function_help(const char *transaction) { buffer_free(wb); } -DICTIONARY *function_query_status_dict = NULL; - -static void function_systemd_journal_progress(BUFFER *wb, const char *transaction, const char *progress_id) { - if(!progress_id || !(*progress_id)) { - netdata_mutex_lock(&stdout_mutex); - pluginsd_function_json_error_to_stdout(transaction, HTTP_RESP_BAD_REQUEST, "missing progress id"); - netdata_mutex_unlock(&stdout_mutex); - return; - } - - const DICTIONARY_ITEM *item = dictionary_get_and_acquire_item(function_query_status_dict, progress_id); - - if(!item) { - netdata_mutex_lock(&stdout_mutex); - pluginsd_function_json_error_to_stdout(transaction, HTTP_RESP_NOT_FOUND, "progress id is not found here"); - netdata_mutex_unlock(&stdout_mutex); - return; - } - - FUNCTION_QUERY_STATUS *fqs = dictionary_acquired_item_value(item); - - usec_t now_monotonic_ut = now_monotonic_usec(); - if(now_monotonic_ut + 10 * USEC_PER_SEC > fqs->stop_monotonic_ut) - fqs->stop_monotonic_ut = now_monotonic_ut + 10 * USEC_PER_SEC; - - usec_t duration_ut = now_monotonic_ut - fqs->started_monotonic_ut; - - size_t files_matched = fqs->files_matched; - size_t file_working = fqs->file_working; - if(file_working > files_matched) - files_matched = file_working; - - size_t rows_read = __atomic_load_n(&fqs->rows_read, __ATOMIC_RELAXED); - size_t bytes_read = __atomic_load_n(&fqs->bytes_read, __ATOMIC_RELAXED); - - buffer_flush(wb); - buffer_json_initialize(wb, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_MINIFY); - buffer_json_member_add_uint64(wb, "status", HTTP_RESP_OK); - buffer_json_member_add_string(wb, "type", "table"); - buffer_json_member_add_uint64(wb, "running_duration_usec", duration_ut); - buffer_json_member_add_double(wb, "progress", (double)file_working * 100.0 / (double)files_matched); - char msg[1024 + 1]; - snprintfz(msg, sizeof(msg) - 1, - "Read %zu rows (%0.0f rows/s), " - "data %0.1f MB (%0.1f MB/s), " - "file %zu of %zu", - rows_read, (double)rows_read / (double)duration_ut * (double)USEC_PER_SEC, - (double)bytes_read / 1024.0 / 1024.0, ((double)bytes_read / (double)duration_ut * (double)USEC_PER_SEC) / 1024.0 / 1024.0, - file_working, files_matched - ); - buffer_json_member_add_string(wb, "message", msg); - buffer_json_finalize(wb); - - netdata_mutex_lock(&stdout_mutex); - pluginsd_function_result_to_stdout(transaction, HTTP_RESP_OK, "application/json", now_realtime_sec() + 1, wb); - netdata_mutex_unlock(&stdout_mutex); - - dictionary_acquired_item_release(function_query_status_dict, item); -} - -void function_systemd_journal(const char *transaction, char *function, int timeout, bool *cancelled) { +void function_systemd_journal(const char *transaction, char *function, usec_t *stop_monotonic_ut, bool *cancelled) { fstat_thread_calls = 0; fstat_thread_cached_responses = 0; @@ -1590,11 +1528,9 @@ void function_systemd_journal(const char *transaction, char *function, int timeo buffer_flush(wb); buffer_json_initialize(wb, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_MINIFY); - usec_t now_monotonic_ut = now_monotonic_usec(); FUNCTION_QUERY_STATUS tmp_fqs = { .cancelled = cancelled, - .started_monotonic_ut = now_monotonic_ut, - .stop_monotonic_ut = now_monotonic_ut + (timeout * USEC_PER_SEC), + .stop_monotonic_ut = stop_monotonic_ut, }; FUNCTION_QUERY_STATUS *fqs = NULL; const DICTIONARY_ITEM *fqs_item = NULL; @@ -1616,8 +1552,6 @@ void function_systemd_journal(const char *transaction, char *function, int timeo facets_accepted_param(facets, JOURNAL_PARAMETER_HISTOGRAM); facets_accepted_param(facets, JOURNAL_PARAMETER_IF_MODIFIED_SINCE); facets_accepted_param(facets, JOURNAL_PARAMETER_DATA_ONLY); - facets_accepted_param(facets, JOURNAL_PARAMETER_ID); - facets_accepted_param(facets, JOURNAL_PARAMETER_PROGRESS); facets_accepted_param(facets, JOURNAL_PARAMETER_DELTA); facets_accepted_param(facets, JOURNAL_PARAMETER_TAIL); facets_accepted_param(facets, JOURNAL_PARAMETER_SAMPLING); @@ -1724,7 +1658,7 @@ void function_systemd_journal(const char *transaction, char *function, int timeo // ------------------------------------------------------------------------ // parse the parameters - bool info = false, data_only = false, progress = false, slice = JOURNAL_DEFAULT_SLICE_MODE, delta = false, tail = false; + bool info = false, data_only = false, slice = JOURNAL_DEFAULT_SLICE_MODE, delta = false, tail = false; time_t after_s = 0, before_s = 0; usec_t anchor = 0; usec_t if_modified_since = 0; @@ -1733,7 +1667,6 @@ void function_systemd_journal(const char *transaction, char *function, int timeo const char *query = NULL; const char *chart = NULL; SIMPLE_PATTERN *sources = NULL; - const char *progress_id = NULL; SD_JOURNAL_FILE_SOURCE_TYPE source_type = SDJF_ALL; size_t filters = 0; size_t sampling = SYSTEMD_JOURNAL_DEFAULT_ITEMS_SAMPLING; @@ -1753,9 +1686,6 @@ void function_systemd_journal(const char *transaction, char *function, int timeo else if(strcmp(keyword, JOURNAL_PARAMETER_INFO) == 0) { info = true; } - else if(strcmp(keyword, JOURNAL_PARAMETER_PROGRESS) == 0) { - progress = true; - } else if(strncmp(keyword, JOURNAL_PARAMETER_DELTA ":", sizeof(JOURNAL_PARAMETER_DELTA ":") - 1) == 0) { char *v = &keyword[sizeof(JOURNAL_PARAMETER_DELTA ":") - 1]; @@ -1791,12 +1721,6 @@ void function_systemd_journal(const char *transaction, char *function, int timeo else slice = true; } - else if(strncmp(keyword, JOURNAL_PARAMETER_ID ":", sizeof(JOURNAL_PARAMETER_ID ":") - 1) == 0) { - char *id = &keyword[sizeof(JOURNAL_PARAMETER_ID ":") - 1]; - - if(*id) - progress_id = id; - } else if(strncmp(keyword, JOURNAL_PARAMETER_SOURCE ":", sizeof(JOURNAL_PARAMETER_SOURCE ":") - 1) == 0) { const char *value = &keyword[sizeof(JOURNAL_PARAMETER_SOURCE ":") - 1]; @@ -1930,15 +1854,8 @@ void function_systemd_journal(const char *transaction, char *function, int timeo // ------------------------------------------------------------------------ // put this request into the progress db - if(progress_id && *progress_id) { - fqs_item = dictionary_set_and_acquire_item(function_query_status_dict, progress_id, &tmp_fqs, sizeof(tmp_fqs)); - fqs = dictionary_acquired_item_value(fqs_item); - } - else { - // no progress id given, proceed without registering our progress in the dictionary - fqs = &tmp_fqs; - fqs_item = NULL; - } + fqs = &tmp_fqs; + fqs_item = NULL; // ------------------------------------------------------------------------ // validate parameters @@ -1969,6 +1886,7 @@ void function_systemd_journal(const char *transaction, char *function, int timeo // ------------------------------------------------------------------------ // set query time-frame, anchors and direction + fqs->transaction = transaction; fqs->after_ut = after_s * USEC_PER_SEC; fqs->before_ut = (before_s * USEC_PER_SEC) + USEC_PER_SEC - 1; fqs->if_modified_since = if_modified_since; @@ -2045,11 +1963,9 @@ void function_systemd_journal(const char *transaction, char *function, int timeo buffer_json_member_add_boolean(wb, JOURNAL_PARAMETER_INFO, false); buffer_json_member_add_boolean(wb, JOURNAL_PARAMETER_SLICE, fqs->slice); buffer_json_member_add_boolean(wb, JOURNAL_PARAMETER_DATA_ONLY, fqs->data_only); - buffer_json_member_add_boolean(wb, JOURNAL_PARAMETER_PROGRESS, false); buffer_json_member_add_boolean(wb, JOURNAL_PARAMETER_DELTA, fqs->delta); buffer_json_member_add_boolean(wb, JOURNAL_PARAMETER_TAIL, fqs->tail); buffer_json_member_add_uint64(wb, JOURNAL_PARAMETER_SAMPLING, fqs->sampling); - buffer_json_member_add_string(wb, JOURNAL_PARAMETER_ID, progress_id); buffer_json_member_add_uint64(wb, "source_type", fqs->source_type); buffer_json_member_add_uint64(wb, JOURNAL_PARAMETER_AFTER, fqs->after_ut / USEC_PER_SEC); buffer_json_member_add_uint64(wb, JOURNAL_PARAMETER_BEFORE, fqs->before_ut / USEC_PER_SEC); @@ -2098,11 +2014,6 @@ void function_systemd_journal(const char *transaction, char *function, int timeo goto output; } - if(progress) { - function_systemd_journal_progress(wb, transaction, progress_id); - goto cleanup; - } - response = netdata_systemd_journal_query(wb, facets, fqs); // ------------------------------------------------------------------------ @@ -2124,16 +2035,4 @@ void function_systemd_journal(const char *transaction, char *function, int timeo simple_pattern_free(sources); facets_destroy(facets); buffer_free(wb); - - if(fqs_item) { - dictionary_del(function_query_status_dict, dictionary_acquired_item_name(fqs_item)); - dictionary_acquired_item_release(function_query_status_dict, fqs_item); - dictionary_garbage_collect(function_query_status_dict); - } -} - -void journal_init_query_status(void) { - function_query_status_dict = dictionary_create_advanced( - DICT_OPTION_DONT_OVERWRITE_VALUE | DICT_OPTION_FIXED_SIZE, - NULL, sizeof(FUNCTION_QUERY_STATUS)); } diff --git a/collectors/systemd-journal.plugin/systemd-main.c b/collectors/systemd-journal.plugin/systemd-main.c index 8d377155d0bb66..3e839749dc2dac 100644 --- a/collectors/systemd-journal.plugin/systemd-main.c +++ b/collectors/systemd-journal.plugin/systemd-main.c @@ -29,7 +29,6 @@ int main(int argc __maybe_unused, char **argv __maybe_unused) { // initialization netdata_systemd_journal_message_ids_init(); - journal_init_query_status(); journal_init_files_and_directories(); if (!journal_data_direcories_exist()) { @@ -46,17 +45,19 @@ int main(int argc __maybe_unused, char **argv __maybe_unused) { journal_files_registry_update(); bool cancelled = false; + usec_t stop_monotonic_ut = now_monotonic_usec() + 600 * USEC_PER_SEC; char buf[] = "systemd-journal after:-8640000 before:0 direction:backward last:200 data_only:false slice:true source:all"; // char buf[] = "systemd-journal after:1695332964 before:1695937764 direction:backward last:100 slice:true source:all DHKucpqUoe1:PtVoyIuX.MU"; // char buf[] = "systemd-journal after:1694511062 before:1694514662 anchor:1694514122024403"; - function_systemd_journal("123", buf, 600, &cancelled); + function_systemd_journal("123", buf, &stop_monotonic_ut, &cancelled); // function_systemd_units("123", "systemd-units", 600, &cancelled); exit(1); } #ifdef ENABLE_SYSTEMD_DBUS if(argc == 2 && strcmp(argv[1], "debug-units") == 0) { bool cancelled = false; - function_systemd_units("123", "systemd-units", 600, &cancelled); + usec_t stop_monotonic_ut = now_monotonic_usec() + 600 * USEC_PER_SEC; + function_systemd_units("123", "systemd-units", &stop_monotonic_ut, &cancelled); exit(1); } #endif @@ -87,12 +88,14 @@ int main(int argc __maybe_unused, char **argv __maybe_unused) { netdata_mutex_lock(&stdout_mutex); - fprintf(stdout, PLUGINSD_KEYWORD_FUNCTION " GLOBAL \"%s\" %d \"%s\"\n", - SYSTEMD_JOURNAL_FUNCTION_NAME, SYSTEMD_JOURNAL_DEFAULT_TIMEOUT, SYSTEMD_JOURNAL_FUNCTION_DESCRIPTION); + fprintf(stdout, PLUGINSD_KEYWORD_FUNCTION " GLOBAL \"%s\" %d \"%s\" \"logs\" \"members\" %d\n", + SYSTEMD_JOURNAL_FUNCTION_NAME, SYSTEMD_JOURNAL_DEFAULT_TIMEOUT, SYSTEMD_JOURNAL_FUNCTION_DESCRIPTION, + RRDFUNCTIONS_PRIORITY_DEFAULT); #ifdef ENABLE_SYSTEMD_DBUS - fprintf(stdout, PLUGINSD_KEYWORD_FUNCTION " GLOBAL \"%s\" %d \"%s\"\n", - SYSTEMD_UNITS_FUNCTION_NAME, SYSTEMD_UNITS_DEFAULT_TIMEOUT, SYSTEMD_UNITS_FUNCTION_DESCRIPTION); + fprintf(stdout, PLUGINSD_KEYWORD_FUNCTION " GLOBAL \"%s\" %d \"%s\" \"top\" \"members\" %d\n", + SYSTEMD_UNITS_FUNCTION_NAME, SYSTEMD_UNITS_DEFAULT_TIMEOUT, SYSTEMD_UNITS_FUNCTION_DESCRIPTION, + RRDFUNCTIONS_PRIORITY_DEFAULT); #endif fflush(stdout); diff --git a/collectors/systemd-journal.plugin/systemd-units.c b/collectors/systemd-journal.plugin/systemd-units.c index dac15881748ab3..5bd3d6c8a83da4 100644 --- a/collectors/systemd-journal.plugin/systemd-units.c +++ b/collectors/systemd-journal.plugin/systemd-units.c @@ -1596,7 +1596,7 @@ void systemd_units_assign_priority(UnitInfo *base) { } } -void function_systemd_units(const char *transaction, char *function, int timeout, bool *cancelled) { +void function_systemd_units(const char *transaction, char *function, usec_t *stop_monotonic_ut __maybe_unused, bool *cancelled __maybe_unused) { char *words[SYSTEMD_UNITS_MAX_PARAMS] = { NULL }; size_t num_words = quoted_strings_splitter_pluginsd(function, words, SYSTEMD_UNITS_MAX_PARAMS); for(int i = 1; i < SYSTEMD_UNITS_MAX_PARAMS ;i++) { diff --git a/daemon/main.c b/daemon/main.c index 7974daab229e48..5e262d51ad8696 100644 --- a/daemon/main.c +++ b/daemon/main.c @@ -1395,6 +1395,7 @@ void replication_initialize(void); void bearer_tokens_init(void); int unittest_rrdpush_compressions(void); int uuid_unittest(void); +int progress_unittest(void); int main(int argc, char **argv) { // initialize the system clocks @@ -1620,6 +1621,10 @@ int main(int argc, char **argv) { unittest_running = true; return unittest_rrdpush_compressions(); } + else if(strcmp(optarg, "progresstest") == 0) { + unittest_running = true; + return progress_unittest(); + } else if(strncmp(optarg, createdataset_string, strlen(createdataset_string)) == 0) { optarg += strlen(createdataset_string); unsigned history_seconds = strtoul(optarg, NULL, 0); diff --git a/database/contexts/api_v2.c b/database/contexts/api_v2.c index 3ca49a319522b2..21851cfbba34d0 100644 --- a/database/contexts/api_v2.c +++ b/database/contexts/api_v2.c @@ -167,6 +167,9 @@ struct function_v2_entry { size_t used; size_t *node_ids; STRING *help; + STRING *tags; + HTTP_ACCESS access; + int priority; }; struct context_v2_entry { @@ -913,8 +916,11 @@ static ssize_t rrdcontext_to_json_v2_add_host(void *data, RRDHOST *host, bool qu .size = 1, .node_ids = &ctl->nodes.ni, .help = NULL, + .tags = NULL, + .access = HTTP_ACCESS_MEMBERS, + .priority = RRDFUNCTIONS_PRIORITY_DEFAULT, }; - host_functions_to_dict(host, ctl->functions.dict, &t, sizeof(t), &t.help); + host_functions_to_dict(host, ctl->functions.dict, &t, sizeof(t), &t.help, &t.tags, &t.access, &t.priority); } if(ctl->mode & CONTEXTS_V2_NODES) { @@ -2062,12 +2068,19 @@ int rrdcontext_to_json_v2(BUFFER *wb, struct api_v2_contexts_request *req, CONTE struct function_v2_entry *t; dfe_start_read(ctl.functions.dict, t) { buffer_json_add_array_item_object(wb); - buffer_json_member_add_string(wb, "name", t_dfe.name); - buffer_json_member_add_string(wb, "help", string2str(t->help)); - buffer_json_member_add_array(wb, "ni"); - for (size_t i = 0; i < t->used; i++) - buffer_json_add_array_item_uint64(wb, t->node_ids[i]); - buffer_json_array_close(wb); + { + buffer_json_member_add_string(wb, "name", t_dfe.name); + buffer_json_member_add_string(wb, "help", string2str(t->help)); + buffer_json_member_add_array(wb, "ni"); + { + for (size_t i = 0; i < t->used; i++) + buffer_json_add_array_item_uint64(wb, t->node_ids[i]); + } + buffer_json_array_close(wb); + buffer_json_member_add_string(wb, "tags", string2str(t->tags)); + buffer_json_member_add_string(wb, "access", http_id2access(t->access)); + buffer_json_member_add_uint64(wb, "priority", t->priority); + } buffer_json_object_close(wb); } dfe_done(t); diff --git a/database/contexts/rrdcontext.h b/database/contexts/rrdcontext.h index 9c497a5a5ee7c6..b0b7060e5999ef 100644 --- a/database/contexts/rrdcontext.h +++ b/database/contexts/rrdcontext.h @@ -299,6 +299,8 @@ typedef struct query_target_request { qt_interrupt_callback_t interrupt_callback; void *interrupt_callback_data; + + uuid_t *transaction; } QUERY_TARGET_REQUEST; #define GROUP_BY_MAX_LABEL_KEYS 10 diff --git a/database/rrd.h b/database/rrd.h index 2f4f2a0d3667dd..25f11431a1743e 100644 --- a/database/rrd.h +++ b/database/rrd.h @@ -966,6 +966,7 @@ void rrdset_memory_file_update(RRDSET *st); const char *rrdset_cache_filename(RRDSET *st); bool rrdset_memory_load_or_create_map_save(RRDSET *st_on_file, RRD_MEMORY_MODE memory_mode); +#include "rrdcollector.h" #include "rrdfunctions.h" // ---------------------------------------------------------------------------- diff --git a/database/rrdcollector.c b/database/rrdcollector.c new file mode 100644 index 00000000000000..b776433c75f852 --- /dev/null +++ b/database/rrdcollector.c @@ -0,0 +1,137 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#define NETDATA_RRDCOLLECTOR_INTERNALS +#include "rrdcollector.h" + +// Each function points to this collector structure +// so that when the collector exits, all of them will +// be invalidated (running == false) +// The last function using this collector +// frees the structure too (or when the collector calls +// rrdset_collector_finished()). + +struct rrd_collector { + int32_t refcount; + int32_t refcount_dispatcher; + pid_t tid; + bool running; +}; + +// Each thread that adds RRDSET functions has to call +// rrdset_collector_started() and rrdset_collector_finished() +// to create the collector structure. + +__thread struct rrd_collector *thread_rrd_collector = NULL; + +inline bool rrd_collector_running(struct rrd_collector *rdc) { + return __atomic_load_n(&rdc->running, __ATOMIC_RELAXED); +} + +inline pid_t rrd_collector_tid(struct rrd_collector *rdc) { + return rdc->tid; +} + +bool rrd_collector_dispatcher_acquire(struct rrd_collector *rdc) { + int32_t expected = __atomic_load_n(&rdc->refcount_dispatcher, __ATOMIC_RELAXED); + int32_t wanted; + do { + if(expected < 0) + return false; + + wanted = expected + 1; + } while(!__atomic_compare_exchange_n(&rdc->refcount_dispatcher, &expected, wanted, false, __ATOMIC_RELAXED, __ATOMIC_RELAXED)); + + return true; +} + +void rrd_collector_dispatcher_release(struct rrd_collector *rdc) { + __atomic_sub_fetch(&rdc->refcount_dispatcher, 1, __ATOMIC_RELAXED); +} + +static void rrd_collector_free(struct rrd_collector *rdc) { + if(rdc->running) + return; + + int32_t expected = 0; + if(!__atomic_compare_exchange_n(&rdc->refcount, &expected, -1, false, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED)) { + // the collector is still referenced by charts. + // leave it hanging there, the last chart will actually free it. + return; + } + + // we can free it now + freez(rdc); +} + +// called once per collector +void rrd_collector_started(void) { + if(!thread_rrd_collector) + thread_rrd_collector = callocz(1, sizeof(struct rrd_collector)); + + thread_rrd_collector->tid = gettid(); + __atomic_store_n(&thread_rrd_collector->running, true, __ATOMIC_RELAXED); +} + +// called once per collector +void rrd_collector_finished(void) { + if(!thread_rrd_collector) + return; + + __atomic_store_n(&thread_rrd_collector->running, false, __ATOMIC_RELAXED); + + // wait for any cancellation requests to be dispatched; + // the problem is that cancellation requests require a structure allocated by the collector, + // so, while cancellation requests are being dispatched, this structure is accessed. + // delaying the exit of the thread is required to avoid cleaning up this structure. + + int32_t expected = 0; + while(!__atomic_compare_exchange_n(&thread_rrd_collector->refcount_dispatcher, &expected, -1, false, __ATOMIC_RELAXED, __ATOMIC_RELAXED)) { + expected = 0; + sleep_usec(1 * USEC_PER_MS); + } + + rrd_collector_free(thread_rrd_collector); + thread_rrd_collector = NULL; +} + +bool rrd_collector_acquire(struct rrd_collector *rdc) { + + int32_t expected = __atomic_load_n(&rdc->refcount, __ATOMIC_RELAXED), wanted = 0; + do { + if(expected < 0 || !rrd_collector_running(rdc)) + return false; + + wanted = expected + 1; + } while(!__atomic_compare_exchange_n(&rdc->refcount, &expected, wanted, false, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED)); + + return true; +} + +struct rrd_collector *rrd_collector_acquire_current_thread(void) { + rrd_collector_started(); + + if(!rrd_collector_acquire(thread_rrd_collector)) + internal_fatal(true, "FUNCTIONS: Trying to acquire a the current thread collector, that is currently exiting."); + + return thread_rrd_collector; +} + +void rrd_collector_release(struct rrd_collector *rdc) { + if(unlikely(!rdc)) return; + + int32_t expected = __atomic_load_n(&rdc->refcount, __ATOMIC_RELAXED), wanted = 0; + do { + if(expected < 0) + return; + + if(expected == 0) { + internal_fatal(true, "FUNCTIONS: Trying to release a collector that is not acquired."); + return; + } + + wanted = expected - 1; + } while(!__atomic_compare_exchange_n(&rdc->refcount, &expected, wanted, false, __ATOMIC_RELEASE, __ATOMIC_RELAXED)); + + if(wanted == 0) + rrd_collector_free(rdc); +} diff --git a/database/rrdcollector.h b/database/rrdcollector.h new file mode 100644 index 00000000000000..aeab1de52eaf5a --- /dev/null +++ b/database/rrdcollector.h @@ -0,0 +1,30 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef NETDATA_RRDCOLLECTOR_H +#define NETDATA_RRDCOLLECTOR_H + +#include "rrd.h" + +#ifdef NETDATA_RRDCOLLECTOR_INTERNALS + +// ---------------------------------------------------------------------------- +// private API + +struct rrd_collector; +struct rrd_collector *rrd_collector_acquire_current_thread(void); +void rrd_collector_release(struct rrd_collector *rdc); +extern __thread struct rrd_collector *thread_rrd_collector; +bool rrd_collector_running(struct rrd_collector *rdc); +pid_t rrd_collector_tid(struct rrd_collector *rdc); +bool rrd_collector_dispatcher_acquire(struct rrd_collector *rdc); +void rrd_collector_dispatcher_release(struct rrd_collector *rdc); + +#endif // NETDATA_RRDCOLLECTOR_INTERNALS + +// ---------------------------------------------------------------------------- +// public API + +void rrd_collector_started(void); +void rrd_collector_finished(void); + +#endif //NETDATA_RRDCOLLECTOR_H diff --git a/database/rrdfunctions.c b/database/rrdfunctions.c index 2659130f036639..a204343c189fe8 100644 --- a/database/rrdfunctions.c +++ b/database/rrdfunctions.c @@ -1,266 +1,68 @@ // SPDX-License-Identifier: GPL-3.0-or-later #define NETDATA_RRD_INTERNALS +#define NETDATA_RRDCOLLECTOR_INTERNALS + #include "rrd.h" #define MAX_FUNCTION_LENGTH (PLUGINSD_LINE_MAX - 512) // we need some space for the rest of the line static unsigned char functions_allowed_chars[256] = { - [0] = '\0', // - [1] = '_', // - [2] = '_', // - [3] = '_', // - [4] = '_', // - [5] = '_', // - [6] = '_', // - [7] = '_', // - [8] = '_', // - [9] = ' ', // Horizontal Tab - [10] = ' ', // Line Feed - [11] = ' ', // Vertical Tab - [12] = ' ', // Form Feed - [13] = ' ', // Carriage Return - [14] = '_', // - [15] = '_', // - [16] = '_', // - [17] = '_', // - [18] = '_', // - [19] = '_', // - [20] = '_', // - [21] = '_', // - [22] = '_', // - [23] = '_', // - [24] = '_', // - [25] = '_', // - [26] = '_', // - [27] = '_', // - [28] = '_', // - [29] = '_', // - [30] = '_', // - [31] = '_', // - [32] = ' ', // SPACE keep - [33] = '!', // ! keep - [34] = '"', // " keep - [35] = '#', // # keep - [36] = '$', // $ keep - [37] = '%', // % keep - [38] = '&', // & keep - [39] = '\'', // ' keep - [40] = '(', // ( keep - [41] = ')', // ) keep - [42] = '*', // * keep - [43] = '+', // + keep - [44] = ',', // , keep - [45] = '-', // - keep - [46] = '.', // . keep - [47] = '/', // / keep - [48] = '0', // 0 keep - [49] = '1', // 1 keep - [50] = '2', // 2 keep - [51] = '3', // 3 keep - [52] = '4', // 4 keep - [53] = '5', // 5 keep - [54] = '6', // 6 keep - [55] = '7', // 7 keep - [56] = '8', // 8 keep - [57] = '9', // 9 keep - [58] = ':', // : keep - [59] = ';', // ; keep - [60] = '<', // < keep - [61] = '=', // = keep - [62] = '>', // > keep - [63] = '?', // ? keep - [64] = '@', // @ keep - [65] = 'A', // A keep - [66] = 'B', // B keep - [67] = 'C', // C keep - [68] = 'D', // D keep - [69] = 'E', // E keep - [70] = 'F', // F keep - [71] = 'G', // G keep - [72] = 'H', // H keep - [73] = 'I', // I keep - [74] = 'J', // J keep - [75] = 'K', // K keep - [76] = 'L', // L keep - [77] = 'M', // M keep - [78] = 'N', // N keep - [79] = 'O', // O keep - [80] = 'P', // P keep - [81] = 'Q', // Q keep - [82] = 'R', // R keep - [83] = 'S', // S keep - [84] = 'T', // T keep - [85] = 'U', // U keep - [86] = 'V', // V keep - [87] = 'W', // W keep - [88] = 'X', // X keep - [89] = 'Y', // Y keep - [90] = 'Z', // Z keep - [91] = '[', // [ keep - [92] = '\\', // backslash keep - [93] = ']', // ] keep - [94] = '^', // ^ keep - [95] = '_', // _ keep - [96] = '`', // ` keep - [97] = 'a', // a keep - [98] = 'b', // b keep - [99] = 'c', // c keep - [100] = 'd', // d keep - [101] = 'e', // e keep - [102] = 'f', // f keep - [103] = 'g', // g keep - [104] = 'h', // h keep - [105] = 'i', // i keep - [106] = 'j', // j keep - [107] = 'k', // k keep - [108] = 'l', // l keep - [109] = 'm', // m keep - [110] = 'n', // n keep - [111] = 'o', // o keep - [112] = 'p', // p keep - [113] = 'q', // q keep - [114] = 'r', // r keep - [115] = 's', // s keep - [116] = 't', // t keep - [117] = 'u', // u keep - [118] = 'v', // v keep - [119] = 'w', // w keep - [120] = 'x', // x keep - [121] = 'y', // y keep - [122] = 'z', // z keep - [123] = '{', // { keep - [124] = '|', // | keep - [125] = '}', // } keep - [126] = '~', // ~ keep - [127] = '_', // - [128] = '_', // - [129] = '_', // - [130] = '_', // - [131] = '_', // - [132] = '_', // - [133] = '_', // - [134] = '_', // - [135] = '_', // - [136] = '_', // - [137] = '_', // - [138] = '_', // - [139] = '_', // - [140] = '_', // - [141] = '_', // - [142] = '_', // - [143] = '_', // - [144] = '_', // - [145] = '_', // - [146] = '_', // - [147] = '_', // - [148] = '_', // - [149] = '_', // - [150] = '_', // - [151] = '_', // - [152] = '_', // - [153] = '_', // - [154] = '_', // - [155] = '_', // - [156] = '_', // - [157] = '_', // - [158] = '_', // - [159] = '_', // - [160] = '_', // - [161] = '_', // - [162] = '_', // - [163] = '_', // - [164] = '_', // - [165] = '_', // - [166] = '_', // - [167] = '_', // - [168] = '_', // - [169] = '_', // - [170] = '_', // - [171] = '_', // - [172] = '_', // - [173] = '_', // - [174] = '_', // - [175] = '_', // - [176] = '_', // - [177] = '_', // - [178] = '_', // - [179] = '_', // - [180] = '_', // - [181] = '_', // - [182] = '_', // - [183] = '_', // - [184] = '_', // - [185] = '_', // - [186] = '_', // - [187] = '_', // - [188] = '_', // - [189] = '_', // - [190] = '_', // - [191] = '_', // - [192] = '_', // - [193] = '_', // - [194] = '_', // - [195] = '_', // - [196] = '_', // - [197] = '_', // - [198] = '_', // - [199] = '_', // - [200] = '_', // - [201] = '_', // - [202] = '_', // - [203] = '_', // - [204] = '_', // - [205] = '_', // - [206] = '_', // - [207] = '_', // - [208] = '_', // - [209] = '_', // - [210] = '_', // - [211] = '_', // - [212] = '_', // - [213] = '_', // - [214] = '_', // - [215] = '_', // - [216] = '_', // - [217] = '_', // - [218] = '_', // - [219] = '_', // - [220] = '_', // - [221] = '_', // - [222] = '_', // - [223] = '_', // - [224] = '_', // - [225] = '_', // - [226] = '_', // - [227] = '_', // - [228] = '_', // - [229] = '_', // - [230] = '_', // - [231] = '_', // - [232] = '_', // - [233] = '_', // - [234] = '_', // - [235] = '_', // - [236] = '_', // - [237] = '_', // - [238] = '_', // - [239] = '_', // - [240] = '_', // - [241] = '_', // - [242] = '_', // - [243] = '_', // - [244] = '_', // - [245] = '_', // - [246] = '_', // - [247] = '_', // - [248] = '_', // - [249] = '_', // - [250] = '_', // - [251] = '_', // - [252] = '_', // - [253] = '_', // - [254] = '_', // - [255] = '_' // + [0] = '\0', [1] = '_', [2] = '_', [3] = '_', [4] = '_', [5] = '_', [6] = '_', [7] = '_', [8] = '_', + + // control + ['\t'] = ' ', ['\n'] = ' ', ['\v'] = ' ', [12] = ' ', ['\r'] = ' ', + + [14] = '_', [15] = '_', [16] = '_', [17] = '_', [18] = '_', [19] = '_', [20] = '_', [21] = '_', + [22] = '_', [23] = '_', [24] = '_', [25] = '_', [26] = '_', [27] = '_', [28] = '_', [29] = '_', + [30] = '_', [31] = '_', + + // symbols + [' '] = ' ', ['!'] = '!', ['"'] = '"', ['#'] = '#', ['$'] = '$', ['%'] = '%', ['&'] = '&', ['\''] = '\'', + ['('] = '(', [')'] = ')', ['*'] = '*', ['+'] = '+', [','] = ',', ['-'] = '-', ['.'] = '.', ['/'] = '/', + + // numbers + ['0'] = '0', ['1'] = '1', ['2'] = '2', ['3'] = '3', ['4'] = '4', ['5'] = '5', ['6'] = '6', ['7'] = '7', + ['8'] = '8', ['9'] = '9', + + // symbols + [':'] = ':', [';'] = ';', ['<'] = '<', ['='] = '=', ['>'] = '>', ['?'] = '?', ['@'] = '@', + + // capitals + ['A'] = 'A', ['B'] = 'B', ['C'] = 'C', ['D'] = 'D', ['E'] = 'E', ['F'] = 'F', ['G'] = 'G', ['H'] = 'H', + ['I'] = 'I', ['J'] = 'J', ['K'] = 'K', ['L'] = 'L', ['M'] = 'M', ['N'] = 'N', ['O'] = 'O', ['P'] = 'P', + ['Q'] = 'Q', ['R'] = 'R', ['S'] = 'S', ['T'] = 'T', ['U'] = 'U', ['V'] = 'V', ['W'] = 'W', ['X'] = 'X', + ['Y'] = 'Y', ['Z'] = 'Z', + + // symbols + ['['] = '[', ['\\'] = '\\', [']'] = ']', ['^'] = '^', ['_'] = '_', ['`'] = '`', + + // lower + ['a'] = 'a', ['b'] = 'b', ['c'] = 'c', ['d'] = 'd', ['e'] = 'e', ['f'] = 'f', ['g'] = 'g', ['h'] = 'h', + ['i'] = 'i', ['j'] = 'j', ['k'] = 'k', ['l'] = 'l', ['m'] = 'm', ['n'] = 'n', ['o'] = 'o', ['p'] = 'p', + ['q'] = 'q', ['r'] = 'r', ['s'] = 's', ['t'] = 't', ['u'] = 'u', ['v'] = 'v', ['w'] = 'w', ['x'] = 'x', + ['y'] = 'y', ['z'] = 'z', + + // symbols + ['{'] = '{', ['|'] = '|', ['}'] = '}', ['~'] = '~', + + // rest + [127] = '_', [128] = '_', [129] = '_', [130] = '_', [131] = '_', [132] = '_', [133] = '_', [134] = '_', + [135] = '_', [136] = '_', [137] = '_', [138] = '_', [139] = '_', [140] = '_', [141] = '_', [142] = '_', + [143] = '_', [144] = '_', [145] = '_', [146] = '_', [147] = '_', [148] = '_', [149] = '_', [150] = '_', + [151] = '_', [152] = '_', [153] = '_', [154] = '_', [155] = '_', [156] = '_', [157] = '_', [158] = '_', + [159] = '_', [160] = '_', [161] = '_', [162] = '_', [163] = '_', [164] = '_', [165] = '_', [166] = '_', + [167] = '_', [168] = '_', [169] = '_', [170] = '_', [171] = '_', [172] = '_', [173] = '_', [174] = '_', + [175] = '_', [176] = '_', [177] = '_', [178] = '_', [179] = '_', [180] = '_', [181] = '_', [182] = '_', + [183] = '_', [184] = '_', [185] = '_', [186] = '_', [187] = '_', [188] = '_', [189] = '_', [190] = '_', + [191] = '_', [192] = '_', [193] = '_', [194] = '_', [195] = '_', [196] = '_', [197] = '_', [198] = '_', + [199] = '_', [200] = '_', [201] = '_', [202] = '_', [203] = '_', [204] = '_', [205] = '_', [206] = '_', + [207] = '_', [208] = '_', [209] = '_', [210] = '_', [211] = '_', [212] = '_', [213] = '_', [214] = '_', + [215] = '_', [216] = '_', [217] = '_', [218] = '_', [219] = '_', [220] = '_', [221] = '_', [222] = '_', + [223] = '_', [224] = '_', [225] = '_', [226] = '_', [227] = '_', [228] = '_', [229] = '_', [230] = '_', + [231] = '_', [232] = '_', [233] = '_', [234] = '_', [235] = '_', [236] = '_', [237] = '_', [238] = '_', + [239] = '_', [240] = '_', [241] = '_', [242] = '_', [243] = '_', [244] = '_', [245] = '_', [246] = '_', + [247] = '_', [248] = '_', [249] = '_', [250] = '_', [251] = '_', [252] = '_', [253] = '_', [254] = '_', + [255] = '_' }; static inline size_t sanitize_function_text(char *dst, const char *src, size_t dst_len) { @@ -278,127 +80,96 @@ typedef enum __attribute__((packed)) { // this is 8-bit } RRD_FUNCTION_OPTIONS; +// ---------------------------------------------------------------------------- + struct rrd_host_function { bool sync; // when true, the function is called synchronously RRD_FUNCTION_OPTIONS options; // RRD_FUNCTION_OPTIONS + HTTP_ACCESS access; STRING *help; + STRING *tags; int timeout; // the default timeout of the function + int priority; rrd_function_execute_cb_t execute_cb; - void *execute_cb_data; - struct rrd_collector *collector; -}; -// Each function points to this collector structure -// so that when the collector exits, all of them will -// be invalidated (running == false) -// The last function that is using this collector -// frees the structure too (or when the collector calls -// rrdset_collector_finished()). - -struct rrd_collector { - int32_t refcount; - int32_t refcount_canceller; - pid_t tid; - bool running; + struct rrd_collector *collector; }; -// Each thread that adds RRDSET functions, has to call -// rrdset_collector_started() and rrdset_collector_finished() -// to create the collector structure. - -static __thread struct rrd_collector *thread_rrd_collector = NULL; - -static void rrd_collector_free(struct rrd_collector *rdc) { - if(rdc->running) - return; - - int32_t expected = 0; - if(!__atomic_compare_exchange_n(&rdc->refcount, &expected, -1, false, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED)) { - // the collector is still referenced by charts. - // leave it hanging there, the last chart will actually free it. - return; - } - - // we can free it now - freez(rdc); -} - -// called once per collector -void rrd_collector_started(void) { - if(!thread_rrd_collector) - thread_rrd_collector = callocz(1, sizeof(struct rrd_collector)); - - thread_rrd_collector->tid = gettid(); - __atomic_store_n(&thread_rrd_collector->running, true, __ATOMIC_RELAXED); -} - -// called once per collector -void rrd_collector_finished(void) { - if(!thread_rrd_collector) - return; - - __atomic_store_n(&thread_rrd_collector->running, false, __ATOMIC_RELAXED); - - int32_t expected = 0; - while(!__atomic_compare_exchange_n(&thread_rrd_collector->refcount_canceller, &expected, -1, false, __ATOMIC_RELAXED, __ATOMIC_RELAXED)) { - expected = 0; - sleep_usec(1 * USEC_PER_MS); - } - - rrd_collector_free(thread_rrd_collector); - thread_rrd_collector = NULL; -} +struct rrd_function_inflight { + bool used; -#define rrd_collector_running(c) __atomic_load_n(&(c)->running, __ATOMIC_RELAXED) + RRDHOST *host; + uuid_t transaction_uuid; + const char *transaction; + const char *cmd; + const char *sanitized_cmd; + size_t sanitized_cmd_length; + int timeout; + bool cancelled; + usec_t stop_monotonic_ut; -static struct rrd_collector *rrd_collector_acquire(void) { - rrd_collector_started(); + const DICTIONARY_ITEM *host_function_acquired; - int32_t expected = __atomic_load_n(&thread_rrd_collector->refcount, __ATOMIC_RELAXED), wanted = 0; - do { - if(expected < 0 || !rrd_collector_running(thread_rrd_collector)) { - internal_fatal(true, "FUNCTIONS: Trying to acquire a collector that is exiting."); - return thread_rrd_collector; - } + // the collector + // we acquire this structure at the beginning, + // and we release it at the end + struct rrd_host_function *rdcf; - wanted = expected + 1; + struct { + BUFFER *wb; - } while(!__atomic_compare_exchange_n(&thread_rrd_collector->refcount, &expected, wanted, false, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED)); + // in async mode, + // the function to call to send the result back + rrd_function_result_callback_t cb; + void *data; + } result; - return thread_rrd_collector; -} + struct { + // to be called in sync mode + // while the function is running + // to check if the function has been canceled + rrd_function_is_cancelled_cb_t cb; + void *data; + } is_cancelled; -static void rrd_collector_release(struct rrd_collector *rdc) { - if(unlikely(!rdc)) return; + struct { + // to be registered by the function itself + // used to signal the function to cancel + rrd_function_cancel_cb_t cb; + void *data; + } canceller; - int32_t expected = __atomic_load_n(&rdc->refcount, __ATOMIC_RELAXED), wanted = 0; - do { - if(expected < 0) { - internal_fatal(true, "FUNCTIONS: Trying to release a collector that is exiting."); - return; - } + struct { + // callback to receive progress reports from function + rrd_function_progress_cb_t cb; + void *data; + } progress; - if(expected == 0) { - internal_fatal(true, "FUNCTIONS: Trying to release a collector that is not acquired."); - return; - } + struct { + // to be registered by the function itself + // used to send progress requests to function + rrd_function_progresser_cb_t cb; + void *data; + } progresser; +}; - wanted = expected - 1; +static DICTIONARY *rrd_functions_inflight_requests = NULL; - } while(!__atomic_compare_exchange_n(&rdc->refcount, &expected, wanted, false, __ATOMIC_RELEASE, __ATOMIC_RELAXED)); +static void rrd_function_cancel_inflight(struct rrd_function_inflight *r); - if(wanted == 0) - rrd_collector_free(rdc); -} +// ---------------------------------------------------------------------------- static void rrd_functions_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, void *func, void *rrdhost) { RRDHOST *host = rrdhost; (void)host; struct rrd_host_function *rdcf = func; rrd_collector_started(); - rdcf->collector = rrd_collector_acquire(); + rdcf->collector = rrd_collector_acquire_current_thread(); + + if(!rdcf->priority) + rdcf->priority = RRDFUNCTIONS_PRIORITY_DEFAULT; // internal_error(true, "FUNCTIONS: adding function '%s' on host '%s', collection tid %d, %s", // dictionary_acquired_item_name(item), rrdhost_hostname(host), @@ -422,26 +193,30 @@ static bool rrd_functions_conflict_callback(const DICTIONARY_ITEM *item __maybe_ bool changed = false; if(rdcf->collector != thread_rrd_collector) { - netdata_log_info("FUNCTIONS: function '%s' of host '%s' changed collector from %d to %d", - dictionary_acquired_item_name(item), rrdhost_hostname(host), rdcf->collector->tid, thread_rrd_collector->tid); + nd_log(NDLS_DAEMON, NDLP_DEBUG, + "FUNCTIONS: function '%s' of host '%s' changed collector from %d to %d", + dictionary_acquired_item_name(item), rrdhost_hostname(host), + rrd_collector_tid(rdcf->collector), rrd_collector_tid(thread_rrd_collector)); struct rrd_collector *old_rdc = rdcf->collector; - rdcf->collector = rrd_collector_acquire(); + rdcf->collector = rrd_collector_acquire_current_thread(); rrd_collector_release(old_rdc); changed = true; } if(rdcf->execute_cb != new_rdcf->execute_cb) { - netdata_log_info("FUNCTIONS: function '%s' of host '%s' changed execute callback", - dictionary_acquired_item_name(item), rrdhost_hostname(host)); + nd_log(NDLS_DAEMON, NDLP_DEBUG, + "FUNCTIONS: function '%s' of host '%s' changed execute callback", + dictionary_acquired_item_name(item), rrdhost_hostname(host)); rdcf->execute_cb = new_rdcf->execute_cb; changed = true; } if(rdcf->help != new_rdcf->help) { - netdata_log_info("FUNCTIONS: function '%s' of host '%s' changed help text", - dictionary_acquired_item_name(item), rrdhost_hostname(host)); + nd_log(NDLS_DAEMON, NDLP_DEBUG, + "FUNCTIONS: function '%s' of host '%s' changed help text", + dictionary_acquired_item_name(item), rrdhost_hostname(host)); STRING *old = rdcf->help; rdcf->help = new_rdcf->help; @@ -451,25 +226,59 @@ static bool rrd_functions_conflict_callback(const DICTIONARY_ITEM *item __maybe_ else string_freez(new_rdcf->help); + if(rdcf->tags != new_rdcf->tags) { + nd_log(NDLS_DAEMON, NDLP_DEBUG, + "FUNCTIONS: function '%s' of host '%s' changed tags", + dictionary_acquired_item_name(item), rrdhost_hostname(host)); + + STRING *old = rdcf->tags; + rdcf->tags = new_rdcf->tags; + string_freez(old); + changed = true; + } + else + string_freez(new_rdcf->tags); + if(rdcf->timeout != new_rdcf->timeout) { - netdata_log_info("FUNCTIONS: function '%s' of host '%s' changed timeout", - dictionary_acquired_item_name(item), rrdhost_hostname(host)); + nd_log(NDLS_DAEMON, NDLP_DEBUG, + "FUNCTIONS: function '%s' of host '%s' changed timeout", + dictionary_acquired_item_name(item), rrdhost_hostname(host)); rdcf->timeout = new_rdcf->timeout; changed = true; } + if(rdcf->priority != new_rdcf->priority) { + nd_log(NDLS_DAEMON, NDLP_DEBUG, + "FUNCTIONS: function '%s' of host '%s' changed priority", + dictionary_acquired_item_name(item), rrdhost_hostname(host)); + + rdcf->priority = new_rdcf->priority; + changed = true; + } + + if(rdcf->access != new_rdcf->access) { + nd_log(NDLS_DAEMON, NDLP_DEBUG, + "FUNCTIONS: function '%s' of host '%s' changed access level", + dictionary_acquired_item_name(item), rrdhost_hostname(host)); + + rdcf->access = new_rdcf->access; + changed = true; + } + if(rdcf->sync != new_rdcf->sync) { - netdata_log_info("FUNCTIONS: function '%s' of host '%s' changed sync/async mode", - dictionary_acquired_item_name(item), rrdhost_hostname(host)); + nd_log(NDLS_DAEMON, NDLP_DEBUG, + "FUNCTIONS: function '%s' of host '%s' changed sync/async mode", + dictionary_acquired_item_name(item), rrdhost_hostname(host)); rdcf->sync = new_rdcf->sync; changed = true; } if(rdcf->execute_cb_data != new_rdcf->execute_cb_data) { - netdata_log_info("FUNCTIONS: function '%s' of host '%s' changed execute callback data", - dictionary_acquired_item_name(item), rrdhost_hostname(host)); + nd_log(NDLS_DAEMON, NDLP_DEBUG, + "FUNCTIONS: function '%s' of host '%s' changed execute callback data", + dictionary_acquired_item_name(item), rrdhost_hostname(host)); rdcf->execute_cb_data = new_rdcf->execute_cb_data; changed = true; @@ -497,12 +306,22 @@ void rrdfunctions_host_destroy(RRDHOST *host) { dictionary_destroy(host->functions); } -void rrd_function_add(RRDHOST *host, RRDSET *st, const char *name, int timeout, const char *help, - bool sync, rrd_function_execute_cb_t execute_cb, void *execute_cb_data) { +// ---------------------------------------------------------------------------- + +void rrd_function_add(RRDHOST *host, RRDSET *st, const char *name, int timeout, int priority, const char *help, const char *tags, + HTTP_ACCESS access, bool sync, rrd_function_execute_cb_t execute_cb, + void *execute_cb_data) { // RRDSET *st may be NULL in this function // to create a GLOBAL function + if(!tags || !*tags) { + if(strcmp(name, "systemd-journal") == 0) + tags = "logs"; + else + tags = "top"; + } + if(st && !st->functions_view) st->functions_view = dictionary_create_view(host->functions); @@ -513,9 +332,12 @@ void rrd_function_add(RRDHOST *host, RRDSET *st, const char *name, int timeout, .sync = sync, .timeout = timeout, .options = (st)?RRD_FUNCTION_LOCAL:RRD_FUNCTION_GLOBAL, + .access = access, .execute_cb = execute_cb, .execute_cb_data = execute_cb_data, .help = string_strdupz(help), + .tags = string_strdupz(tags), + .priority = priority, }; const DICTIONARY_ITEM *item = dictionary_set_and_acquire_item(host->functions, key, &tmp, sizeof(tmp)); @@ -534,10 +356,13 @@ void rrd_functions_expose_rrdpush(RRDSET *st, BUFFER *wb) { struct rrd_host_function *tmp; dfe_start_read(st->functions_view, tmp) { buffer_sprintf(wb - , PLUGINSD_KEYWORD_FUNCTION " \"%s\" %d \"%s\"\n" + , PLUGINSD_KEYWORD_FUNCTION " \"%s\" %d \"%s\" \"%s\" \"%s\" %d\n" , tmp_dfe.name , tmp->timeout , string2str(tmp->help) + , string2str(tmp->tags) + , http_id2access(tmp->access) + , tmp->priority ); } dfe_done(tmp); @@ -552,10 +377,13 @@ void rrd_functions_expose_global_rrdpush(RRDHOST *host, BUFFER *wb) { continue; buffer_sprintf(wb - , PLUGINSD_KEYWORD_FUNCTION " GLOBAL \"%s\" %d \"%s\"\n" + , PLUGINSD_KEYWORD_FUNCTION " GLOBAL \"%s\" %d \"%s\" \"%s\" \"%s\" %d\n" , tmp_dfe.name , tmp->timeout , string2str(tmp->help) + , string2str(tmp->tags) + , http_id2access(tmp->access) + , tmp->priority ); } dfe_done(tmp); @@ -632,7 +460,7 @@ static int rrd_call_function_find(RRDHOST *host, BUFFER *wb, const char *name, s } } - // if s == NULL, set it to the end of the buffer + // if s == NULL, set it to the end of the buffer; // this should happen only the first time if (unlikely(!s)) s = &buffer[key_length - 1]; @@ -663,51 +491,6 @@ static int rrd_call_function_find(RRDHOST *host, BUFFER *wb, const char *name, s // ---------------------------------------------------------------------------- -struct rrd_function_inflight { - bool used; - - RRDHOST *host; - const char *transaction; - const char *cmd; - const char *sanitized_cmd; - size_t sanitized_cmd_length; - int timeout; - bool cancelled; - - const DICTIONARY_ITEM *host_function_acquired; - - // the collector - // we acquire this structure at the beginning, - // and we release it at the end - struct rrd_host_function *rdcf; - - struct { - BUFFER *wb; - - // in async mode, - // the function to call to send the result back - rrd_function_result_callback_t cb; - void *data; - } result; - - struct { - // to be called in sync mode - // while the function is running - // to check if the function has been cancelled - rrd_function_is_cancelled_cb_t cb; - void *data; - } is_cancelled; - - struct { - // to be registered by the function itself - // used to signal the function to cancel - rrd_function_canceller_cb_t cb; - void *data; - } canceller; -}; - -static DICTIONARY *rrd_functions_inflight_requests = NULL; - static void rrd_functions_inflight_delete_cb(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *data __maybe_unused) { struct rrd_function_inflight *r = value; @@ -736,12 +519,18 @@ void rrd_functions_inflight_destroy(void) { rrd_functions_inflight_requests = NULL; } -static void rrd_inflight_async_function_register_canceller_cb(void *register_canceller_cb_data, rrd_function_canceller_cb_t canceller_cb, void *canceller_cb_data) { +static void rrd_inflight_async_function_register_canceller_cb(void *register_canceller_cb_data, rrd_function_cancel_cb_t canceller_cb, void *canceller_cb_data) { struct rrd_function_inflight *r = register_canceller_cb_data; r->canceller.cb = canceller_cb; r->canceller.data = canceller_cb_data; } +static void rrd_inflight_async_function_register_progresser_cb(void *register_progresser_cb_data, rrd_function_progresser_cb_t progresser_cb, void *progresser_cb_data) { + struct rrd_function_inflight *r = register_progresser_cb_data; + r->progresser.cb = progresser_cb; + r->progresser.data = progresser_cb_data; +} + // ---------------------------------------------------------------------------- // waiting for async function completion @@ -758,14 +547,13 @@ struct rrd_function_call_wait { }; static void rrd_inflight_function_cleanup(RRDHOST *host __maybe_unused, - const DICTIONARY_ITEM *host_function_acquired __maybe_unused, const char *transaction) { dictionary_del(rrd_functions_inflight_requests, transaction); dictionary_garbage_collect(rrd_functions_inflight_requests); } static void rrd_function_call_wait_free(struct rrd_function_call_wait *tmp) { - rrd_inflight_function_cleanup(tmp->host, tmp->host_function_acquired, tmp->transaction); + rrd_inflight_function_cleanup(tmp->host, tmp->transaction); freez(tmp->transaction); pthread_cond_destroy(&tmp->cond); @@ -805,7 +593,7 @@ static void rrd_inflight_async_function_nowait_finished(BUFFER *wb, int code, vo if(r->result.cb) r->result.cb(wb, code, r->result.data); - rrd_inflight_function_cleanup(r->host, r->host_function_acquired, r->transaction); + rrd_inflight_function_cleanup(r->host, r->transaction); } static bool rrd_inflight_async_function_is_cancelled(void *data) { @@ -814,27 +602,25 @@ static bool rrd_inflight_async_function_is_cancelled(void *data) { } static inline int rrd_call_function_async_and_dont_wait(struct rrd_function_inflight *r) { - int code = r->rdcf->execute_cb(r->result.wb, r->timeout, r->sanitized_cmd, r->rdcf->execute_cb_data, + int code = r->rdcf->execute_cb(&r->transaction_uuid, r->result.wb, + &r->stop_monotonic_ut, r->sanitized_cmd, r->rdcf->execute_cb_data, rrd_inflight_async_function_nowait_finished, r, + r->progress.cb, r->progress.data, rrd_inflight_async_function_is_cancelled, r, - rrd_inflight_async_function_register_canceller_cb, r); + rrd_inflight_async_function_register_canceller_cb, r, + rrd_inflight_async_function_register_progresser_cb, r); if(code != HTTP_RESP_OK) { if (!buffer_strlen(r->result.wb)) rrd_call_function_error(r->result.wb, "Failed to send request to the collector.", code); - rrd_inflight_function_cleanup(r->host, r->host_function_acquired, r->transaction); + rrd_inflight_function_cleanup(r->host, r->transaction); } return code; } static int rrd_call_function_async_and_wait(struct rrd_function_inflight *r) { - struct timespec tp; - clock_gettime(CLOCK_REALTIME, &tp); - usec_t now_ut = tp.tv_sec * USEC_PER_SEC + tp.tv_nsec / NSEC_PER_USEC; - usec_t end_ut = now_ut + r->timeout * USEC_PER_SEC + RRDFUNCTIONS_TIMEOUT_EXTENSION_UT; - struct rrd_function_call_wait *tmp = mallocz(sizeof(struct rrd_function_call_wait)); tmp->free_with_signal = false; tmp->data_are_ready = false; @@ -844,19 +630,22 @@ static int rrd_call_function_async_and_wait(struct rrd_function_inflight *r) { netdata_mutex_init(&tmp->mutex); pthread_cond_init(&tmp->cond, NULL); - // we need a temporary BUFFER, because we may time out and the caller supplied one may vanish - // so, we create a new one we guarantee will survive until the collector finishes... + // we need a temporary BUFFER, because we may time out and the caller supplied one may vanish, + // so we create a new one we guarantee will survive until the collector finishes... bool we_should_free = true; BUFFER *temp_wb = buffer_create(PLUGINSD_LINE_MAX + 1, &netdata_buffers_statistics.buffers_functions); // we need it because we may give up on it temp_wb->content_type = r->result.wb->content_type; - int code = r->rdcf->execute_cb(temp_wb, r->timeout, r->sanitized_cmd, r->rdcf->execute_cb_data, - // we overwrite the result callbacks, - // so that we can clean up the allocations made + int code = r->rdcf->execute_cb(&r->transaction_uuid, temp_wb, &r->stop_monotonic_ut, + r->sanitized_cmd, r->rdcf->execute_cb_data, + // we overwrite the result callbacks, + // so that we can clean up the allocations made rrd_async_function_signal_when_ready, tmp, + r->progress.cb, r->progress.data, rrd_inflight_async_function_is_cancelled, r, - rrd_inflight_async_function_register_canceller_cb, r); + rrd_inflight_async_function_register_canceller_cb, r, + rrd_inflight_async_function_register_progresser_cb, r); if (code == HTTP_RESP_OK) { netdata_mutex_lock(&tmp->mutex); @@ -864,14 +653,16 @@ static int rrd_call_function_async_and_wait(struct rrd_function_inflight *r) { bool cancelled = false; int rc = 0; while (rc == 0 && !cancelled && !tmp->data_are_ready) { - clock_gettime(CLOCK_REALTIME, &tp); - now_ut = tp.tv_sec * USEC_PER_SEC + tp.tv_nsec / NSEC_PER_USEC; - - if(now_ut >= end_ut) { + usec_t now_mono_ut = now_monotonic_usec(); + usec_t stop_mono_ut = __atomic_load_n(&r->stop_monotonic_ut, __ATOMIC_RELAXED) + RRDFUNCTIONS_TIMEOUT_EXTENSION_UT; + if(now_mono_ut > stop_mono_ut) { rc = ETIMEDOUT; break; } + // wait for 10ms, and loop again... + struct timespec tp; + clock_gettime(CLOCK_REALTIME, &tp); tp.tv_nsec += 10 * NSEC_PER_MSEC; if(tp.tv_nsec > (long)(1 * NSEC_PER_SEC)) { tp.tv_sec++; @@ -883,14 +674,15 @@ static int rrd_call_function_async_and_wait(struct rrd_function_inflight *r) { // the mutex is again ours if(rc == ETIMEDOUT) { + // 10ms have passed + rc = 0; if (!tmp->data_are_ready && r->is_cancelled.cb && r->is_cancelled.cb(r->is_cancelled.data)) { // internal_error(true, "FUNCTIONS: transaction '%s' is cancelled while waiting for response", // r->transaction); - rc = 0; cancelled = true; - rrd_function_cancel(r->transaction); + rrd_function_cancel_inflight(r); break; } } @@ -955,9 +747,10 @@ static inline int rrd_call_function_async(struct rrd_function_inflight *r, bool void call_virtual_function_async(BUFFER *wb, RRDHOST *host, const char *name, const char *payload, rrd_function_result_callback_t callback, void *callback_data); // ---------------------------------------------------------------------------- -int rrd_function_run(RRDHOST *host, BUFFER *result_wb, int timeout, const char *cmd, +int rrd_function_run(RRDHOST *host, BUFFER *result_wb, int timeout_s, HTTP_ACCESS access, const char *cmd, bool wait, const char *transaction, rrd_function_result_callback_t result_cb, void *result_cb_data, + rrd_function_progress_cb_t progress_cb, void *progress_cb_data, rrd_function_is_cancelled_cb_t is_cancelled_cb, void *is_cancelled_cb_data, const char *payload) { int code; @@ -980,42 +773,31 @@ int rrd_function_run(RRDHOST *host, BUFFER *result_wb, int timeout, const char * struct rrd_host_function *rdcf = dictionary_acquired_item_value(host_function_acquired); - if(timeout <= 0) - timeout = rdcf->timeout; + if(access != HTTP_ACCESS_ADMINS && rdcf->access != HTTP_ACCESS_ANY && access > rdcf->access) { + rrd_call_function_error(result_wb, "This function requires a higher access level.", code); + dictionary_acquired_item_release(host->functions, host_function_acquired); + return HTTP_RESP_PRECOND_FAIL; + } + if(timeout_s <= 0) + timeout_s = rdcf->timeout; // ------------------------------------------------------------------------ - // the function can only be executed in sync mode + // validate and parse the transaction, or generate a new transaction id - if(rdcf->sync) { - // the caller has to wait - - code = rdcf->execute_cb(result_wb, timeout, sanitized_cmd, rdcf->execute_cb_data, - result_cb, result_cb_data, - is_cancelled_cb, is_cancelled_cb_data, // it is ok to pass these, we block the caller - NULL, NULL); // no need to pass, we will wait - - if (code != HTTP_RESP_OK && !buffer_strlen(result_wb)) - rrd_call_function_error(result_wb, "Collector reported error.", code); + char uuid_str[UUID_COMPACT_STR_LEN]; + uuid_t uuid; - dictionary_acquired_item_release(host->functions, host_function_acquired); - return code; - } + if(!transaction || !*transaction || uuid_parse_flexi(transaction, uuid) != 0) + uuid_generate_random(uuid); + uuid_unparse_lower_compact(uuid, uuid_str); + transaction = uuid_str; // ------------------------------------------------------------------------ // the function can only be executed in async mode // put the function into the inflight requests - char uuid_str[UUID_COMPACT_STR_LEN]; - if(!transaction) { - uuid_t uuid; - uuid_generate_random(uuid); - uuid_unparse_lower_compact(uuid, uuid_str); - transaction = uuid_str; - } - - // put the request into the inflight requests struct rrd_function_inflight t = { .used = false, .host = host, @@ -1023,8 +805,9 @@ int rrd_function_run(RRDHOST *host, BUFFER *result_wb, int timeout, const char * .sanitized_cmd = strdupz(sanitized_cmd), .sanitized_cmd_length = sanitized_cmd_length, .transaction = strdupz(transaction), - .timeout = timeout, + .timeout = timeout_s, .cancelled = false, + .stop_monotonic_ut = now_monotonic_usec() + timeout_s * USEC_PER_SEC, .host_function_acquired = host_function_acquired, .rdcf = rdcf, .result = { @@ -1035,11 +818,20 @@ int rrd_function_run(RRDHOST *host, BUFFER *result_wb, int timeout, const char * .is_cancelled = { .cb = is_cancelled_cb, .data = is_cancelled_cb_data, - } + }, + .progress = { + .cb = progress_cb, + .data = progress_cb_data, + }, }; + uuid_copy(t.transaction_uuid, uuid); + struct rrd_function_inflight *r = dictionary_set(rrd_functions_inflight_requests, transaction, &t, sizeof(t)); if(r->used) { - netdata_log_info("FUNCTIONS: duplicate transaction '%s', function: '%s'", t.transaction, t.cmd); + nd_log(NDLS_DAEMON, NDLP_NOTICE, + "FUNCTIONS: duplicate transaction '%s', function: '%s'", + t.transaction, t.cmd); + code = rrd_call_function_error(result_wb, "duplicate transaction", HTTP_RESP_BAD_REQUEST); freez((void *)t.transaction); freez((void *)t.cmd); @@ -1050,51 +842,105 @@ int rrd_function_run(RRDHOST *host, BUFFER *result_wb, int timeout, const char * r->used = true; // internal_error(true, "FUNCTIONS: transaction '%s' started", r->transaction); + if(r->rdcf->sync) { + // the caller has to wait + code = r->rdcf->execute_cb(&r->transaction_uuid, r->result.wb, + &r->stop_monotonic_ut, r->sanitized_cmd, r->rdcf->execute_cb_data, + r->result.cb, r->result.data, + r->progress.cb, r->progress.data, + r->is_cancelled.cb, r->is_cancelled.data, // it is ok to pass these, we block the caller + NULL, NULL, // no need to register canceller, we will wait + NULL, NULL // ?? do we need a progresser in this case? + ); + + if(code != HTTP_RESP_OK && !buffer_strlen(result_wb)) + rrd_call_function_error(result_wb, "Collector reported error.", code); + + rrd_inflight_function_cleanup(host, r->transaction); + return code; + } + return rrd_call_function_async(r, wait); } +static void rrd_function_cancel_inflight(struct rrd_function_inflight *r) { + if(!r) + return; + + bool cancelled = __atomic_load_n(&r->cancelled, __ATOMIC_RELAXED); + if(cancelled) { + nd_log(NDLS_DAEMON, NDLP_DEBUG, + "FUNCTIONS: received a CANCEL request for transaction '%s', but it is already cancelled.", + r->transaction); + return; + } + + __atomic_store_n(&r->cancelled, true, __ATOMIC_RELAXED); + + if(!rrd_collector_dispatcher_acquire(r->rdcf->collector)) { + nd_log(NDLS_DAEMON, NDLP_DEBUG, + "FUNCTIONS: received a CANCEL request for transaction '%s', but the collector is not running.", + r->transaction); + return; + } + + if(r->canceller.cb) + r->canceller.cb(r->canceller.data); + + rrd_collector_dispatcher_release(r->rdcf->collector); +} + void rrd_function_cancel(const char *transaction) { // internal_error(true, "FUNCTIONS: request to cancel transaction '%s'", transaction); const DICTIONARY_ITEM *item = dictionary_get_and_acquire_item(rrd_functions_inflight_requests, transaction); if(!item) { - netdata_log_info("FUNCTIONS: received a cancel request for transaction '%s', but the transaction is not running.", - transaction); + nd_log(NDLS_DAEMON, NDLP_DEBUG, + "FUNCTIONS: received a CANCEL request for transaction '%s', but the transaction is not running.", + transaction); return; } struct rrd_function_inflight *r = dictionary_acquired_item_value(item); + rrd_function_cancel_inflight(r); + dictionary_acquired_item_release(rrd_functions_inflight_requests, item); +} - bool cancelled = __atomic_load_n(&r->cancelled, __ATOMIC_RELAXED); - if(cancelled) { - netdata_log_info("FUNCTIONS: received a cancel request for transaction '%s', but it is already cancelled.", - transaction); - goto cleanup; +void rrd_function_progress(const char *transaction) { + const DICTIONARY_ITEM *item = dictionary_get_and_acquire_item(rrd_functions_inflight_requests, transaction); + if(!item) { + nd_log(NDLS_DAEMON, NDLP_DEBUG, + "FUNCTIONS: received a PROGRESS request for transaction '%s', but the transaction is not running.", + transaction); + return; } - __atomic_store_n(&r->cancelled, true, __ATOMIC_RELAXED); + struct rrd_function_inflight *r = dictionary_acquired_item_value(item); - int32_t expected = __atomic_load_n(&r->rdcf->collector->refcount_canceller, __ATOMIC_RELAXED); - int32_t wanted; - do { - if(expected < 0) { - netdata_log_info("FUNCTIONS: received a cancel request for transaction '%s', but the collector is not running.", - transaction); - goto cleanup; - } + if(!rrd_collector_dispatcher_acquire(r->rdcf->collector)) { + nd_log(NDLS_DAEMON, NDLP_DEBUG, + "FUNCTIONS: received a PROGRESS request for transaction '%s', but the collector is not running.", + transaction); + goto cleanup; + } - wanted = expected + 1; - } while(!__atomic_compare_exchange_n(&r->rdcf->collector->refcount_canceller, &expected, wanted, false, __ATOMIC_RELAXED, __ATOMIC_RELAXED)); + functions_stop_monotonic_update_on_progress(&r->stop_monotonic_ut); - if(r->canceller.cb) - r->canceller.cb(r->canceller.data); + if(r->progresser.cb) + r->progresser.cb(r->progresser.data); - __atomic_sub_fetch(&r->rdcf->collector->refcount_canceller, 1, __ATOMIC_RELAXED); + rrd_collector_dispatcher_release(r->rdcf->collector); cleanup: dictionary_acquired_item_release(rrd_functions_inflight_requests, item); } +void rrd_function_call_progresser(uuid_t *transaction) { + char str[UUID_COMPACT_STR_LEN]; + uuid_unparse_lower_compact(*transaction, str); + rrd_function_progress(str); +} + // ---------------------------------------------------------------------------- static void functions2json(DICTIONARY *functions, BUFFER *wb) @@ -1106,18 +952,23 @@ static void functions2json(DICTIONARY *functions, BUFFER *wb) continue; buffer_json_member_add_object(wb, t_dfe.name); - buffer_json_member_add_string_or_empty(wb, "help", string2str(t->help)); - buffer_json_member_add_int64(wb, "timeout", (int64_t)t->timeout); - - char options[65]; - snprintfz( - options, - 64, - "%s%s", - (t->options & RRD_FUNCTION_LOCAL) ? "LOCAL " : "", - (t->options & RRD_FUNCTION_GLOBAL) ? "GLOBAL" : ""); - - buffer_json_member_add_string_or_empty(wb, "options", options); + { + buffer_json_member_add_string_or_empty(wb, "help", string2str(t->help)); + buffer_json_member_add_int64(wb, "timeout", (int64_t) t->timeout); + + char options[65]; + snprintfz( + options, 64 + , "%s%s" + , (t->options & RRD_FUNCTION_LOCAL) ? "LOCAL " : "" + , (t->options & RRD_FUNCTION_GLOBAL) ? "GLOBAL" : "" + ); + + buffer_json_member_add_string_or_empty(wb, "options", options); + buffer_json_member_add_string_or_empty(wb, "tags", string2str(t->tags)); + buffer_json_member_add_string(wb, "access", http_id2access(t->access)); + buffer_json_member_add_uint64(wb, "priority", t->priority); + } buffer_json_object_close(wb); } dfe_done(t); @@ -1139,14 +990,21 @@ void host_functions2json(RRDHOST *host, BUFFER *wb) { if(!rrd_collector_running(t->collector)) continue; buffer_json_member_add_object(wb, t_dfe.name); - buffer_json_member_add_string(wb, "help", string2str(t->help)); - buffer_json_member_add_int64(wb, "timeout", t->timeout); - buffer_json_member_add_array(wb, "options"); - if(t->options & RRD_FUNCTION_GLOBAL) - buffer_json_add_array_item_string(wb, "GLOBAL"); - if(t->options & RRD_FUNCTION_LOCAL) - buffer_json_add_array_item_string(wb, "LOCAL"); - buffer_json_array_close(wb); + { + buffer_json_member_add_string(wb, "help", string2str(t->help)); + buffer_json_member_add_int64(wb, "timeout", t->timeout); + buffer_json_member_add_array(wb, "options"); + { + if (t->options & RRD_FUNCTION_GLOBAL) + buffer_json_add_array_item_string(wb, "GLOBAL"); + if (t->options & RRD_FUNCTION_LOCAL) + buffer_json_add_array_item_string(wb, "LOCAL"); + } + buffer_json_array_close(wb); + buffer_json_member_add_string(wb, "tags", string2str(t->tags)); + buffer_json_member_add_string(wb, "access", http_id2access(t->access)); + buffer_json_member_add_uint64(wb, "priority", t->priority); + } buffer_json_object_close(wb); } dfe_done(t); @@ -1166,7 +1024,7 @@ void chart_functions_to_dict(DICTIONARY *rrdset_functions_view, DICTIONARY *dst, dfe_done(t); } -void host_functions_to_dict(RRDHOST *host, DICTIONARY *dst, void *value, size_t value_size, STRING **help) { +void host_functions_to_dict(RRDHOST *host, DICTIONARY *dst, void *value, size_t value_size, STRING **help, STRING **tags, HTTP_ACCESS *access, int *priority) { if(!host || !host->functions || !dictionary_entries(host->functions) || !dst) return; struct rrd_host_function *t; @@ -1176,6 +1034,15 @@ void host_functions_to_dict(RRDHOST *host, DICTIONARY *dst, void *value, size_t if(help) *help = t->help; + if(tags) + *tags = t->tags; + + if(access) + *access = t->access; + + if(priority) + *priority = t->priority; + dictionary_set(dst, t_dfe.name, value, value_size); } dfe_done(t); @@ -1183,12 +1050,40 @@ void host_functions_to_dict(RRDHOST *host, DICTIONARY *dst, void *value, size_t // ---------------------------------------------------------------------------- -int rrdhost_function_streaming(BUFFER *wb, int timeout __maybe_unused, const char *function __maybe_unused, +int rrdhost_function_progress(uuid_t *transaction __maybe_unused, BUFFER *wb, + usec_t *stop_monotonic_ut __maybe_unused, const char *function __maybe_unused, + void *collector_data __maybe_unused, + rrd_function_result_callback_t result_cb, void *result_cb_data, + rrd_function_progress_cb_t progress_cb __maybe_unused, void *progress_cb_data __maybe_unused, + rrd_function_is_cancelled_cb_t is_cancelled_cb, void *is_cancelled_cb_data, + rrd_function_register_canceller_cb_t register_canceller_cb __maybe_unused, + void *register_canceller_cb_data __maybe_unused, + rrd_function_register_progresser_cb_t register_progresser_cb __maybe_unused, + void *register_progresser_cb_data __maybe_unused) { + + int response = progress_function_result(wb, rrdhost_hostname(localhost)); + + if(is_cancelled_cb && is_cancelled_cb(is_cancelled_cb_data)) { + buffer_flush(wb); + response = HTTP_RESP_CLIENT_CLOSED_REQUEST; + } + + if(result_cb) + result_cb(wb, response, result_cb_data); + + return response; +} + +int rrdhost_function_streaming(uuid_t *transaction __maybe_unused, BUFFER *wb, + usec_t *stop_monotonic_ut __maybe_unused, const char *function __maybe_unused, void *collector_data __maybe_unused, rrd_function_result_callback_t result_cb, void *result_cb_data, + rrd_function_progress_cb_t progress_cb __maybe_unused, void *progress_cb_data __maybe_unused, rrd_function_is_cancelled_cb_t is_cancelled_cb, void *is_cancelled_cb_data, rrd_function_register_canceller_cb_t register_canceller_cb __maybe_unused, - void *register_canceller_cb_data __maybe_unused) { + void *register_canceller_cb_data __maybe_unused, + rrd_function_register_progresser_cb_t register_progresser_cb __maybe_unused, + void *register_progresser_cb_data __maybe_unused) { time_t now = now_realtime_sec(); @@ -1389,19 +1284,19 @@ int rrdhost_function_streaming(BUFFER *wb, int timeout __maybe_unused, const cha buffer_rrdf_table_add_field(wb, field_id++, "dbMetrics", "Time-series Metrics in the DB", RRDF_FIELD_TYPE_INTEGER, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NUMBER, - 0, NULL, max_db_metrics, RRDF_FIELD_SORT_DESCENDING, NULL, + 0, NULL, (double)max_db_metrics, RRDF_FIELD_SORT_DESCENDING, NULL, RRDF_FIELD_SUMMARY_SUM, RRDF_FIELD_FILTER_RANGE, RRDF_FIELD_OPTS_VISIBLE, NULL); buffer_rrdf_table_add_field(wb, field_id++, "dbInstances", "Instances in the DB", RRDF_FIELD_TYPE_INTEGER, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NUMBER, - 0, NULL, max_db_instances, RRDF_FIELD_SORT_DESCENDING, NULL, + 0, NULL, (double)max_db_instances, RRDF_FIELD_SORT_DESCENDING, NULL, RRDF_FIELD_SUMMARY_SUM, RRDF_FIELD_FILTER_RANGE, RRDF_FIELD_OPTS_VISIBLE, NULL); buffer_rrdf_table_add_field(wb, field_id++, "dbContexts", "Contexts in the DB", RRDF_FIELD_TYPE_INTEGER, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NUMBER, - 0, NULL, max_db_contexts, RRDF_FIELD_SORT_DESCENDING, NULL, + 0, NULL, (double)max_db_contexts, RRDF_FIELD_SORT_DESCENDING, NULL, RRDF_FIELD_SUMMARY_SUM, RRDF_FIELD_FILTER_RANGE, RRDF_FIELD_OPTS_VISIBLE, NULL); @@ -1460,7 +1355,7 @@ int rrdhost_function_streaming(BUFFER *wb, int timeout __maybe_unused, const cha buffer_rrdf_table_add_field(wb, field_id++, "InReplInstances", "Inbound Replicating Instances", RRDF_FIELD_TYPE_INTEGER, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NUMBER, - 0, "instances", max_collection_replication_instances, RRDF_FIELD_SORT_DESCENDING, + 0, "instances", (double)max_collection_replication_instances, RRDF_FIELD_SORT_DESCENDING, NULL, RRDF_FIELD_SUMMARY_SUM, RRDF_FIELD_FILTER_RANGE, RRDF_FIELD_OPTS_NONE, NULL); @@ -1535,7 +1430,7 @@ int rrdhost_function_streaming(BUFFER *wb, int timeout __maybe_unused, const cha buffer_rrdf_table_add_field(wb, field_id++, "OutReplInstances", "Outbound Replicating Instances", RRDF_FIELD_TYPE_INTEGER, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NUMBER, - 0, "instances", max_streaming_replication_instances, RRDF_FIELD_SORT_DESCENDING, + 0, "instances", (double)max_streaming_replication_instances, RRDF_FIELD_SORT_DESCENDING, NULL, RRDF_FIELD_SUMMARY_SUM, RRDF_FIELD_FILTER_RANGE, RRDF_FIELD_OPTS_NONE, NULL); @@ -1584,7 +1479,7 @@ int rrdhost_function_streaming(BUFFER *wb, int timeout __maybe_unused, const cha buffer_rrdf_table_add_field(wb, field_id++, "OutTrafficData", "Outbound Metric Data Traffic", RRDF_FIELD_TYPE_INTEGER, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NUMBER, - 0, "bytes", max_sent_bytes_on_this_connection_per_type[STREAM_TRAFFIC_TYPE_DATA], + 0, "bytes", (double)max_sent_bytes_on_this_connection_per_type[STREAM_TRAFFIC_TYPE_DATA], RRDF_FIELD_SORT_DESCENDING, NULL, RRDF_FIELD_SUMMARY_SUM, RRDF_FIELD_FILTER_RANGE, RRDF_FIELD_OPTS_NONE, NULL); @@ -1592,7 +1487,7 @@ int rrdhost_function_streaming(BUFFER *wb, int timeout __maybe_unused, const cha buffer_rrdf_table_add_field(wb, field_id++, "OutTrafficMetadata", "Outbound Metric Metadata Traffic", RRDF_FIELD_TYPE_INTEGER, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NUMBER, 0, "bytes", - max_sent_bytes_on_this_connection_per_type[STREAM_TRAFFIC_TYPE_METADATA], + (double)max_sent_bytes_on_this_connection_per_type[STREAM_TRAFFIC_TYPE_METADATA], RRDF_FIELD_SORT_DESCENDING, NULL, RRDF_FIELD_SUMMARY_SUM, RRDF_FIELD_FILTER_RANGE, RRDF_FIELD_OPTS_NONE, NULL); @@ -1600,7 +1495,7 @@ int rrdhost_function_streaming(BUFFER *wb, int timeout __maybe_unused, const cha buffer_rrdf_table_add_field(wb, field_id++, "OutTrafficReplication", "Outbound Metric Replication Traffic", RRDF_FIELD_TYPE_INTEGER, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NUMBER, 0, "bytes", - max_sent_bytes_on_this_connection_per_type[STREAM_TRAFFIC_TYPE_REPLICATION], + (double)max_sent_bytes_on_this_connection_per_type[STREAM_TRAFFIC_TYPE_REPLICATION], RRDF_FIELD_SORT_DESCENDING, NULL, RRDF_FIELD_SUMMARY_SUM, RRDF_FIELD_FILTER_RANGE, RRDF_FIELD_OPTS_NONE, NULL); @@ -1608,7 +1503,7 @@ int rrdhost_function_streaming(BUFFER *wb, int timeout __maybe_unused, const cha buffer_rrdf_table_add_field(wb, field_id++, "OutTrafficFunctions", "Outbound Metric Functions Traffic", RRDF_FIELD_TYPE_INTEGER, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NUMBER, 0, "bytes", - max_sent_bytes_on_this_connection_per_type[STREAM_TRAFFIC_TYPE_FUNCTIONS], + (double)max_sent_bytes_on_this_connection_per_type[STREAM_TRAFFIC_TYPE_FUNCTIONS], RRDF_FIELD_SORT_DESCENDING, NULL, RRDF_FIELD_SUMMARY_SUM, RRDF_FIELD_FILTER_RANGE, RRDF_FIELD_OPTS_NONE, NULL); @@ -1639,7 +1534,7 @@ int rrdhost_function_streaming(BUFFER *wb, int timeout __maybe_unused, const cha buffer_rrdf_table_add_field(wb, field_id++, "MlAnomalous", "Number of Anomalous Metrics", RRDF_FIELD_TYPE_INTEGER, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NUMBER, 0, "metrics", - max_ml_anomalous, + (double)max_ml_anomalous, RRDF_FIELD_SORT_DESCENDING, NULL, RRDF_FIELD_SUMMARY_SUM, RRDF_FIELD_FILTER_RANGE, RRDF_FIELD_OPTS_NONE, NULL); @@ -1647,7 +1542,7 @@ int rrdhost_function_streaming(BUFFER *wb, int timeout __maybe_unused, const cha buffer_rrdf_table_add_field(wb, field_id++, "MlNormal", "Number of Not Anomalous Metrics", RRDF_FIELD_TYPE_INTEGER, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NUMBER, 0, "metrics", - max_ml_normal, + (double)max_ml_normal, RRDF_FIELD_SORT_DESCENDING, NULL, RRDF_FIELD_SUMMARY_SUM, RRDF_FIELD_FILTER_RANGE, RRDF_FIELD_OPTS_NONE, NULL); @@ -1655,7 +1550,7 @@ int rrdhost_function_streaming(BUFFER *wb, int timeout __maybe_unused, const cha buffer_rrdf_table_add_field(wb, field_id++, "MlTrained", "Number of Trained Metrics", RRDF_FIELD_TYPE_INTEGER, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NUMBER, 0, "metrics", - max_ml_trained, + (double)max_ml_trained, RRDF_FIELD_SORT_DESCENDING, NULL, RRDF_FIELD_SUMMARY_SUM, RRDF_FIELD_FILTER_RANGE, RRDF_FIELD_OPTS_NONE, NULL); @@ -1663,7 +1558,7 @@ int rrdhost_function_streaming(BUFFER *wb, int timeout __maybe_unused, const cha buffer_rrdf_table_add_field(wb, field_id++, "MlPending", "Number of Pending Metrics", RRDF_FIELD_TYPE_INTEGER, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NUMBER, 0, "metrics", - max_ml_pending, + (double)max_ml_pending, RRDF_FIELD_SORT_DESCENDING, NULL, RRDF_FIELD_SUMMARY_SUM, RRDF_FIELD_FILTER_RANGE, RRDF_FIELD_OPTS_NONE, NULL); @@ -1671,7 +1566,7 @@ int rrdhost_function_streaming(BUFFER *wb, int timeout __maybe_unused, const cha buffer_rrdf_table_add_field(wb, field_id++, "MlSilenced", "Number of Silenced Metrics", RRDF_FIELD_TYPE_INTEGER, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NUMBER, 0, "metrics", - max_ml_silenced, + (double)max_ml_silenced, RRDF_FIELD_SORT_DESCENDING, NULL, RRDF_FIELD_SUMMARY_SUM, RRDF_FIELD_FILTER_RANGE, RRDF_FIELD_OPTS_NONE, NULL); diff --git a/database/rrdfunctions.h b/database/rrdfunctions.h index 21ca5c73463380..c3013da8ffbb5e 100644 --- a/database/rrdfunctions.h +++ b/database/rrdfunctions.h @@ -6,53 +6,76 @@ #include "rrd.h" +#define RRDFUNCTIONS_PRIORITY_DEFAULT 100 + #define RRDFUNCTIONS_TIMEOUT_EXTENSION_UT (1 * USEC_PER_SEC) typedef void (*rrd_function_result_callback_t)(BUFFER *wb, int code, void *result_cb_data); typedef bool (*rrd_function_is_cancelled_cb_t)(void *is_cancelled_cb_data); -typedef void (*rrd_function_canceller_cb_t)(void *data); -typedef void (*rrd_function_register_canceller_cb_t)(void *register_cancel_cb_data, rrd_function_canceller_cb_t cancel_cb, void *cancel_cb_data); -typedef int (*rrd_function_execute_cb_t)(BUFFER *wb, int timeout, const char *function, void *collector_data, +typedef void (*rrd_function_cancel_cb_t)(void *data); +typedef void (*rrd_function_register_canceller_cb_t)(void *register_cancel_cb_data, rrd_function_cancel_cb_t cancel_cb, void *cancel_cb_data); +typedef void (*rrd_function_progress_cb_t)(void *data, size_t done, size_t all); +typedef void (*rrd_function_progresser_cb_t)(void *data); +typedef void (*rrd_function_register_progresser_cb_t)(void *register_progresser_cb_data, rrd_function_progresser_cb_t progresser_cb, void *progresser_cb_data); + +typedef int (*rrd_function_execute_cb_t)(uuid_t *transaction, BUFFER *wb, + usec_t *stop_monotonic_ut, const char *function, void *collector_data, rrd_function_result_callback_t result_cb, void *result_cb_data, + rrd_function_progress_cb_t progress_cb, void *progress_cb_data, rrd_function_is_cancelled_cb_t is_cancelled_cb, void *is_cancelled_cb_data, - rrd_function_register_canceller_cb_t register_cancel_cb, void *register_cancel_db_data); + rrd_function_register_canceller_cb_t register_canceller_cb, void *register_canceller_cb_data, + rrd_function_register_progresser_cb_t register_progresser_cb, void *register_progresser_cb_data); void rrd_functions_inflight_init(void); void rrdfunctions_host_init(RRDHOST *host); void rrdfunctions_host_destroy(RRDHOST *host); -void rrd_collector_started(void); -void rrd_collector_finished(void); - // add a function, to be run from the collector -void rrd_function_add(RRDHOST *host, RRDSET *st, const char *name, int timeout, const char *help, - bool sync, rrd_function_execute_cb_t execute_cb, void *execute_cb_data); +void rrd_function_add(RRDHOST *host, RRDSET *st, const char *name, int timeout, int priority, const char *help, const char *tags, + HTTP_ACCESS access, bool sync, rrd_function_execute_cb_t execute_cb, + void *execute_cb_data); // call a function, to be run from anywhere -int rrd_function_run(RRDHOST *host, BUFFER *result_wb, int timeout, const char *cmd, +int rrd_function_run(RRDHOST *host, BUFFER *result_wb, int timeout_s, HTTP_ACCESS access, const char *cmd, bool wait, const char *transaction, rrd_function_result_callback_t result_cb, void *result_cb_data, + rrd_function_progress_cb_t progress_cb, void *progress_cb_data, rrd_function_is_cancelled_cb_t is_cancelled_cb, void *is_cancelled_cb_data, const char *payload); // cancel a running function, to be run from anywhere void rrd_function_cancel(const char *transaction); +void rrd_function_progress(const char *transaction); +void rrd_function_call_progresser(uuid_t *transaction); void rrd_functions_expose_rrdpush(RRDSET *st, BUFFER *wb); void rrd_functions_expose_global_rrdpush(RRDHOST *host, BUFFER *wb); void chart_functions2json(RRDSET *st, BUFFER *wb); void chart_functions_to_dict(DICTIONARY *rrdset_functions_view, DICTIONARY *dst, void *value, size_t value_size); -void host_functions_to_dict(RRDHOST *host, DICTIONARY *dst, void *value, size_t value_size, STRING **help); +void host_functions_to_dict(RRDHOST *host, DICTIONARY *dst, void *value, size_t value_size, STRING **help, STRING **tags, HTTP_ACCESS *access, int *priority); void host_functions2json(RRDHOST *host, BUFFER *wb); uint8_t functions_format_to_content_type(const char *format); const char *functions_content_type_to_format(HTTP_CONTENT_TYPE content_type); int rrd_call_function_error(BUFFER *wb, const char *msg, int code); -int rrdhost_function_streaming(BUFFER *wb, int timeout, const char *function, void *collector_data, +int rrdhost_function_progress(uuid_t *transaction, BUFFER *wb, + usec_t *stop_monotonic_ut, const char *function, void *collector_data, + rrd_function_result_callback_t result_cb, void *result_cb_data, + rrd_function_progress_cb_t progress_cb, void *progress_cb_data, + rrd_function_is_cancelled_cb_t is_cancelled_cb, void *is_cancelled_cb_data, + rrd_function_register_canceller_cb_t register_canceller_cb, void *register_canceller_cb_data, + rrd_function_register_progresser_cb_t register_progresser_cb, + void *register_progresser_cb_data); + +int rrdhost_function_streaming(uuid_t *transaction, BUFFER *wb, + usec_t *stop_monotonic_ut, const char *function, void *collector_data, rrd_function_result_callback_t result_cb, void *result_cb_data, + rrd_function_progress_cb_t progress_cb, void *progress_cb_data, rrd_function_is_cancelled_cb_t is_cancelled_cb, void *is_cancelled_cb_data, - rrd_function_register_canceller_cb_t register_canceller_cb, void *register_canceller_cb_data); + rrd_function_register_canceller_cb_t register_canceller_cb, void *register_canceller_cb_data, + rrd_function_register_progresser_cb_t register_progresser_cb, + void *register_progresser_cb_data); #define RRDFUNCTIONS_STREAMING_HELP "Streaming status for parents and children." diff --git a/database/rrdhost.c b/database/rrdhost.c index a3c2721536b941..c4731a0cff2bc8 100644 --- a/database/rrdhost.c +++ b/database/rrdhost.c @@ -1112,23 +1112,29 @@ int rrd_init(char *hostname, struct rrdhost_system_info *system_info, bool unitt , 0 ); - if (unlikely(!localhost)) { + if (unlikely(!localhost)) return 1; - } // we register this only on localhost // for the other nodes, the origin server should register it rrd_collector_started(); // this creates a collector that runs for as long as netdata runs - rrd_function_add(localhost, NULL, "streaming", 10, - RRDFUNCTIONS_STREAMING_HELP, true, + rrd_function_add(localhost, NULL, "streaming", 10, RRDFUNCTIONS_PRIORITY_DEFAULT + 1, + RRDFUNCTIONS_STREAMING_HELP, "top", + HTTP_ACCESS_MEMBERS, true, rrdhost_function_streaming, NULL); + rrd_function_add(localhost, NULL, "netdata-api-calls", 10, RRDFUNCTIONS_PRIORITY_DEFAULT + 2, + RRDFUNCTIONS_PROGRESS_HELP, "top", + HTTP_ACCESS_MEMBERS, true, + rrdhost_function_progress, NULL); + if (likely(system_info)) { migrate_localhost(&localhost->host_uuid); sql_aclk_sync_init(); web_client_api_v1_management_init(); } - return localhost==NULL; + + return 0; } // ---------------------------------------------------------------------------- diff --git a/libnetdata/buffer/buffer.c b/libnetdata/buffer/buffer.c index 64f9cce47c6d09..3e91fc10714637 100644 --- a/libnetdata/buffer/buffer.c +++ b/libnetdata/buffer/buffer.c @@ -259,23 +259,23 @@ void buffer_free(BUFFER *b) { void buffer_increase(BUFFER *b, size_t free_size_required) { buffer_overflow_check(b); - size_t left = b->size - b->len; - if(left >= free_size_required) return; + size_t remaining = b->size - b->len; + if(remaining >= free_size_required) return; - size_t wanted = free_size_required - left; - size_t minimum = WEB_DATA_LENGTH_INCREASE_STEP; - if(minimum > wanted) wanted = minimum; + size_t increase = free_size_required - remaining; + size_t minimum = 128; + if(minimum > increase) increase = minimum; size_t optimal = (b->size > 5*1024*1024) ? b->size / 2 : b->size; - if(optimal > wanted) wanted = optimal; + if(optimal > increase) increase = optimal; - netdata_log_debug(D_WEB_BUFFER, "Increasing data buffer from size %zu to %zu.", b->size, b->size + wanted); + netdata_log_debug(D_WEB_BUFFER, "Increasing data buffer from size %zu to %zu.", b->size, b->size + increase); - b->buffer = reallocz(b->buffer, b->size + wanted + sizeof(BUFFER_OVERFLOW_EOF) + 2); - b->size += wanted; + b->buffer = reallocz(b->buffer, b->size + increase + sizeof(BUFFER_OVERFLOW_EOF) + 2); + b->size += increase; if(b->statistics) - __atomic_add_fetch(b->statistics, wanted, __ATOMIC_RELAXED); + __atomic_add_fetch(b->statistics, increase, __ATOMIC_RELAXED); buffer_overflow_init(b); buffer_overflow_check(b); diff --git a/libnetdata/buffer/buffer.h b/libnetdata/buffer/buffer.h index 88d3f0282a9670..b9262b6089ecd7 100644 --- a/libnetdata/buffer/buffer.h +++ b/libnetdata/buffer/buffer.h @@ -10,8 +10,6 @@ #include "h2o/memory.h" #endif -#define WEB_DATA_LENGTH_INCREASE_STEP 1024 - #define BUFFER_JSON_MAX_DEPTH 32 // max is 255 extern const char hex_digits[16]; @@ -871,6 +869,26 @@ static inline void buffer_json_add_array_item_string(BUFFER *wb, const char *val wb->json.stack[wb->json.depth].count++; } +static inline void buffer_json_add_array_item_uuid(BUFFER *wb, uuid_t *value) { + if(value && !uuid_is_null(*value)) { + char uuid[GUID_LEN + 1]; + uuid_unparse_lower(*value, uuid); + buffer_json_add_array_item_string(wb, uuid); + } + else + buffer_json_add_array_item_string(wb, NULL); +} + +static inline void buffer_json_add_array_item_uuid_compact(BUFFER *wb, uuid_t *value) { + if(value && !uuid_is_null(*value)) { + char uuid[GUID_LEN + 1]; + uuid_unparse_lower_compact(*value, uuid); + buffer_json_add_array_item_string(wb, uuid); + } + else + buffer_json_add_array_item_string(wb, NULL); +} + static inline void buffer_json_add_array_item_double(BUFFER *wb, NETDATA_DOUBLE value) { buffer_print_json_comma_newline_spacing(wb); diff --git a/libnetdata/dictionary/dictionary.h b/libnetdata/dictionary/dictionary.h index 35212da07d082a..b75013cd11b53e 100644 --- a/libnetdata/dictionary/dictionary.h +++ b/libnetdata/dictionary/dictionary.h @@ -294,7 +294,7 @@ typedef DICTFE_CONST struct dictionary_foreach { DICTFE value ## _dfe = {}; \ (void)(value); /* needed to avoid warning when looping without using this */ \ for((value) = dictionary_foreach_start_rw(&value ## _dfe, (dict), (mode)); \ - (value ## _dfe.item) ; \ + (value ## _dfe.item) || (value) ; \ (value) = dictionary_foreach_next(&value ## _dfe)) \ { diff --git a/libnetdata/dyn_conf/dyn_conf.c b/libnetdata/dyn_conf/dyn_conf.c index ee4b4733a5f78c..ccbd97f23f2c01 100644 --- a/libnetdata/dyn_conf/dyn_conf.c +++ b/libnetdata/dyn_conf/dyn_conf.c @@ -620,7 +620,7 @@ void freez_dyncfg(void *ptr) { #ifdef NETDATA_TEST_DYNCFG static void handle_dyncfg_root(DICTIONARY *plugins_dict, struct uni_http_response *resp, int method) { - if (method != HTTP_METHOD_GET) { + if (method != HTTP_REQUEST_MODE_GET) { resp->content = "method not allowed"; resp->content_length = strlen(resp->content); resp->status = HTTP_RESP_METHOD_NOT_ALLOWED; @@ -640,7 +640,7 @@ static void handle_dyncfg_root(DICTIONARY *plugins_dict, struct uni_http_respons static void handle_plugin_root(struct uni_http_response *resp, int method, struct configurable_plugin *plugin, void *post_payload, size_t post_payload_size) { switch(method) { - case HTTP_METHOD_GET: + case HTTP_REQUEST_MODE_GET: { dyncfg_config_t cfg = plugin->get_config_cb(plugin->cb_usr_ctx, plugin->name); resp->content = mallocz(cfg.data_size); @@ -650,7 +650,7 @@ static void handle_plugin_root(struct uni_http_response *resp, int method, struc resp->content_length = cfg.data_size; return; } - case HTTP_METHOD_PUT: + case HTTP_REQUEST_MODE_PUT: { char *response; if (post_payload == NULL) { @@ -696,7 +696,7 @@ void handle_module_root(struct uni_http_response *resp, int method, struct confi return; } if (strncmp(module, DYN_CONF_MODULE_LIST, sizeof(DYN_CONF_MODULE_LIST)) == 0) { - if (method != HTTP_METHOD_GET) { + if (method != HTTP_REQUEST_MODE_GET) { resp->content = "method not allowed (only GET)"; resp->content_length = strlen(resp->content); resp->status = HTTP_RESP_METHOD_NOT_ALLOWED; @@ -720,7 +720,7 @@ void handle_module_root(struct uni_http_response *resp, int method, struct confi resp->status = HTTP_RESP_NOT_FOUND; return; } - if (method == HTTP_METHOD_GET) { + if (method == HTTP_REQUEST_MODE_GET) { dyncfg_config_t cfg = mod->get_config_cb(mod->config_cb_usr_ctx, plugin->name, mod->name); resp->content = mallocz(cfg.data_size); memcpy(resp->content, cfg.data, cfg.data_size); @@ -728,7 +728,7 @@ void handle_module_root(struct uni_http_response *resp, int method, struct confi resp->content_free = freez_dyncfg; resp->content_length = cfg.data_size; return; - } else if (method == HTTP_METHOD_PUT) { + } else if (method == HTTP_REQUEST_MODE_PUT) { char *response; if (post_payload == NULL) { resp->content = "no payload"; @@ -759,7 +759,7 @@ void handle_module_root(struct uni_http_response *resp, int method, struct confi static inline void _handle_job_root(struct uni_http_response *resp, int method, struct module *mod, const char *job_id, void *post_payload, size_t post_payload_size, struct job *job) { - if (method == HTTP_METHOD_POST) { + if (method == HTTP_REQUEST_MODE_POST) { if (job != NULL) { resp->content = "can't POST, job already exists (use PUT to update?)"; resp->content_length = strlen(resp->content); @@ -794,7 +794,7 @@ static inline void _handle_job_root(struct uni_http_response *resp, int method, return; } switch (method) { - case HTTP_METHOD_GET: + case HTTP_REQUEST_MODE_GET: { dyncfg_config_t cfg = mod->get_job_config_cb(mod->job_config_cb_usr_ctx, mod->plugin->name, mod->name, job->name); resp->content = mallocz(cfg.data_size); @@ -804,7 +804,7 @@ static inline void _handle_job_root(struct uni_http_response *resp, int method, resp->content_length = cfg.data_size; return; } - case HTTP_METHOD_PUT: + case HTTP_REQUEST_MODE_PUT: { if (post_payload == NULL) { resp->content = "missing payload"; @@ -828,7 +828,7 @@ static inline void _handle_job_root(struct uni_http_response *resp, int method, resp->content_length = strlen(resp->content); return; } - case HTTP_METHOD_DELETE: + case HTTP_REQUEST_MODE_DELETE: { if (!remove_job(mod, job)) { resp->content = "failed to remove job"; @@ -876,7 +876,7 @@ void handle_job_root(struct uni_http_response *resp, int method, struct module * resp->status = HTTP_RESP_NOT_FOUND; return; } - if (method != HTTP_METHOD_GET) { + if (method != HTTP_REQUEST_MODE_GET) { resp->content = "method not allowed (only GET)"; resp->content_length = strlen(resp->content); resp->status = HTTP_RESP_METHOD_NOT_ALLOWED; @@ -913,7 +913,7 @@ struct uni_http_response dyn_conf_process_http_request( struct uni_http_response resp = { .status = HTTP_RESP_INTERNAL_SERVER_ERROR, .content_type = CT_TEXT_PLAIN, - .content = HTTP_RESP_INTERNAL_SERVER_ERROR_STR, + .content = (char *) http_response_code2string(HTTP_RESP_INTERNAL_SERVER_ERROR), .content_free = NULL, .content_length = 0 }; diff --git a/libnetdata/facets/facets.c b/libnetdata/facets/facets.c index 4a5f5442bf322a..a5379e68b7ec2c 100644 --- a/libnetdata/facets/facets.c +++ b/libnetdata/facets/facets.c @@ -102,10 +102,7 @@ static inline bool is_valid_string_hash(const char *s) { // hashtable for FACET_VALUE // cleanup hashtable defines -#undef SIMPLE_HASHTABLE_SORT_FUNCTION -#undef SIMPLE_HASHTABLE_VALUE_TYPE -#undef SIMPLE_HASHTABLE_NAME -#undef NETDATA_SIMPLE_HASHTABLE_H +#include "../../libnetdata/simple_hashtable_undef.h" struct facet_value; // #define SIMPLE_HASHTABLE_SORT_FUNCTION compare_facet_value @@ -117,10 +114,7 @@ struct facet_value; // hashtable for FACET_KEY // cleanup hashtable defines -#undef SIMPLE_HASHTABLE_SORT_FUNCTION -#undef SIMPLE_HASHTABLE_VALUE_TYPE -#undef SIMPLE_HASHTABLE_NAME -#undef NETDATA_SIMPLE_HASHTABLE_H +#include "../../libnetdata/simple_hashtable_undef.h" struct facet_key; // #define SIMPLE_HASHTABLE_SORT_FUNCTION compare_facet_key @@ -439,12 +433,12 @@ static inline void FACET_VALUE_ADD_CONFLICT(FACET_KEY *k, FACET_VALUE *v, const } static inline FACET_VALUE *FACET_VALUE_GET_FROM_INDEX(FACET_KEY *k, FACETS_HASH hash) { - SIMPLE_HASHTABLE_SLOT_VALUE *slot = simple_hashtable_get_slot_VALUE(&k->values.ht, hash, true); + SIMPLE_HASHTABLE_SLOT_VALUE *slot = simple_hashtable_get_slot_VALUE(&k->values.ht, hash, NULL, true); return SIMPLE_HASHTABLE_SLOT_DATA(slot); } static inline FACET_VALUE *FACET_VALUE_ADD_TO_INDEX(FACET_KEY *k, const FACET_VALUE * const tv) { - SIMPLE_HASHTABLE_SLOT_VALUE *slot = simple_hashtable_get_slot_VALUE(&k->values.ht, tv->hash, true); + SIMPLE_HASHTABLE_SLOT_VALUE *slot = simple_hashtable_get_slot_VALUE(&k->values.ht, tv->hash, NULL, true); if(SIMPLE_HASHTABLE_SLOT_DATA(slot)) { // already exists @@ -634,7 +628,7 @@ static inline void FACETS_KEYS_INDEX_DESTROY(FACETS *facets) { } static inline FACET_KEY *FACETS_KEY_GET_FROM_INDEX(FACETS *facets, FACETS_HASH hash) { - SIMPLE_HASHTABLE_SLOT_KEY *slot = simple_hashtable_get_slot_KEY(&facets->keys.ht, hash, true); + SIMPLE_HASHTABLE_SLOT_KEY *slot = simple_hashtable_get_slot_KEY(&facets->keys.ht, hash, NULL, true); return SIMPLE_HASHTABLE_SLOT_DATA(slot); } @@ -714,7 +708,7 @@ static inline FACET_KEY *FACETS_KEY_CREATE(FACETS *facets, FACETS_HASH hash, con static inline FACET_KEY *FACETS_KEY_ADD_TO_INDEX(FACETS *facets, FACETS_HASH hash, const char *name, size_t name_length, FACET_KEY_OPTIONS options) { facets->operations.keys.registered++; - SIMPLE_HASHTABLE_SLOT_KEY *slot = simple_hashtable_get_slot_KEY(&facets->keys.ht, hash, true); + SIMPLE_HASHTABLE_SLOT_KEY *slot = simple_hashtable_get_slot_KEY(&facets->keys.ht, hash, NULL, true); if(unlikely(!SIMPLE_HASHTABLE_SLOT_DATA(slot))) { // we have to add it diff --git a/libnetdata/functions_evloop/functions_evloop.c b/libnetdata/functions_evloop/functions_evloop.c index 044556ac670b83..5daf0f73f41caf 100644 --- a/libnetdata/functions_evloop/functions_evloop.c +++ b/libnetdata/functions_evloop/functions_evloop.c @@ -8,6 +8,7 @@ struct functions_evloop_worker_job { bool used; bool running; bool cancelled; + usec_t stop_monotonic_ut; char *cmd; const char *transaction; time_t timeout; @@ -72,7 +73,7 @@ static void *rrd_functions_worker_globals_worker_main(void *arg) { last_acquired = true; j = dictionary_acquired_item_value(acquired); - j->cb(j->transaction, j->cmd, j->timeout, &j->cancelled); + j->cb(j->transaction, j->cmd, &j->stop_monotonic_ut, &j->cancelled); dictionary_del(wg->worker_queue, j->transaction); dictionary_acquired_item_release(wg->worker_queue, acquired); dictionary_garbage_collect(wg->worker_queue); @@ -102,7 +103,7 @@ static void *rrd_functions_worker_globals_reader_main(void *arg) { char *function = get_word(words, num_words, 3); if(!transaction || !*transaction || !timeout_s || !*timeout_s || !function || !*function) { - netdata_log_error("Received incomplete %s (transaction = '%s', timeout = '%s', function = '%s'). Ignoring it.", + nd_log(NDLS_COLLECTORS, NDLP_ERR, "Received incomplete %s (transaction = '%s', timeout = '%s', function = '%s'). Ignoring it.", keyword, transaction?transaction:"(unset)", timeout_s?timeout_s:"(unset)", @@ -111,24 +112,30 @@ static void *rrd_functions_worker_globals_reader_main(void *arg) { else { int timeout = str2i(timeout_s); + const char *msg = "No function with this name found"; bool found = false; struct rrd_functions_expectation *we; for(we = wg->expectations; we ;we = we->next) { if(strncmp(function, we->function, we->function_length) == 0) { + if(timeout <= 0) + timeout = (int)we->default_timeout; + struct functions_evloop_worker_job t = { .cmd = strdupz(function), .transaction = strdupz(transaction), .running = false, .cancelled = false, - .timeout = timeout > 0 ? timeout : we->default_timeout, + .timeout = timeout, + .stop_monotonic_ut = now_monotonic_usec() + (timeout * USEC_PER_SEC), .used = false, .cb = we->cb, }; struct functions_evloop_worker_job *j = dictionary_set(wg->worker_queue, transaction, &t, sizeof(t)); if(j->used) { - netdata_log_error("Received duplicate function transaction '%s'", transaction); + nd_log(NDLS_COLLECTORS, NDLP_WARNING, "Received duplicate function transaction '%s'. Ignoring it.", transaction); freez((void *)t.cmd); freez((void *)t.transaction); + msg = "Duplicate function transaction. Ignoring it."; } else { found = true; @@ -140,8 +147,7 @@ static void *rrd_functions_worker_globals_reader_main(void *arg) { if(!found) { netdata_mutex_lock(wg->stdout_mutex); - pluginsd_function_json_error_to_stdout(transaction, HTTP_RESP_NOT_FOUND, - "No function with this name found."); + pluginsd_function_json_error_to_stdout(transaction, HTTP_RESP_NOT_FOUND, msg); netdata_mutex_unlock(wg->stdout_mutex); } } @@ -157,15 +163,28 @@ static void *rrd_functions_worker_globals_reader_main(void *arg) { dictionary_garbage_collect(wg->worker_queue); } else - netdata_log_error("Received CANCEL for transaction '%s', but it not available here", transaction); + nd_log(NDLS_COLLECTORS, NDLP_NOTICE, "Received CANCEL for transaction '%s', but it not available here", transaction); + } + else if(keyword && strcmp(keyword, PLUGINSD_KEYWORD_FUNCTION_PROGRESS) == 0) { + char *transaction = get_word(words, num_words, 1); + const DICTIONARY_ITEM *acquired = dictionary_get_and_acquire_item(wg->worker_queue, transaction); + if(acquired) { + struct functions_evloop_worker_job *j = dictionary_acquired_item_value(acquired); + + functions_stop_monotonic_update_on_progress(&j->stop_monotonic_ut); + + dictionary_acquired_item_release(wg->worker_queue, acquired); + } + else + nd_log(NDLS_COLLECTORS, NDLP_NOTICE, "Received PROGRESS for transaction '%s', but it not available here", transaction); } else - netdata_log_error("Received unknown command: %s", keyword?keyword:"(unset)"); + nd_log(NDLS_COLLECTORS, NDLP_NOTICE, "Received unknown command: %s", keyword?keyword:"(unset)"); } if(!s || feof(stdin) || ferror(stdin)) { *wg->plugin_should_exit = true; - netdata_log_error("Received error on stdin."); + nd_log(NDLS_COLLECTORS, NDLP_ERR, "Received error on stdin."); } exit(1); diff --git a/libnetdata/functions_evloop/functions_evloop.h b/libnetdata/functions_evloop/functions_evloop.h index e5e83e95ebcf27..d9585a9d0f452e 100644 --- a/libnetdata/functions_evloop/functions_evloop.h +++ b/libnetdata/functions_evloop/functions_evloop.h @@ -7,6 +7,7 @@ #define PLUGINSD_KEYWORD_CHART "CHART" #define PLUGINSD_KEYWORD_CHART_DEFINITION_END "CHART_DEFINITION_END" + #define PLUGINSD_KEYWORD_DIMENSION "DIMENSION" #define PLUGINSD_KEYWORD_BEGIN "BEGIN" #define PLUGINSD_KEYWORD_SET "SET" @@ -18,8 +19,12 @@ #define PLUGINSD_KEYWORD_OVERWRITE "OVERWRITE" #define PLUGINSD_KEYWORD_CLABEL "CLABEL" #define PLUGINSD_KEYWORD_CLABEL_COMMIT "CLABEL_COMMIT" + #define PLUGINSD_KEYWORD_FUNCTION "FUNCTION" +#define PLUGINSD_KEYWORD_FUNCTION_PAYLOAD "FUNCTION_PAYLOAD" +#define PLUGINSD_KEYWORD_FUNCTION_PAYLOAD_END "FUNCTION_PAYLOAD_END" #define PLUGINSD_KEYWORD_FUNCTION_CANCEL "FUNCTION_CANCEL" +#define PLUGINSD_KEYWORD_FUNCTION_PROGRESS "FUNCTION_PROGRESS" #define PLUGINSD_KEYWORD_FUNCTION_RESULT_BEGIN "FUNCTION_RESULT_BEGIN" #define PLUGINSD_KEYWORD_FUNCTION_RESULT_END "FUNCTION_RESULT_END" @@ -50,12 +55,22 @@ #define PLUGINS_FUNCTIONS_TIMEOUT_DEFAULT 10 // seconds -typedef void (*functions_evloop_worker_execute_t)(const char *transaction, char *function, int timeout, bool *cancelled); +typedef void (*functions_evloop_worker_execute_t)(const char *transaction, char *function, usec_t *stop_monotonic_ut, bool *cancelled); struct functions_evloop_worker_job; struct functions_evloop_globals *functions_evloop_init(size_t worker_threads, const char *tag, netdata_mutex_t *stdout_mutex, bool *plugin_should_exit); void functions_evloop_add_function(struct functions_evloop_globals *wg, const char *function, functions_evloop_worker_execute_t cb, time_t default_timeout); void functions_evloop_cancel_threads(struct functions_evloop_globals *wg); +#define FUNCTIONS_EXTENDED_TIME_ON_PROGRESS_UT (10 * USEC_PER_SEC) +static inline void functions_stop_monotonic_update_on_progress(usec_t *stop_monotonic_ut) { + usec_t now_ut = now_monotonic_usec(); + if(now_ut + FUNCTIONS_EXTENDED_TIME_ON_PROGRESS_UT > *stop_monotonic_ut) { + nd_log(NDLS_DAEMON, NDLP_DEBUG, "Extending function timeout due to PROGRESS update..."); + __atomic_store_n(stop_monotonic_ut, now_ut + FUNCTIONS_EXTENDED_TIME_ON_PROGRESS_UT, __ATOMIC_RELAXED); + } + else + nd_log(NDLS_DAEMON, NDLP_DEBUG, "Received PROGRESS update..."); +} #define pluginsd_function_result_begin_to_buffer(wb, transaction, code, content_type, expires) \ buffer_sprintf(wb \ @@ -98,4 +113,10 @@ static inline void pluginsd_function_result_to_stdout(const char *transaction, i fflush(stdout); } +static inline void pluginsd_function_progress_to_stdout(const char *transaction, size_t done, size_t all) { + fprintf(stdout, PLUGINSD_KEYWORD_FUNCTION_PROGRESS " '%s' %zu %zu\n", + transaction, done, all); + fflush(stdout); +} + #endif //NETDATA_FUNCTIONS_EVLOOP_H diff --git a/libnetdata/http/http_access.c b/libnetdata/http/http_access.c new file mode 100644 index 00000000000000..95de91d3520647 --- /dev/null +++ b/libnetdata/http/http_access.c @@ -0,0 +1,38 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "../libnetdata.h" + +static struct { + HTTP_ACCESS access; + const char *name; +} rrd_function_access_levels[] = { + { .access = HTTP_ACCESS_NONE, .name = "none" }, + { .access = HTTP_ACCESS_MEMBERS, .name = "members" }, + { .access = HTTP_ACCESS_ADMINS, .name = "admins" }, + { .access = HTTP_ACCESS_ANY, .name = "any" }, +}; + +HTTP_ACCESS http_access2id(const char *access) { + if(!access || !*access) + return HTTP_ACCESS_MEMBERS; + + size_t entries = sizeof(rrd_function_access_levels) / sizeof(rrd_function_access_levels[0]); + for(size_t i = 0; i < entries ;i++) { + if(strcmp(rrd_function_access_levels[i].name, access) == 0) + return rrd_function_access_levels[i].access; + } + + nd_log(NDLS_DAEMON, NDLP_WARNING, "HTTP access level '%s' is not valid", access); + return HTTP_ACCESS_MEMBERS; +} + +const char *http_id2access(HTTP_ACCESS access) { + size_t entries = sizeof(rrd_function_access_levels) / sizeof(rrd_function_access_levels[0]); + for(size_t i = 0; i < entries ;i++) { + if(access == rrd_function_access_levels[i].access) + return rrd_function_access_levels[i].name; + } + + nd_log(NDLS_DAEMON, NDLP_WARNING, "HTTP access level %d is not valid", access); + return "members"; +} diff --git a/libnetdata/http/http_access.h b/libnetdata/http/http_access.h new file mode 100644 index 00000000000000..894ded257e537d --- /dev/null +++ b/libnetdata/http/http_access.h @@ -0,0 +1,56 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef NETDATA_HTTP_ACCESS_H +#define NETDATA_HTTP_ACCESS_H + +typedef enum __attribute__((packed)) { + HTTP_ACCESS_NONE = 0, + HTTP_ACCESS_ADMINS = 1, + HTTP_ACCESS_MEMBERS = 2, + HTTP_ACCESS_ANY = 3, + + // keep this list so that lower numbers are more strict access levels +} HTTP_ACCESS; + +const char *http_id2access(HTTP_ACCESS access); +HTTP_ACCESS http_access2id(const char *access); + +typedef enum __attribute__((packed)) { + HTTP_ACL_NONE = (0), + HTTP_ACL_NOCHECK = (1 << 0), // Don't check anything - this should work on all channels + HTTP_ACL_DASHBOARD = (1 << 1), + HTTP_ACL_REGISTRY = (1 << 2), + HTTP_ACL_BADGE = (1 << 3), + HTTP_ACL_MGMT = (1 << 4), + HTTP_ACL_STREAMING = (1 << 5), + HTTP_ACL_NETDATACONF = (1 << 6), + HTTP_ACL_SSL_OPTIONAL = (1 << 7), + HTTP_ACL_SSL_FORCE = (1 << 8), + HTTP_ACL_SSL_DEFAULT = (1 << 9), + HTTP_ACL_ACLK = (1 << 10), + HTTP_ACL_WEBRTC = (1 << 11), + HTTP_ACL_BEARER_IF_PROTECTED = (1 << 12), // allow unprotected access if bearer is not enabled in netdata + HTTP_ACL_BEARER_REQUIRED = (1 << 13), // allow access only if a valid bearer is used + HTTP_ACL_BEARER_OPTIONAL = (1 << 14), // the call may or may not need a bearer - will be determined later +} HTTP_ACL; + +#define HTTP_ACL_DASHBOARD_ACLK_WEBRTC (HTTP_ACL_DASHBOARD | HTTP_ACL_ACLK | HTTP_ACL_WEBRTC | HTTP_ACL_BEARER_IF_PROTECTED) +#define HTTP_ACL_ACLK_WEBRTC_DASHBOARD_WITH_OPTIONAL_BEARER (HTTP_ACL_DASHBOARD | HTTP_ACL_ACLK | HTTP_ACL_WEBRTC | HTTP_ACL_BEARER_OPTIONAL) + +#ifdef NETDATA_DEV_MODE +#define ACL_DEV_OPEN_ACCESS HTTP_ACL_NOCHECK +#else +#define ACL_DEV_OPEN_ACCESS 0 +#endif + +#define http_can_access_dashboard(w) ((w)->acl & HTTP_ACL_DASHBOARD) +#define http_can_access_registry(w) ((w)->acl & HTTP_ACL_REGISTRY) +#define http_can_access_badges(w) ((w)->acl & HTTP_ACL_BADGE) +#define http_can_access_mgmt(w) ((w)->acl & HTTP_ACL_MGMT) +#define http_can_access_stream(w) ((w)->acl & HTTP_ACL_STREAMING) +#define http_can_access_netdataconf(w) ((w)->acl & HTTP_ACL_NETDATACONF) +#define http_is_using_ssl_optional(w) ((w)->port_acl & HTTP_ACL_SSL_OPTIONAL) +#define http_is_using_ssl_force(w) ((w)->port_acl & HTTP_ACL_SSL_FORCE) +#define http_is_using_ssl_default(w) ((w)->port_acl & HTTP_ACL_SSL_DEFAULT) + +#endif //NETDATA_HTTP_ACCESS_H diff --git a/libnetdata/http/http_defs.c b/libnetdata/http/http_defs.c new file mode 100644 index 00000000000000..261b5d930ddcfe --- /dev/null +++ b/libnetdata/http/http_defs.c @@ -0,0 +1,255 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "../libnetdata.h" + +const char *http_request_method2string(HTTP_REQUEST_MODE mode) { + switch(mode) { + case HTTP_REQUEST_MODE_OPTIONS: + return "OPTIONS"; + + case HTTP_REQUEST_MODE_GET: + return "GET"; + + case HTTP_REQUEST_MODE_FILECOPY: + return "FILECOPY"; + + case HTTP_REQUEST_MODE_POST: + return "POST"; + + case HTTP_REQUEST_MODE_PUT: + return "PUT"; + + case HTTP_REQUEST_MODE_DELETE: + return "DELETE"; + + case HTTP_REQUEST_MODE_STREAM: + return "STREAM"; + + default: + return "UNKNOWN"; + } +} + +const char *http_response_code2string(int code) { + switch(code) { + case 100: + return "Continue"; + case 101: + return "Switching Protocols"; + case 102: + return "Processing"; + case 103: + return "Early Hints"; + + case 200: + return "OK"; + case 201: + return "Created"; + case 202: + return "Accepted"; + case 203: + return "Non-Authoritative Information"; + case 204: + return "No Content"; + case 205: + return "Reset Content"; + case 206: + return "Partial Content"; + case 207: + return "Multi-Status"; + case 208: + return "Already Reported"; + case 226: + return "IM Used"; + + case 300: + return "Multiple Choices"; + case 301: + return "Moved Permanently"; + case 302: + return "Found"; + case 303: + return "See Other"; + case 304: + return "Not Modified"; + case 305: + return "Use Proxy"; + case 306: + return "Switch Proxy"; + case 307: + return "Temporary Redirect"; + case 308: + return "Permanent Redirect"; + + case 400: + return "Bad Request"; + case 401: + return "Unauthorized"; + case 402: + return "Payment Required"; + case 403: + return "Forbidden"; + case 404: + return "Not Found"; + case 405: + return "Method Not Allowed"; + case 406: + return "Not Acceptable"; + case 407: + return "Proxy Authentication Required"; + case 408: + return "Request Timeout"; + case 409: + return "Conflict"; + case 410: + return "Gone"; + case 411: + return "Length Required"; + case 412: + return "Precondition Failed"; + case 413: + return "Payload Too Large"; + case 414: + return "URI Too Long"; + case 415: + return "Unsupported Media Type"; + case 416: + return "Range Not Satisfiable"; + case 417: + return "Expectation Failed"; + case 418: + return "I'm a teapot"; + case 421: + return "Misdirected Request"; + case 422: + return "Unprocessable Entity"; + case 423: + return "Locked"; + case 424: + return "Failed Dependency"; + case 425: + return "Too Early"; + case 426: + return "Upgrade Required"; + case 428: + return "Precondition Required"; + case 429: + return "Too Many Requests"; + case 431: + return "Request Header Fields Too Large"; + case 451: + return "Unavailable For Legal Reasons"; + case 499: // nginx's extension to the standard + return "Client Closed Request"; + + case 500: + return "Internal Server Error"; + case 501: + return "Not Implemented"; + case 502: + return "Bad Gateway"; + case 503: + return "Service Unavailable"; + case 504: + return "Gateway Timeout"; + case 505: + return "HTTP Version Not Supported"; + case 506: + return "Variant Also Negotiates"; + case 507: + return "Insufficient Storage"; + case 508: + return "Loop Detected"; + case 510: + return "Not Extended"; + case 511: + return "Network Authentication Required"; + + default: + if(code >= 100 && code < 200) + return "Informational"; + + if(code >= 200 && code < 300) + return "Successful"; + + if(code >= 300 && code < 400) + return "Redirection"; + + if(code >= 400 && code < 500) + return "Client Error"; + + if(code >= 500 && code < 600) + return "Server Error"; + + return "Undefined Error"; + } +} + + +static struct { + const char *extension; + uint32_t hash; + HTTP_CONTENT_TYPE contenttype; +} mime_types[] = { + { "html" , 0 , CT_TEXT_HTML } + , { "js" , 0 , CT_APPLICATION_X_JAVASCRIPT } + , { "css" , 0 , CT_TEXT_CSS } + , { "xml" , 0 , CT_TEXT_XML } + , { "xsl" , 0 , CT_TEXT_XSL } + , { "txt" , 0 , CT_TEXT_PLAIN } + , { "svg" , 0 , CT_IMAGE_SVG_XML } + , { "ttf" , 0 , CT_APPLICATION_X_FONT_TRUETYPE } + , { "otf" , 0 , CT_APPLICATION_X_FONT_OPENTYPE } + , { "woff2", 0 , CT_APPLICATION_FONT_WOFF2 } + , { "woff" , 0 , CT_APPLICATION_FONT_WOFF } + , { "eot" , 0 , CT_APPLICATION_VND_MS_FONTOBJ } + , { "png" , 0 , CT_IMAGE_PNG } + , { "jpg" , 0 , CT_IMAGE_JPG } + , { "jpeg" , 0 , CT_IMAGE_JPG } + , { "gif" , 0 , CT_IMAGE_GIF } + , { "bmp" , 0 , CT_IMAGE_BMP } + , { "ico" , 0 , CT_IMAGE_XICON } + , { "icns" , 0 , CT_IMAGE_ICNS } + , { NULL , 0 , 0 } +}; + +HTTP_CONTENT_TYPE contenttype_for_filename(const char *filename) { + // netdata_log_info("checking filename '%s'", filename); + + static int initialized = 0; + int i; + + if(unlikely(!initialized)) { + for (i = 0; mime_types[i].extension; i++) + mime_types[i].hash = simple_hash(mime_types[i].extension); + + initialized = 1; + } + + const char *s = filename, *last_dot = NULL; + + // find the last dot + while(*s) { + if(unlikely(*s == '.')) last_dot = s; + s++; + } + + if(unlikely(!last_dot || !*last_dot || !last_dot[1])) { + // netdata_log_info("no extension for filename '%s'", filename); + return CT_APPLICATION_OCTET_STREAM; + } + last_dot++; + + // netdata_log_info("extension for filename '%s' is '%s'", filename, last_dot); + + uint32_t hash = simple_hash(last_dot); + for(i = 0; mime_types[i].extension ; i++) { + if(unlikely(hash == mime_types[i].hash && !strcmp(last_dot, mime_types[i].extension))) { + // netdata_log_info("matched extension for filename '%s': '%s'", filename, last_dot); + return mime_types[i].contenttype; + } + } + + // netdata_log_info("not matched extension for filename '%s': '%s'", filename, last_dot); + return CT_APPLICATION_OCTET_STREAM; +} diff --git a/libnetdata/http/http_defs.h b/libnetdata/http/http_defs.h index 635f703e40888b..cb26a65a5d8456 100644 --- a/libnetdata/http/http_defs.h +++ b/libnetdata/http/http_defs.h @@ -26,7 +26,6 @@ #define HTTP_RESP_FORBIDDEN 403 #define HTTP_RESP_NOT_FOUND 404 #define HTTP_RESP_METHOD_NOT_ALLOWED 405 -#define HTTP_RESP_METHOD_NOT_ALLOWED_STR "Method Not Allowed" #define HTTP_RESP_CONFLICT 409 #define HTTP_RESP_PRECOND_FAIL 412 #define HTTP_RESP_CONTENT_TOO_LONG 413 @@ -34,14 +33,23 @@ // HTTP_CODES 5XX Server Errors #define HTTP_RESP_INTERNAL_SERVER_ERROR 500 -#define HTTP_RESP_INTERNAL_SERVER_ERROR_STR "Internal Server Error" #define HTTP_RESP_SERVICE_UNAVAILABLE 503 #define HTTP_RESP_GATEWAY_TIMEOUT 504 #define HTTP_RESP_BACKEND_RESPONSE_INVALID 591 -#define HTTP_METHOD_GET (1) -#define HTTP_METHOD_POST (2) -#define HTTP_METHOD_PUT (3) -#define HTTP_METHOD_DELETE (4) +typedef enum __attribute__((__packed__)) { + HTTP_REQUEST_MODE_NONE = 0, + HTTP_REQUEST_MODE_GET = 1, + HTTP_REQUEST_MODE_POST = 2, + HTTP_REQUEST_MODE_PUT = 3, + HTTP_REQUEST_MODE_DELETE = 4, + HTTP_REQUEST_MODE_FILECOPY = 5, + HTTP_REQUEST_MODE_OPTIONS = 6, + HTTP_REQUEST_MODE_STREAM = 7, +} HTTP_REQUEST_MODE; + +const char *http_request_method2string(HTTP_REQUEST_MODE mode); +const char *http_response_code2string(int code); +HTTP_CONTENT_TYPE contenttype_for_filename(const char *filename); #endif /* NETDATA_HTTP_DEFS_H */ diff --git a/libnetdata/libnetdata.h b/libnetdata/libnetdata.h index bef7654e05501f..ee35dc12a8c02d 100644 --- a/libnetdata/libnetdata.h +++ b/libnetdata/libnetdata.h @@ -697,7 +697,7 @@ extern char *netdata_configured_host_prefix; #include "xxhash.h" #include "uuid/uuid.h" - +#include "http/http_access.h" #include "libjudy/src/Judy.h" #include "july/july.h" #include "os.h" @@ -745,6 +745,7 @@ extern char *netdata_configured_host_prefix; #include "facets/facets.h" #include "dyn_conf/dyn_conf.h" #include "functions_evloop/functions_evloop.h" +#include "query_progress/progress.h" // BEWARE: this exists in alarm-notify.sh #define DEFAULT_CLOUD_BASE_URL "https://app.netdata.cloud" diff --git a/libnetdata/log/README.md b/libnetdata/log/README.md index d9ed64374d546a..b6da3297de286e 100644 --- a/libnetdata/log/README.md +++ b/libnetdata/log/README.md @@ -138,6 +138,8 @@ Netdata exposes the following fields to its logs: | `ND_SRC_TRANSPORT` | `src_transport` | `src_transport` | when the event happened during a request, this is the request transport | | `ND_SRC_IP` | `src_ip` | `src_ip` | when the event happened during an inbound request, this is the IP the request came from | | `ND_SRC_PORT` | `src_port` | `src_port` | when the event happened during an inbound request, this is the port the request came from | +| `ND_SRC_FORWARDED_HOST` | `src_forwarded_host` | `src_forwarded_host` | the contents of the HTTP header `X-Forwarded-Host` | +| `ND_SRC_FORWARDED_FOR` | `src_forwarded_for` | `src_forwarded_for` | the contents of the HTTP header `X-Forwarded-For` | | `ND_SRC_CAPABILITIES` | `src_capabilities` | `src_capabilities` | when the request came from a child, this is the communication capabilities of the child | | `ND_DST_TRANSPORT` | `dst_transport` | `dst_transport` | when the event happened during an outbound request, this is the outbound request transport | | `ND_DST_IP` | `dst_ip` | `dst_ip` | when the event happened during an outbound request, this is the IP the request destination | diff --git a/libnetdata/log/log.c b/libnetdata/log/log.c index 95ae30a05cde6b..e510e9559f239f 100644 --- a/libnetdata/log/log.c +++ b/libnetdata/log/log.c @@ -1121,6 +1121,14 @@ static __thread struct log_field thread_log_fields[_NDF_MAX] = { .journal = "ND_SRC_IP", .logfmt = "src_ip", }, + [NDF_SRC_FORWARDED_HOST] = { + .journal = "ND_SRC_FORWARDED_HOST", + .logfmt = "src_forwarded_host", + }, + [NDF_SRC_FORWARDED_FOR] = { + .journal = "ND_SRC_FORWARDED_FOR", + .logfmt = "src_forwarded_for", + }, [NDF_SRC_PORT] = { .journal = "ND_SRC_PORT", .logfmt = "src_port", diff --git a/libnetdata/log/log.h b/libnetdata/log/log.h index ae531d01e76b8a..c626ba2ecc53fe 100644 --- a/libnetdata/log/log.h +++ b/libnetdata/log/log.h @@ -66,6 +66,8 @@ typedef enum __attribute__((__packed__)) { // web server and stream receiver NDF_SRC_IP, // the streaming / web server source IP NDF_SRC_PORT, // the streaming / web server source Port + NDF_SRC_FORWARDED_HOST, + NDF_SRC_FORWARDED_FOR, NDF_SRC_CAPABILITIES, // the stream receiver capabilities // stream sender (established links) diff --git a/libnetdata/query_progress/README.md b/libnetdata/query_progress/README.md new file mode 100644 index 00000000000000..e69de29bb2d1d6 diff --git a/libnetdata/query_progress/progress.c b/libnetdata/query_progress/progress.c new file mode 100644 index 00000000000000..b1fcbefd6a0b27 --- /dev/null +++ b/libnetdata/query_progress/progress.c @@ -0,0 +1,654 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "progress.h" + +#define PROGRESS_CACHE_SIZE 200 + +// ---------------------------------------------------------------------------- +// hashtable for HASHED_KEY + +// cleanup hashtable defines +#include "../simple_hashtable_undef.h" + +struct query; +#define SIMPLE_HASHTABLE_VALUE_TYPE struct query +#define SIMPLE_HASHTABLE_KEY_TYPE uuid_t +#define SIMPLE_HASHTABLE_NAME _QUERY +#define SIMPLE_HASHTABLE_VALUE2KEY_FUNCTION query_transaction +#define SIMPLE_HASHTABLE_COMPARE_KEYS_FUNCTION query_compare_keys +#include "../simple_hashtable.h" + +// ---------------------------------------------------------------------------- + +typedef struct query { + uuid_t transaction; + + BUFFER *query; + BUFFER *payload; + BUFFER *client; + + usec_t started_ut; + usec_t finished_ut; + + HTTP_REQUEST_MODE mode; + HTTP_ACL acl; + + uint32_t sent_size; + uint32_t response_size; + short response_code; + + bool indexed; + + uint32_t updates; + + usec_t duration_ut; + size_t all; + size_t done; + + struct query *prev, *next; +} QUERY_PROGRESS; + +static inline uuid_t *query_transaction(QUERY_PROGRESS *qp) { + return qp ? &qp->transaction : NULL; +} + +static inline bool query_compare_keys(uuid_t *t1, uuid_t *t2) { + if(t1 == t2 || (t1 && t2 && memcmp(t1, t2, sizeof(uuid_t)) == 0)) + return true; + + return false; +} + +static struct progress { + SPINLOCK spinlock; + bool initialized; + + struct { + size_t available; + QUERY_PROGRESS *list; + } cache; + + SIMPLE_HASHTABLE_QUERY hashtable; + +} progress = { + .initialized = false, +}; + +SIMPLE_HASHTABLE_HASH query_hash(uuid_t *transaction) { + struct uuid_hi_lo_t { + uint64_t hi; + uint64_t lo; + } *parts = (struct uuid_hi_lo_t *)transaction; + + return parts->lo; +} + +static void query_progress_init_unsafe(void) { + if(!progress.initialized) { + memset(&progress, 0, sizeof(progress)); + simple_hashtable_init_QUERY(&progress.hashtable, PROGRESS_CACHE_SIZE * 4); + progress.initialized = true; + } +} + +// ---------------------------------------------------------------------------- + +static inline QUERY_PROGRESS *query_progress_find_in_hashtable_unsafe(uuid_t *transaction) { + SIMPLE_HASHTABLE_HASH hash = query_hash(transaction); + SIMPLE_HASHTABLE_SLOT_QUERY *slot = simple_hashtable_get_slot_QUERY(&progress.hashtable, hash, transaction, true); + QUERY_PROGRESS *qp = SIMPLE_HASHTABLE_SLOT_DATA(slot); + + assert(!qp || qp->indexed); + + return qp; +} + +static inline void query_progress_add_to_hashtable_unsafe(QUERY_PROGRESS *qp) { + assert(!qp->indexed); + + SIMPLE_HASHTABLE_HASH hash = query_hash(&qp->transaction); + SIMPLE_HASHTABLE_SLOT_QUERY *slot = + simple_hashtable_get_slot_QUERY(&progress.hashtable, hash, &qp->transaction, true); + + internal_fatal(SIMPLE_HASHTABLE_SLOT_DATA(slot) != NULL && SIMPLE_HASHTABLE_SLOT_DATA(slot) != qp, + "Attempt to overwrite a progress slot, with another value"); + + simple_hashtable_set_slot_QUERY(&progress.hashtable, slot, hash, qp); + + qp->indexed = true; +} + +static inline void query_progress_remove_from_hashtable_unsafe(QUERY_PROGRESS *qp) { + assert(qp->indexed); + + SIMPLE_HASHTABLE_HASH hash = query_hash(&qp->transaction); + SIMPLE_HASHTABLE_SLOT_QUERY *slot = + simple_hashtable_get_slot_QUERY(&progress.hashtable, hash, &qp->transaction, true); + + if(SIMPLE_HASHTABLE_SLOT_DATA(slot) == qp) + simple_hashtable_del_slot_QUERY(&progress.hashtable, slot); + else + internal_fatal(SIMPLE_HASHTABLE_SLOT_DATA(slot) != NULL, + "Attempt to remove from the hashtable a progress slot with a different value"); + + qp->indexed = false; +} + +// ---------------------------------------------------------------------------- + +static QUERY_PROGRESS *query_progress_alloc(uuid_t *transaction) { + QUERY_PROGRESS *qp; + qp = callocz(1, sizeof(*qp)); + uuid_copy(qp->transaction, *transaction); + qp->query = buffer_create(0, NULL); + qp->payload = buffer_create(0, NULL); + qp->client = buffer_create(0, NULL); + return qp; +} + +static void query_progress_free(QUERY_PROGRESS *qp) { + if(!qp) return; + + buffer_free(qp->query); + buffer_free(qp->payload); + buffer_free(qp->client); + freez(qp); +} + +static void query_progress_cleanup_to_reuse(QUERY_PROGRESS *qp, uuid_t *transaction) { + assert(qp && qp->prev == NULL && qp->next == NULL); + assert(!transaction || !qp->indexed); + + buffer_flush(qp->query); + buffer_flush(qp->payload); + buffer_flush(qp->client); + qp->started_ut = qp->finished_ut = qp->duration_ut = 0; + qp->all = qp->done = qp->updates = 0; + qp->acl = 0; + qp->next = qp->prev = NULL; + qp->response_size = qp->sent_size = 0; + qp->response_code = 0; + + if(transaction) + uuid_copy(qp->transaction, *transaction); +} + +static inline void query_progress_update(QUERY_PROGRESS *qp, usec_t started_ut, HTTP_REQUEST_MODE mode, HTTP_ACL acl, const char *query, const char *payload, const char *client) { + qp->mode = mode; + qp->acl = acl; + qp->started_ut = started_ut ? started_ut : now_realtime_usec(); + qp->finished_ut = 0; + qp->duration_ut = 0; + qp->response_size = 0; + qp->sent_size = 0; + qp->response_code = 0; + + if(query && *query && !buffer_strlen(qp->query)) + buffer_strcat(qp->query, query); + + if(payload && *payload && !buffer_strlen(qp->payload)) + buffer_strcat(qp->payload, payload); + + if(client && *client && !buffer_strlen(qp->client)) + buffer_strcat(qp->client, client); +} + +// ---------------------------------------------------------------------------- + +static inline void query_progress_link_to_cache_unsafe(QUERY_PROGRESS *qp) { + assert(!qp->prev && !qp->next); + DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(progress.cache.list, qp, prev, next); + progress.cache.available++; +} + +static inline void query_progress_unlink_from_cache_unsafe(QUERY_PROGRESS *qp) { + assert(qp->prev); + DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(progress.cache.list, qp, prev, next); + progress.cache.available--; +} + +// ---------------------------------------------------------------------------- +// Progress API + +void query_progress_start_or_update(uuid_t *transaction, usec_t started_ut, HTTP_REQUEST_MODE mode, HTTP_ACL acl, const char *query, const char *payload, const char *client) { + if(!transaction) + return; + + spinlock_lock(&progress.spinlock); + query_progress_init_unsafe(); + + QUERY_PROGRESS *qp = query_progress_find_in_hashtable_unsafe(transaction); + if(qp) { + // the transaction is already there + if(qp->prev) { + // reusing a finished transaction + query_progress_unlink_from_cache_unsafe(qp); + query_progress_cleanup_to_reuse(qp, NULL); + } + } + else if (progress.cache.available >= PROGRESS_CACHE_SIZE && progress.cache.list) { + // transaction is not found - get the first available, if any. + qp = progress.cache.list; + query_progress_unlink_from_cache_unsafe(qp); + + query_progress_remove_from_hashtable_unsafe(qp); + query_progress_cleanup_to_reuse(qp, transaction); + } + else { + qp = query_progress_alloc(transaction); + } + + query_progress_update(qp, started_ut, mode, acl, query, payload, client); + + if(!qp->indexed) + query_progress_add_to_hashtable_unsafe(qp); + + spinlock_unlock(&progress.spinlock); +} + +void query_progress_set_finish_line(uuid_t *transaction, size_t all) { + if(!transaction) + return; + + spinlock_lock(&progress.spinlock); + query_progress_init_unsafe(); + + QUERY_PROGRESS *qp = query_progress_find_in_hashtable_unsafe(transaction); + if(qp) { + qp->updates++; + + if(all > qp->all) + qp->all = all; + } + + spinlock_unlock(&progress.spinlock); +} + +void query_progress_done_step(uuid_t *transaction, size_t done) { + if(!transaction) + return; + + spinlock_lock(&progress.spinlock); + query_progress_init_unsafe(); + + QUERY_PROGRESS *qp = query_progress_find_in_hashtable_unsafe(transaction); + if(qp) { + qp->updates++; + qp->done += done; + } + + spinlock_unlock(&progress.spinlock); +} + +void query_progress_finished(uuid_t *transaction, usec_t finished_ut, short int response_code, usec_t duration_ut, size_t response_size, size_t sent_size) { + if(!transaction) + return; + + spinlock_lock(&progress.spinlock); + query_progress_init_unsafe(); + + // find this transaction to update it + { + QUERY_PROGRESS *qp = query_progress_find_in_hashtable_unsafe(transaction); + if(qp) { + qp->sent_size = sent_size; + qp->response_size = response_size; + qp->response_code = response_code; + qp->duration_ut = duration_ut; + qp->finished_ut = finished_ut ? finished_ut : now_realtime_usec(); + + if(qp->prev) + query_progress_unlink_from_cache_unsafe(qp); + + query_progress_link_to_cache_unsafe(qp); + } + } + + // find an item to free + { + QUERY_PROGRESS *qp_to_free = NULL; + if(progress.cache.available > PROGRESS_CACHE_SIZE && progress.cache.list) { + qp_to_free = progress.cache.list; + query_progress_unlink_from_cache_unsafe(qp_to_free); + query_progress_remove_from_hashtable_unsafe(qp_to_free); + } + + spinlock_unlock(&progress.spinlock); + + query_progress_free(qp_to_free); + } +} + +void query_progress_functions_update(uuid_t *transaction, size_t done, size_t all) { + // functions send to the total 'done', not the increment + + if(!transaction) + return; + + spinlock_lock(&progress.spinlock); + query_progress_init_unsafe(); + + QUERY_PROGRESS *qp = query_progress_find_in_hashtable_unsafe(transaction); + + if(qp) { + if(all) + qp->all = all; + + if(done) + qp->done = done; + + qp->updates++; + } + + spinlock_unlock(&progress.spinlock); +} + +// ---------------------------------------------------------------------------- +// /api/v2/progress - to get the progress of a transaction + +int web_api_v2_report_progress(uuid_t *transaction, BUFFER *wb) { + buffer_flush(wb); + buffer_json_initialize(wb, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_MINIFY); + + if(!transaction) { + buffer_json_member_add_uint64(wb, "status", 400); + buffer_json_member_add_string(wb, "message", "No transaction given"); + buffer_json_finalize(wb); + return 400; + } + + spinlock_lock(&progress.spinlock); + query_progress_init_unsafe(); + + QUERY_PROGRESS *qp = query_progress_find_in_hashtable_unsafe(transaction); + if(!qp) { + spinlock_unlock(&progress.spinlock); + buffer_json_member_add_uint64(wb, "status", HTTP_RESP_NOT_FOUND); + buffer_json_member_add_string(wb, "message", "Transaction not found"); + buffer_json_finalize(wb); + return HTTP_RESP_NOT_FOUND; + } + + buffer_json_member_add_uint64(wb, "status", 200); + + if(qp->finished_ut) { + buffer_json_member_add_double(wb, "progress", 100.0); + buffer_json_member_add_uint64(wb, "age_ut", qp->finished_ut - qp->started_ut); + } + else { + buffer_json_member_add_uint64(wb, "age_ut", now_realtime_usec() - qp->started_ut); + + if (qp->all) + buffer_json_member_add_double(wb, "progress", (double) qp->done * 100.0 / (double) qp->all); + else + buffer_json_member_add_uint64(wb, "working", qp->done); + } + + buffer_json_finalize(wb); + + spinlock_unlock(&progress.spinlock); + + return 200; +} + +// ---------------------------------------------------------------------------- +// function to show the progress of all current queries +// and the recent few completed queries + +int progress_function_result(BUFFER *wb, const char *hostname) { + buffer_flush(wb); + wb->content_type = CT_APPLICATION_JSON; + buffer_json_initialize(wb, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_DEFAULT); + + buffer_json_member_add_string(wb, "hostname", hostname); + buffer_json_member_add_uint64(wb, "status", HTTP_RESP_OK); + buffer_json_member_add_string(wb, "type", "table"); + buffer_json_member_add_time_t(wb, "update_every", 1); + buffer_json_member_add_string(wb, "help", RRDFUNCTIONS_PROGRESS_HELP); + buffer_json_member_add_array(wb, "data"); + + spinlock_lock(&progress.spinlock); + query_progress_init_unsafe(); + + usec_t now_ut = now_realtime_usec(); + usec_t max_duration_ut = 0; + size_t max_size = 0, max_sent = 0; + size_t archived = 0, running = 0; + SIMPLE_HASHTABLE_FOREACH_READ_ONLY(&progress.hashtable, sl, _QUERY) { + QUERY_PROGRESS *qp = SIMPLE_HASHTABLE_FOREACH_READ_ONLY_VALUE(sl); + if(unlikely(!qp)) continue; // not really needed, just for completeness + + if(qp->prev) + archived++; + else + running++; + + bool finished = qp->finished_ut ? true : false; + usec_t duration_ut = finished ? qp->duration_ut : now_ut - qp->started_ut; + if(duration_ut > max_duration_ut) + max_duration_ut = duration_ut; + + if(finished) { + if(qp->response_size > max_size) + max_size = qp->response_size; + + if(qp->sent_size > max_sent) + max_sent = qp->sent_size; + } + + buffer_json_add_array_item_array(wb); // row + + buffer_json_add_array_item_uuid_compact(wb, &qp->transaction); + buffer_json_add_array_item_uint64(wb, qp->started_ut); + buffer_json_add_array_item_string(wb, http_request_method2string(qp->mode)); + buffer_json_add_array_item_string(wb, buffer_tostring(qp->query)); + + if(!buffer_strlen(qp->client)) { + if(qp->acl & HTTP_ACL_ACLK) + buffer_json_add_array_item_string(wb, "ACLK"); + else if(qp->acl & HTTP_ACL_WEBRTC) + buffer_json_add_array_item_string(wb, "WEBRTC"); + else + buffer_json_add_array_item_string(wb, "unknown"); + } + else + buffer_json_add_array_item_string(wb, buffer_tostring(qp->client)); + + if(finished) { + buffer_json_add_array_item_string(wb, "finished"); + buffer_json_add_array_item_string(wb, "100.00 %%"); + } + else { + char buf[50]; + + buffer_json_add_array_item_string(wb, "in-progress"); + + if (qp->all) + snprintfz(buf, sizeof(buf), "%0.2f %%", (double) qp->done * 100.0 / (double) qp->all); + else + snprintfz(buf, sizeof(buf), "%zu", qp->done); + + buffer_json_add_array_item_string(wb, buf); + } + + buffer_json_add_array_item_double(wb, (double)duration_ut / USEC_PER_MS); + + if(finished) { + buffer_json_add_array_item_uint64(wb, qp->response_code); + buffer_json_add_array_item_uint64(wb, qp->response_size); + buffer_json_add_array_item_uint64(wb, qp->sent_size); + } + else { + buffer_json_add_array_item_string(wb, NULL); + buffer_json_add_array_item_string(wb, NULL); + buffer_json_add_array_item_string(wb, NULL); + } + + buffer_json_add_array_item_object(wb); // row options + { + char *severity = "notice"; + if(finished) { + if(qp->response_code == HTTP_RESP_NOT_MODIFIED || + qp->response_code == HTTP_RESP_CLIENT_CLOSED_REQUEST || + qp->response_code == HTTP_RESP_CONFLICT) + severity = "debug"; + else if(qp->response_code >= 500 && qp->response_code <= 599) + severity = "error"; + else if(qp->response_code >= 400 && qp->response_code <= 499) + severity = "warning"; + else if(qp->response_code >= 300 && qp->response_code <= 399) + severity = "notice"; + else + severity = "normal"; + } + buffer_json_member_add_string(wb, "severity", severity); + } + buffer_json_object_close(wb); // row options + + buffer_json_array_close(wb); // row + } + + assert(archived == progress.cache.available); + + spinlock_unlock(&progress.spinlock); + + buffer_json_array_close(wb); // data + buffer_json_member_add_object(wb, "columns"); + { + size_t field_id = 0; + + // transaction + buffer_rrdf_table_add_field(wb, field_id++, "Transaction", "Transaction ID", + RRDF_FIELD_TYPE_STRING, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NONE, + 0, NULL, NAN, RRDF_FIELD_SORT_ASCENDING, NULL, + RRDF_FIELD_SUMMARY_COUNT, RRDF_FIELD_FILTER_NONE, + RRDF_FIELD_OPTS_VISIBLE | RRDF_FIELD_OPTS_UNIQUE_KEY, + NULL); + + // timestamp + buffer_rrdf_table_add_field(wb, field_id++, "Started", "Query Start Timestamp", + RRDF_FIELD_TYPE_TIMESTAMP, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_DATETIME_USEC, + 0, NULL, NAN, RRDF_FIELD_SORT_DESCENDING, NULL, + RRDF_FIELD_SUMMARY_MAX, RRDF_FIELD_FILTER_NONE, + RRDF_FIELD_OPTS_VISIBLE, NULL); + + // request method + buffer_rrdf_table_add_field(wb, field_id++, "Method", "Request Method", + RRDF_FIELD_TYPE_STRING, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NONE, + 0, NULL, NAN, RRDF_FIELD_SORT_ASCENDING, NULL, + RRDF_FIELD_SUMMARY_COUNT, RRDF_FIELD_FILTER_MULTISELECT, + RRDF_FIELD_OPTS_VISIBLE, NULL); + + // query + buffer_rrdf_table_add_field(wb, field_id++, "Query", "Query", + RRDF_FIELD_TYPE_STRING, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NONE, + 0, NULL, NAN, RRDF_FIELD_SORT_ASCENDING, NULL, + RRDF_FIELD_SUMMARY_COUNT, RRDF_FIELD_FILTER_NONE, + RRDF_FIELD_OPTS_VISIBLE | RRDF_FIELD_OPTS_FULL_WIDTH | RRDF_FIELD_OPTS_WRAP, NULL); + + // client + buffer_rrdf_table_add_field(wb, field_id++, "Client", "Client", + RRDF_FIELD_TYPE_STRING, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NONE, + 0, NULL, NAN, RRDF_FIELD_SORT_ASCENDING, NULL, + RRDF_FIELD_SUMMARY_COUNT, RRDF_FIELD_FILTER_MULTISELECT, + RRDF_FIELD_OPTS_VISIBLE, NULL); + + // status + buffer_rrdf_table_add_field(wb, field_id++, "Status", "Query Status", + RRDF_FIELD_TYPE_STRING, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NONE, + 0, NULL, NAN, RRDF_FIELD_SORT_ASCENDING, NULL, + RRDF_FIELD_SUMMARY_COUNT, RRDF_FIELD_FILTER_MULTISELECT, + RRDF_FIELD_OPTS_VISIBLE, NULL); + + // progress + buffer_rrdf_table_add_field(wb, field_id++, "Progress", "Query Progress", + RRDF_FIELD_TYPE_STRING, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NONE, + 0, NULL, NAN, RRDF_FIELD_SORT_DESCENDING, NULL, + RRDF_FIELD_SUMMARY_COUNT, RRDF_FIELD_FILTER_NONE, + RRDF_FIELD_OPTS_VISIBLE, NULL); + + // duration + buffer_rrdf_table_add_field(wb, field_id++, "Duration", "Query Duration", + RRDF_FIELD_TYPE_DURATION, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NUMBER, + 2, "ms", (double)max_duration_ut / USEC_PER_MS, RRDF_FIELD_SORT_DESCENDING, NULL, + RRDF_FIELD_SUMMARY_MAX, RRDF_FIELD_FILTER_RANGE, + RRDF_FIELD_OPTS_VISIBLE, NULL); + + // response code + buffer_rrdf_table_add_field(wb, field_id++, "Response", "Query Response Code", + RRDF_FIELD_TYPE_INTEGER, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NONE, + 0, NULL, NAN, RRDF_FIELD_SORT_DESCENDING, NULL, + RRDF_FIELD_SUMMARY_COUNT, RRDF_FIELD_FILTER_MULTISELECT, + RRDF_FIELD_OPTS_VISIBLE, NULL); + + // response size + buffer_rrdf_table_add_field(wb, field_id++, "Size", "Query Response Size", + RRDF_FIELD_TYPE_INTEGER, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NONE, + 0, "bytes", (double)max_size, RRDF_FIELD_SORT_DESCENDING, NULL, + RRDF_FIELD_SUMMARY_SUM, RRDF_FIELD_FILTER_RANGE, + RRDF_FIELD_OPTS_NONE, NULL); + + // sent size + buffer_rrdf_table_add_field(wb, field_id++, "Sent", "Query Response Final Size", + RRDF_FIELD_TYPE_INTEGER, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NONE, + 0, "bytes", (double)max_sent, RRDF_FIELD_SORT_DESCENDING, NULL, + RRDF_FIELD_SUMMARY_SUM, RRDF_FIELD_FILTER_RANGE, + RRDF_FIELD_OPTS_NONE, NULL); + + // row options + buffer_rrdf_table_add_field(wb, field_id++, "rowOptions", "rowOptions", + RRDF_FIELD_TYPE_NONE, RRDR_FIELD_VISUAL_ROW_OPTIONS, RRDF_FIELD_TRANSFORM_NONE, + 0, NULL, NAN, RRDF_FIELD_SORT_FIXED, NULL, + RRDF_FIELD_SUMMARY_COUNT, RRDF_FIELD_FILTER_NONE, + RRDF_FIELD_OPTS_DUMMY, NULL); + } + + buffer_json_object_close(wb); // columns + buffer_json_member_add_string(wb, "default_sort_column", "Started"); + + buffer_json_member_add_time_t(wb, "expires", (time_t)((now_ut / USEC_PER_SEC) + 1)); + buffer_json_finalize(wb); + + return 200; +} + + +// ---------------------------------------------------------------------------- + +int progress_unittest(void) { + size_t permanent = 100; + uuid_t valid[permanent]; + + usec_t started = now_monotonic_usec(); + + for(size_t i = 0; i < permanent ;i++) { + uuid_generate_random(valid[i]); + query_progress_start_or_update(&valid[i], 0, HTTP_REQUEST_MODE_GET, HTTP_ACL_ACLK, "permanent", NULL, "test"); + } + + for(size_t n = 0; n < 5000000 ;n++) { + uuid_t t; + uuid_generate_random(t); + query_progress_start_or_update(&t, 0, HTTP_REQUEST_MODE_OPTIONS, HTTP_ACL_WEBRTC, "ephemeral", NULL, "test"); + query_progress_finished(&t, 0, 200, 1234, 123, 12); + + QUERY_PROGRESS *qp; + for(size_t i = 0; i < permanent ;i++) { + qp = query_progress_find_in_hashtable_unsafe(&valid[i]); + assert(qp); + } + } + + usec_t ended = now_monotonic_usec(); + usec_t duration = ended - started; + + printf("progress hashtable resizes: %zu, size: %zu, used: %zu, deleted: %zu, searches: %zu, collisions: %zu, additions: %zu, deletions: %zu\n", + progress.hashtable.resizes, + progress.hashtable.size, progress.hashtable.used, progress.hashtable.deleted, + progress.hashtable.searches, progress.hashtable.collisions, progress.hashtable.additions, progress.hashtable.deletions); + + double d = (double)duration / USEC_PER_SEC; + printf("hashtable ops: %0.2f / sec\n", (double)progress.hashtable.searches / d); + + return 0; +} diff --git a/libnetdata/query_progress/progress.h b/libnetdata/query_progress/progress.h new file mode 100644 index 00000000000000..a31a58be856b02 --- /dev/null +++ b/libnetdata/query_progress/progress.h @@ -0,0 +1,19 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef NETDATA_QUERY_PROGRESS_H +#define NETDATA_QUERY_PROGRESS_H 1 + +#include "../libnetdata.h" + +void query_progress_start_or_update(uuid_t *transaction, usec_t started_ut, HTTP_REQUEST_MODE mode, HTTP_ACL acl, const char *query, const char *payload, const char *client); +void query_progress_done_step(uuid_t *transaction, size_t done); +void query_progress_set_finish_line(uuid_t *transaction, size_t all); +void query_progress_finished(uuid_t *transaction, usec_t finished_ut, short int response_code, usec_t duration_ut, size_t response_size, size_t sent_size); +void query_progress_functions_update(uuid_t *transaction, size_t done, size_t all); + +int web_api_v2_report_progress(uuid_t *transaction, BUFFER *wb); + +#define RRDFUNCTIONS_PROGRESS_HELP "View the progress on the running and latest Netdata API Requests" +int progress_function_result(BUFFER *wb, const char *hostname); + +#endif // NETDATA_QUERY_PROGRESS_H diff --git a/libnetdata/simple_hashtable.h b/libnetdata/simple_hashtable.h index f6b6db9068e09e..f21c01b151aed3 100644 --- a/libnetdata/simple_hashtable.h +++ b/libnetdata/simple_hashtable.h @@ -3,14 +3,47 @@ #ifndef NETDATA_SIMPLE_HASHTABLE_H #define NETDATA_SIMPLE_HASHTABLE_H -#ifndef XXH_INLINE_ALL -#define XXH_INLINE_ALL -#endif -#include "xxhash.h" - typedef uint64_t SIMPLE_HASHTABLE_HASH; #define SIMPLE_HASHTABLE_HASH_SECOND_HASH_SHIFTS 32 +/* + * CONFIGURATION + * + * SIMPLE_HASHTABLE_NAME + * The name of the hashtable - all functions and defines will have this name appended + * Example: #define SIMPLE_HASHTABLE_NAME _FACET_KEY + * + * SIMPLE_HASHTABLE_VALUE_TYPE and SIMPLE_HASHTABLE_KEY_TYPE + * The data types of values and keys - optional - setting them will enable strict type checking by the compiler. + * If undefined, they both default to void. + * + * SIMPLE_HASHTABLE_SORT_FUNCTION + * A function name that accepts 2x values and compares them for sorting (returning -1, 0, 1). + * When set, the hashtable will maintain an always sorted array of the values in the hashtable. + * Do not use this for non-static hashtables. So, if your data is changing all the time, this can make the + * hashtable quite slower (it memmove()s an array of pointers to keep it sorted, on every single change). + * + * SIMPLE_HASHTABLE_VALUE2KEY_FUNCTION and SIMPLE_HASHTABLE_COMPARE_KEYS_FUNCTION + * The hashtable can either compare just hashes (the default), or hashes and keys (when these are set). + * Both need to be set for this feature to be enabled. + * + * - SIMPLE_HASHTABLE_VALUE2KEY_FUNCTION + * The name of a function accepting SIMPLE_HASHTABLE_VALUE_TYPE pointer. + * It should return a pointer to SIMPLE_HASHTABLE_KEY_TYPE. + * This function is called prior to SIMPLE_HASHTABLE_COMPARE_KEYS_FUNCTION to extract the key from a value. + * It is also called during hashtable resize, to rehash all values in the hashtable. + * + * - SIMPLE_HASHTABLE_COMPARE_KEYS_FUNCTION + * The name of a function accepting 2x SIMPLE_HASHTABLE_KEY_TYPE pointers. + * It should return true when the keys match. + * This function is only called when the hashes match, to verify that the keys also match. + * + * SIMPLE_HASHTABLE_SAMPLE_IMPLEMENTATION + * If defined, 3x functions will be injected for easily working with the hashtable. + * + */ + + #ifndef SIMPLE_HASHTABLE_NAME #define SIMPLE_HASHTABLE_NAME #endif @@ -19,6 +52,22 @@ typedef uint64_t SIMPLE_HASHTABLE_HASH; #define SIMPLE_HASHTABLE_VALUE_TYPE void #endif +#ifndef SIMPLE_HASHTABLE_KEY_TYPE +#define SIMPLE_HASHTABLE_KEY_TYPE void +#endif + +#ifndef SIMPLE_HASHTABLE_VALUE2KEY_FUNCTION +#undef SIMPLE_HASHTABLE_COMPARE_KEYS_FUNCTION +#endif + +#if defined(SIMPLE_HASHTABLE_VALUE2KEY_FUNCTION) +static inline SIMPLE_HASHTABLE_KEY_TYPE *SIMPLE_HASHTABLE_VALUE2KEY_FUNCTION(SIMPLE_HASHTABLE_VALUE_TYPE *); +#endif + +#if defined(SIMPLE_HASHTABLE_COMPARE_KEYS_FUNCTION) +static inline bool SIMPLE_HASHTABLE_COMPARE_KEYS_FUNCTION(SIMPLE_HASHTABLE_KEY_TYPE *, SIMPLE_HASHTABLE_KEY_TYPE *); +#endif + // First layer of macro for token concatenation #define CONCAT_INTERNAL(a, b) a ## b // Second layer of macro, which ensures proper expansion @@ -33,6 +82,7 @@ typedef uint64_t SIMPLE_HASHTABLE_HASH; #define simple_hashtable_named CONCAT(simple_hashtable, SIMPLE_HASHTABLE_NAME) #define SIMPLE_HASHTABLE_NAMED CONCAT(SIMPLE_HASHTABLE, SIMPLE_HASHTABLE_NAME) #define simple_hashtable_resize_named CONCAT(simple_hashtable_resize, SIMPLE_HASHTABLE_NAME) +#define simple_hashtable_can_use_slot_named CONCAT(simple_hashtable_keys_match, SIMPLE_HASHTABLE_NAME) #define simple_hashtable_get_slot_named CONCAT(simple_hashtable_get_slot, SIMPLE_HASHTABLE_NAME) #define simple_hashtable_del_slot_named CONCAT(simple_hashtable_del_slot, SIMPLE_HASHTABLE_NAME) #define simple_hashtable_set_slot_named CONCAT(simple_hashtable_set_slot, SIMPLE_HASHTABLE_NAME) @@ -55,10 +105,12 @@ typedef struct simple_hashtable_named { size_t resizes; size_t searches; size_t collisions; + size_t additions; size_t deletions; size_t deleted; size_t used; size_t size; + bool needs_cleanup; SIMPLE_HASHTABLE_SLOT_NAMED *hashtable; #ifdef SIMPLE_HASHTABLE_SORT_FUNCTION @@ -210,19 +262,53 @@ static void simple_hashtable_destroy_named(SIMPLE_HASHTABLE_NAMED *ht) { static inline void simple_hashtable_resize_named(SIMPLE_HASHTABLE_NAMED *ht); -#define SHTS_DATA_UNSET ((void *)NULL) -#define SHTS_DATA_DELETED ((void *)0x01) -#define SHTS_DATA_USERNULL ((void *)0x02) -#define SHTS_IS_UNSET(sl) ((sl)->data == SHTS_DATA_UNSET) -#define SHTS_IS_DELETED(sl) ((sl)->data == SHTS_DATA_DELETED) -#define SHTS_IS_USERNULL(sl) ((sl)->data == SHTS_DATA_USERNULL) -#define SIMPLE_HASHTABLE_SLOT_DATA(sl) ((SHTS_IS_UNSET(sl) || SHTS_IS_DELETED(sl) || SHTS_IS_USERNULL(sl)) ? NULL : (sl)->data) -#define SIMPLE_HASHTABLE_SLOT_UNSET_OR_DELETED(sl) ((SHTS_IS_UNSET(sl) || SHTS_IS_DELETED(sl)) ? NULL : (sl)->data) +#define simple_hashtable_data_unset ((void *)NULL) +#define simple_hashtable_data_deleted ((void *)UINT64_MAX) +#define simple_hashtable_data_usernull ((void *)(UINT64_MAX - 1)) +#define simple_hashtable_is_slot_unset(sl) ((sl)->data == simple_hashtable_data_unset) +#define simple_hashtable_is_slot_deleted(sl) ((sl)->data == simple_hashtable_data_deleted) +#define simple_hashtable_is_slot_usernull(sl) ((sl)->data == simple_hashtable_data_usernull) +#define SIMPLE_HASHTABLE_SLOT_DATA(sl) ((simple_hashtable_is_slot_unset(sl) || simple_hashtable_is_slot_deleted(sl) || simple_hashtable_is_slot_usernull(sl)) ? NULL : (sl)->data) -// IMPORTANT -// The pointer returned by this call is valid up to the next call of this function (or the resize one) +static inline bool simple_hashtable_can_use_slot_named( + SIMPLE_HASHTABLE_SLOT_NAMED *sl, SIMPLE_HASHTABLE_HASH hash, + SIMPLE_HASHTABLE_KEY_TYPE *key __maybe_unused) { + + if(simple_hashtable_is_slot_unset(sl)) + return true; + + if(simple_hashtable_is_slot_deleted(sl)) + return false; + + if(sl->hash == hash) { +#if defined(SIMPLE_HASHTABLE_COMPARE_KEYS_FUNCTION) && defined(SIMPLE_HASHTABLE_VALUE2KEY_FUNCTION) + return SIMPLE_HASHTABLE_COMPARE_KEYS_FUNCTION(SIMPLE_HASHTABLE_VALUE2KEY_FUNCTION(SIMPLE_HASHTABLE_SLOT_DATA(sl)), key); +#else + return true; +#endif + } + + return false; +} + +#define SIMPLE_HASHTABLE_NEEDS_RESIZE(ht) ((ht)->size <= ((ht)->used - (ht)->deleted) << 1 || (ht)->used >= (ht)->size) + +// IMPORTANT: the pointer returned by this call is valid up to the next call of this function (or the resize one). // If you need to cache something, cache the hash, not the slot pointer. -static inline SIMPLE_HASHTABLE_SLOT_NAMED *simple_hashtable_get_slot_named(SIMPLE_HASHTABLE_NAMED *ht, SIMPLE_HASHTABLE_HASH hash, bool resize) { +static inline SIMPLE_HASHTABLE_SLOT_NAMED *simple_hashtable_get_slot_named( + SIMPLE_HASHTABLE_NAMED *ht, SIMPLE_HASHTABLE_HASH hash, + SIMPLE_HASHTABLE_KEY_TYPE *key, bool resize) { + + // This function finds the requested hash and key in the hashtable. + // It uses a second version of the hash in case of collisions, and then linear probing. + // It may resize the hashtable if it is more than 50% full. + + // Deleted items remain in the hashtable, but they are marked as DELETED. + // Reuse of DELETED slots happens only if the slot to be returned is UNSET. + // So, when looking up for an item, it tries to find it, assuming DELETED + // slots are occupied. If the item to be returned is UNSET, and it has + // encountered a DELETED slot, it returns the DELETED one instead of the UNSET. + ht->searches++; size_t slot; @@ -231,41 +317,68 @@ static inline SIMPLE_HASHTABLE_SLOT_NAMED *simple_hashtable_get_slot_named(SIMPL slot = hash % ht->size; sl = &ht->hashtable[slot]; - deleted = SHTS_IS_DELETED(sl) ? sl : NULL; - if(likely(!SIMPLE_HASHTABLE_SLOT_UNSET_OR_DELETED(sl) || sl->hash == hash)) - return (SHTS_IS_UNSET(sl) && deleted) ? deleted : sl; + deleted = simple_hashtable_is_slot_deleted(sl) ? sl : NULL; + if(likely(simple_hashtable_can_use_slot_named(sl, hash, key))) + return (simple_hashtable_is_slot_unset(sl) && deleted) ? deleted : sl; ht->collisions++; - if(unlikely(resize && (ht->size <= (ht->used << 1) || ht->used >= ht->size))) { + if(unlikely(resize && (ht->needs_cleanup || SIMPLE_HASHTABLE_NEEDS_RESIZE(ht)))) { simple_hashtable_resize_named(ht); + deleted = NULL; // our deleted pointer is not valid anymore slot = hash % ht->size; sl = &ht->hashtable[slot]; - deleted = (!deleted && SHTS_IS_DELETED(sl)) ? sl : deleted; - if(likely(!SIMPLE_HASHTABLE_SLOT_UNSET_OR_DELETED(sl) || sl->hash == hash)) - return (SHTS_IS_UNSET(sl) && deleted) ? deleted : sl; + if(likely(simple_hashtable_can_use_slot_named(sl, hash, key))) + return sl; ht->collisions++; } slot = ((hash >> SIMPLE_HASHTABLE_HASH_SECOND_HASH_SHIFTS) + 1) % ht->size; sl = &ht->hashtable[slot]; - deleted = (!deleted && SHTS_IS_DELETED(sl)) ? sl : deleted; + deleted = (!deleted && simple_hashtable_is_slot_deleted(sl)) ? sl : deleted; // Linear probing until we find it - while (SIMPLE_HASHTABLE_SLOT_UNSET_OR_DELETED(sl) && sl->hash != hash) { + SIMPLE_HASHTABLE_SLOT_NAMED *sl_started = sl; + size_t collisions_started = ht->collisions; + while (!simple_hashtable_can_use_slot_named(sl, hash, key)) { slot = (slot + 1) % ht->size; // Wrap around if necessary sl = &ht->hashtable[slot]; - deleted = (!deleted && SHTS_IS_DELETED(sl)) ? sl : deleted; + deleted = (!deleted && simple_hashtable_is_slot_deleted(sl)) ? sl : deleted; ht->collisions++; + + if(sl == sl_started) { + if(deleted) { + // we looped through all items, and we didn't find a free slot, + // but we have found a deleted slot, so return it. + return deleted; + } + else if(resize) { + // the hashtable is full, without any deleted slots. + // we need to resize it now. + simple_hashtable_resize_named(ht); + return simple_hashtable_get_slot_named(ht, hash, key, false); + } + else { + // the hashtable is full, but resize is false. + // this should never happen. + assert(sl != sl_started); + } + } } - return (SHTS_IS_UNSET(sl) && deleted) ? deleted : sl; + if((ht->collisions - collisions_started) > (ht->size / 2) && ht->deleted >= (ht->size / 3)) { + // we traversed through half of the hashtable to find a slot, + // but we have more than 1/3 deleted items + ht->needs_cleanup = true; + } + + return (simple_hashtable_is_slot_unset(sl) && deleted) ? deleted : sl; } static inline bool simple_hashtable_del_slot_named(SIMPLE_HASHTABLE_NAMED *ht, SIMPLE_HASHTABLE_SLOT_NAMED *sl) { - if(SHTS_IS_UNSET(sl) || SHTS_IS_DELETED(sl)) + if(simple_hashtable_is_slot_unset(sl) || simple_hashtable_is_slot_deleted(sl)) return false; ht->deletions++; @@ -273,25 +386,28 @@ static inline bool simple_hashtable_del_slot_named(SIMPLE_HASHTABLE_NAMED *ht, S simple_hashtable_del_value_sorted_named(ht, SIMPLE_HASHTABLE_SLOT_DATA(sl)); - sl->data = SHTS_DATA_DELETED; + sl->data = simple_hashtable_data_deleted; return true; } -static inline void simple_hashtable_set_slot_named(SIMPLE_HASHTABLE_NAMED *ht, SIMPLE_HASHTABLE_SLOT_NAMED *sl, SIMPLE_HASHTABLE_HASH hash, SIMPLE_HASHTABLE_VALUE_TYPE *data) { +static inline void simple_hashtable_set_slot_named( + SIMPLE_HASHTABLE_NAMED *ht, SIMPLE_HASHTABLE_SLOT_NAMED *sl, + SIMPLE_HASHTABLE_HASH hash, SIMPLE_HASHTABLE_VALUE_TYPE *data) { + if(data == NULL) - data = SHTS_DATA_USERNULL; + data = simple_hashtable_data_usernull; - if(unlikely(data == SHTS_DATA_UNSET || data == SHTS_DATA_DELETED)) { + if(unlikely(data == simple_hashtable_data_unset || data == simple_hashtable_data_deleted)) { simple_hashtable_del_slot_named(ht, sl); return; } - if(likely(SHTS_IS_UNSET(sl))) { + if(likely(simple_hashtable_is_slot_unset(sl))) { simple_hashtable_add_value_sorted_named(ht, data); ht->used++; } - else if(unlikely(SHTS_IS_DELETED(sl))) { + else if(unlikely(simple_hashtable_is_slot_deleted(sl))) { ht->deleted--; } @@ -300,6 +416,7 @@ static inline void simple_hashtable_set_slot_named(SIMPLE_HASHTABLE_NAMED *ht, S sl->hash = hash; sl->data = data; + ht->additions++; } // IMPORTANT @@ -308,19 +425,38 @@ static inline void simple_hashtable_resize_named(SIMPLE_HASHTABLE_NAMED *ht) { SIMPLE_HASHTABLE_SLOT_NAMED *old = ht->hashtable; size_t old_size = ht->size; + size_t new_size = ht->size; + + if(SIMPLE_HASHTABLE_NEEDS_RESIZE(ht)) + new_size = (ht->size << 1) - ((ht->size > 16) ? 1 : 0); + ht->resizes++; - ht->size = (ht->size << 1) - ((ht->size > 16) ? 1 : 0); - ht->hashtable = callocz(ht->size, sizeof(*ht->hashtable)); - ht->used = ht->deleted = 0; + ht->size = new_size; + ht->hashtable = callocz(new_size, sizeof(*ht->hashtable)); + size_t used = 0; for(size_t i = 0 ; i < old_size ; i++) { - if(!SIMPLE_HASHTABLE_SLOT_UNSET_OR_DELETED(&old[i])) + SIMPLE_HASHTABLE_SLOT_NAMED *slot = &old[i]; + if(simple_hashtable_is_slot_unset(slot) || simple_hashtable_is_slot_deleted(slot)) continue; - SIMPLE_HASHTABLE_SLOT_NAMED *slot = simple_hashtable_get_slot_named(ht, old[i].hash, false); - *slot = old[i]; - ht->used++; + SIMPLE_HASHTABLE_KEY_TYPE *key = NULL; + +#if defined(SIMPLE_HASHTABLE_COMPARE_KEYS_FUNCTION) && defined(SIMPLE_HASHTABLE_VALUE2KEY_FUNCTION) + SIMPLE_HASHTABLE_VALUE_TYPE *value = SIMPLE_HASHTABLE_SLOT_DATA(slot); + key = SIMPLE_HASHTABLE_VALUE2KEY_FUNCTION(value); +#endif + + SIMPLE_HASHTABLE_SLOT_NAMED *slot2 = simple_hashtable_get_slot_named(ht, slot->hash, key, false); + *slot2 = *slot; + used++; } + assert(used == ht->used - ht->deleted); + + ht->used = used; + ht->deleted = 0; + ht->needs_cleanup = false; + freez(old); } @@ -329,9 +465,9 @@ static inline void simple_hashtable_resize_named(SIMPLE_HASHTABLE_NAMED *ht) { // the hashtable should not be modified while the traversal is taking place static inline SIMPLE_HASHTABLE_SLOT_NAMED *simple_hashtable_first_read_only_named(SIMPLE_HASHTABLE_NAMED *ht) { - for(size_t i = 0; i < ht->used ;i++) { + for(size_t i = 0; i < ht->size ;i++) { SIMPLE_HASHTABLE_SLOT_NAMED *sl = &ht->hashtable[i]; - if(!SIMPLE_HASHTABLE_SLOT_UNSET_OR_DELETED(sl)) + if(!simple_hashtable_is_slot_unset(sl) && !simple_hashtable_is_slot_deleted(sl)) return sl; } @@ -342,12 +478,12 @@ static inline SIMPLE_HASHTABLE_SLOT_NAMED *simple_hashtable_next_read_only_named if (!last) return NULL; // Calculate the current position in the array - size_t currentIndex = last - ht->hashtable; + size_t index = last - ht->hashtable; // Iterate over the hashtable starting from the next element - for (size_t i = currentIndex + 1; i < ht->size; i++) { + for (size_t i = index + 1; i < ht->size; i++) { SIMPLE_HASHTABLE_SLOT_NAMED *sl = &ht->hashtable[i]; - if (!SIMPLE_HASHTABLE_SLOT_UNSET_OR_DELETED(sl)) { + if (!simple_hashtable_is_slot_unset(sl) && !simple_hashtable_is_slot_deleted(sl)) { return sl; } } @@ -368,29 +504,41 @@ static inline SIMPLE_HASHTABLE_SLOT_NAMED *simple_hashtable_next_read_only_named #ifdef SIMPLE_HASHTABLE_SAMPLE_IMPLEMENTATION +#ifndef XXH_INLINE_ALL +#define XXH_INLINE_ALL +#endif +#include "xxhash.h" + #define simple_hashtable_set_named CONCAT(simple_hashtable_set, SIMPLE_HASHTABLE_NAME) #define simple_hashtable_get_named CONCAT(simple_hashtable_get, SIMPLE_HASHTABLE_NAME) #define simple_hashtable_del_named CONCAT(simple_hashtable_del, SIMPLE_HASHTABLE_NAME) -static inline SIMPLE_HASHTABLE_VALUE_TYPE *simple_hashtable_set_named(SIMPLE_HASHTABLE_NAMED *ht, void *key, size_t key_len, SIMPLE_HASHTABLE_VALUE_TYPE *data) { - XXH64_hash_t hash = XXH3_64bits(key, key_len); - SIMPLE_HASHTABLE_SLOT_NAMED *sl = simple_hashtable_get_slot_named(ht, hash, true); +static inline SIMPLE_HASHTABLE_VALUE_TYPE *simple_hashtable_set_named(SIMPLE_HASHTABLE_NAMED *ht, SIMPLE_HASHTABLE_KEY_TYPE *key, size_t key_len, SIMPLE_HASHTABLE_VALUE_TYPE *data) { + XXH64_hash_t hash = XXH3_64bits((void *)key, key_len); + SIMPLE_HASHTABLE_SLOT_NAMED *sl = simple_hashtable_get_slot_named(ht, hash, key, true); simple_hashtable_set_slot_named(ht, sl, hash, data); return SIMPLE_HASHTABLE_SLOT_DATA(sl); } -static inline SIMPLE_HASHTABLE_VALUE_TYPE *simple_hashtable_get_named(SIMPLE_HASHTABLE_NAMED *ht, void *key, size_t key_len, SIMPLE_HASHTABLE_VALUE_TYPE *data) { - XXH64_hash_t hash = XXH3_64bits(key, key_len); - SIMPLE_HASHTABLE_SLOT_NAMED *sl = simple_hashtable_get_slot_named(ht, hash, true); +static inline SIMPLE_HASHTABLE_VALUE_TYPE *simple_hashtable_get_named(SIMPLE_HASHTABLE_NAMED *ht, SIMPLE_HASHTABLE_KEY_TYPE *key, size_t key_len, SIMPLE_HASHTABLE_VALUE_TYPE *data) { + XXH64_hash_t hash = XXH3_64bits((void *)key, key_len); + SIMPLE_HASHTABLE_SLOT_NAMED *sl = simple_hashtable_get_slot_named(ht, hash, key, true); return SIMPLE_HASHTABLE_SLOT_DATA(sl); } -static inline bool simple_hashtable_del_named(SIMPLE_HASHTABLE_NAMED *ht, void *key, size_t key_len, SIMPLE_HASHTABLE_VALUE_TYPE *data) { - XXH64_hash_t hash = XXH3_64bits(key, key_len); - SIMPLE_HASHTABLE_SLOT_NAMED *sl = simple_hashtable_get_slot_named(ht, hash, true); +static inline bool simple_hashtable_del_named(SIMPLE_HASHTABLE_NAMED *ht, SIMPLE_HASHTABLE_KEY_TYPE *key, size_t key_len, SIMPLE_HASHTABLE_VALUE_TYPE *data) { + XXH64_hash_t hash = XXH3_64bits((void *)key, key_len); + SIMPLE_HASHTABLE_SLOT_NAMED *sl = simple_hashtable_get_slot_named(ht, hash, key, true); return simple_hashtable_del_slot_named(ht, sl); } #endif // SIMPLE_HASHTABLE_SAMPLE_IMPLEMENTATION +// ---------------------------------------------------------------------------- +// Clear the preprocessor defines of simple_hashtable.h +// allowing simple_hashtable.h to be included multiple times +// with different configuration each time. + +#include "simple_hashtable_undef.h" + #endif //NETDATA_SIMPLE_HASHTABLE_H diff --git a/libnetdata/simple_hashtable_undef.h b/libnetdata/simple_hashtable_undef.h new file mode 100644 index 00000000000000..3fe5a708dfc97f --- /dev/null +++ b/libnetdata/simple_hashtable_undef.h @@ -0,0 +1,35 @@ + +// this file clears the preprocessor defines of simple_hashtable.h +// allowing simple_hashtable.h to be included multiple times +// with different configuration each time. + +#undef SIMPLE_HASHTABLE_HASH_SECOND_HASH_SHIFTS + +#undef simple_hashtable_init_named +#undef simple_hashtable_destroy_named +#undef simple_hashtable_slot_named +#undef SIMPLE_HASHTABLE_SLOT_NAMED +#undef simple_hashtable_named +#undef SIMPLE_HASHTABLE_NAMED +#undef simple_hashtable_resize_named +#undef simple_hashtable_can_use_slot_named +#undef simple_hashtable_get_slot_named +#undef simple_hashtable_del_slot_named +#undef simple_hashtable_set_slot_named +#undef simple_hashtable_first_read_only_named +#undef simple_hashtable_next_read_only_named +#undef simple_hashtable_sorted_binary_search_named +#undef simple_hashtable_add_value_sorted_named +#undef simple_hashtable_del_value_sorted_named +#undef simple_hashtable_replace_value_sorted_named +#undef simple_hashtable_sorted_array_first_read_only_named +#undef simple_hashtable_sorted_array_next_read_only_named + +#undef SIMPLE_HASHTABLE_SAMPLE_IMPLEMENTATION +#undef SIMPLE_HASHTABLE_SORT_FUNCTION +#undef SIMPLE_HASHTABLE_VALUE_TYPE +#undef SIMPLE_HASHTABLE_KEY_TYPE +#undef SIMPLE_HASHTABLE_VALUE2KEY_FUNCTION +#undef SIMPLE_HASHTABLE_COMPARE_KEYS_FUNCTION +#undef SIMPLE_HASHTABLE_NAME +#undef NETDATA_SIMPLE_HASHTABLE_H diff --git a/libnetdata/socket/socket.c b/libnetdata/socket/socket.c index ceeb8e60ee10ae..703ab8b383d70d 100644 --- a/libnetdata/socket/socket.c +++ b/libnetdata/socket/socket.c @@ -500,7 +500,7 @@ void listen_sockets_close(LISTEN_SOCKETS *sockets) { * * @param acl is the acl given by the user. */ -WEB_CLIENT_ACL socket_ssl_acl(char *acl) { +HTTP_ACL socket_ssl_acl(char *acl) { char *ssl = strchr(acl,'^'); if(ssl) { //Due the format of the SSL command it is always the last command, @@ -511,34 +511,34 @@ WEB_CLIENT_ACL socket_ssl_acl(char *acl) { if (!strncmp("SSL=",ssl,4)) { ssl += 4; if (!strcmp(ssl,"optional")) { - return WEB_CLIENT_ACL_SSL_OPTIONAL; + return HTTP_ACL_SSL_OPTIONAL; } else if (!strcmp(ssl,"force")) { - return WEB_CLIENT_ACL_SSL_FORCE; + return HTTP_ACL_SSL_FORCE; } } #endif } - return WEB_CLIENT_ACL_NONE; + return HTTP_ACL_NONE; } -WEB_CLIENT_ACL read_acl(char *st) { - WEB_CLIENT_ACL ret = socket_ssl_acl(st); +HTTP_ACL read_acl(char *st) { + HTTP_ACL ret = socket_ssl_acl(st); - if (!strcmp(st,"dashboard")) ret |= WEB_CLIENT_ACL_DASHBOARD; - if (!strcmp(st,"registry")) ret |= WEB_CLIENT_ACL_REGISTRY; - if (!strcmp(st,"badges")) ret |= WEB_CLIENT_ACL_BADGE; - if (!strcmp(st,"management")) ret |= WEB_CLIENT_ACL_MGMT; - if (!strcmp(st,"streaming")) ret |= WEB_CLIENT_ACL_STREAMING; - if (!strcmp(st,"netdata.conf")) ret |= WEB_CLIENT_ACL_NETDATACONF; + if (!strcmp(st,"dashboard")) ret |= HTTP_ACL_DASHBOARD; + if (!strcmp(st,"registry")) ret |= HTTP_ACL_REGISTRY; + if (!strcmp(st,"badges")) ret |= HTTP_ACL_BADGE; + if (!strcmp(st,"management")) ret |= HTTP_ACL_MGMT; + if (!strcmp(st,"streaming")) ret |= HTTP_ACL_STREAMING; + if (!strcmp(st,"netdata.conf")) ret |= HTTP_ACL_NETDATACONF; return ret; } static inline int bind_to_this(LISTEN_SOCKETS *sockets, const char *definition, uint16_t default_port, int listen_backlog) { int added = 0; - WEB_CLIENT_ACL acl_flags = WEB_CLIENT_ACL_NONE; + HTTP_ACL acl_flags = HTTP_ACL_NONE; struct addrinfo hints; struct addrinfo *result = NULL, *rp = NULL; @@ -578,7 +578,7 @@ static inline int bind_to_this(LISTEN_SOCKETS *sockets, const char *definition, sockets->failed++; } else { - acl_flags = WEB_CLIENT_ACL_DASHBOARD | WEB_CLIENT_ACL_REGISTRY | WEB_CLIENT_ACL_BADGE | WEB_CLIENT_ACL_MGMT | WEB_CLIENT_ACL_NETDATACONF | WEB_CLIENT_ACL_STREAMING | WEB_CLIENT_ACL_SSL_DEFAULT; + acl_flags = HTTP_ACL_DASHBOARD | HTTP_ACL_REGISTRY | HTTP_ACL_BADGE | HTTP_ACL_MGMT | HTTP_ACL_NETDATACONF | HTTP_ACL_STREAMING | HTTP_ACL_SSL_DEFAULT; listen_sockets_add(sockets, fd, AF_UNIX, socktype, protocol_str, path, 0, acl_flags); added++; } @@ -628,13 +628,13 @@ static inline int bind_to_this(LISTEN_SOCKETS *sockets, const char *definition, } acl_flags |= read_acl(portconfig); } else { - acl_flags = WEB_CLIENT_ACL_DASHBOARD | WEB_CLIENT_ACL_REGISTRY | WEB_CLIENT_ACL_BADGE | WEB_CLIENT_ACL_MGMT | WEB_CLIENT_ACL_NETDATACONF | WEB_CLIENT_ACL_STREAMING | WEB_CLIENT_ACL_SSL_DEFAULT; + acl_flags = HTTP_ACL_DASHBOARD | HTTP_ACL_REGISTRY | HTTP_ACL_BADGE | HTTP_ACL_MGMT | HTTP_ACL_NETDATACONF | HTTP_ACL_STREAMING | HTTP_ACL_SSL_DEFAULT; } //Case the user does not set the option SSL in the "bind to", but he has //the certificates, I must redirect, so I am assuming here the default option - if(!(acl_flags & WEB_CLIENT_ACL_SSL_OPTIONAL) && !(acl_flags & WEB_CLIENT_ACL_SSL_FORCE)) { - acl_flags |= WEB_CLIENT_ACL_SSL_DEFAULT; + if(!(acl_flags & HTTP_ACL_SSL_OPTIONAL) && !(acl_flags & HTTP_ACL_SSL_FORCE)) { + acl_flags |= HTTP_ACL_SSL_DEFAULT; } uint32_t scope_id = 0; @@ -1463,7 +1463,7 @@ int accept_socket(int fd, int flags, char *client_ip, size_t ipsize, char *clien inline POLLINFO *poll_add_fd(POLLJOB *p , int fd , int socktype - , WEB_CLIENT_ACL port_acl + , HTTP_ACL port_acl , uint32_t flags , const char *client_ip , const char *client_port diff --git a/libnetdata/socket/socket.h b/libnetdata/socket/socket.h index e4ca08d47fede3..debbd766d973ca 100644 --- a/libnetdata/socket/socket.h +++ b/libnetdata/socket/socket.h @@ -9,45 +9,6 @@ #define MAX_LISTEN_FDS 50 #endif -typedef enum web_client_acl { - WEB_CLIENT_ACL_NONE = (0), - WEB_CLIENT_ACL_NOCHECK = (1 << 0), // Don't check anything - this should work on all channels - WEB_CLIENT_ACL_DASHBOARD = (1 << 1), - WEB_CLIENT_ACL_REGISTRY = (1 << 2), - WEB_CLIENT_ACL_BADGE = (1 << 3), - WEB_CLIENT_ACL_MGMT = (1 << 4), - WEB_CLIENT_ACL_STREAMING = (1 << 5), - WEB_CLIENT_ACL_NETDATACONF = (1 << 6), - WEB_CLIENT_ACL_SSL_OPTIONAL = (1 << 7), - WEB_CLIENT_ACL_SSL_FORCE = (1 << 8), - WEB_CLIENT_ACL_SSL_DEFAULT = (1 << 9), - WEB_CLIENT_ACL_ACLK = (1 << 10), - WEB_CLIENT_ACL_WEBRTC = (1 << 11), - WEB_CLIENT_ACL_BEARER_OPTIONAL = (1 << 12), // allow unprotected access if bearer is not enabled in netdata - WEB_CLIENT_ACL_BEARER_REQUIRED = (1 << 13), // allow access only if a valid bearer is used -} WEB_CLIENT_ACL; - -#define WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC (WEB_CLIENT_ACL_DASHBOARD | WEB_CLIENT_ACL_ACLK | WEB_CLIENT_ACL_WEBRTC | WEB_CLIENT_ACL_BEARER_OPTIONAL) -#define WEB_CLIENT_ACL_ACLK_WEBRTC_DASHBOARD_WITH_BEARER (WEB_CLIENT_ACL_DASHBOARD | WEB_CLIENT_ACL_ACLK | WEB_CLIENT_ACL_WEBRTC | WEB_CLIENT_ACL_BEARER_REQUIRED) - -#ifdef NETDATA_DEV_MODE -#define ACL_DEV_OPEN_ACCESS WEB_CLIENT_ACL_NOCHECK -#else -#define ACL_DEV_OPEN_ACCESS 0 -#endif - -#define WEB_CLIENT_ACL_ALL 0xFFFF - -#define web_client_can_access_dashboard(w) ((w)->acl & WEB_CLIENT_ACL_DASHBOARD) -#define web_client_can_access_registry(w) ((w)->acl & WEB_CLIENT_ACL_REGISTRY) -#define web_client_can_access_badges(w) ((w)->acl & WEB_CLIENT_ACL_BADGE) -#define web_client_can_access_mgmt(w) ((w)->acl & WEB_CLIENT_ACL_MGMT) -#define web_client_can_access_stream(w) ((w)->acl & WEB_CLIENT_ACL_STREAMING) -#define web_client_can_access_netdataconf(w) ((w)->acl & WEB_CLIENT_ACL_NETDATACONF) -#define web_client_is_using_ssl_optional(w) ((w)->port_acl & WEB_CLIENT_ACL_SSL_OPTIONAL) -#define web_client_is_using_ssl_force(w) ((w)->port_acl & WEB_CLIENT_ACL_SSL_FORCE) -#define web_client_is_using_ssl_default(w) ((w)->port_acl & WEB_CLIENT_ACL_SSL_DEFAULT) - typedef struct listen_sockets { struct config *config; // the config file to use const char *config_section; // the netdata configuration section to read settings from @@ -61,7 +22,7 @@ typedef struct listen_sockets { char *fds_names[MAX_LISTEN_FDS]; // descriptions for the open sockets int fds_types[MAX_LISTEN_FDS]; // the socktype for the open sockets (SOCK_STREAM, SOCK_DGRAM) int fds_families[MAX_LISTEN_FDS]; // the family of the open sockets (AF_UNIX, AF_INET, AF_INET6) - WEB_CLIENT_ACL fds_acl_flags[MAX_LISTEN_FDS]; // the acl to apply to the open sockets (dashboard, badges, streaming, netdata.conf, management) + HTTP_ACL fds_acl_flags[MAX_LISTEN_FDS]; // the acl to apply to the open sockets (dashboard, badges, streaming, netdata.conf, management) } LISTEN_SOCKETS; char *strdup_client_description(int family, const char *protocol, const char *ip, uint16_t port); @@ -128,7 +89,7 @@ typedef struct pollinfo { int fd; // the file descriptor int socktype; // the client socket type - WEB_CLIENT_ACL port_acl; // the access lists permitted on this web server port (it's -1 for client sockets) + HTTP_ACL port_acl; // the access lists permitted on this web server port (it's -1 for client sockets) char *client_ip; // Max INET6_ADDRSTRLEN bytes char *client_port; // Max NI_MAXSERV bytes char *client_host; // Max NI_MAXHOST bytes @@ -196,7 +157,7 @@ void *poll_default_add_callback(POLLINFO *pi, short int *events, void *data); POLLINFO *poll_add_fd(POLLJOB *p , int fd , int socktype - , WEB_CLIENT_ACL port_acl + , HTTP_ACL port_acl , uint32_t flags , const char *client_ip , const char *client_port diff --git a/logsmanagement/functions.c b/logsmanagement/functions.c index d53c3ed7fb9fd5..8031687b9d31e7 100644 --- a/logsmanagement/functions.c +++ b/logsmanagement/functions.c @@ -25,8 +25,6 @@ #define LOGS_MANAG_FUNC_PARAM_DATA_ONLY "data_only" #define LOGS_MANAG_FUNC_PARAM_SOURCE "source" #define LOGS_MANAG_FUNC_PARAM_INFO "info" -#define LOGS_MANAG_FUNC_PARAM_ID "id" -#define LOGS_MANAG_FUNC_PARAM_PROGRESS "progress" #define LOGS_MANAG_FUNC_PARAM_SLICE "slice" #define LOGS_MANAG_FUNC_PARAM_DELTA "delta" #define LOGS_MANAG_FUNC_PARAM_TAIL "tail" @@ -116,9 +114,7 @@ static DICTIONARY *used_hashes_registry = NULL; typedef struct function_query_status { bool *cancelled; // a pointer to the cancelling boolean - usec_t stop_monotonic_ut; - - usec_t started_monotonic_ut; + usec_t *stop_monotonic_ut; // request STRING *source; @@ -164,7 +160,7 @@ typedef struct function_query_status { "|message" \ "" -static void logsmanagement_function_facets(const char *transaction, char *function, int timeout, bool *cancelled){ +static void logsmanagement_function_facets(const char *transaction, char *function, usec_t *stop_monotonic_ut, bool *cancelled){ struct rusage start, end; getrusage(RUSAGE_THREAD, &start); @@ -175,11 +171,9 @@ static void logsmanagement_function_facets(const char *transaction, char *functi buffer_flush(wb); buffer_json_initialize(wb, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_MINIFY); - usec_t now_monotonic_ut = now_monotonic_usec(); FUNCTION_QUERY_STATUS tmp_fqs = { .cancelled = cancelled, - .started_monotonic_ut = now_monotonic_ut, - .stop_monotonic_ut = now_monotonic_ut + (timeout * USEC_PER_SEC), + .stop_monotonic_ut = stop_monotonic_ut, }; FUNCTION_QUERY_STATUS *fqs = NULL; const DICTIONARY_ITEM *fqs_item = NULL; @@ -201,8 +195,6 @@ static void logsmanagement_function_facets(const char *transaction, char *functi facets_accepted_param(facets, LOGS_MANAG_FUNC_PARAM_HISTOGRAM); facets_accepted_param(facets, LOGS_MANAG_FUNC_PARAM_IF_MODIFIED_SINCE); facets_accepted_param(facets, LOGS_MANAG_FUNC_PARAM_DATA_ONLY); - // facets_accepted_param(facets, LOGS_MANAG_FUNC_PARAM_ID); - // facets_accepted_param(facets, LOGS_MANAG_FUNC_PARAM_PROGRESS); facets_accepted_param(facets, LOGS_MANAG_FUNC_PARAM_DELTA); // facets_accepted_param(facets, JOURNAL_PARAMETER_TAIL); @@ -235,7 +227,6 @@ static void logsmanagement_function_facets(const char *transaction, char *functi bool info = false, data_only = false, - progress = false, /* slice = true, */ delta = false, tail = false; @@ -247,8 +238,7 @@ static void logsmanagement_function_facets(const char *transaction, char *functi const char *query = NULL; const char *chart = NULL; const char *source = NULL; - const char *progress_id = NULL; - // size_t filters = 0; + // size_t filters = 0; buffer_json_member_add_object(wb, "_request"); @@ -265,20 +255,17 @@ static void logsmanagement_function_facets(const char *transaction, char *functi if(!strcmp(keyword, LOGS_MANAG_FUNC_PARAM_HELP)){ - BUFFER *wb = buffer_create(0, NULL); - buffer_sprintf(wb, FUNCTION_LOGSMANAGEMENT_HELP_LONG); + BUFFER *tmp = buffer_create(0, NULL); + buffer_sprintf(tmp, FUNCTION_LOGSMANAGEMENT_HELP_LONG); netdata_mutex_lock(&stdout_mut); - pluginsd_function_result_to_stdout(transaction, HTTP_RESP_OK, "text/plain", now_realtime_sec() + 3600, wb); + pluginsd_function_result_to_stdout(transaction, HTTP_RESP_OK, "text/plain", now_realtime_sec() + 3600, tmp); netdata_mutex_unlock(&stdout_mut); - buffer_free(wb); + buffer_free(tmp); goto cleanup; } else if(!strcmp(keyword, LOGS_MANAG_FUNC_PARAM_INFO)){ info = true; } - else if(!strcmp(keyword, LOGS_MANAG_FUNC_PARAM_PROGRESS)){ - progress = true; - } else if(strncmp(keyword, LOGS_MANAG_FUNC_PARAM_DELTA ":", sizeof(LOGS_MANAG_FUNC_PARAM_DELTA ":") - 1) == 0) { char *v = &keyword[sizeof(LOGS_MANAG_FUNC_PARAM_DELTA ":") - 1]; @@ -314,13 +301,6 @@ static void logsmanagement_function_facets(const char *transaction, char *functi // else // slice = true; // } - else if(strncmp(keyword, LOGS_MANAG_FUNC_PARAM_ID ":", sizeof(LOGS_MANAG_FUNC_PARAM_ID ":") - 1) == 0) { - char *id = &keyword[sizeof(LOGS_MANAG_FUNC_PARAM_ID ":") - 1]; - - if(*id) - progress_id = id; - } - else if(strncmp(keyword, LOGS_MANAG_FUNC_PARAM_SOURCE ":", sizeof(LOGS_MANAG_FUNC_PARAM_SOURCE ":") - 1) == 0) { source = !strcmp("all", &keyword[sizeof(LOGS_MANAG_FUNC_PARAM_SOURCE ":") - 1]) ? NULL : &keyword[sizeof(LOGS_MANAG_FUNC_PARAM_SOURCE ":") - 1]; @@ -393,18 +373,8 @@ static void logsmanagement_function_facets(const char *transaction, char *functi } } - // ------------------------------------------------------------------------ - // put this request into the progress db - - if(progress_id && *progress_id) { - fqs_item = dictionary_set_and_acquire_item(function_query_status_dict, progress_id, &tmp_fqs, sizeof(tmp_fqs)); - fqs = dictionary_acquired_item_value(fqs_item); - } - else { - // no progress id given, proceed without registering our progress in the dictionary - fqs = &tmp_fqs; - fqs_item = NULL; - } + fqs = &tmp_fqs; + fqs_item = NULL; // ------------------------------------------------------------------------ // validate parameters @@ -508,10 +478,8 @@ static void logsmanagement_function_facets(const char *transaction, char *functi buffer_json_member_add_boolean(wb, LOGS_MANAG_FUNC_PARAM_INFO, false); buffer_json_member_add_boolean(wb, LOGS_MANAG_FUNC_PARAM_SLICE, fqs->slice); buffer_json_member_add_boolean(wb, LOGS_MANAG_FUNC_PARAM_DATA_ONLY, fqs->data_only); - buffer_json_member_add_boolean(wb, LOGS_MANAG_FUNC_PARAM_PROGRESS, false); buffer_json_member_add_boolean(wb, LOGS_MANAG_FUNC_PARAM_DELTA, fqs->delta); buffer_json_member_add_boolean(wb, LOGS_MANAG_FUNC_PARAM_TAIL, fqs->tail); - buffer_json_member_add_string(wb, LOGS_MANAG_FUNC_PARAM_ID, progress_id); buffer_json_member_add_string(wb, LOGS_MANAG_FUNC_PARAM_SOURCE, string2str(fqs->source)); buffer_json_member_add_uint64(wb, LOGS_MANAG_FUNC_PARAM_AFTER, fqs->after_ut / USEC_PER_SEC); buffer_json_member_add_uint64(wb, LOGS_MANAG_FUNC_PARAM_BEFORE, fqs->before_ut / USEC_PER_SEC); @@ -556,13 +524,7 @@ static void logsmanagement_function_facets(const char *transaction, char *functi goto output; } - if(progress) { - // TODO: Add progress function - // function_logsmanagement_progress(wb, transaction, progress_id); - goto cleanup; - } - - if(!req_quota) + if(!req_quota) query_params.quota = LOGS_MANAG_QUERY_QUOTA_DEFAULT; else if(req_quota > LOGS_MANAG_QUERY_QUOTA_MAX) query_params.quota = LOGS_MANAG_QUERY_QUOTA_MAX; @@ -590,7 +552,7 @@ static void logsmanagement_function_facets(const char *transaction, char *functi } query_params.cancelled = cancelled; - query_params.stop_monotonic_ut = now_monotonic_usec() + (timeout - 1) * USEC_PER_SEC; + query_params.stop_monotonic_ut = stop_monotonic_ut; query_params.results_buff = buffer_create(query_params.quota, NULL); facets_rows_begin(facets); @@ -736,10 +698,11 @@ struct functions_evloop_globals *logsmanagement_func_facets_init(bool *p_logsman used_hashes_registry = dictionary_create(DICT_OPTION_DONT_OVERWRITE_VALUE); netdata_mutex_lock(&stdout_mut); - fprintf(stdout, PLUGINSD_KEYWORD_FUNCTION " GLOBAL \"%s\" %d \"%s\"\n", + fprintf(stdout, PLUGINSD_KEYWORD_FUNCTION " GLOBAL \"%s\" %d \"%s\" \"logs\" \"members\" %d\n", LOGS_MANAG_FUNC_NAME, LOGS_MANAG_QUERY_TIMEOUT_DEFAULT, - FUNCTION_LOGSMANAGEMENT_HELP_SHORT); + FUNCTION_LOGSMANAGEMENT_HELP_SHORT, + RRDFUNCTIONS_PRIORITY_DEFAULT + 1); netdata_mutex_unlock(&stdout_mut); struct functions_evloop_globals *wg = functions_evloop_init(1, "LGSMNGM", diff --git a/logsmanagement/query.c b/logsmanagement/query.c index a94c9f704c44fb..77b066eab7f237 100644 --- a/logsmanagement/query.c +++ b/logsmanagement/query.c @@ -6,7 +6,9 @@ * logs management querying API. */ +#ifndef _GNU_SOURCE #define _GNU_SOURCE +#endif #include "query.h" #include @@ -107,7 +109,7 @@ bool terminate_logs_manag_query(logs_query_params_t *const p_query_params){ return true; } - if(now_monotonic_usec() > p_query_params->stop_monotonic_ut) + if(now_monotonic_usec() > __atomic_load_n(p_query_params->stop_monotonic_ut, __ATOMIC_RELAXED)) return true; return false; @@ -173,9 +175,6 @@ const logs_qry_res_err_t *execute_logs_manag_query(logs_query_params_t *p_query_ p_query_params->keyword = sanitise_string(p_query_params->keyword); // freez(p_query_params->keyword) in this case } - if(p_query_params->stop_monotonic_ut == 0) - p_query_params->stop_monotonic_ut = now_monotonic_usec() + (LOGS_MANAG_QUERY_TIMEOUT_DEFAULT - 1) * USEC_PER_SEC; - struct rusage ru_start, ru_end; getrusage(RUSAGE_THREAD, &ru_start); diff --git a/logsmanagement/query.h b/logsmanagement/query.h index 0576f86e3fe4df..a9da4368abbeda 100644 --- a/logsmanagement/query.h +++ b/logsmanagement/query.h @@ -113,7 +113,7 @@ typedef struct logs_query_params { int order_by_asc; unsigned long quota; bool *cancelled; - usec_t stop_monotonic_ut; + usec_t *stop_monotonic_ut; char *chartname[LOGS_MANAG_MAX_COMPOUND_QUERY_SOURCES]; char *filename[LOGS_MANAG_MAX_COMPOUND_QUERY_SOURCES]; char *keyword; diff --git a/logsmanagement/unit_test/unit_test.c b/logsmanagement/unit_test/unit_test.c index 9ee50458c441ce..6c1d76e59e459b 100644 --- a/logsmanagement/unit_test/unit_test.c +++ b/logsmanagement/unit_test/unit_test.c @@ -7,7 +7,11 @@ #include "unit_test.h" #include #include + +#ifndef __USE_XOPEN_EXTENDED #define __USE_XOPEN_EXTENDED +#endif + #include #include #include "../circular_buffer.h" diff --git a/streaming/rrdpush.c b/streaming/rrdpush.c index 7bee57dbd1a9f2..e74c06ef49eef5 100644 --- a/streaming/rrdpush.c +++ b/streaming/rrdpush.c @@ -208,14 +208,14 @@ int configured_as_parent() { // chart labels static int send_clabels_callback(const char *name, const char *value, RRDLABEL_SRC ls, void *data) { BUFFER *wb = (BUFFER *)data; - buffer_sprintf(wb, "CLABEL \"%s\" \"%s\" %d\n", name, value, ls & ~(RRDLABEL_FLAG_INTERNAL)); + buffer_sprintf(wb, PLUGINSD_KEYWORD_CLABEL " \"%s\" \"%s\" %d\n", name, value, ls & ~(RRDLABEL_FLAG_INTERNAL)); return 1; } static void rrdpush_send_clabels(BUFFER *wb, RRDSET *st) { if (st->rrdlabels) { if(rrdlabels_walkthrough_read(st->rrdlabels, send_clabels_callback, wb) > 0) - buffer_sprintf(wb, "CLABEL_COMMIT\n"); + buffer_sprintf(wb, PLUGINSD_KEYWORD_CLABEL_COMMIT "\n"); } } @@ -1410,6 +1410,7 @@ static struct { {STREAM_CAP_ZSTD, "ZSTD" }, {STREAM_CAP_GZIP, "GZIP" }, {STREAM_CAP_BROTLI, "BROTLI" }, + {STREAM_CAP_PROGRESS, "PROGRESS" }, {0 , NULL }, }; @@ -1485,7 +1486,7 @@ STREAM_CAPABILITIES stream_our_capabilities(RRDHOST *host, bool sender) { STREAM_CAP_REPLICATION | STREAM_CAP_BINARY | STREAM_CAP_INTERPOLATED | - STREAM_CAP_SLOTS | + STREAM_CAP_SLOTS | STREAM_CAP_PROGRESS | STREAM_CAP_COMPRESSIONS_AVAILABLE | #ifdef NETDATA_TEST_DYNCFG STREAM_CAP_DYNCFG | diff --git a/streaming/rrdpush.h b/streaming/rrdpush.h index 1459c881e9ee75..d2930edb050b6c 100644 --- a/streaming/rrdpush.h +++ b/streaming/rrdpush.h @@ -52,6 +52,7 @@ typedef enum { STREAM_CAP_ZSTD = (1 << 19), // ZSTD compression supported STREAM_CAP_GZIP = (1 << 20), // GZIP compression supported STREAM_CAP_BROTLI = (1 << 21), // BROTLI compression supported + STREAM_CAP_PROGRESS = (1 << 22), // Functions PROGRESS support STREAM_CAP_INVALID = (1 << 30), // used as an invalid value for capabilities when this is set // this must be signed int, so don't use the last bit diff --git a/streaming/sender.c b/streaming/sender.c index 09b67e968cede8..987dd653753c1c 100644 --- a/streaming/sender.c +++ b/streaming/sender.c @@ -1118,9 +1118,8 @@ struct inflight_stream_function { usec_t received_ut; }; -void stream_execute_function_callback(BUFFER *func_wb, int code, void *data) { +static void stream_execute_function_callback(BUFFER *func_wb, int code, void *data) { struct inflight_stream_function *tmp = data; - struct sender_state *s = tmp->sender; if(rrdhost_can_send_definitions_to_parent(s->host)) { @@ -1150,6 +1149,20 @@ void stream_execute_function_callback(BUFFER *func_wb, int code, void *data) { freez(tmp); } +static void stream_execute_function_progress_callback(void *data, size_t done, size_t all) { + struct inflight_stream_function *tmp = data; + struct sender_state *s = tmp->sender; + + if(rrdhost_can_send_definitions_to_parent(s->host)) { + BUFFER *wb = sender_start(s); + + buffer_sprintf(wb, PLUGINSD_KEYWORD_FUNCTION_PROGRESS " '%s' %zu %zu\n", + string2str(tmp->transaction), done, all); + + sender_commit(s, wb, STREAM_TRAFFIC_TYPE_FUNCTIONS); + } +} + // This is just a placeholder until the gap filling state machine is inserted void execute_commands(struct sender_state *s) { worker_is_busy(WORKER_SENDER_JOB_EXECUTE); @@ -1204,8 +1217,12 @@ void execute_commands(struct sender_state *s) { BUFFER *wb = buffer_create(PLUGINSD_LINE_MAX + 1, &netdata_buffers_statistics.buffers_functions); char *payload = s->receiving_function_payload ? (char *)buffer_tostring(s->function_payload.payload) : NULL; - int code = rrd_function_run(s->host, wb, timeout, function, false, transaction, - stream_execute_function_callback, tmp, NULL, NULL, payload); + int code = rrd_function_run(s->host, wb, + timeout, HTTP_ACCESS_ADMINS, function, false, transaction, + stream_execute_function_callback, tmp, + stream_has_capability(s, STREAM_CAP_PROGRESS) ? stream_execute_function_progress_callback : NULL, + stream_has_capability(s, STREAM_CAP_PROGRESS) ? tmp : NULL, + NULL, NULL, payload); if(code != HTTP_RESP_OK) { if (!buffer_strlen(wb)) @@ -1267,6 +1284,14 @@ void execute_commands(struct sender_state *s) { if(transaction && *transaction) rrd_function_cancel(transaction); } + else if(command && strcmp(command, PLUGINSD_KEYWORD_FUNCTION_PROGRESS) == 0) { + worker_is_busy(WORKER_SENDER_JOB_FUNCTION_REQUEST); + nd_log(NDLS_ACCESS, NDLP_DEBUG, NULL); + + char *transaction = get_word(s->line.words, s->line.num_words, 1); + if(transaction && *transaction) + rrd_function_progress(transaction); + } else if (command && strcmp(command, PLUGINSD_KEYWORD_REPLAY_CHART) == 0) { worker_is_busy(WORKER_SENDER_JOB_REPLAY_REQUEST); nd_log(NDLS_ACCESS, NDLP_DEBUG, NULL); diff --git a/web/api/queries/query.c b/web/api/queries/query.c index 27e3b76c025e7c..45eb87bc86d4ac 100644 --- a/web/api/queries/query.c +++ b/web/api/queries/query.c @@ -3468,6 +3468,8 @@ RRDR *rrd2rrdr(ONEWAYALLOC *owa, QUERY_TARGET *qt) { internal_fatal(released_ops, "QUERY: released_ops should be NULL when the query starts"); + query_progress_set_finish_line(qt->request.transaction, qt->query.used); + QUERY_ENGINE_OPS **ops = NULL; if(qt->query.used) ops = onewayalloc_callocz(owa, qt->query.used, sizeof(QUERY_ENGINE_OPS *)); @@ -3639,6 +3641,8 @@ RRDR *rrd2rrdr(ONEWAYALLOC *owa, QUERY_TARGET *qt) { break; } + else + query_progress_done_step(qt->request.transaction, 1); } // free all resources used by the grouping method diff --git a/web/api/queries/weights.c b/web/api/queries/weights.c index 68af2250f9ec4c..451875a5575a28 100644 --- a/web/api/queries/weights.c +++ b/web/api/queries/weights.c @@ -1717,6 +1717,8 @@ static ssize_t weights_for_rrdmetric(void *data, RRDHOST *host, RRDCONTEXT_ACQUI return -1; } + query_progress_done_step(qwr->transaction, 1); + return 1; } diff --git a/web/api/queries/weights.h b/web/api/queries/weights.h index 66bea6ab2f8575..a93519b6fed8a9 100644 --- a/web/api/queries/weights.h +++ b/web/api/queries/weights.h @@ -57,6 +57,8 @@ typedef struct query_weights_request { weights_interrupt_callback_t interrupt_callback; void *interrupt_callback_data; + + uuid_t *transaction; } QUERY_WEIGHTS_REQUEST; int web_api_v12_weights(BUFFER *wb, QUERY_WEIGHTS_REQUEST *qwr); diff --git a/web/api/web_api.c b/web/api/web_api.c index 25c765551e522b..0c4db4dfc4dfd5 100644 --- a/web/api/web_api.c +++ b/web/api/web_api.c @@ -5,39 +5,52 @@ bool netdata_is_protected_by_bearer = false; // this is controlled by cloud, at the point the agent logs in - this should also be saved to /var/lib/netdata DICTIONARY *netdata_authorized_bearers = NULL; -static short int web_client_check_acl_and_bearer(struct web_client *w, WEB_CLIENT_ACL endpoint_acl) { - if(endpoint_acl == WEB_CLIENT_ACL_NONE || (endpoint_acl & WEB_CLIENT_ACL_NOCHECK)) +static short int web_client_check_acl_and_bearer(struct web_client *w, HTTP_ACL endpoint_acl) { + if(endpoint_acl == HTTP_ACL_NONE || (endpoint_acl & HTTP_ACL_NOCHECK)) { // the endpoint is totally public + w->access = HTTP_ACCESS_ADMINS; return HTTP_RESP_OK; + } bool acl_allows = w->acl & endpoint_acl; if(!acl_allows) // the channel we received the request from (w->acl) is not compatible with the endpoint return HTTP_RESP_FORBIDDEN; - if(!netdata_is_protected_by_bearer && !(endpoint_acl & WEB_CLIENT_ACL_BEARER_REQUIRED)) + if(!netdata_is_protected_by_bearer && !(endpoint_acl & (HTTP_ACL_BEARER_REQUIRED | HTTP_ACL_BEARER_OPTIONAL))) { // bearer protection is not enabled and is not required by the endpoint + w->access = HTTP_ACCESS_ANY; return HTTP_RESP_OK; + } - if(!(endpoint_acl & (WEB_CLIENT_ACL_BEARER_REQUIRED|WEB_CLIENT_ACL_BEARER_OPTIONAL))) + if(!(endpoint_acl & (HTTP_ACL_BEARER_REQUIRED | HTTP_ACL_BEARER_OPTIONAL | HTTP_ACL_BEARER_IF_PROTECTED))) { // endpoint does not require a bearer + w->access = HTTP_ACCESS_ANY; return HTTP_RESP_OK; + } - if((w->acl & (WEB_CLIENT_ACL_ACLK|WEB_CLIENT_ACL_WEBRTC))) + if((w->acl & (HTTP_ACL_ACLK | HTTP_ACL_WEBRTC))) { // the request is coming from ACLK or WEBRTC (authorized already), + w->access = HTTP_ACCESS_MEMBERS; return HTTP_RESP_OK; + } - // at this point we need a bearer to serve the request - // either because: - // - // 1. WEB_CLIENT_ACL_BEARER_REQUIRED, or - // 2. netdata_is_protected_by_bearer == true - // + // we now need a bearer to serve the request, + // because: + // 1. HTTP_ACL_BEARER_REQUIRED, or + // 2. netdata_is_protected_by_bearer == true BEARER_STATUS t = api_check_bearer_token(w); - if(t == BEARER_STATUS_AVAILABLE_AND_VALIDATED) + if(t == BEARER_STATUS_AVAILABLE_AND_VALIDATED) { // we have a valid bearer on the request + w->access = HTTP_ACCESS_MEMBERS; return HTTP_RESP_OK; + } + + if(endpoint_acl & HTTP_ACL_BEARER_OPTIONAL) { + w->access = HTTP_ACCESS_ANY; + return HTTP_RESP_OK; + } netdata_log_info("BEARER: bearer is required for request: code %d", t); @@ -278,6 +291,8 @@ int web_client_api_request_weights(RRDHOST *host, struct web_client *w, char *ur .interrupt_callback = web_client_interrupt_callback, .interrupt_callback_data = w, + + .transaction = &w->transaction, }; return web_api_v12_weights(wb, &qwr); diff --git a/web/api/web_api.h b/web/api/web_api.h index a6b3716b760863..99a3ee4a823e5c 100644 --- a/web/api/web_api.h +++ b/web/api/web_api.h @@ -29,7 +29,7 @@ BEARER_STATUS extract_bearer_token_from_request(struct web_client *w, char *dst, struct web_api_command { const char *command; uint32_t hash; - WEB_CLIENT_ACL acl; + HTTP_ACL acl; int (*callback)(RRDHOST *host, struct web_client *w, char *url); unsigned int allow_subpaths; }; diff --git a/web/api/web_api_v1.c b/web/api/web_api_v1.c index e08f8aa2f8729c..d7fa48c8b3f55e 100644 --- a/web/api/web_api_v1.c +++ b/web/api/web_api_v1.c @@ -822,6 +822,7 @@ static inline int web_client_api_request_v1_data(RRDHOST *host, struct web_clien .priority = STORAGE_PRIORITY_NORMAL, .interrupt_callback = web_client_interrupt_callback, .interrupt_callback_data = w, + .transaction = &w->transaction, }; qt = query_target_create(&qtr); @@ -1018,12 +1019,12 @@ inline int web_client_api_request_v1_registry(RRDHOST *host, struct web_client * if(unlikely(action == 'H')) { // HELLO request, dashboard ACL analytics_log_dashboard(); - if(unlikely(!web_client_can_access_dashboard(w))) + if(unlikely(!http_can_access_dashboard(w))) return web_client_permission_denied(w); } else { // everything else, registry ACL - if(unlikely(!web_client_can_access_registry(w))) + if(unlikely(!http_can_access_registry(w))) return web_client_permission_denied(w); if(unlikely(do_not_track)) { @@ -1373,13 +1374,17 @@ static int web_client_api_request_v1_aclk_state(RRDHOST *host, struct web_client } int web_client_api_request_v1_metric_correlations(RRDHOST *host, struct web_client *w, char *url) { - return web_client_api_request_weights(host, w, url, default_metric_correlations_method, - WEIGHTS_FORMAT_CHARTS, 1); + return web_client_api_request_weights(host, w, url, default_metric_correlations_method, WEIGHTS_FORMAT_CHARTS, 1); } int web_client_api_request_v1_weights(RRDHOST *host, struct web_client *w, char *url) { - return web_client_api_request_weights(host, w, url, WEIGHTS_METHOD_ANOMALY_RATE, - WEIGHTS_FORMAT_CONTEXTS, 1); + return web_client_api_request_weights(host, w, url, WEIGHTS_METHOD_ANOMALY_RATE, WEIGHTS_FORMAT_CONTEXTS, 1); +} + +static void web_client_progress_functions_update(void *data, size_t done, size_t all) { + // handle progress updates from the plugin + struct web_client *w = data; + query_progress_functions_update(&w->transaction, done, all); } int web_client_api_request_v1_function(RRDHOST *host, struct web_client *w, char *url) { @@ -1413,8 +1418,9 @@ int web_client_api_request_v1_function(RRDHOST *host, struct web_client *w, char char transaction[UUID_COMPACT_STR_LEN]; uuid_unparse_lower_compact(w->transaction, transaction); - return rrd_function_run(host, wb, timeout, function, true, transaction, + return rrd_function_run(host, wb, timeout, w->access, function, true, transaction, NULL, NULL, + web_client_progress_functions_update, w, web_client_interrupt_callback, w, NULL); } @@ -1547,43 +1553,43 @@ int web_client_api_request_v1_mgmt(RRDHOST *host, struct web_client *w, char *ur } static struct web_api_command api_commands_v1[] = { - { "info", 0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v1_info, 0 }, - { "data", 0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v1_data, 0 }, - { "chart", 0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v1_chart, 0 }, - { "charts", 0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v1_charts, 0 }, - { "context", 0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v1_context, 0 }, - { "contexts", 0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v1_contexts, 0 }, + {"info", 0, HTTP_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v1_info, 0 }, + {"data", 0, HTTP_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v1_data, 0 }, + {"chart", 0, HTTP_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v1_chart, 0 }, + {"charts", 0, HTTP_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v1_charts, 0 }, + {"context", 0, HTTP_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v1_context, 0 }, + {"contexts", 0, HTTP_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v1_contexts, 0 }, // registry checks the ACL by itself, so we allow everything - { "registry", 0, WEB_CLIENT_ACL_NOCHECK, web_client_api_request_v1_registry, 0 }, + {"registry", 0, HTTP_ACL_NOCHECK, web_client_api_request_v1_registry, 0 }, // badges can be fetched with both dashboard and badge permissions - { "badge.svg", 0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC | WEB_CLIENT_ACL_BADGE, web_client_api_request_v1_badge, 0 }, + {"badge.svg", 0, HTTP_ACL_DASHBOARD_ACLK_WEBRTC | HTTP_ACL_BADGE, web_client_api_request_v1_badge, 0 }, - { "alarms", 0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v1_alarms, 0 }, - { "alarms_values", 0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v1_alarms_values, 0 }, - { "alarm_log", 0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v1_alarm_log, 0 }, - { "alarm_variables", 0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v1_alarm_variables, 0 }, - { "alarm_count", 0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v1_alarm_count, 0 }, - { "allmetrics", 0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v1_allmetrics, 0 }, + {"alarms", 0, HTTP_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v1_alarms, 0 }, + {"alarms_values", 0, HTTP_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v1_alarms_values, 0 }, + {"alarm_log", 0, HTTP_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v1_alarm_log, 0 }, + {"alarm_variables", 0, HTTP_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v1_alarm_variables, 0 }, + {"alarm_count", 0, HTTP_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v1_alarm_count, 0 }, + {"allmetrics", 0, HTTP_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v1_allmetrics, 0 }, #if defined(ENABLE_ML) - { "ml_info", 0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v1_ml_info, 0 }, - // { "ml_models", 0, WEB_CLIENT_ACL_DASHBOARD, web_client_api_request_v1_ml_models }, + {"ml_info", 0, HTTP_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v1_ml_info, 0 }, + // { "ml_models", 0, HTTP_ACL_DASHBOARD, web_client_api_request_v1_ml_models }, #endif - {"manage", 0, WEB_CLIENT_ACL_MGMT | WEB_CLIENT_ACL_ACLK, web_client_api_request_v1_mgmt, 1 }, - { "aclk", 0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v1_aclk_state, 0 }, - { "metric_correlations", 0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v1_metric_correlations, 0 }, - { "weights", 0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v1_weights, 0 }, + {"manage", 0, HTTP_ACL_MGMT | HTTP_ACL_ACLK, web_client_api_request_v1_mgmt, 1 }, + {"aclk", 0, HTTP_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v1_aclk_state, 0 }, + {"metric_correlations", 0, HTTP_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v1_metric_correlations, 0 }, + {"weights", 0, HTTP_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v1_weights, 0 }, - {"function", 0, WEB_CLIENT_ACL_ACLK_WEBRTC_DASHBOARD_WITH_BEARER | ACL_DEV_OPEN_ACCESS, web_client_api_request_v1_function, 0 }, - {"functions", 0, WEB_CLIENT_ACL_ACLK_WEBRTC_DASHBOARD_WITH_BEARER | ACL_DEV_OPEN_ACCESS, web_client_api_request_v1_functions, 0 }, + {"function", 0, HTTP_ACL_ACLK_WEBRTC_DASHBOARD_WITH_OPTIONAL_BEARER | ACL_DEV_OPEN_ACCESS, web_client_api_request_v1_function, 0 }, + {"functions", 0, HTTP_ACL_DASHBOARD_ACLK_WEBRTC | ACL_DEV_OPEN_ACCESS, web_client_api_request_v1_functions, 0 }, - { "dbengine_stats", 0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v1_dbengine_stats, 0 }, + {"dbengine_stats", 0, HTTP_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v1_dbengine_stats, 0 }, // terminator - { NULL, 0, WEB_CLIENT_ACL_NONE, NULL, 0 }, + {NULL, 0, HTTP_ACL_NONE, NULL, 0 }, }; inline int web_client_api_request_v1(RRDHOST *host, struct web_client *w, char *url_path_endpoint) { diff --git a/web/api/web_api_v2.c b/web/api/web_api_v2.c index 9daf80b9d29306..5937fc4cb7bc5d 100644 --- a/web/api/web_api_v2.c +++ b/web/api/web_api_v2.c @@ -580,6 +580,8 @@ static int web_client_api_request_v2_data(RRDHOST *host __maybe_unused, struct w .interrupt_callback = web_client_interrupt_callback, .interrupt_callback_data = w, + + .transaction = &w->transaction, }; for(size_t g = 0; g < MAX_QUERY_GROUP_BY_PASSES ;g++) @@ -662,6 +664,31 @@ static int web_client_api_request_v2_webrtc(RRDHOST *host __maybe_unused, struct return webrtc_new_connection(w->post_payload, w->response.data); } +static int web_client_api_request_v2_progress(RRDHOST *host __maybe_unused, struct web_client *w, char *url) { + char *transaction = NULL; + + while(url) { + char *value = strsep_skip_consecutive_separators(&url, "&"); + if(!value || !*value) continue; + + char *name = strsep_skip_consecutive_separators(&value, "="); + if(!name || !*name) continue; + if(!value || !*value) continue; + + // name and value are now the parameters + // they are not null and not empty + + if(!strcmp(name, "transaction")) transaction = value; + } + + uuid_t tr; + uuid_parse_flexi(transaction, tr); + + rrd_function_call_progresser(&tr); + + return web_api_v2_report_progress(&tr, w->response.data); +} + #define CONFIG_API_V2_URL "/api/v2/config" static int web_client_api_request_v2_config(RRDHOST *host __maybe_unused, struct web_client *w, char *query __maybe_unused) { @@ -703,17 +730,11 @@ static int web_client_api_request_v2_config(RRDHOST *host __maybe_unused, struct int http_method; switch (w->mode) { - case WEB_CLIENT_MODE_GET: - http_method = HTTP_METHOD_GET; - break; - case WEB_CLIENT_MODE_POST: - http_method = HTTP_METHOD_POST; - break; - case WEB_CLIENT_MODE_PUT: - http_method = HTTP_METHOD_PUT; - break; - case WEB_CLIENT_MODE_DELETE: - http_method = HTTP_METHOD_DELETE; + case HTTP_REQUEST_MODE_GET: + case HTTP_REQUEST_MODE_POST: + case HTTP_REQUEST_MODE_PUT: + case HTTP_REQUEST_MODE_DELETE: + http_method = w->mode; break; default: buffer_sprintf(w->response.data, "Invalid HTTP method"); @@ -839,35 +860,36 @@ static int web_client_api_request_v2_job_statuses(RRDHOST *host __maybe_unused, } static struct web_api_command api_commands_v2[] = { - {"info", 0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v2_info, 0}, + {"info", 0, HTTP_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v2_info, 0}, - {"data", 0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v2_data, 0}, - {"weights", 0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v2_weights, 0}, + {"data", 0, HTTP_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v2_data, 0}, + {"weights", 0, HTTP_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v2_weights, 0}, - {"contexts", 0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v2_contexts, 0}, - {"nodes", 0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v2_nodes, 0}, - {"node_instances", 0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v2_node_instances, 0}, - {"versions", 0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v2_versions, 0}, - {"functions", 0, WEB_CLIENT_ACL_ACLK_WEBRTC_DASHBOARD_WITH_BEARER | ACL_DEV_OPEN_ACCESS, web_client_api_request_v2_functions, 0}, - {"q", 0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v2_q, 0}, - {"alerts", 0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v2_alerts, 0}, + {"contexts", 0, HTTP_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v2_contexts, 0}, + {"nodes", 0, HTTP_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v2_nodes, 0}, + {"node_instances", 0, HTTP_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v2_node_instances, 0}, + {"versions", 0, HTTP_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v2_versions, 0}, + {"functions", 0, HTTP_ACL_DASHBOARD_ACLK_WEBRTC | ACL_DEV_OPEN_ACCESS, web_client_api_request_v2_functions, 0}, + {"q", 0, HTTP_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v2_q, 0}, + {"alerts", 0, HTTP_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v2_alerts, 0}, - {"alert_transitions", 0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v2_alert_transitions, 0}, - {"alert_config", 0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v2_alert_config, 0}, + {"alert_transitions", 0, HTTP_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v2_alert_transitions, 0}, + {"alert_config", 0, HTTP_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v2_alert_config, 0}, - {"claim", 0, WEB_CLIENT_ACL_NOCHECK, web_client_api_request_v2_claim, 0}, + {"claim", 0, HTTP_ACL_NOCHECK, web_client_api_request_v2_claim, 0}, - {"rtc_offer", 0, WEB_CLIENT_ACL_ACLK | ACL_DEV_OPEN_ACCESS, web_client_api_request_v2_webrtc, 0}, - {"bearer_protection", 0, WEB_CLIENT_ACL_ACLK | ACL_DEV_OPEN_ACCESS, api_v2_bearer_protection, 0}, - {"bearer_get_token", 0, WEB_CLIENT_ACL_ACLK | ACL_DEV_OPEN_ACCESS, api_v2_bearer_token, 0}, + {"rtc_offer", 0, HTTP_ACL_ACLK | ACL_DEV_OPEN_ACCESS, web_client_api_request_v2_webrtc, 0}, + {"bearer_protection", 0, HTTP_ACL_ACLK | ACL_DEV_OPEN_ACCESS, api_v2_bearer_protection, 0}, + {"bearer_get_token", 0, HTTP_ACL_ACLK | ACL_DEV_OPEN_ACCESS, api_v2_bearer_token, 0}, - {"config", 0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v2_config, 1}, - {"job_statuses", 0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v2_job_statuses, 0}, + {"config", 0, HTTP_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v2_config, 1}, + {"job_statuses", 0, HTTP_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v2_job_statuses, 0}, - { "ilove.svg", 0, WEB_CLIENT_ACL_NOCHECK, web_client_api_request_v2_ilove, 0 }, + { "ilove.svg", 0, HTTP_ACL_NOCHECK, web_client_api_request_v2_ilove, 0 }, + { "progress", 0, HTTP_ACL_NOCHECK, web_client_api_request_v2_progress, 0 }, // terminator - {NULL, 0, WEB_CLIENT_ACL_NONE, NULL, 0}, + {NULL, 0, HTTP_ACL_NONE, NULL, 0}, }; inline int web_client_api_request_v2(RRDHOST *host, struct web_client *w, char *url_path_endpoint) { diff --git a/web/rtc/webrtc.c b/web/rtc/webrtc.c index 80fff7e940abce..2d1e0aeaa6345a 100644 --- a/web/rtc/webrtc.c +++ b/web/rtc/webrtc.c @@ -293,15 +293,15 @@ static void webrtc_execute_api_request(WEBRTC_DC *chan, const char *request, siz w->interrupt.callback = web_client_stop_callback; w->interrupt.callback_data = chan; - w->acl = WEB_CLIENT_ACL_WEBRTC; + w->acl = HTTP_ACL_WEBRTC; char *path = (char *)request; if(strncmp(request, "POST ", 5) == 0) { - w->mode = WEB_CLIENT_MODE_POST; + w->mode = HTTP_REQUEST_MODE_POST; path += 10; } else if(strncmp(request, "GET ", 4) == 0) { - w->mode = WEB_CLIENT_MODE_GET; + w->mode = HTTP_REQUEST_MODE_GET; path += 4; } diff --git a/web/server/h2o/http_server.c b/web/server/h2o/http_server.c index 08ad120b147fff..b3e000325f4e5b 100644 --- a/web/server/h2o/http_server.c +++ b/web/server/h2o/http_server.c @@ -210,7 +210,7 @@ static inline int _netdata_uberhandler(h2o_req_t *req, RRDHOST **host) w.response.header = buffer_create(NBUF_INITIAL_SIZE_RESP, NULL); w.url_query_string_decoded = buffer_create(NBUF_INITIAL_SIZE_RESP, NULL); w.url_as_received = buffer_create(NBUF_INITIAL_SIZE_RESP, NULL); - w.acl = WEB_CLIENT_ACL_DASHBOARD; + w.acl = HTTP_ACL_DASHBOARD; char *path_c_str = iovec_to_cstr(&api_command); char *path_unescaped = url_unescape(path_c_str); diff --git a/web/server/static/static-threaded.c b/web/server/static/static-threaded.c index 773d49f15f6187..e6fae9835c233a 100644 --- a/web/server/static/static-threaded.c +++ b/web/server/static/static-threaded.c @@ -142,7 +142,7 @@ static int web_server_file_read_callback(POLLINFO *pi, short int *events) { goto cleanup; } - if(unlikely(w->mode != WEB_CLIENT_MODE_FILECOPY || w->ifd == w->ofd)) { + if(unlikely(w->mode != HTTP_REQUEST_MODE_FILECOPY || w->ifd == w->ofd)) { netdata_log_debug(D_WEB_CLIENT, "%llu: PREVENTED ATTEMPT TO READ FILE ON FD %d, ON NON-FILECOPY WEB CLIENT", w->id, pi->fd); retval = -1; goto cleanup; @@ -296,11 +296,11 @@ static int web_server_rcv_callback(POLLINFO *pi, short int *events) { worker_is_busy(WORKER_JOB_PROCESS); web_client_process_request_from_web_server(w); - if (unlikely(w->mode == WEB_CLIENT_MODE_STREAM)) { + if (unlikely(w->mode == HTTP_REQUEST_MODE_STREAM)) { web_client_send(w); } - else if(unlikely(w->mode == WEB_CLIENT_MODE_FILECOPY)) { + else if(unlikely(w->mode == HTTP_REQUEST_MODE_FILECOPY)) { if(w->pollinfo_filecopy_slot == 0) { netdata_log_debug(D_WEB_CLIENT, "%llu: FILECOPY DETECTED ON FD %d", w->id, pi->fd); diff --git a/web/server/web_client.c b/web/server/web_client.c index 3a869bc62a59b7..b1d94ba19f1977 100644 --- a/web/server/web_client.c +++ b/web/server/web_client.c @@ -138,6 +138,9 @@ static void web_client_reset_allocations(struct web_client *w, bool free_all) { freez(w->forwarded_host); w->forwarded_host = NULL; + freez(w->forwarded_for); + w->forwarded_for = NULL; + freez(w->origin); w->origin = NULL; @@ -163,41 +166,12 @@ static void web_client_reset_allocations(struct web_client *w, bool free_all) { web_client_reset_path_flags(w); } -const char *get_request_method(struct web_client *w) { - switch(w->mode) { - case WEB_CLIENT_MODE_FILECOPY: - return "FILECOPY"; - - case WEB_CLIENT_MODE_OPTIONS: - return "OPTIONS"; - - case WEB_CLIENT_MODE_STREAM: - return "STREAM"; - - case WEB_CLIENT_MODE_POST: - return "POST"; - - case WEB_CLIENT_MODE_PUT: - return "PUT"; - - case WEB_CLIENT_MODE_GET: - return "GET"; - - case WEB_CLIENT_MODE_DELETE: - return "DELETE"; - - default: - return "UNKNOWN"; - } -} - void web_client_log_completed_request(struct web_client *w, bool update_web_stats) { struct timeval tv; now_monotonic_high_precision_timeval(&tv); - size_t size = (w->mode == WEB_CLIENT_MODE_FILECOPY)?w->response.rlen:w->response.data->len; - size_t sent = size; - if(likely(w->response.zoutput)) sent = (size_t)w->response.zstream.total_out; + size_t size = (w->mode == HTTP_REQUEST_MODE_FILECOPY) ? w->response.rlen : w->response.data->len; + size_t sent = w->response.zoutput ? (size_t)w->response.zstream.total_out : size; if(update_web_stats) global_statistics_web_request_completed(dt_usec(&tv, &w->timings.tv_in), @@ -215,7 +189,7 @@ void web_client_log_completed_request(struct web_client *w, bool update_web_stat ND_LOG_FIELD_U64(NDF_CONNECTION_ID, w->id), ND_LOG_FIELD_UUID(NDF_TRANSACTION_ID, &w->transaction), ND_LOG_FIELD_TXT(NDF_NIDL_NODE, w->client_host), - ND_LOG_FIELD_TXT(NDF_REQUEST_METHOD, get_request_method(w)), + ND_LOG_FIELD_TXT(NDF_REQUEST_METHOD, http_request_method2string(w->mode)), ND_LOG_FIELD_BFR(NDF_REQUEST, w->url_as_received), ND_LOG_FIELD_U64(NDF_RESPONSE_CODE, w->response.code), ND_LOG_FIELD_U64(NDF_RESPONSE_SENT_BYTES, sent), @@ -235,13 +209,21 @@ void web_client_log_completed_request(struct web_client *w, bool update_web_stat else if(w->response.code >= 300) prio = NDLP_NOTICE; + // cleanup progress + if(web_client_flag_check(w, WEB_CLIENT_FLAG_PROGRESS_TRACKING)) { + web_client_flag_clear(w, WEB_CLIENT_FLAG_PROGRESS_TRACKING); + query_progress_finished(&w->transaction, 0, w->response.code, total_ut, size, sent); + } + // access log - nd_log(NDLS_ACCESS, prio, NULL); + if(likely(buffer_strlen(w->url_as_received))) + nd_log(NDLS_ACCESS, prio, NULL); } void web_client_request_done(struct web_client *w) { ND_LOG_STACK lgs[] = { ND_LOG_FIELD_TXT(NDF_SRC_IP, w->client_ip), + ND_LOG_FIELD_TXT(NDF_SRC_FORWARDED_FOR, w->forwarded_for), ND_LOG_FIELD_TXT(NDF_SRC_PORT, w->client_port), ND_LOG_FIELD_END(), }; @@ -251,10 +233,9 @@ void web_client_request_done(struct web_client *w) { netdata_log_debug(D_WEB_CLIENT, "%llu: Resetting client.", w->id); - if(likely(buffer_strlen(w->url_as_received))) - web_client_log_completed_request(w, true); + web_client_log_completed_request(w, true); - if(unlikely(w->mode == WEB_CLIENT_MODE_FILECOPY)) { + if(unlikely(w->mode == HTTP_REQUEST_MODE_FILECOPY)) { if(w->ifd != w->ofd) { netdata_log_debug(D_WEB_CLIENT, "%llu: Closing filecopy input file descriptor %d.", w->id, w->ifd); @@ -270,7 +251,7 @@ void web_client_request_done(struct web_client *w) { web_client_reset_allocations(w, false); - w->mode = WEB_CLIENT_MODE_GET; + w->mode = HTTP_REQUEST_MODE_GET; web_client_disable_donottrack(w); web_client_disable_tracking_required(w); @@ -292,74 +273,6 @@ void web_client_request_done(struct web_client *w) { w->statistics.sent_bytes = 0; } -static struct { - const char *extension; - uint32_t hash; - uint8_t contenttype; -} mime_types[] = { - { "html" , 0 , CT_TEXT_HTML} - , {"js" , 0 , CT_APPLICATION_X_JAVASCRIPT} - , {"css" , 0 , CT_TEXT_CSS} - , {"xml" , 0 , CT_TEXT_XML} - , {"xsl" , 0 , CT_TEXT_XSL} - , {"txt" , 0 , CT_TEXT_PLAIN} - , {"svg" , 0 , CT_IMAGE_SVG_XML} - , {"ttf" , 0 , CT_APPLICATION_X_FONT_TRUETYPE} - , {"otf" , 0 , CT_APPLICATION_X_FONT_OPENTYPE} - , {"woff2", 0 , CT_APPLICATION_FONT_WOFF2} - , {"woff" , 0 , CT_APPLICATION_FONT_WOFF} - , {"eot" , 0 , CT_APPLICATION_VND_MS_FONTOBJ} - , {"png" , 0 , CT_IMAGE_PNG} - , {"jpg" , 0 , CT_IMAGE_JPG} - , {"jpeg" , 0 , CT_IMAGE_JPG} - , {"gif" , 0 , CT_IMAGE_GIF} - , {"bmp" , 0 , CT_IMAGE_BMP} - , {"ico" , 0 , CT_IMAGE_XICON} - , {"icns" , 0 , CT_IMAGE_ICNS} - , { NULL, 0, 0} -}; - -static inline uint8_t contenttype_for_filename(const char *filename) { - // netdata_log_info("checking filename '%s'", filename); - - static int initialized = 0; - int i; - - if(unlikely(!initialized)) { - for (i = 0; mime_types[i].extension; i++) - mime_types[i].hash = simple_hash(mime_types[i].extension); - - initialized = 1; - } - - const char *s = filename, *last_dot = NULL; - - // find the last dot - while(*s) { - if(unlikely(*s == '.')) last_dot = s; - s++; - } - - if(unlikely(!last_dot || !*last_dot || !last_dot[1])) { - // netdata_log_info("no extension for filename '%s'", filename); - return CT_APPLICATION_OCTET_STREAM; - } - last_dot++; - - // netdata_log_info("extension for filename '%s' is '%s'", filename, last_dot); - - uint32_t hash = simple_hash(last_dot); - for(i = 0; mime_types[i].extension ; i++) { - if(unlikely(hash == mime_types[i].hash && !strcmp(last_dot, mime_types[i].extension))) { - // netdata_log_info("matched extension for filename '%s': '%s'", filename, last_dot); - return mime_types[i].contenttype; - } - } - - // netdata_log_info("not matched extension for filename '%s': '%s'", filename, last_dot); - return CT_APPLICATION_OCTET_STREAM; -} - static int append_slash_to_url_and_redirect(struct web_client *w) { // this function returns a relative redirect // it finds the last path component on the URL and just appends / to it @@ -508,7 +421,7 @@ static bool find_filename_to_serve(const char *filename, char *dst, size_t dst_l static int mysendfile(struct web_client *w, char *filename) { netdata_log_debug(D_WEB_CLIENT, "%llu: Looking for file '%s/%s'", w->id, netdata_configured_web_dir, filename); - if(!web_client_can_access_dashboard(w)) + if(!http_can_access_dashboard(w)) return web_client_permission_denied(w); // skip leading slashes @@ -576,7 +489,7 @@ static int mysendfile(struct web_client *w, char *filename) { w->response.data->content_type = contenttype_for_filename(web_filename); netdata_log_debug(D_WEB_CLIENT_ACCESS, "%llu: Sending file '%s' (%"PRId64" bytes, ifd %d, ofd %d).", w->id, web_filename, (int64_t)statbuf.st_size, w->ifd, w->ofd); - w->mode = WEB_CLIENT_MODE_FILECOPY; + w->mode = HTTP_REQUEST_MODE_FILECOPY; web_client_enable_wait_receive(w); web_client_disable_wait_send(w); buffer_flush(w->response.data); @@ -719,14 +632,14 @@ static inline int check_host_and_call(RRDHOST *host, struct web_client *w, char } static inline int UNUSED_FUNCTION(check_host_and_dashboard_acl_and_call)(RRDHOST *host, struct web_client *w, char *url, int (*func)(RRDHOST *, struct web_client *, char *)) { - if(!web_client_can_access_dashboard(w)) + if(!http_can_access_dashboard(w)) return web_client_permission_denied(w); return check_host_and_call(host, w, url, func); } static inline int UNUSED_FUNCTION(check_host_and_mgmt_acl_and_call)(RRDHOST *host, struct web_client *w, char *url, int (*func)(RRDHOST *, struct web_client *, char *)) { - if(!web_client_can_access_mgmt(w)) + if(!http_can_access_mgmt(w)) return web_client_permission_denied(w); return check_host_and_call(host, w, url, func); @@ -736,8 +649,10 @@ int web_client_api_request(RRDHOST *host, struct web_client *w, char *url_path_f ND_LOG_STACK lgs[] = { ND_LOG_FIELD_TXT(NDF_SRC_IP, w->client_ip), ND_LOG_FIELD_TXT(NDF_SRC_PORT, w->client_port), + ND_LOG_FIELD_TXT(NDF_SRC_FORWARDED_HOST, w->forwarded_host), + ND_LOG_FIELD_TXT(NDF_SRC_FORWARDED_FOR, w->forwarded_for), ND_LOG_FIELD_TXT(NDF_NIDL_NODE, w->client_host), - ND_LOG_FIELD_TXT(NDF_REQUEST_METHOD, get_request_method(w)), + ND_LOG_FIELD_TXT(NDF_REQUEST_METHOD, http_request_method2string(w->mode)), ND_LOG_FIELD_BFR(NDF_REQUEST, w->url_as_received), ND_LOG_FIELD_U64(NDF_CONNECTION_ID, w->id), ND_LOG_FIELD_UUID(NDF_TRANSACTION_ID, &w->transaction), @@ -745,6 +660,14 @@ int web_client_api_request(RRDHOST *host, struct web_client *w, char *url_path_f }; ND_LOG_STACK_PUSH(lgs); + if(!web_client_flag_check(w, WEB_CLIENT_FLAG_PROGRESS_TRACKING)) { + web_client_flag_set(w, WEB_CLIENT_FLAG_PROGRESS_TRACKING); + query_progress_start_or_update(&w->transaction, 0, w->mode, w->acl, + buffer_tostring(w->url_as_received), + w->post_payload, + w->forwarded_for ? w->forwarded_for : w->client_ip); + } + // get the api version char *tok = strsep_skip_consecutive_separators(&url_path_fragment, "/"); if(tok && *tok) { @@ -853,164 +776,10 @@ const char *web_content_type_to_string(HTTP_CONTENT_TYPE content_type) { } } -const char *web_response_code_to_string(int code) { - switch(code) { - case 100: - return "Continue"; - case 101: - return "Switching Protocols"; - case 102: - return "Processing"; - case 103: - return "Early Hints"; - - case 200: - return "OK"; - case 201: - return "Created"; - case 202: - return "Accepted"; - case 203: - return "Non-Authoritative Information"; - case 204: - return "No Content"; - case 205: - return "Reset Content"; - case 206: - return "Partial Content"; - case 207: - return "Multi-Status"; - case 208: - return "Already Reported"; - case 226: - return "IM Used"; - - case 300: - return "Multiple Choices"; - case 301: - return "Moved Permanently"; - case 302: - return "Found"; - case 303: - return "See Other"; - case 304: - return "Not Modified"; - case 305: - return "Use Proxy"; - case 306: - return "Switch Proxy"; - case 307: - return "Temporary Redirect"; - case 308: - return "Permanent Redirect"; - - case 400: - return "Bad Request"; - case 401: - return "Unauthorized"; - case 402: - return "Payment Required"; - case 403: - return "Forbidden"; - case 404: - return "Not Found"; - case 405: - return "Method Not Allowed"; - case 406: - return "Not Acceptable"; - case 407: - return "Proxy Authentication Required"; - case 408: - return "Request Timeout"; - case 409: - return "Conflict"; - case 410: - return "Gone"; - case 411: - return "Length Required"; - case 412: - return "Precondition Failed"; - case 413: - return "Payload Too Large"; - case 414: - return "URI Too Long"; - case 415: - return "Unsupported Media Type"; - case 416: - return "Range Not Satisfiable"; - case 417: - return "Expectation Failed"; - case 418: - return "I'm a teapot"; - case 421: - return "Misdirected Request"; - case 422: - return "Unprocessable Entity"; - case 423: - return "Locked"; - case 424: - return "Failed Dependency"; - case 425: - return "Too Early"; - case 426: - return "Upgrade Required"; - case 428: - return "Precondition Required"; - case 429: - return "Too Many Requests"; - case 431: - return "Request Header Fields Too Large"; - case 451: - return "Unavailable For Legal Reasons"; - case 499: // nginx's extension to the standard - return "Client Closed Request"; - - case 500: - return "Internal Server Error"; - case 501: - return "Not Implemented"; - case 502: - return "Bad Gateway"; - case 503: - return "Service Unavailable"; - case 504: - return "Gateway Timeout"; - case 505: - return "HTTP Version Not Supported"; - case 506: - return "Variant Also Negotiates"; - case 507: - return "Insufficient Storage"; - case 508: - return "Loop Detected"; - case 510: - return "Not Extended"; - case 511: - return "Network Authentication Required"; - - default: - if(code >= 100 && code < 200) - return "Informational"; - - if(code >= 200 && code < 300) - return "Successful"; - - if(code >= 300 && code < 400) - return "Redirection"; - - if(code >= 400 && code < 500) - return "Client Error"; - - if(code >= 500 && code < 600) - return "Server Error"; - - return "Undefined Error"; - } -} - static inline char *http_header_parse(struct web_client *w, char *s, int parse_useragent) { static uint32_t hash_origin = 0, hash_connection = 0, hash_donottrack = 0, hash_useragent = 0, - hash_authorization = 0, hash_host = 0, hash_forwarded_host = 0, hash_transaction_id = 0; + hash_authorization = 0, hash_host = 0, hash_forwarded_host = 0, hash_forwarded_for = 0, + hash_transaction_id = 0; static uint32_t hash_accept_encoding = 0; if(unlikely(!hash_origin)) { @@ -1022,6 +791,7 @@ static inline char *http_header_parse(struct web_client *w, char *s, int parse_u hash_authorization = simple_uhash("X-Auth-Token"); hash_host = simple_uhash("Host"); hash_forwarded_host = simple_uhash("X-Forwarded-Host"); + hash_forwarded_for = simple_uhash("X-Forwarded-For"); hash_transaction_id = simple_uhash("X-Transaction-ID"); } @@ -1090,6 +860,11 @@ static inline char *http_header_parse(struct web_client *w, char *s, int parse_u strncpyz(buffer, v, ((size_t)(ve - v) < sizeof(buffer) - 1 ? (size_t)(ve - v) : sizeof(buffer) - 1)); w->forwarded_host = strdupz(buffer); } + else if(hash == hash_forwarded_for && !strcasecmp(s, "X-Forwarded-For")) { + char buffer[NI_MAXHOST]; + strncpyz(buffer, v, ((size_t)(ve - v) < sizeof(buffer) - 1 ? (size_t)(ve - v) : sizeof(buffer) - 1)); + w->forwarded_for = strdupz(buffer); + } else if(hash == hash_transaction_id && !strcasecmp(s, "X-Transaction-ID")) { char buffer[UUID_STR_LEN * 2]; strncpyz(buffer, v, ((size_t)(ve - v) < sizeof(buffer) - 1 ? (size_t)(ve - v) : sizeof(buffer) - 1)); @@ -1115,29 +890,29 @@ static inline char *web_client_valid_method(struct web_client *w, char *s) { // is is a valid request? if(!strncmp(s, "GET ", 4)) { s = &s[4]; - w->mode = WEB_CLIENT_MODE_GET; + w->mode = HTTP_REQUEST_MODE_GET; } else if(!strncmp(s, "OPTIONS ", 8)) { s = &s[8]; - w->mode = WEB_CLIENT_MODE_OPTIONS; + w->mode = HTTP_REQUEST_MODE_OPTIONS; } else if(!strncmp(s, "POST ", 5)) { s = &s[5]; - w->mode = WEB_CLIENT_MODE_POST; + w->mode = HTTP_REQUEST_MODE_POST; } else if(!strncmp(s, "PUT ", 4)) { s = &s[4]; - w->mode = WEB_CLIENT_MODE_PUT; + w->mode = HTTP_REQUEST_MODE_PUT; } else if(!strncmp(s, "DELETE ", 7)) { s = &s[7]; - w->mode = WEB_CLIENT_MODE_DELETE; + w->mode = HTTP_REQUEST_MODE_DELETE; } else if(!strncmp(s, "STREAM ", 7)) { s = &s[7]; #ifdef ENABLE_HTTPS - if (!SSL_connection(&w->ssl) && web_client_is_using_ssl_force(w)) { + if (!SSL_connection(&w->ssl) && http_is_using_ssl_force(w)) { w->header_parse_tries = 0; w->header_parse_last_size = 0; web_client_disable_wait_receive(w); @@ -1166,7 +941,7 @@ static inline char *web_client_valid_method(struct web_client *w, char *s) { } #endif - w->mode = WEB_CLIENT_MODE_STREAM; + w->mode = HTTP_REQUEST_MODE_STREAM; } else { s = NULL; @@ -1279,7 +1054,7 @@ static inline HTTP_VALIDATION http_request_validate(struct web_client *w) { #ifdef ENABLE_HTTPS if ( (!web_client_check_unix(w)) && (netdata_ssl_web_server_ctx) ) { - if (!w->ssl.conn && (web_client_is_using_ssl_force(w) || web_client_is_using_ssl_default(w)) && (w->mode != WEB_CLIENT_MODE_STREAM)) { + if (!w->ssl.conn && (http_is_using_ssl_force(w) || http_is_using_ssl_default(w)) && (w->mode != HTTP_REQUEST_MODE_STREAM)) { w->header_parse_tries = 0; w->header_parse_last_size = 0; web_client_disable_wait_receive(w); @@ -1295,7 +1070,7 @@ static inline HTTP_VALIDATION http_request_validate(struct web_client *w) { } // another header line - s = http_header_parse(w, s, (w->mode == WEB_CLIENT_MODE_STREAM)); // parse user agent + s = http_header_parse(w, s, (w->mode == HTTP_REQUEST_MODE_STREAM)); // parse user agent } } @@ -1340,7 +1115,7 @@ void web_client_build_http_header(struct web_client *w) { netdata_log_debug(D_WEB_CLIENT, "%llu: Generating HTTP header with response %d.", w->id, w->response.code); const char *content_type_string = web_content_type_to_string(w->response.data->content_type); - const char *code_msg = web_response_code_to_string(w->response.code); + const char *code_msg = http_response_code2string(w->response.code); // prepare the last modified and expiration dates char rfc7231_date[RFC7231_MAX_LENGTH], rfc7231_expires[RFC7231_MAX_LENGTH]; @@ -1393,7 +1168,7 @@ void web_client_build_http_header(struct web_client *w) { } } - if(w->mode == WEB_CLIENT_MODE_OPTIONS) { + if(w->mode == HTTP_REQUEST_MODE_OPTIONS) { buffer_strcat(w->response.header_output, "Access-Control-Allow-Methods: GET, OPTIONS\r\n" "Access-Control-Allow-Headers: accept, x-requested-with, origin, content-type, cookie, pragma, cache-control, x-auth-token\r\n" @@ -1580,7 +1355,7 @@ int web_client_api_request_with_node_selection(RRDHOST *host, struct web_client // entry point for all API requests ND_LOG_STACK lgs[] = { - ND_LOG_FIELD_TXT(NDF_REQUEST_METHOD, get_request_method(w)), + ND_LOG_FIELD_TXT(NDF_REQUEST_METHOD, http_request_method2string(w->mode)), ND_LOG_FIELD_BFR(NDF_REQUEST, w->url_as_received), ND_LOG_FIELD_U64(NDF_CONNECTION_ID, w->id), ND_LOG_FIELD_UUID(NDF_TRANSACTION_ID, &w->transaction), @@ -1692,7 +1467,7 @@ static inline int web_client_process_url(RRDHOST *host, struct web_client *w, ch return web_client_process_url(host, w, decoded_url_path); } else if(unlikely(hash == hash_netdata_conf && strcmp(tok, "netdata.conf") == 0)) { // netdata.conf - if(unlikely(!web_client_can_access_netdataconf(w))) + if(unlikely(!http_can_access_netdataconf(w))) return web_client_permission_denied(w); netdata_log_debug(D_WEB_CLIENT_ACCESS, "%llu: generating netdata.conf ...", w->id); @@ -1703,7 +1478,7 @@ static inline int web_client_process_url(RRDHOST *host, struct web_client *w, ch } #ifdef NETDATA_INTERNAL_CHECKS else if(unlikely(hash == hash_exit && strcmp(tok, "exit") == 0)) { - if(unlikely(!web_client_can_access_netdataconf(w))) + if(unlikely(!http_can_access_netdataconf(w))) return web_client_permission_denied(w); w->response.data->content_type = CT_TEXT_PLAIN; @@ -1719,7 +1494,7 @@ static inline int web_client_process_url(RRDHOST *host, struct web_client *w, ch return HTTP_RESP_OK; } else if(unlikely(hash == hash_debug && strcmp(tok, "debug") == 0)) { - if(unlikely(!web_client_can_access_netdataconf(w))) + if(unlikely(!http_can_access_netdataconf(w))) return web_client_permission_denied(w); buffer_flush(w->response.data); @@ -1759,7 +1534,7 @@ static inline int web_client_process_url(RRDHOST *host, struct web_client *w, ch return HTTP_RESP_BAD_REQUEST; } else if(unlikely(hash == hash_mirror && strcmp(tok, "mirror") == 0)) { - if(unlikely(!web_client_can_access_netdataconf(w))) + if(unlikely(!http_can_access_netdataconf(w))) return web_client_permission_denied(w); netdata_log_debug(D_WEB_CLIENT_ACCESS, "%llu: Mirroring...", w->id); @@ -1799,8 +1574,10 @@ void web_client_process_request_from_web_server(struct web_client *w) { ND_LOG_FIELD_CB(NDF_SRC_TRANSPORT, web_server_log_transport, w), ND_LOG_FIELD_TXT(NDF_SRC_IP, w->client_ip), ND_LOG_FIELD_TXT(NDF_SRC_PORT, w->client_port), + ND_LOG_FIELD_TXT(NDF_SRC_FORWARDED_HOST, w->forwarded_host), + ND_LOG_FIELD_TXT(NDF_SRC_FORWARDED_FOR, w->forwarded_for), ND_LOG_FIELD_TXT(NDF_NIDL_NODE, w->client_host), - ND_LOG_FIELD_TXT(NDF_REQUEST_METHOD, get_request_method(w)), + ND_LOG_FIELD_TXT(NDF_REQUEST_METHOD, http_request_method2string(w->mode)), ND_LOG_FIELD_BFR(NDF_REQUEST, w->url_as_received), ND_LOG_FIELD_U64(NDF_CONNECTION_ID, w->id), ND_LOG_FIELD_UUID(NDF_TRANSACTION_ID, &w->transaction), @@ -1816,9 +1593,17 @@ void web_client_process_request_from_web_server(struct web_client *w) { switch(http_request_validate(w)) { case HTTP_VALIDATION_OK: + if(!web_client_flag_check(w, WEB_CLIENT_FLAG_PROGRESS_TRACKING)) { + web_client_flag_set(w, WEB_CLIENT_FLAG_PROGRESS_TRACKING); + query_progress_start_or_update(&w->transaction, 0, w->mode, w->acl, + buffer_tostring(w->url_as_received), + w->post_payload, + w->forwarded_for ? w->forwarded_for : w->client_ip); + } + switch(w->mode) { - case WEB_CLIENT_MODE_STREAM: - if(unlikely(!web_client_can_access_stream(w))) { + case HTTP_REQUEST_MODE_STREAM: + if(unlikely(!http_can_access_stream(w))) { web_client_permission_denied(w); return; } @@ -1826,13 +1611,13 @@ void web_client_process_request_from_web_server(struct web_client *w) { w->response.code = rrdpush_receiver_thread_spawn(w, (char *)buffer_tostring(w->url_query_string_decoded), NULL); return; - case WEB_CLIENT_MODE_OPTIONS: + case HTTP_REQUEST_MODE_OPTIONS: if(unlikely( - !web_client_can_access_dashboard(w) && - !web_client_can_access_registry(w) && - !web_client_can_access_badges(w) && - !web_client_can_access_mgmt(w) && - !web_client_can_access_netdataconf(w) + !http_can_access_dashboard(w) && + !http_can_access_registry(w) && + !http_can_access_badges(w) && + !http_can_access_mgmt(w) && + !http_can_access_netdataconf(w) )) { web_client_permission_denied(w); break; @@ -1844,17 +1629,17 @@ void web_client_process_request_from_web_server(struct web_client *w) { w->response.code = HTTP_RESP_OK; break; - case WEB_CLIENT_MODE_FILECOPY: - case WEB_CLIENT_MODE_POST: - case WEB_CLIENT_MODE_GET: - case WEB_CLIENT_MODE_PUT: - case WEB_CLIENT_MODE_DELETE: + case HTTP_REQUEST_MODE_FILECOPY: + case HTTP_REQUEST_MODE_POST: + case HTTP_REQUEST_MODE_GET: + case HTTP_REQUEST_MODE_PUT: + case HTTP_REQUEST_MODE_DELETE: if(unlikely( - !web_client_can_access_dashboard(w) && - !web_client_can_access_registry(w) && - !web_client_can_access_badges(w) && - !web_client_can_access_mgmt(w) && - !web_client_can_access_netdataconf(w) + !http_can_access_dashboard(w) && + !http_can_access_registry(w) && + !http_can_access_badges(w) && + !http_can_access_mgmt(w) && + !http_can_access_netdataconf(w) )) { web_client_permission_denied(w); break; @@ -1888,6 +1673,10 @@ void web_client_process_request_from_web_server(struct web_client *w) { w->response.code = (short)web_client_process_url(localhost, w, path); break; + + default: + web_client_permission_denied(w); + return; } break; @@ -1907,8 +1696,8 @@ void web_client_process_request_from_web_server(struct web_client *w) { // wait for more data // set to normal to prevent web_server_rcv_callback // from going into stream mode - if (w->mode == WEB_CLIENT_MODE_STREAM) - w->mode = WEB_CLIENT_MODE_GET; + if (w->mode == HTTP_REQUEST_MODE_STREAM) + w->mode = HTTP_REQUEST_MODE_GET; return; } break; @@ -1971,22 +1760,22 @@ void web_client_process_request_from_web_server(struct web_client *w) { else web_client_disable_wait_send(w); switch(w->mode) { - case WEB_CLIENT_MODE_STREAM: + case HTTP_REQUEST_MODE_STREAM: netdata_log_debug(D_WEB_CLIENT, "%llu: STREAM done.", w->id); break; - case WEB_CLIENT_MODE_OPTIONS: + case HTTP_REQUEST_MODE_OPTIONS: netdata_log_debug(D_WEB_CLIENT, "%llu: Done preparing the OPTIONS response. Sending data (%zu bytes) to client.", w->id, w->response.data->len); break; - case WEB_CLIENT_MODE_POST: - case WEB_CLIENT_MODE_GET: - case WEB_CLIENT_MODE_PUT: - case WEB_CLIENT_MODE_DELETE: + case HTTP_REQUEST_MODE_POST: + case HTTP_REQUEST_MODE_GET: + case HTTP_REQUEST_MODE_PUT: + case HTTP_REQUEST_MODE_DELETE: netdata_log_debug(D_WEB_CLIENT, "%llu: Done preparing the response. Sending data (%zu bytes) to client.", w->id, w->response.data->len); break; - case WEB_CLIENT_MODE_FILECOPY: + case HTTP_REQUEST_MODE_FILECOPY: if(w->response.rlen) { netdata_log_debug(D_WEB_CLIENT, "%llu: Done preparing the response. Will be sending data file of %zu bytes to client.", w->id, w->response.rlen); web_client_enable_wait_receive(w); @@ -2104,7 +1893,7 @@ ssize_t web_client_send_deflate(struct web_client *w) if(t < 0) return t; } - if(w->mode == WEB_CLIENT_MODE_FILECOPY && web_client_has_wait_receive(w) && w->response.rlen && w->response.rlen > w->response.data->len) { + if(w->mode == HTTP_REQUEST_MODE_FILECOPY && web_client_has_wait_receive(w) && w->response.rlen && w->response.rlen > w->response.data->len) { // we have to wait, more data will come netdata_log_debug(D_WEB_CLIENT, "%llu: Waiting for more data to become available.", w->id); web_client_disable_wait_send(w); @@ -2146,8 +1935,8 @@ ssize_t web_client_send_deflate(struct web_client *w) // ask for FINISH if we have all the input int flush = Z_SYNC_FLUSH; - if((w->mode == WEB_CLIENT_MODE_GET || w->mode == WEB_CLIENT_MODE_POST || w->mode == WEB_CLIENT_MODE_PUT || w->mode == WEB_CLIENT_MODE_DELETE) - || (w->mode == WEB_CLIENT_MODE_FILECOPY && !web_client_has_wait_receive(w) && w->response.data->len == w->response.rlen)) { + if((w->mode == HTTP_REQUEST_MODE_GET || w->mode == HTTP_REQUEST_MODE_POST || w->mode == HTTP_REQUEST_MODE_PUT || w->mode == HTTP_REQUEST_MODE_DELETE) + || (w->mode == HTTP_REQUEST_MODE_FILECOPY && !web_client_has_wait_receive(w) && w->response.data->len == w->response.rlen)) { flush = Z_FINISH; netdata_log_debug(D_DEFLATE, "%llu: Requesting Z_FINISH, if possible.", w->id); } @@ -2212,7 +2001,7 @@ ssize_t web_client_send(struct web_client *w) { // A. we have done everything // B. we temporarily have nothing to send, waiting for the buffer to be filled by ifd - if(w->mode == WEB_CLIENT_MODE_FILECOPY && web_client_has_wait_receive(w) && w->response.rlen && w->response.rlen > w->response.data->len) { + if(w->mode == HTTP_REQUEST_MODE_FILECOPY && web_client_has_wait_receive(w) && w->response.rlen && w->response.rlen > w->response.data->len) { // we have to wait, more data will come netdata_log_debug(D_WEB_CLIENT, "%llu: Waiting for more data to become available.", w->id); web_client_disable_wait_send(w); @@ -2301,7 +2090,7 @@ ssize_t web_client_read_file(struct web_client *w) ssize_t web_client_receive(struct web_client *w) { - if(unlikely(w->mode == WEB_CLIENT_MODE_FILECOPY)) + if(unlikely(w->mode == HTTP_REQUEST_MODE_FILECOPY)) return web_client_read_file(w); ssize_t bytes; @@ -2365,7 +2154,7 @@ void web_client_decode_path_and_query_string(struct web_client *w, const char *p // do not overwrite this if it is already filled buffer_strcat(w->url_as_received, path_and_query_string); - if(w->mode == WEB_CLIENT_MODE_STREAM) { + if(w->mode == HTTP_REQUEST_MODE_STREAM) { // in stream mode, there is no path url_decode_r(buffer, path_and_query_string, NETDATA_WEB_REQUEST_URL_SIZE + 1); diff --git a/web/server/web_client.h b/web/server/web_client.h index 92d512e3d66898..051f780c2caaa3 100644 --- a/web/server/web_client.h +++ b/web/server/web_client.h @@ -12,16 +12,6 @@ extern int web_enable_gzip, web_gzip_level, web_gzip_strategy; extern int respect_web_browser_do_not_track_policy; extern char *web_x_frame_options; -typedef enum web_client_mode { - WEB_CLIENT_MODE_GET = 0, - WEB_CLIENT_MODE_POST = 1, - WEB_CLIENT_MODE_FILECOPY = 2, - WEB_CLIENT_MODE_OPTIONS = 3, - WEB_CLIENT_MODE_STREAM = 4, - WEB_CLIENT_MODE_PUT = 5, - WEB_CLIENT_MODE_DELETE = 6, -} WEB_CLIENT_MODE; - typedef enum { HTTP_VALIDATION_OK, HTTP_VALIDATION_NOT_SUPPORTED, @@ -52,6 +42,7 @@ typedef enum web_client_flags { WEB_CLIENT_FLAG_PATH_IS_V2 = (1 << 15), // v2 dashboard found on the path WEB_CLIENT_FLAG_PATH_HAS_TRAILING_SLASH = (1 << 16), // the path has a trailing hash WEB_CLIENT_FLAG_PATH_HAS_FILE_EXTENSION = (1 << 17), // the path ends with a filename extension + WEB_CLIENT_FLAG_PROGRESS_TRACKING = (1 << 18), // when set we track the progress of this transaction } WEB_CLIENT_FLAGS; #define WEB_CLIENT_FLAG_PATH_WITH_VERSION (WEB_CLIENT_FLAG_PATH_IS_V0|WEB_CLIENT_FLAG_PATH_IS_V1|WEB_CLIENT_FLAG_PATH_IS_V2) @@ -139,8 +130,9 @@ struct web_client { uuid_t transaction; WEB_CLIENT_FLAGS flags; // status flags for the client - WEB_CLIENT_MODE mode; // the operational mode of the client - WEB_CLIENT_ACL acl; // the access list of the client + HTTP_REQUEST_MODE mode; // the operational mode of the client + HTTP_ACL acl; // the access list of the client + HTTP_ACCESS access; // the access level of the client int port_acl; // the operations permitted on the port the client connected to size_t header_parse_tries; size_t header_parse_last_size; @@ -160,7 +152,8 @@ struct web_client { // THESE NEED TO BE FREED char *auth_bearer_token; // the Bearer auth token (if sent) char *server_host; // the Host: header - char *forwarded_host; // the X-Forwarded-For: header + char *forwarded_host; // the X-Forwarded-Host: header + char *forwarded_for; // the X-Forwarded-For: header char *origin; // the Origin: header char *user_agent; // the User-Agent: header diff --git a/web/server/web_client_cache.c b/web/server/web_client_cache.c index 364fc76b36ebde..2a4256b9b1ed85 100644 --- a/web/server/web_client_cache.c +++ b/web/server/web_client_cache.c @@ -114,7 +114,7 @@ struct web_client *web_client_get_from_cache(void) { // initialize it w->use_count++; - w->mode = WEB_CLIENT_MODE_GET; + w->mode = HTTP_REQUEST_MODE_GET; memset(w->transaction, 0, sizeof(w->transaction)); return w; diff --git a/web/server/web_server.c b/web/server/web_server.c index 8ac0c6595fc19e..ed88e700d1c329 100644 --- a/web/server/web_server.c +++ b/web/server/web_server.c @@ -41,13 +41,13 @@ void debug_sockets() { int i; for(i = 0 ; i < (int)api_sockets.opened ; i++) { - buffer_strcat(wb, (api_sockets.fds_acl_flags[i] & WEB_CLIENT_ACL_NOCHECK)?"NONE ":""); - buffer_strcat(wb, (api_sockets.fds_acl_flags[i] & WEB_CLIENT_ACL_DASHBOARD)?"dashboard ":""); - buffer_strcat(wb, (api_sockets.fds_acl_flags[i] & WEB_CLIENT_ACL_REGISTRY)?"registry ":""); - buffer_strcat(wb, (api_sockets.fds_acl_flags[i] & WEB_CLIENT_ACL_BADGE)?"badges ":""); - buffer_strcat(wb, (api_sockets.fds_acl_flags[i] & WEB_CLIENT_ACL_MGMT)?"management ":""); - buffer_strcat(wb, (api_sockets.fds_acl_flags[i] & WEB_CLIENT_ACL_STREAMING)?"streaming ":""); - buffer_strcat(wb, (api_sockets.fds_acl_flags[i] & WEB_CLIENT_ACL_NETDATACONF)?"netdata.conf ":""); + buffer_strcat(wb, (api_sockets.fds_acl_flags[i] & HTTP_ACL_NOCHECK) ? "NONE " : ""); + buffer_strcat(wb, (api_sockets.fds_acl_flags[i] & HTTP_ACL_DASHBOARD) ? "dashboard " : ""); + buffer_strcat(wb, (api_sockets.fds_acl_flags[i] & HTTP_ACL_REGISTRY) ? "registry " : ""); + buffer_strcat(wb, (api_sockets.fds_acl_flags[i] & HTTP_ACL_BADGE) ? "badges " : ""); + buffer_strcat(wb, (api_sockets.fds_acl_flags[i] & HTTP_ACL_MGMT) ? "management " : ""); + buffer_strcat(wb, (api_sockets.fds_acl_flags[i] & HTTP_ACL_STREAMING) ? "streaming " : ""); + buffer_strcat(wb, (api_sockets.fds_acl_flags[i] & HTTP_ACL_NETDATACONF) ? "netdata.conf " : ""); netdata_log_debug(D_WEB_CLIENT, "Socket fd %d name '%s' acl_flags: %s", i, api_sockets.fds_names[i], @@ -91,37 +91,37 @@ SIMPLE_PATTERN *web_allow_netdataconf_from = NULL; int web_allow_netdataconf_dns; void web_client_update_acl_matches(struct web_client *w) { - w->acl = WEB_CLIENT_ACL_NONE; + w->acl = HTTP_ACL_NONE; if (!web_allow_dashboard_from || connection_allowed(w->ifd, w->client_ip, w->client_host, sizeof(w->client_host), web_allow_dashboard_from, "dashboard", web_allow_dashboard_dns)) - w->acl |= WEB_CLIENT_ACL_DASHBOARD; + w->acl |= HTTP_ACL_DASHBOARD; if (!web_allow_registry_from || connection_allowed(w->ifd, w->client_ip, w->client_host, sizeof(w->client_host), web_allow_registry_from, "registry", web_allow_registry_dns)) - w->acl |= WEB_CLIENT_ACL_REGISTRY; + w->acl |= HTTP_ACL_REGISTRY; if (!web_allow_badges_from || connection_allowed(w->ifd, w->client_ip, w->client_host, sizeof(w->client_host), web_allow_badges_from, "badges", web_allow_badges_dns)) - w->acl |= WEB_CLIENT_ACL_BADGE; + w->acl |= HTTP_ACL_BADGE; if (!web_allow_mgmt_from || connection_allowed(w->ifd, w->client_ip, w->client_host, sizeof(w->client_host), web_allow_mgmt_from, "management", web_allow_mgmt_dns)) - w->acl |= WEB_CLIENT_ACL_MGMT; + w->acl |= HTTP_ACL_MGMT; if (!web_allow_streaming_from || connection_allowed(w->ifd, w->client_ip, w->client_host, sizeof(w->client_host), web_allow_streaming_from, "streaming", web_allow_streaming_dns)) - w->acl |= WEB_CLIENT_ACL_STREAMING; + w->acl |= HTTP_ACL_STREAMING; if (!web_allow_netdataconf_from || connection_allowed(w->ifd, w->client_ip, w->client_host, sizeof(w->client_host), web_allow_netdataconf_from, "netdata.conf", web_allow_netdataconf_dns)) - w->acl |= WEB_CLIENT_ACL_NETDATACONF; + w->acl |= HTTP_ACL_NETDATACONF; w->acl &= w->port_acl; } @@ -139,6 +139,8 @@ void web_server_log_connection(struct web_client *w, const char *msg) { #endif ND_LOG_FIELD_TXT(NDF_SRC_IP, w->client_ip), ND_LOG_FIELD_TXT(NDF_SRC_PORT, w->client_port), + ND_LOG_FIELD_TXT(NDF_SRC_FORWARDED_HOST, w->forwarded_host), + ND_LOG_FIELD_TXT(NDF_SRC_FORWARDED_FOR, w->forwarded_for), ND_LOG_FIELD_END(), }; ND_LOG_STACK_PUSH(lgs);