Merge pull request #982 from criblio/feature/970-binary-console-config

Add configuration control for binary redaction of console to scope.yml

cli/libscope/config.go

@@ -39,10 +39,11 @@ type ScopePayloadConfig struct {
 
 // ScopeWatchConfig represents a watch configuration
 type ScopeWatchConfig struct {
-	WatchType string `mapstructure:"type" json:"type" yaml:"type"`
-	Name      string `mapstructure:"name" json:"name" yaml:"name"`
-	Field     string `mapstructure:"field,omitempty" json:"field,omitempty" yaml:"field,omitempty"`
-	Value     string `mapstructure:"value" json:"value" yaml:"value"`
+	WatchType   string `mapstructure:"type" json:"type" yaml:"type"`
+	Name        string `mapstructure:"name" json:"name" yaml:"name"`
+	Field       string `mapstructure:"field,omitempty" json:"field,omitempty" yaml:"field,omitempty"`
+	Value       string `mapstructure:"value" json:"value" yaml:"value"`
+	AllowBinary bool   `mapstructure:"allowbinary,omitempty" json:"allowbinary,omitempty" yaml:"allowbinary,omitempty"`
 }
 
 // ScopeLibscopeConfig represents how to configure libscope

cli/run/scopeconfig.go

@@ -49,9 +49,10 @@ func (c *Config) SetDefault() error {
 					Value:     ".*",
 				},
 				{
-					WatchType: "console",
-					Name:      "(stdout|stderr)",
-					Value:     ".*",
+					WatchType:   "console",
+					Name:        "(stdout|stderr)",
+					Value:       ".*",
+					AllowBinary: true,
 				},
 				{
 					WatchType: "net",

cli/run/setup_test.go

@@ -170,6 +170,7 @@ event:
   - type: console
     name: (stdout|stderr)
     value: .*
+    allowbinary: true
   - type: net
     name: .*
     field: .*

conf/scope.yml

@@ -373,9 +373,13 @@ event:
     # category. The regular expressions can be set with
     # $SCOPE_EVENT_CONSOLE_NAME and $SCOPE_EVENT_CONSOLE_VALUE.
     #
+    # Set $SCOPE_ALLOW_BINARY_CONSOLE to true or false to allow or disallow
+    # emiting binary data for console events.
+    #
     - type: console
       name: (stdout)|(stderr) # matches the output stream
       value: .*               # matches data written
+      allowbinary: true
 
     # The net category includes open and close events on network connections.
     # The name filter is applied to the sourcetype in the event which are only

docs/schemas/definitions/data.schema.json

@@ -9,6 +9,12 @@
       "description": "IP address(es) of the domain for which the scoped app sent a DNS request.",
       "type": "array"
     },
+    "allowbinary": {
+      "title": "allowbinary",
+      "description": "TBD allowbinary",
+      "type": "string",
+      "enum": ["true", "false"]
+    },
     "binary": {
       "title": "binary",
       "description": "TBD binary",

docs/schemas/event_start_msg.schema.json

@@ -4,7 +4,7 @@
   "type": "object",
   "title": "AppScope Start message",
   "description": "TBD Structure of the start message",
-  "examples": [{"format":"ndjson","info":{"process":{"libscopever":"v1.1.0","pid":35673,"ppid":3390,"gid":1000,"groupname":"test_user","uid":1000,"username":"test_user","hostname":"test_user","procname":"ls","cmd":"ls --color=auto","id":"test_user-ls-ls --color=auto"},"configuration":{"current":{"metric":{"enable":"true","transport":{"type":"udp","host":"127.0.0.1","port":"8125","tls":{"enable":"false","validateserver":"true","cacertpath":""}},"format":{"type":"statsd","statsdprefix":"","statsdmaxlen":512,"verbosity":4},"watch":[{"type":"statsd"}]},"libscope":{"log":{"level":"info","transport":{"type":"file","path":"/tmp/scope.log","buffering":"line"}},"configevent":"true","summaryperiod":10,"commanddir":"/tmp"},"event":{"enable":"true","transport":{"type":"tcp","host":"127.0.0.1","port":"9109","tls":{"enable":"false","validateserver":"true","cacertpath":""}},"format":{"type":"ndjson","maxeventpersec":10000,"enhancefs":"true"},"watch":[{"type":"file","name":"(\\/logs?\\/)|(\\.log$)|(\\.log[.\\d])","field":".*","value":".*"},{"type":"console","name":"(stdout)|(stderr)","field":".*","value":".*"},{"type":"http","name":".*","field":".*","value":".*","headers":[]},{"type":"net","name":".*","field":".*","value":".*"},{"type":"fs","name":".*","field":".*","value":".*"},{"type":"dns","name":".*","field":".*","value":".*"}]},"payload":{"enable":"false","dir":"/tmp"},"tags":{},"protocol":[],"cribl":{"enable":"false","transport":{"type":"edge"},"authtoken":""}}},"environment":{}}}],
+  "examples": [{"format":"ndjson","info":{"process":{"libscopever":"v1.1.0","pid":35673,"ppid":3390,"gid":1000,"groupname":"test_user","uid":1000,"username":"test_user","hostname":"test_user","procname":"ls","cmd":"ls --color=auto","id":"test_user-ls-ls --color=auto"},"configuration":{"current":{"metric":{"enable":"true","transport":{"type":"udp","host":"127.0.0.1","port":"8125","tls":{"enable":"false","validateserver":"true","cacertpath":""}},"format":{"type":"statsd","statsdprefix":"","statsdmaxlen":512,"verbosity":4},"watch":[{"type":"statsd"}]},"libscope":{"log":{"level":"info","transport":{"type":"file","path":"/tmp/scope.log","buffering":"line"}},"configevent":"true","summaryperiod":10,"commanddir":"/tmp"},"event":{"enable":"true","transport":{"type":"tcp","host":"127.0.0.1","port":"9109","tls":{"enable":"false","validateserver":"true","cacertpath":""}},"format":{"type":"ndjson","maxeventpersec":10000,"enhancefs":"true"},"watch":[{"type":"file","name":"(\\/logs?\\/)|(\\.log$)|(\\.log[.\\d])","field":".*","value":".*"},{"type":"console","name":"(stdout)|(stderr)","field":".*","value":".*","allowbinary":"true"},{"type":"http","name":".*","field":".*","value":".*","headers":[]},{"type":"net","name":".*","field":".*","value":".*"},{"type":"fs","name":".*","field":".*","value":".*"},{"type":"dns","name":".*","field":".*","value":".*"}]},"payload":{"enable":"false","dir":"/tmp"},"tags":{},"protocol":[],"cribl":{"enable":"false","transport":{"type":"edge"},"authtoken":""}}},"environment":{}}}],
   "required": [
     "format",
     "info"
@@ -335,6 +335,9 @@
                             },
                             "value": {
                               "$ref": "definitions/data.schema.json#/$defs/value"
+                            },
+                            "allowbinary": {
+                              "$ref": "definitions/data.schema.json#/$defs/allowbinary"
                             }
                           }
                         },

src/cfg.c

@@ -56,6 +56,7 @@ struct _config_t
         unsigned src[CFG_SRC_MAX];
         size_t numHeaders;
         header_extract_t **hextract;
+        unsigned allowbinaryconsole;
     } evt;
 
     struct {
@@ -204,6 +205,7 @@ cfgCreateDefault()
 
     c->evt.hextract = DEFAULT_SRC_HTTP_HEADER;
     c->evt.numHeaders = 0;
+    c->evt.allowbinaryconsole = DEFAULT_ALLOW_BINARY_CONSOLE;
 
     which_transport_t tp;
     for (tp=CFG_MTC; tp<CFG_WHICH_MAX; tp++) {
@@ -461,6 +463,12 @@ cfgEvtFormatSourceEnabled(config_t *cfg, watch_t src)
     return srcEnabledDefault[CFG_SRC_FILE];
 }
 
+unsigned
+cfgEvtAllowBinaryConsole(config_t *cfg)
+{
+    return (cfg) ? cfg->evt.allowbinaryconsole : DEFAULT_ALLOW_BINARY_CONSOLE;
+}
+
 unsigned
 cfgMtcVerbosity(config_t* cfg)
 {
@@ -840,6 +848,13 @@ cfgEvtFormatSourceEnabledSet(config_t* cfg, watch_t src, unsigned val)
     cfg->evt.src[src] = val;
 }
 
+void
+cfgEvtAllowBinaryConsoleSet(config_t *cfg, unsigned val)
+{
+    if (!cfg || val < 0 || val > 1) return;
+    cfg->evt.allowbinaryconsole = val;
+}
+
 void
 cfgTransportTypeSet(config_t* cfg, which_transport_t t, cfg_transport_t type)
 {

src/cfg.h

@@ -47,6 +47,7 @@ cfg_log_level_t     cfgLogLevel(config_t*);
 unsigned int        cfgPayEnable(config_t*);
 const char *        cfgPayDir(config_t*);
 const char *        cfgEvtFormatHeader(config_t *, int);
+unsigned            cfgEvtAllowBinaryConsole(config_t *);
 unsigned            cfgLogStreamEnable(config_t *);
 unsigned            cfgLogStreamCloud(config_t *);
 size_t              cfgEvtFormatNumHeaders(config_t *);
@@ -84,6 +85,7 @@ void                cfgLogLevelSet(config_t*, cfg_log_level_t);
 void                cfgPayEnableSet(config_t*, unsigned int);
 void                cfgPayDirSet(config_t*, const char *);
 void                cfgEvtFormatHeaderSet(config_t *, const char *);
+void                cfgEvtAllowBinaryConsoleSet(config_t *, unsigned);
 void                cfgLogStreamEnableSet(config_t *, unsigned);
 void                cfgLogStreamCloudSet(config_t *, unsigned);
 void                cfgAuthTokenSet(config_t *, const char *);

src/cfgutils.c

@@ -65,6 +65,7 @@
 #define FIELD_NODE                   "field"
 #define VALUE_NODE                   "value"
 #define EX_HEADERS                   "headers"
+#define ALLOW_BINARY_NODE            "allowbinary"
 
 #define PAYLOAD_NODE         "payload"
 #define ENABLE_NODE              "enable"
@@ -169,6 +170,7 @@ void cfgEvtEnableSetFromStr(config_t*, const char*);
 void cfgEventFormatSetFromStr(config_t*, const char*);
 void cfgEvtRateLimitSetFromStr(config_t*, const char*);
 void cfgEnhanceFsSetFromStr(config_t*, const char*);
+void cfgAllowBinaryConsoleSetFromStr(config_t *, const char *);
 void cfgEvtFormatValueFilterSetFromStr(config_t*, watch_t, const char*);
 void cfgEvtFormatFieldFilterSetFromStr(config_t*, watch_t, const char*);
 void cfgEvtFormatNameFilterSetFromStr(config_t*, watch_t, const char*);
@@ -519,6 +521,8 @@ processEnvStyleInput(config_t *cfg, const char *env_line)
         cfgEvtRateLimitSetFromStr(cfg, value);
     } else if (!scope_strcmp(env_name, "SCOPE_ENHANCE_FS")) {
         cfgEnhanceFsSetFromStr(cfg, value);
+    } else if (!scope_strcmp(env_name, "SCOPE_ALLOW_BINARY_CONSOLE")) {
+        cfgAllowBinaryConsoleSetFromStr(cfg, value);
     } else if (!scope_strcmp(env_name, "SCOPE_EVENT_LOGFILE_NAME")) {
         cfgEvtFormatNameFilterSetFromStr(cfg, CFG_SRC_FILE, value);
     } else if (!scope_strcmp(env_name, "SCOPE_EVENT_CONSOLE_NAME")) {
@@ -753,6 +757,13 @@ cfgEnhanceFsSetFromStr(config_t* cfg, const char* value)
     cfgEnhanceFsSet(cfg, strToVal(boolMap, value));
 }
 
+void
+cfgAllowBinaryConsoleSetFromStr(config_t *cfg, const char *value)
+{
+    if (!cfg || !value) return;
+    cfgEvtAllowBinaryConsoleSet(cfg, strToVal(boolMap, value));
+}
+
 void
 cfgEvtFormatValueFilterSetFromStr(config_t* cfg, watch_t src, const char* value)
 {
@@ -1411,6 +1422,16 @@ processEvtWatchHeader(config_t *config, yaml_document_t *doc, yaml_node_t *node)
     }
 }
 
+static void
+processEvtWatchBinary(config_t *config, yaml_document_t *doc, yaml_node_t *node){
+    // watch binary is only valid for console
+    if (node->type != YAML_SCALAR_NODE || watch_context!= CFG_SRC_CONSOLE) return;
+
+    char* value = stringVal(node);
+    cfgAllowBinaryConsoleSetFromStr(config, value);
+    if (value) scope_free(value);
+}
+
 static int
 isWatchType(yaml_document_t* doc, yaml_node_pair_t* pair)
 {
@@ -1430,6 +1451,7 @@ processSource(config_t* config, yaml_document_t* doc, yaml_node_t* node)
         {YAML_SCALAR_NODE,    FIELD_NODE,           processEvtWatchField},
         {YAML_SCALAR_NODE,    VALUE_NODE,           processEvtWatchValue},
         {YAML_SEQUENCE_NODE,  EX_HEADERS,           processEvtWatchHeader},
+        {YAML_SCALAR_NODE,    ALLOW_BINARY_NODE,    processEvtWatchBinary},
         {YAML_NO_NODE,        NULL,                 NULL}
     };
 
@@ -2343,12 +2365,15 @@ createWatchObjectJson(config_t *cfg, watch_t src)
             for (i = 0; i < numhead; i++) {
                 char *hstr = (char *)cfgEvtFormatHeader(cfg, i);
                 if (hstr) {
-                    cJSON_AddStringToObjLN(headers, "headers", hstr);
+                    cJSON_AddStringToObjLN(headers, EX_HEADERS, hstr);
                 }
             }
         }
 
-        cJSON_AddItemToObject(root, "headers", headers);
+        cJSON_AddItemToObject(root, EX_HEADERS, headers);
+    } else if (src == CFG_SRC_CONSOLE) {
+    if (!cJSON_AddStringToObjLN(root, ALLOW_BINARY_NODE,
+                                  valToStr(boolMap, cfgEvtAllowBinaryConsole(cfg)))) goto err;
     }
 
     return root;
@@ -2756,6 +2781,7 @@ initCtl(config_t *cfg)
     ctlEnhanceFsSet(ctl, cfgEnhanceFs(cfg));
     ctlPayEnableSet(ctl, cfgPayEnable(cfg));
     ctlPayDirSet(ctl,    cfgPayDir(cfg));
+    ctlAllowBinaryConsoleSet(ctl, cfgEvtAllowBinaryConsole(cfg));
 
     return ctl;
 }

src/ctl.c

@@ -610,7 +610,7 @@ ctlCreate()
     }
 
     ctl->enhancefs = DEFAULT_ENHANCE_FS;
-    ctl->allow_binary_console = checkEnv("SCOPE_ALLOW_BINARY_CONSOLE", "true");
+    ctl->allow_binary_console = DEFAULT_ALLOW_BINARY_CONSOLE;
 
     ctl->payload.enable = DEFAULT_PAYLOAD_ENABLE;
     ctl->payload.dir = (DEFAULT_PAYLOAD_DIR) ? scope_strdup(DEFAULT_PAYLOAD_DIR) : NULL;
@@ -1297,6 +1297,13 @@ ctlPayDirSet(ctl_t *ctl, const char *dir)
     ctl->payload.dir = scope_strdup(dir);
 }
 
+void
+ctlAllowBinaryConsoleSet(ctl_t *ctl, unsigned val)
+{
+    if (!ctl || val < 0 || val > 1) return;
+    ctl->allow_binary_console = val;
+}
+
 
 uint64_t
 ctlGetEvent(ctl_t *ctl)

src/ctl.h

@@ -139,6 +139,7 @@ unsigned int    ctlPayEnable(ctl_t *);
 void            ctlPayEnableSet(ctl_t *, unsigned int);
 const char *    ctlPayDir(ctl_t *);
 void            ctlPayDirSet(ctl_t *, const char *);
+void            ctlAllowBinaryConsoleSet(ctl_t *, unsigned);
 
 // Retrieve events
 uint64_t   ctlGetEvent(ctl_t *);

src/scopetypes.h

@@ -112,6 +112,7 @@ typedef unsigned int bool;
 #define DEFAULT_SRC_NET_NAME ".*"
 #define DEFAULT_SRC_FS_NAME ".*"
 #define DEFAULT_SRC_DNS_NAME ".*"
+#define DEFAULT_ALLOW_BINARY_CONSOLE TRUE
 #define DEFAULT_MTC_IPPORT_VERBOSITY 1
 
 #define DEFAULT_SRC_FILE TRUE
@@ -186,7 +187,6 @@ typedef unsigned int bool;
 //    SCOPE_ALLOW_CONSTRUCT_DBG      allows debug inside the constructor
 //    SCOPE_ERROR_SIGNAL_HANDLER     allows to register SIGSEGV&SIGBUS handler
 //    SCOPE_QUEUE_LENGTH             override default circular buffer sizes
-//    SCOPE_ALLOW_BINARY_CONSOLE     "true" outputs all console data, always
 
 #define SCOPE_PID_ENV "SCOPE_PID"
 #define PRESERVE_PERF_REPORTING "SCOPE_PERF_PRESERVE"

test/cfgtest.c

@@ -29,6 +29,7 @@ verifyDefaults(config_t* config)
     assert_int_equal       (cfgEventFormat(config), DEFAULT_CTL_FORMAT);
     assert_int_equal       (cfgEvtRateLimit(config), DEFAULT_MAXEVENTSPERSEC);
     assert_int_equal       (cfgEnhanceFs(config), DEFAULT_ENHANCE_FS);
+    assert_int_equal       (cfgEvtAllowBinaryConsole(config), DEFAULT_ALLOW_BINARY_CONSOLE);
     assert_string_equal    (cfgEvtFormatValueFilter(config, CFG_SRC_FILE), DEFAULT_SRC_FILE_VALUE);
     assert_string_equal    (cfgEvtFormatValueFilter(config, CFG_SRC_CONSOLE), DEFAULT_SRC_CONSOLE_VALUE);
     assert_string_equal    (cfgEvtFormatValueFilter(config, CFG_SRC_SYSLOG), DEFAULT_SRC_SYSLOG_VALUE);
@@ -302,6 +303,23 @@ cfgEnhanceFsSetAndGet(void** state)
     cfgDestroy(&config);
 }
 
+static void
+cfgEvtAllowBinaryConsoleSetAndGet(void** state)
+{
+    config_t *config = cfgCreateDefault();
+    cfgEvtAllowBinaryConsoleSet(config, TRUE);
+    assert_int_equal(cfgEvtAllowBinaryConsole(config), TRUE);
+
+    cfgEvtAllowBinaryConsoleSet(config, FALSE);
+    assert_int_equal(cfgEvtAllowBinaryConsole(config), FALSE);
+
+    // 2 is outside of allowed range; should be ignored.
+    cfgEvtAllowBinaryConsoleSet(config, 2);
+    assert_int_equal(cfgEvtAllowBinaryConsole(config), FALSE);
+
+    cfgDestroy(&config);
+}
+
 typedef struct
 {
     watch_t   src;
@@ -677,6 +695,7 @@ main(int argc, char* argv[])
         cmocka_unit_test(cfgEventFormatSetAndGet),
         cmocka_unit_test(cfgEvtRateLimitSetAndGet),
         cmocka_unit_test(cfgEnhanceFsSetAndGet),
+        cmocka_unit_test(cfgEvtAllowBinaryConsoleSetAndGet),
 
         cmocka_unit_test_prestate(cfgEvtFormatValueFilterSetAndGet, &log),
         cmocka_unit_test_prestate(cfgEvtFormatValueFilterSetAndGet, &con),

test/ctltest.c

@@ -774,6 +774,8 @@ ctlSendLogConsoleNoneAsciiData(void **state)
     assert_non_null(ctl);
     bool b_res = ctlEvtSourceEnabled(ctl, CFG_SRC_CONSOLE);
     assert_true(b_res);
+    // with ALLOW_BINARY false, verify that the binary_data_event_msg appears.
+    ctlAllowBinaryConsoleSet(ctl, FALSE);
     allow_copy_buf_data(TRUE);
 
     ctlSendLog(ctl, STDOUT_FILENO, console_path, non_basic_ascii_text, strlen(non_basic_ascii_text), 0, &proc);
@@ -787,7 +789,7 @@ ctlSendLogConsoleNoneAsciiData(void **state)
     // and verify that the binary_data_event_msg does *not* appear.
     memset(cbuf_data, '\0', sizeof(cbuf_data));
 
-    setenv("SCOPE_ALLOW_BINARY_CONSOLE", "true", 1);
+    ctlAllowBinaryConsoleSet(ctl, TRUE);
     ctl = ctlCreate();
     assert_non_null(ctl);
     b_res = ctlEvtSourceEnabled(ctl, CFG_SRC_CONSOLE);
@@ -798,7 +800,6 @@ ctlSendLogConsoleNoneAsciiData(void **state)
     assert_string_not_equal(binary_data_event_msg, val);
     assert_string_equal(non_basic_ascii_text, val);
     ctlDestroy(&ctl);
-    unsetenv("SCOPE_ALLOW_BINARY_CONSOLE");
 
     scope_free(non_basic_ascii_text);
     allow_copy_buf_data(FALSE);

test/integration/cli/expected.yml

@@ -39,6 +39,7 @@ event:
   - type: console
     name: (stdout|stderr)
     value: .*
+    allowbinary: true
   - type: net
     name: .*
     field: .*

test/integration/console/scope-test

@@ -77,12 +77,15 @@ fi
 endtest
 
 #
-# cat binary file
+# cat binary file with ALLOW_BINARY env false
+#   (ensure the $BINARY_MSG appears)
 #
 
 starttest cat_binary
 
+export SCOPE_ALLOW_BINARY_CONSOLE=false
 ldscope cat /bin/bash > /dev/null
+unset SCOPE_ALLOW_BINARY_CONSOLE
 evaltest
 
 grep -- "$BINARY_MSG" $EVT_FILE > /dev/null
@@ -92,7 +95,7 @@ endtest
 
 
 #
-# cat binary file with ALLOW_BINARY env set
+# cat binary file with ALLOW_BINARY env true
 #   (ensure the $BINARY_MSG does not appear)
 #
 starttest cat_binary_with_ALLOW_BINARY_env
@@ -109,6 +112,13 @@ fi
 
 endtest
 
+#########################################################################
+#
+#     For all remaining tests, set SCOPE_ALLOW_BINARY_CONSOLE false
+#
+#########################################################################
+export SCOPE_ALLOW_BINARY_CONSOLE=false
+
 
 #
 # curl png