diff --git a/test/falco_test.py b/test/falco_test.py index f08030b5a8d..1e0da0d6ddd 100644 --- a/test/falco_test.py +++ b/test/falco_test.py @@ -77,6 +77,10 @@ def setUp(self): else: self.stderr_not_contains = [self.stderr_not_contains] + self.validate_ok = self.params.get('validate_ok', '*', default='') + self.validate_warnings = self.params.get('validate_warnings', '*', default='') + self.validate_errors = self.params.get('validate_errors', '*', default='') + self.exit_status = self.params.get('exit_status', '*', default=0) self.should_detect = self.params.get('detect', '*', default=False) self.check_detection_counts = self.params.get('check_detection_counts', '*', default=True) @@ -105,6 +109,9 @@ def setUp(self): if self.validate_rules_file == False: self.validate_rules_file = [] else: + # Always enable json output when validating rules + # files. Makes parsing errors/warnings easier + self.json_output = True if not isinstance(self.validate_rules_file, list): self.validate_rules_file = [self.validate_rules_file] @@ -153,13 +160,6 @@ def setUp(self): detect_counts[key] = value self.detect_counts = detect_counts - self.rules_warning = self.params.get( - 'rules_warning', '*', default=False) - if self.rules_warning == False: - self.rules_warning = set() - else: - self.rules_warning = set(self.rules_warning) - # Maps from rule name to set of evttypes self.rules_events = self.params.get('rules_events', '*', default=False) if self.rules_events == False: @@ -265,22 +265,6 @@ def tearDown(self): if self.package != 'None': self.uninstall_package() - def check_rules_warnings(self, res): - - found_warning = set() - - for match in re.finditer('Rule ([^:]+): warning \(([^)]+)\):', res.stderr.decode("utf-8")): - rule = match.group(1) - warning = match.group(2) - found_warning.add(rule) - - self.log.debug("Expected warning rules: {}".format(self.rules_warning)) - self.log.debug("Actual warning rules: {}".format(found_warning)) - - if found_warning != self.rules_warning: - self.fail("Expected rules with warnings {} does not match actual rules with warnings {}".format( - self.rules_warning, found_warning)) - def check_rules_events(self, res): found_events = {} @@ -376,7 +360,68 @@ def check_outputs(self): return True - def check_json_output(self, res): + def get_validate_json(self, res): + if self.validate_json is None: + # The first line of stdout should be the validation result as json + self.validate_json = json.loads(res.stdout.decode("utf-8").partition('\n')[0]) + return self.validate_json + + def check_validate_ok(self, res): + if self.validate_ok != '': + vobj = self.get_validate_json(res) + for expected in self.validate_ok: + found = False + for vres in vobj["falco_load_results"]: + if vres["successful"] and os.path.basename(vres["name"]) == expected: + found = True + break + if not found: + self.fail("Validation json did not contain a successful result for file '{}'".format(expected)) + + def check_validate_warnings(self, res): + if self.validate_warnings != '': + vobj = self.get_validate_json(res) + for warnobj in self.validate_warnings: + found = False + for vres in vobj["falco_load_results"]: + for warning in vres["warnings"]: + if warning["code"] == warnobj["code"]: + if ("message" in warnobj and warning["message"] == warnobj["message"]) or ("message_contains" in warnobj and warnobj["message_contains"] in warning["message"]): + for loc in warning["context"]["locations"]: + if loc["item_type"] == warnobj["item_type"] and loc["item_name"] == warnobj["item_name"]: + found = True + break + if not found: + if "message" in warnobj: + self.fail("Validation json did not contain a warning '{}' for '{}' '{}' with message '{}'".format( + warnobj["code"], warnobj["item_type"], warnobj["item_name"], warnobj["message"])) + else: + self.fail("Validation json did not contain a warning '{}' for '{}' '{}' with message containing '{}'".format( + warnobj["code"], warnobj["item_type"], warnobj["item_name"], warnobj["message_contains"])) + + def check_validate_errors(self, res): + if self.validate_errors != '': + vobj = self.get_validate_json(res) + for errobj in self.validate_errors: + found = False + for vres in vobj["falco_load_results"]: + for error in vres["errors"]: + if error["code"] == errobj["code"]: + if ("message" in errobj and error["message"] == errobj["message"]) or ("message_contains" in errobj and errobj["message_contains"] in error["message"]): + for loc in error["context"]["locations"]: + if loc["item_type"] == errobj["item_type"] and loc["item_name"] == errobj["item_name"]: + found = True + break + if not found: + if "message" in errobj: + self.fail("Validation json did not contain a error '{}' for '{}' '{}' with message '{}'".format( + errobj["code"], errobj["item_type"], errobj["item_name"], errobj["message"])) + else: + self.fail("Validation json did not contain a error '{}' for '{}' '{}' with message containing '{}'".format( + errobj["code"], errobj["item_type"], errobj["item_name"], errobj["message_contains"])) + + + def check_json_event_output(self, res): if self.json_output: # Just verify that any lines starting with '{' are valid json objects. # Doesn't do any deep inspection of the contents. @@ -578,6 +623,8 @@ def test(self): # This sets falco_binary_path as a side-effect. self.install_package() + self.validate_json = None + trace_arg = self.trace_file if self.trace_file: @@ -644,18 +691,22 @@ def test(self): self.error("Falco command \"{}\" exited with unexpected return value {} (!= {})".format( cmd, res.exit_status, self.exit_status)) + self.check_validate_ok(res) + self.check_validate_errors(res) + self.check_validate_warnings(res) + # No need to check any outputs if the falco process exited abnormally. if res.exit_status != 0: return - self.check_rules_warnings(res) if len(self.rules_events) > 0: self.check_rules_events(res) if len(self.validate_rules_file) == 0 and self.check_detection_counts: self.check_detections(res) if len(self.detect_counts) > 0: self.check_detections_by_rule(res) - self.check_json_output(res) + if not self.validate_rules_file: + self.check_json_event_output(res) self.check_outputs() self.check_output_strictly_contains(res) self.check_grpc() diff --git a/test/falco_tests.yaml b/test/falco_tests.yaml index e38ff43286a..51442cba4de 100644 --- a/test/falco_tests.yaml +++ b/test/falco_tests.yaml @@ -20,22 +20,55 @@ trace_files: !mux builtin_rules_no_warnings: detect: False trace_file: trace_files/empty.scap - rules_warning: False + # The rules_events part of this test was mistakenly disabled when + # generic events (e.g. k8s_audit support) was added (#1715). + # The implementation no longer prints messages of the form: + # "Event types for rule (): () + # And without that output, none of the checks below rules_events + # are considered. + # XXX/mstemm add it back test_warnings: detect: False trace_file: trace_files/empty.scap - rules_file: rules/falco_rules_warnings.yaml - rules_warning: - - no_evttype - - evttype_not_equals - - leading_not - - not_equals_at_end - - not_at_end - - not_equals_and_not - - leading_in_not_equals_at_evttype - - not_with_evttypes - - not_with_evttypes_addl + validate_rules_file: rules/falco_rules_warnings.yaml + validate_warnings: + - item_type: rule + item_name: no_evttype + code: LOAD_NO_EVTTYPE + message: "Rule matches too many evt.type values. This has a significant performance penalty." + - item_type: rule + item_name: evttype_not_equals + code: LOAD_NO_EVTTYPE + message: "Rule matches too many evt.type values. This has a significant performance penalty." + - item_type: rule + item_name: leading_not + code: LOAD_NO_EVTTYPE + message: "Rule matches too many evt.type values. This has a significant performance penalty." + - item_type: rule + item_name: not_equals_at_end + code: LOAD_NO_EVTTYPE + message: "Rule matches too many evt.type values. This has a significant performance penalty." + - item_type: rule + item_name: not_at_end + code: LOAD_NO_EVTTYPE + message: "Rule matches too many evt.type values. This has a significant performance penalty." + - item_type: rule + item_name: not_equals_and_not + code: LOAD_NO_EVTTYPE + message: "Rule matches too many evt.type values. This has a significant performance penalty." + - item_type: rule + item_name: leading_in_not_equals_at_evttype + code: LOAD_NO_EVTTYPE + message: "Rule matches too many evt.type values. This has a significant performance penalty." + - item_type: rule + item_name: not_with_evttypes + code: LOAD_NO_EVTTYPE + message: "Rule matches too many evt.type values. This has a significant performance penalty." + - item_type: rule + item_name: not_with_evttypes_addl + code: LOAD_NO_EVTTYPE + message: "Rule matches too many evt.type values. This has a significant performance penalty." rules_events: - no_warnings: [execve] - no_evttype: [all] @@ -251,159 +284,149 @@ trace_files: !mux invalid_not_yaml: exit_status: 1 - stdout_is: |+ - 1 errors: - Rules content is not yaml + validate_errors: + - item_type: file + item_name: "" + code: LOAD_ERR_YAML_VALIDATE + message: "Rules content is not yaml" validate_rules_file: - rules/invalid_not_yaml.yaml trace_file: trace_files/cat_write.scap invalid_not_array: exit_status: 1 - stdout_is: |+ - 1 errors: - Rules content is not yaml array of objects + validate_errors: + - item_type: file + item_name: "" + code: LOAD_ERR_YAML_VALIDATE + message: "Rules content is not yaml array of objects" validate_rules_file: - rules/invalid_not_array.yaml trace_file: trace_files/cat_write.scap invalid_array_item_not_object: exit_status: 1 - stdout_is: |+ - 1 errors: - Unexpected element type. Each element should be a yaml associative array. - --- - - foo - --- + validate_errors: + - item_type: item + item_name: "" + code: LOAD_ERR_YAML_VALIDATE + message: "Unexpected element type. Each element should be a yaml associative array." validate_rules_file: - rules/invalid_array_item_not_object.yaml trace_file: trace_files/cat_write.scap invalid_engine_version_not_number: exit_status: 1 - stdout_is: |+ - 1 errors: - Value of required_engine_version must be a number - --- - - required_engine_version: not-a-number - --- + validate_errors: + - item_type: required_engine_version + item_name: "" + code: LOAD_ERR_YAML_VALIDATE + message: "Can't decode YAML scalar value" validate_rules_file: - rules/invalid_engine_version_not_number.yaml trace_file: trace_files/cat_write.scap invalid_yaml_parse_error: exit_status: 1 + validate_errors: + - item_type: file + item_name: "" + code: LOAD_ERR_YAML_PARSE + message: "yaml-cpp: error at line 1, column 11: illegal map value" validate_rules_file: - rules/invalid_yaml_parse_error.yaml trace_file: trace_files/cat_write.scap invalid_list_without_items: exit_status: 1 - stdout_is: |+ - 1 errors: - List must have property items - --- - - list: bad_list - no_items: foo - --- + validate_errors: + - item_type: list + item_name: bad_list + code: LOAD_ERR_YAML_VALIDATE + message: "Item has no mapping for key 'items'" validate_rules_file: - rules/invalid_list_without_items.yaml trace_file: trace_files/cat_write.scap invalid_macro_without_condition: exit_status: 1 - stdout_is: |+ - 1 errors: - Macro must have property condition - --- - - macro: bad_macro - nope: 1 - --- + validate_errors: + - item_type: macro + item_name: bad_macro + code: LOAD_ERR_YAML_VALIDATE + message: "Item has no mapping for key 'condition'" validate_rules_file: - rules/invalid_macro_without_condition.yaml trace_file: trace_files/cat_write.scap invalid_rule_without_output: exit_status: 1 - stdout_is: |+ - 1 errors: - Rule must have properties 'condition', 'output', 'desc', and 'priority' - --- - - rule: no output rule - desc: some desc - condition: evt.type=fork - priority: INFO - --- + validate_errors: + - item_type: rule + item_name: no output rule + code: LOAD_ERR_YAML_VALIDATE + message: "Item has no mapping for key 'output'" validate_rules_file: - rules/invalid_rule_without_output.yaml trace_file: trace_files/cat_write.scap invalid_append_rule_without_condition: exit_status: 1 - stdout_is: |+ - 1 errors: - Appended rule must have exceptions or condition property - --- - - rule: no condition rule - append: true - --- + validate_errors: + - item_type: rule + item_name: no condition rule + code: LOAD_ERR_VALIDATE + message: "Appended rule must have exceptions or condition property" validate_rules_file: - rules/invalid_append_rule_without_condition.yaml trace_file: trace_files/cat_write.scap invalid_append_macro_dangling: exit_status: 1 - stdout_is: |+ - 1 errors: - Macro dangling append has 'append' key but no macro by that name already exists - --- - - macro: dangling append - condition: and evt.type=execve - append: true - --- + validate_errors: + - item_type: macro + item_name: dangling append + code: LOAD_ERR_VALIDATE + message: "Macro has 'append' key but no macro by that name already exists" validate_rules_file: - rules/invalid_append_macro_dangling.yaml trace_file: trace_files/cat_write.scap invalid_list_append_dangling: exit_status: 1 - stdout_is: |+ - 1 errors: - List my_list has 'append' key but no list by that name already exists - --- - - list: my_list - items: [not-cat] - append: true - --- + validate_errors: + - item_type: list + item_name: my_list + code: LOAD_ERR_VALIDATE + message: "List has 'append' key but no list by that name already exists" validate_rules_file: - rules/list_append_failure.yaml trace_file: trace_files/cat_write.scap invalid_rule_append_dangling: exit_status: 1 - stdout_is: |+ - 1 errors: - Rule my_rule has 'append' key but no rule by that name already exists - --- - - rule: my_rule - condition: evt.type=open - append: true - --- + validate_errors: + - item_type: rule + item_name: my_rule + code: LOAD_ERR_VALIDATE + message: "Rule has 'append' key but no rule by that name already exists" validate_rules_file: - rules/rule_append_failure.yaml trace_file: trace_files/cat_write.scap invalid_overwrite_macro: exit_status: 1 - stdout_contains: |+ - .*invalid_base_macro.yaml: Ok - .*invalid_overwrite_macro.yaml: 1 errors: - Compilation error when compiling "foo": Undefined macro 'foo' used in filter. - --- - - macro: some macro - condition: foo - append: false - --- + validate_ok: [invalid_base_macro.yaml] + validate_errors: + - item_type: macro + item_name: some macro + code: LOAD_ERR_VALIDATE + message: "Undefined macro 'foo' used in filter." + validate_warnings: + - item_type: macro + item_name: some macro + code: LOAD_UNUSED_MACRO + message: "Macro not referred to by any other rule/macro" validate_rules_file: - rules/invalid_base_macro.yaml - rules/invalid_overwrite_macro.yaml @@ -411,18 +434,17 @@ trace_files: !mux invalid_append_macro: exit_status: 1 - stdout_contains: |+ - .*invalid_base_macro.yaml: Ok - .*invalid_append_macro.yaml: 1 errors: - Compilation error when compiling "evt.type=execve foo": 17: unexpected token after 'execve', expecting 'or', 'and' - --- - - macro: some macro - condition: evt.type=execve - - - macro: some macro - condition: foo - append: true - --- + validate_ok: [invalid_base_macro.yaml] + validate_errors: + - item_type: macro + item_name: some macro + code: LOAD_ERR_COMPILE_CONDITION + message: "unexpected token after 'execve', expecting 'or', 'and'" + validate_warnings: + - item_type: macro + item_name: some macro + code: LOAD_UNUSED_MACRO + message: "Macro not referred to by any other rule/macro" validate_rules_file: - rules/invalid_base_macro.yaml - rules/invalid_append_macro.yaml @@ -430,49 +452,39 @@ trace_files: !mux invalid_overwrite_macro_multiple_docs: exit_status: 1 - stdout_is: |+ - 1 errors: - Compilation error when compiling "foo": Undefined macro 'foo' used in filter. - --- - - macro: some macro - condition: foo - append: false - --- + validate_errors: + - item_type: macro + item_name: some macro + code: LOAD_ERR_VALIDATE + message: "Undefined macro 'foo' used in filter." + validate_warnings: + - item_type: macro + item_name: some macro + code: LOAD_UNUSED_MACRO + message: "Macro not referred to by any other rule/macro" validate_rules_file: - rules/invalid_overwrite_macro_multiple_docs.yaml trace_file: trace_files/cat_write.scap invalid_append_macro_multiple_docs: exit_status: 1 - stdout_is: |+ - 1 errors: - Compilation error when compiling "evt.type=execve foo": 17: unexpected token after 'execve', expecting 'or', 'and' - --- - - macro: some macro - condition: evt.type=execve - - - macro: some macro - condition: foo - append: true - --- + validate_errors: + - item_type: macro + item_name: some macro + code: LOAD_ERR_COMPILE_CONDITION + message: "unexpected token after 'execve', expecting 'or', 'and'" validate_rules_file: - rules/invalid_append_macro_multiple_docs.yaml trace_file: trace_files/cat_write.scap invalid_overwrite_rule: exit_status: 1 - stdout_contains: |+ - .*invalid_base_rule.yaml: Ok - .*invalid_overwrite_rule.yaml: 1 errors: - Undefined macro 'bar' used in filter. - --- - - rule: some rule - desc: some desc - condition: bar - output: some output - priority: INFO - append: false - --- + validate_ok: [invalid_base_rule.yaml] + validate_errors: + - item_type: rule + item_name: some rule + code: LOAD_ERR_VALIDATE + message: "Undefined macro 'bar' used in filter." validate_rules_file: - rules/invalid_base_rule.yaml - rules/invalid_overwrite_rule.yaml @@ -480,24 +492,12 @@ trace_files: !mux invalid_append_rule: exit_status: 1 - stdout_contains: |+ - .*invalid_base_rule.yaml: Ok - .*invalid_append_rule.yaml: 1 errors: - Compilation error when compiling "evt.type=open bar": 15: unexpected token after 'open', expecting 'or', 'and' - --- - - rule: some rule - desc: some desc - condition: evt.type=open - output: some output - priority: INFO - - - rule: some rule - desc: some desc - condition: bar - output: some output - priority: INFO - append: true - --- + validate_ok: [invalid_base_rule.yaml] + validate_errors: + - item_type: rule + item_name: some rule + code: LOAD_ERR_COMPILE_CONDITION + message: "unexpected token after 'open', expecting 'or', 'and'" validate_rules_file: - rules/invalid_base_rule.yaml - rules/invalid_append_rule.yaml @@ -505,96 +505,66 @@ trace_files: !mux invalid_overwrite_rule_multiple_docs: exit_status: 1 - stdout_is: |+ - 1 errors: - Undefined macro 'bar' used in filter. - --- - - rule: some rule - desc: some desc - condition: bar - output: some output - priority: INFO - append: false - --- + validate_errors: + - item_type: rule + item_name: some rule + code: LOAD_ERR_VALIDATE + message: "Undefined macro 'bar' used in filter." validate_rules_file: - rules/invalid_overwrite_rule_multiple_docs.yaml trace_file: trace_files/cat_write.scap invalid_append_rule_multiple_docs: exit_status: 1 - stdout_contains: |+ - Compilation error when compiling "evt.type=open bar": 15: unexpected token after 'open', expecting 'or', 'and' - --- - - rule: some rule - desc: some desc - condition: evt.type=open - output: some output - priority: INFO - - - rule: some rule - desc: some desc - condition: bar - output: some output - priority: INFO - append: true - --- + validate_errors: + - item_type: rule + item_name: some rule + code: LOAD_ERR_COMPILE_CONDITION + message: "unexpected token after 'open', expecting 'or', 'and'" validate_rules_file: - rules/invalid_append_rule_multiple_docs.yaml trace_file: trace_files/cat_write.scap invalid_missing_rule_name: exit_status: 1 - stdout_is: |+ - 1 errors: - Rule name is empty - --- - - rule: - desc: some desc - condition: evt.type=execve - output: some output - --- + validate_errors: + - item_type: rule + item_name: "" + code: LOAD_ERR_YAML_VALIDATE + message: "Mapping for key 'rule' is empty" validate_rules_file: - rules/invalid_missing_rule_name.yaml trace_file: trace_files/cat_write.scap invalid_missing_list_name: exit_status: 1 - stdout_is: |+ - 1 errors: - List name is empty - --- - - list: - items: [foo] - --- + validate_errors: + - item_type: list + item_name: "" + code: LOAD_ERR_YAML_VALIDATE + message: "Mapping for key 'list' is empty" validate_rules_file: - rules/invalid_missing_list_name.yaml trace_file: trace_files/cat_write.scap invalid_missing_macro_name: exit_status: 1 - stdout_is: |+ - 1 errors: - Macro name is empty - --- - - macro: - condition: evt.type=execve - --- + validate_errors: + - item_type: macro + item_name: "" + code: LOAD_ERR_YAML_VALIDATE + message: "Mapping for key 'macro' is empty" validate_rules_file: - rules/invalid_missing_macro_name.yaml trace_file: trace_files/cat_write.scap invalid_rule_output: exit_status: 1 - stdout_is: |+ - 1 errors: - Invalid output format 'An open was seen %not_a_real_field': 'invalid formatting token not_a_real_field' - --- - - rule: rule_with_invalid_output - desc: A rule with an invalid output field - condition: evt.type=open - output: "An open was seen %not_a_real_field" - priority: WARNING - --- + validate_errors: + - item_type: rule + item_name: rule_with_invalid_output + code: LOAD_ERR_COMPILE_OUTPUT + message: "invalid formatting token not_a_real_field" validate_rules_file: - rules/invalid_rule_output.yaml trace_file: trace_files/cat_write.scap @@ -622,13 +592,13 @@ trace_files: !mux rules_file: - rules/single_rule_enabled_flag.yaml trace_file: trace_files/cat_write.scap - + disabled_rule_using_false_enabled_flag_only: detect: False rules_file: - rules/disabled_rule_using_enabled_flag_only.yaml trace_file: trace_files/cat_write.scap - + enabled_rule_using_false_enabled_flag_only: detect: True detect_level: WARNING @@ -1025,13 +995,6 @@ trace_files: !mux - open_12: 0 - open_13: 0 - list_append_failure: - exit_status: 1 - stderr_contains: "List my_list has 'append' key but no list by that name already exists" - rules_file: - - rules/list_append_failure.yaml - trace_file: trace_files/cat_write.scap - list_append: detect: True detect_level: WARNING @@ -1045,13 +1008,6 @@ trace_files: !mux - rules/list_append_false.yaml trace_file: trace_files/cat_write.scap - macro_append_failure: - exit_status: 1 - stderr_contains: "Macro my_macro has 'append' key but no macro by that name already exists" - rules_file: - - rules/macro_append_failure.yaml - trace_file: trace_files/cat_write.scap - macro_append: detect: True detect_level: WARNING @@ -1065,13 +1021,6 @@ trace_files: !mux - rules/macro_append_false.yaml trace_file: trace_files/cat_write.scap - rule_append_failure: - exit_status: 1 - stderr_contains: "Rule my_rule has 'append' key but no rule by that name already exists" - rules_file: - - rules/rule_append_failure.yaml - trace_file: trace_files/cat_write.scap - rule_append_skipped: detect: False priority: ERROR @@ -1152,10 +1101,18 @@ trace_files: !mux dev_null: 0 trace_file: trace_files/cat_write.scap - skip_unknown_noevt: + validate_skip_unknown_noevt: + validate_warnings: + - item_type: rule + item_name: "Contains Unknown Event And Skipping" + code: LOAD_UNKNOWN_FIELD + message: "filter_check called with nonexistent field proc.nobody" + validate_rules_file: + - rules/skip_unknown_evt.yaml + trace_file: trace_files/cat_write.scap + + detect_skip_unknown_noevt: detect: False - rules_warning: - - Contains Unknown Event And Skipping rules_file: - rules/skip_unknown_evt.yaml trace_file: trace_files/cat_write.scap @@ -1168,41 +1125,34 @@ trace_files: !mux skip_unknown_error: exit_status: 1 - stderr_contains: |+ - Could not load rules file.*skip_unknown_error.yaml: 1 errors: - Rule Contains Unknown Event And Not Skipping: error filter_check called with nonexistent field proc.nobody - --- - - rule: Contains Unknown Event And Not Skipping - desc: Contains an unknown event - condition: proc.nobody=cat - output: Never - skip-if-unknown-filter: false - priority: INFO - --- - rules_file: + validate_errors: + - item_type: rule + item_name: "Contains Unknown Event And Not Skipping" + code: LOAD_ERR_COMPILE_CONDITION + message: "filter_check called with nonexistent field proc.nobody" + validate_rules_file: - rules/skip_unknown_error.yaml trace_file: trace_files/cat_write.scap skip_unknown_unspec_error: exit_status: 1 - stderr_contains: |+ - Could not load rules file .*skip_unknown_unspec.yaml: 1 errors: - Rule Contains Unknown Event And Unspecified: error filter_check called with nonexistent field proc.nobody - --- - - rule: Contains Unknown Event And Unspecified - desc: Contains an unknown event - condition: proc.nobody=cat - output: Never - priority: INFO - --- - rules_file: + validate_errors: + - item_type: rule + item_name: "Contains Unknown Event And Unspecified" + code: LOAD_ERR_COMPILE_CONDITION + message: "filter_check called with nonexistent field proc.nobody" + validate_rules_file: - rules/skip_unknown_unspec.yaml trace_file: trace_files/cat_write.scap engine_version_mismatch: exit_status: 1 - stderr_contains: Rules require engine version 9999999, but engine version is - rules_file: + validate_errors: + - item_type: required_engine_version + item_name: "" + code: LOAD_ERR_VALIDATE + message_contains: "Rules require engine version 9999999, but engine version is" + validate_rules_file: - rules/engine_version_mismatch.yaml trace_file: trace_files/cat_write.scap diff --git a/test/falco_tests_exceptions.yaml b/test/falco_tests_exceptions.yaml index 188d478e2b7..67f8ce9e6c6 100644 --- a/test/falco_tests_exceptions.yaml +++ b/test/falco_tests_exceptions.yaml @@ -20,175 +20,99 @@ trace_files: !mux rule_exception_no_fields: exit_status: 1 - stdout_is: |+ - 1 errors: - Rule exception item ex1: must have fields property with a list of fields - --- - - rule: My Rule - desc: Some desc - condition: evt.type=open and proc.name=cat - output: Some output - exceptions: - - name: ex1 - priority: error - --- + validate_errors: + - item_type: exception + item_name: ex1 + code: LOAD_ERR_YAML_VALIDATE + message: "Item has no mapping for key 'fields'" validate_rules_file: - rules/exceptions/item_no_fields.yaml trace_file: trace_files/cat_write.scap rule_exception_no_name: exit_status: 1 - stdout_is: |+ - 1 errors: - Rule exception item must have name property - --- - - rule: My Rule - desc: Some desc - condition: evt.type=open and proc.name=cat - output: Some output - exceptions: - - fields: [proc.name, fd.filename] - priority: error - --- + validate_errors: + - item_type: exception + item_name: "" + code: LOAD_ERR_YAML_VALIDATE + message: "Item has no mapping for key 'name'" validate_rules_file: - rules/exceptions/item_no_name.yaml trace_file: trace_files/cat_write.scap rule_exception_append_no_name: exit_status: 1 - stdout_is: |+ - 1 errors: - Rule exception item must have name property - --- - - rule: My Rule - exceptions: - - values: - - [nginx, /tmp/foo] - append: true - --- + validate_errors: + - item_type: exception + item_name: "" + code: LOAD_ERR_YAML_VALIDATE + message: "Item has no mapping for key 'name'" validate_rules_file: - rules/exceptions/append_item_no_name.yaml trace_file: trace_files/cat_write.scap rule_exception_unknown_fields: exit_status: 1 - stdout_is: |+ - 1 errors: - Rule exception item ex1: field name not.exist is not a supported filter field - --- - - rule: My Rule - desc: Some desc - condition: evt.type=open and proc.name=cat - output: Some output - exceptions: - - name: ex1 - fields: [not.exist] - priority: error - --- + validate_errors: + - item_type: exception + item_name: ex1 + code: LOAD_ERR_VALIDATE + message: "'not.exist' is not a supported filter field" validate_rules_file: - rules/exceptions/item_unknown_fields.yaml trace_file: trace_files/cat_write.scap rule_exception_comps_fields_len_mismatch: exit_status: 1 - stdout_is: |+ - 1 errors: - Rule exception item ex1: fields and comps lists must have equal length - --- - - rule: My Rule - desc: Some desc - condition: evt.type=open and proc.name=cat - output: Some output - exceptions: - - name: ex1 - fields: [proc.name, fd.filename] - comps: [=] - priority: error - --- + validate_errors: + - item_type: exception + item_name: ex1 + code: LOAD_ERR_VALIDATE + message: "Fields and comps lists must have equal length" validate_rules_file: - rules/exceptions/item_comps_fields_len_mismatch.yaml trace_file: trace_files/cat_write.scap rule_exception_unknown_comp: exit_status: 1 - stdout_is: |+ - 1 errors: - Rule exception item ex1: comparison operator no-comp is not a supported comparison operator - --- - - rule: My Rule - desc: Some desc - condition: evt.type=open and proc.name=cat - output: Some output - exceptions: - - name: ex1 - fields: [proc.name, fd.filename] - comps: [=, no-comp] - priority: error - --- + validate_errors: + - item_type: exception + item_name: ex1 + code: LOAD_ERR_VALIDATE + message: "'no-comp' is not a supported comparison operator" validate_rules_file: - rules/exceptions/item_unknown_comp.yaml trace_file: trace_files/cat_write.scap rule_exception_fields_values_len_mismatch: exit_status: 1 - stdout_is: |+ - 1 errors: - Exception item ex1: fields and values lists must have equal length - --- - - rule: My Rule - desc: Some desc - condition: evt.type=open and proc.name=cat - output: Some output - exceptions: - - name: ex1 - fields: [proc.name, fd.filename] - values: - - [nginx] - priority: error - --- + validate_errors: + - item_type: exception + item_name: ex1 + code: LOAD_ERR_VALIDATE + message: "Fields and values lists must have equal length" validate_rules_file: - rules/exceptions/item_fields_values_len_mismatch.yaml trace_file: trace_files/cat_write.scap rule_exception_append_fields_values_len_mismatch: exit_status: 1 - stdout_is: |+ - 1 errors: - Exception item ex1: fields and values lists must have equal length - --- - - rule: My Rule - desc: Some desc - condition: evt.type=open and proc.name=cat - output: Some output - exceptions: - - name: ex1 - fields: [proc.name, fd.filename] - priority: error - - - rule: My Rule - exceptions: - - name: ex1 - values: - - [nginx] - append: true - --- + validate_errors: + - item_type: exception + item_name: ex1 + code: LOAD_ERR_VALIDATE + message: "Fields and values lists must have equal length" validate_rules_file: - rules/exceptions/append_item_fields_values_len_mismatch.yaml trace_file: trace_files/cat_write.scap rule_exception_append_item_not_in_rule: exit_status: 1 - stdout_is: |+ - 1 errors: - Rule exception new item ex2: must have fields property with a list of fields - --- - - rule: My Rule - exceptions: - - name: ex2 - values: - - [apache, /tmp] - append: true - --- + validate_errors: + - item_type: exception + item_name: ex2 + code: LOAD_ERR_VALIDATE + message: "Rule exception must have fields property with a list of fields" validate_rules_file: - rules/exceptions/append_item_not_in_rule.yaml trace_file: trace_files/cat_write.scap @@ -325,7 +249,7 @@ trace_files: !mux rules_file: - rules/exceptions/rule_exception_new_single_field_append.yaml trace_file: trace_files/cat_write.scap - + rule_exception_new_second_field_append: detect: False detect_level: WARNING @@ -335,18 +259,11 @@ trace_files: !mux rule_exception_new_append_no_field: exit_status: 1 - stdout_is: |+ - 1 errors: - Rule exception new item proc_cmdline: must have fields property with a list of fields - --- - - rule: Open From Cat - exceptions: - - name: proc_cmdline - comps: in - values: - - "cat /dev/null" - append: true - --- + validate_errors: + - item_type: exception + item_name: proc_cmdline + code: LOAD_ERR_VALIDATE + message: "Rule exception must have fields property with a list of fields" validate_rules_file: - rules/exceptions/rule_exception_new_no_field_append.yaml trace_file: trace_files/cat_write.scap diff --git a/test/falco_tests_plugins.yaml b/test/falco_tests_plugins.yaml index f59f3aa6c10..59c732f2a1f 100644 --- a/test/falco_tests_plugins.yaml +++ b/test/falco_tests_plugins.yaml @@ -95,21 +95,24 @@ trace_files: !mux - rules/plugins/cloudtrail_incompat_plugin_version.yaml no_plugins_unknown_source: - detect: False - rules_file: + exit_status: 0 + validate_warnings: + - item_type: rule + item_name: Cloudtrail Create Instance + code: LOAD_UNKNOWN_SOURCE + message: "Unknown source aws_cloudtrail, skipping" + validate_rules_file: - rules/plugins/cloudtrail_create_instances.yaml - trace_file: trace_files/empty.scap - rules_warning: - - Cloudtrail Create Instance - stderr_contains: "Rule Cloudtrail Create Instance: warning .unknown-source.: unknown source aws_cloudtrail, skipping" no_plugins_unknown_source_rule_exception: - detect: False - rules_file: + exit_status: 0 + validate_warnings: + - item_type: rule + item_name: Cloudtrail Create Instance + code: LOAD_UNKNOWN_SOURCE + message: "Unknown source aws_cloudtrail, skipping" + validate_rules_file: - rules/plugins/cloudtrail_create_instances_exceptions.yaml - trace_file: trace_files/empty.scap - rules_warning: - - Cloudtrail Create Instance - stderr_contains: "Rule Cloudtrail Create Instance: warning .unknown-source.: unknown source aws_cloudtrail, skipping" + diff --git a/test/rules/macro_append_failure.yaml b/test/rules/macro_append_failure.yaml deleted file mode 100644 index d70ddc7e02d..00000000000 --- a/test/rules/macro_append_failure.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# -# Copyright (C) 2019 The Falco Authors. -# -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -- macro: my_macro - condition: proc.name=not-cat - append: true