From 72ee106c61bbffc2877adadac94da3fc24087f95 Mon Sep 17 00:00:00 2001 From: Jason Dellaluce Date: Thu, 14 Apr 2022 13:58:41 +0000 Subject: [PATCH 01/10] refactor(userspace/engine): introduce interface for rulesets and their factory This interface will allow us to use different ruleset implementations inside the same engine. The goal is to define API boundaries that will allow swapping the current evttype-index ruleset implementation more easily. Key benefits include: smaller component with less responsibilities, easier substituibility, more testable design, opportunity to adopt different index strategies depending on the ruleset implementation. Signed-off-by: Jason Dellaluce --- userspace/engine/ruleset.h | 217 ++++++++++++++++++++----------------- 1 file changed, 116 insertions(+), 101 deletions(-) diff --git a/userspace/engine/ruleset.h b/userspace/engine/ruleset.h index fe70a6c4f81..edd163f5edf 100644 --- a/userspace/engine/ruleset.h +++ b/userspace/engine/ruleset.h @@ -1,5 +1,5 @@ /* -Copyright (C) 2019 The Falco Authors. +Copyright (C) 2022 The Falco Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -16,108 +16,123 @@ limitations under the License. #pragma once -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include "falco_rule.h" -#include "sinsp.h" -#include "filter.h" -#include "event.h" - -#include "gen_filter.h" - -class falco_ruleset +/*! + \brief Represents a manager for rulesets.A ruleset represents a set of + enabled rules that is able to process events and find potential rule + matches. By convention, the ruleset with id = 0 is the default one. +*/ +class filter_ruleset { public: - falco_ruleset(); - virtual ~falco_ruleset(); - - void add(string &source, - std::string &name, - std::set &tags, - set &evttypes, - std::shared_ptr filter); - - // rulesets are arbitrary numbers and should be managed by the caller. - // Note that rulesets are used to index into a std::vector so - // specifying unnecessarily large rulesets will result in - // unnecessarily large vectors. - - // Find those rules matching the provided substring and set - // their enabled status to enabled. If match_exact is true, - // substring must be an exact match for a given rule - // name. Otherwise, any rules having substring as a substring - // in the rule name are enabled/disabled. - void enable(const std::string &substring, bool match_exact, bool enabled, uint16_t ruleset = 0); - - // Find those rules that have a tag in the set of tags and set - // their enabled status to enabled. Note that the enabled - // status is on the rules, and not the tags--if a rule R has - // tags (a, b), and you call enable_tags([a], true) and then - // enable_tags([b], false), R will be disabled despite the - // fact it has tag a and was enabled by the first call to - // enable_tags. - void enable_tags(const std::set &tags, bool enabled, uint16_t ruleset = 0); - - - // Return the number of falco rules enabled for the provided ruleset - uint64_t num_rules_for_ruleset(uint16_t ruleset = 0); - - // Match all filters against the provided event. - bool run(gen_event *evt, uint16_t ruleset = 0); - - // Populate the provided set of event types used by this ruleset. - void evttypes_for_ruleset(std::set &evttypes, uint16_t ruleset); - -private: - - class filter_wrapper { - public: - std::string source; - std::string name; - std::set tags; - std::set evttypes; - std::shared_ptr filter; - }; - - typedef std::list> filter_wrapper_list; - - // A group of filters all having the same ruleset - class ruleset_filters { - public: - ruleset_filters(); - - virtual ~ruleset_filters(); - - void add_filter(std::shared_ptr wrap); - void remove_filter(std::shared_ptr wrap); - - uint64_t num_filters(); - - bool run(gen_event *evt); - - void evttypes_for_ruleset(std::set &evttypes); - - private: - void add_wrapper_to_list(filter_wrapper_list &wrappers, std::shared_ptr wrap); - void remove_wrapper_from_list(filter_wrapper_list &wrappers, std::shared_ptr wrap); - - // Vector indexes from event type to a set of filters. There can - // be multiple filters for a given event type. - // NOTE: This is used only when the event sub-type is 0. - std::vector m_filter_by_event_type; - - filter_wrapper_list m_filter_all_event_types; - - // All filters added. Used to make num_filters() fast. - std::set> m_filters; - }; - - // Vector indexes from ruleset id to set of rules. - std::vector> m_rulesets; + virtual ~filter_ruleset() = default; + + /*! + \brief Adds a rule and its filtering condition inside the manager. + An exception is thrown is case of error. This method only adds the rule + inside the internal collection, but does not enable it for any ruleset. + The rule must be enabled for one or more rulesets with the enable() or + enable_tags() methods. + \param rule The rule to be added + \param condition The AST representing the rule's filtering condition + */ + virtual void add( + const falco_rule& rule, + std::shared_ptr condition) = 0; + + /*! + \brief Erases the internal state. All rules are disabled in each + ruleset, and all the rules defined with add() are removed. + */ + virtual void clear() = 0; + + /*! + \brief Returns the numeric id of a ruleset given its name. + If a ruleset has been used before, its previously assigned id is + returned. Otherwise, a new ruleset is created with the given name + and a new id is assigned to it. + */ + virtual uint16_t ruleset_id(const std::string &name) = 0; + + /*! + \brief This is meant to be called after all rules have been added + with add() and enabled on the given ruleset with enable()/enable_tags(). + */ + virtual void on_loading_complete() = 0; + + /*! + \brief Processes an event and tries to find a match in a given ruleset. + \return true if a match is found, false otherwise + \param evt The event to be processed + \param match If true is returned, this is filled-out with the rule + that matched the event + \param ruleset The id of the ruleset to be used + */ + virtual bool run( + gen_event *evt, + falco_rule& match, + uint16_t ruleset = 0) = 0; + + /*! + \brief Returns the number of rules enabled in a given ruleset + \param ruleset The id of the ruleset to be used + */ + virtual uint64_t enabled_count(uint16_t ruleset = 0) = 0; + + /*! + \brief Returns the union of the evttypes of all the rules enabled + in a given ruleset + \param ruleset The id of the ruleset to be used + */ + virtual void enabled_evttypes( + std::set &evttypes, + uint16_t ruleset) = 0; + + /*! + \brief Find those rules matching the provided substring and set + their enabled status to enabled in a given ruleset. + \param substring Substring used to match rule names. + If empty, all rules are matched. + \param match_exact If true, substring must be an exact match for a + given rule name. Otherwise, any rules having substring as a substring + in the rule name are enabled/disabled. + \param enabled The enabled status to set on all matching rules + \param ruleset The id of the ruleset to be used + */ + virtual void enable( + const std::string &substring, + bool match_exact, + bool enabled, + uint16_t ruleset = 0) = 0; + + /*! + \brief Find those rules that have a tag in the set of tags and + set their enabled status to enabled. Note that the enabled + status is on the rules, and not the tags--if a rule R has + tags (a, b), and you call enable_tags([a], true) and then + enable_tags([b], false), R will be disabled despite the + fact it has tag a and was enabled by the first call to + enable_tags. + \param substring Tags used to match ruless + \param enabled The enabled status to set on all matching rules + \param ruleset The id of the ruleset to be used + */ + virtual void enable_tags( + const std::set &tags, + bool enabled, + uint16_t ruleset = 0) = 0; +}; - // All filters added. The set of enabled filters is held in m_rulesets - std::set> m_filters; +/*! + \brief Represents a factory that creates filter_ruleset objects +*/ +class filter_ruleset_factory +{ +public: + virtual std::shared_ptr new_ruleset() = 0; }; From b35b7ef2dbeea83ce6e3d3da6308c73a72ca1ea6 Mon Sep 17 00:00:00 2001 From: Jason Dellaluce Date: Thu, 14 Apr 2022 14:02:31 +0000 Subject: [PATCH 02/10] refactor(userspace/engine): adapt existing ruleset implementation to new filter_ruleset interface Signed-off-by: Jason Dellaluce --- userspace/engine/CMakeLists.txt | 2 +- ...{ruleset.cpp => evttype_index_ruleset.cpp} | 103 ++++++++----- userspace/engine/evttype_index_ruleset.h | 140 ++++++++++++++++++ userspace/engine/falco_rule.h | 1 + 4 files changed, 211 insertions(+), 35 deletions(-) rename userspace/engine/{ruleset.cpp => evttype_index_ruleset.cpp} (55%) create mode 100644 userspace/engine/evttype_index_ruleset.h diff --git a/userspace/engine/CMakeLists.txt b/userspace/engine/CMakeLists.txt index 89202e52900..d08f83dd1bc 100644 --- a/userspace/engine/CMakeLists.txt +++ b/userspace/engine/CMakeLists.txt @@ -15,7 +15,7 @@ set(FALCO_ENGINE_SOURCE_FILES falco_engine.cpp falco_utils.cpp json_evt.cpp - ruleset.cpp + evttype_index_ruleset.cpp formats.cpp filter_macro_resolver.cpp filter_evttype_resolver.cpp diff --git a/userspace/engine/ruleset.cpp b/userspace/engine/evttype_index_ruleset.cpp similarity index 55% rename from userspace/engine/ruleset.cpp rename to userspace/engine/evttype_index_ruleset.cpp index 00b8838bccc..04aef217ac9 100644 --- a/userspace/engine/ruleset.cpp +++ b/userspace/engine/evttype_index_ruleset.cpp @@ -14,30 +14,32 @@ See the License for the specific language governing permissions and limitations under the License. */ -#include "ruleset.h" +#include "evttype_index_ruleset.h" +#include "filter_evttype_resolver.h" #include "banned.h" // This raises a compilation error when certain functions are used #include using namespace std; -falco_ruleset::falco_ruleset() +evttype_index_ruleset::evttype_index_ruleset( + std::shared_ptr f): m_filter_factory(f) { } -falco_ruleset::~falco_ruleset() +evttype_index_ruleset::~evttype_index_ruleset() { } -falco_ruleset::ruleset_filters::ruleset_filters() +evttype_index_ruleset::ruleset_filters::ruleset_filters() { } -falco_ruleset::ruleset_filters::~ruleset_filters() +evttype_index_ruleset::ruleset_filters::~ruleset_filters() { } -void falco_ruleset::ruleset_filters::add_wrapper_to_list(filter_wrapper_list &wrappers, std::shared_ptr wrap) +void evttype_index_ruleset::ruleset_filters::add_wrapper_to_list(filter_wrapper_list &wrappers, std::shared_ptr wrap) { // This is O(n) but it's also uncommon // (when loading rules only). @@ -51,7 +53,7 @@ void falco_ruleset::ruleset_filters::add_wrapper_to_list(filter_wrapper_list &wr } } -void falco_ruleset::ruleset_filters::remove_wrapper_from_list(filter_wrapper_list &wrappers, std::shared_ptr wrap) +void evttype_index_ruleset::ruleset_filters::remove_wrapper_from_list(filter_wrapper_list &wrappers, std::shared_ptr wrap) { // This is O(n) but it's also uncommon // (when loading rules only). @@ -64,7 +66,7 @@ void falco_ruleset::ruleset_filters::remove_wrapper_from_list(filter_wrapper_lis } } -void falco_ruleset::ruleset_filters::add_filter(std::shared_ptr wrap) +void evttype_index_ruleset::ruleset_filters::add_filter(std::shared_ptr wrap) { if(wrap->evttypes.empty()) { @@ -87,7 +89,7 @@ void falco_ruleset::ruleset_filters::add_filter(std::shared_ptr m_filters.insert(wrap); } -void falco_ruleset::ruleset_filters::remove_filter(std::shared_ptr wrap) +void evttype_index_ruleset::ruleset_filters::remove_filter(std::shared_ptr wrap) { if(wrap->evttypes.empty()) { @@ -107,12 +109,12 @@ void falco_ruleset::ruleset_filters::remove_filter(std::shared_ptrget_type() < m_filter_by_event_type.size()) { @@ -120,6 +122,7 @@ bool falco_ruleset::ruleset_filters::run(gen_event *evt) { if(wrap->filter->run(evt)) { + match = wrap->rule; return true; } } @@ -130,6 +133,7 @@ bool falco_ruleset::ruleset_filters::run(gen_event *evt) { if(wrap->filter->run(evt)) { + match = wrap->rule; return true; } } @@ -137,7 +141,7 @@ bool falco_ruleset::ruleset_filters::run(gen_event *evt) return false; } -void falco_ruleset::ruleset_filters::evttypes_for_ruleset(std::set &evttypes) +void evttype_index_ruleset::ruleset_filters::evttypes_for_ruleset(std::set &evttypes) { evttypes.clear(); @@ -147,23 +151,54 @@ void falco_ruleset::ruleset_filters::evttypes_for_ruleset(std::set &ev } } -void falco_ruleset::add(string &source, - string &name, - set &tags, - set &evttypes, - std::shared_ptr filter) +void evttype_index_ruleset::add( + const falco_rule& rule, + std::shared_ptr condition) { - std::shared_ptr wrap(new filter_wrapper()); - wrap->source = source; - wrap->name = name; - wrap->tags = tags; - wrap->filter = filter; - wrap->evttypes = evttypes; + try + { + sinsp_filter_compiler compiler(m_filter_factory, condition.get()); + shared_ptr filter(compiler.compile()); + std::shared_ptr wrap(new filter_wrapper()); + filter_evttype_resolver resolver; + wrap->rule = rule; + wrap->filter = filter; + resolver.evttypes(condition, wrap->evttypes); + m_filters.insert(wrap); + } + catch (const sinsp_exception& e) + { + throw falco_exception(string(e.what())); + } +} - m_filters.insert(wrap); +uint16_t evttype_index_ruleset::ruleset_id(const std::string &name) +{ + auto it = find(m_ruleset_names.begin(), m_ruleset_names.end(), name); + if (it != m_ruleset_names.end()) + { + return it - m_ruleset_names.begin(); + } + m_ruleset_names.push_back(name); + return m_ruleset_names.size() - 1; +} + +void evttype_index_ruleset::on_loading_complete() +{ + // nothing to do for now +} + +void evttype_index_ruleset::clear() +{ + for (size_t i = 0; i < m_rulesets.size(); i++) + { + std::shared_ptr r(new ruleset_filters()); + m_rulesets[i] = r; + } + m_filters.clear(); } -void falco_ruleset::enable(const string &substring, bool match_exact, bool enabled, uint16_t ruleset) +void evttype_index_ruleset::enable(const string &substring, bool match_exact, bool enabled, uint16_t ruleset) { while(m_rulesets.size() < (size_t)ruleset + 1) { @@ -176,14 +211,14 @@ void falco_ruleset::enable(const string &substring, bool match_exact, bool enabl if(match_exact) { - size_t pos = wrap->name.find(substring); + size_t pos = wrap->rule.name.find(substring); matches = (substring == "" || (pos == 0 && - substring.size() == wrap->name.size())); + substring.size() == wrap->rule.name.size())); } else { - matches = (substring == "" || (wrap->name.find(substring) != string::npos)); + matches = (substring == "" || (wrap->rule.name.find(substring) != string::npos)); } if(matches) @@ -200,7 +235,7 @@ void falco_ruleset::enable(const string &substring, bool match_exact, bool enabl } } -void falco_ruleset::enable_tags(const set &tags, bool enabled, uint16_t ruleset) +void evttype_index_ruleset::enable_tags(const set &tags, bool enabled, uint16_t ruleset) { while(m_rulesets.size() < (size_t)ruleset + 1) { @@ -212,7 +247,7 @@ void falco_ruleset::enable_tags(const set &tags, bool enabled, uint16_t std::set intersect; set_intersection(tags.begin(), tags.end(), - wrap->tags.begin(), wrap->tags.end(), + wrap->rule.tags.begin(), wrap->rule.tags.end(), inserter(intersect, intersect.begin())); if(!intersect.empty()) @@ -229,7 +264,7 @@ void falco_ruleset::enable_tags(const set &tags, bool enabled, uint16_t } } -uint64_t falco_ruleset::num_rules_for_ruleset(uint16_t ruleset) +uint64_t evttype_index_ruleset::enabled_count(uint16_t ruleset) { while(m_rulesets.size() < (size_t)ruleset + 1) { @@ -239,17 +274,17 @@ uint64_t falco_ruleset::num_rules_for_ruleset(uint16_t ruleset) return m_rulesets[ruleset]->num_filters(); } -bool falco_ruleset::run(gen_event *evt, uint16_t ruleset) +bool evttype_index_ruleset::run(gen_event *evt, falco_rule& match, uint16_t ruleset) { if(m_rulesets.size() < (size_t)ruleset + 1) { return false; } - return m_rulesets[ruleset]->run(evt); + return m_rulesets[ruleset]->run(evt, match); } -void falco_ruleset::evttypes_for_ruleset(set &evttypes, uint16_t ruleset) +void evttype_index_ruleset::enabled_evttypes(set &evttypes, uint16_t ruleset) { if(m_rulesets.size() < (size_t)ruleset + 1) { diff --git a/userspace/engine/evttype_index_ruleset.h b/userspace/engine/evttype_index_ruleset.h new file mode 100644 index 00000000000..9ae944563de --- /dev/null +++ b/userspace/engine/evttype_index_ruleset.h @@ -0,0 +1,140 @@ +/* +Copyright (C) 2019 The Falco Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +#pragma once + +#include +#include +#include +#include +#include + +#include "ruleset.h" +#include "sinsp.h" +#include "filter.h" +#include "event.h" + +#include "gen_filter.h" + +/*! + \brief A filter_ruleset that indexes enabled rules by event type, + and performs linear search on each event type bucket +*/ +class evttype_index_ruleset: public filter_ruleset +{ +public: + evttype_index_ruleset(std::shared_ptr factory); + virtual ~evttype_index_ruleset(); + + void add( + const falco_rule& rule, + std::shared_ptr condition) override; + + void clear() override; + + uint16_t ruleset_id(const std::string &name) override; + + bool run(gen_event *evt, falco_rule& match, uint16_t ruleset = 0); + + uint64_t enabled_count(uint16_t ruleset = 0) override; + + void on_loading_complete() override; + + void enable( + const std::string &substring, + bool match_exact, + bool enabled, + uint16_t ruleset = 0) override; + + void enable_tags( + const std::set &tags, + bool enabled, + uint16_t ruleset = 0) override; + + // evttypes for a ruleset + void enabled_evttypes( + std::set &evttypes, + uint16_t ruleset) override; + +private: + + struct filter_wrapper + { + falco_rule rule; + std::set evttypes; + std::shared_ptr filter; + }; + + typedef std::list> filter_wrapper_list; + + // A group of filters all having the same ruleset + class ruleset_filters { + public: + ruleset_filters(); + + virtual ~ruleset_filters(); + + void add_filter(std::shared_ptr wrap); + void remove_filter(std::shared_ptr wrap); + + uint64_t num_filters(); + + bool run(gen_event *evt, falco_rule& match); + + void evttypes_for_ruleset(std::set &evttypes); + + private: + void add_wrapper_to_list(filter_wrapper_list &wrappers, std::shared_ptr wrap); + void remove_wrapper_from_list(filter_wrapper_list &wrappers, std::shared_ptr wrap); + + // Vector indexes from event type to a set of filters. There can + // be multiple filters for a given event type. + // NOTE: This is used only when the event sub-type is 0. + std::vector m_filter_by_event_type; + + filter_wrapper_list m_filter_all_event_types; + + // All filters added. Used to make num_filters() fast. + std::set> m_filters; + }; + + // Vector indexes from ruleset id to set of rules. + std::vector> m_rulesets; + + // All filters added. The set of enabled filters is held in m_rulesets + std::set> m_filters; + + std::shared_ptr m_filter_factory; + std::vector m_ruleset_names; +}; + +class evttype_index_ruleset_factory: public filter_ruleset_factory +{ +public: + inline evttype_index_ruleset_factory( + std::shared_ptr factory + ): m_filter_factory(factory) { } + + inline std::shared_ptr new_ruleset() override + { + std::shared_ptr ret( + new evttype_index_ruleset(m_filter_factory)); + return ret; + } + +private: + std::shared_ptr m_filter_factory; +}; \ No newline at end of file diff --git a/userspace/engine/falco_rule.h b/userspace/engine/falco_rule.h index b998771f098..77aa8c79dd2 100644 --- a/userspace/engine/falco_rule.h +++ b/userspace/engine/falco_rule.h @@ -22,6 +22,7 @@ limitations under the License. struct falco_rule { + size_t id; std::string source; std::string name; std::string description; From 1f2f4c4556909f9bc0e48f15e33a5dae7a2a4749 Mon Sep 17 00:00:00 2001 From: Jason Dellaluce Date: Thu, 14 Apr 2022 14:06:46 +0000 Subject: [PATCH 03/10] refactor(userspace/engine): update falco engine to use new ruleset interface and have one ruleset for each source This also fixes a couple of bugs. With the current implementation, the multi-ruleset feature is broken with multiple sources. Signed-off-by: Jason Dellaluce --- userspace/engine/falco_engine.cpp | 207 +++++++++++------------------- userspace/engine/falco_engine.h | 56 +++----- userspace/engine/falco_source.h | 41 ++++++ 3 files changed, 135 insertions(+), 169 deletions(-) create mode 100644 userspace/engine/falco_source.h diff --git a/userspace/engine/falco_engine.cpp b/userspace/engine/falco_engine.cpp index fb2ab162a4d..0f8726d76cf 100644 --- a/userspace/engine/falco_engine.cpp +++ b/userspace/engine/falco_engine.cpp @@ -31,14 +31,14 @@ limitations under the License. #include "utils.h" #include "banned.h" // This raises a compilation error when certain functions are used +#include "evttype_index_ruleset.h" const std::string falco_engine::s_default_ruleset = "falco-default-ruleset"; using namespace std; falco_engine::falco_engine(bool seed_rng) - : m_next_ruleset_id(0), - m_min_priority(falco_common::PRIORITY_DEBUG), + : m_min_priority(falco_common::PRIORITY_DEBUG), m_sampling_ratio(1), m_sampling_multiplier(0), m_replace_container_info(false) { @@ -46,8 +46,6 @@ falco_engine::falco_engine(bool seed_rng) { srandom((unsigned) getpid()); } - - m_default_ruleset_id = find_ruleset_id(s_default_ruleset); } falco_engine::~falco_engine() @@ -62,6 +60,26 @@ uint32_t falco_engine::engine_version() return (uint32_t) FALCO_ENGINE_VERSION; } +falco_source& falco_engine::find_source(const std::string& name) +{ + auto ret = m_sources.at(name); + if(!ret) + { + throw falco_exception("Unknown event source " + name); + } + return *ret; +} + +falco_source& falco_engine::find_source(std::size_t index) +{ + auto ret = m_sources.at(index); + if(!ret) + { + throw falco_exception("Unknown event source index " + to_string(index)); + } + return *ret; +} + // Return a key that uniquely represents a field class. // For now, we assume name + shortdesc is unique. static std::string fieldclass_key(const gen_event_filter_factory::filter_fieldclass_info &fld_info) @@ -77,16 +95,16 @@ void falco_engine::list_fields(std::string &source, bool verbose, bool names_onl // Do a first pass to group together classes that are // applicable to multiple event sources. - for(auto &it : m_filter_factories) + for(auto &it : m_sources) { - if(source != "" && source != it.first) + if(source != "" && source != it.name) { continue; } - for(auto &fld_class : it.second->get_fields()) + for(auto &fld_class : it.filter_factory->get_fields()) { - fieldclass_event_sources[fieldclass_key(fld_class)].insert(it.first); + fieldclass_event_sources[fieldclass_key(fld_class)].insert(it.name); } } @@ -96,14 +114,14 @@ void falco_engine::list_fields(std::string &source, bool verbose, bool names_onl // In the second pass, actually print info, skipping duplicate // field classes and also printing info on supported sources. - for(auto &it : m_filter_factories) + for(auto &it : m_sources) { - if(source != "" && source != it.first) + if(source != "" && source != it.name) { continue; } - for(auto &fld_class : it.second->get_fields()) + for(auto &fld_class : it.filter_factory->get_fields()) { std::string key = fieldclass_key(fld_class); @@ -221,7 +239,7 @@ void falco_engine::enable_rule(const string &substring, bool enabled, const stri uint16_t ruleset_id = find_ruleset_id(ruleset); bool match_exact = false; - for(auto &it : m_rulesets) + for(auto &it : m_sources) { it.ruleset->enable(substring, match_exact, enabled, ruleset_id); } @@ -232,7 +250,7 @@ void falco_engine::enable_rule_exact(const string &rule_name, bool enabled, cons uint16_t ruleset_id = find_ruleset_id(ruleset); bool match_exact = true; - for(auto &it : m_rulesets) + for(auto &it : m_sources) { it.ruleset->enable(rule_name, match_exact, enabled, ruleset_id); } @@ -242,7 +260,7 @@ void falco_engine::enable_rule_by_tag(const set &tags, bool enabled, con { uint16_t ruleset_id = find_ruleset_id(ruleset); - for(auto &it : m_rulesets) + for(auto &it : m_sources) { it.ruleset->enable_tags(tags, enabled, ruleset_id); } @@ -253,60 +271,32 @@ void falco_engine::set_min_priority(falco_common::priority_type priority) m_min_priority = priority; } -uint16_t falco_engine::find_ruleset_id(const std::string &ruleset) +uint16_t falco_engine::find_ruleset_id( + const std::string &ruleset, const std::string &source) { - auto it = m_known_rulesets.lower_bound(ruleset); - - if(it == m_known_rulesets.end() || - it->first != ruleset) - { - it = m_known_rulesets.emplace_hint(it, - std::make_pair(ruleset, m_next_ruleset_id++)); - } - - return it->second; + return find_source(source).ruleset->ruleset_id(ruleset); } uint64_t falco_engine::num_rules_for_ruleset(const std::string &ruleset) { - uint16_t ruleset_id = find_ruleset_id(ruleset); - uint64_t ret = 0; - for(auto &it : m_rulesets) + for (auto &src : m_sources) { - ret += it.ruleset->num_rules_for_ruleset(ruleset_id); + ret += src.ruleset->enabled_count(src.ruleset->ruleset_id(ruleset)); } - return ret; } void falco_engine::evttypes_for_ruleset(std::string &source, std::set &evttypes, const std::string &ruleset) { - uint16_t ruleset_id = find_ruleset_id(ruleset); - - auto it = find_ruleset(source); - if(it == m_rulesets.end()) - { - string err = "Unknown event source " + source; - throw falco_exception(err); - } - - it->ruleset->evttypes_for_ruleset(evttypes, ruleset_id); - + auto src = find_source(source); + src.ruleset->enabled_evttypes(evttypes, src.ruleset->ruleset_id(ruleset)); } std::shared_ptr falco_engine::create_formatter(const std::string &source, const std::string &output) { - auto it = m_format_factories.find(source); - - if(it == m_format_factories.end()) - { - string err = "Unknown event source " + source; - throw falco_exception(err); - } - - return it->second->create_formatter(output); + return find_source(source).formatter_factory->create_formatter(output); } unique_ptr falco_engine::process_event(std::size_t source_idx, gen_event *ev, uint16_t ruleset_id) @@ -316,67 +306,52 @@ unique_ptr falco_engine::process_event(std::size_t so return unique_ptr(); } - try + falco_rule rule; + if(!find_source(source_idx).ruleset->run(ev, rule, ruleset_id)) { - auto &r = m_rulesets.at(source_idx); - if(!r.ruleset->run(ev, ruleset_id)) - { - return unique_ptr(); - } - - unique_ptr res(new rule_result()); - // note: indexes are 0-based, whereas check_ids are not - auto rule_idx = ev->get_check_id() - 1; - auto rule = m_rules.at(rule_idx); - if (!rule) - { - throw falco_exception("populate_rule_result error: unknown rule id " - + to_string(rule_idx)); - } - res->evt = ev; - res->rule = rule->name; - res->source = rule->source; - res->format = rule->output; - res->priority_num = rule->priority; - res->tags = rule->tags; - res->exception_fields = rule->exception_fields; - m_rule_stats_manager.on_event(m_rules, rule_idx); - return res; - } - catch(std::out_of_range const &exc) - { - std::string err = "Unknown event source index " + std::to_string(source_idx); - throw falco_exception(err); + return unique_ptr(); } + + unique_ptr res(new rule_result()); + res->evt = ev; + res->rule = rule.name; + res->source = rule.source; + res->format = rule.output; + res->priority_num = rule.priority; + res->tags = rule.tags; + res->exception_fields = rule.exception_fields; + m_rule_stats_manager.on_event(m_rules, rule.id); + return res; } unique_ptr falco_engine::process_event(std::size_t source_idx, gen_event *ev) { - return process_event(source_idx, ev, m_default_ruleset_id); + return process_event(source_idx, ev, find_source(source_idx).default_ruleset_id); } std::size_t falco_engine::add_source(const std::string &source, std::shared_ptr filter_factory, std::shared_ptr formatter_factory) -{ - m_filter_factories[source] = filter_factory; - m_format_factories[source] = formatter_factory; - - auto idx = m_rulesets.size(); - m_rulesets.emplace_back(source, new falco_ruleset); - // here we just trust the caller they won't add the same source more than once - return idx; +{ + // evttype_index_ruleset is the default ruleset implementation + std::shared_ptr ruleset_factory( + new evttype_index_ruleset_factory(filter_factory)); + return add_source(source, filter_factory, formatter_factory, ruleset_factory); } -std::shared_ptr falco_engine::get_filter_factory( - const std::string &source) +std::size_t falco_engine::add_source(const std::string &source, + std::shared_ptr filter_factory, + std::shared_ptr formatter_factory, + std::shared_ptr ruleset_factory) { - auto it = m_filter_factories.find(source); - if(it == m_filter_factories.end()) - { - throw falco_exception(string("unknown event source: ") + source); - } - return it->second; + falco_source src; + src.name = source; + src.filter_factory = filter_factory; + src.formatter_factory = formatter_factory; + src.ruleset_factory = ruleset_factory; + src.ruleset = ruleset_factory->new_ruleset(); + src.default_ruleset_id = src.ruleset->ruleset_id(s_default_ruleset); + return m_sources.insert(src, source); } void falco_engine::describe_rule(string *rule) @@ -398,7 +373,6 @@ void falco_engine::describe_rule(string *rule) auto str = falco::utils::wrap_text(r->description, 51, 110) + "\n"; fprintf(stdout, rule_fmt, r->name.c_str(), str.c_str()); } - } void falco_engine::print_stats() @@ -409,25 +383,9 @@ void falco_engine::print_stats() fprintf(stdout, "%s", out.c_str()); } -void falco_engine::add_filter(std::shared_ptr filter, - std::string &rule, - std::string &source, - std::set &evttypes, - std::set &tags) -{ - auto it = find_ruleset(source); - if(it == m_rulesets.end()) - { - string err = "Unknown event source " + source; - throw falco_exception(err); - } - - it->ruleset->add(source, rule, tags, evttypes, filter); -} - bool falco_engine::is_source_valid(const std::string &source) { - return (find_ruleset(source) != m_rulesets.end()); + return m_sources.at(source) != nullptr; } bool falco_engine::check_plugin_requirements( @@ -475,9 +433,10 @@ bool falco_engine::check_plugin_requirements( void falco_engine::clear_filters() { - for(auto &it : m_rulesets) + for(auto &src : m_sources) { - it.ruleset.reset(new falco_ruleset); + src.ruleset = src.ruleset_factory->new_ruleset(); + src.default_ruleset_id = src.ruleset->ruleset_id(s_default_ruleset); } } @@ -517,17 +476,3 @@ inline bool falco_engine::should_drop_evt() double coin = (random() * (1.0/RAND_MAX)); return (coin >= (1.0/(m_sampling_multiplier * m_sampling_ratio))); } - -inline std::vector::iterator falco_engine::find_ruleset(const std::string &source) -{ - return std::find_if( - m_rulesets.begin(), m_rulesets.end(), - [&source](const ruleset_node &r) { return r.source == source; }); -} - -inline std::vector::const_iterator falco_engine::find_ruleset(const std::string &source) const -{ - return std::find_if( - m_rulesets.cbegin(), m_rulesets.cend(), - [&source](const ruleset_node &r) { return r.source == source; }); -} diff --git a/userspace/engine/falco_engine.h b/userspace/engine/falco_engine.h index 1fd6e38ebab..8df24fce68a 100644 --- a/userspace/engine/falco_engine.h +++ b/userspace/engine/falco_engine.h @@ -33,6 +33,7 @@ limitations under the License. #include "rule_loader.h" #include "stats_manager.h" #include "falco_common.h" +#include "falco_source.h" // // This class acts as the primary interface between a program and the @@ -76,6 +77,7 @@ class falco_engine // context of the provided ruleset. The ruleset (id) can later // be passed as an argument to process_event(). This allows // for different sets of rules being active at once. + // The rules are matched against the rulesets of all the defined sources. // void enable_rule(const std::string &substring, bool enabled, const std::string &ruleset = s_default_ruleset); @@ -97,10 +99,13 @@ class falco_engine // to enable_rule/enable_rule_by_tag(), you should look up the // ruleset id and pass it to process_event(). // - uint16_t find_ruleset_id(const std::string &ruleset); + uint16_t find_ruleset_id( + const std::string &ruleset, + const std::string &source = falco_common::syscall_source); // // Return the number of falco rules enabled for the provided ruleset + // across all sources. // uint64_t num_rules_for_ruleset(const std::string &ruleset); @@ -187,25 +192,20 @@ class falco_engine std::size_t add_source(const std::string &source, std::shared_ptr filter_factory, std::shared_ptr formatter_factory); - - // todo(jasondellaluce): this is here for internal use, and - // will possibly be removed in the future - std::shared_ptr get_filter_factory( - const std::string &source); + + // + // Equivalent to above, but allows specifying a ruleset factory + // for the newly added source. + // + std::size_t add_source(const std::string &source, + std::shared_ptr filter_factory, + std::shared_ptr formatter_factory, + std::shared_ptr ruleset_factory); // Return whether or not there is a valid filter/formatter // factory for this source. bool is_source_valid(const std::string &source); - // - // Add a filter for the provided event source to the engine - // - void add_filter(std::shared_ptr filter, - std::string &rule, - std::string &source, - std::set &evttypes, - std::set &tags); - // // Given an event source and ruleset, fill in a bitset // containing the event types for which this ruleset can run. @@ -237,14 +237,10 @@ class falco_engine std::string& err); private: - struct ruleset_node - { - ruleset_node(const std::string &n, falco_ruleset *p): - source(n), ruleset(p) {} + indexed_vector m_sources; - std::string source; - mutable std::shared_ptr ruleset; - }; + falco_source& find_source(std::size_t index); + falco_source& find_source(const std::string& name); // // Determine whether the given event should be matched at all @@ -253,27 +249,12 @@ class falco_engine // inline bool should_drop_evt(); - inline std::vector::iterator find_ruleset(const std::string &source); - inline std::vector::const_iterator find_ruleset(const std::string &source) const; - - // Maps from event source to object that can generate filters from rules - std::map> m_filter_factories; - - // Maps from event source to object that can format output strings in rules - std::map> m_format_factories; - - // Maps from event source to the set of rules for that event source - std::vector m_rulesets; - rule_loader m_rule_loader; indexed_vector m_rules; stats_manager m_rule_stats_manager; - uint16_t m_next_ruleset_id; - std::map m_known_rulesets; falco_common::priority_type m_min_priority; - // // Here's how the sampling ratio and multiplier influence // whether or not an event is dropped in @@ -298,7 +279,6 @@ class falco_engine double m_sampling_multiplier; static const std::string s_default_ruleset; - uint32_t m_default_ruleset_id; std::string m_extra; bool m_replace_container_info; diff --git a/userspace/engine/falco_source.h b/userspace/engine/falco_source.h new file mode 100644 index 00000000000..ca70db942a7 --- /dev/null +++ b/userspace/engine/falco_source.h @@ -0,0 +1,41 @@ +/* +Copyright (C) 2022 The Falco Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +#pragma once + +#include +#include "ruleset.h" + +struct falco_source +{ + std::string name; + uint16_t default_ruleset_id; + std::shared_ptr ruleset; + std::shared_ptr ruleset_factory; + std::shared_ptr filter_factory; + std::shared_ptr formatter_factory; + + inline bool is_field_defined(std::string field) const + { + auto *chk = filter_factory->new_filtercheck(field.c_str()); + if (chk) + { + delete(chk); + return true; + } + return false; + } +}; From e27f75bfae3d2c4d6064e06a543d67acbeeeb3cd Mon Sep 17 00:00:00 2001 From: Jason Dellaluce Date: Thu, 14 Apr 2022 14:12:27 +0000 Subject: [PATCH 04/10] refactor(userspace/engine): update rule loader to use new filter_ruleset interface Signed-off-by: Jason Dellaluce --- userspace/engine/evttype_index_ruleset.cpp | 11 ++- userspace/engine/falco_engine.cpp | 7 +- userspace/engine/rule_loader.cpp | 108 ++++++++------------- userspace/engine/rule_loader.h | 6 +- userspace/engine/ruleset.h | 4 +- 5 files changed, 57 insertions(+), 79 deletions(-) diff --git a/userspace/engine/evttype_index_ruleset.cpp b/userspace/engine/evttype_index_ruleset.cpp index 04aef217ac9..31b6be56098 100644 --- a/userspace/engine/evttype_index_ruleset.cpp +++ b/userspace/engine/evttype_index_ruleset.cpp @@ -160,10 +160,17 @@ void evttype_index_ruleset::add( sinsp_filter_compiler compiler(m_filter_factory, condition.get()); shared_ptr filter(compiler.compile()); std::shared_ptr wrap(new filter_wrapper()); - filter_evttype_resolver resolver; wrap->rule = rule; wrap->filter = filter; - resolver.evttypes(condition, wrap->evttypes); + if(rule.source == falco_common::syscall_source) + { + filter_evttype_resolver resolver; + resolver.evttypes(condition, wrap->evttypes); + } + else + { + wrap->evttypes = { ppm_event_type::PPME_PLUGINEVENT_E }; + } m_filters.insert(wrap); } catch (const sinsp_exception& e) diff --git a/userspace/engine/falco_engine.cpp b/userspace/engine/falco_engine.cpp index 0f8726d76cf..44aeb53b550 100644 --- a/userspace/engine/falco_engine.cpp +++ b/userspace/engine/falco_engine.cpp @@ -169,7 +169,7 @@ void falco_engine::load_rules(const string &rules_content, bool verbose, bool al void falco_engine::load_rules(const string &rules_content, bool verbose, bool all_events, uint64_t &required_engine_version) { rule_loader::configuration cfg(rules_content); - cfg.engine = this; + cfg.sources = m_sources; cfg.min_priority = m_min_priority; cfg.output_extra = m_extra; cfg.replace_output_container_info = m_replace_container_info; @@ -179,7 +179,10 @@ void falco_engine::load_rules(const string &rules_content, bool verbose, bool al bool success = reader.load(cfg, m_rule_loader); if (success) { - clear_filters(); + for (auto &s : m_sources) + { + s.ruleset->clear(); + } m_rules.clear(); success = m_rule_loader.compile(cfg, m_rules); } diff --git a/userspace/engine/rule_loader.cpp b/userspace/engine/rule_loader.cpp index 1d55ceca7db..103aacfcc4a 100644 --- a/userspace/engine/rule_loader.cpp +++ b/userspace/engine/rule_loader.cpp @@ -48,22 +48,6 @@ static void paren_item(string& e) } } -static bool is_field_defined( - falco_engine *engine, const string& source, string field) -{ - auto factory = engine->get_filter_factory(source); - if(factory) - { - auto *chk = factory->new_filtercheck(field.c_str()); - if (chk) - { - delete(chk); - return true; - } - } - return false; -} - static inline bool is_operator_defined(const string& op) { auto ops = libsinsp::filter::parser::supported_operators(); @@ -76,13 +60,12 @@ static inline bool is_operator_for_list(const string& op) return find(ops.begin(), ops.end(), op) != ops.end(); } -static bool is_format_valid( - falco_engine* e, const string& src, const string& fmt, string& err) +static bool is_format_valid(const falco_source& source, string fmt, string& err) { try { shared_ptr formatter; - formatter = e->create_formatter(src, fmt); + formatter = source.formatter_factory->create_formatter(fmt); return true; } catch(exception &e) @@ -118,9 +101,8 @@ static inline void append_info(T* prev, T& info, uint32_t id) } static void validate_exception_info( - rule_loader::configuration& cfg, - rule_loader::rule_exception_info &ex, - const string& source) + const falco_source& source, + rule_loader::rule_exception_info &ex) { if (ex.fields.is_list) { @@ -133,7 +115,7 @@ static void validate_exception_info( } } THROW(ex.fields.items.size() != ex.comps.items.size(), - "Rule exception item " + ex.name + "Rule exception item " + ex.name + ": fields and comps lists must have equal length"); for (auto &v : ex.comps.items) { @@ -143,7 +125,7 @@ static void validate_exception_info( } for (auto &v : ex.fields.items) { - THROW(!is_field_defined(cfg.engine, source, v.item), + THROW(!source.is_field_defined(v.item), "Rule exception item " + ex.name + ": field name " + v.item + " is not a supported filter field"); } @@ -160,7 +142,7 @@ static void validate_exception_info( THROW(!is_operator_defined(ex.comps.item), "Rule exception item " + ex.name + ": comparison operator " + ex.comps.item + " is not a supported comparison operator"); - THROW(!is_field_defined(cfg.engine, source, ex.fields.item), + THROW(!source.is_field_defined(ex.fields.item), "Rule exception item " + ex.name + ": field name " + ex.fields.item + " is not a supported filter field"); } @@ -365,37 +347,11 @@ static shared_ptr parse_condition( } catch (const sinsp_exception& e) { - throw falco_exception("Compilation error when compiling \"" + throw falco_exception("Compilation error when compiling \"" + condition + "\": " + to_string(p.get_pos().col) + ": " + e.what()); } } -static shared_ptr compile_condition( - falco_engine* engine, - uint32_t id, - shared_ptr cnd, - string src, - string& err) -{ - try - { - auto factory = engine->get_filter_factory(src); - sinsp_filter_compiler compiler(factory, cnd.get()); - compiler.set_check_id(id); - shared_ptr ret(compiler.compile()); - return ret; - } - catch (const sinsp_exception& e) - { - err = e.what(); - } - catch (const falco_exception& e) - { - err = e.what(); - } - return nullptr; -} - static void apply_output_substitutions( rule_loader::configuration& cfg, string& out) @@ -457,7 +413,7 @@ void rule_loader::append(configuration& cfg, list_info& info) void rule_loader::define(configuration& cfg, macro_info& info) { - if (!cfg.engine->is_source_valid(info.source)) + if (!cfg.sources.at(info.source)) { cfg.warnings.push_back("Macro " + info.name + ": warning (unknown-source): unknown source " @@ -484,7 +440,8 @@ void rule_loader::append(configuration& cfg, macro_info& info) void rule_loader::define(configuration& cfg, rule_info& info) { - if (!cfg.engine->is_source_valid(info.source)) + auto source = cfg.sources.at(info.source); + if (!source) { cfg.warnings.push_back("Rule " + info.name + ": warning (unknown-source): unknown source " @@ -500,7 +457,7 @@ void rule_loader::define(configuration& cfg, rule_info& info) { THROW(!ex.fields.is_valid(), "Rule exception item " + ex.name + ": must have fields property with a list of fields"); - validate_exception_info(cfg, ex, info.source); + validate_exception_info(*source, ex); } define_info(m_rule_infos, info, m_cur_index++); @@ -514,6 +471,11 @@ void rule_loader::append(configuration& cfg, rule_info& info) THROW(info.cond.empty() && info.exceptions.empty(), "Appended rule must have exceptions or condition property"); + auto source = cfg.sources.at(prev->source); + // note: this is not supposed to happen + THROW(!source, "Rule " + prev->name + + ": error (unknown-source): unknown source " + prev->source); + if (!info.cond.empty()) { prev->cond += " "; @@ -531,7 +493,7 @@ void rule_loader::append(configuration& cfg, rule_info& info) + ex.name + ": must have fields property with a list of fields"); THROW(ex.values.empty(), "Rule exception new item " + ex.name + ": must have fields property with a list of values"); - validate_exception_info(cfg, ex, prev->source); + validate_exception_info(*source, ex); prev->exceptions.push_back(ex); } else @@ -649,9 +611,15 @@ void rule_loader::compile_rule_infos( continue; } + auto source = cfg.sources.at(r.source); + // note: this is not supposed to happen + THROW(!source, "Rule " + r.name + + ": error (unknown-source): unknown source " + r.source); + // build filter AST by parsing the condition, building exceptions, // and resolving lists and macros falco_rule rule; + condition = r.cond; if (!r.exceptions.empty()) { @@ -679,7 +647,8 @@ void rule_loader::compile_rule_infos( { apply_output_substitutions(cfg, rule.output); } - THROW(!is_format_valid(cfg.engine, r.source, rule.output, err), + + THROW(!is_format_valid(cfg.engine, r.source, rule.output, err), "Invalid output format '" + rule.output + "': '" + err + "'"); // construct rule definition and compile it to a filter @@ -688,13 +657,18 @@ void rule_loader::compile_rule_infos( rule.description = r.desc; rule.priority = r.priority; rule.tags = r.tags; - // note: indexes are 0-based, but 0 is not an acceptable rule_id - auto id = out.insert(rule, rule.name) + 1; - auto filter = compile_condition(cfg.engine, id, ast, rule.source, err); - if (!filter) + try + { + auto rule_id = out.insert(rule, rule.name); + out.at(rule_id)->id = rule_id; + source->ruleset->add(*out.at(rule_id), ast); + source->ruleset->enable(rule.name, false, r.enabled); + } + catch (falco_exception& e) { - if (r.skip_if_unknown_filter - && err.find("nonexistent field") != string::npos) + string err = e.what(); + if (err.find("nonexistent field") != string::npos + && r.skip_if_unknown_filter) { cfg.warnings.push_back( "Rule " + rule.name + ": warning (unknown-field):"); @@ -705,7 +679,7 @@ void rule_loader::compile_rule_infos( throw falco_exception("Rule " + rule.name + ": error " + err); } } - + // populate set of event types and emit an special warning set evttypes = { ppm_event_type::PPME_PLUGINEVENT_E }; if(rule.source == falco_common::syscall_source) @@ -721,10 +695,6 @@ void rule_loader::compile_rule_infos( + " This has a significant performance penalty."); } } - - // add rule and its filter in the engine - cfg.engine->add_filter(filter, rule.name, rule.source, evttypes, rule.tags); - cfg.engine->enable_rule(rule.name, r.enabled); } catch (exception& e) { @@ -769,4 +739,4 @@ bool rule_loader::compile(configuration& cfg, indexed_vector& out) c } } return true; -} \ No newline at end of file +} diff --git a/userspace/engine/rule_loader.h b/userspace/engine/rule_loader.h index 29c88eac45a..57964da3b78 100644 --- a/userspace/engine/rule_loader.h +++ b/userspace/engine/rule_loader.h @@ -21,11 +21,9 @@ limitations under the License. #include #include #include "falco_rule.h" +#include "falco_source.h" #include "indexed_vector.h" -// todo(jasondellaluce): remove this cyclic dependency -class falco_engine; - /*! \brief Ruleset loader of the falco engine @@ -73,9 +71,9 @@ class rule_loader std::string output_extra; bool replace_output_container_info; falco_common::priority_type min_priority; + indexed_vector sources; std::vector warnings; std::vector errors; - falco_engine* engine; }; /*! diff --git a/userspace/engine/ruleset.h b/userspace/engine/ruleset.h index edd163f5edf..33bdb27340d 100644 --- a/userspace/engine/ruleset.h +++ b/userspace/engine/ruleset.h @@ -16,11 +16,11 @@ limitations under the License. #pragma once +#include "falco_rule.h" +#include #include #include #include -#include -#include "falco_rule.h" /*! \brief Represents a manager for rulesets.A ruleset represents a set of From 54745334d22aa4cad1253025cca578e22f77ad8b Mon Sep 17 00:00:00 2001 From: Jason Dellaluce Date: Mon, 23 May 2022 10:03:22 +0000 Subject: [PATCH 05/10] refactor(userspace/engine): leverage falco_rule def in stats manager Signed-off-by: Jason Dellaluce --- userspace/engine/stats_manager.cpp | 27 ++++++++------------------- userspace/engine/stats_manager.h | 8 +++----- 2 files changed, 11 insertions(+), 24 deletions(-) diff --git a/userspace/engine/stats_manager.cpp b/userspace/engine/stats_manager.cpp index d130f6d3a69..47a37fc57b7 100644 --- a/userspace/engine/stats_manager.cpp +++ b/userspace/engine/stats_manager.cpp @@ -38,10 +38,9 @@ void stats_manager::clear() void stats_manager::format( const indexed_vector& rules, - string& out) + string& out) const { string fmt; - string name; out = "Events detected: " + to_string(m_total) + "\n"; out += "Rule counts by severity:\n"; for (size_t i = 0; i < m_by_priority.size(); i++) @@ -66,27 +65,17 @@ void stats_manager::format( } } -void stats_manager::on_event( - const indexed_vector& rules, - uint32_t rule_id) +void stats_manager::on_event(const falco_rule& rule) { - auto *rule = rules.at(rule_id); - if (!rule) + if (m_by_rule_id.size() <= rule.id) { - throw falco_exception( - "on_event(): event with invalid rule_id: " + rule_id); + m_by_rule_id.resize(rule.id + 1, (uint64_t) 0); } - if (m_by_rule_id.size() <= rule_id) + if (m_by_priority.size() <= (size_t) rule.priority) { - m_by_rule_id.resize(rule_id + 1); - m_by_rule_id[rule_id] = 0; - } - if (m_by_priority.size() <= (size_t) rule->priority) - { - m_by_priority.resize((size_t) rule->priority + 1); - m_by_priority[(size_t) rule->priority] = 0; + m_by_priority.resize((size_t) rule.priority + 1, (uint64_t) 0); } m_total++; - m_by_rule_id[rule_id]++; - m_by_priority[(size_t) rule->priority]++; + m_by_rule_id[rule.id]++; + m_by_priority[(size_t) rule.priority]++; } diff --git a/userspace/engine/stats_manager.h b/userspace/engine/stats_manager.h index 5be98cc75eb..23f9bcca618 100644 --- a/userspace/engine/stats_manager.h +++ b/userspace/engine/stats_manager.h @@ -36,18 +36,16 @@ class stats_manager virtual void clear(); /*! - \brief Callback for when a rule with a given index matches an event + \brief Callback for when a given rule matches an event */ - virtual void on_event( - const indexed_vector& rules, - uint32_t index); + virtual void on_event(const falco_rule& rule); /*! \brief Formats the internal statistics into the out string */ virtual void format( const indexed_vector& rules, - std::string& out); + std::string& out) const; private: uint64_t m_total; From 71e639225732414f66692eadb0c7ad76484d7fa7 Mon Sep 17 00:00:00 2001 From: Jason Dellaluce Date: Mon, 23 May 2022 10:17:05 +0000 Subject: [PATCH 06/10] refactor(userspace/engine): improve ruleset interface definitions The filter_ruleset interface its implementation evt_type_index_ruleset have been modified as follows: - Only keep track of ruleset ids and not names. The falco engine will take care of mapping easy-to-remember ruleset names to ruleset ids. To emphasize this, use ruleset_id everywhere and not ruleset. Also, make it non-optional. - Have explicit separate functions to enable/disable rules, instead of a single enable() method combined with a boolean flag. This does *not* change the falco_engine interface, which has similar methods, to avoid breaking API changes. Signed-off-by: Jason Dellaluce Co-authored-by: Mark Stemm --- userspace/engine/evttype_index_ruleset.cpp | 65 +++++++++++-------- userspace/engine/evttype_index_ruleset.h | 38 ++++++++--- userspace/engine/ruleset.h | 74 ++++++++++++++-------- 3 files changed, 111 insertions(+), 66 deletions(-) diff --git a/userspace/engine/evttype_index_ruleset.cpp b/userspace/engine/evttype_index_ruleset.cpp index 31b6be56098..e52655eb3f6 100644 --- a/userspace/engine/evttype_index_ruleset.cpp +++ b/userspace/engine/evttype_index_ruleset.cpp @@ -179,17 +179,6 @@ void evttype_index_ruleset::add( } } -uint16_t evttype_index_ruleset::ruleset_id(const std::string &name) -{ - auto it = find(m_ruleset_names.begin(), m_ruleset_names.end(), name); - if (it != m_ruleset_names.end()) - { - return it - m_ruleset_names.begin(); - } - m_ruleset_names.push_back(name); - return m_ruleset_names.size() - 1; -} - void evttype_index_ruleset::on_loading_complete() { // nothing to do for now @@ -205,9 +194,19 @@ void evttype_index_ruleset::clear() m_filters.clear(); } -void evttype_index_ruleset::enable(const string &substring, bool match_exact, bool enabled, uint16_t ruleset) +void evttype_index_ruleset::enable(const string &substring, bool match_exact, uint16_t ruleset_id) { - while(m_rulesets.size() < (size_t)ruleset + 1) + enable_disable(substring, match_exact, true, ruleset_id); +} + +void evttype_index_ruleset::disable(const string &substring, bool match_exact, uint16_t ruleset_id) +{ + enable_disable(substring, match_exact, false, ruleset_id); +} + +void evttype_index_ruleset::enable_disable(const string &substring, bool match_exact, bool enabled, uint16_t ruleset_id) +{ + while(m_rulesets.size() < (size_t)ruleset_id + 1) { m_rulesets.emplace_back(new ruleset_filters()); } @@ -232,19 +231,29 @@ void evttype_index_ruleset::enable(const string &substring, bool match_exact, bo { if(enabled) { - m_rulesets[ruleset]->add_filter(wrap); + m_rulesets[ruleset_id]->add_filter(wrap); } else { - m_rulesets[ruleset]->remove_filter(wrap); + m_rulesets[ruleset_id]->remove_filter(wrap); } } } } -void evttype_index_ruleset::enable_tags(const set &tags, bool enabled, uint16_t ruleset) +void evttype_index_ruleset::enable_tags(const set &tags, uint16_t ruleset_id) +{ + enable_disable_tags(tags, true, ruleset_id); +} + +void evttype_index_ruleset::disable_tags(const set &tags, uint16_t ruleset_id) +{ + enable_disable_tags(tags, false, ruleset_id); +} + +void evttype_index_ruleset::enable_disable_tags(const set &tags, bool enabled, uint16_t ruleset_id) { - while(m_rulesets.size() < (size_t)ruleset + 1) + while(m_rulesets.size() < (size_t)ruleset_id + 1) { m_rulesets.emplace_back(new ruleset_filters()); } @@ -261,42 +270,42 @@ void evttype_index_ruleset::enable_tags(const set &tags, bool enabled, u { if(enabled) { - m_rulesets[ruleset]->add_filter(wrap); + m_rulesets[ruleset_id]->add_filter(wrap); } else { - m_rulesets[ruleset]->remove_filter(wrap); + m_rulesets[ruleset_id]->remove_filter(wrap); } } } } -uint64_t evttype_index_ruleset::enabled_count(uint16_t ruleset) +uint64_t evttype_index_ruleset::enabled_count(uint16_t ruleset_id) { - while(m_rulesets.size() < (size_t)ruleset + 1) + while(m_rulesets.size() < (size_t)ruleset_id + 1) { m_rulesets.emplace_back(new ruleset_filters()); } - return m_rulesets[ruleset]->num_filters(); + return m_rulesets[ruleset_id]->num_filters(); } -bool evttype_index_ruleset::run(gen_event *evt, falco_rule& match, uint16_t ruleset) +bool evttype_index_ruleset::run(gen_event *evt, falco_rule& match, uint16_t ruleset_id) { - if(m_rulesets.size() < (size_t)ruleset + 1) + if(m_rulesets.size() < (size_t)ruleset_id + 1) { return false; } - return m_rulesets[ruleset]->run(evt, match); + return m_rulesets[ruleset_id]->run(evt, match); } -void evttype_index_ruleset::enabled_evttypes(set &evttypes, uint16_t ruleset) +void evttype_index_ruleset::enabled_evttypes(set &evttypes, uint16_t ruleset_id) { - if(m_rulesets.size() < (size_t)ruleset + 1) + if(m_rulesets.size() < (size_t)ruleset_id + 1) { return; } - return m_rulesets[ruleset]->evttypes_for_ruleset(evttypes); + return m_rulesets[ruleset_id]->evttypes_for_ruleset(evttypes); } diff --git a/userspace/engine/evttype_index_ruleset.h b/userspace/engine/evttype_index_ruleset.h index 9ae944563de..acdd1aaf1fb 100644 --- a/userspace/engine/evttype_index_ruleset.h +++ b/userspace/engine/evttype_index_ruleset.h @@ -42,27 +42,32 @@ class evttype_index_ruleset: public filter_ruleset void add( const falco_rule& rule, std::shared_ptr condition) override; - + void clear() override; - uint16_t ruleset_id(const std::string &name) override; + bool run(gen_event *evt, falco_rule& match, uint16_t rulset_id); - bool run(gen_event *evt, falco_rule& match, uint16_t ruleset = 0); - - uint64_t enabled_count(uint16_t ruleset = 0) override; + uint64_t enabled_count(uint16_t ruleset_id) override; void on_loading_complete() override; void enable( const std::string &substring, bool match_exact, - bool enabled, - uint16_t ruleset = 0) override; + uint16_t rulset_id) override; + + void disable( + const std::string &substring, + bool match_exact, + uint16_t rulset_id) override; void enable_tags( const std::set &tags, - bool enabled, - uint16_t ruleset = 0) override; + uint16_t rulset_id) override; + + void disable_tags( + const std::set &tags, + uint16_t rulset_id) override; // evttypes for a ruleset void enabled_evttypes( @@ -71,6 +76,19 @@ class evttype_index_ruleset: public filter_ruleset private: + // Helper used by enable()/disable() + void enable_disable( + const std::string &substring, + bool match_exact, + bool enabled, + uint16_t rulset_id); + + // Helper used by enable_tags()/disable_tags() + void enable_disable_tags( + const std::set &tags, + bool enabled, + uint16_t rulset_id); + struct filter_wrapper { falco_rule rule; @@ -137,4 +155,4 @@ class evttype_index_ruleset_factory: public filter_ruleset_factory private: std::shared_ptr m_filter_factory; -}; \ No newline at end of file +}; diff --git a/userspace/engine/ruleset.h b/userspace/engine/ruleset.h index 33bdb27340d..8ae5475e2c8 100644 --- a/userspace/engine/ruleset.h +++ b/userspace/engine/ruleset.h @@ -44,21 +44,13 @@ class filter_ruleset virtual void add( const falco_rule& rule, std::shared_ptr condition) = 0; - + /*! \brief Erases the internal state. All rules are disabled in each ruleset, and all the rules defined with add() are removed. */ virtual void clear() = 0; - /*! - \brief Returns the numeric id of a ruleset given its name. - If a ruleset has been used before, its previously assigned id is - returned. Otherwise, a new ruleset is created with the given name - and a new id is assigned to it. - */ - virtual uint16_t ruleset_id(const std::string &name) = 0; - /*! \brief This is meant to be called after all rules have been added with add() and enabled on the given ruleset with enable()/enable_tags(). @@ -71,61 +63,87 @@ class filter_ruleset \param evt The event to be processed \param match If true is returned, this is filled-out with the rule that matched the event - \param ruleset The id of the ruleset to be used + \param ruleset_id The id of the ruleset to be used */ virtual bool run( gen_event *evt, falco_rule& match, - uint16_t ruleset = 0) = 0; - + uint16_t ruleset_id) = 0; + /*! \brief Returns the number of rules enabled in a given ruleset - \param ruleset The id of the ruleset to be used + \param ruleset_id The id of the ruleset to be used */ - virtual uint64_t enabled_count(uint16_t ruleset = 0) = 0; + virtual uint64_t enabled_count(uint16_t ruleset_id) = 0; /*! \brief Returns the union of the evttypes of all the rules enabled in a given ruleset - \param ruleset The id of the ruleset to be used + \param ruleset_id The id of the ruleset to be used */ virtual void enabled_evttypes( std::set &evttypes, uint16_t ruleset) = 0; /*! - \brief Find those rules matching the provided substring and set - their enabled status to enabled in a given ruleset. + \brief Find those rules matching the provided substring and enable + them in the provided ruleset. \param substring Substring used to match rule names. If empty, all rules are matched. \param match_exact If true, substring must be an exact match for a given rule name. Otherwise, any rules having substring as a substring in the rule name are enabled/disabled. - \param enabled The enabled status to set on all matching rules - \param ruleset The id of the ruleset to be used + \param ruleset_id The id of the ruleset to be used */ virtual void enable( const std::string &substring, bool match_exact, - bool enabled, - uint16_t ruleset = 0) = 0; + uint16_t ruleset_id) = 0; + + /*! + \brief Find those rules matching the provided substring and disable + them in the provided ruleset. + \param substring Substring used to match rule names. + If empty, all rules are matched. + \param match_exact If true, substring must be an exact match for a + given rule name. Otherwise, any rules having substring as a substring + in the rule name are enabled/disabled. + \param ruleset_id The id of the ruleset to be used + */ + virtual void disable( + const std::string &substring, + bool match_exact, + uint16_t ruleset_id) = 0; /*! \brief Find those rules that have a tag in the set of tags and - set their enabled status to enabled. Note that the enabled + enable them for the provided ruleset. Note that the enabled status is on the rules, and not the tags--if a rule R has - tags (a, b), and you call enable_tags([a], true) and then - enable_tags([b], false), R will be disabled despite the + tags (a, b), and you call enable_tags([a]) and then + disable_tags([b]), R will be disabled despite the fact it has tag a and was enabled by the first call to enable_tags. \param substring Tags used to match ruless - \param enabled The enabled status to set on all matching rules - \param ruleset The id of the ruleset to be used + \param ruleset_id The id of the ruleset to be used */ virtual void enable_tags( const std::set &tags, - bool enabled, - uint16_t ruleset = 0) = 0; + uint16_t ruleset_id) = 0; + + /*! + \brief Find those rules that have a tag in the set of tags and + disable them for the provided ruleset. Note that the disabled + status is on the rules, and not the tags--if a rule R has + tags (a, b), and you call enable_tags([a]) and then + disable_tags([b]), R will be disabled despite the + fact it has tag a and was enabled by the first call to + enable_tags. + \param substring Tags used to match ruless + \param ruleset_id The id of the ruleset to be used + */ + virtual void disable_tags( + const std::set &tags, + uint16_t ruleset_id) = 0; }; /*! From 059b3ad9fcb69de83b8402630579ab80c0944872 Mon Sep 17 00:00:00 2001 From: Jason Dellaluce Date: Mon, 23 May 2022 10:35:35 +0000 Subject: [PATCH 07/10] test(userspace/engine): adapt test_rulesets to new definitions Signed-off-by: Jason Dellaluce --- tests/engine/test_rulesets.cpp | 202 +++++++++++++++++---------------- 1 file changed, 105 insertions(+), 97 deletions(-) diff --git a/tests/engine/test_rulesets.cpp b/tests/engine/test_rulesets.cpp index ca03c5d0716..d782a1764ac 100644 --- a/tests/engine/test_rulesets.cpp +++ b/tests/engine/test_rulesets.cpp @@ -14,198 +14,206 @@ See the License for the specific language governing permissions and limitations under the License. */ -#include "ruleset.h" +#include "falco_common.h" +#include "evttype_index_ruleset.h" +#include #include static bool exact_match = true; static bool substring_match = false; -static bool enabled = true; -static bool disabled = false; static uint16_t default_ruleset = 0; static uint16_t non_default_ruleset = 3; static uint16_t other_non_default_ruleset = 2; static std::set tags = {"some_tag", "some_other_tag"}; static std::set evttypes = { ppm_event_type::PPME_GENERIC_E }; -static std::shared_ptr create_filter() +static std::shared_ptr create_filter() { - // The actual contents of the filters don't matter here. - sinsp_filter_compiler compiler(NULL, "evt.type=open"); - sinsp_filter *f = compiler.compile(); - - std::shared_ptr ret(f); + libsinsp::filter::parser parser("evt.type=open"); + std::shared_ptr ret(parser.parse()); + return ret; +} +static std::shared_ptr create_ruleset() +{ + std::shared_ptr f(new sinsp_filter_factory(NULL)); + std::shared_ptr ret(new evttype_index_ruleset(f)); return ret; } TEST_CASE("Should enable/disable on ruleset", "[rulesets]") { - falco_ruleset r; - std::shared_ptr filter = create_filter(); - string rule_name = "one_rule"; - string source = "syscall"; + auto r = create_ruleset(); + auto filter = create_filter(); + falco_rule rule; + rule.name = "one_rule"; + rule.source = falco_common::syscall_source; + rule.tags = tags; - r.add(source, rule_name, tags, evttypes, filter); + r->add(rule, filter); SECTION("Should enable/disable for exact match w/ default ruleset") { - r.enable("one_rule", exact_match, enabled); - REQUIRE(r.num_rules_for_ruleset(default_ruleset) == 1); + r->enable("one_rule", exact_match, default_ruleset); + REQUIRE(r->enabled_count(default_ruleset) == 1); - r.enable("one_rule", exact_match, disabled); - REQUIRE(r.num_rules_for_ruleset(default_ruleset) == 0); + r->disable("one_rule", exact_match, default_ruleset); + REQUIRE(r->enabled_count(default_ruleset) == 0); } SECTION("Should enable/disable for exact match w/ specific ruleset") { - r.enable("one_rule", exact_match, enabled, non_default_ruleset); - REQUIRE(r.num_rules_for_ruleset(non_default_ruleset) == 1); - REQUIRE(r.num_rules_for_ruleset(default_ruleset) == 0); - REQUIRE(r.num_rules_for_ruleset(other_non_default_ruleset) == 0); - - r.enable("one_rule", exact_match, disabled, non_default_ruleset); - REQUIRE(r.num_rules_for_ruleset(non_default_ruleset) == 0); - REQUIRE(r.num_rules_for_ruleset(default_ruleset) == 0); - REQUIRE(r.num_rules_for_ruleset(other_non_default_ruleset) == 0); + r->enable("one_rule", exact_match, non_default_ruleset); + REQUIRE(r->enabled_count(non_default_ruleset) == 1); + REQUIRE(r->enabled_count(default_ruleset) == 0); + REQUIRE(r->enabled_count(other_non_default_ruleset) == 0); + + r->disable("one_rule", exact_match, non_default_ruleset); + REQUIRE(r->enabled_count(non_default_ruleset) == 0); + REQUIRE(r->enabled_count(default_ruleset) == 0); + REQUIRE(r->enabled_count(other_non_default_ruleset) == 0); } SECTION("Should not enable for exact match different rule name") { - r.enable("some_other_rule", exact_match, enabled); - REQUIRE(r.num_rules_for_ruleset(default_ruleset) == 0); + r->enable("some_other_rule", exact_match, default_ruleset); + REQUIRE(r->enabled_count(default_ruleset) == 0); } SECTION("Should enable/disable for exact match w/ substring and default ruleset") { - r.enable("one_rule", substring_match, enabled); - REQUIRE(r.num_rules_for_ruleset(default_ruleset) == 1); + r->enable("one_rule", substring_match, default_ruleset); + REQUIRE(r->enabled_count(default_ruleset) == 1); - r.enable("one_rule", substring_match, disabled); - REQUIRE(r.num_rules_for_ruleset(default_ruleset) == 0); + r->disable("one_rule", substring_match, default_ruleset); + REQUIRE(r->enabled_count(default_ruleset) == 0); } SECTION("Should not enable for substring w/ exact_match") { - r.enable("one_", exact_match, enabled); - REQUIRE(r.num_rules_for_ruleset(default_ruleset) == 0); + r->enable("one_", exact_match, default_ruleset); + REQUIRE(r->enabled_count(default_ruleset) == 0); } SECTION("Should enable/disable for prefix match w/ default ruleset") { - r.enable("one_", substring_match, enabled); - REQUIRE(r.num_rules_for_ruleset(default_ruleset) == 1); + r->enable("one_", substring_match, default_ruleset); + REQUIRE(r->enabled_count(default_ruleset) == 1); - r.enable("one_", substring_match, disabled); - REQUIRE(r.num_rules_for_ruleset(default_ruleset) == 0); + r->disable("one_", substring_match, default_ruleset); + REQUIRE(r->enabled_count(default_ruleset) == 0); } SECTION("Should enable/disable for suffix match w/ default ruleset") { - r.enable("_rule", substring_match, enabled); - REQUIRE(r.num_rules_for_ruleset(default_ruleset) == 1); + r->enable("_rule", substring_match, default_ruleset); + REQUIRE(r->enabled_count(default_ruleset) == 1); - r.enable("_rule", substring_match, disabled); - REQUIRE(r.num_rules_for_ruleset(default_ruleset) == 0); + r->disable("_rule", substring_match, default_ruleset); + REQUIRE(r->enabled_count(default_ruleset) == 0); } SECTION("Should enable/disable for substring match w/ default ruleset") { - r.enable("ne_ru", substring_match, enabled); - REQUIRE(r.num_rules_for_ruleset(default_ruleset) == 1); + r->enable("ne_ru", substring_match, default_ruleset); + REQUIRE(r->enabled_count(default_ruleset) == 1); - r.enable("ne_ru", substring_match, disabled); - REQUIRE(r.num_rules_for_ruleset(default_ruleset) == 0); + r->disable("ne_ru", substring_match, default_ruleset); + REQUIRE(r->enabled_count(default_ruleset) == 0); } SECTION("Should enable/disable for substring match w/ specific ruleset") { - r.enable("ne_ru", substring_match, enabled, non_default_ruleset); - REQUIRE(r.num_rules_for_ruleset(non_default_ruleset) == 1); - REQUIRE(r.num_rules_for_ruleset(default_ruleset) == 0); - REQUIRE(r.num_rules_for_ruleset(other_non_default_ruleset) == 0); - - r.enable("ne_ru", substring_match, disabled, non_default_ruleset); - REQUIRE(r.num_rules_for_ruleset(non_default_ruleset) == 0); - REQUIRE(r.num_rules_for_ruleset(default_ruleset) == 0); - REQUIRE(r.num_rules_for_ruleset(other_non_default_ruleset) == 0); + r->enable("ne_ru", substring_match, non_default_ruleset); + REQUIRE(r->enabled_count(non_default_ruleset) == 1); + REQUIRE(r->enabled_count(default_ruleset) == 0); + REQUIRE(r->enabled_count(other_non_default_ruleset) == 0); + + r->disable("ne_ru", substring_match, non_default_ruleset); + REQUIRE(r->enabled_count(non_default_ruleset) == 0); + REQUIRE(r->enabled_count(default_ruleset) == 0); + REQUIRE(r->enabled_count(other_non_default_ruleset) == 0); } SECTION("Should enable/disable for tags w/ default ruleset") { std::set want_tags = {"some_tag"}; - r.enable_tags(want_tags, enabled); - REQUIRE(r.num_rules_for_ruleset(default_ruleset) == 1); + r->enable_tags(want_tags, default_ruleset); + REQUIRE(r->enabled_count(default_ruleset) == 1); - r.enable_tags(want_tags, disabled); - REQUIRE(r.num_rules_for_ruleset(default_ruleset) == 0); + r->disable_tags(want_tags, default_ruleset); + REQUIRE(r->enabled_count(default_ruleset) == 0); } SECTION("Should enable/disable for tags w/ specific ruleset") { std::set want_tags = {"some_tag"}; - r.enable_tags(want_tags, enabled, non_default_ruleset); - REQUIRE(r.num_rules_for_ruleset(non_default_ruleset) == 1); - REQUIRE(r.num_rules_for_ruleset(default_ruleset) == 0); - REQUIRE(r.num_rules_for_ruleset(other_non_default_ruleset) == 0); + r->enable_tags(want_tags, non_default_ruleset); + REQUIRE(r->enabled_count(non_default_ruleset) == 1); + REQUIRE(r->enabled_count(default_ruleset) == 0); + REQUIRE(r->enabled_count(other_non_default_ruleset) == 0); - r.enable_tags(want_tags, disabled, non_default_ruleset); - REQUIRE(r.num_rules_for_ruleset(non_default_ruleset) == 0); - REQUIRE(r.num_rules_for_ruleset(default_ruleset) == 0); - REQUIRE(r.num_rules_for_ruleset(other_non_default_ruleset) == 0); + r->disable_tags(want_tags, non_default_ruleset); + REQUIRE(r->enabled_count(non_default_ruleset) == 0); + REQUIRE(r->enabled_count(default_ruleset) == 0); + REQUIRE(r->enabled_count(other_non_default_ruleset) == 0); } SECTION("Should not enable for different tags") { std::set want_tags = {"some_different_tag"}; - r.enable_tags(want_tags, enabled); - REQUIRE(r.num_rules_for_ruleset(non_default_ruleset) == 0); + r->enable_tags(want_tags, default_ruleset); + REQUIRE(r->enabled_count(non_default_ruleset) == 0); } SECTION("Should enable/disable for overlapping tags") { std::set want_tags = {"some_tag", "some_different_tag"}; - r.enable_tags(want_tags, enabled); - REQUIRE(r.num_rules_for_ruleset(default_ruleset) == 1); + r->enable_tags(want_tags, default_ruleset); + REQUIRE(r->enabled_count(default_ruleset) == 1); - r.enable_tags(want_tags, disabled); - REQUIRE(r.num_rules_for_ruleset(default_ruleset) == 0); + r->disable_tags(want_tags, default_ruleset); + REQUIRE(r->enabled_count(default_ruleset) == 0); } } TEST_CASE("Should enable/disable on ruleset for incremental adding tags", "[rulesets]") { - falco_ruleset r; - string source = "syscall"; - - std::shared_ptr rule1_filter = create_filter(); - string rule1_name = "one_rule"; - std::set rule1_tags = {"rule1_tag"}; - r.add(source, rule1_name, rule1_tags, evttypes, rule1_filter); - - std::shared_ptr rule2_filter = create_filter(); - string rule2_name = "two_rule"; - std::set rule2_tags = {"rule2_tag"}; - r.add(source, rule2_name, rule2_tags, evttypes, rule2_filter); + auto r = create_ruleset(); + + auto rule1_filter = create_filter(); + falco_rule rule1; + rule1.name = "one_rule"; + rule1.source = falco_common::syscall_source; + rule1.tags = {"rule1_tag"}; + r->add(rule1, rule1_filter); + + auto rule2_filter = create_filter(); + falco_rule rule2; + rule2.name = "two_rule"; + rule2.source = falco_common::syscall_source; + rule2.tags = {"rule2_tag"}; + r->add(rule2, rule2_filter); std::set want_tags; - want_tags = rule1_tags; - r.enable_tags(want_tags, enabled); - REQUIRE(r.num_rules_for_ruleset(default_ruleset) == 1); + want_tags = rule1.tags; + r->enable_tags(want_tags, default_ruleset); + REQUIRE(r->enabled_count(default_ruleset) == 1); - want_tags = rule2_tags; - r.enable_tags(want_tags, enabled); - REQUIRE(r.num_rules_for_ruleset(default_ruleset) == 2); + want_tags = rule2.tags; + r->enable_tags(want_tags, default_ruleset); + REQUIRE(r->enabled_count(default_ruleset) == 2); - r.enable_tags(want_tags, disabled); - REQUIRE(r.num_rules_for_ruleset(default_ruleset) == 1); + r->disable_tags(want_tags, default_ruleset); + REQUIRE(r->enabled_count(default_ruleset) == 1); - want_tags = rule1_tags; - r.enable_tags(want_tags, disabled); - REQUIRE(r.num_rules_for_ruleset(default_ruleset) == 0); + want_tags = rule1.tags; + r->disable_tags(want_tags, default_ruleset); + REQUIRE(r->enabled_count(default_ruleset) == 0); } From 33e4dfc7c90159fb054acdc439030f2d364999ab Mon Sep 17 00:00:00 2001 From: Jason Dellaluce Date: Mon, 23 May 2022 13:49:36 +0000 Subject: [PATCH 08/10] refactor(userspace/engine): refactor engine interface and internals This updates the engine to comply and work properly with the newly-introduced interface design. Signed-off-by: Jason Dellaluce --- userspace/engine/falco_engine.cpp | 96 ++++++++++++++++++------------- userspace/engine/falco_engine.h | 33 ++++++----- userspace/engine/falco_rule.h | 6 +- userspace/engine/falco_source.h | 8 ++- userspace/engine/rule_loader.cpp | 13 ++++- userspace/engine/rule_loader.h | 13 +++-- userspace/engine/ruleset.h | 5 +- 7 files changed, 106 insertions(+), 68 deletions(-) diff --git a/userspace/engine/falco_engine.cpp b/userspace/engine/falco_engine.cpp index 44aeb53b550..da1b234f534 100644 --- a/userspace/engine/falco_engine.cpp +++ b/userspace/engine/falco_engine.cpp @@ -38,7 +38,8 @@ const std::string falco_engine::s_default_ruleset = "falco-default-ruleset"; using namespace std; falco_engine::falco_engine(bool seed_rng) - : m_min_priority(falco_common::PRIORITY_DEBUG), + : m_next_ruleset_id(0), + m_min_priority(falco_common::PRIORITY_DEBUG), m_sampling_ratio(1), m_sampling_multiplier(0), m_replace_container_info(false) { @@ -46,6 +47,8 @@ falco_engine::falco_engine(bool seed_rng) { srandom((unsigned) getpid()); } + + m_default_ruleset_id = find_ruleset_id(s_default_ruleset); } falco_engine::~falco_engine() @@ -53,6 +56,7 @@ falco_engine::~falco_engine() m_rules.clear(); m_rule_loader.clear(); m_rule_stats_manager.clear(); + m_sources.clear(); } uint32_t falco_engine::engine_version() @@ -60,24 +64,24 @@ uint32_t falco_engine::engine_version() return (uint32_t) FALCO_ENGINE_VERSION; } -falco_source& falco_engine::find_source(const std::string& name) +falco_source* falco_engine::find_source(const std::string& name) { auto ret = m_sources.at(name); if(!ret) { throw falco_exception("Unknown event source " + name); } - return *ret; + return ret; } -falco_source& falco_engine::find_source(std::size_t index) +falco_source* falco_engine::find_source(std::size_t index) { auto ret = m_sources.at(index); if(!ret) { throw falco_exception("Unknown event source index " + to_string(index)); } - return *ret; + return ret; } // Return a key that uniquely represents a field class. @@ -168,20 +172,20 @@ void falco_engine::load_rules(const string &rules_content, bool verbose, bool al void falco_engine::load_rules(const string &rules_content, bool verbose, bool all_events, uint64_t &required_engine_version) { - rule_loader::configuration cfg(rules_content); - cfg.sources = m_sources; + rule_loader::configuration cfg(rules_content, m_sources); cfg.min_priority = m_min_priority; cfg.output_extra = m_extra; cfg.replace_output_container_info = m_replace_container_info; + cfg.default_ruleset_id = m_default_ruleset_id; std::ostringstream os; rule_reader reader; bool success = reader.load(cfg, m_rule_loader); if (success) { - for (auto &s : m_sources) + for (auto &src : m_sources) { - s.ruleset->clear(); + src.ruleset = src.ruleset_factory->new_ruleset(); } m_rules.clear(); success = m_rule_loader.compile(cfg, m_rules); @@ -244,7 +248,14 @@ void falco_engine::enable_rule(const string &substring, bool enabled, const stri for(auto &it : m_sources) { - it.ruleset->enable(substring, match_exact, enabled, ruleset_id); + if(enabled) + { + it.ruleset->enable(substring, match_exact, ruleset_id); + } + else + { + it.ruleset->disable(substring, match_exact, ruleset_id); + } } } @@ -255,7 +266,14 @@ void falco_engine::enable_rule_exact(const string &rule_name, bool enabled, cons for(auto &it : m_sources) { - it.ruleset->enable(rule_name, match_exact, enabled, ruleset_id); + if(enabled) + { + it.ruleset->enable(rule_name, match_exact, ruleset_id); + } + else + { + it.ruleset->disable(rule_name, match_exact, ruleset_id); + } } } @@ -265,7 +283,14 @@ void falco_engine::enable_rule_by_tag(const set &tags, bool enabled, con for(auto &it : m_sources) { - it.ruleset->enable_tags(tags, enabled, ruleset_id); + if(enabled) + { + it.ruleset->enable_tags(tags, ruleset_id); + } + else + { + it.ruleset->disable_tags(tags, ruleset_id); + } } } @@ -274,47 +299,47 @@ void falco_engine::set_min_priority(falco_common::priority_type priority) m_min_priority = priority; } -uint16_t falco_engine::find_ruleset_id( - const std::string &ruleset, const std::string &source) +uint16_t falco_engine::find_ruleset_id(const std::string &ruleset) { - return find_source(source).ruleset->ruleset_id(ruleset); + auto it = m_known_rulesets.lower_bound(ruleset); + if(it == m_known_rulesets.end() || it->first != ruleset) + { + it = m_known_rulesets.emplace_hint(it, + std::make_pair(ruleset, m_next_ruleset_id++)); + } + return it->second; } uint64_t falco_engine::num_rules_for_ruleset(const std::string &ruleset) { + uint16_t ruleset_id = find_ruleset_id(ruleset); uint64_t ret = 0; for (auto &src : m_sources) { - ret += src.ruleset->enabled_count(src.ruleset->ruleset_id(ruleset)); + ret += src.ruleset->enabled_count(ruleset_id); } return ret; } void falco_engine::evttypes_for_ruleset(std::string &source, std::set &evttypes, const std::string &ruleset) { - auto src = find_source(source); - src.ruleset->enabled_evttypes(evttypes, src.ruleset->ruleset_id(ruleset)); + find_source(source)->ruleset->enabled_evttypes(evttypes, find_ruleset_id(ruleset)); } std::shared_ptr falco_engine::create_formatter(const std::string &source, const std::string &output) { - return find_source(source).formatter_factory->create_formatter(output); + return find_source(source)->formatter_factory->create_formatter(output); } unique_ptr falco_engine::process_event(std::size_t source_idx, gen_event *ev, uint16_t ruleset_id) { - if(should_drop_evt()) - { - return unique_ptr(); - } - falco_rule rule; - if(!find_source(source_idx).ruleset->run(ev, rule, ruleset_id)) + if(should_drop_evt() || !find_source(source_idx)->ruleset->run(ev, rule, ruleset_id)) { return unique_ptr(); } - + unique_ptr res(new rule_result()); res->evt = ev; res->rule = rule.name; @@ -323,19 +348,19 @@ unique_ptr falco_engine::process_event(std::size_t so res->priority_num = rule.priority; res->tags = rule.tags; res->exception_fields = rule.exception_fields; - m_rule_stats_manager.on_event(m_rules, rule.id); + m_rule_stats_manager.on_event(rule); return res; } unique_ptr falco_engine::process_event(std::size_t source_idx, gen_event *ev) { - return process_event(source_idx, ev, find_source(source_idx).default_ruleset_id); + return process_event(source_idx, ev, m_default_ruleset_id); } std::size_t falco_engine::add_source(const std::string &source, std::shared_ptr filter_factory, std::shared_ptr formatter_factory) -{ +{ // evttype_index_ruleset is the default ruleset implementation std::shared_ptr ruleset_factory( new evttype_index_ruleset_factory(filter_factory)); @@ -353,7 +378,6 @@ std::size_t falco_engine::add_source(const std::string &source, src.formatter_factory = formatter_factory; src.ruleset_factory = ruleset_factory; src.ruleset = ruleset_factory->new_ruleset(); - src.default_ruleset_id = src.ruleset->ruleset_id(s_default_ruleset); return m_sources.insert(src, source); } @@ -434,20 +458,14 @@ bool falco_engine::check_plugin_requirements( return true; } -void falco_engine::clear_filters() +void falco_engine::complete_rule_loading() { - for(auto &src : m_sources) + for (auto &src : m_sources) { - src.ruleset = src.ruleset_factory->new_ruleset(); - src.default_ruleset_id = src.ruleset->ruleset_id(s_default_ruleset); + src.ruleset->on_loading_complete(); } } -void falco_engine::clear_loader() -{ - m_rule_loader.clear(); -} - void falco_engine::set_sampling_ratio(uint32_t sampling_ratio) { m_sampling_ratio = sampling_ratio; diff --git a/userspace/engine/falco_engine.h b/userspace/engine/falco_engine.h index 8df24fce68a..40782a9ee6c 100644 --- a/userspace/engine/falco_engine.h +++ b/userspace/engine/falco_engine.h @@ -90,6 +90,16 @@ class falco_engine // void enable_rule_by_tag(const std::set &tags, bool enabled, const std::string &ruleset = s_default_ruleset); + // + // Must be called after the engine has been configured and all rulesets + // have been loaded and enabled/disabled. + // This does not change the engine configuration nor the loaded/enabled rule + // setup, and does not affect the functional behavior. + // Internally, this can be used to release unused resources before starting + // processing events with process_event(). + // + void complete_rule_loading(); + // Only load rules having this priority or more severe. void set_min_priority(falco_common::priority_type priority); @@ -99,9 +109,7 @@ class falco_engine // to enable_rule/enable_rule_by_tag(), you should look up the // ruleset id and pass it to process_event(). // - uint16_t find_ruleset_id( - const std::string &ruleset, - const std::string &source = falco_common::syscall_source); + uint16_t find_ruleset_id(const std::string &ruleset); // // Return the number of falco rules enabled for the provided ruleset @@ -120,18 +128,6 @@ class falco_engine // void print_stats(); - // Clear all existing filters. - void clear_filters(); - - // - // Clear all the definitions of the internal rule loader (e.g. defined - // rules, macros, lists, engine/plugin version requirements). This is meant - // to be used to free-up memory at runtime when the definitions are not - // used anymore. Calling this between successive invocations of load_rules - // or load_rules_file can cause failures of features like appending. - // - void clear_loader(); - // // Set the sampling ratio, which can affect which events are // matched against the set of rules. @@ -239,8 +235,8 @@ class falco_engine private: indexed_vector m_sources; - falco_source& find_source(std::size_t index); - falco_source& find_source(const std::string& name); + falco_source* find_source(std::size_t index); + falco_source* find_source(const std::string& name); // // Determine whether the given event should be matched at all @@ -253,6 +249,8 @@ class falco_engine indexed_vector m_rules; stats_manager m_rule_stats_manager; + uint16_t m_next_ruleset_id; + std::map m_known_rulesets; falco_common::priority_type m_min_priority; // @@ -279,6 +277,7 @@ class falco_engine double m_sampling_multiplier; static const std::string s_default_ruleset; + uint32_t m_default_ruleset_id; std::string m_extra; bool m_replace_container_info; diff --git a/userspace/engine/falco_rule.h b/userspace/engine/falco_rule.h index 77aa8c79dd2..7cdbcf8a581 100644 --- a/userspace/engine/falco_rule.h +++ b/userspace/engine/falco_rule.h @@ -20,9 +20,13 @@ limitations under the License. #include #include "falco_common.h" +/*! + \brief Represents a rule in the Falco Engine. + The rule ID must be unique across all the rules loaded in the engine. +*/ struct falco_rule { - size_t id; + std::size_t id; std::string source; std::string name; std::string description; diff --git a/userspace/engine/falco_source.h b/userspace/engine/falco_source.h index ca70db942a7..826f278319b 100644 --- a/userspace/engine/falco_source.h +++ b/userspace/engine/falco_source.h @@ -19,14 +19,18 @@ limitations under the License. #include #include "ruleset.h" +/*! + \brief Represents a given data source used by the engine. + The ruleset of a source should be created through the ruleset factory + of the same data source. +*/ struct falco_source { std::string name; - uint16_t default_ruleset_id; std::shared_ptr ruleset; std::shared_ptr ruleset_factory; std::shared_ptr filter_factory; - std::shared_ptr formatter_factory; + std::shared_ptr formatter_factory; inline bool is_field_defined(std::string field) const { diff --git a/userspace/engine/rule_loader.cpp b/userspace/engine/rule_loader.cpp index 103aacfcc4a..ba7f686dbf0 100644 --- a/userspace/engine/rule_loader.cpp +++ b/userspace/engine/rule_loader.cpp @@ -648,7 +648,7 @@ void rule_loader::compile_rule_infos( apply_output_substitutions(cfg, rule.output); } - THROW(!is_format_valid(cfg.engine, r.source, rule.output, err), + THROW(!is_format_valid(*cfg.sources.at(r.source), rule.output, err), "Invalid output format '" + rule.output + "': '" + err + "'"); // construct rule definition and compile it to a filter @@ -662,7 +662,16 @@ void rule_loader::compile_rule_infos( auto rule_id = out.insert(rule, rule.name); out.at(rule_id)->id = rule_id; source->ruleset->add(*out.at(rule_id), ast); - source->ruleset->enable(rule.name, false, r.enabled); + + // By default rules are enabled/disabled for the default ruleset + if(r.enabled) + { + source->ruleset->enable(rule.name, true, cfg.default_ruleset_id); + } + else + { + source->ruleset->disable(rule.name, true, cfg.default_ruleset_id); + } } catch (falco_exception& e) { diff --git a/userspace/engine/rule_loader.h b/userspace/engine/rule_loader.h index 57964da3b78..47bb2c27414 100644 --- a/userspace/engine/rule_loader.h +++ b/userspace/engine/rule_loader.h @@ -66,14 +66,19 @@ class rule_loader */ struct configuration { - explicit configuration(const std::string& cont): content(cont) {} + explicit configuration( + const std::string& cont, + const indexed_vector& srcs) + : content(cont), sources(srcs) {} + const std::string& content; + const indexed_vector& sources; + std::vector errors; + std::vector warnings; std::string output_extra; + uint16_t default_ruleset_id; bool replace_output_container_info; falco_common::priority_type min_priority; - indexed_vector sources; - std::vector warnings; - std::vector errors; }; /*! diff --git a/userspace/engine/ruleset.h b/userspace/engine/ruleset.h index 8ae5475e2c8..82001c44f32 100644 --- a/userspace/engine/ruleset.h +++ b/userspace/engine/ruleset.h @@ -23,9 +23,8 @@ limitations under the License. #include /*! - \brief Represents a manager for rulesets.A ruleset represents a set of - enabled rules that is able to process events and find potential rule - matches. By convention, the ruleset with id = 0 is the default one. + \brief Manages a set of rulesets. A ruleset is a set of + enabled rules that is able to process events and find matches for those rules. */ class filter_ruleset { From 2f6f1a2736e1d933c9acfee93c9010aa3f9e70dc Mon Sep 17 00:00:00 2001 From: Jason Dellaluce Date: Mon, 23 May 2022 13:50:24 +0000 Subject: [PATCH 09/10] update(userspace/falco): sync falco with new engine definitions Signed-off-by: Jason Dellaluce --- userspace/falco/app_actions/load_rules_files.cpp | 3 --- userspace/falco/app_actions/open_inspector.cpp | 3 +++ 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/userspace/falco/app_actions/load_rules_files.cpp b/userspace/falco/app_actions/load_rules_files.cpp index 83a338b181c..68576eeba5a 100644 --- a/userspace/falco/app_actions/load_rules_files.cpp +++ b/userspace/falco/app_actions/load_rules_files.cpp @@ -123,9 +123,6 @@ application::run_result application::load_rules_files() return run_result::fatal(plugin_vers_err); } - // Free-up memory for the rule loader, which is not used from now on - m_state->engine->clear_loader(); - for (const auto& substring : m_options.disabled_rule_substrings) { falco_logger::log(LOG_INFO, "Disabling rules matching substring: " + substring + "\n"); diff --git a/userspace/falco/app_actions/open_inspector.cpp b/userspace/falco/app_actions/open_inspector.cpp index bbb2e20b8fb..0f9aa85d84a 100644 --- a/userspace/falco/app_actions/open_inspector.cpp +++ b/userspace/falco/app_actions/open_inspector.cpp @@ -26,6 +26,9 @@ typedef std::function inspector)> open_t; application::run_result application::open_inspector() { + // Notify engine that we finished loading and enabling all rules + m_state->engine->complete_rule_loading(); + if(is_capture_mode()) { // Try to open the trace file as a From 2de6182002c04fb1235fc767318036965c3d87f4 Mon Sep 17 00:00:00 2001 From: Jason Dellaluce Date: Tue, 24 May 2022 14:33:32 +0000 Subject: [PATCH 10/10] update(userspace/engine): rename ruleset.h in filter_ruleset.h Signed-off-by: Jason Dellaluce Co-authored-by: Leonardo Grasso --- userspace/engine/evttype_index_ruleset.h | 2 +- userspace/engine/falco_engine.h | 2 +- userspace/engine/falco_source.h | 2 +- userspace/engine/{ruleset.h => filter_ruleset.h} | 0 4 files changed, 3 insertions(+), 3 deletions(-) rename userspace/engine/{ruleset.h => filter_ruleset.h} (100%) diff --git a/userspace/engine/evttype_index_ruleset.h b/userspace/engine/evttype_index_ruleset.h index acdd1aaf1fb..176bab09600 100644 --- a/userspace/engine/evttype_index_ruleset.h +++ b/userspace/engine/evttype_index_ruleset.h @@ -22,7 +22,7 @@ limitations under the License. #include #include -#include "ruleset.h" +#include "filter_ruleset.h" #include "sinsp.h" #include "filter.h" #include "event.h" diff --git a/userspace/engine/falco_engine.h b/userspace/engine/falco_engine.h index 40782a9ee6c..81a307b7126 100644 --- a/userspace/engine/falco_engine.h +++ b/userspace/engine/falco_engine.h @@ -29,7 +29,7 @@ limitations under the License. #include #include "gen_filter.h" -#include "ruleset.h" +#include "filter_ruleset.h" #include "rule_loader.h" #include "stats_manager.h" #include "falco_common.h" diff --git a/userspace/engine/falco_source.h b/userspace/engine/falco_source.h index 826f278319b..cff54cf8083 100644 --- a/userspace/engine/falco_source.h +++ b/userspace/engine/falco_source.h @@ -17,7 +17,7 @@ limitations under the License. #pragma once #include -#include "ruleset.h" +#include "filter_ruleset.h" /*! \brief Represents a given data source used by the engine. diff --git a/userspace/engine/ruleset.h b/userspace/engine/filter_ruleset.h similarity index 100% rename from userspace/engine/ruleset.h rename to userspace/engine/filter_ruleset.h