new(metrics): add rules_counters_enabled option

falco.yaml

@@ -981,6 +981,8 @@ syscall_event_drops:
 # as Falco does not automatically rotate the file. It can be used in combination
 # with `output_rule`.
 #
+# `rules_counters_enabled`: Emit counts for each rule.
+#
 # `resource_utilization_enabled`: Emit CPU and memory usage metrics. CPU usage
 # is reported as a percentage of one CPU and can be normalized to the total
 # number of CPUs to determine overall usage. Memory metrics are provided in raw
@@ -1025,16 +1027,19 @@ syscall_event_drops:
 # values are included in the output.
 #
 # If metrics are enabled, the web server can be configured to activate the
-# corresponding Prometheus endpoint using webserver.prometheus_metrics_enabled.
+# corresponding Prometheus endpoint using `webserver.prometheus_metrics_enabled`.
+# Prometheus output can be used in combination with the other output options.
 #
 # todo: syscall_counters_enabled option
 metrics:
   enabled: false
   interval: 1h
   # Typically, in production, you only use `output_rule` or `output_file`, but not both. 
   # However, if you have a very unique use case, you can use both together.
+  # Set `webserver.prometheus_metrics_enabled` for Prometheus output.
   output_rule: true
   # output_file: /tmp/falco_stats.jsonl
+  rules_counters_enabled: true
   resource_utilization_enabled: true
   state_counters_enabled: true
   kernel_event_counters_enabled: true

unit_tests/engine/test_falco_utils.cpp

@@ -73,6 +73,14 @@ TEST(FalcoUtils, parse_prometheus_interval)
 	ASSERT_EQ(falco::utils::parse_prometheus_interval("200"), 0UL);
 }
 
+TEST(FalcoUtils, sanitize_metric_name)
+{
+	ASSERT_EQ(falco::utils::sanitize_metric_name("Testing rule   2 (CVE-2244)"), "Testing_rule_2_CVE_2244");
+	ASSERT_EQ(falco::utils::sanitize_metric_name("Testing rule__:2)"), "Testing_rule_:2");
+	ASSERT_EQ(falco::utils::sanitize_metric_name("This@is_a$test rule123"), "This_is_a_test_rule123");
+	ASSERT_EQ(falco::utils::sanitize_metric_name("RULEwith:special#characters"), "RULEwith:special_characters");
+}
+
 TEST(FalcoUtils, matches_wildcard)
 {
 	ASSERT_TRUE(falco::utils::matches_wildcard("*", "anything"));

userspace/engine/falco_engine.cpp

@@ -907,6 +907,11 @@ void falco_engine::print_stats() const
 	fprintf(stdout, "%s", out.c_str());
 }
 
+const stats_manager& falco_engine::get_rule_stats_manager() const
+{
+    return m_rule_stats_manager;
+}
+
 bool falco_engine::is_source_valid(const std::string &source) const
 {
 	return m_sources.at(source) != nullptr;

userspace/engine/falco_engine.h

@@ -149,11 +149,21 @@ class falco_engine
 	//
 	nlohmann::json describe_rule(std::string *rule_name, const std::vector<std::shared_ptr<sinsp_plugin>>& plugins) const;
 
+	//
+	// Return const /ref to rules stored in the Falco engine.
+	//
+	inline const indexed_vector<falco_rule>& get_rules() const { return m_rules; }
+
 	//
 	// Print statistics on how many events matched each rule.
 	//
 	void print_stats() const;
 
+	//
+	// Return const /ref to stats_manager to access current rules stats (how many events matched each rule so far).
+	//
+	const stats_manager& get_rule_stats_manager() const;
+
 	//
 	// Set the sampling ratio, which can affect which events are
 	// matched against the set of rules.

userspace/engine/falco_utils.cpp

@@ -151,6 +151,18 @@ std::string calculate_file_sha256sum(const std::string& filename)
 }
 #endif
 
+std::string sanitize_metric_name(const std::string& name)
+{
+	std::string sanitized_name = name;
+	RE2::GlobalReplace(&sanitized_name, "[^a-zA-Z0-9_:]", "_");
+	RE2::GlobalReplace(&sanitized_name, "_+", "_");
+	if (!sanitized_name.empty() && sanitized_name.back() == '_')
+	{
+		sanitized_name.pop_back();
+	}
+	return sanitized_name;
+}
+
 std::string wrap_text(const std::string& in, uint32_t indent, uint32_t line_len)
 {
 	std::istringstream is(in);

userspace/engine/falco_utils.h

@@ -31,6 +31,8 @@ uint64_t parse_prometheus_interval(std::string interval_str);
 std::string calculate_file_sha256sum(const std::string& filename);
 #endif
 
+std::string sanitize_metric_name(const std::string& name);
+
 std::string wrap_text(const std::string& in, uint32_t indent, uint32_t linelen);
 
 void readfile(const std::string& filename, std::string& data);

userspace/engine/stats_manager.h

@@ -63,6 +63,23 @@ class stats_manager
 		const indexed_vector<falco_rule>& rules,
 		std::string& out) const;
 
+	// Getter functions
+	inline const std::atomic<uint64_t>& get_total() const
+	{
+		return m_total;
+	}
+
+	inline const std::vector<std::unique_ptr<std::atomic<uint64_t>>>& get_by_priority() const
+	{
+		return m_by_priority;
+	}
+
+	inline const std::vector<std::unique_ptr<std::atomic<uint64_t>>>& get_by_rule_id() const
+	{
+		return m_by_rule_id;
+	}
+
+
 private:
 	std::atomic<uint64_t> m_total;
 	std::vector<std::unique_ptr<std::atomic<uint64_t>>> m_by_priority;

userspace/falco/app/actions/process_events.cpp

@@ -436,7 +436,7 @@ falco::app::run_result falco::app::actions::process_events(falco::app::state& s)
 	s.engine->complete_rule_loading();
 
 	// Initialize stats writer
-	auto statsw = std::make_shared<stats_writer>(s.outputs, s.config);
+	auto statsw = std::make_shared<stats_writer>(s.outputs, s.config, s.engine);
 	auto res = init_stats_writer(statsw, s.config, s.options.dry_run);
 
 	if (s.options.dry_run)

userspace/falco/configuration.cpp

@@ -72,7 +72,7 @@ falco_configuration::falco_configuration():
 	m_metrics_interval(5000),
 	m_metrics_stats_rule_enabled(false),
 	m_metrics_output_file(""),
-	m_metrics_flags((METRICS_V2_KERNEL_COUNTERS | METRICS_V2_LIBBPF_STATS | METRICS_V2_RESOURCE_UTILIZATION | METRICS_V2_STATE_COUNTERS)),
+	m_metrics_flags((METRICS_V2_KERNEL_COUNTERS | METRICS_V2_LIBBPF_STATS | METRICS_V2_RESOURCE_UTILIZATION | METRICS_V2_STATE_COUNTERS | METRICS_V2_RULE_COUNTERS)),
 	m_metrics_convert_memory_to_mb(true),
 	m_metrics_include_empty_values(false)
 {
@@ -535,20 +535,21 @@ void falco_configuration::load_yaml(const std::string& config_name)
 	m_metrics_output_file = config.get_scalar<std::string>("metrics.output_file", "");
 
 	m_metrics_flags = 0;
+	if (config.get_scalar<bool>("metrics.rules_counters_enabled", true))
+	{
+		m_metrics_flags |= METRICS_V2_RULE_COUNTERS;
+	}
 	if (config.get_scalar<bool>("metrics.resource_utilization_enabled", true))
 	{
 		m_metrics_flags |= METRICS_V2_RESOURCE_UTILIZATION;
-
 	}
 	if (config.get_scalar<bool>("metrics.state_counters_enabled", true))
 	{
 		m_metrics_flags |= METRICS_V2_STATE_COUNTERS;
-
 	}
 	if (config.get_scalar<bool>("metrics.kernel_event_counters_enabled", true))
 	{
 		m_metrics_flags |= METRICS_V2_KERNEL_COUNTERS;
-
 	}
 	if (config.get_scalar<bool>("metrics.libbpf_stats_enabled", true))
 	{

userspace/falco/falco_metrics.cpp

@@ -17,6 +17,8 @@ limitations under the License.
 
 #include "falco_metrics.h"
 
+#include "falco_utils.h"
+
 #include "app/state.h"
 
 #include <libsinsp/sinsp.h>
@@ -57,15 +59,17 @@ std::string falco_metrics::to_text(const falco::app::state& state)
 	for (const auto& source_info: state.source_infos)
 	{
 		sinsp *source_inspector = source_info.inspector.get();
-		inspectors.push_back(source_inspector);
-		metrics_collectors.push_back(libs::metrics::libs_metrics_collector(source_inspector, state.config->m_metrics_flags));
+		inspectors.emplace_back(source_inspector);
+		metrics_collectors.emplace_back(libs::metrics::libs_metrics_collector(source_inspector, state.config->m_metrics_flags));
 	}
 
 	libs::metrics::prometheus_metrics_converter prometheus_metrics_converter;
 	std::string prometheus_text;
 
 	for (auto* inspector: inspectors)
 	{
+		// Falco wrapper metrics
+		//
 		for (size_t i = 0; i < sizeof(all_driver_engines) / sizeof(const char*); i++)
 		{
 			if (inspector->check_current_engine(all_driver_engines[i]))
@@ -89,15 +93,15 @@ std::string falco_metrics::to_text(const falco::app::state& state)
 		{
 			fs::path fs_path = item.first;
 			std::string metric_name_file_sha256 = fs_path.filename().stem();
-			metric_name_file_sha256 = "falco.sha256_rule_file." + metric_name_file_sha256;
+			metric_name_file_sha256 = "falco.sha256_rule_file." + falco::utils::sanitize_metric_name(metric_name_file_sha256);
 			prometheus_text += prometheus_metrics_converter.convert_metric_to_text_prometheus(metric_name_file_sha256, "falcosecurity", "falco", {{metric_name_file_sha256, item.second}});
 		}
 
 		for (const auto& item : state.config.get()->m_loaded_configs_filenames_sha256sum)
 		{
 			fs::path fs_path = item.first;
 			std::string metric_name_file_sha256 = fs_path.filename().stem();
-			metric_name_file_sha256 = "falco.sha256_config_file." + metric_name_file_sha256;
+			metric_name_file_sha256 = "falco.sha256_config_file." + falco::utils::sanitize_metric_name(metric_name_file_sha256);
 			prometheus_text += prometheus_metrics_converter.convert_metric_to_text_prometheus(metric_name_file_sha256, "falcosecurity", "falco", {{metric_name_file_sha256, item.second}});
 		}
 #endif
@@ -106,26 +110,27 @@ std::string falco_metrics::to_text(const falco::app::state& state)
 		{
 			prometheus_text += prometheus_metrics_converter.convert_metric_to_text_prometheus("evt_source", "falcosecurity", "falco", {{"evt_source", source}});
 		}
-		std::vector<metrics_v2> static_metrics;
-		static_metrics.push_back(libs_metrics_collector.new_metric("start_ts",
+		std::vector<metrics_v2> additional_wrapper_metrics;
+
+		additional_wrapper_metrics.emplace_back(libs_metrics_collector.new_metric("start_ts",
 																	METRICS_V2_MISC,
 																	METRIC_VALUE_TYPE_U64,
 																	METRIC_VALUE_UNIT_TIME_TIMESTAMP_NS,
 																	METRIC_VALUE_METRIC_TYPE_NON_MONOTONIC_CURRENT,
 																	agent_info->start_ts_epoch));
-		static_metrics.push_back(libs_metrics_collector.new_metric("host_boot_ts",
+		additional_wrapper_metrics.emplace_back(libs_metrics_collector.new_metric("host_boot_ts",
 																	METRICS_V2_MISC,
 																	METRIC_VALUE_TYPE_U64,
 																	METRIC_VALUE_UNIT_TIME_TIMESTAMP_NS,
 																	METRIC_VALUE_METRIC_TYPE_NON_MONOTONIC_CURRENT,
 																	machine_info->boot_ts_epoch));
-		static_metrics.push_back(libs_metrics_collector.new_metric("host_num_cpus",
+		additional_wrapper_metrics.emplace_back(libs_metrics_collector.new_metric("host_num_cpus",
 																	METRICS_V2_MISC,
 																	METRIC_VALUE_TYPE_U32,
 																	METRIC_VALUE_UNIT_COUNT,
 																	METRIC_VALUE_METRIC_TYPE_NON_MONOTONIC_CURRENT,
 																	machine_info->num_cpus));
-		static_metrics.push_back(libs_metrics_collector.new_metric("outputs_queue_num_drops",
+		additional_wrapper_metrics.emplace_back(libs_metrics_collector.new_metric("outputs_queue_num_drops",
 																	METRICS_V2_MISC,
 																	METRIC_VALUE_TYPE_U64,
 																	METRIC_VALUE_UNIT_COUNT,
@@ -134,34 +139,78 @@ std::string falco_metrics::to_text(const falco::app::state& state)
 
 		auto now = std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::system_clock::now().time_since_epoch()).count();
 
-		static_metrics.push_back(libs_metrics_collector.new_metric("duration_sec",
+		additional_wrapper_metrics.emplace_back(libs_metrics_collector.new_metric("duration_sec",
 																	METRICS_V2_MISC,
 																	METRIC_VALUE_TYPE_U64,
 																	METRIC_VALUE_UNIT_TIME_S_COUNT,
 																	METRIC_VALUE_METRIC_TYPE_MONOTONIC,
 																	(uint64_t)((now - agent_info->start_ts_epoch) / ONE_SECOND_IN_NS)));
 
-		for (auto metrics: static_metrics)
+		for (auto metric: additional_wrapper_metrics)
 		{
-			prometheus_metrics_converter.convert_metric_to_unit_convention(metrics);
-			prometheus_text += prometheus_metrics_converter.convert_metric_to_text_prometheus(metrics, "falcosecurity", "falco");
+			prometheus_metrics_converter.convert_metric_to_unit_convention(metric);
+			prometheus_text += prometheus_metrics_converter.convert_metric_to_text_prometheus(metric, "falcosecurity", "falco");
+		}
+
+		// Falco metrics categories
+		//
+		// rules_counters_enabled
+		if(state.config->m_metrics_flags & METRICS_V2_RULE_COUNTERS)
+		{
+			const stats_manager& rule_stats_manager = state.engine->get_rule_stats_manager();
+			const indexed_vector<falco_rule>& rules = state.engine->get_rules();
+			auto metric = libs_metrics_collector.new_metric("rules.matches_total",
+																	METRICS_V2_RULE_COUNTERS,
+																	METRIC_VALUE_TYPE_U64,
+																	METRIC_VALUE_UNIT_COUNT,
+																	METRIC_VALUE_METRIC_TYPE_MONOTONIC,
+																	rule_stats_manager.get_total().load());
+
+			prometheus_metrics_converter.convert_metric_to_unit_convention(metric);
+			prometheus_text += prometheus_metrics_converter.convert_metric_to_text_prometheus(metric, "falcosecurity", "falco");
+			const std::vector<std::unique_ptr<std::atomic<uint64_t>>>& rules_by_id = rule_stats_manager.get_by_rule_id();
+			for (size_t i = 0; i < rules_by_id.size(); i++)
+			{
+				auto rule = rules.at(i);
+				std::string rules_metric_name = "rules." + falco::utils::sanitize_metric_name(rule->name);
+				// Separate processing of rules counter metrics given we add extra tags
+				auto metric = libs_metrics_collector.new_metric(rules_metric_name.c_str(),
+																	METRICS_V2_RULE_COUNTERS,
+																	METRIC_VALUE_TYPE_U64,
+																	METRIC_VALUE_UNIT_COUNT,
+																	METRIC_VALUE_METRIC_TYPE_MONOTONIC,
+																	rules_by_id[i]->load());
+				prometheus_metrics_converter.convert_metric_to_unit_convention(metric);
+				const std::map<std::string, std::string>& const_labels = {
+					{"priority", std::to_string(rule->priority)},
+					{"source", rule->source},
+					{"tags", concat_set_in_order(rule->tags)}
+				};
+				prometheus_text += prometheus_metrics_converter.convert_metric_to_text_prometheus(metric, "falcosecurity", "falco", const_labels);
+			}
 		}
 	}
 
+	// Libs metrics categories
+	//
+	// resource_utilization_enabled
+	// state_counters_enabled
+	// kernel_event_counters_enabled
+	// libbpf_stats_enabled
 	for (auto metrics_collector: metrics_collectors)
 	{
 		metrics_collector.snapshot();
 		auto metrics_snapshot = metrics_collector.get_metrics();
 
-		for (auto& metrics: metrics_snapshot)
+		for (auto& metric: metrics_snapshot)
 		{
-			prometheus_metrics_converter.convert_metric_to_unit_convention(metrics);
+			prometheus_metrics_converter.convert_metric_to_unit_convention(metric);
 			std::string namespace_name = "scap";
-			if (metrics.flags & METRICS_V2_RESOURCE_UTILIZATION || metrics.flags & METRICS_V2_KERNEL_COUNTERS)
+			if (metric.flags & METRICS_V2_RESOURCE_UTILIZATION || metric.flags & METRICS_V2_KERNEL_COUNTERS)
 			{
 				namespace_name = "falco";
 			}
-			prometheus_text += prometheus_metrics_converter.convert_metric_to_text_prometheus(metrics, "falcosecurity", namespace_name);
+			prometheus_text += prometheus_metrics_converter.convert_metric_to_text_prometheus(metric, "falcosecurity", namespace_name);
 		}
 
 	}