From b354c616dc8a6366744b0063dc1974de1f38286b Mon Sep 17 00:00:00 2001 From: Leonardo Parente Date: Thu, 10 Feb 2022 13:28:58 -0400 Subject: [PATCH] add configure group method and not pass that in constructor --- src/AbstractMetricsManager.h | 22 +++++++++++-- src/StreamHandler.h | 5 +-- src/handlers/dhcp/DhcpStreamHandler.h | 4 +-- src/handlers/dns/DnsStreamHandler.cpp | 47 +++++++++++++-------------- src/handlers/dns/DnsStreamHandler.h | 17 +++------- src/handlers/mock/MockStreamHandler.h | 4 +-- src/handlers/net/NetStreamHandler.h | 4 +-- src/handlers/pcap/PcapStreamHandler.h | 4 +-- src/tests/test_metrics.cpp | 2 +- 9 files changed, 59 insertions(+), 50 deletions(-) diff --git a/src/AbstractMetricsManager.h b/src/AbstractMetricsManager.h index e2786ce6b..0bce0d7e8 100644 --- a/src/AbstractMetricsManager.h +++ b/src/AbstractMetricsManager.h @@ -22,6 +22,8 @@ namespace visor { +constexpr size_t GROUP_SIZE = 64; + using json = nlohmann::json; class PeriodException : public std::runtime_error @@ -59,6 +61,8 @@ class AbstractMetricsBucket bool _recorded_stream = false; protected: + const std::bitset *_groups; + // merge the metrics of the specialized metric bucket virtual void specialized_merge(const AbstractMetricsBucket &other) = 0; @@ -195,6 +199,10 @@ class AbstractMetricsBucket } } + void configure_groups(const std::bitset *groups) { + _groups = groups; + } + virtual void to_json(json &j) const = 0; virtual void to_prometheus(std::stringstream &out, Metric::LabelMap add_labels = {}) const = 0; }; @@ -230,6 +238,7 @@ class AbstractMetricsManager */ bool _recorded_stream = false; + const std::bitset *_groups; private: /** * window maintenance @@ -314,7 +323,7 @@ class AbstractMetricsManager } public: - AbstractMetricsManager(const Configurable *window_config, const std::bitset<64> groups = std::bitset<64>()) + AbstractMetricsManager(const Configurable *window_config) : _metric_buckets{} , _deep_sampling_now{true} , _last_shift_tstamp{0, 0} @@ -339,7 +348,7 @@ class AbstractMetricsManager _next_shift_tstamp = _last_shift_tstamp; _next_shift_tstamp.tv_sec += AbstractMetricsManager::PERIOD_SEC; - _metric_buckets.emplace_front(std::make_unique(groups)); + _metric_buckets.emplace_front(std::make_unique()); } @@ -406,6 +415,15 @@ class AbstractMetricsManager return _metric_buckets.at(period).get(); } + void configure_groups(const std::bitset *groups) { + std::unique_lock wl(_base_mutex); + std::shared_lock rl(_bucket_mutex); + _groups = groups; + for (const auto &bucket : _metric_buckets) { + bucket->configure_groups(groups); + } + } + MetricsBucketClass *live_bucket() { // CRITICAL PATH diff --git a/src/StreamHandler.h b/src/StreamHandler.h index d67ac40ec..b435d7a39 100644 --- a/src/StreamHandler.h +++ b/src/StreamHandler.h @@ -44,6 +44,7 @@ class StreamMetricsHandler : public StreamHandler protected: std::unique_ptr _metrics; + std::bitset _groups; void common_info_json(json &j) const { @@ -76,10 +77,10 @@ class StreamMetricsHandler : public StreamHandler } public: - StreamMetricsHandler(const std::string &name, const Configurable *window_config, const std::bitset<64> groups = std::bitset<64>()) + StreamMetricsHandler(const std::string &name, const Configurable *window_config) : StreamHandler(name) { - _metrics = std::make_unique(window_config, groups); + _metrics = std::make_unique(window_config); } const MetricsManagerClass *metrics() const diff --git a/src/handlers/dhcp/DhcpStreamHandler.h b/src/handlers/dhcp/DhcpStreamHandler.h index bef5c3ed3..960283d65 100644 --- a/src/handlers/dhcp/DhcpStreamHandler.h +++ b/src/handlers/dhcp/DhcpStreamHandler.h @@ -43,7 +43,7 @@ class DhcpMetricsBucket final : public visor::AbstractMetricsBucket counters _counters; public: - DhcpMetricsBucket(const std::bitset<64> groups = std::bitset<64>()) + DhcpMetricsBucket() { set_event_rate_info("dhcp", {"rates", "total"}, "Rate of all DHCP wire packets (combined ingress and egress) per second"); set_num_events_info("dhcp", {"wire_packets", "total"}, "Total DHCP wire packets"); @@ -70,7 +70,7 @@ class DhcpMetricsBucket final : public visor::AbstractMetricsBucket class DhcpMetricsManager final : public visor::AbstractMetricsManager { public: - DhcpMetricsManager(const Configurable *window_config, const std::bitset<64> groups) + DhcpMetricsManager(const Configurable *window_config) : visor::AbstractMetricsManager(window_config) { } diff --git a/src/handlers/dns/DnsStreamHandler.cpp b/src/handlers/dns/DnsStreamHandler.cpp index d246306d1..07729ff46 100644 --- a/src/handlers/dns/DnsStreamHandler.cpp +++ b/src/handlers/dns/DnsStreamHandler.cpp @@ -21,7 +21,7 @@ namespace visor::handler::dns { DnsStreamHandler::DnsStreamHandler(const std::string &name, InputStream *stream, const Configurable *window_config, StreamHandler *handler) - : StreamMetricsHandler(name, window_config, _process_dns_groups(window_config)) + : StreamMetricsHandler(name, window_config) { if (handler) { throw StreamHandlerException(fmt::format("DnsStreamHandler: unsupported upstream chained stream handler {}", handler->name())); @@ -35,8 +35,6 @@ DnsStreamHandler::DnsStreamHandler(const std::string &name, InputStream *stream, if (!_pcap_stream && !_mock_stream && !_dnstap_stream) { throw StreamHandlerException(fmt::format("DnsStreamHandler: unsupported input stream {}", stream->name())); } - - _g_enabled = _process_dns_groups(window_config); } void DnsStreamHandler::start() @@ -45,6 +43,8 @@ void DnsStreamHandler::start() return; } + _process_dns_groups(); + // Setup Filters if (config_exists("exclude_noerror") && config_get("exclude_noerror")) { _f_enabled.set(Filters::ExcludingRCode); @@ -224,13 +224,11 @@ void DnsStreamHandler::tcp_message_ready_cb(int8_t side, const pcpp::TcpStreamDa // data is freed upon return }; - if (_g_enabled.test(group::DnsMetrics::TopDnsWire)) { - if (!iter->second.sessionData[side]) { - iter->second.sessionData[side] = std::make_unique(got_dns_message); - } - - iter->second.sessionData[side]->receive_dns_wire_data(tcpData.getData(), tcpData.getDataLength()); + if (!iter->second.sessionData[side]) { + iter->second.sessionData[side] = std::make_unique(got_dns_message); } + + iter->second.sessionData[side]->receive_dns_wire_data(tcpData.getData(), tcpData.getDataLength()); } void DnsStreamHandler::tcp_connection_start_cb(const pcpp::ConnectionData &connectionData) @@ -313,28 +311,27 @@ bool DnsStreamHandler::_filtering(DnsLayer &payload, [[maybe_unused]] PacketDire return true; } -const std::bitset<64> DnsStreamHandler::_process_dns_groups(const Configurable *metrics_config) +void DnsStreamHandler::_process_dns_groups() { // default enabled groups - std::bitset<64> groups; - groups.set(group::DnsMetrics::Counters); - groups.set(group::DnsMetrics::TopDnsWire); - groups.set(group::DnsMetrics::DnsTransactions); - groups.set(group::DnsMetrics::TopQnames); - - if (metrics_config->config_exists("enable")) { - for (const auto &group : metrics_config->config_get("enable")) { - groups.set(_group_metrics.at(group)); + _groups.set(group::DnsMetrics::Counters); + _groups.set(group::DnsMetrics::TopDnsWire); + _groups.set(group::DnsMetrics::DnsTransactions); + _groups.set(group::DnsMetrics::TopQnames); + + if (config_exists("enable")) { + for (const auto &group : config_get("enable")) { + _groups.set(_group_metrics.at(group)); } } - if (metrics_config->config_exists("disable")) { - for (const auto &group : metrics_config->config_get("disable")) { - groups.reset(_group_metrics.at(group)); + if (config_exists("disable")) { + for (const auto &group : config_get("disable")) { + _groups.reset(_group_metrics.at(group)); } } - return groups; + _metrics->configure_groups(&_groups); } void DnsMetricsBucket::specialized_merge(const AbstractMetricsBucket &o) @@ -594,7 +591,7 @@ void DnsMetricsBucket::process_dns_layer(bool deep, DnsLayer &payload, bool dnst _dns_qnameCard.update(name); _dns_topQType.update(query->getDnsType()); - if (_groups.test(group::DnsMetrics::TopQnames)) { + if (_groups->test(group::DnsMetrics::TopQnames)) { if (payload.getDnsHeader()->queryOrResponse == response) { switch (payload.getDnsHeader()->responseCode) { case SrvFail: @@ -728,7 +725,7 @@ void DnsMetricsManager::process_dns_layer(DnsLayer &payload, PacketDirection dir // process in the "live" bucket. this will parse the resources if we are deep sampling live_bucket()->process_dns_layer(_deep_sampling_now, payload, false, l3, l4, port); - if (_groups.test(group::DnsMetrics::DnsTransactions)) { + if (_groups->test(group::DnsMetrics::DnsTransactions)) { // handle dns transactions (query/response pairs) if (payload.getDnsHeader()->queryOrResponse == QR::response) { auto xact = _qr_pair_manager.maybe_end_transaction(flowkey, payload.getDnsHeader()->transactionID, stamp); diff --git a/src/handlers/dns/DnsStreamHandler.h b/src/handlers/dns/DnsStreamHandler.h index 0d65347d7..0c20e95b0 100644 --- a/src/handlers/dns/DnsStreamHandler.h +++ b/src/handlers/dns/DnsStreamHandler.h @@ -42,8 +42,6 @@ class DnsMetricsBucket final : public visor::AbstractMetricsBucket protected: mutable std::shared_mutex _mutex; - const std::bitset<64> _groups; - Quantile _dnsXactFromTimeUs; Quantile _dnsXactToTimeUs; @@ -102,9 +100,8 @@ class DnsMetricsBucket final : public visor::AbstractMetricsBucket counters _counters; public: - DnsMetricsBucket(const std::bitset<64> groups = std::bitset<64>()) - : _groups(groups) - , _dnsXactFromTimeUs("dns", {"xact", "out", "quantiles_us"}, "Quantiles of transaction timing (query/reply pairs) when host is client, in microseconds") + DnsMetricsBucket() + : _dnsXactFromTimeUs("dns", {"xact", "out", "quantiles_us"}, "Quantiles of transaction timing (query/reply pairs) when host is client, in microseconds") , _dnsXactToTimeUs("dns", {"xact", "in", "quantiles_us"}, "Quantiles of transaction timing (query/reply pairs) when host is server, in microseconds") , _dns_qnameCard("dns", {"cardinality", "qname"}, "Cardinality of unique QNAMES, both ingress and egress") , _dns_topQname2("dns", "qname", {"top_qname2"}, "Top QNAMES, aggregated at a depth of two labels") @@ -161,16 +158,13 @@ class DnsMetricsBucket final : public visor::AbstractMetricsBucket class DnsMetricsManager final : public visor::AbstractMetricsManager { - - const std::bitset<64> _groups; QueryResponsePairMgr _qr_pair_manager; float _to90th{0.0}; float _from90th{0.0}; public: - DnsMetricsManager(const Configurable *window_config, const std::bitset<64> groups) - : visor::AbstractMetricsManager(window_config, groups) - , _groups(groups) + DnsMetricsManager(const Configurable *window_config) + : visor::AbstractMetricsManager(window_config) { } @@ -280,10 +274,9 @@ class DnsStreamHandler final : public visor::StreamMetricsHandler _g_enabled; bool _filtering(DnsLayer &payload, PacketDirection dir, pcpp::ProtocolType l3, pcpp::ProtocolType l4, uint16_t port, timespec stamp); - const std::bitset<64> _process_dns_groups(const Configurable *metrics_config); + void _process_dns_groups(); public: DnsStreamHandler(const std::string &name, InputStream *stream, const Configurable *window_config, StreamHandler *handler = nullptr); diff --git a/src/handlers/mock/MockStreamHandler.h b/src/handlers/mock/MockStreamHandler.h index 0d440114d..b318d84eb 100644 --- a/src/handlers/mock/MockStreamHandler.h +++ b/src/handlers/mock/MockStreamHandler.h @@ -35,7 +35,7 @@ class MockMetricsBucket final : public visor::AbstractMetricsBucket counters _counters; public: - MockMetricsBucket(const std::bitset<64> groups = std::bitset<64>()) + MockMetricsBucket() { } @@ -57,7 +57,7 @@ class MockMetricsBucket final : public visor::AbstractMetricsBucket class MockMetricsManager final : public visor::AbstractMetricsManager { public: - MockMetricsManager(const Configurable *window_config, const std::bitset<64> groups) + MockMetricsManager(const Configurable *window_config) : visor::AbstractMetricsManager(window_config) { } diff --git a/src/handlers/net/NetStreamHandler.h b/src/handlers/net/NetStreamHandler.h index 7efc698be..d8169dfaa 100644 --- a/src/handlers/net/NetStreamHandler.h +++ b/src/handlers/net/NetStreamHandler.h @@ -62,7 +62,7 @@ class NetworkMetricsBucket final : public visor::AbstractMetricsBucket Rate _rate_out; public: - NetworkMetricsBucket(const std::bitset<64> groups = std::bitset<64>()) + NetworkMetricsBucket() : _srcIPCard("packets", {"cardinality", "src_ips_in"}, "Source IP cardinality") , _dstIPCard("packets", {"cardinality", "dst_ips_out"}, "Destination IP cardinality") , _topGeoLoc("packets", "geo_loc", {"top_geoLoc"}, "Top GeoIP locations") @@ -105,7 +105,7 @@ class NetworkMetricsBucket final : public visor::AbstractMetricsBucket class NetworkMetricsManager final : public visor::AbstractMetricsManager { public: - NetworkMetricsManager(const Configurable *window_config, const std::bitset<64> groups) + NetworkMetricsManager(const Configurable *window_config) : visor::AbstractMetricsManager(window_config) { } diff --git a/src/handlers/pcap/PcapStreamHandler.h b/src/handlers/pcap/PcapStreamHandler.h index acefce30d..02c0c578d 100644 --- a/src/handlers/pcap/PcapStreamHandler.h +++ b/src/handlers/pcap/PcapStreamHandler.h @@ -42,7 +42,7 @@ class PcapMetricsBucket final : public visor::AbstractMetricsBucket counters _counters; public: - PcapMetricsBucket(const std::bitset<64> groups = std::bitset<64>()) + PcapMetricsBucket() { } @@ -65,7 +65,7 @@ class PcapMetricsBucket final : public visor::AbstractMetricsBucket class PcapMetricsManager final : public visor::AbstractMetricsManager { public: - PcapMetricsManager(const Configurable *window_config, const std::bitset<64> groups) + PcapMetricsManager(const Configurable *window_config) : visor::AbstractMetricsManager(window_config) { } diff --git a/src/tests/test_metrics.cpp b/src/tests/test_metrics.cpp index f48976c34..c5835474d 100644 --- a/src/tests/test_metrics.cpp +++ b/src/tests/test_metrics.cpp @@ -6,7 +6,7 @@ using namespace visor; class TestMetricsBucket : public AbstractMetricsBucket { public: - TestMetricsBucket(const std::bitset<64> groups = std::bitset<64>()) + TestMetricsBucket() { } void specialized_merge([[maybe_unused]] const AbstractMetricsBucket &other)