Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

fix global configuration of period and deep sample for window config #211

Merged
merged 2 commits into from
Feb 10, 2022
Merged

fix global configuration of period and deep sample for window config #211

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
565481a
fix global configuration of period and deep sample for window config
weyrick Feb 9, 2022
c3f2210
explicit template type to fix linux
weyrick Feb 9, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
66 changes: 39 additions & 27 deletions cmd/pktvisord/main.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,8 +54,8 @@ static const char USAGE[] =
--no-track Don't send lightweight, anonymous usage metrics
--version Show version
Web Server Options:
-l HOST Run web server on the given host or IP [default: localhost]
-p PORT Run web server on the given port [default: 10853]
-l HOST Run web server on the given host or IP (default: localhost)
-p PORT Run web server on the given port (default: 10853)
--tls Enable TLS on the web server
--tls-cert FILE Use given TLS cert. Required if --tls is enabled.
--tls-key FILE Use given TLS private key. Required if --tls is enabled.
Expand All @@ -78,8 +78,8 @@ static const char USAGE[] =
--prometheus Ignored, Prometheus output always enabled (left for backwards compatibility)
--prom-instance ID Optionally set the 'instance' label to given ID
Handler Module Defaults:
--max-deep-sample N Never deep sample more than N% of streams (an int between 0 and 100) [default: 100]
--periods P Hold this many 60 second time periods of history in memory [default: 5]
--max-deep-sample N Never deep sample more than N% of streams (an int between 0 and 100) (default: 100)
--periods P Hold this many 60 second time periods of history in memory (default: 5)
pcap Input Module Options: (applicable to default policy when IFACE is specified only)
-b BPF Filter packets using the given tcpdump compatible filter expression. Example: "port 53"
-H HOSTSPEC Specify subnets (comma separated) to consider HOST, in CIDR form. In live capture this
Expand Down Expand Up @@ -150,7 +150,7 @@ void fill_cmd_options(std::map<std::string, docopt::value> args, CmdOptions &opt
exit(EXIT_FAILURE);
}

options.config = std::make_pair(true, config_file);
options.config = {true, config_file};

if (config_file["visor"]["config"] && config_file["visor"]["config"].IsMap()) {
config = config_file["visor"]["config"];
Expand All @@ -170,74 +170,82 @@ void fill_cmd_options(std::map<std::string, docopt::value> args, CmdOptions &opt
options.prometheus = (config["prometheus"] && config["prometheus"].as<bool>()) || args["--prometheus"].asBool();

if (args["--log-file"]) {
options.log_file = std::make_pair(true, args["--log-file"].asString());
options.log_file = {true, args["--log-file"].asString()};
} else if (config["log_file"]) {
options.log_file = std::make_pair(true, config["log_file"].as<std::string>());
options.log_file = {true, config["log_file"].as<std::string>()};
}

if (args["--prom-instance"]) {
options.prom_instance = std::make_pair(true, args["--prom-instance"].asString());
options.prom_instance = {true, args["--prom-instance"].asString()};
} else if (config["prom_instance"]) {
options.prom_instance = std::make_pair(true, config["prom_instance"].as<std::string>());
options.prom_instance = {true, config["prom_instance"].as<std::string>()};
}

if (args["--geo-city"]) {
options.geo_city = std::make_pair(true, args["--geo-city"].asString());
options.geo_city = {true, args["--geo-city"].asString()};
} else if (config["geo_city"]) {
options.geo_city = std::make_pair(true, config["geo_city"].as<std::string>());
options.geo_city = {true, config["geo_city"].as<std::string>()};
}

if (args["--geo-asn"]) {
options.geo_asn = std::make_pair(true, args["--geo-asn"].asString());
options.geo_asn = {true, args["--geo-asn"].asString()};
} else if (config["geo_asn"]) {
options.geo_asn = std::make_pair(true, config["geo_asn"].as<std::string>());
options.geo_asn = {true, config["geo_asn"].as<std::string>()};
}

if (args["--max-deep-sample"]) {
options.max_deep_sample = std::make_pair(true, static_cast<unsigned int>(args["--max-deep-sample"].asLong()));
options.max_deep_sample = {true, static_cast<unsigned int>(args["--max-deep-sample"].asLong())};
} else if (config["max_deep_sample"]) {
options.max_deep_sample = std::make_pair(true, config["max_deep_sample"].as<unsigned int>());
options.max_deep_sample = {true, config["max_deep_sample"].as<unsigned int>()};
} else {
options.max_deep_sample = {false, 100};
}

if (args["--periods"]) {
options.periods = std::make_pair(true, static_cast<unsigned int>(args["--periods"].asLong()));
options.periods = {true, static_cast<unsigned int>(args["--periods"].asLong())};
} else if (config["periods"]) {
options.periods = std::make_pair(true, config["periods"].as<unsigned int>());
options.periods = {true, config["periods"].as<unsigned int>()};
} else {
options.periods = {false, 5};
}

options.web_server.tls_support = (config["tls"] && config["tls"].as<bool>()) || args["--tls"].asBool();
options.web_server.admin_api = (config["admin_api"] && config["admin_api"].as<bool>()) || args["--admin-api"].asBool();

if (args["-p"]) {
options.web_server.port = std::make_pair(true, static_cast<unsigned int>(args["-p"].asLong()));
options.web_server.port = {true, static_cast<unsigned int>(args["-p"].asLong())};
} else if (config["port"]) {
options.web_server.port = std::make_pair(true, config["port"].as<unsigned int>());
options.web_server.port = {true, config["port"].as<unsigned int>()};
} else {
options.web_server.port = {false, 10853};
}

if (args["-l"]) {
options.web_server.host = std::make_pair(true, args["-l"].asString());
options.web_server.host = {true, args["-l"].asString()};
} else if (config["host"]) {
options.web_server.host = std::make_pair(true, config["host"].as<std::string>());
options.web_server.host = {true, config["host"].as<std::string>()};
} else {
options.web_server.host = {false, "localhost"};
}

if (args["--tls-cert"]) {
options.web_server.tls_cert = std::make_pair(true, args["--tls-cert"].asString());
options.web_server.tls_cert = {true, args["--tls-cert"].asString()};
} else if (config["tls_cert"]) {
options.web_server.tls_cert = std::make_pair(true, config["tls_cert"].as<std::string>());
options.web_server.tls_cert = {true, config["tls_cert"].as<std::string>()};
}

if (args["--tls-key"]) {
options.web_server.tls_key = std::make_pair(true, args["--tls-key"].asString());
options.web_server.tls_key = {true, args["--tls-key"].asString()};
} else if (config["tls_key"]) {
options.web_server.tls_key = std::make_pair(true, config["tls_key"].as<std::string>());
options.web_server.tls_key = {true, config["tls_key"].as<std::string>()};
}

options.module.list = (config["module_list"] && config["module_list"].as<bool>()) || args["--module-list"].asBool();

if (args["--module-dir"]) {
options.module.dir = std::make_pair(true, args["--module-dir"].asString());
options.module.dir = {true, args["--module-dir"].asString()};
} else if (config["module_dir"]) {
options.module.dir = std::make_pair(true, config["module_dir"].as<std::string>());
options.module.dir = {true, config["module_dir"].as<std::string>()};
}
}

Expand Down Expand Up @@ -384,6 +392,10 @@ int main(int argc, char *argv[])
registry.handler_plugin_registry()->setPluginDirectory(options.module.dir.second);
}

// window config defaults for all policies
registry.policy_manager()->set_default_deep_sample_rate(options.max_deep_sample.second);
registry.policy_manager()->set_default_num_periods(options.periods.second);

logger->info("{} starting up", VISOR_VERSION);

// if we are demonized, change to root directory now that (potentially) logs are open
Expand Down
17 changes: 15 additions & 2 deletions golang/pkg/client/client.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import (
"io/ioutil"
"log"
"net/http"
"regexp"
"time"
)

Expand All @@ -35,6 +36,7 @@ type ClientConfig struct {
type client struct {
config ClientConfig
serverVersion string
periods string
}

func (c *client) GetPolicy() string {
Expand All @@ -61,7 +63,8 @@ func (c *client) GetServerVersion() string {

func New(config ClientConfig) (Client, error) {
return &client{
config: config,
config: config,
periods: "5",
}, nil
}

Expand Down Expand Up @@ -89,6 +92,16 @@ func (c *client) getMetrics(url string, payload interface{}) error {
err := json.Unmarshal(body, &jsonBody)
if err == nil {
if errMsg, ok := jsonBody["error"]; ok {
if res.StatusCode == 425 {
// attempt to resolve a period exception
r, _ := regexp.Compile("invalid metrics period, specify \\[\\d, (\\d)\\]")
match := r.FindStringSubmatch(errMsg.(string))
if match != nil && match[1] != c.periods {
// try new period on next request
c.periods = match[1]
return nil
}
}
return errors.New(fmt.Sprintf("%d %s", res.StatusCode, errMsg))
}
}
Expand All @@ -105,7 +118,7 @@ func (c *client) getMetrics(url string, payload interface{}) error {

func (c *client) GetStats() (*StatSnapshot, error) {
var rawStats map[string]map[string]map[string]interface{}
err := c.getMetrics(fmt.Sprintf("%s://%s:%d/api/v1/policies/%s/metrics/window/5", c.config.Protocol, c.config.Host, c.config.Port, c.config.DefaultPolicy), &rawStats)
err := c.getMetrics(fmt.Sprintf("%s://%s:%d/api/v1/policies/%s/metrics/window/%s", c.config.Protocol, c.config.Host, c.config.Port, c.config.DefaultPolicy, c.periods), &rawStats)
if err != nil {
return nil, err
}
Expand Down
3 changes: 3 additions & 0 deletions src/Policies.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -169,6 +169,9 @@ std::vector<Policy *> PolicyManager::load(const YAML::Node &policy_yaml)
} catch (ConfigException &e) {
throw PolicyException(fmt::format("invalid stream handler window config: {}", e.what()));
}
} else {
window_config.config_set<uint64_t>("num_periods", _default_num_periods);
window_config.config_set<uint64_t>("deep_sample_rate", _default_deep_sample_rate);
}

std::vector<std::unique_ptr<StreamHandler>> handler_modules;
Expand Down
25 changes: 25 additions & 0 deletions src/Policies.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,12 @@ class PolicyManager : public AbstractManager<Policy>

CoreRegistry *_registry;

/**
* the default number of periods we will maintain in the window for handlers
*/
unsigned int _default_num_periods{5};
uint32_t _default_deep_sample_rate{100};

public:
PolicyManager(CoreRegistry *registry)
: _registry(registry)
Expand All @@ -89,6 +95,25 @@ class PolicyManager : public AbstractManager<Policy>
{
}

void set_default_num_periods(unsigned int n)
{
_default_num_periods = n;
}
void set_default_deep_sample_rate(uint32_t r)
{
_default_deep_sample_rate = r;
}

unsigned int default_num_periods() const
{
return _default_num_periods;
}

uint32_t default_deep_sample_rate() const
{
return _default_deep_sample_rate;
}

std::vector<Policy *> load_from_str(const std::string &str);
std::vector<Policy *> load(const YAML::Node &tap_yaml);
void remove_policy(const std::string &name);
Expand Down