Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

refact "cscli metrics" part 2 #2806

Merged
merged 5 commits into from
Feb 2, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
243 changes: 129 additions & 114 deletions cmd/crowdsec-cli/metrics.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,27 @@
"github.com/crowdsecurity/go-cs-lib/trace"
)

type (
statAcquis map[string]map[string]int
statParser map[string]map[string]int
statBucket map[string]map[string]int
statLapi map[string]map[string]int
statLapiMachine map[string]map[string]map[string]int
statLapiBouncer map[string]map[string]map[string]int
statLapiDecision map[string]struct {
NonEmpty int
Empty int
}
statDecision map[string]map[string]map[string]int
statAppsecEngine map[string]map[string]int
statAppsecRule map[string]map[string]map[string]int
statAlert map[string]int
statStash map[string]struct {
Type string
Count int
}
)

type cliMetrics struct {
cfg configGetter
}
Expand All @@ -29,7 +50,6 @@
}
}


// FormatPrometheusMetrics is a complete rip from prom2json
func FormatPrometheusMetrics(out io.Writer, url string, formatType string, noUnit bool) error {
mfChan := make(chan *dto.MetricFamily, 1024)
Expand Down Expand Up @@ -63,24 +83,19 @@

log.Debugf("Finished reading prometheus output, %d entries", len(result))
/*walk*/
lapi_decisions_stats := map[string]struct {
NonEmpty int
Empty int
}{}
acquis_stats := map[string]map[string]int{}
parsers_stats := map[string]map[string]int{}
buckets_stats := map[string]map[string]int{}
lapi_stats := map[string]map[string]int{}
lapi_machine_stats := map[string]map[string]map[string]int{}
lapi_bouncer_stats := map[string]map[string]map[string]int{}
decisions_stats := map[string]map[string]map[string]int{}
appsec_engine_stats := map[string]map[string]int{}
appsec_rule_stats := map[string]map[string]map[string]int{}
alerts_stats := map[string]int{}
stash_stats := map[string]struct {
Type string
Count int
}{}

mAcquis := statAcquis{}
mParser := statParser{}
mBucket := statBucket{}
mLapi := statLapi{}
mLapiMachine := statLapiMachine{}
mLapiBouncer := statLapiBouncer{}
mLapiDecision := statLapiDecision{}
mDecision := statDecision{}
mAppsecEngine := statAppsecEngine{}
mAppsecRule := statAppsecRule{}
mAlert := statAlert{}
mStash := statStash{}

for idx, fam := range result {
if !strings.HasPrefix(fam.Name, "cs_") {
Expand Down Expand Up @@ -127,138 +142,138 @@
switch fam.Name {
/*buckets*/
case "cs_bucket_created_total":
if _, ok := buckets_stats[name]; !ok {
buckets_stats[name] = make(map[string]int)
if _, ok := mBucket[name]; !ok {
mBucket[name] = make(map[string]int)

Check warning on line 146 in cmd/crowdsec-cli/metrics.go

View check run for this annotation

Codecov / codecov/patch

cmd/crowdsec-cli/metrics.go#L145-L146

Added lines #L145 - L146 were not covered by tests
}
buckets_stats[name]["instantiation"] += ival
mBucket[name]["instantiation"] += ival

Check warning on line 148 in cmd/crowdsec-cli/metrics.go

View check run for this annotation

Codecov / codecov/patch

cmd/crowdsec-cli/metrics.go#L148

Added line #L148 was not covered by tests
case "cs_buckets":
if _, ok := buckets_stats[name]; !ok {
buckets_stats[name] = make(map[string]int)
if _, ok := mBucket[name]; !ok {
mBucket[name] = make(map[string]int)

Check warning on line 151 in cmd/crowdsec-cli/metrics.go

View check run for this annotation

Codecov / codecov/patch

cmd/crowdsec-cli/metrics.go#L150-L151

Added lines #L150 - L151 were not covered by tests
}
buckets_stats[name]["curr_count"] += ival
mBucket[name]["curr_count"] += ival

Check warning on line 153 in cmd/crowdsec-cli/metrics.go

View check run for this annotation

Codecov / codecov/patch

cmd/crowdsec-cli/metrics.go#L153

Added line #L153 was not covered by tests
case "cs_bucket_overflowed_total":
if _, ok := buckets_stats[name]; !ok {
buckets_stats[name] = make(map[string]int)
if _, ok := mBucket[name]; !ok {
mBucket[name] = make(map[string]int)

Check warning on line 156 in cmd/crowdsec-cli/metrics.go

View check run for this annotation

Codecov / codecov/patch

cmd/crowdsec-cli/metrics.go#L155-L156

Added lines #L155 - L156 were not covered by tests
}
buckets_stats[name]["overflow"] += ival
mBucket[name]["overflow"] += ival

Check warning on line 158 in cmd/crowdsec-cli/metrics.go

View check run for this annotation

Codecov / codecov/patch

cmd/crowdsec-cli/metrics.go#L158

Added line #L158 was not covered by tests
case "cs_bucket_poured_total":
if _, ok := buckets_stats[name]; !ok {
buckets_stats[name] = make(map[string]int)
if _, ok := mBucket[name]; !ok {
mBucket[name] = make(map[string]int)

Check warning on line 161 in cmd/crowdsec-cli/metrics.go

View check run for this annotation

Codecov / codecov/patch

cmd/crowdsec-cli/metrics.go#L160-L161

Added lines #L160 - L161 were not covered by tests
}
if _, ok := acquis_stats[source]; !ok {
acquis_stats[source] = make(map[string]int)
if _, ok := mAcquis[source]; !ok {
mAcquis[source] = make(map[string]int)

Check warning on line 164 in cmd/crowdsec-cli/metrics.go

View check run for this annotation

Codecov / codecov/patch

cmd/crowdsec-cli/metrics.go#L163-L164

Added lines #L163 - L164 were not covered by tests
}
buckets_stats[name]["pour"] += ival
acquis_stats[source]["pour"] += ival
mBucket[name]["pour"] += ival
mAcquis[source]["pour"] += ival

Check warning on line 167 in cmd/crowdsec-cli/metrics.go

View check run for this annotation

Codecov / codecov/patch

cmd/crowdsec-cli/metrics.go#L166-L167

Added lines #L166 - L167 were not covered by tests
case "cs_bucket_underflowed_total":
if _, ok := buckets_stats[name]; !ok {
buckets_stats[name] = make(map[string]int)
if _, ok := mBucket[name]; !ok {
mBucket[name] = make(map[string]int)

Check warning on line 170 in cmd/crowdsec-cli/metrics.go

View check run for this annotation

Codecov / codecov/patch

cmd/crowdsec-cli/metrics.go#L169-L170

Added lines #L169 - L170 were not covered by tests
}
buckets_stats[name]["underflow"] += ival
mBucket[name]["underflow"] += ival

Check warning on line 172 in cmd/crowdsec-cli/metrics.go

View check run for this annotation

Codecov / codecov/patch

cmd/crowdsec-cli/metrics.go#L172

Added line #L172 was not covered by tests
/*acquis*/
case "cs_parser_hits_total":
if _, ok := acquis_stats[source]; !ok {
acquis_stats[source] = make(map[string]int)
if _, ok := mAcquis[source]; !ok {
mAcquis[source] = make(map[string]int)

Check warning on line 176 in cmd/crowdsec-cli/metrics.go

View check run for this annotation

Codecov / codecov/patch

cmd/crowdsec-cli/metrics.go#L175-L176

Added lines #L175 - L176 were not covered by tests
}
acquis_stats[source]["reads"] += ival
mAcquis[source]["reads"] += ival

Check warning on line 178 in cmd/crowdsec-cli/metrics.go

View check run for this annotation

Codecov / codecov/patch

cmd/crowdsec-cli/metrics.go#L178

Added line #L178 was not covered by tests
case "cs_parser_hits_ok_total":
if _, ok := acquis_stats[source]; !ok {
acquis_stats[source] = make(map[string]int)
if _, ok := mAcquis[source]; !ok {
mAcquis[source] = make(map[string]int)

Check warning on line 181 in cmd/crowdsec-cli/metrics.go

View check run for this annotation

Codecov / codecov/patch

cmd/crowdsec-cli/metrics.go#L180-L181

Added lines #L180 - L181 were not covered by tests
}
acquis_stats[source]["parsed"] += ival
mAcquis[source]["parsed"] += ival

Check warning on line 183 in cmd/crowdsec-cli/metrics.go

View check run for this annotation

Codecov / codecov/patch

cmd/crowdsec-cli/metrics.go#L183

Added line #L183 was not covered by tests
case "cs_parser_hits_ko_total":
if _, ok := acquis_stats[source]; !ok {
acquis_stats[source] = make(map[string]int)
if _, ok := mAcquis[source]; !ok {
mAcquis[source] = make(map[string]int)

Check warning on line 186 in cmd/crowdsec-cli/metrics.go

View check run for this annotation

Codecov / codecov/patch

cmd/crowdsec-cli/metrics.go#L185-L186

Added lines #L185 - L186 were not covered by tests
}
acquis_stats[source]["unparsed"] += ival
mAcquis[source]["unparsed"] += ival

Check warning on line 188 in cmd/crowdsec-cli/metrics.go

View check run for this annotation

Codecov / codecov/patch

cmd/crowdsec-cli/metrics.go#L188

Added line #L188 was not covered by tests
case "cs_node_hits_total":
if _, ok := parsers_stats[name]; !ok {
parsers_stats[name] = make(map[string]int)
if _, ok := mParser[name]; !ok {
mParser[name] = make(map[string]int)

Check warning on line 191 in cmd/crowdsec-cli/metrics.go

View check run for this annotation

Codecov / codecov/patch

cmd/crowdsec-cli/metrics.go#L190-L191

Added lines #L190 - L191 were not covered by tests
}
parsers_stats[name]["hits"] += ival
mParser[name]["hits"] += ival

Check warning on line 193 in cmd/crowdsec-cli/metrics.go

View check run for this annotation

Codecov / codecov/patch

cmd/crowdsec-cli/metrics.go#L193

Added line #L193 was not covered by tests
case "cs_node_hits_ok_total":
if _, ok := parsers_stats[name]; !ok {
parsers_stats[name] = make(map[string]int)
if _, ok := mParser[name]; !ok {
mParser[name] = make(map[string]int)

Check warning on line 196 in cmd/crowdsec-cli/metrics.go

View check run for this annotation

Codecov / codecov/patch

cmd/crowdsec-cli/metrics.go#L195-L196

Added lines #L195 - L196 were not covered by tests
}
parsers_stats[name]["parsed"] += ival
mParser[name]["parsed"] += ival

Check warning on line 198 in cmd/crowdsec-cli/metrics.go

View check run for this annotation

Codecov / codecov/patch

cmd/crowdsec-cli/metrics.go#L198

Added line #L198 was not covered by tests
case "cs_node_hits_ko_total":
if _, ok := parsers_stats[name]; !ok {
parsers_stats[name] = make(map[string]int)
if _, ok := mParser[name]; !ok {
mParser[name] = make(map[string]int)

Check warning on line 201 in cmd/crowdsec-cli/metrics.go

View check run for this annotation

Codecov / codecov/patch

cmd/crowdsec-cli/metrics.go#L200-L201

Added lines #L200 - L201 were not covered by tests
}
parsers_stats[name]["unparsed"] += ival
mParser[name]["unparsed"] += ival

Check warning on line 203 in cmd/crowdsec-cli/metrics.go

View check run for this annotation

Codecov / codecov/patch

cmd/crowdsec-cli/metrics.go#L203

Added line #L203 was not covered by tests
case "cs_lapi_route_requests_total":
if _, ok := lapi_stats[route]; !ok {
lapi_stats[route] = make(map[string]int)
if _, ok := mLapi[route]; !ok {
mLapi[route] = make(map[string]int)
}
lapi_stats[route][method] += ival
mLapi[route][method] += ival
case "cs_lapi_machine_requests_total":
if _, ok := lapi_machine_stats[machine]; !ok {
lapi_machine_stats[machine] = make(map[string]map[string]int)
if _, ok := mLapiMachine[machine]; !ok {
mLapiMachine[machine] = make(map[string]map[string]int)

Check warning on line 211 in cmd/crowdsec-cli/metrics.go

View check run for this annotation

Codecov / codecov/patch

cmd/crowdsec-cli/metrics.go#L210-L211

Added lines #L210 - L211 were not covered by tests
}
if _, ok := lapi_machine_stats[machine][route]; !ok {
lapi_machine_stats[machine][route] = make(map[string]int)
if _, ok := mLapiMachine[machine][route]; !ok {
mLapiMachine[machine][route] = make(map[string]int)

Check warning on line 214 in cmd/crowdsec-cli/metrics.go

View check run for this annotation

Codecov / codecov/patch

cmd/crowdsec-cli/metrics.go#L213-L214

Added lines #L213 - L214 were not covered by tests
}
lapi_machine_stats[machine][route][method] += ival
mLapiMachine[machine][route][method] += ival

Check warning on line 216 in cmd/crowdsec-cli/metrics.go

View check run for this annotation

Codecov / codecov/patch

cmd/crowdsec-cli/metrics.go#L216

Added line #L216 was not covered by tests
case "cs_lapi_bouncer_requests_total":
if _, ok := lapi_bouncer_stats[bouncer]; !ok {
lapi_bouncer_stats[bouncer] = make(map[string]map[string]int)
if _, ok := mLapiBouncer[bouncer]; !ok {
mLapiBouncer[bouncer] = make(map[string]map[string]int)

Check warning on line 219 in cmd/crowdsec-cli/metrics.go

View check run for this annotation

Codecov / codecov/patch

cmd/crowdsec-cli/metrics.go#L218-L219

Added lines #L218 - L219 were not covered by tests
}
if _, ok := lapi_bouncer_stats[bouncer][route]; !ok {
lapi_bouncer_stats[bouncer][route] = make(map[string]int)
if _, ok := mLapiBouncer[bouncer][route]; !ok {
mLapiBouncer[bouncer][route] = make(map[string]int)

Check warning on line 222 in cmd/crowdsec-cli/metrics.go

View check run for this annotation

Codecov / codecov/patch

cmd/crowdsec-cli/metrics.go#L221-L222

Added lines #L221 - L222 were not covered by tests
}
lapi_bouncer_stats[bouncer][route][method] += ival
mLapiBouncer[bouncer][route][method] += ival

Check warning on line 224 in cmd/crowdsec-cli/metrics.go

View check run for this annotation

Codecov / codecov/patch

cmd/crowdsec-cli/metrics.go#L224

Added line #L224 was not covered by tests
case "cs_lapi_decisions_ko_total", "cs_lapi_decisions_ok_total":
if _, ok := lapi_decisions_stats[bouncer]; !ok {
lapi_decisions_stats[bouncer] = struct {
if _, ok := mLapiDecision[bouncer]; !ok {
mLapiDecision[bouncer] = struct {

Check warning on line 227 in cmd/crowdsec-cli/metrics.go

View check run for this annotation

Codecov / codecov/patch

cmd/crowdsec-cli/metrics.go#L226-L227

Added lines #L226 - L227 were not covered by tests
NonEmpty int
Empty int
}{}
}
x := lapi_decisions_stats[bouncer]
x := mLapiDecision[bouncer]

Check warning on line 232 in cmd/crowdsec-cli/metrics.go

View check run for this annotation

Codecov / codecov/patch

cmd/crowdsec-cli/metrics.go#L232

Added line #L232 was not covered by tests
if fam.Name == "cs_lapi_decisions_ko_total" {
x.Empty += ival
} else if fam.Name == "cs_lapi_decisions_ok_total" {
x.NonEmpty += ival
}
lapi_decisions_stats[bouncer] = x
mLapiDecision[bouncer] = x

Check warning on line 238 in cmd/crowdsec-cli/metrics.go

View check run for this annotation

Codecov / codecov/patch

cmd/crowdsec-cli/metrics.go#L238

Added line #L238 was not covered by tests
case "cs_active_decisions":
if _, ok := decisions_stats[reason]; !ok {
decisions_stats[reason] = make(map[string]map[string]int)
if _, ok := mDecision[reason]; !ok {
mDecision[reason] = make(map[string]map[string]int)

Check warning on line 241 in cmd/crowdsec-cli/metrics.go

View check run for this annotation

Codecov / codecov/patch

cmd/crowdsec-cli/metrics.go#L240-L241

Added lines #L240 - L241 were not covered by tests
}
if _, ok := decisions_stats[reason][origin]; !ok {
decisions_stats[reason][origin] = make(map[string]int)
if _, ok := mDecision[reason][origin]; !ok {
mDecision[reason][origin] = make(map[string]int)

Check warning on line 244 in cmd/crowdsec-cli/metrics.go

View check run for this annotation

Codecov / codecov/patch

cmd/crowdsec-cli/metrics.go#L243-L244

Added lines #L243 - L244 were not covered by tests
}
decisions_stats[reason][origin][action] += ival
mDecision[reason][origin][action] += ival

Check warning on line 246 in cmd/crowdsec-cli/metrics.go

View check run for this annotation

Codecov / codecov/patch

cmd/crowdsec-cli/metrics.go#L246

Added line #L246 was not covered by tests
case "cs_alerts":
/*if _, ok := alerts_stats[scenario]; !ok {
alerts_stats[scenario] = make(map[string]int)
/*if _, ok := mAlert[scenario]; !ok {
mAlert[scenario] = make(map[string]int)

Check warning on line 249 in cmd/crowdsec-cli/metrics.go

View check run for this annotation

Codecov / codecov/patch

cmd/crowdsec-cli/metrics.go#L248-L249

Added lines #L248 - L249 were not covered by tests
}*/
alerts_stats[reason] += ival
mAlert[reason] += ival

Check warning on line 251 in cmd/crowdsec-cli/metrics.go

View check run for this annotation

Codecov / codecov/patch

cmd/crowdsec-cli/metrics.go#L251

Added line #L251 was not covered by tests
case "cs_cache_size":
stash_stats[name] = struct {
mStash[name] = struct {

Check warning on line 253 in cmd/crowdsec-cli/metrics.go

View check run for this annotation

Codecov / codecov/patch

cmd/crowdsec-cli/metrics.go#L253

Added line #L253 was not covered by tests
Type string
Count int
}{Type: mtype, Count: ival}
case "cs_appsec_reqs_total":
if _, ok := appsec_engine_stats[metric.Labels["appsec_engine"]]; !ok {
appsec_engine_stats[metric.Labels["appsec_engine"]] = make(map[string]int, 0)
if _, ok := mAppsecEngine[metric.Labels["appsec_engine"]]; !ok {
mAppsecEngine[metric.Labels["appsec_engine"]] = make(map[string]int, 0)

Check warning on line 259 in cmd/crowdsec-cli/metrics.go

View check run for this annotation

Codecov / codecov/patch

cmd/crowdsec-cli/metrics.go#L258-L259

Added lines #L258 - L259 were not covered by tests
}
appsec_engine_stats[metric.Labels["appsec_engine"]]["processed"] = ival
mAppsecEngine[metric.Labels["appsec_engine"]]["processed"] = ival

Check warning on line 261 in cmd/crowdsec-cli/metrics.go

View check run for this annotation

Codecov / codecov/patch

cmd/crowdsec-cli/metrics.go#L261

Added line #L261 was not covered by tests
case "cs_appsec_block_total":
if _, ok := appsec_engine_stats[metric.Labels["appsec_engine"]]; !ok {
appsec_engine_stats[metric.Labels["appsec_engine"]] = make(map[string]int, 0)
if _, ok := mAppsecEngine[metric.Labels["appsec_engine"]]; !ok {
mAppsecEngine[metric.Labels["appsec_engine"]] = make(map[string]int, 0)

Check warning on line 264 in cmd/crowdsec-cli/metrics.go

View check run for this annotation

Codecov / codecov/patch

cmd/crowdsec-cli/metrics.go#L263-L264

Added lines #L263 - L264 were not covered by tests
}
appsec_engine_stats[metric.Labels["appsec_engine"]]["blocked"] = ival
mAppsecEngine[metric.Labels["appsec_engine"]]["blocked"] = ival

Check warning on line 266 in cmd/crowdsec-cli/metrics.go

View check run for this annotation

Codecov / codecov/patch

cmd/crowdsec-cli/metrics.go#L266

Added line #L266 was not covered by tests
case "cs_appsec_rule_hits":
appsecEngine := metric.Labels["appsec_engine"]
ruleID := metric.Labels["rule_name"]
if _, ok := appsec_rule_stats[appsecEngine]; !ok {
appsec_rule_stats[appsecEngine] = make(map[string]map[string]int, 0)
if _, ok := mAppsecRule[appsecEngine]; !ok {
mAppsecRule[appsecEngine] = make(map[string]map[string]int, 0)

Check warning on line 271 in cmd/crowdsec-cli/metrics.go

View check run for this annotation

Codecov / codecov/patch

cmd/crowdsec-cli/metrics.go#L270-L271

Added lines #L270 - L271 were not covered by tests
}
if _, ok := appsec_rule_stats[appsecEngine][ruleID]; !ok {
appsec_rule_stats[appsecEngine][ruleID] = make(map[string]int, 0)
if _, ok := mAppsecRule[appsecEngine][ruleID]; !ok {
mAppsecRule[appsecEngine][ruleID] = make(map[string]int, 0)

Check warning on line 274 in cmd/crowdsec-cli/metrics.go

View check run for this annotation

Codecov / codecov/patch

cmd/crowdsec-cli/metrics.go#L273-L274

Added lines #L273 - L274 were not covered by tests
}
appsec_rule_stats[appsecEngine][ruleID]["triggered"] = ival
mAppsecRule[appsecEngine][ruleID]["triggered"] = ival

Check warning on line 276 in cmd/crowdsec-cli/metrics.go

View check run for this annotation

Codecov / codecov/patch

cmd/crowdsec-cli/metrics.go#L276

Added line #L276 was not covered by tests
default:
log.Debugf("unknown: %+v", fam.Name)
continue
Expand All @@ -267,33 +282,33 @@
}

if formatType == "human" {
acquisStatsTable(out, acquis_stats, noUnit)
bucketStatsTable(out, buckets_stats, noUnit)
parserStatsTable(out, parsers_stats, noUnit)
lapiStatsTable(out, lapi_stats)
lapiMachineStatsTable(out, lapi_machine_stats)
lapiBouncerStatsTable(out, lapi_bouncer_stats)
lapiDecisionStatsTable(out, lapi_decisions_stats)
decisionStatsTable(out, decisions_stats)
alertStatsTable(out, alerts_stats)
stashStatsTable(out, stash_stats)
appsecMetricsToTable(out, appsec_engine_stats, noUnit)
appsecRulesToTable(out, appsec_rule_stats, noUnit)
mAcquis.table(out, noUnit)
mBucket.table(out, noUnit)
mParser.table(out, noUnit)
mLapi.table(out)
mLapiMachine.table(out)
mLapiBouncer.table(out)
mLapiDecision.table(out)
mDecision.table(out)
mAlert.table(out)
mStash.table(out)
mAppsecEngine.table(out, noUnit)
mAppsecRule.table(out, noUnit)
return nil
}

stats := make(map[string]any)

stats["acquisition"] = acquis_stats
stats["buckets"] = buckets_stats
stats["parsers"] = parsers_stats
stats["lapi"] = lapi_stats
stats["lapi_machine"] = lapi_machine_stats
stats["lapi_bouncer"] = lapi_bouncer_stats
stats["lapi_decisions"] = lapi_decisions_stats
stats["decisions"] = decisions_stats
stats["alerts"] = alerts_stats
stats["stash"] = stash_stats
stats["acquisition"] = mAcquis
stats["buckets"] = mBucket
stats["parsers"] = mParser
stats["lapi"] = mLapi
stats["lapi_machine"] = mLapiMachine
stats["lapi_bouncer"] = mLapiBouncer
stats["lapi_decisions"] = mLapiDecision
stats["decisions"] = mDecision
stats["alerts"] = mAlert
stats["stash"] = mStash

Check warning on line 311 in cmd/crowdsec-cli/metrics.go

View check run for this annotation

Codecov / codecov/patch

cmd/crowdsec-cli/metrics.go#L302-L311

Added lines #L302 - L311 were not covered by tests

switch formatType {
case "json":
Expand Down
Loading
Loading