From b621499199cf7039cc9575c83f32cb81f5764085 Mon Sep 17 00:00:00 2001 From: Sebastien Blot Date: Thu, 23 Mar 2023 00:22:50 +0100 Subject: [PATCH 01/11] wip --- go.mod | 8 ++++++-- go.sum | 8 ++++++-- pkg/acquisition/acquisition.go | 3 +++ pkg/fflag/crowdsec.go | 6 ++++++ pkg/parser/node.go | 4 ++-- pkg/parser/parsing_test.go | 4 ++-- pkg/parser/unix_parser.go | 2 ++ pkg/types/grok_pattern.go | 2 +- 8 files changed, 28 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index 8b94ef948b4..d137fdf0d0b 100644 --- a/go.mod +++ b/go.mod @@ -67,6 +67,7 @@ require ( github.com/Masterminds/semver v1.5.0 github.com/Masterminds/sprig/v3 v3.2.2 github.com/aquasecurity/table v1.8.0 + github.com/aws/aws-lambda-go v1.38.0 github.com/beevik/etree v1.1.0 github.com/blackfireio/osinfo v1.0.3 github.com/bluele/gcache v0.0.2 @@ -98,7 +99,6 @@ require ( github.com/ahmetalpbalkan/dlog v0.0.0-20170105205344-4fb5f8204f26 // indirect github.com/apparentlymart/go-textseg/v13 v13.0.0 // indirect github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef // indirect - github.com/aws/aws-lambda-go v1.38.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/containerd/containerd v1.6.18 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect @@ -143,6 +143,7 @@ require ( github.com/klauspost/compress v1.15.7 // indirect github.com/leodido/go-urn v1.2.1 // indirect github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect + github.com/magefile/mage v1.14.0 // indirect github.com/mailru/easyjson v0.7.6 // indirect github.com/mattn/go-colorable v0.1.12 // indirect github.com/mattn/go-runewidth v0.0.13 // indirect @@ -170,12 +171,13 @@ require ( github.com/shopspring/decimal v1.2.0 // indirect github.com/spf13/cast v1.3.1 // indirect github.com/spf13/pflag v1.0.5 // indirect + github.com/tetratelabs/wazero v1.0.0-pre.5 // indirect github.com/tidwall/gjson v1.13.0 // indirect github.com/tklauser/go-sysconf v0.3.11 // indirect github.com/tklauser/numcpus v0.6.0 // indirect github.com/ugorji/go/codec v1.2.6 // indirect - github.com/vjeantet/grok v1.0.1 // indirect github.com/vmihailenco/msgpack v4.0.4+incompatible // indirect + github.com/wasilibs/go-re2 v0.1.0 // indirect github.com/yusufpapurcu/wmi v1.2.2 // indirect github.com/zclconf/go-cty v1.8.0 // indirect go.mongodb.org/mongo-driver v1.9.0 // indirect @@ -197,3 +199,5 @@ require ( ) replace golang.org/x/time/rate => github.com/crowdsecurity/crowdsec/pkg/time/rate v0.0.0 + +replace github.com/crowdsecurity/grokky => ../grokky diff --git a/go.sum b/go.sum index 3b2bc3b13b3..2e6c2af90e1 100644 --- a/go.sum +++ b/go.sum @@ -177,8 +177,6 @@ github.com/creack/pty v1.1.11 h1:07n33Z8lZxZ2qwegKbObQohDhXDQxiMMz1NOUGYlesw= github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26 h1:r97WNVC30Uen+7WnLs4xDScS/Ex988+id2k6mDf8psU= github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26/go.mod h1:zpv7r+7KXwgVUZnUNjyP22zc/D7LKjyoY02weH2RBbk= -github.com/crowdsecurity/grokky v0.1.0 h1:jLUzZd3vKxYrM4hQ8n5HWLfvs5ag4UP08eT9OTekI4U= -github.com/crowdsecurity/grokky v0.1.0/go.mod h1:fx5UYUYAFIrOUNAkFCUOM2wJcsp9EWSQE9R0/9kaFJg= github.com/crowdsecurity/machineid v1.0.2 h1:wpkpsUghJF8Khtmn/tg6GxgdhLA1Xflerh5lirI+bdc= github.com/crowdsecurity/machineid v1.0.2/go.mod h1:XWUSlnS0R0+u/JK5ulidwlbceNT3ZOCKteoVQEn6Luo= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -644,6 +642,8 @@ github.com/lithammer/dedent v1.1.0 h1:VNzHMVCBNG1j0fh3OrsFRkVUwStdDArbgBWoPAffkt github.com/lithammer/dedent v1.1.0/go.mod h1:jrXYCQtgg0nJiN+StA2KgR7w6CiQNv9Fd/Z9BP0jIOc= github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4= github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= +github.com/magefile/mage v1.14.0 h1:6QDX3g6z1YvJ4olPhT1wksUcSa/V0a1B+pJb73fBjyo= +github.com/magefile/mage v1.14.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= @@ -888,6 +888,8 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= +github.com/tetratelabs/wazero v1.0.0-pre.5 h1:ChZsQewsazFwF+NT6qAKPwnqqiOlf2MN0aah1dlD8Bk= +github.com/tetratelabs/wazero v1.0.0-pre.5/go.mod h1:u8wrFmpdrykiFK0DFPiFm5a4+0RzsdmXYVtijBKqUVo= github.com/texttheater/golang-levenshtein/levenshtein v0.0.0-20200805054039-cae8b0eaed6c h1:HelZ2kAFadG0La9d+4htN4HzQ68Bm2iM9qKMSMES6xg= github.com/texttheater/golang-levenshtein/levenshtein v0.0.0-20200805054039-cae8b0eaed6c/go.mod h1:JlzghshsemAMDGZLytTFY8C1JQxQPhnatWqNwUXjggo= github.com/tidwall/gjson v1.12.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= @@ -918,6 +920,8 @@ github.com/vmihailenco/msgpack v4.0.4+incompatible h1:dSLoQfGFAo3F6OoNhwUmLwVgaU github.com/vmihailenco/msgpack v4.0.4+incompatible/go.mod h1:fy3FlTQTDXWkZ7Bh6AcGMlsjHatGryHQYUTf1ShIgkk= github.com/vmihailenco/msgpack/v4 v4.3.12/go.mod h1:gborTTJjAo/GWTqqRjrLCn9pgNN+NXzzngzBKDPIqw4= github.com/vmihailenco/tagparser v0.1.1/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI= +github.com/wasilibs/go-re2 v0.1.0 h1:+pcjlvge6E9WWU4eaMlULRikEUkhPDXSFfCF5p7htMI= +github.com/wasilibs/go-re2 v0.1.0/go.mod h1:F91Yac+zPNDFrrd8fl4mSd7+TTu2tYiX56BEIEPQl2M= github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= github.com/xdg-go/scram v1.0.2/go.mod h1:1WAq6h33pAW+iRreB34OORO2Nf7qel3VV3fjBj+hCSs= github.com/xdg-go/stringprep v1.0.2/go.mod h1:8F9zXuvzgwmyT5DUm4GUfZGDdT3W+LCvS6+da4O5kxM= diff --git a/pkg/acquisition/acquisition.go b/pkg/acquisition/acquisition.go index 5ffa2d441df..4dc8142f953 100644 --- a/pkg/acquisition/acquisition.go +++ b/pkg/acquisition/acquisition.go @@ -6,6 +6,7 @@ import ( "io" "os" "strings" + "time" "github.com/prometheus/client_golang/prometheus" log "github.com/sirupsen/logrus" @@ -228,7 +229,9 @@ func StartAcquisition(sources []DataSource, output chan types.Event, AcquisTomb if subsrc.GetMode() == configuration.TAIL_MODE { err = subsrc.StreamingAcquisition(output, AcquisTomb) } else { + start := time.Now() err = subsrc.OneShotAcquisition(output, AcquisTomb) + log.Infof("acquisition of %s took %s", subsrc.GetName(), time.Since(start)) } if err != nil { //if one of the acqusition returns an error, we kill the others to properly shutdown diff --git a/pkg/fflag/crowdsec.go b/pkg/fflag/crowdsec.go index 9f4db57b014..bcbb7eb5525 100644 --- a/pkg/fflag/crowdsec.go +++ b/pkg/fflag/crowdsec.go @@ -6,6 +6,7 @@ var CscliSetup = &Feature{Name: "cscli_setup", Description: "Enable cscli setup var DisableHttpRetryBackoff = &Feature{Name: "disable_http_retry_backoff", Description: "Disable http retry backoff"} var ChunkedDecisionsStream = &Feature{Name: "chunked_decisions_stream", Description: "Enable chunked decisions stream"} var PapiClient = &Feature{Name: "papi_client", Description: "Enable Polling API client"} +var Re2Support = &Feature{Name: "re2_support", Description: "Enable RE2 support for GROK patterns"} func RegisterAllFeatures() error { err := Crowdsec.RegisterFeature(CscliSetup) @@ -24,5 +25,10 @@ func RegisterAllFeatures() error { if err != nil { return err } + err = Crowdsec.RegisterFeature(Re2Support) + if err != nil { + return err + } + return nil } diff --git a/pkg/parser/node.go b/pkg/parser/node.go index cdd6f711914..0965f247c04 100644 --- a/pkg/parser/node.go +++ b/pkg/parser/node.go @@ -509,7 +509,7 @@ func (n *Node) compile(pctx *UnixParserCtx, ectx EnricherCtx) error { if n.Grok.RunTimeRegexp == nil { return fmt.Errorf("empty grok '%s'", n.Grok.RegexpName) } - n.Logger.Tracef("%s regexp: %s", n.Grok.RegexpName, n.Grok.RunTimeRegexp.Regexp.String()) + n.Logger.Tracef("%s regexp: %s", n.Grok.RegexpName, n.Grok.RunTimeRegexp.String()) valid = true } else if n.Grok.RegexpValue != "" { if strings.HasSuffix(n.Grok.RegexpValue, "\n") { @@ -523,7 +523,7 @@ func (n *Node) compile(pctx *UnixParserCtx, ectx EnricherCtx) error { // We shouldn't be here because compilation succeeded, so regexp shouldn't be nil return fmt.Errorf("grok compilation failure: %s", n.Grok.RegexpValue) } - n.Logger.Tracef("%s regexp : %s", n.Grok.RegexpValue, n.Grok.RunTimeRegexp.Regexp.String()) + n.Logger.Tracef("%s regexp : %s", n.Grok.RegexpValue, n.Grok.RunTimeRegexp.String()) valid = true } diff --git a/pkg/parser/parsing_test.go b/pkg/parser/parsing_test.go index 34d36f07dad..089b95835b5 100644 --- a/pkg/parser/parsing_test.go +++ b/pkg/parser/parsing_test.go @@ -372,10 +372,10 @@ func TestGeneratePatternsDoc(t *testing.T) { } log.Infof("-> %s", spew.Sdump(pctx)) /*don't judge me, we do it for the users*/ - p := make(PairList, len(pctx.Grok)) + p := make(PairList, len(pctx.Grok.Patterns)) i := 0 - for key, val := range pctx.Grok { + for key, val := range pctx.Grok.Patterns { p[i] = Pair{key, val} p[i].Value = strings.ReplaceAll(p[i].Value, "{%{", "\\{\\%\\{") i++ diff --git a/pkg/parser/unix_parser.go b/pkg/parser/unix_parser.go index 3779b0941c0..0396e0e829b 100644 --- a/pkg/parser/unix_parser.go +++ b/pkg/parser/unix_parser.go @@ -9,6 +9,7 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/csconfig" "github.com/crowdsecurity/crowdsec/pkg/cwhub" + "github.com/crowdsecurity/crowdsec/pkg/fflag" "github.com/crowdsecurity/grokky" log "github.com/sirupsen/logrus" @@ -34,6 +35,7 @@ type Parsers struct { func Init(c map[string]interface{}) (*UnixParserCtx, error) { r := UnixParserCtx{} r.Grok = grokky.NewBase() + r.Grok.UseRe2 = fflag.Re2Support.IsEnabled() files, err := os.ReadDir(c["patterns"].(string)) if err != nil { return nil, err diff --git a/pkg/types/grok_pattern.go b/pkg/types/grok_pattern.go index 74c612b1fd8..bb51d2d9be0 100644 --- a/pkg/types/grok_pattern.go +++ b/pkg/types/grok_pattern.go @@ -34,7 +34,7 @@ type GrokPattern struct { //a proper grok pattern RegexpValue string `yaml:"pattern,omitempty"` //the runtime form of regexpname / regexpvalue - RunTimeRegexp *grokky.Pattern `json:"-"` //the actual regexp + RunTimeRegexp grokky.Pattern `json:"-"` //the actual regexp //the output of the expression is going to be the source for regexp ExpValue string `yaml:"expression,omitempty"` RunTimeValue *vm.Program `json:"-"` //the actual compiled filter From 2e87e380e51efe650d0851f539d1e13d5a19431e Mon Sep 17 00:00:00 2001 From: Sebastien Blot Date: Thu, 23 Mar 2023 11:15:20 +0100 Subject: [PATCH 02/11] update grokky --- go.mod | 4 ++-- go.sum | 5 ++--- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index d137fdf0d0b..a66f91d770e 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,7 @@ require ( github.com/confluentinc/bincover v0.2.0 github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26 - github.com/crowdsecurity/grokky v0.1.0 + github.com/crowdsecurity/grokky v0.2.0 github.com/crowdsecurity/machineid v1.0.2 github.com/davecgh/go-spew v1.1.1 github.com/dghubble/sling v1.3.0 @@ -171,7 +171,7 @@ require ( github.com/shopspring/decimal v1.2.0 // indirect github.com/spf13/cast v1.3.1 // indirect github.com/spf13/pflag v1.0.5 // indirect - github.com/tetratelabs/wazero v1.0.0-pre.5 // indirect + github.com/tetratelabs/wazero v1.0.0-rc.2 // indirect github.com/tidwall/gjson v1.13.0 // indirect github.com/tklauser/go-sysconf v0.3.11 // indirect github.com/tklauser/numcpus v0.6.0 // indirect diff --git a/go.sum b/go.sum index 2e6c2af90e1..bde3a1126c5 100644 --- a/go.sum +++ b/go.sum @@ -888,8 +888,8 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= -github.com/tetratelabs/wazero v1.0.0-pre.5 h1:ChZsQewsazFwF+NT6qAKPwnqqiOlf2MN0aah1dlD8Bk= -github.com/tetratelabs/wazero v1.0.0-pre.5/go.mod h1:u8wrFmpdrykiFK0DFPiFm5a4+0RzsdmXYVtijBKqUVo= +github.com/tetratelabs/wazero v1.0.0-rc.2 h1:OA3UUynnoqxrjCQ94mpAtdO4/oMxFQVNL2BXDMOc66Q= +github.com/tetratelabs/wazero v1.0.0-rc.2/go.mod h1:wYx2gNRg8/WihJfSDxA1TIL8H+GkfLYm+bIfbblu9VQ= github.com/texttheater/golang-levenshtein/levenshtein v0.0.0-20200805054039-cae8b0eaed6c h1:HelZ2kAFadG0La9d+4htN4HzQ68Bm2iM9qKMSMES6xg= github.com/texttheater/golang-levenshtein/levenshtein v0.0.0-20200805054039-cae8b0eaed6c/go.mod h1:JlzghshsemAMDGZLytTFY8C1JQxQPhnatWqNwUXjggo= github.com/tidwall/gjson v1.12.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= @@ -915,7 +915,6 @@ github.com/umahmood/haversine v0.0.0-20151105152445-808ab04add26 h1:UFHFmFfixpmf github.com/umahmood/haversine v0.0.0-20151105152445-808ab04add26/go.mod h1:IGhd0qMDsUa9acVjsbsT7bu3ktadtGOHI79+idTew/M= github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw= github.com/vjeantet/grok v1.0.1 h1:2rhIR7J4gThTgcZ1m2JY4TrJZNgjn985U28kT2wQrJ4= -github.com/vjeantet/grok v1.0.1/go.mod h1:ax1aAchzC6/QMXMcyzHQGZWaW1l195+uMYIkCWPCNIo= github.com/vmihailenco/msgpack v4.0.4+incompatible h1:dSLoQfGFAo3F6OoNhwUmLwVgaUXK79GlxNBwueZn0xI= github.com/vmihailenco/msgpack v4.0.4+incompatible/go.mod h1:fy3FlTQTDXWkZ7Bh6AcGMlsjHatGryHQYUTf1ShIgkk= github.com/vmihailenco/msgpack/v4 v4.3.12/go.mod h1:gborTTJjAo/GWTqqRjrLCn9pgNN+NXzzngzBKDPIqw4= From 181f5e0168b5f1f5d89bacd913b80e9ce0235309 Mon Sep 17 00:00:00 2001 From: Sebastien Blot Date: Thu, 23 Mar 2023 11:15:54 +0100 Subject: [PATCH 03/11] rename flag --- pkg/fflag/crowdsec.go | 4 ++-- pkg/parser/unix_parser.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/fflag/crowdsec.go b/pkg/fflag/crowdsec.go index bcbb7eb5525..8dd49cdd1a9 100644 --- a/pkg/fflag/crowdsec.go +++ b/pkg/fflag/crowdsec.go @@ -6,7 +6,7 @@ var CscliSetup = &Feature{Name: "cscli_setup", Description: "Enable cscli setup var DisableHttpRetryBackoff = &Feature{Name: "disable_http_retry_backoff", Description: "Disable http retry backoff"} var ChunkedDecisionsStream = &Feature{Name: "chunked_decisions_stream", Description: "Enable chunked decisions stream"} var PapiClient = &Feature{Name: "papi_client", Description: "Enable Polling API client"} -var Re2Support = &Feature{Name: "re2_support", Description: "Enable RE2 support for GROK patterns"} +var Re2GrokSupport = &Feature{Name: "re2_grok_support", Description: "Enable RE2 support for GROK patterns"} func RegisterAllFeatures() error { err := Crowdsec.RegisterFeature(CscliSetup) @@ -25,7 +25,7 @@ func RegisterAllFeatures() error { if err != nil { return err } - err = Crowdsec.RegisterFeature(Re2Support) + err = Crowdsec.RegisterFeature(Re2GrokSupport) if err != nil { return err } diff --git a/pkg/parser/unix_parser.go b/pkg/parser/unix_parser.go index 0396e0e829b..b0f38f81e6c 100644 --- a/pkg/parser/unix_parser.go +++ b/pkg/parser/unix_parser.go @@ -35,7 +35,7 @@ type Parsers struct { func Init(c map[string]interface{}) (*UnixParserCtx, error) { r := UnixParserCtx{} r.Grok = grokky.NewBase() - r.Grok.UseRe2 = fflag.Re2Support.IsEnabled() + r.Grok.UseRe2 = fflag.Re2GrokSupport.IsEnabled() files, err := os.ReadDir(c["patterns"].(string)) if err != nil { return nil, err From 1d6215b1cbd0f6d06055d9b9f68f6917ae29b441 Mon Sep 17 00:00:00 2001 From: Sebastien Blot Date: Thu, 23 Mar 2023 12:06:35 +0100 Subject: [PATCH 04/11] debug log --- pkg/acquisition/acquisition.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/acquisition/acquisition.go b/pkg/acquisition/acquisition.go index 4dc8142f953..59a015818e6 100644 --- a/pkg/acquisition/acquisition.go +++ b/pkg/acquisition/acquisition.go @@ -231,7 +231,7 @@ func StartAcquisition(sources []DataSource, output chan types.Event, AcquisTomb } else { start := time.Now() err = subsrc.OneShotAcquisition(output, AcquisTomb) - log.Infof("acquisition of %s took %s", subsrc.GetName(), time.Since(start)) + log.Debugf("acquisition of %s took %s", subsrc.GetName(), time.Since(start)) } if err != nil { //if one of the acqusition returns an error, we kill the others to properly shutdown From b09f3e394a47960413c2ebb4e1ddb1aa83ace993 Mon Sep 17 00:00:00 2001 From: Sebastien Blot Date: Thu, 23 Mar 2023 12:08:40 +0100 Subject: [PATCH 05/11] remove local module replace --- go.mod | 2 -- go.sum | 2 ++ 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index a66f91d770e..79715d16863 100644 --- a/go.mod +++ b/go.mod @@ -199,5 +199,3 @@ require ( ) replace golang.org/x/time/rate => github.com/crowdsecurity/crowdsec/pkg/time/rate v0.0.0 - -replace github.com/crowdsecurity/grokky => ../grokky diff --git a/go.sum b/go.sum index bde3a1126c5..8bdc67ea261 100644 --- a/go.sum +++ b/go.sum @@ -177,6 +177,8 @@ github.com/creack/pty v1.1.11 h1:07n33Z8lZxZ2qwegKbObQohDhXDQxiMMz1NOUGYlesw= github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26 h1:r97WNVC30Uen+7WnLs4xDScS/Ex988+id2k6mDf8psU= github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26/go.mod h1:zpv7r+7KXwgVUZnUNjyP22zc/D7LKjyoY02weH2RBbk= +github.com/crowdsecurity/grokky v0.2.0 h1:8u3dCU1kqSlOu2phP6S+HF2OTohZ/JBGcZCWu1dhnF0= +github.com/crowdsecurity/grokky v0.2.0/go.mod h1:qdyzFHtnY6nmdYZJFHVbdms2HMFLiEXzmqq5tCUmiGE= github.com/crowdsecurity/machineid v1.0.2 h1:wpkpsUghJF8Khtmn/tg6GxgdhLA1Xflerh5lirI+bdc= github.com/crowdsecurity/machineid v1.0.2/go.mod h1:XWUSlnS0R0+u/JK5ulidwlbceNT3ZOCKteoVQEn6Luo= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= From 7f108049dc231756f6776bb6022751829940269b Mon Sep 17 00:00:00 2001 From: Sebastien Blot Date: Thu, 23 Mar 2023 15:04:23 +0100 Subject: [PATCH 06/11] add fflag for re2 in expr helpers --- pkg/exprhelpers/exprlib.go | 33 +++++++++++++++++++++++++++------ pkg/fflag/crowdsec.go | 5 +++++ 2 files changed, 32 insertions(+), 6 deletions(-) diff --git a/pkg/exprhelpers/exprlib.go b/pkg/exprhelpers/exprlib.go index e1d82be4e5b..e3d289ee9e1 100644 --- a/pkg/exprhelpers/exprlib.go +++ b/pkg/exprhelpers/exprlib.go @@ -23,11 +23,15 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/cache" "github.com/crowdsecurity/crowdsec/pkg/database" + "github.com/crowdsecurity/crowdsec/pkg/fflag" "github.com/crowdsecurity/crowdsec/pkg/types" + + "github.com/wasilibs/go-re2" ) var dataFile map[string][]string var dataFileRegex map[string][]*regexp.Regexp +var dataFileRe2 map[string][]*re2.Regexp // This is used to (optionally) cache regexp results for RegexpInFile operations var dataFileRegexCache map[string]gcache.Cache = make(map[string]gcache.Cache) @@ -171,6 +175,7 @@ func Distance(lat1 string, long1 string, lat2 string, long2 string) (float64, er func Init(databaseClient *database.Client) error { dataFile = make(map[string][]string) dataFileRegex = make(map[string][]*regexp.Regexp) + dataFileRe2 = make(map[string][]*re2.Regexp) dbClient = databaseClient return nil } @@ -249,7 +254,11 @@ func FileInit(fileFolder string, filename string, fileType string) error { } switch fileType { case "regex", "regexp": - dataFileRegex[filename] = append(dataFileRegex[filename], regexp.MustCompile(scanner.Text())) + if fflag.Re2ExprHelperSupport.IsEnabled() { + dataFileRe2[filename] = append(dataFileRe2[filename], re2.MustCompile(scanner.Text())) + } else { + dataFileRegex[filename] = append(dataFileRegex[filename], regexp.MustCompile(scanner.Text())) + } case "string": dataFile[filename] = append(dataFile[filename], scanner.Text()) default: @@ -312,12 +321,24 @@ func RegexpInFile(data string, filename string) bool { } if _, ok := dataFileRegex[filename]; ok { - for _, re := range dataFileRegex[filename] { - if re.Match([]byte(data)) { - if hasCache { - dataFileRegexCache[filename].Set(hash, true) + switch fflag.Re2ExprHelperSupport.IsEnabled() { + case true: + for _, re := range dataFileRe2[filename] { + if re.MatchString(data) { + if hasCache { + dataFileRegexCache[filename].Set(hash, true) + } + return true + } + } + case false: + for _, re := range dataFileRegex[filename] { + if re.Match([]byte(data)) { + if hasCache { + dataFileRegexCache[filename].Set(hash, true) + } + return true } - return true } } } else { diff --git a/pkg/fflag/crowdsec.go b/pkg/fflag/crowdsec.go index 8dd49cdd1a9..0c1636f451b 100644 --- a/pkg/fflag/crowdsec.go +++ b/pkg/fflag/crowdsec.go @@ -7,6 +7,7 @@ var DisableHttpRetryBackoff = &Feature{Name: "disable_http_retry_backoff", Descr var ChunkedDecisionsStream = &Feature{Name: "chunked_decisions_stream", Description: "Enable chunked decisions stream"} var PapiClient = &Feature{Name: "papi_client", Description: "Enable Polling API client"} var Re2GrokSupport = &Feature{Name: "re2_grok_support", Description: "Enable RE2 support for GROK patterns"} +var Re2ExprHelperSupport = &Feature{Name: "re2_expr_helper_support", Description: "Enable RE2 support for expr helpers"} func RegisterAllFeatures() error { err := Crowdsec.RegisterFeature(CscliSetup) @@ -29,6 +30,10 @@ func RegisterAllFeatures() error { if err != nil { return err } + err = Crowdsec.RegisterFeature(Re2ExprHelperSupport) + if err != nil { + return err + } return nil } From 0ca47939ce82426b4b64924b1f3dc3e4d8ea6c99 Mon Sep 17 00:00:00 2001 From: Sebastien Blot Date: Thu, 23 Mar 2023 15:15:35 +0100 Subject: [PATCH 07/11] rename fflag --- pkg/exprhelpers/exprlib.go | 4 ++-- pkg/fflag/crowdsec.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/exprhelpers/exprlib.go b/pkg/exprhelpers/exprlib.go index e3d289ee9e1..bbdd06aa2a6 100644 --- a/pkg/exprhelpers/exprlib.go +++ b/pkg/exprhelpers/exprlib.go @@ -254,7 +254,7 @@ func FileInit(fileFolder string, filename string, fileType string) error { } switch fileType { case "regex", "regexp": - if fflag.Re2ExprHelperSupport.IsEnabled() { + if fflag.Re2RegexpInfileSupport.IsEnabled() { dataFileRe2[filename] = append(dataFileRe2[filename], re2.MustCompile(scanner.Text())) } else { dataFileRegex[filename] = append(dataFileRegex[filename], regexp.MustCompile(scanner.Text())) @@ -321,7 +321,7 @@ func RegexpInFile(data string, filename string) bool { } if _, ok := dataFileRegex[filename]; ok { - switch fflag.Re2ExprHelperSupport.IsEnabled() { + switch fflag.Re2RegexpInfileSupport.IsEnabled() { case true: for _, re := range dataFileRe2[filename] { if re.MatchString(data) { diff --git a/pkg/fflag/crowdsec.go b/pkg/fflag/crowdsec.go index 0c1636f451b..889f62dcf2f 100644 --- a/pkg/fflag/crowdsec.go +++ b/pkg/fflag/crowdsec.go @@ -7,7 +7,7 @@ var DisableHttpRetryBackoff = &Feature{Name: "disable_http_retry_backoff", Descr var ChunkedDecisionsStream = &Feature{Name: "chunked_decisions_stream", Description: "Enable chunked decisions stream"} var PapiClient = &Feature{Name: "papi_client", Description: "Enable Polling API client"} var Re2GrokSupport = &Feature{Name: "re2_grok_support", Description: "Enable RE2 support for GROK patterns"} -var Re2ExprHelperSupport = &Feature{Name: "re2_expr_helper_support", Description: "Enable RE2 support for expr helpers"} +var Re2RegexpInfileSupport = &Feature{Name: "re2_regexp_in_file_support", Description: "Enable RE2 support for RegexpInFile expr helper"} func RegisterAllFeatures() error { err := Crowdsec.RegisterFeature(CscliSetup) @@ -30,7 +30,7 @@ func RegisterAllFeatures() error { if err != nil { return err } - err = Crowdsec.RegisterFeature(Re2ExprHelperSupport) + err = Crowdsec.RegisterFeature(Re2RegexpInfileSupport) if err != nil { return err } From 33eccb2afcf6bf689fa03f3a0f16e4f888f47882 Mon Sep 17 00:00:00 2001 From: Sebastien Blot Date: Thu, 23 Mar 2023 15:49:14 +0100 Subject: [PATCH 08/11] fix re2 with regexpinfile --- pkg/exprhelpers/exprlib.go | 38 ++++++++++++++++++++------------------ 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/pkg/exprhelpers/exprlib.go b/pkg/exprhelpers/exprlib.go index bbdd06aa2a6..4930ef0b105 100644 --- a/pkg/exprhelpers/exprlib.go +++ b/pkg/exprhelpers/exprlib.go @@ -311,6 +311,7 @@ func RegexpInFile(data string, filename string) bool { var hash uint64 hasCache := false + matched := false if _, ok := dataFileRegexCache[filename]; ok { hasCache = true @@ -320,35 +321,36 @@ func RegexpInFile(data string, filename string) bool { } } - if _, ok := dataFileRegex[filename]; ok { - switch fflag.Re2RegexpInfileSupport.IsEnabled() { - case true: + switch fflag.Re2RegexpInfileSupport.IsEnabled() { + case true: + if _, ok := dataFileRe2[filename]; ok { for _, re := range dataFileRe2[filename] { if re.MatchString(data) { - if hasCache { - dataFileRegexCache[filename].Set(hash, true) - } - return true + matched = true + break } } - case false: + } else { + log.Errorf("file '%s' (type:regexp) not found in expr library", filename) + log.Errorf("expr library : %s", spew.Sdump(dataFileRegex)) + } + case false: + if _, ok := dataFileRegex[filename]; ok { for _, re := range dataFileRegex[filename] { - if re.Match([]byte(data)) { - if hasCache { - dataFileRegexCache[filename].Set(hash, true) - } - return true + if re.MatchString(data) { + matched = true + break } } + } else { + log.Errorf("file '%s' (type:regexp) not found in expr library", filename) + log.Errorf("expr library : %s", spew.Sdump(dataFileRegex)) } - } else { - log.Errorf("file '%s' (type:regexp) not found in expr library", filename) - log.Errorf("expr library : %s", spew.Sdump(dataFileRegex)) } if hasCache { - dataFileRegexCache[filename].Set(hash, false) + dataFileRegexCache[filename].Set(hash, matched) } - return false + return matched } func IpInRange(ip string, ipRange string) bool { From 9a324f58a402f8b4efcb40e1167286326549428c Mon Sep 17 00:00:00 2001 From: Sebastien Blot Date: Fri, 24 Mar 2023 11:31:32 +0100 Subject: [PATCH 09/11] up --- go.mod | 4 ++-- go.sum | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 79715d16863..e34cb169b89 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,7 @@ require ( github.com/confluentinc/bincover v0.2.0 github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26 - github.com/crowdsecurity/grokky v0.2.0 + github.com/crowdsecurity/grokky v0.2.1 github.com/crowdsecurity/machineid v1.0.2 github.com/davecgh/go-spew v1.1.1 github.com/dghubble/sling v1.3.0 @@ -177,7 +177,7 @@ require ( github.com/tklauser/numcpus v0.6.0 // indirect github.com/ugorji/go/codec v1.2.6 // indirect github.com/vmihailenco/msgpack v4.0.4+incompatible // indirect - github.com/wasilibs/go-re2 v0.1.0 // indirect + github.com/wasilibs/go-re2 v0.2.1 // indirect github.com/yusufpapurcu/wmi v1.2.2 // indirect github.com/zclconf/go-cty v1.8.0 // indirect go.mongodb.org/mongo-driver v1.9.0 // indirect diff --git a/go.sum b/go.sum index 8bdc67ea261..f24b1e2646d 100644 --- a/go.sum +++ b/go.sum @@ -179,6 +179,8 @@ github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26 h1:r97WNVC30Uen github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26/go.mod h1:zpv7r+7KXwgVUZnUNjyP22zc/D7LKjyoY02weH2RBbk= github.com/crowdsecurity/grokky v0.2.0 h1:8u3dCU1kqSlOu2phP6S+HF2OTohZ/JBGcZCWu1dhnF0= github.com/crowdsecurity/grokky v0.2.0/go.mod h1:qdyzFHtnY6nmdYZJFHVbdms2HMFLiEXzmqq5tCUmiGE= +github.com/crowdsecurity/grokky v0.2.1 h1:t4VYnDlAd0RjDM2SlILalbwfCrQxtJSMGdQOR0zwkE4= +github.com/crowdsecurity/grokky v0.2.1/go.mod h1:33usDIYzGDsgX1kHAThCbseso6JuWNJXOzRQDGXHtWM= github.com/crowdsecurity/machineid v1.0.2 h1:wpkpsUghJF8Khtmn/tg6GxgdhLA1Xflerh5lirI+bdc= github.com/crowdsecurity/machineid v1.0.2/go.mod h1:XWUSlnS0R0+u/JK5ulidwlbceNT3ZOCKteoVQEn6Luo= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -923,6 +925,8 @@ github.com/vmihailenco/msgpack/v4 v4.3.12/go.mod h1:gborTTJjAo/GWTqqRjrLCn9pgNN+ github.com/vmihailenco/tagparser v0.1.1/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI= github.com/wasilibs/go-re2 v0.1.0 h1:+pcjlvge6E9WWU4eaMlULRikEUkhPDXSFfCF5p7htMI= github.com/wasilibs/go-re2 v0.1.0/go.mod h1:F91Yac+zPNDFrrd8fl4mSd7+TTu2tYiX56BEIEPQl2M= +github.com/wasilibs/go-re2 v0.2.1 h1:bZVXxlqGHXnHlQt5ys6CsHyypsCsP952m1oKBB5O9Dc= +github.com/wasilibs/go-re2 v0.2.1/go.mod h1:jrB0h+KSTmTwsL0lI2sEyu8TcRi1htLiR/TbSURMwMM= github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= github.com/xdg-go/scram v1.0.2/go.mod h1:1WAq6h33pAW+iRreB34OORO2Nf7qel3VV3fjBj+hCSs= github.com/xdg-go/stringprep v1.0.2/go.mod h1:8F9zXuvzgwmyT5DUm4GUfZGDdT3W+LCvS6+da4O5kxM= From 18e62c9c8cbeed1d3e63880433197eb4ed5e28fa Mon Sep 17 00:00:00 2001 From: Sebastien Blot Date: Tue, 28 Mar 2023 10:55:38 +0200 Subject: [PATCH 10/11] increase workflow timeout for hub-test: we currently use go-re2 with its WASM runtime which significantly increase startup time --- .github/workflows/bats-hub.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/bats-hub.yml b/.github/workflows/bats-hub.yml index cda25b660cb..66ee409db00 100644 --- a/.github/workflows/bats-hub.yml +++ b/.github/workflows/bats-hub.yml @@ -19,7 +19,7 @@ jobs: name: "Build + tests" runs-on: ubuntu-latest - timeout-minutes: 20 + timeout-minutes: 30 steps: - name: "Force machineid" From dc3e06fac124798b8a7cc75441086be0423942fa Mon Sep 17 00:00:00 2001 From: bui Date: Tue, 28 Mar 2023 14:53:36 +0200 Subject: [PATCH 11/11] minor fixes --- pkg/acquisition/acquisition.go | 3 --- pkg/exprhelpers/helpers.go | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/pkg/acquisition/acquisition.go b/pkg/acquisition/acquisition.go index 59a015818e6..5ffa2d441df 100644 --- a/pkg/acquisition/acquisition.go +++ b/pkg/acquisition/acquisition.go @@ -6,7 +6,6 @@ import ( "io" "os" "strings" - "time" "github.com/prometheus/client_golang/prometheus" log "github.com/sirupsen/logrus" @@ -229,9 +228,7 @@ func StartAcquisition(sources []DataSource, output chan types.Event, AcquisTomb if subsrc.GetMode() == configuration.TAIL_MODE { err = subsrc.StreamingAcquisition(output, AcquisTomb) } else { - start := time.Now() err = subsrc.OneShotAcquisition(output, AcquisTomb) - log.Debugf("acquisition of %s took %s", subsrc.GetName(), time.Since(start)) } if err != nil { //if one of the acqusition returns an error, we kill the others to properly shutdown diff --git a/pkg/exprhelpers/helpers.go b/pkg/exprhelpers/helpers.go index cd10f74361e..4e2bf23f8a8 100644 --- a/pkg/exprhelpers/helpers.go +++ b/pkg/exprhelpers/helpers.go @@ -316,7 +316,7 @@ func RegexpInFile(params ...any) (any, error) { } } else { log.Errorf("file '%s' (type:regexp) not found in expr library", filename) - log.Errorf("expr library : %s", spew.Sdump(dataFileRegex)) + log.Errorf("expr library : %s", spew.Sdump(dataFileRe2)) } case false: if _, ok := dataFileRegex[filename]; ok {