From 1be9bbfd2db9d39cddbb89784f898d3187162a3f Mon Sep 17 00:00:00 2001 From: Marc Guasch Date: Thu, 2 Dec 2021 12:02:21 +0100 Subject: [PATCH 1/6] Add allowed ip check --- internal/fields/validate.go | 63 ++++++++++++++++++++++++++++++++++++- 1 file changed, 62 insertions(+), 1 deletion(-) diff --git a/internal/fields/validate.go b/internal/fields/validate.go index 6ceee110c..cdc7950df 100644 --- a/internal/fields/validate.go +++ b/internal/fields/validate.go @@ -7,6 +7,7 @@ package fields import ( "encoding/json" "fmt" + "net" "os" "path/filepath" "regexp" @@ -306,7 +307,7 @@ func (v *Validator) parseElementValue(key string, definition FieldDefinition, va if err := ensurePatternMatches(key, valStr, definition.Pattern); err != nil { return err } - case "date", "ip", "keyword", "text": + case "date", "keyword", "text": var valStr string valStr, valid = val.(string) if !valid { @@ -316,6 +317,20 @@ func (v *Validator) parseElementValue(key string, definition FieldDefinition, va if err := ensurePatternMatches(key, valStr, definition.Pattern); err != nil { return err } + case "ip": + var valStr string + valStr, valid = val.(string) + if !valid { + break + } + + if err := ensurePatternMatches(key, valStr, definition.Pattern); err != nil { + return err + } + + if !isAllowedIPValue(valStr) { + return fmt.Errorf("the IP %q is not one of the allowed test IPs", valStr) + } case "float", "long", "double": _, valid = val.(float64) default: @@ -328,6 +343,52 @@ func (v *Validator) parseElementValue(key string, definition FieldDefinition, va return nil } +// isAllowedIPValue checks if the provided IP is allowed for testing +// The set of allowed IPs are: +// - private IPs as described in RFC 1918 & RFC 4193 +// - public IPs allowed by MaxMind for testing +// - 0.0.0.0 and 255.255.255.255 +func isAllowedIPValue(v string) bool { + allowedIPs := map[string]struct{}{ + "0.0.0.0": {}, + "255.255.255.255": {}, + + // maxmind allowed set found at: + // https://github.com/elastic/elastic-package/blob/master/docs/howto/ingest_geoip.md + "1.128.3.4": {}, + "175.16.199.1": {}, + "216.160.83.57": {}, + "216.160.83.61": {}, + "81.2.69.143": {}, + "81.2.69.144": {}, + "81.2.69.145": {}, + "81.2.69.193": {}, + "89.160.20.112": {}, + "89.160.20.156": {}, + "67.43.156.12": {}, + "67.43.156.13": {}, + "67.43.156.14": {}, + "67.43.156.15": {}, + "2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6": {}, + } + + if _, found := allowedIPs[v]; found { + return true + } + + ip := net.ParseIP(v) + if ip == nil { + return false + } + + if ip.IsPrivate() || ip.IsLoopback() || + ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() { + return true + } + + return false +} + // ensureSingleElementValue extracts single entity from a potential array, which is a valid field representation // in Elasticsearch. For type assertion we need a single value. func ensureSingleElementValue(val interface{}) (interface{}, bool) { From da9e6c1fef01c1f5ed7f2a0afcdb476f8fca6cd8 Mon Sep 17 00:00:00 2001 From: Marc Guasch Date: Thu, 2 Dec 2021 15:08:41 +0100 Subject: [PATCH 2/6] Disable allowed ip checks for system tests --- internal/fields/validate.go | 13 +++++++++++++ internal/testrunner/runners/system/runner.go | 4 +++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/internal/fields/validate.go b/internal/fields/validate.go index cdc7950df..66c352641 100644 --- a/internal/fields/validate.go +++ b/internal/fields/validate.go @@ -32,6 +32,7 @@ type Validator struct { numericKeywordFields map[string]struct{} disabledDependencyManagement bool + disabledAllowedIPCheck bool } // ValidatorOption represents an optional flag that can be passed to CreateValidatorForDataStream. @@ -65,6 +66,14 @@ func WithDisabledDependencyManagement() ValidatorOption { } } +// WithDisabledAllowedIPCheck configures the validator to ignore external fields and won't follow dependencies. +func WithDisabledAllowedIPCheck() ValidatorOption { + return func(v *Validator) error { + v.disabledAllowedIPCheck = true + return nil + } +} + // CreateValidatorForDataStream function creates a validator for the data stream. func CreateValidatorForDataStream(dataStreamRootPath string, opts ...ValidatorOption) (v *Validator, err error) { v = new(Validator) @@ -328,6 +337,10 @@ func (v *Validator) parseElementValue(key string, definition FieldDefinition, va return err } + if v.disabledAllowedIPCheck { + break + } + if !isAllowedIPValue(valStr) { return fmt.Errorf("the IP %q is not one of the allowed test IPs", valStr) } diff --git a/internal/testrunner/runners/system/runner.go b/internal/testrunner/runners/system/runner.go index a192c55e7..005528688 100644 --- a/internal/testrunner/runners/system/runner.go +++ b/internal/testrunner/runners/system/runner.go @@ -438,7 +438,9 @@ func (r *runner) runTest(config *testConfig, ctxt servicedeployer.ServiceContext // Validate fields in docs fieldsValidator, err := fields.CreateValidatorForDataStream(serviceOptions.DataStreamRootPath, - fields.WithNumericKeywordFields(config.NumericKeywordFields)) + fields.WithNumericKeywordFields(config.NumericKeywordFields), + fields.WithDisabledAllowedIPCheck(), + ) if err != nil { return result.WithError(errors.Wrapf(err, "creating fields validator for data stream failed (path: %s)", serviceOptions.DataStreamRootPath)) } From 686d8628307780161ba798650845b612187dfdcb Mon Sep 17 00:00:00 2001 From: Marc Guasch Date: Thu, 2 Dec 2021 15:55:04 +0100 Subject: [PATCH 3/6] Embed IPs file and change docs --- docs/howto/ingest_geoip.md | 21 +------ internal/fields/_static/allowed_geo_ips.txt | 15 +++++ internal/fields/validate.go | 63 +++++++++---------- .../testrunner/runners/pipeline/runner.go | 6 +- internal/testrunner/runners/system/runner.go | 4 +- 5 files changed, 50 insertions(+), 59 deletions(-) create mode 100644 internal/fields/_static/allowed_geo_ips.txt diff --git a/docs/howto/ingest_geoip.md b/docs/howto/ingest_geoip.md index 51c956fc3..3dd9cd4af 100644 --- a/docs/howto/ingest_geoip.md +++ b/docs/howto/ingest_geoip.md @@ -2,26 +2,7 @@ Elasticsearch provides default GeoIP databases that can be downloaded in runtime and which weights ~70 MB. This can be a root cause of flakiness of package tests, so elastic-package embeds small samples of GeoIP databases, that can identify -accurately only few ranges of IP addresses: - -``` -1.128.3.4 -175.16.199.1 -216.160.83.57 -216.160.83.61 -67.43.156.12 -81.2.69.143 -81.2.69.144 -81.2.69.145 -81.2.69.193 -89.160.20.112 -89.160.20.156 -67.43.156.12 -67.43.156.13 -67.43.156.14 -67.43.156.15 -2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6 -``` +accurately only few ranges of IP addresses included [here](../../internal/fields/_static/allowed_geo_ips.txt) If you want the ingest pipeline to include a "geo" section in the event, feel free to use one of above IP addresses. Embedded databases contain information about: cities, countries and ASNs. \ No newline at end of file diff --git a/internal/fields/_static/allowed_geo_ips.txt b/internal/fields/_static/allowed_geo_ips.txt new file mode 100644 index 000000000..500b8b6bb --- /dev/null +++ b/internal/fields/_static/allowed_geo_ips.txt @@ -0,0 +1,15 @@ +1.128.3.4 +175.16.199.1 +216.160.83.57 +216.160.83.61 +81.2.69.143 +81.2.69.144 +81.2.69.145 +81.2.69.193 +89.160.20.112 +89.160.20.156 +67.43.156.12 +67.43.156.13 +67.43.156.14 +67.43.156.15 +2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6 \ No newline at end of file diff --git a/internal/fields/validate.go b/internal/fields/validate.go index 66c352641..d62398170 100644 --- a/internal/fields/validate.go +++ b/internal/fields/validate.go @@ -5,6 +5,7 @@ package fields import ( + _ "embed" "encoding/json" "fmt" "net" @@ -32,7 +33,9 @@ type Validator struct { numericKeywordFields map[string]struct{} disabledDependencyManagement bool - disabledAllowedIPCheck bool + + enabledAllowedIPCheck bool + allowedIPs map[string]struct{} } // ValidatorOption represents an optional flag that can be passed to CreateValidatorForDataStream. @@ -66,10 +69,10 @@ func WithDisabledDependencyManagement() ValidatorOption { } } -// WithDisabledAllowedIPCheck configures the validator to ignore external fields and won't follow dependencies. -func WithDisabledAllowedIPCheck() ValidatorOption { +// WithEnabledAllowedIPCheck configures the validator to ignore external fields and won't follow dependencies. +func WithEnabledAllowedIPCheck() ValidatorOption { return func(v *Validator) error { - v.disabledAllowedIPCheck = true + v.enabledAllowedIPCheck = true return nil } } @@ -82,6 +85,9 @@ func CreateValidatorForDataStream(dataStreamRootPath string, opts ...ValidatorOp return nil, err } } + + v.allowedIPs = initializeAllowedIPsList() + v.Schema, err = loadFieldsForDataStream(dataStreamRootPath) if err != nil { return nil, errors.Wrapf(err, "can't load fields for data stream (path: %s)", dataStreamRootPath) @@ -109,6 +115,20 @@ func CreateValidatorForDataStream(dataStreamRootPath string, opts ...ValidatorOp return v, nil } +//go:embed _static/allowed_geo_ips.txt +var allowedGeoIPs string + +func initializeAllowedIPsList() map[string]struct{} { + m := map[string]struct{}{ + "0.0.0.0": {}, "255.255.255.255": {}, + } + for _, ip := range strings.Split(allowedGeoIPs, "\n") { + m[ip] = struct{}{} + } + + return m +} + func loadFieldsForDataStream(dataStreamRootPath string) ([]FieldDefinition, error) { fieldsDir := filepath.Join(dataStreamRootPath, "fields") files, err := filepath.Glob(filepath.Join(fieldsDir, "*.yml")) @@ -337,11 +357,7 @@ func (v *Validator) parseElementValue(key string, definition FieldDefinition, va return err } - if v.disabledAllowedIPCheck { - break - } - - if !isAllowedIPValue(valStr) { + if v.enabledAllowedIPCheck && !v.isAllowedIPValue(valStr) { return fmt.Errorf("the IP %q is not one of the allowed test IPs", valStr) } case "float", "long", "double": @@ -361,35 +377,12 @@ func (v *Validator) parseElementValue(key string, definition FieldDefinition, va // - private IPs as described in RFC 1918 & RFC 4193 // - public IPs allowed by MaxMind for testing // - 0.0.0.0 and 255.255.255.255 -func isAllowedIPValue(v string) bool { - allowedIPs := map[string]struct{}{ - "0.0.0.0": {}, - "255.255.255.255": {}, - - // maxmind allowed set found at: - // https://github.com/elastic/elastic-package/blob/master/docs/howto/ingest_geoip.md - "1.128.3.4": {}, - "175.16.199.1": {}, - "216.160.83.57": {}, - "216.160.83.61": {}, - "81.2.69.143": {}, - "81.2.69.144": {}, - "81.2.69.145": {}, - "81.2.69.193": {}, - "89.160.20.112": {}, - "89.160.20.156": {}, - "67.43.156.12": {}, - "67.43.156.13": {}, - "67.43.156.14": {}, - "67.43.156.15": {}, - "2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6": {}, - } - - if _, found := allowedIPs[v]; found { +func (v *Validator) isAllowedIPValue(s string) bool { + if _, found := v.allowedIPs[s]; found { return true } - ip := net.ParseIP(v) + ip := net.ParseIP(s) if ip == nil { return false } diff --git a/internal/testrunner/runners/pipeline/runner.go b/internal/testrunner/runners/pipeline/runner.go index 85faf51e3..70c45a76d 100644 --- a/internal/testrunner/runners/pipeline/runner.go +++ b/internal/testrunner/runners/pipeline/runner.go @@ -131,7 +131,11 @@ func (r *runner) run() ([]testrunner.TestResult, error) { tr.TimeElapsed = time.Since(startTime) fieldsValidator, err := fields.CreateValidatorForDataStream(dataStreamPath, - fields.WithNumericKeywordFields(tc.config.NumericKeywordFields)) + fields.WithNumericKeywordFields(tc.config.NumericKeywordFields), + // explicitly enabled for pipeline tests only + // since system tests can have dynamic public IPs + fields.WithEnabledAllowedIPCheck(), + ) if err != nil { return nil, errors.Wrapf(err, "creating fields validator for data stream failed (path: %s, test case file: %s)", dataStreamPath, testCaseFile) } diff --git a/internal/testrunner/runners/system/runner.go b/internal/testrunner/runners/system/runner.go index 005528688..a192c55e7 100644 --- a/internal/testrunner/runners/system/runner.go +++ b/internal/testrunner/runners/system/runner.go @@ -438,9 +438,7 @@ func (r *runner) runTest(config *testConfig, ctxt servicedeployer.ServiceContext // Validate fields in docs fieldsValidator, err := fields.CreateValidatorForDataStream(serviceOptions.DataStreamRootPath, - fields.WithNumericKeywordFields(config.NumericKeywordFields), - fields.WithDisabledAllowedIPCheck(), - ) + fields.WithNumericKeywordFields(config.NumericKeywordFields)) if err != nil { return result.WithError(errors.Wrapf(err, "creating fields validator for data stream failed (path: %s)", serviceOptions.DataStreamRootPath)) } From 5801ce014b77d30b9c0061246adcbe35930b13c5 Mon Sep 17 00:00:00 2001 From: Marc Guasch Date: Thu, 2 Dec 2021 16:01:12 +0100 Subject: [PATCH 4/6] Update validate.go --- internal/fields/validate.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/fields/validate.go b/internal/fields/validate.go index d62398170..4499ec686 100644 --- a/internal/fields/validate.go +++ b/internal/fields/validate.go @@ -69,7 +69,7 @@ func WithDisabledDependencyManagement() ValidatorOption { } } -// WithEnabledAllowedIPCheck configures the validator to ignore external fields and won't follow dependencies. +// WithEnabledAllowedIPCheck configures the validator to perform check on the IP values against an allowed list. func WithEnabledAllowedIPCheck() ValidatorOption { return func(v *Validator) error { v.enabledAllowedIPCheck = true From 6839c885e4b35fbc4721db651e9f027947dc8c45 Mon Sep 17 00:00:00 2001 From: Marc Guasch Date: Fri, 3 Dec 2021 10:12:32 +0100 Subject: [PATCH 5/6] Add allowed for ipv6 --- internal/fields/validate.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/internal/fields/validate.go b/internal/fields/validate.go index 4499ec686..6323716e4 100644 --- a/internal/fields/validate.go +++ b/internal/fields/validate.go @@ -121,6 +121,7 @@ var allowedGeoIPs string func initializeAllowedIPsList() map[string]struct{} { m := map[string]struct{}{ "0.0.0.0": {}, "255.255.255.255": {}, + "0:0:0:0:0:0:0:0": {}, "ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff": {}, } for _, ip := range strings.Split(allowedGeoIPs, "\n") { m[ip] = struct{}{} @@ -376,7 +377,8 @@ func (v *Validator) parseElementValue(key string, definition FieldDefinition, va // The set of allowed IPs are: // - private IPs as described in RFC 1918 & RFC 4193 // - public IPs allowed by MaxMind for testing -// - 0.0.0.0 and 255.255.255.255 +// - 0.0.0.0 and 255.255.255.255 for IPv4 +// - 0:0:0:0:0:0:0:0 and ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff for IPv6 func (v *Validator) isAllowedIPValue(s string) bool { if _, found := v.allowedIPs[s]; found { return true From 86ef3b0ffa078d655de6b1a35b5754b627902fbf Mon Sep 17 00:00:00 2001 From: Marc Guasch Date: Mon, 13 Dec 2021 15:41:50 +0100 Subject: [PATCH 6/6] Trim possible empty IPs --- internal/fields/validate.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/internal/fields/validate.go b/internal/fields/validate.go index 6323716e4..77cd03e77 100644 --- a/internal/fields/validate.go +++ b/internal/fields/validate.go @@ -124,6 +124,10 @@ func initializeAllowedIPsList() map[string]struct{} { "0:0:0:0:0:0:0:0": {}, "ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff": {}, } for _, ip := range strings.Split(allowedGeoIPs, "\n") { + ip = strings.Trim(ip, " \n\t") + if ip == "" { + continue + } m[ip] = struct{}{} }