[processor/lsminterval] Define cardinality limits and handle overflows

elastic · Nov 30, 2024 · d47a60c · d47a60c
1 parent 3651d70
commit d47a60c
Show file tree

Hide file tree

Showing 28 changed files with 1,665 additions and 280 deletions.
diff --git a/processor/lsmintervalprocessor/config.go → ...sor/lsmintervalprocessor/config/config.go b/processor/lsmintervalprocessor/config.go → ...sor/lsmintervalprocessor/config/config.go
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-package lsmintervalprocessor // import "github.com/elastic/opentelemetry-collector-components/processor/lsmintervalprocessor"
+package config // import "github.com/elastic/opentelemetry-collector-components/processor/lsmintervalprocessor/config"
 
 import (
 	"time"
@@ -39,7 +39,10 @@ type Config struct {
 	// TODO (lahsivjar): Make specifying interval easier. We can just
 	// optimize the timer to run on differnt times and remove any
 	// restriction on different interval configuration.
-	Intervals []IntervalConfig `mapstructure:"intervals"`
+	Intervals       []IntervalConfig `mapstructure:"intervals"`
+	ResourceLimits  LimitConfig      `mapstructure:"resource_limit"`
+	ScopeLimits     LimitConfig      `mapstructure:"scope_limit"`
+	DatapointLimits LimitConfig      `mapstructure:"datapoint_limit"`
 }
 
 // PassThrough determines whether metrics should be passed through as they
@@ -63,6 +66,21 @@ type IntervalConfig struct {
 	Statements []string `mapstructure:"statements"`
 }
 
+type LimitConfig struct {
+	Attributes     map[string]struct{} `mapstructure:"attributes"`
+	MaxCardinality uint64              `mapstructure:"max_cardinality"`
+	Overflow       OverflowConfig      `mapstructure:"overflow"`
+}
+
+type OverflowConfig struct {
+	Attributes []Attribute `mapstructure:"attributes"`
+}
+
+type Attribute struct {
+	Key   string `mapstructure:"key"`
+	Value any    `mapstructure:"value"`
+}
+
 func (config *Config) Validate() error {
 	// TODO (lahsivjar): Add validation for interval duration
 	return nil

diff --git a/processor/lsmintervalprocessor/factory.go b/processor/lsmintervalprocessor/factory.go
@@ -26,6 +26,7 @@ import (
 	"go.opentelemetry.io/collector/consumer"
 	"go.opentelemetry.io/collector/processor"
 
+	"github.com/elastic/opentelemetry-collector-components/processor/lsmintervalprocessor/config"
 	"github.com/elastic/opentelemetry-collector-components/processor/lsmintervalprocessor/internal/metadata"
 	"github.com/open-telemetry/opentelemetry-collector-contrib/pkg/ottl"
 	"github.com/open-telemetry/opentelemetry-collector-contrib/pkg/ottl/contexts/ottldatapoint"
@@ -41,8 +42,8 @@ func NewFactory() processor.Factory {
 }
 
 func createDefaultConfig() component.Config {
-	return &Config{
-		Intervals: []IntervalConfig{
+	return &config.Config{
+		Intervals: []config.IntervalConfig{
 			{Duration: 60 * time.Second},
 		},
 	}
@@ -54,7 +55,7 @@ func createMetricsProcessor(
 	cfg component.Config,
 	nextConsumer consumer.Metrics,
 ) (processor.Metrics, error) {
-	processorConfig, ok := cfg.(*Config)
+	processorConfig, ok := cfg.(*config.Config)
 	if !ok {
 		return nil, fmt.Errorf("configuration parsing error")
 	}

diff --git a/processor/lsmintervalprocessor/go.mod b/processor/lsmintervalprocessor/go.mod
@@ -3,6 +3,7 @@ module github.com/elastic/opentelemetry-collector-components/processor/lsminterv
 go 1.22.0
 
 require (
+	github.com/axiomhq/hyperloglog v0.2.0
 	github.com/cockroachdb/pebble v1.1.2
 	github.com/google/go-cmp v0.6.0
 	github.com/open-telemetry/opentelemetry-collector-contrib/pkg/golden v0.114.0
@@ -35,6 +36,7 @@ require (
 	github.com/cockroachdb/redact v1.1.5 // indirect
 	github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/dgryski/go-metro v0.0.0-20180109044635-280f6062b5bc // indirect
 	github.com/elastic/go-grok v0.3.1 // indirect
 	github.com/elastic/lunes v0.1.0 // indirect
 	github.com/getsentry/sentry-go v0.27.0 // indirect

diff --git a/processor/lsmintervalprocessor/go.sum b/processor/lsmintervalprocessor/go.sum
@@ -10,6 +10,8 @@ github.com/antchfx/xmlquery v1.4.2 h1:MZKd9+wblwxfQ1zd1AdrTsqVaMjMCwow3IqkCSe00K
 github.com/antchfx/xmlquery v1.4.2/go.mod h1:QXhvf5ldTuGqhd1SHNvvtlhhdQLks4dD0awIVhXIDTA=
 github.com/antchfx/xpath v1.3.2 h1:LNjzlsSjinu3bQpw9hWMY9ocB80oLOWuQqFvO6xt51U=
 github.com/antchfx/xpath v1.3.2/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs=
+github.com/axiomhq/hyperloglog v0.2.0 h1:u1XT3yyY1rjzlWuP6NQIrV4bRYHOaqZaovqjcBEvZJo=
+github.com/axiomhq/hyperloglog v0.2.0/go.mod h1:GcgMjz9gaDKZ3G0UMS6Fq/VkZ4l7uGgcJyxA7M+omIM=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
@@ -32,6 +34,8 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dgryski/go-metro v0.0.0-20180109044635-280f6062b5bc h1:8WFBn63wegobsYAX0YjD+8suexZDga5CctH4CCTx2+8=
+github.com/dgryski/go-metro v0.0.0-20180109044635-280f6062b5bc/go.mod h1:c9O8+fpSOX1DM8cPNSkX/qsBWdkD4yd2dpciOWQjpBw=
 github.com/elastic/go-grok v0.3.1 h1:WEhUxe2KrwycMnlvMimJXvzRa7DoByJB4PVUIE1ZD/U=
 github.com/elastic/go-grok v0.3.1/go.mod h1:n38ls8ZgOboZRgKcjMY8eFeZFMmcL9n2lP0iHhIDk64=
 github.com/elastic/lunes v0.1.0 h1:amRtLPjwkWtzDF/RKzcEPMvSsSseLDLW+bnhfNSLRe4=