From bccb601ebda3fe467d665792eee38a4aec7fd374 Mon Sep 17 00:00:00 2001 From: kehindesalaam Date: Mon, 15 Jul 2024 21:45:53 +0200 Subject: [PATCH 01/16] Add `otelcol.processor.groupbyattrs` component --- CHANGELOG.md | 3 + .../otelcol/otelcol.processor.groupbyattrs.md | 88 ++++++++++++++++++ go.mod | 24 ++--- go.sum | 44 +++++---- internal/component/all/all.go | 1 + .../processor/groupbyattrs/groupbyattrs.go | 92 +++++++++++++++++++ .../groupbyattrs/groupbyattrs_test.go | 80 ++++++++++++++++ 7 files changed, 301 insertions(+), 31 deletions(-) create mode 100644 docs/sources/reference/components/otelcol/otelcol.processor.groupbyattrs.md create mode 100644 internal/component/otelcol/processor/groupbyattrs/groupbyattrs.go create mode 100644 internal/component/otelcol/processor/groupbyattrs/groupbyattrs_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 99edfe611b..4627d08e4a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -39,6 +39,9 @@ Main (unreleased) - (_Public preview_) Added rate limiting of cluster state changes to reduce the number of unnecessary, intermediate state updates. (@thampiotr) +- Add a `otelcol.processor.groupbyattrs` component to reassociate collected metrics that match specified attributes + from opentelemetry. (@kehindesalaam) + ### Bugfixes - Fixed a clustering mode issue where a fatal startup failure of the clustering service diff --git a/docs/sources/reference/components/otelcol/otelcol.processor.groupbyattrs.md b/docs/sources/reference/components/otelcol/otelcol.processor.groupbyattrs.md new file mode 100644 index 0000000000..243cb64fa1 --- /dev/null +++ b/docs/sources/reference/components/otelcol/otelcol.processor.groupbyattrs.md @@ -0,0 +1,88 @@ +# otelcol.processor.groupbyattrs + +`otelcol.processor.groupbyattrs` accepts telemetry data from other `otelcol` +components and reassociates spans, log records and metric datapoints to a Resource that matches with the specified attributes. It groups telemetry data by specified attributes. + +{{% admonition type="note" %}} +`otelcol.processor.groupbyattrs` is a wrapper over the upstream OpenTelemetry +Collector `groupbyattrs` processor. If necessary, bug reports or feature requests +will be redirected to the upstream repository. +{{% /admonition %}} + +It is recommended to use the groupbyattrs processor together with batch processor, as a consecutive step, as this will reduce the fragmentation of data (by grouping records together under matching Resource/Instrumentation Library) + +You can specify multiple `otelcol.processor.groupbyattrs` components by giving them +different labels. + +## Usage + +```river +otelcol.processor.groupbyattrs "LABEL" { + output { + metrics = [...] + logs = [...] + traces = [...] + } +} +``` + +## Arguments + +The following arguments are supported: + +Name | Type | Description | Default | Required +---- |-----------------|---------------------------------------------------------------------------------------|------------| -------- +`keys` | `array(string)` | Keys that will be used to group the spans, log records or metric data points together | | no + + +## Exported fields + +The following fields are exported and can be referenced by other components: + +Name | Type | Description +--------|--------------------|----------------------------------------------------------------- +`input` | `otelcol.Consumer` | A value that other components can use to send telemetry data to. + +`input` accepts `otelcol.Consumer` data for any telemetry signal (metrics, +logs, or traces). + +## Component health + +`otelcol.processor.groupbyattrs` is only reported as unhealthy if given an invalid +configuration. + +## Debug information + +`otelcol.processor.groupbyattrs` does not expose any component-specific debug +information. + +## Debug metrics + +`otelcol.processor.groupbyattrs` does not expose any component-specific debug metrics. + +## Examples + +### Grouping metrics by an attribute + +This example reassociates the metrics based on the value of the `host.name` attribute. + +```alloy +otelcol.processor.groupbyattrs "default" { + keys = [ + "host.name", + ] + + output { + metrics = [otelcol.exporter.otlp.default.input] + logs = [otelcol.exporter.otlp.default.input] + traces = [otelcol.exporter.otlp.default.input] + } +} +``` + +## Notes +- The data points with different data types would not be merged under the same metric (e.g a gauge and sum metric would not be merged). +- The data points without the specified keys would remain under their respective resources. +- New resources would inherit the attributes of the original resource and the specified attributes in the keys array. +- The grouping attributes in the keys array would be removed from the output metrics. +- If the keys array is empty, the processor would perform compaction and re-associate all spans with matching Resource and InstrumentationLibrary. diff --git a/go.mod b/go.mod index 0a1ef96bee..1b12fe1eda 100644 --- a/go.mod +++ b/go.mod @@ -115,7 +115,7 @@ require ( github.com/open-telemetry/opentelemetry-collector-contrib/extension/sigv4authextension v0.102.0 github.com/open-telemetry/opentelemetry-collector-contrib/pkg/ottl v0.102.0 github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatatest v0.102.0 - github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatautil v0.102.0 + github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatautil v0.104.0 github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/loki v0.102.0 github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/prometheus v0.102.0 github.com/open-telemetry/opentelemetry-collector-contrib/processor/attributesprocessor v0.102.0 @@ -151,7 +151,7 @@ require ( github.com/prometheus/blackbox_exporter v0.24.1-0.20230623125439-bd22efa1c900 github.com/prometheus/client_golang v1.19.1 github.com/prometheus/client_model v0.6.1 - github.com/prometheus/common v0.53.0 + github.com/prometheus/common v0.54.0 github.com/prometheus/common/sigv4 v0.1.0 github.com/prometheus/consul_exporter v0.8.0 github.com/prometheus/memcached_exporter v0.13.0 @@ -181,8 +181,8 @@ require ( github.com/wk8/go-ordered-map v0.2.0 github.com/xdg-go/scram v1.1.2 github.com/zeebo/xxh3 v1.0.2 - go.opentelemetry.io/collector v0.102.1 - go.opentelemetry.io/collector/component v0.102.1 + go.opentelemetry.io/collector v0.104.0 + go.opentelemetry.io/collector/component v0.104.0 go.opentelemetry.io/collector/config/configauth v0.102.1 go.opentelemetry.io/collector/config/configcompression v1.9.0 go.opentelemetry.io/collector/config/configgrpc v0.102.1 @@ -190,23 +190,23 @@ require ( go.opentelemetry.io/collector/config/confignet v0.102.1 go.opentelemetry.io/collector/config/configopaque v1.9.0 go.opentelemetry.io/collector/config/configretry v0.102.1 - go.opentelemetry.io/collector/config/configtelemetry v0.102.1 + go.opentelemetry.io/collector/config/configtelemetry v0.104.0 go.opentelemetry.io/collector/config/configtls v0.102.1 - go.opentelemetry.io/collector/confmap v0.102.1 + go.opentelemetry.io/collector/confmap v0.104.0 go.opentelemetry.io/collector/confmap/converter/expandconverter v0.102.1 go.opentelemetry.io/collector/confmap/provider/yamlprovider v0.102.1 go.opentelemetry.io/collector/connector v0.102.1 - go.opentelemetry.io/collector/consumer v0.102.1 + go.opentelemetry.io/collector/consumer v0.104.0 go.opentelemetry.io/collector/exporter v0.102.1 go.opentelemetry.io/collector/exporter/loggingexporter v0.102.1 go.opentelemetry.io/collector/exporter/otlpexporter v0.102.1 go.opentelemetry.io/collector/exporter/otlphttpexporter v0.102.1 go.opentelemetry.io/collector/extension v0.102.1 go.opentelemetry.io/collector/extension/auth v0.102.1 - go.opentelemetry.io/collector/featuregate v1.9.0 + go.opentelemetry.io/collector/featuregate v1.11.0 go.opentelemetry.io/collector/otelcol v0.102.1 - go.opentelemetry.io/collector/pdata v1.9.0 - go.opentelemetry.io/collector/processor v0.102.1 + go.opentelemetry.io/collector/pdata v1.11.0 + go.opentelemetry.io/collector/processor v0.104.0 go.opentelemetry.io/collector/processor/batchprocessor v0.102.1 go.opentelemetry.io/collector/processor/memorylimiterprocessor v0.102.1 go.opentelemetry.io/collector/receiver v0.102.1 @@ -238,7 +238,7 @@ require ( golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d google.golang.org/api v0.180.0 google.golang.org/grpc v1.64.0 - google.golang.org/protobuf v1.34.1 + google.golang.org/protobuf v1.34.2 gopkg.in/yaml.v2 v2.4.0 gopkg.in/yaml.v3 v3.0.1 gotest.tools v2.2.0+incompatible @@ -707,6 +707,8 @@ require ( sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect ) +require github.com/open-telemetry/opentelemetry-collector-contrib/processor/groupbyattrsprocessor v0.104.0 + // NOTE: replace directives below must always be *temporary*. // // Adding a replace directive to change a module to a fork of a module will diff --git a/go.sum b/go.sum index dcc2393583..1adf426450 100644 --- a/go.sum +++ b/go.sum @@ -1741,8 +1741,8 @@ github.com/open-telemetry/opentelemetry-collector-contrib/pkg/ottl v0.102.0 h1:E github.com/open-telemetry/opentelemetry-collector-contrib/pkg/ottl v0.102.0/go.mod h1:qnLc/+jOVcsL1dF17ztBcf3juQ3f9bt6Wuf+Xxbrd9w= github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatatest v0.102.0 h1:vJL6lDaeI3pVA7ADnWKD3HMpI80BSrZ2UnGc+qkwqoY= github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatatest v0.102.0/go.mod h1:xtE7tds5j8PtI/wMuGb+Em5K9rJH8hm6t28Qe4QrpoU= -github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatautil v0.102.0 h1:TvJYcU/DLRFCgHr7nT98k5D+qkZ4syKVxc8OJjv+K4c= -github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatautil v0.102.0/go.mod h1:WzD3Ox7tywAQHknxAFpAC1oZJGItMp5mbvgUGjvzNY8= +github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatautil v0.104.0 h1:Pl4rXXpRG/xJuNWUS3I/w1jViHcrssMf47bGX/Ug/KY= +github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatautil v0.104.0/go.mod h1:tP4dyc5+g/qoXYb8lmNj+y+Nhphn4MkL23/np0Zhx2g= github.com/open-telemetry/opentelemetry-collector-contrib/pkg/resourcetotelemetry v0.102.0 h1:iVdVcLq5uCvvG6bmOwdbRQbjWPsaQY/caDaIE4rJV80= github.com/open-telemetry/opentelemetry-collector-contrib/pkg/resourcetotelemetry v0.102.0/go.mod h1:gSlq0MAX1balwTobJjaQtk/Znm3We2muLNaSLELHxUQ= github.com/open-telemetry/opentelemetry-collector-contrib/pkg/sampling v0.102.0 h1:Nue1wHi8PobP90PXeB8vqoITOCZA/+Hs5Sy3fKfaTKo= @@ -1765,6 +1765,8 @@ github.com/open-telemetry/opentelemetry-collector-contrib/processor/deltatocumul github.com/open-telemetry/opentelemetry-collector-contrib/processor/deltatocumulativeprocessor v0.102.0/go.mod h1:4mjsDJoPFf7MDE6bQpDEr25D/U2HTaD4OZKwo7gt8t8= github.com/open-telemetry/opentelemetry-collector-contrib/processor/filterprocessor v0.102.0 h1:DaEYlVCn58GtkyYVK0IT/ZMjRFJ+BfmR0p9I0Eq42aQ= github.com/open-telemetry/opentelemetry-collector-contrib/processor/filterprocessor v0.102.0/go.mod h1:u9x08rUCWdgI8Nle5XOMTCmxd0K26KTZvMMA5H8Xjyg= +github.com/open-telemetry/opentelemetry-collector-contrib/processor/groupbyattrsprocessor v0.104.0 h1:Dust79ten+1/q92AgOLdnBnQDCsVT5rjl0/eSKFLdYo= +github.com/open-telemetry/opentelemetry-collector-contrib/processor/groupbyattrsprocessor v0.104.0/go.mod h1:evkdeLcXutvRAz202BdwNk2g3+Y/mxVqX7hhDIJId5k= github.com/open-telemetry/opentelemetry-collector-contrib/processor/k8sattributesprocessor v0.102.0 h1:mkRDKVWXfG1gTxwg69ttJoGmXOKNHAGsGms06DrwTlQ= github.com/open-telemetry/opentelemetry-collector-contrib/processor/k8sattributesprocessor v0.102.0/go.mod h1:5F6hpHujLkLuEYmbbUXel2i3mBpwRJHmy8KTY3cbOVg= github.com/open-telemetry/opentelemetry-collector-contrib/processor/probabilisticsamplerprocessor v0.102.0 h1:ErBYnmZUSyPQjHPlyAeUOtQDax0tH2Ax/zOuklZp5Y8= @@ -1947,8 +1949,8 @@ github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9 github.com/prometheus/common v0.29.0/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= github.com/prometheus/common v0.31.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= -github.com/prometheus/common v0.53.0 h1:U2pL9w9nmJwJDa4qqLQ3ZaePJ6ZTwt7cMD3AG3+aLCE= -github.com/prometheus/common v0.53.0/go.mod h1:BrxBKv3FWBIGXw89Mg1AeBq7FSyRzXWI3l3e7W3RN5U= +github.com/prometheus/common v0.54.0 h1:ZlZy0BgJhTwVZUn7dLOkwCZHUkrAqd3WYtcFCWnM1D8= +github.com/prometheus/common v0.54.0/go.mod h1:/TQgMJP5CuVYveyT7n/0Ix8yLNNXy9yRSkhnLTHPDIQ= github.com/prometheus/common/sigv4 v0.1.0 h1:qoVebwtwwEhS85Czm2dSROY5fTo2PAPEVdDeppTwGX4= github.com/prometheus/common/sigv4 v0.1.0/go.mod h1:2Jkxxk9yYvCkE5G1sQT7GuEXm57JrvHu9k5YwTjsNtI= github.com/prometheus/consul_exporter v0.8.0 h1:2z3drFic65WFoHaJRKkmnJRRlBLmmxVqT8L9LO2yxAo= @@ -2289,8 +2291,8 @@ go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= -go.opentelemetry.io/collector/component v0.102.1 h1:66z+LN5dVCXhvuVKD1b56/3cYLK+mtYSLIwlskYA9IQ= -go.opentelemetry.io/collector/component v0.102.1/go.mod h1:XfkiSeImKYaewT2DavA80l0VZ3JjvGndZ8ayPXfp8d0= +go.opentelemetry.io/collector/component v0.104.0 h1:jqu/X9rnv8ha0RNZ1a9+x7OU49KwSMsPbOuIEykHuQE= +go.opentelemetry.io/collector/component v0.104.0/go.mod h1:1C7C0hMVSbXyY1ycCmaMUAR9fVwpgyiNQqxXtEWhVpw= go.opentelemetry.io/collector/config/configauth v0.102.1 h1:LuzijaZulMu4xmAUG8WA00ZKDlampH+ERjxclb40Q9g= go.opentelemetry.io/collector/config/configauth v0.102.1/go.mod h1:kTzfI5fnbMJpm2wycVtQeWxFAtb7ns4HksSb66NIhX8= go.opentelemetry.io/collector/config/configcompression v1.9.0 h1:B2q6XMO6xiF2s+14XjqAQHGY5UefR+PtkZ0WAlmSqpU= @@ -2305,14 +2307,14 @@ go.opentelemetry.io/collector/config/configopaque v1.9.0 h1:jocenLdK/rVG9UoGlnpi go.opentelemetry.io/collector/config/configopaque v1.9.0/go.mod h1:8v1yaH4iYjcigbbyEaP/tzVXeFm4AaAsKBF9SBeqaG4= go.opentelemetry.io/collector/config/configretry v0.102.1 h1:J5/tXBL8P7d7HT5dxsp2H+//SkwDXR66Z9UTgRgtAzk= go.opentelemetry.io/collector/config/configretry v0.102.1/go.mod h1:P+RA0IA+QoxnDn4072uyeAk1RIoYiCbxYsjpKX5eFC4= -go.opentelemetry.io/collector/config/configtelemetry v0.102.1 h1:f/CYcrOkaHd+COIJ2lWnEgBCHfhEycpbow4ZhrGwAlA= -go.opentelemetry.io/collector/config/configtelemetry v0.102.1/go.mod h1:WxWKNVAQJg/Io1nA3xLgn/DWLE/W1QOB2+/Js3ACi40= +go.opentelemetry.io/collector/config/configtelemetry v0.104.0 h1:eHv98XIhapZA8MgTiipvi+FDOXoFhCYOwyKReOt+E4E= +go.opentelemetry.io/collector/config/configtelemetry v0.104.0/go.mod h1:WxWKNVAQJg/Io1nA3xLgn/DWLE/W1QOB2+/Js3ACi40= go.opentelemetry.io/collector/config/configtls v0.102.1 h1:7fr+PU9BRg0HRc1Pn3WmDW/4WBHRjuo7o1CdG2vQKoA= go.opentelemetry.io/collector/config/configtls v0.102.1/go.mod h1:KHdrvo3cwosgDxclyiLWmtbovIwqvaIGeTXr3p5721A= go.opentelemetry.io/collector/config/internal v0.102.1 h1:HFsFD3xpHUuNHb8/UTz5crJw1cMHzsJQf/86sgD44hw= go.opentelemetry.io/collector/config/internal v0.102.1/go.mod h1:Vig3dfeJJnuRe1kBNpszBzPoj5eYnR51wXbeq36Zfpg= -go.opentelemetry.io/collector/confmap v0.102.1 h1:wZuH+d/P11Suz8wbp+xQCJ0BPE9m5pybtUe74c+rU7E= -go.opentelemetry.io/collector/confmap v0.102.1/go.mod h1:KgpS7UxH5rkd69CzAzlY2I1heH8Z7eNCZlHmwQBMxNg= +go.opentelemetry.io/collector/confmap v0.104.0 h1:d3yuwX+CHpoyCh0iMv3rqb/vwAekjSm4ZDL6UK1nZSA= +go.opentelemetry.io/collector/confmap v0.104.0/go.mod h1:F8Lue+tPPn2oldXcfqI75PPMJoyzgUsKVtM/uHZLA4w= go.opentelemetry.io/collector/confmap/converter/expandconverter v0.102.1 h1:s0RxnaABoRxtfvUeimZ0OOsF83wD/EK1tR2N5GZyst0= go.opentelemetry.io/collector/confmap/converter/expandconverter v0.102.1/go.mod h1:ZwSMlOSIzmrrSSVNoMPDr21SQx7E52bZFMQJSOZ+EhY= go.opentelemetry.io/collector/confmap/provider/envprovider v0.102.1 h1:4KLw0pTChIqDfw0ckZ411aQDw98pu2dDOqgBHXfJm8M= @@ -2327,8 +2329,8 @@ go.opentelemetry.io/collector/confmap/provider/yamlprovider v0.102.1 h1:qmdaBIz0 go.opentelemetry.io/collector/confmap/provider/yamlprovider v0.102.1/go.mod h1:nAckG/FkzAaPuwtEN2Na2+ij+2hdTjtXUtFBnlUqpFk= go.opentelemetry.io/collector/connector v0.102.1 h1:7lEwXmhzqtyZwz2bBUHzwV/CZqA8bhPPVJOi0cm9+Fk= go.opentelemetry.io/collector/connector v0.102.1/go.mod h1:DRlDYJXsFx1FKKxkdM2Ja52/xe+0bgmy0hA+wgKRUVI= -go.opentelemetry.io/collector/consumer v0.102.1 h1:0CkgHhxwx4lI/m+hWjh607xyjooW5CObZ8hFQy5vvo0= -go.opentelemetry.io/collector/consumer v0.102.1/go.mod h1:HoXqmrRV13jLnP3/Gg3fYNdRkDPoO7UW58hKiLyFF60= +go.opentelemetry.io/collector/consumer v0.104.0 h1:Z1ZjapFp5mUcbkGEL96ljpqLIUMhRgQQpYKkDRtxy+4= +go.opentelemetry.io/collector/consumer v0.104.0/go.mod h1:60zcIb0W9GW0z9uJCv6NmjpSbCfBOeRUyrtEwqK6Hzo= go.opentelemetry.io/collector/exporter v0.102.1 h1:4VURYgBNJscxfMhZWitzcwA1cig5a6pH0xZSpdECDnM= go.opentelemetry.io/collector/exporter v0.102.1/go.mod h1:1pmNxvrvvbWDW6PiGObICdj0eOSGV4Fzwpm5QA1GU54= go.opentelemetry.io/collector/exporter/loggingexporter v0.102.1 h1:LblufdV22DxB5NZa66CGCQZjadYTVxT+O5NR9YjNQ9Y= @@ -2345,12 +2347,14 @@ go.opentelemetry.io/collector/extension/zpagesextension v0.99.0 h1:4lG8GKJLuc/3W go.opentelemetry.io/collector/extension/zpagesextension v0.99.0/go.mod h1:mQF+VTrjF0VQDbJM/MeWA6QhTunEU5R4unKi0+Vn+pE= go.opentelemetry.io/collector/filter v0.102.1 h1:qHVt97V3iCfAwzAzddbgWH9Xm5k2sGaU3hPRHB7uSwE= go.opentelemetry.io/collector/filter v0.102.1/go.mod h1:6vrr9XoD+fJekeTz5G01mCy6XqMBsARgbJruXcUnhQU= -go.opentelemetry.io/collector/pdata v1.9.0 h1:qyXe3HEVYYxerIYu0rzgo1Tx2d1Zs6iF+TCckbHLFOw= -go.opentelemetry.io/collector/pdata v1.9.0/go.mod h1:vk7LrfpyVpGZrRWcpjyy0DDZzL3SZiYMQxfap25551w= -go.opentelemetry.io/collector/pdata/testdata v0.102.1 h1:S3idZaJxy8M7mCC4PG4EegmtiSaOuh6wXWatKIui8xU= -go.opentelemetry.io/collector/pdata/testdata v0.102.1/go.mod h1:JEoSJTMgeTKyGxoMRy48RMYyhkA5vCCq/abJq9B6vXs= -go.opentelemetry.io/collector/processor v0.102.1 h1:79NWs7kTgmgxOIQacuZyDf+mYWuoJZS07SHwZT7sZ4Y= -go.opentelemetry.io/collector/processor v0.102.1/go.mod h1:sNM41tEHgv3YA/Dz9/6F8oCeObrqnKCGOMs7wS6Ldus= +go.opentelemetry.io/collector/pdata v1.11.0 h1:rzYyV1zfTQQz1DI9hCiaKyyaczqawN75XO9mdXmR/hE= +go.opentelemetry.io/collector/pdata v1.11.0/go.mod h1:IHxHsp+Jq/xfjORQMDJjSH6jvedOSTOyu3nbxqhWSYE= +go.opentelemetry.io/collector/pdata/pprofile v0.104.0 h1:MYOIHvPlKEJbWLiBKFQWGD0xd2u22xGVLt4jPbdxP4Y= +go.opentelemetry.io/collector/pdata/pprofile v0.104.0/go.mod h1:7WpyHk2wJZRx70CGkBio8klrYTTXASbyIhf+rH4FKnA= +go.opentelemetry.io/collector/pdata/testdata v0.104.0 h1:BKTZ7hIyAX5DMPecrXkVB2e86HwWtJyOlXn/5vSVXNw= +go.opentelemetry.io/collector/pdata/testdata v0.104.0/go.mod h1:3SnYKu8gLfxURJMWS/cFEUFs+jEKS6jvfqKXnOZsdkQ= +go.opentelemetry.io/collector/processor v0.104.0 h1:KSvMDu4DWmK1/k2z2rOzMtTvAa00jnTabtPEK9WOSYI= +go.opentelemetry.io/collector/processor v0.104.0/go.mod h1:qU2/xCCYdvVORkN6aq0H/WUWkvo505VGYg2eOwPvaTg= go.opentelemetry.io/collector/processor/batchprocessor v0.102.1 h1:s7TjD8k2d58x/Oj6P6PIm6R4zyBRdUPNbD9Zhiv0x0E= go.opentelemetry.io/collector/processor/batchprocessor v0.102.1/go.mod h1:RDgJIY8J6xstSncSDzvzkOSFoNGK8RqeuHfdoWxu6a8= go.opentelemetry.io/collector/processor/memorylimiterprocessor v0.102.1 h1:aUDHYAMJFQR/NRTqerzJjHk4bbDLwReQnMQmMMyuYLo= @@ -3054,8 +3058,8 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg= -google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= diff --git a/internal/component/all/all.go b/internal/component/all/all.go index 1039d59904..fbe09721e6 100644 --- a/internal/component/all/all.go +++ b/internal/component/all/all.go @@ -81,6 +81,7 @@ import ( _ "github.com/grafana/alloy/internal/component/otelcol/processor/deltatocumulative" // Import otelcol.processor.deltatocumulative _ "github.com/grafana/alloy/internal/component/otelcol/processor/discovery" // Import otelcol.processor.discovery _ "github.com/grafana/alloy/internal/component/otelcol/processor/filter" // Import otelcol.processor.filter + _ "github.com/grafana/alloy/internal/component/otelcol/processor/groupbyattrs" // Import otelcol.processor.groupbyattrs _ "github.com/grafana/alloy/internal/component/otelcol/processor/k8sattributes" // Import otelcol.processor.k8sattributes _ "github.com/grafana/alloy/internal/component/otelcol/processor/memorylimiter" // Import otelcol.processor.memory_limiter _ "github.com/grafana/alloy/internal/component/otelcol/processor/probabilistic_sampler" // Import otelcol.processor.probabilistic_sampler diff --git a/internal/component/otelcol/processor/groupbyattrs/groupbyattrs.go b/internal/component/otelcol/processor/groupbyattrs/groupbyattrs.go new file mode 100644 index 0000000000..af57d77827 --- /dev/null +++ b/internal/component/otelcol/processor/groupbyattrs/groupbyattrs.go @@ -0,0 +1,92 @@ +// Package attributes provides an otelcol.processor.groupbyattrs component. +package groupbyattrs + +import ( + "github.com/grafana/alloy/internal/component" + "github.com/grafana/alloy/internal/component/otelcol" + otelcolCfg "github.com/grafana/alloy/internal/component/otelcol/config" + "github.com/grafana/alloy/internal/component/otelcol/processor" + "github.com/grafana/alloy/internal/featuregate" + + "github.com/open-telemetry/opentelemetry-collector-contrib/processor/groupbyattrsprocessor" + "github.com/open-telemetry/opentelemetry-collector-contrib/processor/k8sattributesprocessor" + otelcomponent "go.opentelemetry.io/collector/component" + otelextension "go.opentelemetry.io/collector/extension" +) + +func init() { + component.Register(component.Registration{ + Name: "otelcol.processor.groupbyattrs", + Stability: featuregate.StabilityExperimental, + Exports: otelcol.ConsumerExports{}, + Args: Arguments{}, + + Build: func(opts component.Options, args component.Arguments) (component.Component, error) { + fact := k8sattributesprocessor.NewFactory() + return processor.New(opts, fact, args.(Arguments)) + }, + }) +} + +type Arguments struct { + + // Keys is a list of attributes to group metrics by. + Keys []string `alloy:"keys,attr,optional"` + + // Output configures where to send processed data. Required. + Output *otelcol.ConsumerArguments `alloy:"output,block"` + + // DebugMetrics configures component internal metrics. Optional. + DebugMetrics otelcolCfg.DebugMetricsArguments `alloy:"debug_metrics,block,optional"` +} + +var ( + _ processor.Arguments = Arguments{} + + // DefaultArguments holds default settings for Arguments. + DefaultArguments = Arguments{ + Keys: []string{}, + } +) + +// SetToDefault implements syntax.Defaulter. +func (args *Arguments) SetToDefault() { + *args = DefaultArguments + args.DebugMetrics.SetToDefault() +} + +// Validate implements syntax.Validator. +func (args *Arguments) Validate() error { + return nil +} + +// Convert implements processor.Arguments. +func (args Arguments) Convert() (otelcomponent.Config, error) { + return args.convertImpl() +} + +func (args Arguments) convertImpl() (*groupbyattrsprocessor.Config, error) { + return &groupbyattrsprocessor.Config{ + GroupByKeys: args.Keys, + }, nil +} + +// Extensions implements processor.Arguments. +func (args Arguments) Extensions() map[otelcomponent.ID]otelextension.Extension { + return nil +} + +// Exporters implements processor.Arguments. +func (args Arguments) Exporters() map[otelcomponent.DataType]map[otelcomponent.ID]otelcomponent.Component { + return nil +} + +// NextConsumers implements processor.Arguments. +func (args Arguments) NextConsumers() *otelcol.ConsumerArguments { + return args.Output +} + +// DebugMetricsConfig implements processor.Arguments. +func (args Arguments) DebugMetricsConfig() otelcolCfg.DebugMetricsArguments { + return args.DebugMetrics +} diff --git a/internal/component/otelcol/processor/groupbyattrs/groupbyattrs_test.go b/internal/component/otelcol/processor/groupbyattrs/groupbyattrs_test.go new file mode 100644 index 0000000000..fc49b247d2 --- /dev/null +++ b/internal/component/otelcol/processor/groupbyattrs/groupbyattrs_test.go @@ -0,0 +1,80 @@ +package groupbyattrs_test + +import ( + "testing" + + "github.com/grafana/alloy/internal/component/otelcol/processor/groupbyattrs" + "github.com/grafana/alloy/syntax" + + "github.com/mitchellh/mapstructure" + "github.com/open-telemetry/opentelemetry-collector-contrib/processor/groupbyattrsprocessor" + "github.com/stretchr/testify/require" +) + +func TestArguments_UnmarshalAlloy(t *testing.T) { + tests := []struct { + testName string + cfg string + expected map[string]interface{} + errMsg string + }{ + { + testName: "Default", + cfg: ` + output {} + `, + expected: map[string]interface{}{ + "keys": []string{}, + }, + }, + { + testName: "SingleKey", + cfg: ` + keys = ["key1"] + output {} + `, + expected: map[string]interface{}{ + "keys": []string{ + "key1", + }, + }, + }, + { + testName: "MultipleKeys", + cfg: ` + keys = ["key1", "key2"] + output {} + `, + expected: map[string]interface{}{ + "keys": []string{ + "key1", + "key2", + }, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.testName, func(t *testing.T) { + var args groupbyattrs.Arguments + err := syntax.Unmarshal([]byte(tt.cfg), &args) + if tt.errMsg != "" { + require.Error(t, err) + require.Contains(t, err.Error(), tt.errMsg) + return + } + require.NoError(t, err) + + actualPtr, err := args.Convert() + require.NoError(t, err) + + actual := actualPtr.(*groupbyattrsprocessor.Config) + + var expectedCfg groupbyattrsprocessor.Config + err = mapstructure.Decode(tt.expected, &expectedCfg) + require.NoError(t, err) + + require.Equal(t, expectedCfg, *actual) + }) + } +} From 7f0ca5ffac87806ba21a073cf5f1bd6618bb0322 Mon Sep 17 00:00:00 2001 From: kehindesalaam Date: Tue, 23 Jul 2024 09:33:10 +0200 Subject: [PATCH 02/16] Make changes from review - Use Generally avaialble stability - Import Correct package --- .../otelcol/processor/groupbyattrs/groupbyattrs.go | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/internal/component/otelcol/processor/groupbyattrs/groupbyattrs.go b/internal/component/otelcol/processor/groupbyattrs/groupbyattrs.go index af57d77827..2206a19213 100644 --- a/internal/component/otelcol/processor/groupbyattrs/groupbyattrs.go +++ b/internal/component/otelcol/processor/groupbyattrs/groupbyattrs.go @@ -9,7 +9,6 @@ import ( "github.com/grafana/alloy/internal/featuregate" "github.com/open-telemetry/opentelemetry-collector-contrib/processor/groupbyattrsprocessor" - "github.com/open-telemetry/opentelemetry-collector-contrib/processor/k8sattributesprocessor" otelcomponent "go.opentelemetry.io/collector/component" otelextension "go.opentelemetry.io/collector/extension" ) @@ -17,12 +16,12 @@ import ( func init() { component.Register(component.Registration{ Name: "otelcol.processor.groupbyattrs", - Stability: featuregate.StabilityExperimental, + Stability: featuregate.StabilityGenerallyAvailable, Exports: otelcol.ConsumerExports{}, Args: Arguments{}, Build: func(opts component.Options, args component.Arguments) (component.Component, error) { - fact := k8sattributesprocessor.NewFactory() + fact := groupbyattrsprocessor.NewFactory() return processor.New(opts, fact, args.(Arguments)) }, }) @@ -62,10 +61,6 @@ func (args *Arguments) Validate() error { // Convert implements processor.Arguments. func (args Arguments) Convert() (otelcomponent.Config, error) { - return args.convertImpl() -} - -func (args Arguments) convertImpl() (*groupbyattrsprocessor.Config, error) { return &groupbyattrsprocessor.Config{ GroupByKeys: args.Keys, }, nil From 280f1aa80bf97e16de7b36713f3f57ea1ae9cb96 Mon Sep 17 00:00:00 2001 From: kehindesalaam Date: Tue, 23 Jul 2024 09:34:33 +0200 Subject: [PATCH 03/16] Import Correct package version --- go.mod | 3 +-- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index 1b12fe1eda..6b7aa53cb0 100644 --- a/go.mod +++ b/go.mod @@ -121,6 +121,7 @@ require ( github.com/open-telemetry/opentelemetry-collector-contrib/processor/attributesprocessor v0.102.0 github.com/open-telemetry/opentelemetry-collector-contrib/processor/deltatocumulativeprocessor v0.102.0 github.com/open-telemetry/opentelemetry-collector-contrib/processor/filterprocessor v0.102.0 + github.com/open-telemetry/opentelemetry-collector-contrib/processor/groupbyattrsprocessor v0.102.0 github.com/open-telemetry/opentelemetry-collector-contrib/processor/k8sattributesprocessor v0.102.0 github.com/open-telemetry/opentelemetry-collector-contrib/processor/probabilisticsamplerprocessor v0.102.0 github.com/open-telemetry/opentelemetry-collector-contrib/processor/resourcedetectionprocessor v0.102.0 @@ -707,8 +708,6 @@ require ( sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect ) -require github.com/open-telemetry/opentelemetry-collector-contrib/processor/groupbyattrsprocessor v0.104.0 - // NOTE: replace directives below must always be *temporary*. // // Adding a replace directive to change a module to a fork of a module will diff --git a/go.sum b/go.sum index 1adf426450..5618e63d05 100644 --- a/go.sum +++ b/go.sum @@ -1765,8 +1765,8 @@ github.com/open-telemetry/opentelemetry-collector-contrib/processor/deltatocumul github.com/open-telemetry/opentelemetry-collector-contrib/processor/deltatocumulativeprocessor v0.102.0/go.mod h1:4mjsDJoPFf7MDE6bQpDEr25D/U2HTaD4OZKwo7gt8t8= github.com/open-telemetry/opentelemetry-collector-contrib/processor/filterprocessor v0.102.0 h1:DaEYlVCn58GtkyYVK0IT/ZMjRFJ+BfmR0p9I0Eq42aQ= github.com/open-telemetry/opentelemetry-collector-contrib/processor/filterprocessor v0.102.0/go.mod h1:u9x08rUCWdgI8Nle5XOMTCmxd0K26KTZvMMA5H8Xjyg= -github.com/open-telemetry/opentelemetry-collector-contrib/processor/groupbyattrsprocessor v0.104.0 h1:Dust79ten+1/q92AgOLdnBnQDCsVT5rjl0/eSKFLdYo= -github.com/open-telemetry/opentelemetry-collector-contrib/processor/groupbyattrsprocessor v0.104.0/go.mod h1:evkdeLcXutvRAz202BdwNk2g3+Y/mxVqX7hhDIJId5k= +github.com/open-telemetry/opentelemetry-collector-contrib/processor/groupbyattrsprocessor v0.102.0 h1:huh7V8uqMakQGdnbOqTSZihfoDeOIbNHfFt62HMsk5k= +github.com/open-telemetry/opentelemetry-collector-contrib/processor/groupbyattrsprocessor v0.102.0/go.mod h1:IIKgEx+D91XNJYN33/wXzGullskvjJzvzKHIP3/+zDQ= github.com/open-telemetry/opentelemetry-collector-contrib/processor/k8sattributesprocessor v0.102.0 h1:mkRDKVWXfG1gTxwg69ttJoGmXOKNHAGsGms06DrwTlQ= github.com/open-telemetry/opentelemetry-collector-contrib/processor/k8sattributesprocessor v0.102.0/go.mod h1:5F6hpHujLkLuEYmbbUXel2i3mBpwRJHmy8KTY3cbOVg= github.com/open-telemetry/opentelemetry-collector-contrib/processor/probabilisticsamplerprocessor v0.102.0 h1:ErBYnmZUSyPQjHPlyAeUOtQDax0tH2Ax/zOuklZp5Y8= From 644c98f433c681a7291fd03145975ab91e5080dc Mon Sep 17 00:00:00 2001 From: kehindesalaam Date: Tue, 23 Jul 2024 09:35:57 +0200 Subject: [PATCH 04/16] Add groupbyattrs converter --- .../converter_groupbyattrsprocessor.go | 63 +++++++++++++++++++ .../testdata/groupbyattrs.alloy | 27 ++++++++ .../otelcolconvert/testdata/groupbyattrs.yaml | 30 +++++++++ 3 files changed, 120 insertions(+) create mode 100644 internal/converter/internal/otelcolconvert/converter_groupbyattrsprocessor.go create mode 100644 internal/converter/internal/otelcolconvert/testdata/groupbyattrs.alloy create mode 100644 internal/converter/internal/otelcolconvert/testdata/groupbyattrs.yaml diff --git a/internal/converter/internal/otelcolconvert/converter_groupbyattrsprocessor.go b/internal/converter/internal/otelcolconvert/converter_groupbyattrsprocessor.go new file mode 100644 index 0000000000..49f47de12d --- /dev/null +++ b/internal/converter/internal/otelcolconvert/converter_groupbyattrsprocessor.go @@ -0,0 +1,63 @@ +package otelcolconvert + +import ( + "fmt" + + "github.com/grafana/alloy/internal/component/otelcol" + "github.com/grafana/alloy/internal/component/otelcol/processor/groupbyattrs" + "github.com/grafana/alloy/internal/converter/diag" + "github.com/grafana/alloy/internal/converter/internal/common" + + "github.com/open-telemetry/opentelemetry-collector-contrib/processor/groupbyattrsprocessor" + "go.opentelemetry.io/collector/component" +) + +func init() { + converters = append(converters, groupByAttrsConverter{}) +} + +type groupByAttrsConverter struct{} + +func (groupByAttrsConverter) Factory() component.Factory { + return groupbyattrsprocessor.NewFactory() +} + +func (groupByAttrsConverter) InputComponentName() string { + return "otelcol.processor.groupbyattrs" +} + +func (groupByAttrsConverter) ConvertAndAppend(state *State, id component.InstanceID, cfg component.Config) diag.Diagnostics { + var diags diag.Diagnostics + + label := state.AlloyComponentLabel() + + args := toGroupByAttrsProcessor(state, id, cfg.(*groupbyattrsprocessor.Config)) + block := common.NewBlockWithOverride([]string{"otelcol", "processor", "groupbyattrs"}, label, args) + + diags.Add( + diag.SeverityLevelInfo, + fmt.Sprintf("Converted %s into %s", StringifyInstanceID(id), StringifyBlock(block)), + ) + + state.Body().AppendBlock(block) + return diags +} + +func toGroupByAttrsProcessor(state *State, id component.InstanceID, cfg *groupbyattrsprocessor.Config) *groupbyattrs.Arguments { + var ( + nextMetrics = state.Next(id, component.DataTypeMetrics) + nextLogs = state.Next(id, component.DataTypeLogs) + nextTraces = state.Next(id, component.DataTypeTraces) + ) + + return &groupbyattrs.Arguments{ + Keys: cfg.GroupByKeys, + Output: &otelcol.ConsumerArguments{ + Metrics: ToTokenizedConsumers(nextMetrics), + Logs: ToTokenizedConsumers(nextLogs), + Traces: ToTokenizedConsumers(nextTraces), + }, + + DebugMetrics: common.DefaultValue[groupbyattrs.Arguments]().DebugMetrics, + } +} diff --git a/internal/converter/internal/otelcolconvert/testdata/groupbyattrs.alloy b/internal/converter/internal/otelcolconvert/testdata/groupbyattrs.alloy new file mode 100644 index 0000000000..13551b88ff --- /dev/null +++ b/internal/converter/internal/otelcolconvert/testdata/groupbyattrs.alloy @@ -0,0 +1,27 @@ +otelcol.receiver.otlp "default" { + grpc { } + + http { } + + output { + metrics = [otelcol.processor.groupbyattrs.default.input] + logs = [otelcol.processor.groupbyattrs.default.input] + traces = [otelcol.processor.groupbyattrs.default.input] + } +} + +otelcol.processor.groupbyattrs "default" { + keys = ["k8s.deployment.name", "k8s.namespace.name"] + + output { + metrics = [otelcol.exporter.otlp.default.input] + logs = [otelcol.exporter.otlp.default.input] + traces = [otelcol.exporter.otlp.default.input] + } +} + +otelcol.exporter.otlp "default" { + client { + endpoint = "database:4317" + } +} diff --git a/internal/converter/internal/otelcolconvert/testdata/groupbyattrs.yaml b/internal/converter/internal/otelcolconvert/testdata/groupbyattrs.yaml new file mode 100644 index 0000000000..cea627d286 --- /dev/null +++ b/internal/converter/internal/otelcolconvert/testdata/groupbyattrs.yaml @@ -0,0 +1,30 @@ +receivers: + otlp: + protocols: + grpc: + http: + +exporters: + otlp: + endpoint: database:4317 + +processors: + groupbyattrs: + keys: + - k8s.namespace.name + - k8s.deployment.name +service: + pipelines: + metrics: + receivers: [otlp] + processors: [groupbyattrs] + exporters: [otlp] + logs: + receivers: [otlp] + processors: [groupbyattrs] + exporters: [otlp] + traces: + receivers: [otlp] + processors: [groupbyattrs] + exporters: [otlp] + From f70fdfd63c01c54f0217819b6945b83d8eb9547a Mon Sep 17 00:00:00 2001 From: Salaam Kehinde Date: Wed, 31 Jul 2024 22:01:00 +0200 Subject: [PATCH 05/16] Apply suggestions from code review Co-authored-by: Clayton Cornell <131809008+clayton-cornell@users.noreply.github.com> --- .../otelcol/otelcol.processor.groupbyattrs.md | 22 ++++++++++++------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/docs/sources/reference/components/otelcol/otelcol.processor.groupbyattrs.md b/docs/sources/reference/components/otelcol/otelcol.processor.groupbyattrs.md index 243cb64fa1..58b2bc84bb 100644 --- a/docs/sources/reference/components/otelcol/otelcol.processor.groupbyattrs.md +++ b/docs/sources/reference/components/otelcol/otelcol.processor.groupbyattrs.md @@ -1,11 +1,17 @@ +--- +canonical: https://grafana.com/docs/alloy/latest/reference/components/otelcol/otelcol.processor.groupbyattrs/ +description: Learn about otelcol.processor.groupbyattrs +title: otelcol.processor.groupbyattrs +--- + # otelcol.processor.groupbyattrs `otelcol.processor.groupbyattrs` accepts telemetry data from other `otelcol` -components and reassociates spans, log records and metric datapoints to a Resource that matches with the specified attributes. It groups telemetry data by specified attributes. +components and reassociates spans, log records, and metric datapoints to a resource that matches the specified attributes. It groups telemetry data by specified attributes. {{% admonition type="note" %}} `otelcol.processor.groupbyattrs` is a wrapper over the upstream OpenTelemetry -Collector `groupbyattrs` processor. If necessary, bug reports or feature requests +Collector `groupbyattrs` processor. If necessary, bug reports or feature requests will be redirected to the upstream repository. {{% /admonition %}} @@ -41,7 +47,7 @@ The following fields are exported and can be referenced by other components: Name | Type | Description --------|--------------------|----------------------------------------------------------------- -`input` | `otelcol.Consumer` | A value that other components can use to send telemetry data to. +`input` | `otelcol.Consumer` | Accepts `otelcol.Consumer` data for metrics, logs, or traces. `input` accepts `otelcol.Consumer` data for any telemetry signal (metrics, logs, or traces). @@ -81,8 +87,8 @@ otelcol.processor.groupbyattrs "default" { ``` ## Notes -- The data points with different data types would not be merged under the same metric (e.g a gauge and sum metric would not be merged). -- The data points without the specified keys would remain under their respective resources. -- New resources would inherit the attributes of the original resource and the specified attributes in the keys array. -- The grouping attributes in the keys array would be removed from the output metrics. -- If the keys array is empty, the processor would perform compaction and re-associate all spans with matching Resource and InstrumentationLibrary. +- The data points with different data types aren't merged under the same metric. For example, a gauge and sum metric would not be merged. +- The data points without the specified keys remain under their respective resources. +- New resources inherit the attributes of the original resource and the specified attributes in the keys array. +- The grouping attributes in the keys array are removed from the output metrics. +- If the keys array is empty, the processor performs compaction and reassociates all spans with matching Resource and InstrumentationLibrary. From 930555bfcd35a6f651aa0a17e19441dddd61684a Mon Sep 17 00:00:00 2001 From: kehindesalaam Date: Wed, 31 Jul 2024 23:06:59 +0200 Subject: [PATCH 06/16] Update documentation --- .../otelcol/otelcol.processor.groupbyattrs.md | 33 ++++++++++++++----- 1 file changed, 25 insertions(+), 8 deletions(-) diff --git a/docs/sources/reference/components/otelcol/otelcol.processor.groupbyattrs.md b/docs/sources/reference/components/otelcol/otelcol.processor.groupbyattrs.md index 58b2bc84bb..eba148b435 100644 --- a/docs/sources/reference/components/otelcol/otelcol.processor.groupbyattrs.md +++ b/docs/sources/reference/components/otelcol/otelcol.processor.groupbyattrs.md @@ -15,7 +15,7 @@ Collector `groupbyattrs` processor. If necessary, bug reports or feature request will be redirected to the upstream repository. {{% /admonition %}} -It is recommended to use the groupbyattrs processor together with batch processor, as a consecutive step, as this will reduce the fragmentation of data (by grouping records together under matching Resource/Instrumentation Library) +It is recommended to use the groupbyattrs processor together with [otelcol.processor.batch][], as a consecutive step, as this will reduce the fragmentation of data (by grouping records together under matching Resource/Instrumentation Library) You can specify multiple `otelcol.processor.groupbyattrs` components by giving them different labels. @@ -36,18 +36,34 @@ otelcol.processor.groupbyattrs "LABEL" { The following arguments are supported: -Name | Type | Description | Default | Required ----- |-----------------|---------------------------------------------------------------------------------------|------------| -------- -`keys` | `array(string)` | Keys that will be used to group the spans, log records or metric data points together | | no +| Name | Type | Description | Default | Required | +|-----------------|-------------------|---------------------------------------------------------------------------------------|---------|----------| +| `keys` | `array(string)` | Keys that will be used to group the spans, log records or metric data points together | | no | +| `output` | [output][] | Configures where to send received telemetry data. | yes | | +| `debug_metrics` | [debug_metrics][] | Configures the metrics that this component generates to monitor its state. | no | | +[output]: #output-block +[debug_metrics]: #debug_metrics-block + +### keys +`keys` is a string array that is used for grouping the data. If it is empty, the processor performs compaction and reassociates all spans with matching Resource and InstrumentationLibrary. + + +### output block + +{{< docs/shared lookup="reference/components/output-block.md" source="alloy" version="" >}} + +### debug_metrics block + +{{< docs/shared lookup="reference/components/otelcol-debug-metrics-block.md" source="alloy" version="" >}} ## Exported fields The following fields are exported and can be referenced by other components: -Name | Type | Description ---------|--------------------|----------------------------------------------------------------- -`input` | `otelcol.Consumer` | Accepts `otelcol.Consumer` data for metrics, logs, or traces. +| Name | Type | Description | +|---------|--------------------|---------------------------------------------------------------| +| `input` | `otelcol.Consumer` | Accepts `otelcol.Consumer` data for metrics, logs, or traces. | `input` accepts `otelcol.Consumer` data for any telemetry signal (metrics, logs, or traces). @@ -91,4 +107,5 @@ otelcol.processor.groupbyattrs "default" { - The data points without the specified keys remain under their respective resources. - New resources inherit the attributes of the original resource and the specified attributes in the keys array. - The grouping attributes in the keys array are removed from the output metrics. -- If the keys array is empty, the processor performs compaction and reassociates all spans with matching Resource and InstrumentationLibrary. + +[otelcol.processor.batch]: https://grafana.com/docs/alloy/latest/reference/components/otelcol/otelcol.processor.batch/ From eaf85ba74f5a51ad8b89969becf796f4561627b8 Mon Sep 17 00:00:00 2001 From: kehindesalaam Date: Thu, 1 Aug 2024 09:41:29 +0200 Subject: [PATCH 07/16] fix order of keys in the test and update go.sum --- go.sum | 2 ++ .../internal/otelcolconvert/testdata/groupbyattrs.alloy | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.sum b/go.sum index 7909740041..7acd84f1f8 100644 --- a/go.sum +++ b/go.sum @@ -1770,6 +1770,8 @@ github.com/open-telemetry/opentelemetry-collector-contrib/processor/deltatocumul github.com/open-telemetry/opentelemetry-collector-contrib/processor/deltatocumulativeprocessor v0.105.0/go.mod h1:PKox+dLnO2bWc1qUN6WZnyHPV0MpWZ10arqGV5v69kI= github.com/open-telemetry/opentelemetry-collector-contrib/processor/filterprocessor v0.105.0 h1:oRa+acTM4f5rjTT3+hjOVM1LYrlwrm6CSNG4o/RIqcA= github.com/open-telemetry/opentelemetry-collector-contrib/processor/filterprocessor v0.105.0/go.mod h1:66cZFd4X8vQBTmvm1hPHxrSNHS474iUEsAVbYk9xQBU= +github.com/open-telemetry/opentelemetry-collector-contrib/processor/groupbyattrsprocessor v0.105.0 h1:OYsGaSC9G7pAVYKTd1+D0f7HTHcxuQfoEHyQy+a1NKk= +github.com/open-telemetry/opentelemetry-collector-contrib/processor/groupbyattrsprocessor v0.105.0/go.mod h1:WCesGEakYveZYZH4o3cUTLt3UB7JxE+yDiiphRHoJoc= github.com/open-telemetry/opentelemetry-collector-contrib/processor/k8sattributesprocessor v0.105.0 h1:ScIwuYg6l79Ta+deOyZIADXrBlXSdeAZ7sp3MXhm7JY= github.com/open-telemetry/opentelemetry-collector-contrib/processor/k8sattributesprocessor v0.105.0/go.mod h1:pranRmnWRkzDsn9a16BzSqX6HJ6XjjVVFmMhyZPEzt0= github.com/open-telemetry/opentelemetry-collector-contrib/processor/probabilisticsamplerprocessor v0.105.0 h1:mFAlBmDFELQJS8uj1M8csB/vQqjpq6W9/9k9izh9Hr4= diff --git a/internal/converter/internal/otelcolconvert/testdata/groupbyattrs.alloy b/internal/converter/internal/otelcolconvert/testdata/groupbyattrs.alloy index 13551b88ff..c1d2932849 100644 --- a/internal/converter/internal/otelcolconvert/testdata/groupbyattrs.alloy +++ b/internal/converter/internal/otelcolconvert/testdata/groupbyattrs.alloy @@ -11,7 +11,7 @@ otelcol.receiver.otlp "default" { } otelcol.processor.groupbyattrs "default" { - keys = ["k8s.deployment.name", "k8s.namespace.name"] + keys = ["k8s.namespace.name", "k8s.deployment.name"] output { metrics = [otelcol.exporter.otlp.default.input] From 8947ecefcfa8ab3146eb2fe53057046679945f04 Mon Sep 17 00:00:00 2001 From: Salaam Kehinde Date: Thu, 1 Aug 2024 19:48:04 +0200 Subject: [PATCH 08/16] Apply suggestions from code review Co-authored-by: Clayton Cornell <131809008+clayton-cornell@users.noreply.github.com> --- .../components/otelcol/otelcol.processor.groupbyattrs.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/sources/reference/components/otelcol/otelcol.processor.groupbyattrs.md b/docs/sources/reference/components/otelcol/otelcol.processor.groupbyattrs.md index eba148b435..f3aa263d96 100644 --- a/docs/sources/reference/components/otelcol/otelcol.processor.groupbyattrs.md +++ b/docs/sources/reference/components/otelcol/otelcol.processor.groupbyattrs.md @@ -15,7 +15,7 @@ Collector `groupbyattrs` processor. If necessary, bug reports or feature request will be redirected to the upstream repository. {{% /admonition %}} -It is recommended to use the groupbyattrs processor together with [otelcol.processor.batch][], as a consecutive step, as this will reduce the fragmentation of data (by grouping records together under matching Resource/Instrumentation Library) +We recommend you use the groupbyattrs processor together with [otelcol.processor.batch][], as a consecutive step. This will reduce the fragmentation of data by grouping records together under the matching Resource/Instrumentation Library. You can specify multiple `otelcol.processor.groupbyattrs` components by giving them different labels. @@ -38,7 +38,7 @@ The following arguments are supported: | Name | Type | Description | Default | Required | |-----------------|-------------------|---------------------------------------------------------------------------------------|---------|----------| -| `keys` | `array(string)` | Keys that will be used to group the spans, log records or metric data points together | | no | +| `keys` | `array(string)` | Keys that will be used to group the spans, log records, or metric data points together. | | no | | `output` | [output][] | Configures where to send received telemetry data. | yes | | | `debug_metrics` | [debug_metrics][] | Configures the metrics that this component generates to monitor its state. | no | | @@ -75,12 +75,12 @@ configuration. ## Debug information -`otelcol.processor.groupbyattrs` does not expose any component-specific debug +`otelcol.processor.groupbyattrs` doesn't expose any component-specific debug information. ## Debug metrics -`otelcol.processor.groupbyattrs` does not expose any component-specific debug metrics. +`otelcol.processor.groupbyattrs` doesn't expose any component-specific debug metrics. ## Examples From c7edec460e1e5eb3dc36b519a62f7c66acbba029 Mon Sep 17 00:00:00 2001 From: Salaam Kehinde Date: Thu, 15 Aug 2024 21:53:08 +0200 Subject: [PATCH 09/16] Update internal/converter/internal/otelcolconvert/testdata/groupbyattrs.alloy Co-authored-by: Paulin Todev --- .../internal/otelcolconvert/testdata/groupbyattrs.alloy | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/internal/converter/internal/otelcolconvert/testdata/groupbyattrs.alloy b/internal/converter/internal/otelcolconvert/testdata/groupbyattrs.alloy index c1d2932849..ebf910d335 100644 --- a/internal/converter/internal/otelcolconvert/testdata/groupbyattrs.alloy +++ b/internal/converter/internal/otelcolconvert/testdata/groupbyattrs.alloy @@ -1,7 +1,11 @@ otelcol.receiver.otlp "default" { - grpc { } + grpc { + endpoint = "localhost:4317" + } - http { } + http { + endpoint = "localhost:4318" + } output { metrics = [otelcol.processor.groupbyattrs.default.input] @@ -25,3 +29,4 @@ otelcol.exporter.otlp "default" { endpoint = "database:4317" } } + From 02a40455aefda5fbdf0133884a0db0a4a6ed4e1e Mon Sep 17 00:00:00 2001 From: kehindesalaam Date: Thu, 15 Aug 2024 22:09:32 +0200 Subject: [PATCH 10/16] Make changes from second review --- .../sources/reference/compatibility/_index.md | 2 + .../otelcol/otelcol.processor.groupbyattrs.md | 100 ++++++++++++++---- .../processor/groupbyattrs/groupbyattrs.go | 10 +- .../testdata/groupbyattrs.alloy | 8 +- 4 files changed, 93 insertions(+), 27 deletions(-) diff --git a/docs/sources/reference/compatibility/_index.md b/docs/sources/reference/compatibility/_index.md index 0da83a17a2..1a56f2799d 100644 --- a/docs/sources/reference/compatibility/_index.md +++ b/docs/sources/reference/compatibility/_index.md @@ -300,6 +300,7 @@ The following components, grouped by namespace, _export_ OpenTelemetry `otelcol. - [otelcol.processor.deltatocumulative](../components/otelcol/otelcol.processor.deltatocumulative) - [otelcol.processor.discovery](../components/otelcol/otelcol.processor.discovery) - [otelcol.processor.filter](../components/otelcol/otelcol.processor.filter) +- [otelcol.processor.groupbyattrs](../components/otelcol/otelcol.processor.groupbyattrs) - [otelcol.processor.k8sattributes](../components/otelcol/otelcol.processor.k8sattributes) - [otelcol.processor.memory_limiter](../components/otelcol/otelcol.processor.memory_limiter) - [otelcol.processor.probabilistic_sampler](../components/otelcol/otelcol.processor.probabilistic_sampler) @@ -336,6 +337,7 @@ The following components, grouped by namespace, _consume_ OpenTelemetry `otelcol - [otelcol.processor.deltatocumulative](../components/otelcol/otelcol.processor.deltatocumulative) - [otelcol.processor.discovery](../components/otelcol/otelcol.processor.discovery) - [otelcol.processor.filter](../components/otelcol/otelcol.processor.filter) +- [otelcol.processor.groupbyattrs](../components/otelcol/otelcol.processor.groupbyattrs) - [otelcol.processor.k8sattributes](../components/otelcol/otelcol.processor.k8sattributes) - [otelcol.processor.memory_limiter](../components/otelcol/otelcol.processor.memory_limiter) - [otelcol.processor.probabilistic_sampler](../components/otelcol/otelcol.processor.probabilistic_sampler) diff --git a/docs/sources/reference/components/otelcol/otelcol.processor.groupbyattrs.md b/docs/sources/reference/components/otelcol/otelcol.processor.groupbyattrs.md index eba148b435..4be331c118 100644 --- a/docs/sources/reference/components/otelcol/otelcol.processor.groupbyattrs.md +++ b/docs/sources/reference/components/otelcol/otelcol.processor.groupbyattrs.md @@ -6,8 +6,8 @@ title: otelcol.processor.groupbyattrs # otelcol.processor.groupbyattrs -`otelcol.processor.groupbyattrs` accepts telemetry data from other `otelcol` -components and reassociates spans, log records, and metric datapoints to a resource that matches the specified attributes. It groups telemetry data by specified attributes. +`otelcol.processor.groupbyattrs` accepts spans, metrics, and traces from other `otelcol` +components and groups them under the same resource. {{% admonition type="note" %}} `otelcol.processor.groupbyattrs` is a wrapper over the upstream OpenTelemetry @@ -22,7 +22,7 @@ different labels. ## Usage -```river +```alloy otelcol.processor.groupbyattrs "LABEL" { output { metrics = [...] @@ -38,14 +38,13 @@ The following arguments are supported: | Name | Type | Description | Default | Required | |-----------------|-------------------|---------------------------------------------------------------------------------------|---------|----------| -| `keys` | `array(string)` | Keys that will be used to group the spans, log records or metric data points together | | no | +| `keys` | `list(string)` | Keys that will be used to group the spans, log records or metric data points together | | no | | `output` | [output][] | Configures where to send received telemetry data. | yes | | | `debug_metrics` | [debug_metrics][] | Configures the metrics that this component generates to monitor its state. | no | | [output]: #output-block [debug_metrics]: #debug_metrics-block -### keys `keys` is a string array that is used for grouping the data. If it is empty, the processor performs compaction and reassociates all spans with matching Resource and InstrumentationLibrary. @@ -84,28 +83,93 @@ information. ## Examples -### Grouping metrics by an attribute +### Grouping metrics + +Consider the following metrics, all originally associated to the same Resource: + +``` +Resource {host.name="localhost",source="prom"} + Metric "gauge-1" (GAUGE) + DataPoint {host.name="host-A",id="eth0"} + DataPoint {host.name="host-A",id="eth0"} + DataPoint {host.name="host-B",id="eth0"} + Metric "gauge-1" (GAUGE) // Identical to previous Metric + DataPoint {host.name="host-A",id="eth0"} + DataPoint {host.name="host-A",id="eth0"} + DataPoint {host.name="host-B",id="eth0"} + Metric "mixed-type" (GAUGE) + DataPoint {host.name="host-A",id="eth0"} + DataPoint {host.name="host-A",id="eth0"} + DataPoint {host.name="host-B",id="eth0"} + Metric "mixed-type" (SUM) + DataPoint {host.name="host-A",id="eth0"} + DataPoint {host.name="host-A",id="eth0"} + Metric "dont-move" (Gauge) + DataPoint {id="eth0"} +``` -This example reassociates the metrics based on the value of the `host.name` attribute. +With the following configuration, the groupbyattrs will re-associate the metrics with either `host-A` or `host-B`, based on the value of the `host.name` attribute. ```alloy otelcol.processor.groupbyattrs "default" { - keys = [ - "host.name", - ] - + keys = [ "host.name" ] output { metrics = [otelcol.exporter.otlp.default.input] - logs = [otelcol.exporter.otlp.default.input] - traces = [otelcol.exporter.otlp.default.input] } } ``` -## Notes -- The data points with different data types aren't merged under the same metric. For example, a gauge and sum metric would not be merged. -- The data points without the specified keys remain under their respective resources. -- New resources inherit the attributes of the original resource and the specified attributes in the keys array. -- The grouping attributes in the keys array are removed from the output metrics. +The output of the processor will therefore be: + +``` +Resource {host.name="localhost",source="prom"} + Metric "dont-move" (Gauge) + DataPoint {id="eth0"} +Resource {host.name="host-A",source="prom"} + Metric "gauge-1" + DataPoint {id="eth0"} + DataPoint {id="eth0"} + DataPoint {id="eth0"} + DataPoint {id="eth0"} + Metric "mixed-type" (GAUGE) + DataPoint {id="eth0"} + DataPoint {id="eth0"} + Metric "mixed-type" (SUM) + DataPoint {id="eth0"} + DataPoint {id="eth0"} +Resource {host.name="host-B",source="prom"} + Metric "gauge-1" + DataPoint {id="eth0"} + DataPoint {id="eth0"} + Metric "mixed-type" (GAUGE) + DataPoint {id="eth0"} +``` + +This output demonstrates how `otelcol.processor.groupbyattrs` works in various situations: + +- The DataPoints for the `gauge-1` (GAUGE) metric were originally split under 2 Metric instances and have been merged in the output. +- The DataPoints of the `mixed-type` (GAUGE) and `mixed-type` (SUM) metrics have not been merged under the same Metric, because their DataType is different. +- The `dont-move` metric DataPoints don't have a `host.name` attribute and therefore remained under the original Resource. +- The new Resources inherited the attributes from the original Resource (`source="prom"`), plus the specified attributes from the processed metrics (`host.name="host-A"` or `host.name="host-B"`). +- The specified "grouping" attributes that are set on the new Resources are also removed from the metric DataPoints. +- While not shown in the above example, the processor also merges collections of records under matching InstrumentationLibrary. [otelcol.processor.batch]: https://grafana.com/docs/alloy/latest/reference/components/otelcol/otelcol.processor.batch/ + + +## Compatible components + +`otelcol.processor.groupbyattrs` can accept arguments from the following components: + +- Components that export [OpenTelemetry `otelcol.Consumer`](../../../compatibility/#opentelemetry-otelcolconsumer-exporters) + +`otelcol.processor.groupbyattrs` has exports that can be consumed by the following components: + +- Components that consume [OpenTelemetry `otelcol.Consumer`](../../../compatibility/#opentelemetry-otelcolconsumer-consumers) + +{{< admonition type="note" >}} +Connecting some components may not be sensible or components may require further configuration to make the connection work correctly. +Refer to the linked documentation for more details. +{{< /admonition >}} + + diff --git a/internal/component/otelcol/processor/groupbyattrs/groupbyattrs.go b/internal/component/otelcol/processor/groupbyattrs/groupbyattrs.go index 2206a19213..ade0a4779c 100644 --- a/internal/component/otelcol/processor/groupbyattrs/groupbyattrs.go +++ b/internal/component/otelcol/processor/groupbyattrs/groupbyattrs.go @@ -1,4 +1,3 @@ -// Package attributes provides an otelcol.processor.groupbyattrs component. package groupbyattrs import ( @@ -41,16 +40,13 @@ type Arguments struct { var ( _ processor.Arguments = Arguments{} - - // DefaultArguments holds default settings for Arguments. - DefaultArguments = Arguments{ - Keys: []string{}, - } ) // SetToDefault implements syntax.Defaulter. func (args *Arguments) SetToDefault() { - *args = DefaultArguments + *args = Arguments{ + Keys: []string{}, + } args.DebugMetrics.SetToDefault() } diff --git a/internal/converter/internal/otelcolconvert/testdata/groupbyattrs.alloy b/internal/converter/internal/otelcolconvert/testdata/groupbyattrs.alloy index c1d2932849..350ee4d663 100644 --- a/internal/converter/internal/otelcolconvert/testdata/groupbyattrs.alloy +++ b/internal/converter/internal/otelcolconvert/testdata/groupbyattrs.alloy @@ -1,7 +1,11 @@ otelcol.receiver.otlp "default" { - grpc { } + grpc { + endpoint = "localhost:4317" + } - http { } + http { + endpoint = "localhost:4318" + } output { metrics = [otelcol.processor.groupbyattrs.default.input] From 0cb3070af54ad9365d4c738b95999c5ef67c686a Mon Sep 17 00:00:00 2001 From: Salaam Kehinde Date: Mon, 19 Aug 2024 19:26:05 +0200 Subject: [PATCH 11/16] Update docs/sources/reference/components/otelcol/otelcol.processor.groupbyattrs.md Co-authored-by: Paulin Todev --- .../components/otelcol/otelcol.processor.groupbyattrs.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sources/reference/components/otelcol/otelcol.processor.groupbyattrs.md b/docs/sources/reference/components/otelcol/otelcol.processor.groupbyattrs.md index 86dc1abde9..584282b0ca 100644 --- a/docs/sources/reference/components/otelcol/otelcol.processor.groupbyattrs.md +++ b/docs/sources/reference/components/otelcol/otelcol.processor.groupbyattrs.md @@ -38,7 +38,7 @@ The following arguments are supported: | Name | Type | Description | Default | Required | |-----------------|-------------------|---------------------------------------------------------------------------------------|---------|----------| -| `keys` | `list(string)` | Keys that will be used to group the spans, log records or metric data points together | | no | +| `keys` | `list(string)` | Keys that will be used to group the spans, log records or metric data points together. | | no | | `output` | [output][] | Configures where to send received telemetry data. | yes | | | `debug_metrics` | [debug_metrics][] | Configures the metrics that this component generates to monitor its state. | no | | From 685fb20d7287f45fbfc195efcb339c50cf3dd2ae Mon Sep 17 00:00:00 2001 From: Salaam Kehinde Date: Mon, 19 Aug 2024 19:26:33 +0200 Subject: [PATCH 12/16] Update docs/sources/reference/components/otelcol/otelcol.processor.groupbyattrs.md Co-authored-by: Paulin Todev --- .../components/otelcol/otelcol.processor.groupbyattrs.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sources/reference/components/otelcol/otelcol.processor.groupbyattrs.md b/docs/sources/reference/components/otelcol/otelcol.processor.groupbyattrs.md index 584282b0ca..19d9a58c5c 100644 --- a/docs/sources/reference/components/otelcol/otelcol.processor.groupbyattrs.md +++ b/docs/sources/reference/components/otelcol/otelcol.processor.groupbyattrs.md @@ -154,7 +154,7 @@ This output demonstrates how `otelcol.processor.groupbyattrs` works in various s - The specified "grouping" attributes that are set on the new Resources are also removed from the metric DataPoints. - While not shown in the above example, the processor also merges collections of records under matching InstrumentationLibrary. -[otelcol.processor.batch]: https://grafana.com/docs/alloy/latest/reference/components/otelcol/otelcol.processor.batch/ +[otelcol.processor.batch]: ../otelcol.processor.batch/ ## Compatible components From a2ac0bcc6826b2acf48823596e5d7c693d47df02 Mon Sep 17 00:00:00 2001 From: kehindesalaam Date: Mon, 15 Jul 2024 21:45:53 +0200 Subject: [PATCH 13/16] Add `otelcol.processor.groupbyattrs` component --- CHANGELOG.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index febd3327eb..e24143d13e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,8 @@ Main (unreleased) - Updated Snowflake exporter with performance improvements for larger environments. Also added a new panel to track deleted tables to the Snowflake mixin. (@Caleb-Hurshman) +- Add a `otelcol.processor.groupbyattrs` component to reassociate collected metrics that match specified attributes + from opentelemetry. (@kehindesalaam) ### Bugfixes @@ -162,9 +164,6 @@ v1.3.0 - `otelcol.connector.spanmetrics`: Produce delta temporality span metrics with StartTimeUnixNano and TimeUnixNano values representing an uninterrupted series. https://github.com/open-telemetry/opentelemetry-collector-contrib/pull/31780 -- Add a `otelcol.processor.groupbyattrs` component to reassociate collected metrics that match specified attributes - from opentelemetry. (@kehindesalaam) - - Upgrade Beyla component v1.6.3 to v1.7.0 - Reporting application process metrics - New supported protocols: SQL, Redis, Kafka From 39cfc5c36366d042b0768debcc57a58850f6fe56 Mon Sep 17 00:00:00 2001 From: kehindesalaam Date: Mon, 19 Aug 2024 20:25:59 +0200 Subject: [PATCH 14/16] remove white space --- .../internal/otelcolconvert/testdata/groupbyattrs.alloy | 1 - 1 file changed, 1 deletion(-) diff --git a/internal/converter/internal/otelcolconvert/testdata/groupbyattrs.alloy b/internal/converter/internal/otelcolconvert/testdata/groupbyattrs.alloy index ebf910d335..48ebc9b079 100644 --- a/internal/converter/internal/otelcolconvert/testdata/groupbyattrs.alloy +++ b/internal/converter/internal/otelcolconvert/testdata/groupbyattrs.alloy @@ -29,4 +29,3 @@ otelcol.exporter.otlp "default" { endpoint = "database:4317" } } - From b1b562c4f33818d69e43a4b7a2e1395ba9e9216d Mon Sep 17 00:00:00 2001 From: Salaam Kehinde Date: Tue, 20 Aug 2024 20:26:43 +0200 Subject: [PATCH 15/16] Update docs/sources/reference/components/otelcol/otelcol.processor.groupbyattrs.md Co-authored-by: Paulin Todev --- .../otelcol/otelcol.processor.groupbyattrs.md | 60 +++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/docs/sources/reference/components/otelcol/otelcol.processor.groupbyattrs.md b/docs/sources/reference/components/otelcol/otelcol.processor.groupbyattrs.md index 19d9a58c5c..ea7e61531b 100644 --- a/docs/sources/reference/components/otelcol/otelcol.processor.groupbyattrs.md +++ b/docs/sources/reference/components/otelcol/otelcol.processor.groupbyattrs.md @@ -154,6 +154,66 @@ This output demonstrates how `otelcol.processor.groupbyattrs` works in various s - The specified "grouping" attributes that are set on the new Resources are also removed from the metric DataPoints. - While not shown in the above example, the processor also merges collections of records under matching InstrumentationLibrary. +### Compaction + +Sometimes telemetry data can become fragmented due to multiple duplicated ResourceSpans/ResourceLogs/ResourceMetrics objects. +This leads to additional memory consumption, increased processing costs, inefficient serialization and increase of the export requests. +In such situations, `otelcol.processor.groupbyattrs` can be used to compact the data with matching Resource and InstrumentationLibrary properties. + +For example, consider this input data: + +``` +Resource {host.name="localhost"} + InstrumentationLibrary {name="MyLibrary"} + Spans + Span {span_id=1, ...} + InstrumentationLibrary {name="OtherLibrary"} + Spans + Span {span_id=2, ...} + +Resource {host.name="localhost"} + InstrumentationLibrary {name="MyLibrary"} + Spans + Span {span_id=3, ...} + +Resource {host.name="localhost"} + InstrumentationLibrary {name="MyLibrary"} + Spans + Span {span_id=4, ...} + +Resource {host.name="otherhost"} + InstrumentationLibrary {name="MyLibrary"} + Spans + Span {span_id=5, ...} +``` + +You can use `otelcol.processor.groupbyattrs` with its default configuration to compact the data: +```alloy +otelcol.processor.groupbyattrs "default" { + output { + metrics = [otelcol.exporter.otlp.default.input] + } +} +``` + +The output will be: + +``` +Resource {host.name="localhost"} + InstrumentationLibrary {name="MyLibrary"} + Spans + Span {span_id=1, ...} + Span {span_id=3, ...} + Span {span_id=4, ...} + InstrumentationLibrary {name="OtherLibrary"} + Spans + Span {span_id=2, ...} + +Resource {host.name="otherhost"} + InstrumentationLibrary {name="MyLibrary"} + Spans + Span {span_id=5, ...} +``` [otelcol.processor.batch]: ../otelcol.processor.batch/ From d7b759335ecc2ae40cdf398e4aad2c3bc44c9d0e Mon Sep 17 00:00:00 2001 From: Paulin Todev Date: Wed, 21 Aug 2024 11:36:00 +0100 Subject: [PATCH 16/16] Move blocks to the Blocks section --- .../otelcol/otelcol.processor.groupbyattrs.md | 23 ++++++++++++------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/docs/sources/reference/components/otelcol/otelcol.processor.groupbyattrs.md b/docs/sources/reference/components/otelcol/otelcol.processor.groupbyattrs.md index ea7e61531b..ccc9b2151e 100644 --- a/docs/sources/reference/components/otelcol/otelcol.processor.groupbyattrs.md +++ b/docs/sources/reference/components/otelcol/otelcol.processor.groupbyattrs.md @@ -36,17 +36,24 @@ otelcol.processor.groupbyattrs "LABEL" { The following arguments are supported: -| Name | Type | Description | Default | Required | -|-----------------|-------------------|---------------------------------------------------------------------------------------|---------|----------| -| `keys` | `list(string)` | Keys that will be used to group the spans, log records or metric data points together. | | no | -| `output` | [output][] | Configures where to send received telemetry data. | yes | | -| `debug_metrics` | [debug_metrics][] | Configures the metrics that this component generates to monitor its state. | no | | +| Name | Type | Description | Default | Required | +|-----------------|-------------------|-----------------------------------------------------------------------------------------|---------|----------| +| `keys` | `list(string)` | Keys that will be used to group the spans, log records, or metric data points together. | `[]` | no | -[output]: #output-block -[debug_metrics]: #debug_metrics-block +`keys` is a string array that is used for grouping the data. +If it is empty, the processor performs compaction and reassociates all spans with matching Resource and InstrumentationLibrary. + +## Blocks -`keys` is a string array that is used for grouping the data. If it is empty, the processor performs compaction and reassociates all spans with matching Resource and InstrumentationLibrary. +The following blocks are supported inside the definition of `otelcol.processor.groupbyattrs`: +Hierarchy | Block | Description | Required +--------- | ----------- | ------------------------------------------------- | -------- +output | [output][] | Configures where to send received telemetry data. | yes +debug_metrics | [debug_metrics][] | Configures the metrics that this component generates to monitor its state. | no + +[output]: #output-block +[debug_metrics]: #debug_metrics-block ### output block