diff --git a/NOTICE-fips.txt b/NOTICE-fips.txt index ce6cb2a792c..d7c2b1b92bf 100644 --- a/NOTICE-fips.txt +++ b/NOTICE-fips.txt @@ -3152,6 +3152,217 @@ Contents of probable licence file $GOMODCACHE/github.com/elastic/opentelemetry-c limitations under the License. +-------------------------------------------------------------------------------- +Dependency : github.com/elastic/opentelemetry-collector-components/extension/beatsauthextension +Version: v0.2.0 +Licence type (autodetected): Apache-2.0 +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/elastic/opentelemetry-collector-components/extension/beatsauthextension@v0.2.0/LICENSE: + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2018 Elasticsearch BV + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + -------------------------------------------------------------------------------- Dependency : github.com/elastic/opentelemetry-collector-components/processor/elasticinframetricsprocessor Version: v0.16.0 diff --git a/NOTICE.txt b/NOTICE.txt index 1c856fa4d84..86e92ab6d0b 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -3152,6 +3152,217 @@ Contents of probable licence file $GOMODCACHE/github.com/elastic/opentelemetry-c limitations under the License. +-------------------------------------------------------------------------------- +Dependency : github.com/elastic/opentelemetry-collector-components/extension/beatsauthextension +Version: v0.2.0 +Licence type (autodetected): Apache-2.0 +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/elastic/opentelemetry-collector-components/extension/beatsauthextension@v0.2.0/LICENSE: + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2018 Elasticsearch BV + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + -------------------------------------------------------------------------------- Dependency : github.com/elastic/opentelemetry-collector-components/processor/elasticinframetricsprocessor Version: v0.16.0 diff --git a/go.mod b/go.mod index df0494c9037..7b703ac5e9d 100644 --- a/go.mod +++ b/go.mod @@ -29,6 +29,7 @@ require ( github.com/elastic/opentelemetry-collector-components/connector/elasticapmconnector v0.6.0 github.com/elastic/opentelemetry-collector-components/extension/apikeyauthextension v0.4.1 github.com/elastic/opentelemetry-collector-components/extension/apmconfigextension v0.6.0 + github.com/elastic/opentelemetry-collector-components/extension/beatsauthextension v0.2.0 github.com/elastic/opentelemetry-collector-components/processor/elasticinframetricsprocessor v0.16.0 github.com/elastic/opentelemetry-collector-components/processor/elastictraceprocessor v0.9.0 github.com/elastic/opentelemetry-collector-components/receiver/elasticapmintakereceiver v0.2.1 diff --git a/go.sum b/go.sum index b02f6ea66f5..14fd8368d5c 100644 --- a/go.sum +++ b/go.sum @@ -549,6 +549,8 @@ github.com/elastic/opentelemetry-collector-components/extension/apikeyauthextens github.com/elastic/opentelemetry-collector-components/extension/apikeyauthextension v0.4.1/go.mod h1:4CogfV72wu4glmT7/Gr6XzzJtSoGQs/cDAeSbb7UZYc= github.com/elastic/opentelemetry-collector-components/extension/apmconfigextension v0.6.0 h1:UBAq2kilCpYBKkRQovA8AG5N5AAnQY5IA3ZEhv4vMAo= github.com/elastic/opentelemetry-collector-components/extension/apmconfigextension v0.6.0/go.mod h1:E1uPMjGeBL8PbYCqX4WHMHGqZ3Oo+gK1OKe5Mp1146M= +github.com/elastic/opentelemetry-collector-components/extension/beatsauthextension v0.2.0 h1:cQ4Bu5iyJn5jk68OdwpGIifqVwAZUCoYpN3ZVbVRGBA= +github.com/elastic/opentelemetry-collector-components/extension/beatsauthextension v0.2.0/go.mod h1:aG7w7AA2CydjMxGG6zUZggXCPa+jVRhVYo/92wiDx4Q= github.com/elastic/opentelemetry-collector-components/internal/sharedcomponent v0.0.0-20250220025958-386ba0c4bced h1:XcWi/S3OoeE5Qwmj381AoO3nr3AVXl4Z4QO0mqyjlzU= github.com/elastic/opentelemetry-collector-components/internal/sharedcomponent v0.0.0-20250220025958-386ba0c4bced/go.mod h1:8e9NcGfE2xeor8r+WV9a+hKBEkzJEDnqZN7tqb3GUe8= github.com/elastic/opentelemetry-collector-components/internal/testutil v0.0.0-20250613082151-282de5af1c9b h1:NWuTKdMCJlU9ehRH8V0w1Kk1QI5Vn+9OcJWIO9wI+pE= diff --git a/internal/pkg/otel/README.md b/internal/pkg/otel/README.md index a4135513e95..7e68746c2cc 100644 --- a/internal/pkg/otel/README.md +++ b/internal/pkg/otel/README.md @@ -92,6 +92,7 @@ This section provides a summary of components included in the Elastic Distributi | [apikeyauthextension](https://github.com/elastic/opentelemetry-collector-components/blob/extension/apikeyauthextension/v0.4.1/extension/apikeyauthextension/README.md) | v0.4.1 | | [apmconfigextension](https://github.com/elastic/opentelemetry-collector-components/blob/extension/apmconfigextension/v0.6.0/extension/apmconfigextension/README.md) | v0.6.0 | | [bearertokenauthextension](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/extension/bearertokenauthextension/v0.132.0/extension/bearertokenauthextension/README.md) | v0.132.0 | +| [beatsauthextension](https://github.com/elastic/opentelemetry-collector-components/blob/extension/beatsauthextension/v0.2.0/extension/beatsauthextension/README.md) | v0.2.0 | | [filestorage](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/extension/storage/filestorage/v0.132.0/extension/storage/filestorage/README.md) | v0.132.0 | | [healthcheckextension](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/extension/healthcheckextension/v0.132.0/extension/healthcheckextension/README.md) | v0.132.0 | | [healthcheckv2extension](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/extension/healthcheckv2extension/v0.132.0/extension/healthcheckv2extension/README.md) | v0.132.0 | diff --git a/internal/pkg/otel/components.go b/internal/pkg/otel/components.go index 5393e9274fe..38069a5c646 100644 --- a/internal/pkg/otel/components.go +++ b/internal/pkg/otel/components.go @@ -79,6 +79,7 @@ import ( forwardconnector "go.opentelemetry.io/collector/connector/forwardconnector" elasticapmconnector "github.com/elastic/opentelemetry-collector-components/connector/elasticapmconnector" + beatsauthextension "github.com/elastic/opentelemetry-collector-components/extension/beatsauthextension" ) func components(extensionFactories ...extension.Factory) func() (otelcol.Factories, error) { @@ -174,6 +175,7 @@ func components(extensionFactories ...extension.Factory) func() (otelcol.Factori k8sobserver.NewFactory(), apikeyauthextension.NewFactory(), apmconfigextension.NewFactory(), + beatsauthextension.NewFactory(), } extensions = append(extensions, extensionFactories...) factories.Extensions, err = otelcol.MakeFactoryMap[extension.Factory](extensions...) diff --git a/internal/pkg/otel/manager/manager.go b/internal/pkg/otel/manager/manager.go index 14fcfcd0ead..5588940ef52 100644 --- a/internal/pkg/otel/manager/manager.go +++ b/internal/pkg/otel/manager/manager.go @@ -268,7 +268,7 @@ func (m *OTelManager) Run(ctx context.Context) error { // and reset the retry count m.recoveryTimer.Stop() m.recoveryRetries.Store(0) - mergedCfg, err := buildMergedConfig(cfgUpdate, m.agentInfo, m.beatMonitoringConfigGetter) + mergedCfg, err := buildMergedConfig(cfgUpdate, m.agentInfo, m.beatMonitoringConfigGetter, m.baseLogger) if err != nil { reportErr(ctx, m.errCh, err) continue @@ -300,7 +300,7 @@ func (m *OTelManager) Errors() <-chan error { } // buildMergedConfig combines collector configuration with component-derived configuration. -func buildMergedConfig(cfgUpdate configUpdate, agentInfo info.Agent, monitoringConfigGetter translate.BeatMonitoringConfigGetter) (*confmap.Conf, error) { +func buildMergedConfig(cfgUpdate configUpdate, agentInfo info.Agent, monitoringConfigGetter translate.BeatMonitoringConfigGetter, logger *logp.Logger) (*confmap.Conf, error) { mergedOtelCfg := confmap.New() // Generate component otel config if there are components @@ -308,7 +308,7 @@ func buildMergedConfig(cfgUpdate configUpdate, agentInfo info.Agent, monitoringC if len(cfgUpdate.components) > 0 { model := &component.Model{Components: cfgUpdate.components} var err error - componentOtelCfg, err = translate.GetOtelConfig(model, agentInfo, monitoringConfigGetter) + componentOtelCfg, err = translate.GetOtelConfig(model, agentInfo, monitoringConfigGetter, logger) if err != nil { return nil, fmt.Errorf("failed to generate otel config: %w", err) } diff --git a/internal/pkg/otel/manager/manager_test.go b/internal/pkg/otel/manager/manager_test.go index bb69f493fa5..dcb27f2108a 100644 --- a/internal/pkg/otel/manager/manager_test.go +++ b/internal/pkg/otel/manager/manager_test.go @@ -31,6 +31,7 @@ import ( "gopkg.in/yaml.v2" "github.com/elastic/elastic-agent-libs/logp" + "github.com/elastic/elastic-agent-libs/logp/logptest" "github.com/elastic/elastic-agent/pkg/core/logger" "github.com/elastic/elastic-agent/pkg/core/logger/loggertest" @@ -767,7 +768,7 @@ func TestOTelManager_buildMergedConfig(t *testing.T) { collectorCfg: tt.collectorCfg, components: tt.components, } - result, err := buildMergedConfig(cfgUpdate, commonAgentInfo, commonBeatMonitoringConfigGetter) + result, err := buildMergedConfig(cfgUpdate, commonAgentInfo, commonBeatMonitoringConfigGetter, logptest.NewTestingLogger(t, "")) if tt.expectedErrorString != "" { assert.Error(t, err) diff --git a/internal/pkg/otel/translate/otelconfig.go b/internal/pkg/otel/translate/otelconfig.go index d9ceec48bc0..6bef6f4a6ae 100644 --- a/internal/pkg/otel/translate/otelconfig.go +++ b/internal/pkg/otel/translate/otelconfig.go @@ -22,6 +22,7 @@ import ( "golang.org/x/exp/maps" elasticsearchtranslate "github.com/elastic/beats/v7/libbeat/otelbeat/oteltranslate/outputs/elasticsearch" + "github.com/elastic/beats/v7/libbeat/outputs/elasticsearch" "github.com/elastic/beats/v7/x-pack/filebeat/fbreceiver" "github.com/elastic/beats/v7/x-pack/libbeat/management" "github.com/elastic/beats/v7/x-pack/metricbeat/mbreceiver" @@ -39,7 +40,7 @@ const OtelNamePrefix = "_agent-component/" // BeatMonitoringConfigGetter is a function that returns the monitoring configuration for a beat receiver. type BeatMonitoringConfigGetter func(unitID, binary string) map[string]any -type exporterConfigTranslationFunc func(*config.C) (map[string]any, error) +type exporterConfigTranslationFunc func(*config.C, *logp.Logger) (map[string]any, error) var ( OtelSupportedOutputTypes = []string{"elasticsearch"} @@ -56,18 +57,31 @@ func GetOtelConfig( model *component.Model, info info.Agent, beatMonitoringConfigGetter BeatMonitoringConfigGetter, + logger *logp.Logger, ) (*confmap.Conf, error) { components := getSupportedComponents(model) if len(components) == 0 { return nil, nil } - otelConfig := confmap.New() // base config, nothing here for now + otelConfig := confmap.New() // base config, nothing here for now + extensionList := []interface{}{} // we have to maintain a list because otel does not merge lists, it overrides them. This is a known issue: see https://github.com/open-telemetry/opentelemetry-collector/issues/8754 for _, comp := range components { - componentConfig, compErr := getCollectorConfigForComponent(comp, info, beatMonitoringConfigGetter) + componentConfig, compErr := getCollectorConfigForComponent(comp, info, beatMonitoringConfigGetter, logger) if compErr != nil { return nil, compErr } + + // logic to merge extension list + if componentConfig.IsSet("service::extensions") { + extensionList = append(extensionList, componentConfig.Get("service::extensions").([]interface{})...) + extensions := confmap.NewFromStringMap(map[string]any{"service::extensions": extensionList}) + err := componentConfig.Merge(extensions) + if err != nil { + return nil, fmt.Errorf("error merging otel extensions for component %s: %w", comp.ID, err) + } + } + // the assumption here is that each component will define its own receivers, and the shared exporters // will be merged mergeErr := otelConfig.Merge(componentConfig) @@ -119,15 +133,24 @@ func getExporterID(exporterType otelcomponent.Type, outputName string) otelcompo return otelcomponent.NewIDWithName(exporterType, exporterName) } +// getBeatsAuthExtensionID returns the id for beatsauth extension +// outputName here is name of the output defined in elastic-agent.yml. For ex: default, monitoring +func getBeatsAuthExtensionID(outputName string) otelcomponent.ID { + extensionName := fmt.Sprintf("%s%s", OtelNamePrefix, outputName) + return otelcomponent.NewIDWithName(otelcomponent.MustNewType("beatsauth"), extensionName) +} + // getCollectorConfigForComponent returns the Otel collector config required to run the given component. // This function returns a full, valid configuration that can then be merged with configurations for other components. +// Note: Lists are not merged and should be handled by the caller of the method func getCollectorConfigForComponent( comp *component.Component, info info.Agent, beatMonitoringConfigGetter BeatMonitoringConfigGetter, + logger *logp.Logger, ) (*confmap.Conf, error) { - exportersConfig, outputQueueConfig, err := getExportersConfigForComponent(comp) + exportersConfig, outputQueueConfig, extensionConfig, err := getExportersConfigForComponent(comp, logger) if err != nil { return nil, err } @@ -147,13 +170,22 @@ func getCollectorConfigForComponent( }, } + // we need to convert []string to []interface for this to work + extensionKey := make([]interface{}, len(maps.Keys(extensionConfig))) + for i, v := range maps.Keys(extensionConfig) { + extensionKey[i] = v + } + fullConfig := map[string]any{ - "receivers": receiversConfig, - "exporters": exportersConfig, + "receivers": receiversConfig, + "exporters": exportersConfig, + "extensions": extensionConfig, "service": map[string]any{ - "pipelines": pipelinesConfig, + "extensions": extensionKey, + "pipelines": pipelinesConfig, }, } + return confmap.NewFromStringMap(fullConfig), nil } @@ -256,26 +288,31 @@ func getReceiversConfigForComponent( // getReceiversConfigForComponent returns the exporters configuration and queue settings for a component. Usually this will be a single // exporter, but in principle it could be more. -func getExportersConfigForComponent(comp *component.Component) (exporterCfg map[string]any, queueCfg map[string]any, err error) { +func getExportersConfigForComponent(comp *component.Component, logger *logp.Logger) (exporterCfg map[string]any, queueCfg map[string]any, extensionCfg map[string]any, err error) { exportersConfig := map[string]any{} + extensionConfig := map[string]any{} exporterType, err := getExporterTypeForComponent(comp) if err != nil { - return nil, nil, err + return nil, nil, nil, err } var queueSettings map[string]any for _, unit := range comp.Units { if unit.Type == client.UnitTypeOutput { var unitExportersConfig map[string]any - unitExportersConfig, queueSettings, err = unitToExporterConfig(unit, exporterType, comp.InputType) + var unitExtensionConfig map[string]any + unitExportersConfig, queueSettings, unitExtensionConfig, err = unitToExporterConfig(unit, exporterType, comp.InputType, logger) if err != nil { - return nil, nil, err + return nil, nil, nil, err } for k, v := range unitExportersConfig { exportersConfig[k] = v } + for k, v := range unitExtensionConfig { + extensionConfig[k] = v + } } } - return exportersConfig, queueSettings, nil + return exportersConfig, queueSettings, extensionConfig, nil } // GetBeatNameForComponent returns the beat binary name that would be used to run this component. @@ -323,13 +360,13 @@ func getExporterTypeForComponent(comp *component.Component) (otelcomponent.Type, } // unitToExporterConfig translates a component.Unit to return an otel exporter configuration and output queue settings -func unitToExporterConfig(unit component.Unit, exporterType otelcomponent.Type, inputType string) (exportersCfg map[string]any, queueSettings map[string]any, err error) { +func unitToExporterConfig(unit component.Unit, exporterType otelcomponent.Type, inputType string, logger *logp.Logger) (exportersCfg map[string]any, queueSettings map[string]any, extensionCfg map[string]any, err error) { if unit.Type == client.UnitTypeInput { - return nil, nil, fmt.Errorf("unit type is an input, expected output: %v", unit) + return nil, nil, nil, fmt.Errorf("unit type is an input, expected output: %v", unit) } configTranslationFunc, ok := configTranslationFuncForExporter[exporterType] if !ok { - return nil, nil, fmt.Errorf("no config translation function for exporter type: %s", exporterType) + return nil, nil, nil, fmt.Errorf("no config translation function for exporter type: %s", exporterType) } // we'd like to use the same exporter for all outputs with the same name, so we parse out the name for the unit id // these will be deduplicated by the configuration merging process at the end @@ -340,30 +377,51 @@ func unitToExporterConfig(unit component.Unit, exporterType otelcomponent.Type, unitConfigMap := unit.Config.GetSource().AsMap() // this is what beats do in libbeat/management/generate.go outputCfgC, err := config.NewConfigFrom(unitConfigMap) if err != nil { - return nil, nil, fmt.Errorf("error translating config for output: %s, unit: %s, error: %w", outputName, unit.ID, err) + return nil, nil, nil, fmt.Errorf("error translating config for output: %s, unit: %s, error: %w", outputName, unit.ID, err) } + // Config translation function can mutate queue settings defined under output config - exporterConfig, err := configTranslationFunc(outputCfgC) + exporterConfig, err := configTranslationFunc(outputCfgC, logger) if err != nil { - return nil, nil, fmt.Errorf("error translating config for output: %s, unit: %s, error: %w", outputName, unit.ID, err) - } - - exportersCfg = map[string]any{ - exporterId.String(): exporterConfig, + return nil, nil, nil, fmt.Errorf("error translating config for output: %s, unit: %s, error: %w", outputName, unit.ID, err) } // If output config contains queue settings defined by user/preset field, it should be promoted to the receiver section if ok := outputCfgC.HasField("queue"); ok { err := outputCfgC.Unpack(&queueSettings) if err != nil { - return nil, nil, fmt.Errorf("error unpacking queue settings for output: %s, unit: %s, error: %w", outputName, unit.ID, err) + return nil, nil, nil, fmt.Errorf("error unpacking queue settings for output: %s, unit: %s, error: %w", outputName, unit.ID, err) } if queue, ok := queueSettings["queue"].(map[string]any); ok { queueSettings = queue } } - return exportersCfg, queueSettings, nil + // beatsauth extension is not tested with output other than elasticsearch + if exporterType.String() == "elasticsearch" { + // get extension ID + extensionID := getBeatsAuthExtensionID(outputName) + extensionConfig, err := getBeatsAuthExtensionConfig(outputCfgC) + if err != nil { + return nil, nil, nil, fmt.Errorf("error supporting http parameters for output: %s, unit: %s, error: %w", outputName, unit.ID, err) + } + + // sets extensionCfg + extensionCfg = map[string]any{ + extensionID.String(): extensionConfig, + } + // add authenticator to ES config + exporterConfig["auth"] = map[string]any{ + "authenticator": extensionID.String(), + } + + } + + exportersCfg = map[string]any{ + exporterId.String(): exporterConfig, + } + + return exportersCfg, queueSettings, extensionCfg, nil } // getInputsForUnit returns the beat inputs for a unit. These can directly be plugged into a beats receiver config. @@ -410,9 +468,8 @@ func getDefaultDatastreamTypeForComponent(comp *component.Component) (string, er } // translateEsOutputToExporter translates an elasticsearch output configuration to an elasticsearch exporter configuration. -func translateEsOutputToExporter(cfg *config.C) (map[string]any, error) { - // TODO: Figure out a way to avoid needing a logger for this function - esConfig, err := elasticsearchtranslate.ToOTelConfig(cfg, logp.L()) +func translateEsOutputToExporter(cfg *config.C, logger *logp.Logger) (map[string]any, error) { + esConfig, err := elasticsearchtranslate.ToOTelConfig(cfg, logger) if err != nil { return nil, err } @@ -429,3 +486,26 @@ func translateEsOutputToExporter(cfg *config.C) (map[string]any, error) { func BeatDataPath(componentId string) string { return filepath.Join(paths.Run(), componentId) } + +// getBeatsAuthExtensionConfig sets http transport settings on beatsauth +// currently this is only supported for elasticsearch output +func getBeatsAuthExtensionConfig(cfg *config.C) (map[string]any, error) { + defaultTransportSettings := elasticsearch.ESDefaultTransportSettings() + err := cfg.Unpack(&defaultTransportSettings) + if err != nil { + return nil, err + } + + newConfig, err := config.NewConfigFrom(defaultTransportSettings) + if err != nil { + return nil, err + } + + var newMap map[string]any + err = newConfig.Unpack(&newMap) + if err != nil { + return nil, err + } + + return newMap, nil +} diff --git a/internal/pkg/otel/translate/otelconfig_test.go b/internal/pkg/otel/translate/otelconfig_test.go index e6f70345b37..2e765c33e59 100644 --- a/internal/pkg/otel/translate/otelconfig_test.go +++ b/internal/pkg/otel/translate/otelconfig_test.go @@ -15,6 +15,7 @@ import ( "go.opentelemetry.io/collector/confmap" "github.com/elastic/elastic-agent-client/v7/pkg/client" + "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/mapstr" "github.com/elastic/elastic-agent/internal/pkg/agent/application/info" @@ -209,17 +210,66 @@ func TestGetOtelConfig(t *testing.T) { }, }, } - esOutputConfig := map[string]any{ - "type": "elasticsearch", - "hosts": []any{"localhost:9200"}, - "username": "elastic", - "password": "password", - "preset": "balanced", - "queue.mem.events": 3200, + + type extraParams struct { + key string + value any + } + // pass ssl params as extra args to this method + esOutputConfig := func(extra ...extraParams) map[string]any { + finalOutput := map[string]any{ + "type": "elasticsearch", + "hosts": []any{"localhost:9200"}, + "username": "elastic", + "password": "password", + "preset": "balanced", + "queue.mem.events": 3200, + "ssl.enabled": true, + } + + for _, v := range extra { + finalOutput[v.key] = v.value + } + return finalOutput } - expectedESConfig := map[string]any{ - "elasticsearch/_agent-component/default": map[string]any{ + expectedExtensionConfig := func(extra ...extraParams) map[string]any { + finalOutput := map[string]any{ + "idle_connection_timeout": "3s", + "proxy_disable": false, + "ssl": map[string]interface{}{ + "ca_sha256": []interface{}{}, + "ca_trusted_fingerprint": "", + "certificate": "", + "certificate_authorities": []interface{}{}, + "cipher_suites": []interface{}{}, + "curve_types": []interface{}{}, + "enabled": true, + "key": "", + "key_passphrase": "", + "key_passphrase_path": "", + "renegotiation": int64(0), + "supported_protocols": []interface{}{}, + "verification_mode": uint64(0), + }, + "timeout": "1m30s", + } + for _, v := range extra { + // accepts one level deep parameters to replace + if _, ok := v.value.(map[string]any); ok { + for newkey, newvalue := range v.value.(map[string]any) { + // this is brittle - it is expected that developers will pass expected params correctly here + finalOutput[v.key].(map[string]any)[newkey] = newvalue + } + continue + } + finalOutput[v.key] = v.value + } + return finalOutput + } + + expectedESConfig := func(outputName string) map[string]any { + return map[string]any{ "compression": "gzip", "compression_params": map[string]any{ "level": 1, @@ -254,7 +304,14 @@ func TestGetOtelConfig(t *testing.T) { }, "timeout": 90 * time.Second, "idle_conn_timeout": 3 * time.Second, - }, + "auth": map[string]any{ + "authenticator": "beatsauth/_agent-component/" + outputName, + }, + "tls": map[string]any{ + "min_version": "1.2", + "max_version": "1.3", + }, + } } defaultProcessors := func(streamId, dataset string, namespace string) []any { @@ -314,6 +371,73 @@ func TestGetOtelConfig(t *testing.T) { } } + // expects input id + expectedFilestreamConfig := func(id string) map[string]any { + return map[string]any{ + "filebeat": map[string]any{ + "inputs": []map[string]any{ + { + "id": "test-1", + "type": "filestream", + "data_stream": map[string]any{ + "dataset": "generic-1", + }, + "paths": []any{ + "/var/log/*.log", + }, + "index": "logs-generic-1-default", + "processors": defaultProcessors("test-1", "generic-1", "logs"), + }, + { + "id": "test-2", + "type": "filestream", + "data_stream": map[string]any{ + "dataset": "generic-2", + }, + "paths": []any{ + "/var/log/*.log", + }, + "index": "logs-generic-2-default", + "processors": defaultProcessors("test-2", "generic-2", "logs"), + }, + }, + }, + "output": map[string]any{ + "otelconsumer": map[string]any{}, + }, + "path": map[string]any{ + "data": filepath.Join(paths.Run(), id), + }, + "queue": map[string]any{ + "mem": map[string]any{ + "events": uint64(3200), + "flush": map[string]any{ + "min_events": uint64(1600), + "timeout": "10s", + }, + }, + }, + "logging": map[string]any{ + "with_fields": map[string]any{ + "component": map[string]any{ + "binary": "filebeat", + "dataset": "elastic_agent.filebeat", + "type": "filestream", + "id": id, + }, + "log": map[string]any{ + "source": id, + }, + }, + }, + "http": map[string]any{ + "enabled": true, + "host": "localhost", + }, + } + + } + getBeatMonitoringConfig := func(_, _ string) map[string]any { return map[string]any{ "http": map[string]any{ @@ -367,83 +491,112 @@ func TestGetOtelConfig(t *testing.T) { { ID: "filestream-default", Type: client.UnitTypeOutput, - Config: component.MustExpectedConfig(esOutputConfig), + Config: component.MustExpectedConfig(esOutputConfig()), }, }, }, }, }, expectedConfig: confmap.NewFromStringMap(map[string]any{ - "exporters": expectedESConfig, + "exporters": map[string]any{ + "elasticsearch/_agent-component/default": expectedESConfig("default"), + }, + "extensions": map[string]any{ + "beatsauth/_agent-component/default": expectedExtensionConfig(), + }, "receivers": map[string]any{ - "filebeatreceiver/_agent-component/filestream-default": map[string]any{ - "filebeat": map[string]any{ - "inputs": []map[string]any{ - { - "id": "test-1", - "type": "filestream", - "data_stream": map[string]any{ - "dataset": "generic-1", - }, - "paths": []any{ - "/var/log/*.log", - }, - "index": "logs-generic-1-default", - "processors": defaultProcessors("test-1", "generic-1", "logs"), - }, - { - "id": "test-2", - "type": "filestream", - "data_stream": map[string]any{ - "dataset": "generic-2", - }, - "paths": []any{ - "/var/log/*.log", - }, - "index": "logs-generic-2-default", - "processors": defaultProcessors("test-2", "generic-2", "logs"), + "filebeatreceiver/_agent-component/filestream-default": expectedFilestreamConfig("filestream-default"), + }, + "service": map[string]any{ + "extensions": []interface{}{"beatsauth/_agent-component/default"}, + "pipelines": map[string]any{ + "logs/_agent-component/filestream-default": map[string][]string{ + "exporters": []string{"elasticsearch/_agent-component/default"}, + "receivers": []string{"filebeatreceiver/_agent-component/filestream-default"}, + }, + }, + }, + }), + }, + { + name: "multiple filestream inputs and output types", + model: &component.Model{ + Components: []component.Component{ + { + ID: "filestream-primaryOutput", + InputType: "filestream", + OutputType: "elasticsearch", + InputSpec: &component.InputRuntimeSpec{ + BinaryName: "agentbeat", + Spec: component.InputSpec{ + Command: &component.CommandSpec{ + Args: []string{"filebeat"}, }, }, }, - "output": map[string]any{ - "otelconsumer": map[string]any{}, - }, - "path": map[string]any{ - "data": filepath.Join(paths.Run(), "filestream-default"), - }, - "queue": map[string]any{ - "mem": map[string]any{ - "events": uint64(3200), - "flush": map[string]any{ - "min_events": uint64(1600), - "timeout": "10s", - }, + Units: []component.Unit{ + { + ID: "filestream-unit", + Type: client.UnitTypeInput, + Config: component.MustExpectedConfig(fileStreamConfig), + }, + { + ID: "filestream-primaryOutput", + Type: client.UnitTypeOutput, + Config: component.MustExpectedConfig(esOutputConfig(extraParams{"ssl.verification_mode", "certificate"})), }, }, - "logging": map[string]any{ - "with_fields": map[string]any{ - "component": map[string]any{ - "binary": "filebeat", - "dataset": "elastic_agent.filebeat", - "type": "filestream", - "id": "filestream-default", - }, - "log": map[string]any{ - "source": "filestream-default", + }, + { + ID: "filestream-secondaryOutput", + InputType: "filestream", + OutputType: "elasticsearch", + InputSpec: &component.InputRuntimeSpec{ + BinaryName: "agentbeat", + Spec: component.InputSpec{ + Command: &component.CommandSpec{ + Args: []string{"filebeat"}, }, }, }, - "http": map[string]any{ - "enabled": true, - "host": "localhost", + Units: []component.Unit{ + { + ID: "filestream-unit-2", + Type: client.UnitTypeInput, + Config: component.MustExpectedConfig(fileStreamConfig), + }, + { + ID: "filestream-secondaryOutput", + Type: client.UnitTypeOutput, + Config: component.MustExpectedConfig(esOutputConfig(extraParams{"ssl.ca_trusted_fingerprint", "b9a10bbe64ee9826abeda6546fc988c8bf798b41957c33d05db736716513dc9c"})), + }, }, }, }, + }, + expectedConfig: confmap.NewFromStringMap(map[string]any{ + "exporters": map[string]any{ + "elasticsearch/_agent-component/primaryOutput": expectedESConfig("primaryOutput"), + "elasticsearch/_agent-component/secondaryOutput": expectedESConfig("secondaryOutput"), + }, + "extensions": map[string]any{ + "beatsauth/_agent-component/primaryOutput": expectedExtensionConfig(extraParams{"ssl", map[string]any{"verification_mode": uint64(2)}}), + "beatsauth/_agent-component/secondaryOutput": expectedExtensionConfig(extraParams{"ssl", map[string]any{"ca_trusted_fingerprint": "b9a10bbe64ee9826abeda6546fc988c8bf798b41957c33d05db736716513dc9c"}}), + }, + "receivers": map[string]any{ + "filebeatreceiver/_agent-component/filestream-primaryOutput": expectedFilestreamConfig("filestream-primaryOutput"), + "filebeatreceiver/_agent-component/filestream-secondaryOutput": expectedFilestreamConfig("filestream-secondaryOutput"), + }, "service": map[string]any{ + "extensions": []interface{}{"beatsauth/_agent-component/primaryOutput", "beatsauth/_agent-component/secondaryOutput"}, "pipelines": map[string]any{ - "logs/_agent-component/filestream-default": map[string][]string{ - "exporters": []string{"elasticsearch/_agent-component/default"}, - "receivers": []string{"filebeatreceiver/_agent-component/filestream-default"}, + "logs/_agent-component/filestream-primaryOutput": map[string][]string{ + "exporters": []string{"elasticsearch/_agent-component/primaryOutput"}, + "receivers": []string{"filebeatreceiver/_agent-component/filestream-primaryOutput"}, + }, + "logs/_agent-component/filestream-secondaryOutput": map[string][]string{ + "exporters": []string{"elasticsearch/_agent-component/secondaryOutput"}, + "receivers": []string{"filebeatreceiver/_agent-component/filestream-secondaryOutput"}, }, }, }, @@ -474,14 +627,19 @@ func TestGetOtelConfig(t *testing.T) { { ID: "beat/metrics-default", Type: client.UnitTypeOutput, - Config: component.MustExpectedConfig(esOutputConfig), + Config: component.MustExpectedConfig(esOutputConfig()), }, }, }, }, }, expectedConfig: confmap.NewFromStringMap(map[string]any{ - "exporters": expectedESConfig, + "exporters": map[string]any{ + "elasticsearch/_agent-component/default": expectedESConfig("default"), + }, + "extensions": map[string]any{ + "beatsauth/_agent-component/default": expectedExtensionConfig(), + }, "receivers": map[string]any{ "metricbeatreceiver/_agent-component/beat-metrics-monitoring": map[string]any{ "metricbeat": map[string]any{ @@ -533,6 +691,7 @@ func TestGetOtelConfig(t *testing.T) { }, }, "service": map[string]any{ + "extensions": []interface{}{"beatsauth/_agent-component/default"}, "pipelines": map[string]any{ "logs/_agent-component/beat-metrics-monitoring": map[string][]string{ "exporters": []string{"elasticsearch/_agent-component/default"}, @@ -567,14 +726,19 @@ func TestGetOtelConfig(t *testing.T) { { ID: "system/metrics-default", Type: client.UnitTypeOutput, - Config: component.MustExpectedConfig(esOutputConfig), + Config: component.MustExpectedConfig(esOutputConfig()), }, }, }, }, }, expectedConfig: confmap.NewFromStringMap(map[string]any{ - "exporters": expectedESConfig, + "exporters": map[string]any{ + "elasticsearch/_agent-component/default": expectedESConfig("default"), + }, + "extensions": map[string]any{ + "beatsauth/_agent-component/default": expectedExtensionConfig(), + }, "receivers": map[string]any{ "metricbeatreceiver/_agent-component/system-metrics": map[string]any{ "metricbeat": map[string]any{ @@ -637,6 +801,7 @@ func TestGetOtelConfig(t *testing.T) { }, }, "service": map[string]any{ + "extensions": []interface{}{"beatsauth/_agent-component/default"}, "pipelines": map[string]any{ "logs/_agent-component/system-metrics": map[string][]string{ "exporters": []string{"elasticsearch/_agent-component/default"}, @@ -649,7 +814,7 @@ func TestGetOtelConfig(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - actualConf, actualError := GetOtelConfig(tt.model, agentInfo, getBeatMonitoringConfig) + actualConf, actualError := GetOtelConfig(tt.model, agentInfo, getBeatMonitoringConfig, logp.NewNopLogger()) if actualConf == nil || tt.expectedConfig == nil { assert.Equal(t, tt.expectedConfig, actualConf) } else { // this gives a nicer diff @@ -874,5 +1039,3 @@ func TestGetReceiversConfigForComponent(t *testing.T) { }) } } - -// TODO: Add unit tests for other config generation functions diff --git a/testing/integration/ess/otel_test.go b/testing/integration/ess/otel_test.go index 5f9881650dd..09bea912d07 100644 --- a/testing/integration/ess/otel_test.go +++ b/testing/integration/ess/otel_test.go @@ -10,6 +10,7 @@ import ( "bytes" "context" "encoding/base64" + "encoding/pem" "errors" "fmt" "os" @@ -31,6 +32,7 @@ import ( "github.com/elastic/elastic-agent-libs/mapstr" "github.com/elastic/elastic-agent-libs/testing/estools" + "github.com/elastic/elastic-agent-libs/transport/tlscommontest" "github.com/elastic/elastic-agent/pkg/control/v2/client" aTesting "github.com/elastic/elastic-agent/pkg/testing" "github.com/elastic/elastic-agent/pkg/testing/define" @@ -1040,7 +1042,8 @@ func TestOtelFilestreamInput(t *testing.T) { require.True(t, len(esApiKey.Encoded) > 1, "api key is invalid %q", esApiKey) decodedApiKey, err := getDecodedApiKey(esApiKey) require.NoError(t, err) - configTemplate := `inputs: + configTemplate := ` +inputs: - type: filestream id: filestream-e2e use_output: default @@ -1059,6 +1062,8 @@ outputs: hosts: [{{.ESEndpoint}}] api_key: "{{.ESApiKey}}" preset: "balanced" + ssl.enabled: true + ssl.verification_mode: full monitoring: type: elasticsearch hosts: [{{.ESEndpoint}}] @@ -1919,3 +1924,160 @@ service: fixtureWg.Wait() require.True(t, err == nil || errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded), "Retrieved unexpected error: %s", err.Error()) } + +func TestOtelBeatsAuthExtension(t *testing.T) { + info := define.Require(t, define.Requirements{ + Group: integration.Default, + Local: true, + OS: []define.OS{ + // {Type: define.Windows}, we don't support otel on Windows yet + {Type: define.Linux}, + {Type: define.Darwin}, + }, + Stack: &define.Stack{}, + }) + + // Create the otel configuration file + type otelConfigOptions struct { + ESEndpoint string + ESApiKey string + Index string + CAFile string + } + esEndpoint, err := integration.GetESHost() + require.NoError(t, err, "error getting elasticsearch endpoint") + esApiKey, err := createESApiKey(info.ESClient) + require.NoError(t, err, "error creating API key") + require.True(t, len(esApiKey.Encoded) > 1, "api key is invalid %q", esApiKey) + index := "logs-integration-" + info.Namespace + + fixture, err := define.NewFixtureFromLocalBuild(t, define.Version()) + require.NoError(t, err) + + ctx, cancel := testcontext.WithDeadline(t, t.Context(), time.Now().Add(5*time.Minute)) + defer cancel() + err = fixture.Prepare(ctx) + require.NoError(t, err) + + // create ca-cert + caCert, err := tlscommontest.GenCA() + if err != nil { + t.Fatalf("could not generate root CA certificate: %s", err) + } + + caFilePath := filepath.Join(t.TempDir(), "ca.pem") + os.WriteFile(caFilePath, pem.EncodeToMemory(&pem.Block{ + Type: "CERTIFICATE", + Bytes: caCert.Leaf.Raw}), 0o777) + + // we pass an incorrect CA to es-exporter + // but we expect beatsauthextension to replace the exporter's + // roundtripper with how beats implements it (with given http configuration block) + // hence we expect events to be indexed to elasticsearch + // if authextension is not used - this test fails + otelConfigTemplate := ` +extensions: + beatsauth: + ssl: + enabled: true + verification_mode: none +receivers: + metricbeatreceiver: + metricbeat: + modules: + - module: system + enabled: true + period: 1s + processes: + - '.*' + metricsets: + - cpu + output: + otelconsumer: + queue.mem.flush.timeout: 0s +exporters: + elasticsearch/log: + endpoints: + - {{.ESEndpoint}} + api_key: {{.ESApiKey}} + logs_index: {{.Index}} + batcher: + enabled: true + flush_timeout: 1s + min_size: 1 + tls: + ca_file: {{ .CAFile }} + auth: + authenticator: beatsauth + mapping: + mode: bodymap +service: + extensions: [beatsauth] + pipelines: + logs: + receivers: + - metricbeatreceiver + exporters: + - elasticsearch/log +` + var otelConfigBuffer bytes.Buffer + require.NoError(t, + template.Must(template.New("otelConfig").Parse(otelConfigTemplate)).Execute(&otelConfigBuffer, + otelConfigOptions{ + ESEndpoint: esEndpoint, + ESApiKey: esApiKey.Encoded, + Index: index, + CAFile: caFilePath, + })) + + // configure elastic-agent.yml + err = fixture.Configure(ctx, otelConfigBuffer.Bytes()) + + // prepare agent command + cmd, err := fixture.PrepareAgentCommand(ctx, nil) + require.NoError(t, err, "cannot prepare Elastic-Agent command: %w", err) + + output := strings.Builder{} + cmd.Stderr = &output + cmd.Stdout = &output + + // start elastic-agent + err = cmd.Start() + require.NoError(t, err) + + t.Cleanup(func() { + if t.Failed() { + t.Log("Elastic-Agent output:") + t.Log(output.String()) + } + }) + + require.Eventually(t, func() bool { + err = fixture.IsHealthy(ctx) + if err != nil { + t.Logf("waiting for agent healthy: %s", err.Error()) + return false + } + return true + }, 30*time.Second, 1*time.Second) + + // Make sure find the logs + actualHits := &struct{ Hits int }{} + require.Eventually(t, + func() bool { + findCtx, findCancel := context.WithTimeout(t.Context(), 10*time.Second) + defer findCancel() + + docs, err := estools.GetLogsForIndexWithContext(findCtx, info.ESClient, ".ds-"+index+"*", map[string]interface{}{ + "metricset.name": "cpu", + }) + require.NoError(t, err) + + actualHits.Hits = docs.Hits.Total.Value + return actualHits.Hits >= 1 + }, + 2*time.Minute, 1*time.Second, + "Expected at least %d logs, got %v", 1, actualHits) + + cancel() +}