diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 403aa9b948b..1fbdac27c44 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -15,6 +15,9 @@ /dev-tools/kubernetes @elastic/elastic-agent-control-plane /docs/release-notes @elastic/ingest-docs /docs/docset.yml @elastic/ingest-docs +/.github/workflows/update-docs.yml @elastic/ingest-docs +/docs/reference/edot-collector @elastic/ingest-docs +/docs/scripts/update-docs @elastic/ingest-docs /internal/pkg/composable/providers/kubernetes @elastic/elastic-agent-control-plane /internal/pkg/otel/samples/darwin/autoops_es.yml @elastic/opex /internal/pkg/otel/samples/linux/autoops_es.yml @elastic/opex diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 02064b5f601..c3ced3b9219 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -32,3 +32,16 @@ updates: - Team:Elastic-Agent-Control-Plane - backport-active-all open-pull-requests-limit: 10 + + - package-ecosystem: "pip" + directory: "/docs/scripts/update-docs" + schedule: + interval: "weekly" + day: "monday" + time: "08:00" + labels: + - automation + - skip-changelog + - Team:Docs + - documentation + open-pull-requests-limit: 5 diff --git a/.github/workflows/update-docs.yml b/.github/workflows/update-docs.yml new file mode 100644 index 00000000000..b3a48214a12 --- /dev/null +++ b/.github/workflows/update-docs.yml @@ -0,0 +1,56 @@ +name: Update documentation + +on: + schedule: + # Run weekly on Monday at 8:00 AM UTC + - cron: '0 8 * * 1' + workflow_dispatch: + # Allow manual triggering + +jobs: + update-docs: + runs-on: ubuntu-latest + permissions: + contents: write + pull-requests: write + issues: write + steps: + - name: Checkout repository + uses: actions/checkout@v5 + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: '3.13' + cache: 'pip' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r docs/scripts/update-docs/requirements.txt + + - name: Run documentation generator + run: | + cd docs/scripts/update-docs + python update-components-docs.py + + - name: Create Pull Request + uses: peter-evans/create-pull-request@271a8d0340265f705b14b6d32b9829c1cb33d45e # v7.0.8 + with: + token: ${{ secrets.GITHUB_TOKEN }} + commit-message: "docs: update generated documentation" + title: "Update generated documentation" + body: | + This PR updates the generated documentation based on the latest local repository data. + + - Updates EDOT Collector component tables + - Updates OpenTelemetry Collector Builder (OCB) configuration + - Uses local version from version.go file + + This is an automated PR created by the documentation update workflow. + branch: update-docs + base: main + delete-branch: true + labels: | + documentation + automated-pr diff --git a/.gitignore b/.gitignore index 8e1dd040e4c..c0dbe41b62b 100644 --- a/.gitignore +++ b/.gitignore @@ -27,6 +27,8 @@ fleet.yml fleet.enc fleet.enc.lock env.sh +.artifacts +/docs/.artifacts # Editor swap files *.swp diff --git a/docs/docset.yml b/docs/docset.yml index 2a04e9a92cb..8691d789f94 100644 --- a/docs/docset.yml +++ b/docs/docset.yml @@ -7,8 +7,11 @@ exclude: cross_links: - docs-content - beats + - opentelemetry + - ecs toc: - toc: release-notes + - toc: reference/edot-collector subs: agent: "Elastic Agent" agent-issue: "https://github.com/elastic/elastic-agent/issues/" @@ -17,3 +20,14 @@ subs: beats: "Beats" es: "Elasticsearch" fleet: "Fleet" + motlp: Elastic Cloud Managed OTLP Endpoint + edot: Elastic Distribution of OpenTelemetry + ecloud: "Elastic Cloud" + edot-cf: "EDOT Cloud Forwarder" + ech: "Elastic Cloud Hosted" + ess: "Elasticsearch Service" + ece: "Elastic Cloud Enterprise" + serverless-full: "Elastic Cloud Serverless" + stack: "Elastic Stack" + kib: "Kibana" + ls: "Logstash" \ No newline at end of file diff --git a/docs/reference/edot-collector/_snippets/edot-collector-auth.md b/docs/reference/edot-collector/_snippets/edot-collector-auth.md new file mode 100644 index 00000000000..560421291de --- /dev/null +++ b/docs/reference/edot-collector/_snippets/edot-collector-auth.md @@ -0,0 +1,22 @@ +```yaml +extensions: + bearertokenauth: + scheme: "APIKey" + token: "" + + apmconfig: + opamp: + protocols: + http: + # Default is localhost:4320 + # endpoint: "" + source: + elasticsearch: + endpoint: "" + auth: + authenticator: bearertokenauth +``` + +:::{note} +For comprehensive authentication configuration options, see [Authentication methods](/reference/edot-collector/config/authentication-methods.md). +::: \ No newline at end of file diff --git a/docs/reference/edot-collector/_snippets/guided-instructions.md b/docs/reference/edot-collector/_snippets/guided-instructions.md new file mode 100644 index 00000000000..877754b1409 --- /dev/null +++ b/docs/reference/edot-collector/_snippets/guided-instructions.md @@ -0,0 +1,6 @@ +Use the **Add data** screen in Elastic Observability to generate install commands that are already configured with the values you need. + +1. Open Elastic Observability. +2. Go to **Add data**. +3. Select what you want to monitor. +4. Follow the instructions. \ No newline at end of file diff --git a/docs/reference/edot-collector/_snippets/process-config.md b/docs/reference/edot-collector/_snippets/process-config.md new file mode 100644 index 00000000000..ef316f3520a --- /dev/null +++ b/docs/reference/edot-collector/_snippets/process-config.md @@ -0,0 +1,17 @@ +Process metrics are turned off by default to avoid generating a large volume of timeseries data. To turn on process metrics, uncomment or add the following section inside the `hostmetrics` receiver configuration: + +```yaml + process: + mute_process_exe_error: true + mute_process_io_error: true + mute_process_user_error: true + metrics: + process.threads: + enabled: true + process.open_file_descriptors: + enabled: true + process.memory.utilization: + enabled: true + process.disk.operations: + enabled: true +``` \ No newline at end of file diff --git a/docs/reference/edot-collector/_snippets/retrieve-credentials.md b/docs/reference/edot-collector/_snippets/retrieve-credentials.md new file mode 100644 index 00000000000..30c636e0a37 --- /dev/null +++ b/docs/reference/edot-collector/_snippets/retrieve-credentials.md @@ -0,0 +1,9 @@ +Retrieve your {{es}} URL and your API key: + +1. Retrieve the {{es}} URL for your {{ecloud}} deployment: + + 1. Go to the [{{ecloud}} console](https://cloud.elastic.co/). + 2. Next to your deployment, select **Manage**. + 3. Under **Applications** next to **{{es}}**, select **Copy endpoint**. + +2. Create an API Key following [these instructions](docs-content://deploy-manage/api-keys/elasticsearch-api-keys.md). \ No newline at end of file diff --git a/docs/reference/edot-collector/_snippets/serverless-endpoint-api.md b/docs/reference/edot-collector/_snippets/serverless-endpoint-api.md new file mode 100644 index 00000000000..b36a1151aee --- /dev/null +++ b/docs/reference/edot-collector/_snippets/serverless-endpoint-api.md @@ -0,0 +1,7 @@ +Follow these steps to retrieve the managed OTLP endpoint URL for your Serverless project: + +1. In {{serverless-full}}, open your Observability project. +2. Go to **Add data** → **Application** → **OpenTelemetry**. +3. Select **Managed OTLP Endpoint** in the second step. +4. Copy the OTLP endpoint configuration value. +5. Select **Create API Key** to generate an API key. \ No newline at end of file diff --git a/docs/reference/edot-collector/component-docs.yml b/docs/reference/edot-collector/component-docs.yml new file mode 100644 index 00000000000..af6ed79fccc --- /dev/null +++ b/docs/reference/edot-collector/component-docs.yml @@ -0,0 +1,27 @@ +# Component Documentation Registry +# Maps component names to their documentation files +# Used by tools.py to generate links to local documentation + +components: + # Exporters + elasticsearchexporter: + doc_path: "/reference/edot-collector/components/elasticsearchexporter.md" + title: "Elasticsearch exporter" + + # Receivers + elasticapmintakereceiver: + doc_path: "/reference/edot-collector/components/elasticapmintakereceiver.md" + title: "Elastic APM intake receiver" + + apikeyauthextension: + doc_path: "/reference/edot-collector/config/authentication-methods.md" + title: "Authentication methods" + + bearertokenauthextension: + doc_path: "/reference/edot-collector/config/authentication-methods.md" + title: "Authentication methods" + + # Add more components as documentation is created + # elasticapmconnector: + # doc_path: "/reference/edot-collector/components/elasticapmconnector" + # title: "Elastic APM connector" diff --git a/docs/reference/edot-collector/components.md b/docs/reference/edot-collector/components.md new file mode 100644 index 00000000000..4c454b07d78 --- /dev/null +++ b/docs/reference/edot-collector/components.md @@ -0,0 +1,94 @@ +--- +navigation_title: Components +description: List of components included in the EDOT Collector, categorized as Core or Extended. +applies_to: + stack: + serverless: + observability: + product: + edot_collector: ga +products: + - id: cloud-serverless + - id: observability + - id: edot-collector +--- + +# Components included in the EDOT Collector + +The {{edot}} (EDOT) Collector includes embedded Collector components from the [OTel Collector Core](https://github.com/open-telemetry/opentelemetry-collector), +[OTel Collector Contrib](https://github.com/open-telemetry/opentelemetry-collector-contrib) and the [Elastic Collector Components](https://github.com/elastic/opentelemetry-collector-components) repositories. + +The components included in the EDOT Collector are categorized into **[Core]** and **[Extended]** components. The following table describes the current components included in the EDOT Collector, their source, and support status. + +% start:edot-collector-components-table +| Component | GitHub Repo | Support status | Version | +|:---|:---|:---|:---| +|***Receivers***|||| +| [dockerstatsreceiver ](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/receiver/dockerstatsreceiver) | [OTel Contrib Repo](https://github.com/open-telemetry/opentelemetry-collector-contrib) | [Extended] | v0.130.0 | +| [elasticapmintakereceiver ](/reference/edot-collector/components/elasticapmintakereceiver.md) | [Elastic Repo](https://github.com/elastic/opentelemetry-collector-components) | [Core] | v0.2.1 | +| [filelogreceiver ](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/receiver/filelogreceiver) | [OTel Contrib Repo](https://github.com/open-telemetry/opentelemetry-collector-contrib) | [Core] | v0.130.0 | +| [hostmetricsreceiver ](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/receiver/hostmetricsreceiver) | [OTel Contrib Repo](https://github.com/open-telemetry/opentelemetry-collector-contrib) | [Core] | v0.130.0 | +| [httpcheckreceiver ](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/receiver/httpcheckreceiver) | [OTel Contrib Repo](https://github.com/open-telemetry/opentelemetry-collector-contrib) | [Extended] | v0.130.0 | +| [jaegerreceiver ](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/receiver/jaegerreceiver) | [OTel Contrib Repo](https://github.com/open-telemetry/opentelemetry-collector-contrib) | [Extended] | v0.130.0 | +| [jmxreceiver ](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/receiver/jmxreceiver) | [OTel Contrib Repo](https://github.com/open-telemetry/opentelemetry-collector-contrib) | [Extended] | v0.130.0 | +| [k8sclusterreceiver ](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/receiver/k8sclusterreceiver) | [OTel Contrib Repo](https://github.com/open-telemetry/opentelemetry-collector-contrib) | [Core] | v0.130.0 | +| [k8sobjectsreceiver ](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/receiver/k8sobjectsreceiver) | [OTel Contrib Repo](https://github.com/open-telemetry/opentelemetry-collector-contrib) | [Core] | v0.130.0 | +| [kafkareceiver ](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/receiver/kafkareceiver) | [OTel Contrib Repo](https://github.com/open-telemetry/opentelemetry-collector-contrib) | [Extended] | v0.130.0 | +| [kubeletstatsreceiver ](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/receiver/kubeletstatsreceiver) | [OTel Contrib Repo](https://github.com/open-telemetry/opentelemetry-collector-contrib) | [Core] | v0.130.0 | +| [nginxreceiver ](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/receiver/nginxreceiver) | [OTel Contrib Repo](https://github.com/open-telemetry/opentelemetry-collector-contrib) | [Extended] | v0.130.0 | +| [nopreceiver ](https://github.com/open-telemetry/opentelemetry-collector/tree/main/receiver/nopreceiver) | [OTel Core Repo](https://github.com/open-telemetry/opentelemetry-collector) | [Extended] | v0.130.0 | +| [otlpreceiver ](https://github.com/open-telemetry/opentelemetry-collector/tree/main/receiver/otlpreceiver) | [OTel Core Repo](https://github.com/open-telemetry/opentelemetry-collector) | [Core] | v0.130.0 | +| [prometheusreceiver ](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/receiver/prometheusreceiver) | [OTel Contrib Repo](https://github.com/open-telemetry/opentelemetry-collector-contrib) | [Extended] | v0.130.0 | +| [receivercreator ](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/receiver/receivercreator) | [OTel Contrib Repo](https://github.com/open-telemetry/opentelemetry-collector-contrib) | [Extended] | v0.130.0 | +| [redisreceiver ](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/receiver/redisreceiver) | [OTel Contrib Repo](https://github.com/open-telemetry/opentelemetry-collector-contrib) | [Extended] | v0.130.0 | +| [zipkinreceiver ](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/receiver/zipkinreceiver) | [OTel Contrib Repo](https://github.com/open-telemetry/opentelemetry-collector-contrib) | [Extended] | v0.130.0 | +|***Exporters***|||| +| [debugexporter ](https://github.com/open-telemetry/opentelemetry-collector/tree/main/exporter/debugexporter) | [OTel Core Repo](https://github.com/open-telemetry/opentelemetry-collector) | [Extended] | v0.132.0 | +| [elasticsearchexporter ](/reference/edot-collector/components/elasticsearchexporter.md) | [OTel Contrib Repo](https://github.com/open-telemetry/opentelemetry-collector-contrib) | [Core] | v0.132.0 | +| [fileexporter ](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/exporter/fileexporter) | [OTel Contrib Repo](https://github.com/open-telemetry/opentelemetry-collector-contrib) | [Extended] | v0.130.0 | +| [kafkaexporter ](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/exporter/kafkaexporter) | [OTel Contrib Repo](https://github.com/open-telemetry/opentelemetry-collector-contrib) | [Extended] | v0.130.0 | +| [loadbalancingexporter ](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/exporter/loadbalancingexporter) | [OTel Contrib Repo](https://github.com/open-telemetry/opentelemetry-collector-contrib) | [Extended] | v0.130.0 | +| [nopexporter ](https://github.com/open-telemetry/opentelemetry-collector/tree/main/exporter/nopexporter) | [OTel Core Repo](https://github.com/open-telemetry/opentelemetry-collector) | [Extended] | v0.130.0 | +| [otlpexporter ](https://github.com/open-telemetry/opentelemetry-collector/tree/main/exporter/otlpexporter) | [OTel Core Repo](https://github.com/open-telemetry/opentelemetry-collector) | [Core] | v0.130.0 | +| [otlphttpexporter ](https://github.com/open-telemetry/opentelemetry-collector/tree/main/exporter/otlphttpexporter) | [OTel Core Repo](https://github.com/open-telemetry/opentelemetry-collector) | [Core] | v0.130.0 | +|***Processors***|||| +| [attributesprocessor ](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/processor/attributesprocessor) | [OTel Contrib Repo](https://github.com/open-telemetry/opentelemetry-collector-contrib) | [Core] | v0.130.0 | +| [batchprocessor ](https://github.com/open-telemetry/opentelemetry-collector/tree/main/processor/batchprocessor) | [OTel Core Repo](https://github.com/open-telemetry/opentelemetry-collector) | [Core] | v0.130.0 | +| [cumulativetodeltaprocessor ](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/processor/cumulativetodeltaprocessor) | [OTel Contrib Repo](https://github.com/open-telemetry/opentelemetry-collector-contrib) | [Extended] | v0.130.0 | +| [elasticinframetricsprocessor ](https://github.com/elastic/opentelemetry-collector-components/tree/main/processor/elasticinframetricsprocessor) | [Elastic Repo](https://github.com/elastic/opentelemetry-collector-components) | [Core] | v0.16.0 | +| [elastictraceprocessor ](https://github.com/elastic/opentelemetry-collector-components/tree/main/processor/elastictraceprocessor) | [Elastic Repo](https://github.com/elastic/opentelemetry-collector-components) | [Core] | v0.9.0 | +| [filterprocessor ](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/processor/filterprocessor) | [OTel Contrib Repo](https://github.com/open-telemetry/opentelemetry-collector-contrib) | [Extended] | v0.130.0 | +| [geoipprocessor ](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/processor/geoipprocessor) | [OTel Contrib Repo](https://github.com/open-telemetry/opentelemetry-collector-contrib) | [Extended] | v0.130.0 | +| [k8sattributesprocessor ](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/processor/k8sattributesprocessor) | [OTel Contrib Repo](https://github.com/open-telemetry/opentelemetry-collector-contrib) | [Core] | v0.130.0 | +| [memorylimiterprocessor ](https://github.com/open-telemetry/opentelemetry-collector/tree/main/processor/memorylimiterprocessor) | [OTel Core Repo](https://github.com/open-telemetry/opentelemetry-collector) | [Extended] | v0.130.0 | +| [resourcedetectionprocessor ](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/processor/resourcedetectionprocessor) | [OTel Contrib Repo](https://github.com/open-telemetry/opentelemetry-collector-contrib) | [Core] | v0.130.0 | +| [resourceprocessor ](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/processor/resourceprocessor) | [OTel Contrib Repo](https://github.com/open-telemetry/opentelemetry-collector-contrib) | [Core] | v0.130.0 | +| [tailsamplingprocessor ](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/processor/tailsamplingprocessor) | [OTel Contrib Repo](https://github.com/open-telemetry/opentelemetry-collector-contrib) | [Extended] | v0.130.0 | +| [transformprocessor ](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/processor/transformprocessor) | [OTel Contrib Repo](https://github.com/open-telemetry/opentelemetry-collector-contrib) | [Core] | v0.130.0 | +|***Connectors***|||| +| [elasticapmconnector ](https://github.com/elastic/opentelemetry-collector-components/tree/main/connector/elasticapmconnector) | [Elastic Repo](https://github.com/elastic/opentelemetry-collector-components) | [Core] | v0.6.0 | +| [forwardconnector ](https://github.com/open-telemetry/opentelemetry-collector/tree/main/connector/forwardconnector) | [OTel Core Repo](https://github.com/open-telemetry/opentelemetry-collector) | [Extended] | v0.130.0 | +| [routingconnector ](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/connector/routingconnector) | [OTel Contrib Repo](https://github.com/open-telemetry/opentelemetry-collector-contrib) | [Core] | v0.130.0 | +| [spanmetricsconnector ](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/connector/spanmetricsconnector) | [OTel Contrib Repo](https://github.com/open-telemetry/opentelemetry-collector-contrib) | [Extended] | v0.130.0 | +|***Extensions***|||| +| [apikeyauthextension ](/reference/edot-collector/config/authentication-methods.md) | [Elastic Repo](https://github.com/elastic/opentelemetry-collector-components) | [Extended] | v0.4.1 | +| [apmconfigextension ](https://github.com/elastic/opentelemetry-collector-components/tree/main/extension/apmconfigextension) | [Elastic Repo](https://github.com/elastic/opentelemetry-collector-components) | [Extended] | v0.6.0 | +| [bearertokenauthextension ](/reference/edot-collector/config/authentication-methods.md) | [OTel Contrib Repo](https://github.com/open-telemetry/opentelemetry-collector-contrib) | [Extended] | v0.130.0 | +| [filestorage ](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/extension/storage/filestorage) | [OTel Contrib Repo](https://github.com/open-telemetry/opentelemetry-collector-contrib) | [Core] | v0.130.0 | +| [healthcheckextension ](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/extension/healthcheckextension) | [OTel Contrib Repo](https://github.com/open-telemetry/opentelemetry-collector-contrib) | [Extended] | v0.130.0 | +| [healthcheckv2extension ](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/extension/healthcheckv2extension) | [OTel Contrib Repo](https://github.com/open-telemetry/opentelemetry-collector-contrib) | [Extended] | v0.130.0 | +| [k8sleaderelector ](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/extension/k8sleaderelector) | [OTel Contrib Repo](https://github.com/open-telemetry/opentelemetry-collector-contrib) | [Extended] | v0.130.0 | +| [k8sobserver ](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/extension/observer/k8sobserver) | [OTel Contrib Repo](https://github.com/open-telemetry/opentelemetry-collector-contrib) | [Extended] | v0.130.0 | +| [memorylimiterextension ](https://github.com/open-telemetry/opentelemetry-collector/tree/main/extension/memorylimiterextension) | [OTel Core Repo](https://github.com/open-telemetry/opentelemetry-collector) | [Extended] | v0.130.0 | +| [pprofextension ](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/extension/pprofextension) | [OTel Contrib Repo](https://github.com/open-telemetry/opentelemetry-collector-contrib) | [Extended] | v0.130.0 | +|***Providers***|||| +| [envprovider ](https://github.com/open-telemetry/opentelemetry-collector/tree/main/confmap/provider/envprovider) | [OTel Core Repo](https://github.com/open-telemetry/opentelemetry-collector) | [Core] | v1.36.0 | +| [fileprovider ](https://github.com/open-telemetry/opentelemetry-collector/tree/main/confmap/provider/fileprovider) | [OTel Core Repo](https://github.com/open-telemetry/opentelemetry-collector) | [Core] | v1.38.0 | +| [httpprovider ](https://github.com/open-telemetry/opentelemetry-collector/tree/main/confmap/provider/httpprovider) | [OTel Core Repo](https://github.com/open-telemetry/opentelemetry-collector) | [Core] | v1.36.0 | +| [httpsprovider ](https://github.com/open-telemetry/opentelemetry-collector/tree/main/confmap/provider/httpsprovider) | [OTel Core Repo](https://github.com/open-telemetry/opentelemetry-collector) | [Core] | v1.35.0 | +| [yamlprovider ](https://github.com/open-telemetry/opentelemetry-collector/tree/main/confmap/provider/yamlprovider) | [OTel Core Repo](https://github.com/open-telemetry/opentelemetry-collector) | [Core] | v1.38.0 | + +% end:edot-collector-components-table + +[Extended]: opentelemetry://reference/compatibility/nomenclature.md#extended-components +[Core]: opentelemetry://reference/compatibility/nomenclature.md#core-components diff --git a/docs/reference/edot-collector/components/elasticapmintakereceiver.md b/docs/reference/edot-collector/components/elasticapmintakereceiver.md new file mode 100644 index 00000000000..5d01ea11e17 --- /dev/null +++ b/docs/reference/edot-collector/components/elasticapmintakereceiver.md @@ -0,0 +1,90 @@ +--- +navigation_title: Elastic APM intake receiver +description: The Elastic APM intake receiver is an OpenTelemetry Collector component that receives APM data from Elastic APM Agents. +applies_to: + stack: ga 9.2 + serverless: + observability: + product: + edot_collector: ga 9.2 +products: + - id: cloud-serverless + - id: observability + - id: edot-collector +--- + +# Elastic APM intake receiver + +The Elastic APM intake receiver is an OpenTelemetry Collector component that receives APM data from classic Elastic APM Agents. The receiver supports the [Elastic Intake v2 protocol](https://github.com/elastic/apm-data/tree/main/input/elasticapm/docs/spec/v2) and behaves like the Elastic APM Server, so that telemetry is stored in the same format and using the same indices while going through the Collector. This allows users of classic APM agents to gradually migrate to OpenTelemetry and adapt their instrumentation to the new OTel-based approach. + +:::{important} +Real user monitoring (RUM) intake and older intake protocols are not supported. +::: + +## Get started + +To use the Elastic APM intake receiver, include it in the receiver definitions of the [Collector configuration](/reference/edot-collector/config/index.md): + +```yaml +receivers: + elasticapmintake: + agent_config: + enabled: false +``` + +## Configuration + +The Elastic APM intake receiver supports standard HTTP server configuration, including TLS/mTLS and authentication. + +### TLS and mTLS settings + +You can turn on TLS or mutual TLS to encrypt data in transit between Elastic APM agents and the receiver. Refer to [OpenTelemetry TLS server configuration](https://github.com/open-telemetry/opentelemetry-collector/blob/main/config/configtls/README.md#server-configuration) for more details. + +For example: + +```yaml +receivers: + elasticapmintake: + tls: + cert_file: server.crt + key_file: server.key + agent_config: + enabled: false +``` + +Refer to [OpenTelemetry TLS server configuration](https://github.com/open-telemetry/opentelemetry-collector/blob/main/config/configtls/README.md#server-configuration) for more details. + +### Authentication settings + +In addition to TLS, you can configure authentication to make sure that only authorized agents can send data to the receiver. The Elastic APM intake receiver supports any `configauth` authenticator. + +Use the recommended`apikeyauth` extension to validate Elastic APM API keys. For example: + +```yaml +extensions: + apikeyauth: + endpoint: "" + application_privileges: + - application: "apm" + privileges: + - "event:write" + resources: + - "-" +receivers: + elasticapmintake: + auth: + authenticator: apikeyauth + tls: + cert_file: server.crt + key_file: server.key + agent_config: + enabled: false +``` + +### Agent environment variables + +The Elastic APM intake receiver supports the following environment variables: + +- `ELASTIC_APM_API_KEY`: The API key to use for authentication. +- `ELASTIC_APM_SERVER_URL`: The URL of the Elastic APM server. +- `ELASTIC_APM_SERVER_CERT`: The path to the server certificate file. diff --git a/docs/reference/edot-collector/components/elasticsearchexporter.md b/docs/reference/edot-collector/components/elasticsearchexporter.md new file mode 100644 index 00000000000..c25282d4f56 --- /dev/null +++ b/docs/reference/edot-collector/components/elasticsearchexporter.md @@ -0,0 +1,244 @@ +--- +navigation_title: Elasticsearch exporter +description: The Elasticsearch exporter is an OpenTelemetry Collector component that sends telemetry data to Elasticsearch. +applies_to: + stack: + serverless: + observability: + product: + edot_collector: +products: + - id: cloud-serverless + - id: observability + - id: edot-collector +--- + +# Elasticsearch exporter + +The Elasticsearch exporter is an OpenTelemetry Collector component that sends logs, metrics, and traces to {{es}}. The exporter supports multiple mapping modes and provides flexible configuration options for data routing, authentication, and performance tuning. + +## Get started + +To use the Elasticsearch exporter, include it in the exporter definitions of the [Collector configuration](/reference/edot-collector/config/index.md). The exporter is already included in the [default configuration](/reference/edot-collector/config/default-config-standalone.md). + +## Configuration + +The Elasticsearch exporter supports various configuration options for connecting to Elasticsearch, mapping data, and optimizing performance. + +### Connection settings + +You must specify exactly one of the following connection methods: + +- `endpoint`: A single Elasticsearch URL. For example, `https://elasticsearch:9200`. +- `endpoints`: A list of Elasticsearch URLs for round-robin load balancing. +- `cloudid`: An [Elastic Cloud ID](docs-content://deploy-manage/deploy/elastic-cloud/find-cloud-id.md) for connecting to {{ecloud}}. + +If none of the previous settings are specified, the exporter relies on the `ELASTICSEARCH_URL` environment variable. + +### Authentication settings + +The exporter supports standard OpenTelemetry [authentication configuration](https://github.com/open-telemetry/opentelemetry-collector/blob/main/config/configauth/README.md#authentication-configuration). You can also use these simplified authentication options: + +- `user` and `password`: For HTTP Basic Authentication +- `api_key`: For [Elasticsearch API key authentication](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-security-create-api-key) + +### TLS and security settings + +The exporter supports standard OpenTelemetry TLS configuration for secure connections. You can configure TLS certificates, client authentication, and other security settings through the standard [TLS configuration options](https://github.com/open-telemetry/opentelemetry-collector/blob/main/config/configtls/README.md#tls-configuration-settings). + +## Mapping modes + +The exporter supports several mapping modes that determine how your telemetry data is preprocessed and stored in Elastic. You can configure the mapping mode through the `mapping` setting: + +| Setting | Default | Description | +|---------|---------|-------------| +| `mapping::mode` | `otel` | The default mapping mode. Valid modes are: `none`, `ecs`, `otel`, `raw`, `bodymap`. | +| `mapping::allowed_modes` | All mapping modes | A list of allowed mapping modes that can be requested through client metadata or scope attributes. | + +### OTel mapping mode + +```{applies_to} +stack: ga 8.12 +``` + +The default mapping mode is `otel`. In the `otel` mapping mode, the Elasticsearch Exporter stores documents in Elastic's preferred OTel-native schema. In this mapping mode, documents use the original attribute names and closely follow the event structure from the OTLP events. + +### ECS mapping mode + +In `ecs` mapping mode, the Elasticsearch Exporter maps fields from OpenTelemetry Semantic Conventions (SemConv) to Elastic Common Schema (ECS) where possible. This mode can be used for compatibility with existing dashboards that work with ECS. Refer to [ECS & OpenTelemetry](ecs://reference/ecs-opentelemetry.md) for more details. + +### Bodymap mapping mode + +In `bodymap` mapping mode, the Elasticsearch Exporter supports only logs and takes the body of a log record as the exact content of the Elasticsearch document without any transformation. Use this mapping mode when you want to have complete control over the Elasticsearch document structure. + +### None mapping mode + +In the `none` mapping mode the Elasticsearch Exporter produces documents with the original field names from the OTLP data structures. + +### Raw mapping mode + +The `raw` mapping mode is identical to `none`, except for two differences: + + - In `none` mode attributes are mapped with an `Attributes.` prefix, while in `raw` mode they are not. + - In `none` mode span events are mapped with an `Events.` prefix, while in `raw` mode they are not. + +## Document routing + +Documents are statically or dynamically routed to the target index or data stream. The first routing mode that applies is used, in the following order: + +### Static mode + +Static mode routes documents to `logs_index` for log records, `metrics_index` for data points, and `traces_index` for spans, if these configs aren't empty respectively. + +### Dynamic mode (Index attribute) + +Dynamic mode (Index attribute) routes documents to index name specified in `elasticsearch.index` attribute, with the following order of precedence: log record / data point / span attribute -> scope attribute -> resource attribute if the attribute exists. + +### Dynamic mode (Data stream routing) + +Dynamic mode (Data stream routing) routes documents to data stream constructed from `${data_stream.type}-${data_stream.dataset}-${data_stream.namespace}`, +where `data_stream.type` is `logs` for log records, `metrics` for data points, and `traces` for spans, and is static. The following rules apply: + +- `data_stream.dataset` or `data_stream.namespace` in attributes, with the following order of precedence: log record / data point / span attribute -> scope attribute -> resource attribute +- Otherwise, if the scope name matches the `/receiver/(\w*receiver)` regular expression, `data_stream.dataset` is the first capture group. +- Otherwise, `data_stream.dataset` falls back to `generic` and `data_stream.namespace` falls back to `default`. + +If the mapping mode is set to `bodymap`, the `data_stream.type` field can be dynamically set from attributes. The resulting documents contain the corresponding `data_stream.*` fields. Refer to [Data Stream Fields](ecs://reference/ecs-data_stream.md) for the restrictions applied to the data stream fields. + +### Document routing settings + +These settings allow you to customize document routing: + +| Setting | Default | Description | +|---------|---------|-------------| +| `logs_index` | - | The index or data stream name to publish logs (and span events in OTel mapping mode) to. Should be empty unless all logs are to be sent to the same index. | +| `metrics_index` | - | The index or data stream name to publish metrics to. Should be empty unless all metrics should be sent to the same index. | +| `traces_index` | - | The index or data stream name to publish traces to. Should be empty unless all traces should be sent to the same index. | +| `logstash_format::enabled` | `false` | Turns on or off Logstash format compatibility. When active, the index name is composed using the dynamic routing rules as prefix and the date as suffix. For example, `logs-generic-default-YYYY.MM.DD`. | +| `logstash_format::prefix_separator` | `-` | Set a separator between logstash prefix and date. | +| `logstash_format::date_format` | `%Y.%m.%d` | Time format based on strftime to generate the second part of the index name. | +| `logs_dynamic_id::enabled` | `false` | Turns on or off dynamic ID for log records. If `elasticsearch.document_id` exists and isn't empty in log record attributes, it's used as the document ID. Otherwise, Elasticsearch generates the ID. The attribute is removed from the final document when using `otel` mapping mode. | + +### Document routing exceptions + +When using the default OpenTelemetry mapping mode, additional handling is applied to the previous document routing rules: + +1. Static mode: Span events are separate documents routed to `logs_index` if non-empty. +2. Dynamic - Index attribute mode: Span events are separate documents routed using attribute `elasticsearch.index`, with the following order of precedence: span event attribute -> scope attribute -> resource attribute if the attribute exists. +3. Dynamic - Data stream routing mode: For all documents, `data_stream.dataset` always ends with `.otel`. Span events are separate documents that have `data_stream.type: logs` and are routed using data stream attributes, with the following order of precedence: span event attribute -> scope attribute -> resource attribute. + +The `elasticsearch.index` attribute is removed from the final document if it exists. + +## Performance and batching + +The exporter supports both internal batching and OpenTelemetry's standard `sending_queue` configuration: + +### Internal batching (default) + +By default, the exporter performs its own buffering and batching, as configured through the `flush` setting, unless the `sending_queue` and `batcher` settings are defined. + +### Using sending queue + +The Elasticsearch exporter supports the `sending_queue` setting, which supports both queueing and batching. However, the sending queue is currently deactivated by default. You can turn on the sending queue by setting `sending_queue` to true. Batching support in sending queue is also deactivated by default and can be turned on by defining `sending_queue::batch`. For example: + +```yaml subs=true +exporters: + elasticsearch: + endpoint: https://elasticsearch:9200 + sending_queue: + enabled: true + batch: + enabled: true + min_size: 1000 + max_size: 10000 + timeout: 5s +``` + +## Bulk indexing + +The Elasticsearch exporter uses the [Elasticsearch Bulk API](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-bulk) for indexing documents. Configure the behavior of bulk indexing with the following settings: + +| Setting | Default | Description | +|---------|---------|-------------| +| `num_workers` | `runtime.NumCPU()` | Number of workers publishing bulk requests concurrently. Note this isn't applicable if `batcher::enabled` is `true` or `false`. | +| `flush::bytes` | `5000000` | Write buffer flush size limit before compression. A bulk request are sent immediately when its buffer exceeds this limit. This value should be much lower than Elasticsearch's `http.max_content_length` config to avoid HTTP 413 Entity Too Large error. Keep this value under 5 MB. | +| `flush::interval` | `10s` | Write buffer flush time limit. | +| `retry::enabled` | `true` | Turns on or off request retry on error. Failed requests are retried with exponential backoff. | +| `retry::max_requests` | DEPRECATED | Number of HTTP request retries including the initial attempt. If used, `retry::max_retries` is set to `max_requests - 1`. Use `retry::max_retries` instead. | +| `retry::max_retries` | `2` | Number of HTTP request retries. To turn off retries, set `retry::enabled` to `false` instead of setting `max_retries` to `0`. | +| `retry::initial_interval` | `100ms` | Initial waiting time if an HTTP request failed. | +| `retry::max_interval` | `1m` | Max waiting time if an HTTP request failed. | +| `retry::retry_on_status` | `[429]` | Status codes that trigger request or document level retries. Request level retry and document level retry status codes are shared and cannot be configured separately. To avoid duplicates, it defaults to `[429]`. | + +:::{note} +The `flush::interval` config is ignored when `batcher::enabled` config is explicitly set to true or false. +::: + +Starting from Elasticsearch 8.18 and higher, the [`include_source_on_error`](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-bulk#operation-bulk-include_source_on_error) query parameter allows users to receive the source document in the error response if there were parsing errors in the bulk request. In the exporter, the equivalent configuration is also named `include_source_on_error`. + +- `include_source_on_error`: + - `true`: Turns on bulk index responses to include source document on error. {applies_to}`stack: ga 8.18` + - `false`: Turns off including source document on bulk index error responses. {applies_to}`stack: ga 8.18` + - `null` (default): Backward-compatible option for older Elasticsearch versions. By default, the error reason is discarded from bulk index responses entirely. Only the error type is returned. + +:::{warning} +The exporter might log error responses containing request payload, causing potential sensitive data to be exposed in logs. +::: + +## Ingest pipeline support + +Documents can be passed through an [Elasticsearch Ingest pipeline] before indexing. Use these settings to configure the ingest pipeline: + +| Setting | Default | Description | +|---------|---------|-------------| +| `pipeline` | - | ID of an Elasticsearch Ingest pipeline used for processing documents published by the exporter. | +| `logs_dynamic_pipeline::enabled` | `false` | Turn on or off the dynamic pipeline. If `elasticsearch.ingest_pipeline` attribute exists in log record attributes and isn't empty, it's used as the Elasticsearch ingest pipeline. This currently only applies to the log signal. The attribute is removed from the final document when using `otel` mapping mode. | + +For example: + +```yaml subs=true +exporters: + elasticsearch: + endpoint: https://elasticsearch:9200 + pipeline: "my-custom-pipeline" +``` + +## Elasticsearch node discovery + +The Elasticsearch Exporter regularly checks Elasticsearch for available nodes. Newly discovered nodes are automatically used for load balancing. + +The following settings are related to node discovery: + +- `discover`: + - `on_start` (optional): If enabled the exporter queries Elasticsearch + for all known nodes in the cluster on startup. + - `interval` (optional): Interval to update the list of Elasticsearch nodes. + +To turn off node discovery, set `discover.interval` to `0`. + +## Known limitations + +The following are some known limitations of the Elasticsearch exporter: + +- Metrics support is currently in development and might have limitations. +- Profile support requires Universal Profiling to be installed in {{es}}. +- Some mapping modes might have reduced functionality for certain telemetry types. +- The `bodymap` mode only supports logs and ignores other telemetry types. + + +## Known issues + +The following are the main known issues with the Elasticsearch exporter: + +| Issue | Cause | Solution | +|-------|-------|----------| +| **version_conflict_engine_exception** | TSDB data streams require unique documents per timestamp. Occurs with OTel mapping mode on Elasticsearch 8.16+ or ECS mode with system integration streams. | Update to Elasticsearch version 8.17.6 or higher and the Elasticsearch exporter version 0.121.0 or higher, or install a custom component template. Remove batch processors to prevent metric splitting. | +| **flush failed (400) illegal_argument_exception** | OTel mapping mode, which is default from version 0.122.0, requires Elasticsearch 8.12 or higher. | Upgrade Elasticsearch to 8.12 or higher or use alternative mapping modes. | + +## Troubleshooting + +When you encounter issues with the Elasticsearch exporter, you can try the following: + +- Make sure your Elasticsearch version is compatible with your chosen mapping mode. +- Verify your API keys or credentials are valid and have appropriate permissions. +- Check that your Elasticsearch cluster supports the required features for your mapping mode. diff --git a/docs/reference/edot-collector/config/authentication-methods.md b/docs/reference/edot-collector/config/authentication-methods.md new file mode 100644 index 00000000000..7513be12589 --- /dev/null +++ b/docs/reference/edot-collector/config/authentication-methods.md @@ -0,0 +1,241 @@ +--- +navigation_title: Authentication methods +description: Learn how to configure authentication for the EDOT Collector using API key authentication, bearer token authentication, and other security methods. +applies_to: + stack: ga + serverless: ga + product: + edot_collector: ga +products: + - id: cloud-serverless + - id: observability + - id: edot-collector +--- + +# Authentication methods for the EDOT Collector + +The {{edot}} Collector supports multiple authentication methods to secure connections and ensure only authorized clients can send telemetry data. This guide covers the available authentication extensions and how to configure them. + +## Overview + +Authentication in the EDOT Collector is handled through extensions that implement the `extensionauth` interface. These extensions can be used to: + +- Authenticate incoming requests from SDKs and other collectors. +- Authenticate outgoing requests to external services. + +## Available authentication extensions + +The EDOT Collector supports the following authentication extensions: + +### Elastic API Key Authentication (`apikeyauth`) + +The `apikeyauth` extension is an Elastic-specific authentication method that validates Elasticsearch API keys against your {{es}} cluster. This extension is ideal for authenticating requests from EDOT SDKs and other Collectors that use Elasticsearch API keys. + +### Bearer Token Authentication (`bearertokenauth`) + +The `bearertokenauth` extension is an contrib OpenTelemetry authentication method that supports static bearer tokens. This extension is useful for token-based authentication scenarios. + +## Configuration examples + +These examples show how to configure the `apikeyauth` and `bearertokenauth` extensions. + +### Elastic API key authentication + +Configure the `apikeyauth` extension to authenticate incoming requests: + +```yaml subs=true +extensions: + apikeyauth: + endpoint: "https://example.com:9200" + application_privileges: + - application: "apm" + privileges: ["config_agent:read"] + resources: ["*"] + cache: + capacity: 1000 + ttl: "5m" + pbkdf2_iterations: 10000 + key_headers: ["X-Tenant-Id"] + +receivers: + otlp: + protocols: + grpc: + auth: + authenticator: apikeyauth + http: + auth: + authenticator: apikeyauth + +service: + extensions: [apikeyauth] +``` + +#### Configuration options + +The following configuration options are available for the `apikeyauth` extension: + +| Option | Type | Description | Default | +|--------|------|-------------|---------| +| `endpoint` | string | The Elasticsearch endpoint for API key validation | Required | +| `application_privileges` | array | List of required application privileges and resources | Required | +| `application_privileges.application` | string | Name of the application for which privileges are defined | `""` | +| `application_privileges.privileges` | array | List of application-specific privileges that the API Key must have to be considered valid | `[]` | +| `application_privileges.resources` | array | List of application-specific resources that the API Key must have access to be considered valid | `[]` | +| `cache.capacity` | integer | Maximum number of cached entries | 1000 | +| `cache.ttl` | duration | Time-to-live for cached entries | 30s | +| `cache.pbkdf2_iterations` | integer | Number of PBKDF2 iterations for key derivation | 10000 | +| `cache.key_headers` | array | Optional headers to include in cache key generation | `[]` | + +### Bearer token authentication + +Configure the `bearertokenauth` extension for bearer token-based authentication: + +```yaml subs=true +extensions: + bearertokenauth: + scheme: "Bearer" + token: "your-secret-token" + header: "Authorization" + +receivers: + otlp: + protocols: + grpc: + auth: + authenticator: bearertokenauth + http: + auth: + authenticator: bearertokenauth + +service: + extensions: [bearertokenauth] +``` + +#### Configuration options + +The following configuration options are available for the `bearertokenauth` extension: + +| Option | Type | Description | Default | +|--------|------|-------------|---------| +| `scheme` | string | Authentication scheme | "Bearer" | +| `token` | string | Static token for authentication. Only required if `tokens` is an empty list and `filename` is empty. | `""` | +| `tokens` | string array | List of multiple tokens. Only required if `token` and `filename` are empty. | `[]` | +| `filename` | string | Path to file containing the token, required if `token` and `tokens` are left unset. | `""` | +| `header` | string | Custom header name | "Authorization" | + +#### File-based token storage + +For enhanced security, store tokens in files instead of configuration: + +```yaml subs=true +extensions: + bearertokenauth: + scheme: "Bearer" + filename: "/path/to/token/file" + header: "Authorization" + +service: + extensions: [bearertokenauth] +``` + +The extension automatically monitors the token file for changes and reloads the token when the file is modified. + +## Use cases + +These use cases show how to configure the `apikeyauth` and `bearertokenauth` extensions for different scenarios. + +### Authenticating EDOT SDKs + +When using EDOT SDKs, configure the `apikeyauth` extension to validate API keys: + +```yaml subs=true +extensions: + apikeyauth: + endpoint: "${ELASTIC_ENDPOINT}" + application_privileges: + - application: "apm" + privileges: ["config_agent:read"] + resources: ["*"] + +receivers: + otlp: + protocols: + grpc: + auth: + authenticator: apikeyauth + http: + auth: + authenticator: apikeyauth +``` + +### Securing collector-to-collector communication + +Use bearer token authentication for secure communication between collectors: + +```yaml subs=true +extensions: + bearertokenauth: + scheme: "Collector" + token: "collector-secret-token" + +receivers: + otlp: + protocols: + grpc: + auth: + authenticator: bearertokenauth +``` + +### Multi-tenant authentication + +For multi-tenant environments, use the `apikeyauth` extension with tenant-specific headers: + +```yaml subs=true +extensions: + apikeyauth: + endpoint: "${ELASTIC_ENDPOINT}" + application_privileges: + - application: "apm" + privileges: ["config_agent:read"] + resources: ["*"] + cache: + key_headers: ["X-Tenant-Id", "X-Organization-Id"] +``` + +## Security considerations + +In general, be aware of the following security considerations: + +### API Key security + +- Store API keys securely using environment variables or secret management systems. +- Use the minimum required privileges for API keys. +- Regularly rotate API keys. +- Monitor API key usage and access patterns. + +### Token security + +- Use strong, randomly generated tokens. +- Store tokens in secure files with appropriate permissions. +- Avoid hardcoding tokens in configuration files. +- Consider using token rotation mechanisms. + +## Troubleshooting + +The following issues might occur. + +:::{dropdown} API key validation failures +- Verify the Elasticsearch endpoint is accessible. +- Check API key permissions and application privileges. +- Ensure the API key is valid and not expired. +- Verify network connectivity and firewall rules. +::: + +:::{dropdown} Bearer token authentication failures +- Confirm the token is correct and not expired. +- Check the authentication scheme matches expectations. +- Verify the token file exists and is readable. +- Ensure the custom header is properly configured. +::: + diff --git a/docs/reference/edot-collector/config/configure-logs-collection.md b/docs/reference/edot-collector/config/configure-logs-collection.md new file mode 100644 index 00000000000..e3f0a8a20ac --- /dev/null +++ b/docs/reference/edot-collector/config/configure-logs-collection.md @@ -0,0 +1,395 @@ +--- +navigation_title: Configure Logs Collection +description: Learn how to configure and customize logs collection through the Elastic Distribution of OpenTelemetry Collector. +applies_to: + stack: + serverless: + observability: + product: + edot_collector: ga +products: + - id: cloud-serverless + - id: observability + - id: edot-collector +--- + +# Configure logs collection + +Learn how to configure and customize logs collection through the {{edot}} Collector. + +:::{note} +{{es}} Ingest Pipelines are not yet applicable to OTel-native data. Use OTel Collector processing pipelines for pre-processing and parsing of logs. +::: + +## Parse JSON logs + +You can parse logs that come in JSON format through +[filelog](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/v0.123.0/receiver/filelogreceiver/README.md) +receiver's operators. Use the [`router`](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/v0.123.0/pkg/stanza/docs/operators/router.md) to check if the format is JSON and route the logs to [`json-parser`](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/v0.123.0/pkg/stanza/docs/operators/json_parser.md). For example: + +```yaml +# ... +receivers: + filelog: + # ... + operators: + # Check if format is json and route properly + - id: get-format + routes: + - expr: body matches "^\\{" + output: json-parser + type: router + # Parse body as JSON https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/pkg/stanza/docs/operators/json_parser.md + - type: json_parser + id: json-parser + on_error: send_quiet + parse_from: body + parse_to: body + + # ... +``` + +## Parse multiline logs + +You can parse multiline logs using the +[`multiline`](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/v0.123.0/receiver/filelogreceiver/README.md#multiline-configuration) +option as in the following example: + +```yaml +receivers: + filelog: + include: + - /var/log/example/multiline.log + multiline: + line_start_pattern: ^Exception +``` + +The previous configuration can parse the following logs that span across multiple lines and recombine them properly into one single log message: + +``` +Exception in thread 1 "main" java.lang.NullPointerException + at com.example.myproject.Book.getTitle(Book.java:16) + at com.example.myproject.Author.getBookTitles(Author.java:25) + at com.example.myproject.Bootstrap.main(Bootstrap.java:14) +Exception in thread 2 "main" java.lang.NullPointerException + at com.example.myproject.Book.getTitle(Book.java:16) + at com.example.myproject.Author.getBookTitles(Author.java:25) + at com.example.myproject.Bootstrap.main(Bootstrap.java:44) +``` + +## Parse OTLP logs in JSON format + +You can configure applications instrumented with OpenTelemetry SDKs to write their logs in `OTLP/JSON` format in files stored on disk. The [filelog](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/v0.123.0/receiver/filelogreceiver/README.md) receiver can then collect and parse the logs and +forward them to the [`otlpjson`](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/v0.123.0/connector/otlpjsonconnector) +connector, which extracts the `OTLP` logs from the `OTLP/JSON` log lines. + +An example `OTLP/JSON` log is the following: + +```json +{ + "resourceLogs": [ + { + "resource": { + "attributes": [ + { + "key": "deployment.environment.name", + "value": { + "stringValue": "staging" + } + }, + { + "key": "service.instance.id", + "value": { + "stringValue": "6ad88e10-238c-4fb7-bf97-38df19053366" + } + }, + { + "key": "service.name", + "value": { + "stringValue": "checkout" + } + }, + { + "key": "service.namespace", + "value": { + "stringValue": "shop" + } + }, + { + "key": "service.version", + "value": { + "stringValue": "1.1" + } + } + ] + }, + "scopeLogs": [ + { + "scope": { + "name": "com.mycompany.checkout.CheckoutServiceServer$CheckoutServiceImpl", + "attributes": [] + }, + "logRecords": [ + { + "timeUnixNano": "1730435085776869000", + "observedTimeUnixNano": "1730435085776944000", + "severityNumber": 9, + "severityText": "INFO", + "body": { + "stringValue": "Order order-12035 successfully placed" + }, + "attributes": [ + { + "key": "customerId", + "value": { + "stringValue": "customer-49" + } + }, + { + "key": "thread.id", + "value": { + "intValue": "44" + } + }, + { + "key": "thread.name", + "value": { + "stringValue": "grpc-default-executor-1" + } + } + ], + "flags": 1, + "traceId": "42de1f0dd124e27619a9f3c10bccac1c", + "spanId": "270984d03e94bb8b" + } + ] + } + ], + "schemaUrl": "https://opentelemetry.io/schemas/1.24.0" + } + ] +} +``` + +You can use the following configuration to properly parse and extract the `OTLP` content from these log lines: + +```yaml +receivers: + filelog/otlpjson: + include: [/path/to/myapp/otlpjson.log] + +connectors: + otlpjson: + +service: + pipelines: + logs/otlpjson: + receivers: [filelog/otlpjson] + processors: [] + exporters: [otlpjson] + logs: + receivers: [otlp, otlpjson] + processors: [] + exporters: [debug] +... +``` + +## Parse apache logs + +You can parse logs of a known technology, like Apache logs, through filelog receiver's operators. Use the [`regex_parser`](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/v0.123.0/pkg/stanza/docs/operators/regex_parser.md) operator to parse the logs that follow the specific pattern: + +```yaml +receivers: + # Receiver to read the Apache logs + filelog: + include: + - /var/log/*apache*.log + start_at: end + operators: + # Operator to parse the Apache logs + # This operator uses a regex to parse the logs + - id: apache-logs + type: regex_parser + regex: ^(?P\d+\.\d+.\d+\.\d+)\s+-\s+-\s+\[(?P\d+/\w+/\d+:\d+:\d+:\d+\s+\+\d+)\]\s"(?P\w+)\s+(?P.*)\s+(?P.*)"\s+(?P\d+)\s+(?P\d+)$ +``` + +## Customize logs parsing on Kubernetes + +The OpenTelemetry Collector also supports dynamic logs collection for Kubernetes Pods by defining Pods annotations. For detailed examples refer to [Dynamic workload discovery on Kubernetes now supported with EDOT Collector](https://www.elastic.co/observability-labs/blog/k8s-discovery-with-EDOT-collector) and the +[Collector documentation](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/receiver/receivercreator/README.md#supported-logs-annotations). + +Make sure that the Collector configuration includes the [`k8s_observer`](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/v0.123.0/extension/observer/k8sobserver) and the [`receiver_creator`](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/v0.123.0/receiver/receivercreator): + +```yaml +receivers: + receiver_creator/logs: + watch_observers: [k8s_observer] + discovery: + enabled: true + receivers: + +# ... + +extensions: + k8s_observer: + +# ... + +service: + extensions: [k8s_observer] + pipelines: + logs: + receivers: [ receiver_creator/logs] +``` + +In addition, make sure to remove or comment out any static filelog receiver. Restrict the log file pattern to avoid log duplication. + +Annotating the pod activates custom log collection targeted only for the specific Pod. + +```yaml +# ... +metadata: + annotations: + io.opentelemetry.discovery.logs/enabled: "true" + io.opentelemetry.discovery.logs/config: | + operators: + - id: container-parser + type: container + # Check if format is json and route properly + - id: get-format + routes: + - expr: body matches "^\\{" + output: json-parser + type: router + - id: json-parser + type: json_parser + on_error: send_quiet + parse_from: body + parse_to: body + - id: custom-value + type: add + field: attributes.tag + value: custom-value +spec: + containers: + # ... +``` + +Targeting a single container's scope is also possible by scoping the annotation using containers' names, like `io.opentelemetry.discovery.logs.my-container/enabled: "true"`. Refer to the [Collector's documentation](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/receiver/receivercreator/README.md#supported-logs-annotations) for additional information. + +### Collect Apache logs using annotations discovery + +Use the following example to collect and parse Apache logs by annotating Apache containers: + +```yaml +metadata: + annotations: + io.opentelemetry.discovery.logs.apache/enabled: "true" + io.opentelemetry.discovery.logs.apache/config: | + operators: + - type: container + id: container-parser + - id: apache-logs + type: regex_parser + regex: ^(?P\d+\.\d+.\d+\.\d+)\s+-\s+-\s+\[(?P\d+/\w+/\d+:\d+:\d+:\d+\s+\+\d+)\]\s"(?P\w+)\s+(?P.*)\s+(?P.*)"\s+(?P\d+)\s+(?P\d+)$ +spec: + containers: + - name: apache + # ... +``` + +## Use processors and OTTL for logs processing + +You can use [OpenTelemetry Transform Language (OTTL)](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/pkg/ottl/README.md) functions in the transform processor to parse logs of a specific format or logs that follow a specific pattern. + +### Parse JSON logs using OTTL + +Use the following [`transform`](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/v0.123.0/processor/transformprocessor) +processor configuration to parse logs in `JSON` format: + +```yaml +processors: + transform: + error_mode: ignore + log_statements: + - context: log + statements: + # Parse body as JSON and merge the resulting map with the cache map, ignoring non-json bodies. + # cache is a field exposed by OTTL that is a temporary storage place for complex operations. + - merge_maps(cache, ParseJSON(body), "upsert") where IsMatch(body, "^\\{") + - set(body,cache["log"]) where cache["log"] != nil +``` + +### Parse Apache logs using OTTL + +The following configuration can parse Apache access log using OTTL and the transform processor: + +```yaml +exporters: + debug: + verbosity: detailed +receivers: + filelog: + include: + - /Users/chrismark/otelcol/log/apache.log + + +processors: + transform/apache_logs: + error_mode: ignore + log_statements: + - context: log + statements: + - 'merge_maps(attributes, ExtractPatterns(body, "^(?P\\d+\\.\\d+.\\d+\\.\\d+)\\s+-\\s+-\\s+\\[(?P\\d+/\\w+/\\d+:\\d+:\\d+:\\d+\\s+\\+\\d+)\\]\\s\"(?P\\w+)\\s+(?P.*)\\s+(?P.*)\"\\s+(?P\\d+)\\s+(?P\\d+)$"), "upsert")' +service: + pipelines: + logs: + receivers: [filelog] + processors: [transform/apache_logs] + exporters: [debug] + + +``` + +A more detailed example about using OTTL and the transform processor can be found at the +[nginx_ingress_controller_otel](https://github.com/elastic/integrations/blob/main/packages/nginx_ingress_controller_otel/docs/README.md) +integration. + +## Exclude paths from logs collection [exclude-logs-paths] + +To exclude specific paths from logs collection, use the `exclude` field in the `logs` pipeline configuration. Exclude patterns are applied against the paths matched by include patterns. For example: + +::::{tab-set} +:::{tab-item} Standalone +```yaml +receivers: + # Receiver for platform specific log files + filelog/platformlogs: + include: [/var/log/*.log] + retry_on_failure: + enabled: true + start_at: end + storage: file_storage + exclude: + # Paths support glob patterns + - /var/log/ignore_this.log + - /var/log/another_path/* +``` +::: + +:::{tab-item} Kubernetes +```yaml +mode: daemonset + +presets: + logsCollection: + enabled: true +config: + receivers: + filelog: + exclude: + # Paths support glob patterns + - /var/log/pods/my-nodejs-app-namespace_my-nodejs-app-pod-name_*/*/*.log +``` +::: +:::: diff --git a/docs/reference/edot-collector/config/configure-metrics-collection.md b/docs/reference/edot-collector/config/configure-metrics-collection.md new file mode 100644 index 00000000000..654d213ff34 --- /dev/null +++ b/docs/reference/edot-collector/config/configure-metrics-collection.md @@ -0,0 +1,218 @@ +--- +navigation_title: Configure Metrics Collection +description: Learn how to configure and customize metrics collection through the Elastic Distribution of OpenTelemetry Collector. +applies_to: + stack: + serverless: + observability: + product: + edot_collector: ga +products: + - id: cloud-serverless + - id: observability + - id: edot-collector +--- + +# Configure metrics collection + +Learn how to configure and customize metrics collection through the {{edot}} Collector. + +:::{note} +{{es}} Ingest Pipelines are not yet applicable to OTel-native data. Use OTel Collector processing pipelines for pre-processing metrics. +::: + +## OTLP metrics + +Any application emitting metrics through OpenTelemetry Protocol (OTLP) can forward them to the EDOT Collector using the OTLP receiver. This is the recommended method for collecting application-level telemetry. + +The following OTLP receiver configuration turns on both gRPC and HTTP protocols for incoming OTLP traffic: + +```yaml +# [OTLP Receiver](https://github.com/open-telemetry/opentelemetry-collector/tree/main/receiver/otlpreceiver) +receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 +``` + +Configure your application to export metrics using the OTLP protocol, targeting the endpoints provided in the previous example. + +## Host metrics + +The [hostmetrics receiver](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/receiver/hostmetricsreceiver) turns on the collection of host-level metrics such as CPU use, memory use, and filesystem stats. + +The following configuration collects a standard set of host metrics that aligns with Elastic's Infrastructure dashboards in {{kib}}: + +```yaml +hostmetrics: + collection_interval: 10s + root_path: /proc # Mounted node's root file system + scrapers: + cpu: + metrics: + system.cpu.utilization: + enabled: true + system.cpu.logical.count: + enabled: true + memory: + metrics: + system.memory.utilization: + enabled: true + network: {} + processes: {} + load: {} + disk: {} + filesystem: + exclude_mount_points: + mount_points: + - /dev/* + - /proc/* + - /sys/* + - /run/k3s/containerd/* + - /var/lib/docker/* + - /var/lib/kubelet/* + - /snap/* + match_type: regexp + exclude_fs_types: + fs_types: + - autofs + - binfmt_misc + - bpf + - cgroup2 + - configfs + - debugfs + - devpts + - devtmpfs + - fusectl + - hugetlbfs + - iso9660 + - mqueue + - nsfs + - overlay + - proc + - procfs + - pstore + - rpc_pipefs + - securityfs + - selinuxfs + - squashfs + - sysfs + - tracefs + match_type: strict +``` + +You must grant access to the `/proc` filesystem to the receiver by running the Collector with privileged access and mounting /proc and /sys appropriately. Refer to the hostmetrics container use [guide](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/receiver/hostmetricsreceiver#collecting-host-metrics-from-inside-a-container-linux-only) (Linux only). + +Turning on the process scraper might significantly increase the volume of scraped metrics, potentially impacting performance. Refer to the contrib issue [#39423](https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/39423) for discussion. + +To ensure compatibility with {{kib}}'s Infrastructure dashboards, include the [elasticinframetrics processor](https://github.com/elastic/opentelemetry-collector-components/tree/main/processor/elasticinframetricsprocessor) in your pipeline: + + ```yaml + service: + pipelines: + metrics/infra: + receivers: + - hostmetrics + processors: + - elasticinframetrics + ``` + +### Process metrics + +:::{include} ../_snippets/process-config.md +::: + +## Kubernetes metrics + +You can collect Kubernetes metrics using multiple receivers depending on the type and source of the metrics. Each receiver might require specific Kubernetes permissions and require a deployment as DaemonSets or singletons. + +As with host metrics, use the [elasticinframetrics processor](https://github.com/elastic/opentelemetry-collector-components/tree/main/processor/elasticinframetricsprocessor) to ensure metrics align with the {{kib}} Infrastructure inventory. + +### Kubelet metrics + +The [kubeletstats](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/receiver/kubeletstatsreceiver) receiver collects resource usage stats directly from the Kubelet's /stats/summary endpoint. Stats include pod-level and node-level metrics. + +```yaml +kubeletstats: + auth_type: serviceAccount # Authentication mechanism with the Kubelet endpoint, refer to: https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/receiver/kubeletstatsreceiver#configuration + collection_interval: 20s + endpoint: ${env:OTEL_K8S_NODE_NAME}:10250 + node: '${env:OTEL_K8S_NODE_NAME}' + # Required to work for all CSPs without an issue + insecure_skip_verify: true + k8s_api_config: + auth_type: serviceAccount + metrics: + k8s.pod.memory.node.utilization: + enabled: true + k8s.pod.cpu.node.utilization: + enabled: true + k8s.container.cpu_limit_utilization: + enabled: true + k8s.pod.cpu_limit_utilization: + enabled: true + k8s.container.cpu_request_utilization: + enabled: true + k8s.container.memory_limit_utilization: + enabled: true + k8s.pod.memory_limit_utilization: + enabled: true + k8s.container.memory_request_utilization: + enabled: true + k8s.node.uptime: + enabled: true + k8s.node.cpu.usage: + enabled: true + k8s.pod.cpu.usage: + enabled: true + extra_metadata_labels: + - container.id +``` + +To capture stats from every node in the cluster, deploy the Collector with the kubeletstats receiver as a DaemonSet. + +### Cluster metrics + +The [k8sclusterreceiver](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/receiver/k8sclusterreceiver) gathers metrics and entity events directly from the Kubernetes API server. It captures cluster-wide resources like nodes, deployments, pods, and more. + +```yaml +k8s_cluster: + auth_type: serviceAccount # Determines how to authenticate to the K8s API server. This can be one of none (for no auth), serviceAccount (to use the standard service account token provided to the agent pod), or kubeConfig to use credentials from ~/.kube/config. + node_conditions_to_report: + - Ready + - MemoryPressure + allocatable_types_to_report: + - cpu + - memory + metrics: + k8s.pod.status_reason: + enabled: true + resource_attributes: + k8s.kubelet.version: + enabled: true + os.description: + enabled: true + os.type: + enabled: true + k8s.container.status.last_terminated_reason: + enabled: true +``` + +Run a single instance of this receiver, for example as a Deployment, with sufficient permissions to access the K8s API server. + +## Other metrics + +The EDOT Collector supports a wide range of metrics receivers for popular software systems, including: + + - Redis ([redisreceiver](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/receiver/redisreceiver)): Retrieve Redis INFO data from a single Redis instance. + + - JMX-based applications ([jmxreceiver](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/receiver/jmxreceiver)): Open a child Java process running the JMX Metric Gatherer configured with your specified JMX connection information and target Groovy script. It then reports metrics to an implicitly created OTLP receiver. + + - Prometheus scrape targets ([prometheusreceiver](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/receiver/prometheusreceiver)): Receives metric data in [Prometheus](https://prometheus.io/) format. + + - Kafka ([kafkareceiver](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/receiver/kafkareceiver)): Receives telemetry data from Kafka, with configurable topics and encodings. + +For a full list of supported receivers, see the EDOT Collector components [reference](/reference/edot-collector/components.md). diff --git a/docs/reference/edot-collector/config/default-config-k8s.md b/docs/reference/edot-collector/config/default-config-k8s.md new file mode 100644 index 00000000000..13871706400 --- /dev/null +++ b/docs/reference/edot-collector/config/default-config-k8s.md @@ -0,0 +1,94 @@ +--- +navigation_title: Default config (Kubernetes) +description: Default configuration of the EDOT Collector for Kubernetes. +applies_to: + stack: + serverless: + observability: + product: + edot_collector: ga +products: + - id: cloud-serverless + - id: observability + - id: edot-collector +--- + +# Default configuration of the EDOT Collector (Kubernetes) + +The [Kubernetes setup](docs-content://solutions/observability/get-started/opentelemetry/quickstart/index.md) uses the OpenTelemetry Operator to automate orchestration of EDOT Collectors: + +* [EDOT Collector Cluster](#cluster-collector-pipeline): Collection of cluster metrics. +* [EDOT Collector Daemon](#daemonset-collectors-pipeline): Collection of node metrics, logs and application telemetry. +* [EDOT Collector Gateway](#gateway-collectors-pipeline): Pre-processing, aggregation and ingestion of data into Elastic. + +The following `values.yaml` files are used depending on the ingest scenario: + +* [Direct ingestion into Elasticsearch](https://github.com/elastic/elastic-agent/blob/main/deploy/helm/edot-collector/kube-stack/values.yaml) +* [Managed OTLP Endpoint](https://github.com/elastic/elastic-agent/blob/main/deploy/helm/edot-collector/kube-stack/managed_otlp/values.yaml) + +The following sections describe the default pipelines for the different roles of EDOT collectors in a Kubernetes setup. + +## Cluster Collector pipeline + +The main purpose of the Cluster collector is to collect Kubernetes cluster-level metrics and events using the [`k8s_cluster`] and the [`k8sobjects`] receivers. + +The [`resource`] and [`resourcedetection`] processors enrich the cluster-level data with corresponding meta information. Data then goes to the Gateway Collector through `OTLP`. + +## Daemonset collectors pipeline + +The Daemonset collectors gather telemetry associated with corresponding, individual Kubernetes nodes: + +### Host metrics and container logs + +The [`filelog`] and [`hostmetrics`] receivers are used to gather container logs and host metrics, respectively. The [`kubeletstats`] receiver collects additional Kubernetes Node, Pod and Container metrics. + +Logs and metrics are batched for better performance ([`batch`] processor) and then enriched with meta information using the [`k8sattributes`], [`resourcedetection`] and [`resource`] processors. + +::::{important} +:::{include} ../_snippets/process-config.md +::: +:::: + +:::{note} +The `from_context: client_metadata` option in the `resource` processor only applies to transport-level metadata. It cannot extract custom application attributes. To propagate such values into your telemetry, set them explicitly in your application code using EDOT SDK instrumentation. For more information, refer to [EDOT Collector doesn’t propagate client metadata](docs-content://troubleshoot/ingest/opentelemetry/edot-collector/metadata.md). +::: + +### Application telemetry through OTLP from OTel SDKs + +The Daemonset collectors also receive the application telemetry from OTel SDKs that instrument services and pods running on corresponding Kubernetes nodes. + +The Daemonset collectors receive that data through [`OTLP`], batch the data ([`batch`] processor) and pass it on to the Gateway Collector through the OTLP exporter. + +## Gateway collectors pipeline + +The Gateway collectors pipelines differ between the two different deployment use cases, direct ingestion into {{es}} and using the [{{motlp}}](opentelemetry://reference/motlp.md). + +### Direct ingestion into Elasticsearch + +In self-managed and {{ech}} Stack deployment use cases, the main purpose of the Gateway Collector is the central enrichment of data before the OpenTelemetry data is being ingested directly into {{es}} using the [`elasticsearch`] exporter. + +The Gateway Collector configuration comprises the pipelines for data enrichment of [application telemetry](/reference/edot-collector/config/default-config-standalone.md#application-and-traces-collection-pipeline) and [host metrics](/reference/edot-collector/config/default-config-standalone.md#host-metrics-collection-pipeline). For more details, refer to the linked descriptions of the corresponding standalone use cases. + +The [`routing`] connector separates the infrastructure metrics from other metrics and routes them into the ECS-based pipeline, with ECS-compatibility exporter mode. Other metrics are exported in OTel-native format to {{es}}. + +### Managed OTLP Endpoint + +With the managed OTLP Endpoint, the Gateway Collector configuration pipes all the data from the [`OTLP`] receiver through a [`batch`] processor before the data is being exported through `OTLP` to the managed endpoint. + +With this scenario there's no need to do any Elastic-specific enrichment in your Kubernetes cluster, as all of that happens behind the managed OTLP endpoint. + +[`hostmetrics`]: https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/receiver/hostmetricsreceiver +[`elasticsearch`]: https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/exporter/elasticsearchexporter +[`elasticinframetrics`]: https://github.com/elastic/opentelemetry-collector-components/tree/main/processor/elasticinframetricsprocessor +[`elasticsearch`]: https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/exporter/elasticsearchexporter +[`k8s_cluster`]: https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/receiver/k8sclusterreceiver +[`k8sobjects`]: https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/receiver/k8sobjectsreceiver +[`resource`]: https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/processor/resourceprocessor +[`k8sattributes`]: https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/processor/k8sattributesprocessor +[`resourcedetection`]: https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/processor/resourcedetectionprocessor +[`filelog`]: https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/receiver/filelogreceiver +[`hostmetrics`]: https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/receiver/hostmetricsreceiver +[`kubeletstats`]: https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/receiver/kubeletstatsreceiver +[`batch`]: https://github.com/open-telemetry/opentelemetry-collector/tree/main/processor/batchprocessor +[`OTLP`]: https://github.com/open-telemetry/opentelemetry-collector/tree/main/receiver/otlpreceiver +[`routing`]: https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/connector/routingconnector diff --git a/docs/reference/edot-collector/config/default-config-standalone.md b/docs/reference/edot-collector/config/default-config-standalone.md new file mode 100644 index 00000000000..d3e1f769491 --- /dev/null +++ b/docs/reference/edot-collector/config/default-config-standalone.md @@ -0,0 +1,326 @@ +--- +navigation_title: Default config (Standalone) +description: Default configuration of the EDOT Collector in standalone mode. +applies_to: + stack: + serverless: + observability: + product: + edot_collector: ga +products: + - id: cloud-serverless + - id: observability + - id: edot-collector +--- + +# Default configuration of the EDOT Collector (Standalone) + +The default configuration of the {{edot}} (EDOT) Collector includes pipelines for the collection of logs, host metrics, and data from OpenTelemetry SDKs. + +The EDOT Collector can run in [Agent](https://opentelemetry.io/docs/collector/deployment/agent/) or [Gateway](https://opentelemetry.io/docs/collector/deployment/gateway/) mode: + +- Agent mode: The EDOT Collector ingests data from infrastructure and SDKs and forwards it to Elastic or to another collector running in Gateway mode. +- Gateway mode: The EDOT Collector ingests data from other collectors running in Agent mode and forwards it to Elastic. + +## Agent mode + +The following sample config files for Agent mode are available: + +| Use Cases | Direct ingestion into {{es}} | Managed OTLP Endpoint | +|---|---|---| +| Platform logs | [Logs - ES] | [Logs - OTLP] | +| Platform logs and host metrics | [Logs | Metrics - ES] | [Logs | Metrics - OTLP] | +| Platform logs, host metrics,
and application telemetry | [Logs | Metrics | App - ES]
(*default*) | [Logs | Metrics | App - OTLP]
(*default*) | + +Use the previous example configurations as a reference when configuring your contrib Collector or customizing your EDOT Collector configuration. + +The following sections describe the default pipelines by use cases. + +### Direct ingestion into Elasticsearch + +For self-managed and {{ech}} stack deployment use cases, ingest OpenTelemetry data from the EDOT Collector directly into {{es}} using the [`elasticsearch`] exporter. + +Learn more about the configuration options for the `elasticsearch` exporter in the [corresponding documentation](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/exporter/elasticsearchexporter/README.md#configuration-options). + +The `elasticsearch` exporter comes with two relevant data ingestion modes: + +- `ecs`: Writes data in backwards compatible Elastic Common Schema (ECS) format. Original attribute names and semantics might be lost during translation. +- `otel`: OTel attribute names and semantics are preserved. + +The goal of EDOT is to preserve OTel data formats and semantics as much as possible, so `otel` is the default mode for the EDOT Collector. Some use cases might require data to be exported in ECS format for backwards compatibility. + +#### Logs collection pipeline + +For logs collection, the default configuration uses the [`filelog`] receiver to read log entries from files. In addition, the [`resourcedetection`] processor enriches the log entries with metadata about the corresponding host and operating system. + +:::{note} +The `from_context: client_metadata` option in the `resource` processor only applies to transport-level metadata. It cannot extract custom application attributes. + +To propagate such values into your telemetry, set them explicitly in your application code using EDOT SDK instrumentation. For more information, refer to [EDOT Collector doesn’t propagate client metadata](docs-content://troubleshoot/ingest/opentelemetry/edot-collector/metadata.md). +::: + +Data is exported directly to {{es}} using the [`elasticsearch`] exporter in `OTel-native` mode. + +#### Application and traces collection pipeline + +The application pipeline in the EDOT Collector receives data from OTel SDKs through the [`OTLP`] receiver. While logs and metrics are exported verbatim into {{es}}, traces require two additional components. + +The [`elastictrace`] processor enriches trace data with additional attributes that improve the user experience in the Elastic Observability UIs. In addition, the [`elasticapm`] connector generates pre-aggregated APM metrics from tracing data. + +Application-related OTel data is ingested into {{es}} in OTel-native format using the [`elasticsearch`] exporter. + +:::{note} +Both components, `elastictrace` and `elasticapm` are required for Elastic APM UIs to work properly. As they aren't included in the OpenTelemetry [Collector Contrib repository](https://github.com/open-telemetry/opentelemetry-collector-contrib), you can: + +* Use the EDOT Collector with the available configuration to ingest data into {{es}}. +* [Build a custom, EDOT-like Collector](/reference/edot-collector/custom-collector.md) for ingesting data into {{es}}. +* Use Elastic's [managed OTLP endpoint](docs-content://solutions/observability/get-started/opentelemetry/quickstart/serverless/index.md) that does the enrichment for you. +::: + +#### Host metrics collection pipeline + +The host metrics pipeline uses the [`hostmetrics`] receiver to collect `disk`, `filesystem`, `cpu`, `memory`, `process` and `network` metrics for the corresponding host. + +For backwards compatibility, host metrics are translated into ECS-compatible system metrics using the [`elasticinframetrics`] processor. Finally, metrics are ingested in `ecs` format through the [`elasticsearch`] exporter. + +The [`resourcedetection`] processor enriches the metrics with meta information about the corresponding host and operating system. The [`attributes`] and [`resource`] processor are used to set some fields for proper routing of the ECS-based system metrics data into corresponding {{es}} data streams. + +::::{important} +:::{include} ../_snippets/process-config.md +::: +:::: + +### Using the Managed OTLP Endpoint + +When ingesting OTel data through the [{{motlp}}](opentelemetry://reference/motlp.md), all the enrichment that is required for an optimal experience in the Elastic solutions happens at the endpoint level and is transparent to users. + +The Collector configuration for all use cases that involve the {{motlp}} is only concerned with local data collection and context enrichment. + +Platform logs are scraped with the [`filelog`] receiver, host metrics are collected through the [`hostmetrics`] receiver and both signals are enriched with meta information through the [`resourcedetection`] processor. + +Data from OTel SDKs is piped through the [`OTLP`] receiver directly to the OTLP exporter that sends data for all signals to the {{motlp}}. + +With the {{motlp}}, there is no need to configure any Elastic-specific components, such as [`elasticinframetrics`], [`elastictrace`] processors, the [`elasticapm`] connector, or the [`elasticsearch`] exporter. Edge setup and configuration can be 100% vendor agnostic. + +## Gateway mode + +In Gateway mode, the Collector ingests data from other Collectors running in Agent mode and forwards it to Elastic. + +## Example configuration + +The following example configuration files are available for the Gateway mode: + +| Version | Configuration | +|---------|----------------| +| 8.17 | [Gateway mode](https://raw.githubusercontent.com/elastic/elastic-agent/refs/tags/v9.0.4/internal/pkg/otel/samples/linux/gateway.yml) | +| 8.18 | [Gateway mode](https://raw.githubusercontent.com/elastic/elastic-agent/refs/tags/v9.0.4/internal/pkg/otel/samples/linux/gateway.yml) | +| 9.0 | [Gateway mode](https://raw.githubusercontent.com/elastic/elastic-agent/refs/tags/v9.0.4/internal/pkg/otel/samples/linux/gateway.yml) | +| 8.19 | [Gateway mode](https://raw.githubusercontent.com/elastic/elastic-agent/refs/tags/v8.19.0/internal/pkg/otel/samples/linux/gateway.yml) | +| 9.1 | [Gateway mode](https://raw.githubusercontent.com/elastic/elastic-agent/refs/tags/v9.1.0/internal/pkg/otel/samples/linux/gateway.yml) | + +Use the previous example configuration as a reference when configuring your Gateway Collector or customizing your EDOT Collector configuration. + +### Data collection and processing + +The EDOT Collector in Gateway mode collects data from other Collectors using the OTLP protocol. By default, the sample Gateway configuration listens on port `4317` for gRPC and port `4318` for HTTP. + +```yaml +receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 # Listen on all interfaces + http: + endpoint: 0.0.0.0:4318 # Listen on all interfaces +``` + +The routing connector splits infrastructure metrics from other metrics and routes them to the appropriate Elastic Common Schema pipelines. Other metrics are exported in OTel-native format through the [`elasticsearch`] exporter. + +```yaml +connectors: + routing: + default_pipelines: [metrics/otel] + error_mode: ignore + table: + - context: metric + statement: route() where IsMatch(instrumentation_scope.name, "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/hostmetricsreceiver/internal/scraper/*") + pipelines: [metrics/infra/ecs, metrics/otel] + elasticapm: {} +``` + +### Data processing and transformation + +The Gateway configuration includes several processors to transform and optimize the collected data: + +```yaml +processors: + elasticinframetrics: + add_system_metrics: true + drop_original: true + attributes/dataset: + actions: + - key: event.dataset + from_attribute: data_stream.dataset + action: upsert + resource/process: + attributes: + - key: process.executable.name + action: delete + - key: process.executable.path + action: delete + batch: + send_batch_size: 1000 + timeout: 1s + send_batch_max_size: 1500 + batch/metrics: + send_batch_max_size: 0 # Prevents splitting metrics requests + timeout: 1s + elastictrace: {} +``` + +### Data export + +The Gateway exports data to Elasticsearch in two formats: + +- OTel-native format using the `elasticsearch/otel` exporter. +- Elastic Common Schema (ECS) format using the `elasticsearch/ecs` exporter. + +```yaml +exporters: + elasticsearch/otel: + endpoints: + - ${ELASTIC_ENDPOINT} + api_key: ${ELASTIC_API_KEY} + mapping: + mode: otel + elasticsearch/ecs: + endpoints: + - ${ELASTIC_ENDPOINT} + api_key: ${ELASTIC_API_KEY} + mapping: + mode: ecs +``` + +### Pipeline configuration + +The service section defines separate pipelines for different telemetry types: + +- Metrics pipelines for infrastructure and OTel metrics +- Logs pipeline +- Traces pipeline +- Aggregated OTel metrics pipeline + +Each pipeline connects specific receivers, processors, and exporters to handle different data types appropriately. + + +## Central configuration + +The EDOT Collector can be configured to use [APM Agent Central Configuration](docs-content://solutions/observability/apm/apm-agent-central-configuration.md). Refer to [Central configuration docs](opentelemetry://reference/central-configuration.md) for more details. + +To activate the central configuration feature, add the [`apmconfig`](https://github.com/elastic/opentelemetry-collector-components/blob/main/extension/apmconfigextension/README.md). For example: + +:::{include} ../_snippets/edot-collector-auth.md +::: + +Create an API Key following [these instructions](docs-content://deploy-manage/api-keys/elasticsearch-api-keys.md). The API key must have `config_agent:read` permissions and resources set to `-`. + +## Secure connection + +To secure the connection between the EDOT Collector and Elastic, you can use TLS or mutual TLS, as well as the `apikeyauth` extension. + +### TLS configuration + +You can turn on TLS or mutual TLS to encrypt data in transit between EDOT SDKs and the extension. Refer to [OpenTelemetry TLS server configuration](https://github.com/open-telemetry/opentelemetry-collector/blob/main/config/configtls/README.md#server-configuration) for more details. + +For example: + +```yaml +extensions: + apmconfig: + opamp: + protocols: + http: + endpoint: ":4320" + tls: + cert_file: server.crt + key_file: server.key + ... +``` + +### Authentication settings + +In addition to TLS, you can configure authentication to ensure that only authorized agents can communicate with the extension and retrieve their corresponding remote configurations. + +The `apmconfig` extension supports any configauth authenticator. Use the `apikeyauth` extension to authenticate with Elasticsearch API keys: + +```yaml +extensions: + apikeyauth: + endpoint: "" + application_privileges: + - application: "apm" + privileges: + - "config_agent:read" + resources: + - "-" + apmconfig: + opamp: + protocols: + http: + auth: + authenticator: apikeyauth + ... +``` + +Create an API key with the minimum required application permissions through {{kib}} under **Observability** → **Applications** → **Settings** → **Agent Keys**, or by using the Elasticsearch Security API: + +::::{dropdown} Example JSON payload +```json +POST /_security/api_key +{ + "name": "apmconfig-opamp-test-sdk", + "metadata": { + "application": "apm" + }, + "role_descriptors": { + "apm": { + "cluster": [], + "indices": [], + "applications": [ + { + "application": "apm", + "privileges": [ + "config_agent:read" + ], + "resources": [ + "*" + ] + } + ], + "run_as": [], + "metadata": {} + } + } +} +``` +:::: + +The server expects incoming HTTP requests to include an API key with sufficient privileges, using the following header format: `Authorization: ApiKey `. + +[`attributes`]: https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/processor/attributesprocessor +[`filelog`]: https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/receiver/filelogreceiver +[`hostmetrics`]: https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/receiver/hostmetricsreceiver +[`elasticsearch`]: https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/exporter/elasticsearchexporter +[`elasticinframetrics`]: https://github.com/elastic/opentelemetry-collector-components/tree/main/processor/elasticinframetricsprocessor +[`elastictrace`]: https://github.com/elastic/opentelemetry-collector-components/tree/main/processor/elastictraceprocessor +[`elasticapm`]: https://github.com/elastic/opentelemetry-collector-components/tree/main/connector/elasticapmconnector +[`resource`]: https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/processor/resourceprocessor +[`resourcedetection`]: https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/processor/resourcedetectionprocessor +[`OTLP`]: https://github.com/open-telemetry/opentelemetry-collector/tree/main/receiver/otlpreceiver +[Logs - ES]: https://raw.githubusercontent.com/elastic/elastic-agent/refs/tags/v{{version.edot_collector}}/internal/pkg/otel/samples/linux/platformlogs.yml +[Logs - OTLP]: https://raw.githubusercontent.com/elastic/elastic-agent/refs/tags/v{{version.edot_collector}}/internal/pkg/otel/samples/linux/managed_otlp/platformlogs.yml +[Logs | Metrics - ES]: https://raw.githubusercontent.com/elastic/elastic-agent/refs/tags/v{{version.edot_collector}}/internal/pkg/otel/samples/linux/platformlogs_hostmetrics.yml +[Logs | Metrics - OTLP]: https://raw.githubusercontent.com/elastic/elastic-agent/refs/tags/v{{version.edot_collector}}/internal/pkg/otel/samples/linux/managed_otlp/platformlogs_hostmetrics.yml +[Logs | Metrics | App - ES]: https://raw.githubusercontent.com/elastic/elastic-agent/refs/tags/v{{version.edot_collector}}/internal/pkg/otel/samples/linux/logs_metrics_traces.yml +[Logs | Metrics | App - OTLP]: https://raw.githubusercontent.com/elastic/elastic-agent/refs/tags/v{{version.edot_collector}}/internal/pkg/otel/samples/linux/managed_otlp/logs_metrics_traces.yml +[Gateway mode]: https://raw.githubusercontent.com/elastic/elastic-agent/refs/heads/main/internal/pkg/otel/samples/linux/gateway.yml diff --git a/docs/reference/edot-collector/config/index.md b/docs/reference/edot-collector/config/index.md new file mode 100644 index 00000000000..910f2b7db29 --- /dev/null +++ b/docs/reference/edot-collector/config/index.md @@ -0,0 +1,26 @@ +--- +navigation_title: Configuration +description: Default configurations of the EDOT Collector in different environments and scenarios, as well as examples of EDOT Collector configuration for different custom use cases. +applies_to: + stack: + serverless: + observability: + product: + edot_collector: ga +products: + - id: cloud-serverless + - id: observability + - id: edot-collector +--- + +# EDOT Collector configuration + +The following pages provide insights into the default configurations of the EDOT Collector in different environments and scenarios, as well as examples of EDOT Collector configuration for different custom use cases. + +* [Default Configuration - Standalone EDOT Collector](/reference/edot-collector/config/default-config-standalone.md) +* [Default Configuration - EDOT Collectors on Kubernetes](/reference/edot-collector/config/default-config-k8s.md) +* [Configure Metrics Collection](/reference/edot-collector/config/configure-metrics-collection.md) +* [Configure Logs Collection](/reference/edot-collector/config/configure-logs-collection.md) +* [Authentication Methods](/reference/edot-collector/config/authentication-methods.md) + + diff --git a/docs/reference/edot-collector/config/proxy.md b/docs/reference/edot-collector/config/proxy.md new file mode 100644 index 00000000000..fed29aa75d7 --- /dev/null +++ b/docs/reference/edot-collector/config/proxy.md @@ -0,0 +1,88 @@ +--- +navigation_title: Proxy settings +description: Configuration of the EDOT Collector's proxy settings. +applies_to: + stack: + serverless: + observability: + product: + edot_collector: ga +products: + - id: cloud-serverless + - id: observability + - id: edot-collector +--- + +# Configure proxy settings for EDOT components + +When running Elastic Distributions of OpenTelemetry (EDOT) in environments that require outbound traffic to go through a proxy, you must explicitly configure proxy settings. + +You may need to configure a proxy if: + +- Your app is deployed behind a corporate proxy or firewall. +- Your telemetry is sent to Elastic APM in Elastic Cloud or another hosted destination. +- Network errors such as `Connection timed out` or `SSL handshake failed` appear in logs. + +## Available proxy variables + +| Variable | Description | +|--------------|---------------------------------------------| +| HTTP_PROXY | URL of the proxy server for HTTP requests | +| HTTPS_PROXY | URL of the proxy server for HTTPS requests | +| NO_PROXY | Comma-separated list of hosts to exclude | + +## Configure proxy settings for the EDOT Collector + +Most EDOT components honor common proxy environment variables. The following examples show how to configure them: + +::::{tab-set} + +:::{tab-item} Docker run +```bash +docker run -e HTTP_PROXY=http://: \ + -e HTTPS_PROXY=http://: \ + otel/opentelemetry-collector:latest +``` +::: + +:::{tab-item} Docker compose +```yaml +services: + edotcollector: + environment: + - HTTP_PROXY=http://: + - HTTPS_PROXY=http://: +``` +::: + +:::{tab-item} Kubernetes pod manifest +```yaml +env: + - name: HTTP_PROXY + value: ':' + - name: HTTPS_PROXY + value: ':' +``` +::: + +:::{tab-item} systemmd [Service] unit file +``` +[Service] +Environment="HTTP_PROXY=http://:" +Environment="HTTPS_PROXY=http://:" +Environment="NO_PROXY=," +``` +::: + +:::: + +:::{{note}} +For the EDOT Collector, proxy support applies to all exporters, including those using gRPC. No special configuration is needed beyond the environment variables. + +If you're using an SDK that doesn't support proxy variables directly, consider routing telemetry through an EDOT Collector configured with proxy settings. This ensures consistent proxy handling. For more information, refer to [Proxy settings for EDOT SDKs](docs-content://troubleshoot/ingest/opentelemetry/edot-sdks/proxy.md). +::: + + +## Resources + +[Proxy support - contrib documentation](https://opentelemetry.io/docs/collector/configuration/#proxy-support) diff --git a/docs/reference/edot-collector/custom-collector.md b/docs/reference/edot-collector/custom-collector.md new file mode 100644 index 00000000000..2a6f066939e --- /dev/null +++ b/docs/reference/edot-collector/custom-collector.md @@ -0,0 +1,203 @@ +--- +navigation_title: Custom Collector +description: How to build a custom OpenTelemetry Collector distribution similar to EDOT. +applies_to: + stack: + serverless: + observability: + product: + edot_collector: ga +products: + - id: cloud-serverless + - id: observability + - id: edot-collector +--- + +# Build a Custom EDOT-like Collector + +You can build and configure a [custom Collector](https://opentelemetry.io/docs/collector/custom-collector/) or extend the [OpenTelemetry Collector Contrib ](https://github.com/open-telemetry/opentelemetry-collector-contrib) distribution to collect logs and metrics and send them to Elastic Observability. + +For a more seamless experience, use the Elastic Distribution of the OpenTelemetry Collector. Refer to the [configuration](/reference/edot-collector/config/index.md) docs for more information on configuring the EDOT Collector. + +## Build a custom Collector + +To build a custom Collector to collect your telemetry data and send it to Elastic Observability, you need to: + +1. Install the OpenTelemetry Collector builder, `ocb`. +1. Create a builder configuration file. +1. Build the Collector. + +Refer to the following sections to complete these steps. + +### Install the OpenTelemetry Collector builder + +Install `ocb` using the command that aligns with your system from the [OpenTelemetry building a custom Collector documentation](https://opentelemetry.io/docs/collector/custom-collector/#step-1---install-the-builder). + +:::{important} +Make sure to install the version of OpenTelemetry Collector Builder that matches the desired components' version. +::: + +### Create a builder configuration file + +Create a builder configuration file,`builder-config.yml`, to define the custom Collector. This file specifies the components, such as extensions, exporters, processors, receivers, and connectors, included in your custom Collector. + +The following example, `builder-config.yml`, contains the components needed to send your telemetry data to Elastic Observability. For more information on these components, refer to the [components](/reference/edot-collector/components.md) documentation. Keep or remove components from the example configuration file to fit your needs. + +% start:edot-collector-components-ocb +```yaml +dist: + otelcol_edot: + description: "Elastic Distribution of OpenTelemetry Collector" + output_path: ./dist/otelcol_edot + builds: + - name: otelcol_edot + goos: [linux, darwin, windows] + goarch: [amd64, arm64] + output_path: ./dist/otelcol_edot_{{ .OS }}_{{ .Arch }} + env: + - CGO_ENABLED=0 + - GOOS={{ .OS }} + - GOARCH={{ .Arch }} + ldflags: + - -s -w + - -X go.opentelemetry.io/collector/otelcol.buildTimestamp={{ .BuildTimestamp }} + - -X go.opentelemetry.io/collector/otelcol.version={{ .Version }} + +receivers: + dockerstatsreceiver : + gomod: github.com/open-telemetry/opentelemetry-collector-contrib/receiver/dockerstatsreceiver v0.130.0 + elasticapmintakereceiver : + gomod: github.com/elastic/opentelemetry-collector-components/receiver/elasticapmintakereceiver v0.2.1 + filelogreceiver : + gomod: github.com/open-telemetry/opentelemetry-collector-contrib/receiver/filelogreceiver v0.130.0 + hostmetricsreceiver : + gomod: github.com/open-telemetry/opentelemetry-collector-contrib/receiver/hostmetricsreceiver v0.130.0 + httpcheckreceiver : + gomod: github.com/open-telemetry/opentelemetry-collector-contrib/receiver/httpcheckreceiver v0.130.0 + jaegerreceiver : + gomod: github.com/open-telemetry/opentelemetry-collector-contrib/receiver/jaegerreceiver v0.130.0 + jmxreceiver : + gomod: github.com/open-telemetry/opentelemetry-collector-contrib/receiver/jmxreceiver v0.130.0 + k8sclusterreceiver : + gomod: github.com/open-telemetry/opentelemetry-collector-contrib/receiver/k8sclusterreceiver v0.130.0 + k8sobjectsreceiver : + gomod: github.com/open-telemetry/opentelemetry-collector-contrib/receiver/k8sobjectsreceiver v0.130.0 + kafkareceiver : + gomod: github.com/open-telemetry/opentelemetry-collector-contrib/receiver/kafkareceiver v0.130.0 + kubeletstatsreceiver : + gomod: github.com/open-telemetry/opentelemetry-collector-contrib/receiver/kubeletstatsreceiver v0.130.0 + nginxreceiver : + gomod: github.com/open-telemetry/opentelemetry-collector-contrib/receiver/nginxreceiver v0.130.0 + nopreceiver : + gomod: go.opentelemetry.io/collector/receiver/nopreceiver v0.130.0 + otlpreceiver : + gomod: go.opentelemetry.io/collector/receiver/otlpreceiver v0.130.0 + prometheusreceiver : + gomod: github.com/open-telemetry/opentelemetry-collector-contrib/receiver/prometheusreceiver v0.130.0 + receivercreator : + gomod: github.com/open-telemetry/opentelemetry-collector-contrib/receiver/receivercreator v0.130.0 + redisreceiver : + gomod: github.com/open-telemetry/opentelemetry-collector-contrib/receiver/redisreceiver v0.130.0 + zipkinreceiver : + gomod: github.com/open-telemetry/opentelemetry-collector-contrib/receiver/zipkinreceiver v0.130.0 + +processors: + attributesprocessor : + gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/attributesprocessor v0.130.0 + batchprocessor : + gomod: go.opentelemetry.io/collector/processor/batchprocessor v0.130.0 + cumulativetodeltaprocessor : + gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/cumulativetodeltaprocessor v0.130.0 + elasticinframetricsprocessor : + gomod: github.com/elastic/opentelemetry-collector-components/processor/elasticinframetricsprocessor v0.16.0 + elastictraceprocessor : + gomod: github.com/elastic/opentelemetry-collector-components/processor/elastictraceprocessor v0.9.0 + filterprocessor : + gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/filterprocessor v0.130.0 + geoipprocessor : + gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/geoipprocessor v0.130.0 + k8sattributesprocessor : + gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/k8sattributesprocessor v0.130.0 + memorylimiterprocessor : + gomod: go.opentelemetry.io/collector/processor/memorylimiterprocessor v0.130.0 + resourcedetectionprocessor : + gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/resourcedetectionprocessor v0.130.0 + resourceprocessor : + gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/resourceprocessor v0.130.0 + tailsamplingprocessor : + gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/tailsamplingprocessor v0.130.0 + transformprocessor : + gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/transformprocessor v0.130.0 + +exporters: + debugexporter : + gomod: go.opentelemetry.io/collector/exporter/debugexporter v0.132.0 + elasticsearchexporter : + gomod: github.com/open-telemetry/opentelemetry-collector-contrib/exporter/elasticsearchexporter v0.132.0 + fileexporter : + gomod: github.com/open-telemetry/opentelemetry-collector-contrib/exporter/fileexporter v0.130.0 + kafkaexporter : + gomod: github.com/open-telemetry/opentelemetry-collector-contrib/exporter/kafkaexporter v0.130.0 + loadbalancingexporter : + gomod: github.com/open-telemetry/opentelemetry-collector-contrib/exporter/loadbalancingexporter v0.130.0 + nopexporter : + gomod: go.opentelemetry.io/collector/exporter/nopexporter v0.130.0 + otlpexporter : + gomod: go.opentelemetry.io/collector/exporter/otlpexporter v0.130.0 + otlphttpexporter : + gomod: go.opentelemetry.io/collector/exporter/otlphttpexporter v0.130.0 + +connectors: + elasticapmconnector : + gomod: github.com/elastic/opentelemetry-collector-components/connector/elasticapmconnector v0.6.0 + forwardconnector : + gomod: go.opentelemetry.io/collector/connector/forwardconnector v0.130.0 + routingconnector : + gomod: github.com/open-telemetry/opentelemetry-collector-contrib/connector/routingconnector v0.130.0 + spanmetricsconnector : + gomod: github.com/open-telemetry/opentelemetry-collector-contrib/connector/spanmetricsconnector v0.130.0 + +extensions: + apikeyauthextension : + gomod: github.com/elastic/opentelemetry-collector-components/extension/apikeyauthextension v0.4.1 + apmconfigextension : + gomod: github.com/elastic/opentelemetry-collector-components/extension/apmconfigextension v0.6.0 + bearertokenauthextension : + gomod: github.com/open-telemetry/opentelemetry-collector-contrib/extension/bearertokenauthextension v0.130.0 + filestorage : + gomod: github.com/open-telemetry/opentelemetry-collector-contrib/extension/storage/filestorage v0.130.0 + healthcheckextension : + gomod: github.com/open-telemetry/opentelemetry-collector-contrib/extension/healthcheckextension v0.130.0 + healthcheckv2extension : + gomod: github.com/open-telemetry/opentelemetry-collector-contrib/extension/healthcheckv2extension v0.130.0 + k8sleaderelector : + gomod: github.com/open-telemetry/opentelemetry-collector-contrib/extension/k8sleaderelector v0.130.0 + k8sobserver : + gomod: github.com/open-telemetry/opentelemetry-collector-contrib/extension/observer/k8sobserver v0.130.0 + memorylimiterextension : + gomod: go.opentelemetry.io/collector/extension/memorylimiterextension v0.130.0 + pprofextension : + gomod: github.com/open-telemetry/opentelemetry-collector-contrib/extension/pprofextension v0.130.0 + +providers: + envprovider : + gomod: go.opentelemetry.io/collector/confmap/provider/envprovider v1.36.0 + fileprovider : + gomod: go.opentelemetry.io/collector/confmap/provider/fileprovider v1.38.0 + httpprovider : + gomod: go.opentelemetry.io/collector/confmap/provider/httpprovider v1.36.0 + httpsprovider : + gomod: go.opentelemetry.io/collector/confmap/provider/httpsprovider v1.35.0 + yamlprovider : + gomod: go.opentelemetry.io/collector/confmap/provider/yamlprovider v1.38.0 +``` +% end:edot-collector-components-ocb + +### Build the Collector + +Build your custom Collector using the `ocb` tool and the configuration file by running the following command: `builder --config builder-config.yml`. + +The command generates a new Collector in the specified output path, `otelcol-dev`. The generated Collector includes the components you specified in the configuration file. + +For general information on building a custom Collector, refer to the [OpenTelemetry documentation](https://opentelemetry.io/docs/collector/custom-collector/#step-1---install-the-builder). + diff --git a/docs/reference/edot-collector/customization.md b/docs/reference/edot-collector/customization.md new file mode 100644 index 00000000000..de86513053c --- /dev/null +++ b/docs/reference/edot-collector/customization.md @@ -0,0 +1,31 @@ +--- +navigation_title: Customization +description: Options for customizing the EDOT Collector, including building a custom Collector or requesting new components. +applies_to: + stack: + serverless: + observability: + product: + edot_collector: ga +products: + - id: cloud-serverless + - id: observability + - id: edot-collector +--- + +# EDOT Collector customization + +The EDOT Collector comes with a [curated list](/reference/edot-collector/components.md) of OTel Collector components and some opinionated [configuration samples](https://github.com/elastic/elastic-agent/tree/main/internal/pkg/otel/samples). + +If your use case requires additional components, you have two options: + +1. [Build your custom, EDOT-like Collector](/reference/edot-collector/custom-collector.md) +2. [Open a request](https://github.com/elastic/elastic-agent/issues/new/choose) to add those components to EDOT. + +Requests for adding new components to the EDOT Collector will be reviewed and decided on the basis of the popularity of the requests, technical suitability and other criteria. + +For instructions on how to build a custom Collector, refer to the [OpenTelemetry documentation](https://opentelemetry.io/docs/collector/custom-collector/). + +:::{warning} +Custom Collector builds are not covered through [Elastic's Support](https://www.elastic.co/support_policy). +::: diff --git a/docs/reference/edot-collector/download.md b/docs/reference/edot-collector/download.md new file mode 100644 index 00000000000..356e75bae0b --- /dev/null +++ b/docs/reference/edot-collector/download.md @@ -0,0 +1,40 @@ +--- +navigation_title: Download +description: Direct download links for EDOT Collector binaries for various operating systems and architectures. +applies_to: + stack: + serverless: + observability: + product: + edot_collector: ga +products: + - id: cloud-serverless + - id: observability + - id: edot-collector +--- + +# Download the EDOT Collector + +The {{edot}} (EDOT) Collector is embedded in the {{agent}} package as a separate binary that invokes OpenTelemetry Collector components. + +The following table contains direct download links for the latest EDOT Collector version for different operating systems and architectures. + +| Platform | Architecture | Download link | +|--------------|--------------|---------------| +| Windows | x86_64 | [Download (Zip)](https://artifacts.elastic.co/downloads/beats/elastic-agent/elastic-agent-{{version.edot_collector}}-windows-x86_64.zip) | +| macOS | x86_64 | [Download (Tar.gz)](https://artifacts.elastic.co/downloads/beats/elastic-agent/elastic-agent-{{version.edot_collector}}-darwin-x86_64.tar.gz) | +| macOS | aarch64 | [Download (Tar.gz)](https://artifacts.elastic.co/downloads/beats/elastic-agent/elastic-agent-{{version.edot_collector}}-darwin-aarch64.tar.gz) | +| Linux | x86_64 | [Download (Tar.gz)](https://artifacts.elastic.co/downloads/beats/elastic-agent/elastic-agent-{{version.edot_collector}}-linux-x86_64.tar.gz) | +| Linux | aarch64 | [Download (Tar.gz)](https://artifacts.elastic.co/downloads/beats/elastic-agent/elastic-agent-{{version.edot_collector}}-linux-arm64.tar.gz) | +| Linux (DEB) | x86_64 | [Download (Deb)](https://artifacts.elastic.co/downloads/beats/elastic-agent/elastic-agent-{{version.edot_collector}}-amd64.deb) | +| Linux (DEB) | aarch64 | [Download (Deb)](https://artifacts.elastic.co/downloads/beats/elastic-agent/elastic-agent-{{version.edot_collector}}-arm64.deb) | +| Linux (RPM) | x86_64 | [Download (Rpm)](https://artifacts.elastic.co/downloads/beats/elastic-agent/elastic-agent-{{version.edot_collector}}-x86_64.rpm) | +| Linux (RPM) | aarch64 | [Download (Rpm)](https://artifacts.elastic.co/downloads/beats/elastic-agent/elastic-agent-{{version.edot_collector}}-aarch64.rpm) | + +After you've downloaded and uncompressed the file, you can get EDOT Collector running with the following command: + +``` +sudo ./otelcol --config otel.yml +``` + +For specific configuration, refer to the [Quickstart guides](docs-content://solutions/observability/get-started/opentelemetry/quickstart/index.md) or refer to [Configuration](/reference/edot-collector/config/index.md). diff --git a/docs/reference/edot-collector/index.md b/docs/reference/edot-collector/index.md new file mode 100644 index 00000000000..3e7ce8c5991 --- /dev/null +++ b/docs/reference/edot-collector/index.md @@ -0,0 +1,48 @@ +--- +navigation_title: EDOT Collector +description: Introduction to the Elastic Distribution of OpenTelemetry (EDOT) Collector, a curated and supported distribution of the OpenTelemetry Collector. +applies_to: + stack: + serverless: + observability: + product: + edot_collector: ga +products: + - id: cloud-serverless + - id: observability + - id: edot-collector +--- + +# Elastic Distribution of OpenTelemetry Collector + +The {{edot}} (EDOT) Collector is an open-source distribution of the OpenTelemetry Collector. + +## Get started + +To install the EDOT Collector with basic settings in your environment, follow the [quickstart guides](docs-content://solutions/observability/get-started/opentelemetry/quickstart/index.md). + +## Deployment modes + +You can deploy the EDOT Collector in different modes to meet your architectural needs. The two primary Collector modes are Agent and Gateway. Depending on your Elastic deployment type, various EDOT Collector instances might be required in each mode to support the target architecture. Refer to [Deployment modes](/reference/edot-collector/modes.md) for more details. + +## Configure the Collector + +You can configure the EDOT Collector to use the standard OTel Collector configuration file or `values.yml` file if you have deployed it using Helm. + +For full details on each option, see [Configuration](/reference/edot-collector/config/index.md) + +## Collector components + +Built on OpenTelemetry’s modular architecture, the EDOT Collector offers a curated and fully supported selection of components designed for production-grade reliability. + +Refer to [Components](/reference/edot-collector/components.md) for the full list of components embedded in the EDOT Collector. + +To request a component to be added to EDOT Collector, submit a [GitHub issue here](https://github.com/elastic/elastic-agent/issues/new/choose). + +## Limitations + +The EDOT Collector inherits the same limitations from the contrib components. Refer to [Limitations](opentelemetry://reference/compatibility/limitations.md) for a complete list. + +## License + +For details on the EDOT Collector license, see the [LICENSE.txt](https://github.com/elastic/elastic-agent/blob/main/LICENSE.txt) file. diff --git a/docs/reference/edot-collector/modes.md b/docs/reference/edot-collector/modes.md new file mode 100644 index 00000000000..0fee4f603bc --- /dev/null +++ b/docs/reference/edot-collector/modes.md @@ -0,0 +1,96 @@ +--- +navigation_title: Deployment modes +description: Deployment modes for the Elastic Distribution of OpenTelemetry (EDOT) Collector, including Agent and Gateway modes and when to use each. +applies_to: + stack: + serverless: + observability: + product: + edot_collector: ga +products: + - id: cloud-serverless + - id: observability + - id: edot-collector +--- + +# EDOT Collector deployment modes + +You can deploy the EDOT Collector in different modes to meet your architectural needs. The two primary Collector modes are Agent and Gateway. Depending on your Elastic deployment type — Elastic self-managed, {{ecloud}}, or {{serverless-full}} — various EDOT Collector instances might be required in each mode to support the target architecture. + +Use the information in the following sections to better understand deployment modes and patterns for your specific environment. + +## EDOT Collector as Agent + +In Agent mode, the EDOT Collector runs close to the data source, collecting telemetry data directly from the local environment. A Collector in Agent mode usually runs on the same host or virtual machine as the application or infrastructure component it is monitoring, or as a sidecar container or daemonset in Kubernetes. + +Use the EDOT Collector in Agent mode when: + +- You need to collect data directly from hosts or applications. +- You have a deployment with a small number of hosts. + +## EDOT Collector as Gateway + +In Gateway mode, the EDOT Collector acts as a central aggregation point, receiving data from multiple EDOT Agent collectors or instrumented applications before forwarding it to Elastic. The flexibility of the Gateway mode allows the Collector to centralize the scaling needs and data transformation operations of the data pipeline. + +Use the EDOT Collector in Gateway mode when: + +- You have multiple data sources or agents that need centralized processing or enrichment. +- You need to implement organization-wide processing rules. +- You want to reduce the number of connections to your Elastic backend. +- You need advanced pre-processing before data reaches Elastic. +- You're using a self-managed {{es}} deployment (required for APM functionality). +- You want to filter telemetry before it is shipped over the network to Elastic. + +The Gateway pattern isn't exclusive to self-managed Elastic deployments. It's a general OpenTelemetry pattern that provides benefits in various scenarios: + +- Kubernetes deployments: A Gateway collector centralizes cluster-level telemetry from multiple node-level Agent collectors. +- Multi-region deployments: Regional Gateway collectors aggregate data from multiple Agents before sending to a central destination. +- High-volume environments: Gateway collectors provide buffering and batching to handle high volumes of telemetry data. +- Complex processing: When advanced data transformation or filtering is needed before data reaches its destination. + +### Gateway requirements for self-managed environments + +For self-managed Elastic environments, you need a Gateway Collector deployed alongside your {{stack}}. The EDOT Collector in Gateway mode exposes a scalable OTLP endpoint, and performs data processing required for APM functionality. + +This is the only case where using the {{es}} exporter is recommended. In all other EDOT Collector deployments described in this guide, use the OTLP exporter. + +#### Required components for APM functionality in self-managed Elastic + +The following components are required for APM functionality in self-managed Elastic: + +- The `elastictrace` processor enriches trace data with additional attributes that improve the user experience in Elastic Observability UIs. +- The `elasticapm` connector generates pre-aggregated APM metrics from trace data. + +In this case, EDOT Collector as a Gateway also handles routing of the different types of telemetry data to the relevant indices. + +## Deployment in Kubernetes environments + +In Kubernetes environments, EDOT Collectors are typically deployed in three distinct modes that work together to provide comprehensive observability: + +### Agent mode in Kubernetes + +In Kubernetes, the Agent mode is implemented in two forms: + +| Form | Deployment | Functions | +|------|------------|-----------| +| Daemon form | DaemonSet on every node | - Collects node-local logs and host metrics.
- Receives telemetry data from applications instrumented with OpenTelemetry SDKs running on the node.
- Enriches application telemetry data with resource information such as host and Kubernetes metadata.
- Forwards all data to the Gateway Collector using the OTLP protocol. | +| Cluster form | Centralized service | - Collects Kubernetes cluster-level metrics from the Kubernetes API.
- Monitors cluster-wide resources that aren't specific to individual nodes.
- Forwards collected data to the Gateway Collector using the OTLP protocol. | + +### Gateway mode in Kubernetes + +The Gateway Collector in Kubernetes receives data from all Daemon and Cluster collectors. The Gateway performs additional pre-processing and aggregation for self-managed and {{ech}} deployments, and handles the final export to the appropriate Elastic backend. + +This multi-tier architecture in Kubernetes provides an efficient way to collect and process telemetry data at different levels of the infrastructure while minimizing resource usage and network traffic. + +For more details on Kubernetes deployment architecture, see [Kubernetes environments](opentelemetry://reference/architecture/k8s.md). + +## Direct EDOT SDK to Managed OTLP (No Collector) + +In some scenarios, you don't need a Collector at all. EDOT SDKs can send telemetry data directly to the [{{motlp}}](opentelemetry://reference/motlp.md). This is the simplest deployment pattern for getting application telemetry data into Elastic Observability. + +Use direct EDOT SDK export to Managed OTLP when: + +- You're sending data to the Managed OTLP Endpoint. +- You only need to collect application telemetry data. +- You want the simplest possible deployment. +- You don't need local data processing or filtering. diff --git a/docs/reference/edot-collector/toc.yml b/docs/reference/edot-collector/toc.yml new file mode 100644 index 00000000000..4e686156d97 --- /dev/null +++ b/docs/reference/edot-collector/toc.yml @@ -0,0 +1,22 @@ +toc: + - file: index.md + - file: download.md + - file: modes.md + - file: config/index.md + children: + - file: config/default-config-standalone.md + - file: config/default-config-k8s.md + - file: config/configure-logs-collection.md + - file: config/configure-metrics-collection.md + - file: config/authentication-methods.md + - file: config/proxy.md + - file: components.md + children: + - file: components/elasticsearchexporter.md + - file: components/elasticapmintakereceiver.md + - file: customization.md + children: + - file: custom-collector.md + - file: upstream-collector.md + - title: Troubleshooting + crosslink: docs-content://troubleshoot/ingest/opentelemetry/edot-collector/index.md \ No newline at end of file diff --git a/docs/reference/edot-collector/upstream-collector.md b/docs/reference/edot-collector/upstream-collector.md new file mode 100644 index 00000000000..98e4f82e333 --- /dev/null +++ b/docs/reference/edot-collector/upstream-collector.md @@ -0,0 +1,138 @@ +--- +navigation_title: Use the contrib Collector +description: Learn how to send data to Elastic Observability using the contrib OpenTelemetry Collector instead of EDOT. +applies_to: + stack: + serverless: + observability: + product: + edot_collector: ga +products: + - id: cloud-serverless + - id: observability + - id: edot-collector +--- + +# Send data to {{serverless-full}} using the contrib Collector + +While the {{edot}} (EDOT) Collector provides a streamlined experience with pre-selected components, you can also use the contrib OpenTelemetry Collector or a custom distribution to send data to Elastic Observability. This approach requires more configuration but gives you more control over your OpenTelemetry setup. + +## Overview + +The contrib OpenTelemetry Collector is the community-maintained version that provides the foundation for all OpenTelemetry distributions. To configure it to work with Elastic Observability, you need to: + +- Manually select and configure components. +- Set up proper data processing pipelines. +- Handle authentication and connection details. +- Ensure required components have been properly configured in accordance to your use case. + +## Deployment scenarios + +The configuration requirements vary depending on your use case and the Elastic deployment model you want to send data to. The following sections outline what you need for each scenario. + +### Elastic Cloud Serverless + +{{serverless-full}} provides a [Managed OTLP Endpoint](opentelemetry://reference/motlp.md) that accepts OpenTelemetry data in its native format. This makes it the simplest scenario for using contrib components because scaling and signal processing (for example producing metrics from events) is handled by Elastic. + +The following configuration example shows how to send data to the Managed OTLP Endpoint: + +```yaml +exporters: + otlp: + endpoint: "https://your-deployment.elastic-cloud.com:443" + headers: + authorization: "Bearer YOUR_API_KEY" + +service: + pipelines: + traces: + exporters: [otlp] + metrics: + exporters: [otlp] + logs: + exporters: [otlp] +``` + +### Elastic Cloud Hosted (ECH) + +Because {{motlp}} is not yet available for {{ech}}, you need to setup an instance of EDOT that works as a gateway, handling processing required for some use cases, like deriving metrics from events in APM, and writing data directly to Elasticsearch. + +Point your contrib Collector OTLP exporter to the EDOT gateway. Refer to [Gateway configuration](/reference/edot-collector/config/default-config-standalone.md#gateway-mode) for more information. + +### Self-managed Elastic Stack + +Self-managed deployments have similar requirements to ECH but with your own Elasticsearch instance. The configuration is similar to ECH. You also need to: + +- Point to your self-managed Elasticsearch instance. +- Configure appropriate security settings. +- Ensure your Elasticsearch version is compatible. +- Set up proper index templates and mappings. + +## Configuration best practices + +When using the contrib OpenTelemetry Collector with Elastic Observability, follow these best practices: + +### Resource detection + +Always include the `resourcedetectionprocessor` to automatically add host, cloud, and Kubernetes metadata: + +```yaml +processors: + resourcedetection: + detectors: [env, system, gcp, ecs, ec2, azure, aks, eks, gke] + timeout: 5s + override: true +``` + +### Attribute processing + +Use the `attributesprocessor` to ensure consistent attribute naming and add required metadata: + +```yaml +processors: + attributes: + actions: + - key: service.name + value: "your-service-name" + action: insert + - key: service.version + value: "1.0.0" + action: insert +``` + +### Batching + +Configure the `batchprocessor` for optimal performance: + +```yaml +processors: + batch: + timeout: 1s + send_batch_size: 1024 + send_batch_max_size: 2048 +``` + +### Security + +For production deployments, always use secure connections: + +```yaml +exporters: + elasticsearch: + tls: + insecure: false + ca_file: "/path/to/ca.crt" + user: "elastic" + password: "YOUR_PASSWORD" +``` + +## Limitations and considerations + +Using the contrib OpenTelemetry Collector instead of EDOT comes with some trade-offs. Refer to [EDOT compared to contrib Collector](opentelemetry://reference/compatibility/edot-vs-upstream.md) for more information. + +## Next steps + +- [Build a custom EDOT-like collector](/reference/edot-collector/custom-collector.md) for more control. +- [Configure the EDOT Collector](/reference/edot-collector/config/index.md) for optimal Elastic integration. +- [Learn about EDOT components](/reference/edot-collector/components.md) to understand what's included. +- [Explore deployment architectures](opentelemetry://reference/architecture/index.md) for different environments. diff --git a/docs/scripts/update-docs/requirements.txt b/docs/scripts/update-docs/requirements.txt new file mode 100644 index 00000000000..d5d8a286fd1 --- /dev/null +++ b/docs/scripts/update-docs/requirements.txt @@ -0,0 +1,2 @@ +Jinja2==3.1.6 +PyYAML==6.0.2 diff --git a/docs/scripts/update-docs/templates/components-table.jinja2 b/docs/scripts/update-docs/templates/components-table.jinja2 new file mode 100644 index 00000000000..7ff77cb73bc --- /dev/null +++ b/docs/scripts/update-docs/templates/components-table.jinja2 @@ -0,0 +1,43 @@ +| Component | GitHub Repo | Support status | Version | +|:---|:---|:---|:---| +{%if grouped_components['Receivers'] -%} +|***Receivers***|||| +{% for comp in grouped_components['Receivers'] -%} +| [{{ comp['name'] }}]({% if comp.get('doc_link') %}{{ comp['doc_link'] }}{% else %}{{ comp['html_url'] }}{% endif %}) | {{ comp['repo_link'] }} | {{ comp['support_status'] }} | {{ comp['version'] }} | +{% endfor %} +{%- endif -%} + +{%- if grouped_components['Exporters'] -%} +|***Exporters***|||| +{% for comp in grouped_components['Exporters'] -%} +| [{{ comp['name'] }}]({% if comp.get('doc_link') %}{{ comp['doc_link'] }}{% else %}{{ comp['html_url'] }}{% endif %}) | {{ comp['repo_link'] }} | {{ comp['support_status'] }} | {{ comp['version'] }} | +{% endfor %} +{%- endif -%} + +{%- if grouped_components['Processors'] -%} +|***Processors***|||| +{% for comp in grouped_components['Processors'] -%} +| [{{ comp['name'] }}]({% if comp.get('doc_link') %}{{ comp['doc_link'] }}{% else %}{{ comp['html_url'] }}{% endif %}) | {{ comp['repo_link'] }} | {{ comp['support_status'] }} | {{ comp['version'] }} | +{% endfor %} +{%- endif -%} + +{%- if grouped_components['Connectors'] -%} +|***Connectors***|||| +{% for comp in grouped_components['Connectors'] -%} +| [{{ comp['name'] }}]({% if comp.get('doc_link') %}{{ comp['doc_link'] }}{% else %}{{ comp['html_url'] }}{% endif %}) | {{ comp['repo_link'] }} | {{ comp['support_status'] }} | {{ comp['version'] }} | +{% endfor %} +{%- endif -%} + +{%- if grouped_components['Extensions'] -%} +|***Extensions***|||| +{% for comp in grouped_components['Extensions'] -%} +| [{{ comp['name'] }}]({% if comp.get('doc_link') %}{{ comp['doc_link'] }}{% else %}{{ comp['html_url'] }}{% endif %}) | {{ comp['repo_link'] }} | {{ comp['support_status'] }} | {{ comp['version'] }} | +{% endfor %} +{%- endif -%} + +{%- if grouped_components['Providers'] -%} +|***Providers***|||| +{% for comp in grouped_components['Providers'] -%} +| [{{ comp['name'] }}]({% if comp.get('doc_link') %}{{ comp['doc_link'] }}{% else %}{{ comp['html_url'] }}{% endif %}) | {{ comp['repo_link'] }} | {{ comp['support_status'] }} | {{ comp['version'] }} | +{% endfor %} +{%- endif -%} \ No newline at end of file diff --git a/docs/scripts/update-docs/templates/ocb.jinja2 b/docs/scripts/update-docs/templates/ocb.jinja2 new file mode 100644 index 00000000000..c538f274950 --- /dev/null +++ b/docs/scripts/update-docs/templates/ocb.jinja2 @@ -0,0 +1,55 @@ +```yaml +dist: + otelcol_edot: + description: "Elastic Distribution of OpenTelemetry Collector" + output_path: ./dist/otelcol_edot + builds: + - name: otelcol_edot + goos: [linux, darwin, windows] + goarch: [amd64, arm64] + output_path: ./dist/otelcol_edot_{{ "{{ .OS }}" }}_{{ "{{ .Arch }}" }} + env: + - CGO_ENABLED=0 + - GOOS={{ "{{ .OS }}" }} + - GOARCH={{ "{{ .Arch }}" }} + ldflags: + - -s -w + - -X go.opentelemetry.io/collector/otelcol.buildTimestamp={{ "{{ .BuildTimestamp }}" }} + - -X go.opentelemetry.io/collector/otelcol.version={{ "{{ .Version }}" }} + +receivers: +{%- for comp in grouped_components.get('Receivers', []) %} + {{ comp.name }}: + gomod: {{ comp.dep }} +{%- endfor %} + +processors: +{%- for comp in grouped_components.get('Processors', []) %} + {{ comp.name }}: + gomod: {{ comp.dep }} +{%- endfor %} + +exporters: +{%- for comp in grouped_components.get('Exporters', []) %} + {{ comp.name }}: + gomod: {{ comp.dep }} +{%- endfor %} + +connectors: +{%- for comp in grouped_components.get('Connectors', []) %} + {{ comp.name }}: + gomod: {{ comp.dep }} +{%- endfor %} + +extensions: +{%- for comp in grouped_components.get('Extensions', []) %} + {{ comp.name }}: + gomod: {{ comp.dep }} +{%- endfor %} + +providers: +{%- for comp in grouped_components.get('Providers', []) %} + {{ comp.name }}: + gomod: {{ comp.dep }} +{%- endfor %} +``` \ No newline at end of file diff --git a/docs/scripts/update-docs/update-components-docs.py b/docs/scripts/update-docs/update-components-docs.py new file mode 100644 index 00000000000..4f9a3973906 --- /dev/null +++ b/docs/scripts/update-docs/update-components-docs.py @@ -0,0 +1,316 @@ +""" +EDOT Collector Documentation Generation Tools + +This module provides tools for automatically generating and updating EDOT Collector documentation +based on local data sources. It includes functionality for: + +- Component table generation from local Elastic Agent go.mod files +- OCB (OpenTelemetry Collector Builder) file generation + +The scripts read data from local Elastic Agent repository files and generate documentation +using Jinja2 templates. +""" + +from jinja2 import Environment, FileSystemLoader +from collections import defaultdict +import yaml +import re +from pathlib import Path + +TABLE_TAG = 'edot-collector-components-table' +DEPS_TAG = 'edot-collector-components-ocb' + +EDOT_COLLECTOR_DIR = '../../../docs/reference/edot-collector' +TEMPLATE_COLLECTOR_COMPONENTS_TABLE = 'templates/components-table.jinja2' +TEMPLATE_COLLECTOR_OCB_FILE = 'templates/ocb.jinja2' +COMPONENT_DOCS_YAML = '../../../docs/reference/edot-collector/component-docs.yml' + + +def get_core_components(version='main'): + """Read and parse the local core-components.yaml file to determine support status""" + # Use local file path instead of GitHub URL + core_components_path = '../../../internal/pkg/otel/core-components.yaml' + print(f"Reading core components from local file: {core_components_path}") + + try: + with open(core_components_path, 'r', encoding='utf-8') as file: + content = file.read() + except FileNotFoundError: + print(f"Could not find core components file at {core_components_path}") + return [] + except Exception as e: + print(f"Error reading core components file: {e}") + return [] + + try: + data = yaml.safe_load(content) + return data.get('components', []) + except yaml.YAMLError as e: + print(f"Error parsing core-components.yaml: {e}") + return [] + +def dep_to_component(dep): + url = dep[:dep.rfind(' v')].strip() + html_url = url + repo_link = '[OTel Contrib Repo](https://github.com/open-telemetry/opentelemetry-collector-contrib)' + if url.startswith('github.com/'): + pattern = r'github.com/(?P[^/]*)/(?P[^/]*)/(?P[^/]*)/(?P.*)' + match = re.search(pattern, url) + if match: + html_url = f'https://github.com/{match.group("org")}/{match.group("repo")}/tree/main/{match.group("comp_type")}/{match.group("comp_name")}' + if match.group("repo") == 'opentelemetry-collector-components': + repo_link = '[Elastic Repo](https://github.com/elastic/opentelemetry-collector-components)' + elif url.startswith('go.opentelemetry.io/collector'): + pattern = r'go.opentelemetry.io/collector/(?P[^/]*)/(?P.*)' + match = re.search(pattern, url) + if match: + html_url = f'https://github.com/open-telemetry/opentelemetry-collector/tree/main/{match.group("comp_type")}/{match.group("comp_name")}' + repo_link = '[OTel Core Repo](https://github.com/open-telemetry/opentelemetry-collector)' + + comp = { + 'name': dep[(dep.rfind('/')+1):(dep.rfind(' ')+1)], + 'version': dep[(dep.rfind(' ')+1):], + 'html_url': html_url, + 'repo_link': repo_link, + 'dep': dep.strip() + } + return comp + +def get_otel_col_upstream_version(): + """Read the OpenTelemetry Collector version from the local go.mod file""" + go_mod_path = '../../../go.mod' + print(f"Reading go.mod from local file: {go_mod_path}") + + try: + with open(go_mod_path, 'r', encoding='utf-8') as file: + content = file.read() + except FileNotFoundError: + print(f"Could not find go.mod file at {go_mod_path}") + return '' + except Exception as e: + print(f"Error reading go.mod file: {e}") + return '' + + lines = content.splitlines() + for line in lines: + if 'go.opentelemetry.io/collector/otelcol ' in line: + return line[(line.rfind('v')+1):] + + return '' + +def get_collector_version(): + """Read the version from the local version.go file""" + version_file_path = '../../../version/version.go' + print(f"Reading version from local file: {version_file_path}") + + try: + with open(version_file_path, 'r', encoding='utf-8') as file: + content = file.read() + + lines = content.splitlines() + for line in lines: + if line.strip().startswith('const defaultBeatVersion ='): + # Extract version from line like: const defaultBeatVersion = "9.2.0" + version_match = re.search(r'"([^"]+)"', line) + if version_match: + return version_match.group(1) + + # If we get here, the version line was not found + raise ValueError(f"Could not find 'const defaultBeatVersion =' in {version_file_path}") + except FileNotFoundError: + print(f"Error: Could not find version file at {version_file_path}") + raise + except Exception as e: + print(f"Error reading version file: {e}") + raise + +def get_otel_components(version='main', component_docs_mapping=None): + """Read OpenTelemetry components from the local go.mod file""" + go_mod_path = '../../../go.mod' + print(f"Reading go.mod from local file: {go_mod_path}") + + try: + with open(go_mod_path, 'r', encoding='utf-8') as file: + elastic_agent_go_mod = file.read() + except FileNotFoundError: + print(f"Could not find go.mod file at {go_mod_path}") + return None + except Exception as e: + print(f"Error reading go.mod file: {e}") + return None + + # Get the list of core components + core_components = get_core_components(version) + print(f"Found {len(core_components)} core components") + + lines = elastic_agent_go_mod.splitlines() + components_type = ['receiver', 'connector', 'processor', 'exporter', 'extension', 'provider'] + otel_deps = [line for line in lines if (not line.endswith('// indirect') and ("=>" not in line) and (any(f'/{comp}/' in line for comp in components_type)))] + otel_components = list(map(dep_to_component, otel_deps)) + + # Add support status and documentation links to each component + for comp in otel_components: + # Extract the component name without the suffix (e.g., 'filelogreceiver' from 'filelogreceiver ') + comp_name = comp['name'].strip() + # Check if this component is in the core components list + if comp_name in core_components: + comp['support_status'] = '[Core]' + else: + comp['support_status'] = '[Extended]' + + # Add documentation link if available + if component_docs_mapping and comp_name in component_docs_mapping: + comp['doc_link'] = component_docs_mapping[comp_name]['doc_path'] + else: + comp['doc_link'] = None + + components_grouped = defaultdict(list) + + for comp in otel_components: + for substring in components_type: + if f'/{substring}/' in comp['dep']: + components_grouped[f'{substring.capitalize()}s'].append(comp) + break # Assumes each string matches only one group + + components_grouped = dict(components_grouped) + + for key, group in components_grouped.items(): + components_grouped[key] = sorted(group, key=lambda comp: comp['name']) + + return components_grouped + +def find_files_with_substring(directory, substring): + matching_files = [] + # Compile the substring into a regular expression for case-insensitive search + pattern = re.compile(re.escape(substring), re.IGNORECASE) + # Use pathlib to iterate over all files in the directory and subdirectories + for file_path in Path(directory).rglob('*'): + if file_path.is_file(): + try: + with open(file_path, 'r', encoding='utf-8') as file: + content = file.read() + if pattern.search(content): + matching_files.append(str(file_path)) + except (UnicodeDecodeError, PermissionError) as e: + # Skip files that can't be read due to encoding issues or permission errors + print(f"Skipping {file_path}: {e}") + return matching_files + +def render_markdown(data, template): + # Set up the Jinja2 environment + env = Environment(loader=FileSystemLoader('.')) + + # Load the template + template = env.get_template(template) + + # Define the data to pass to the template + + return template.render(data) + +def render_components_into_file(dir, data, template, tag): + output = render_markdown(data, template) + start_tag = f'% start:{tag}' + end_tag = f'% end:{tag}' + + filesPaths = find_files_with_substring(dir, start_tag) + + for filePath in filesPaths: + with open(filePath, 'r', encoding='utf-8') as file: + content = file.read() + + pattern = start_tag + r'.*?' + end_tag + new_content = f'{start_tag}\n{output}\n{end_tag}' + updated_content = re.sub(pattern, new_content, content, flags=re.DOTALL) + + with open(filePath, 'w', encoding='utf-8') as file: + file.write(updated_content) + +def check_markdown_generation(dir, data, template, tag): + output = render_markdown(data, template) + start_tag = f'% start:{tag}' + end_tag = f'% end:{tag}' + + filesPaths = find_files_with_substring(dir, start_tag) + + for filePath in filesPaths: + with open(filePath, 'r', encoding='utf-8') as file: + content = file.read() + + pattern = start_tag + r'(.*?)' + end_tag + + matches = re.findall(pattern, content, re.DOTALL) + + for match in matches: + if match.strip() != output.strip(): + print(f'Warning: Generated markdown is outdated in file {filePath}! Regenerate markdown by running `make generate`!') + return False; + + return True; + +def get_component_docs_mapping(source_file): + """Load component documentation mapping from YAML file""" + try: + with open(source_file, 'r') as file: + data = yaml.safe_load(file) + return data.get('components', {}) + except FileNotFoundError: + print(f"Component docs mapping file not found: {source_file}") + return {} + except yaml.YAMLError as exc: + print(f"Error reading component docs YAML file: {exc}") + return {} + +def check_markdown(): + col_version = get_collector_version() + print(f"Collector version: {col_version}") + + # Load component documentation mapping + component_docs_mapping = get_component_docs_mapping(COMPONENT_DOCS_YAML) + print(f"Loaded {len(component_docs_mapping)} component documentation mappings") + + # Read components from local files + components = get_otel_components(col_version, component_docs_mapping) + + if components is None: + print("Failed to read components from local files") + return False + + otel_col_version = get_otel_col_upstream_version() + data = { + 'grouped_components': components, + 'otel_col_version': otel_col_version + } + tables = check_markdown_generation(EDOT_COLLECTOR_DIR, data, TEMPLATE_COLLECTOR_COMPONENTS_TABLE, TABLE_TAG) + ocb = check_markdown_generation(EDOT_COLLECTOR_DIR, data, TEMPLATE_COLLECTOR_OCB_FILE, DEPS_TAG) + + return tables and ocb + +def generate_markdown(): + col_version = get_collector_version() + print(f"Collector version: {col_version}") + + # Load component documentation mapping + component_docs_mapping = get_component_docs_mapping(COMPONENT_DOCS_YAML) + print(f"Loaded {len(component_docs_mapping)} component documentation mappings") + + # Read components from local files + components = get_otel_components(col_version, component_docs_mapping) + + if components is None: + print("Failed to read components from local files") + return + + otel_col_version = get_otel_col_upstream_version() + data = { + 'grouped_components': components, + 'otel_col_version': otel_col_version + } + render_components_into_file(EDOT_COLLECTOR_DIR, data, TEMPLATE_COLLECTOR_COMPONENTS_TABLE, TABLE_TAG) + render_components_into_file(EDOT_COLLECTOR_DIR, data, TEMPLATE_COLLECTOR_OCB_FILE, DEPS_TAG) + +if __name__ == "__main__": + import sys + if len(sys.argv) > 1 and sys.argv[1] == "check": + check_markdown() + else: + generate_markdown()