From 71c57f69608aadbe5b843083752f96a062a3e160 Mon Sep 17 00:00:00 2001 From: Rohan Gujarathi Date: Fri, 25 Oct 2024 11:39:33 -0700 Subject: [PATCH 01/32] add processor and translator for kueue metrics (#844) Co-authored-by: Rohan Gujarathi --- .github/workflows/PR-build.yml | 69 --- internal/containerinsightscommon/const.go | 13 + internal/containerinsightscommon/util.go | 2 +- plugins/processors/kueueattributes/config.go | 19 + .../processors/kueueattributes/config_test.go | 18 + plugins/processors/kueueattributes/factory.go | 56 ++ .../kueueattributes/factory_test.go | 45 ++ .../processors/kueueattributes/processor.go | 94 ++++ .../kueueattributes/processor_test.go | 121 ++++ service/defaultcomponents/components.go | 2 + service/defaultcomponents/components_test.go | 1 + .../appsignals_and_eks_config.json | 3 +- .../appsignals_and_eks_config.yaml | 1 + .../appsignals_and_k8s_config.json | 3 +- .../appsignals_and_k8s_config.yaml | 1 + .../appsignals_fallback_and_eks_config.json | 3 +- .../appsignals_fallback_and_eks_config.yaml | 1 + .../appsignals_over_fallback_config.json | 3 +- .../appsignals_over_fallback_config.yaml | 1 + .../base_container_insights_config.json | 3 +- .../base_container_insights_config.yaml | 1 + .../emf_and_kubernetes_config.json | 3 +- .../emf_and_kubernetes_config.yaml | 1 + .../emf_and_kubernetes_with_gpu_config.json | 3 +- .../emf_and_kubernetes_with_gpu_config.yaml | 1 + .../emf_and_kubernetes_with_kueue_config.conf | 27 + .../emf_and_kubernetes_with_kueue_config.json | 21 + .../emf_and_kubernetes_with_kueue_config.yaml | 517 ++++++++++++++++++ .../kubernetes_on_prem_config.json | 3 +- .../kubernetes_on_prem_config.yaml | 1 + .../sampleConfig/log_ecs_metric_only.yaml | 1 + .../logs_and_kubernetes_config.json | 3 +- .../logs_and_kubernetes_config.yaml | 1 + translator/tocwconfig/tocwconfig_test.go | 12 + translator/translate/otel/common/common.go | 1 + .../pipeline/containerinsights/translator.go | 5 + .../containerinsights/translator_test.go | 12 +- .../otel/processor/kueue/translator.go | 33 ++ .../awscontainerinsight/translator.go | 1 + .../receiver/awscontainerinsight/utils.go | 4 + 40 files changed, 1028 insertions(+), 82 deletions(-) create mode 100644 plugins/processors/kueueattributes/config.go create mode 100644 plugins/processors/kueueattributes/config_test.go create mode 100644 plugins/processors/kueueattributes/factory.go create mode 100644 plugins/processors/kueueattributes/factory_test.go create mode 100644 plugins/processors/kueueattributes/processor.go create mode 100644 plugins/processors/kueueattributes/processor_test.go create mode 100644 translator/tocwconfig/sampleConfig/emf_and_kubernetes_with_kueue_config.conf create mode 100644 translator/tocwconfig/sampleConfig/emf_and_kubernetes_with_kueue_config.json create mode 100644 translator/tocwconfig/sampleConfig/emf_and_kubernetes_with_kueue_config.yaml create mode 100644 translator/translate/otel/processor/kueue/translator.go diff --git a/.github/workflows/PR-build.yml b/.github/workflows/PR-build.yml index ec4b07249c..75c71b1b1a 100644 --- a/.github/workflows/PR-build.yml +++ b/.github/workflows/PR-build.yml @@ -68,72 +68,3 @@ jobs: - name: Check license and imports if: needs.changes.outputs.lint == 'true' run: make simple-lint - - build: - needs: [lint, changes] - name: Build ${{ matrix.os }} - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - os: [ ubuntu-latest, windows-2019, windows-latest, macos-12] - include: - - os: ubuntu-latest - family: linux - cache-path: | - ~/.cache/go-build - ~/go/pkg/mod - - os: macos-12 - family: darwin - cache-path: | - ~/Library/Caches/go-build - ~/go/pkg/mod - - os: windows-2019 - family: windows - cache-path: | - ~\AppData\Local\go-build - ~\go\pkg\mod - - os: windows-latest - family: windows - cache-path: | - ~\AppData\Local\go-build - ~\go\pkg\mod - steps: - - name: Set up Go 1.x - if: needs.changes.outputs.build == 'true' - uses: actions/setup-go@v4 - with: - go-version: ~1.22.2 - cache: false - - - name: Check out code - if: needs.changes.outputs.build == 'true' - uses: actions/checkout@v3 - - - name: Cache binaries - id: cached_binaries - if: needs.changes.outputs.build == 'true' - uses: actions/cache@v3 - with: - key: "cached-binaries-${{ matrix.os }}-${{ github.sha }}" - path: go.mod - - - name: Cache build output - if: steps.cached_binaries.outputs.cache-hit != 'true' && needs.changes.outputs.build == 'true' - uses: actions/cache@v3 - with: - path: ${{ matrix.cache-path }} - key: v1-go-pkg-mod-${{ matrix.os }}-${{ hashFiles('**/go.sum') }} - - - name: Install make - if: matrix.family == 'windows' && steps.cached_binaries.outputs.cache-hit != 'true' && needs.changes.outputs.build == 'true' - run: choco install make - - - name: Unit Test - if: steps.cached_binaries.outputs.cache-hit != 'true' && needs.changes.outputs.build == 'true' - run: make test - - - name: Build - if: steps.cached_binaries.outputs.cache-hit != 'true' && needs.changes.outputs.build == 'true' - run: make amazon-cloudwatch-agent-${{ matrix.family }} - \ No newline at end of file diff --git a/internal/containerinsightscommon/const.go b/internal/containerinsightscommon/const.go index 6a0a44706f..d957f3152f 100644 --- a/internal/containerinsightscommon/const.go +++ b/internal/containerinsightscommon/const.go @@ -18,6 +18,12 @@ const ( SourcesKey = "Sources" GpuDeviceKey = "GpuDevice" + ClusterQueueNameKey = "ClusterQueue" + ClusterQueueStatusKey = "Status" + ClusterQueueReasonKey = "Reason" + ClusterQueueResourceKey = "Resource" + Flavor = "Flavor" + // metric collected CpuTotal = "cpu_usage_total" CpuUser = "cpu_usage_user" @@ -102,10 +108,17 @@ const ( NeuronHardware = "neuron_hardware" NeuronExecutionLatency = "neuron_execution_latency" + KueuePendingWorkloads = "kueue_pending_workloads" + KueueEvictedWorkloadsTotal = "kueue_evicted_workloads_total" + KueueAdmittedActiveWorkloads = "kueue_admitted_active_workloads" + KueueClusterQueueResourceUsage = "kueue_cluster_queue_resource_usage" + KueueClusterQueueNominalUsage = "kueue_cluster_queue_nominal_quota" + TypeCluster = "Cluster" TypeClusterService = "ClusterService" TypeClusterNamespace = "ClusterNamespace" TypeService = "Service" + TypeClusterQueue = "ClusterQueue" // Both TypeInstance and TypeNode mean EC2 Instance, they are used in ECS and EKS separately TypeInstance = "Instance" diff --git a/internal/containerinsightscommon/util.go b/internal/containerinsightscommon/util.go index 352bbcb0a8..7e7d65b159 100644 --- a/internal/containerinsightscommon/util.go +++ b/internal/containerinsightscommon/util.go @@ -50,7 +50,7 @@ func MetricName(mType string, name string) string { prefix = containerPrefix case TypeService: prefix = service - case TypeCluster, TypeGpuCluster: + case TypeCluster, TypeGpuCluster, TypeClusterQueue: prefix = cluster case K8sNamespace: prefix = namespace diff --git a/plugins/processors/kueueattributes/config.go b/plugins/processors/kueueattributes/config.go new file mode 100644 index 0000000000..b2c04331b7 --- /dev/null +++ b/plugins/processors/kueueattributes/config.go @@ -0,0 +1,19 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +package kueueattributes + +import ( + "go.opentelemetry.io/collector/component" +) + +type Config struct{} + +// Verify Config implements Processor interface. +var _ component.Config = (*Config)(nil) + +// Validate does not check for unsupported dimension key-value pairs, because those +// get silently dropped and ignored during translation. +func (cfg *Config) Validate() error { + return nil +} diff --git a/plugins/processors/kueueattributes/config_test.go b/plugins/processors/kueueattributes/config_test.go new file mode 100644 index 0000000000..5be7a7c8f2 --- /dev/null +++ b/plugins/processors/kueueattributes/config_test.go @@ -0,0 +1,18 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +package kueueattributes + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "go.opentelemetry.io/collector/confmap" +) + +func TestUnmarshalDefaultConfig(t *testing.T) { + factory := NewFactory() + cfg := factory.CreateDefaultConfig() + assert.NoError(t, confmap.New().Unmarshal(cfg)) + assert.Equal(t, factory.CreateDefaultConfig(), cfg) +} diff --git a/plugins/processors/kueueattributes/factory.go b/plugins/processors/kueueattributes/factory.go new file mode 100644 index 0000000000..511b525e28 --- /dev/null +++ b/plugins/processors/kueueattributes/factory.go @@ -0,0 +1,56 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +package kueueattributes + +import ( + "context" + "fmt" + + "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/consumer" + "go.opentelemetry.io/collector/processor" + "go.opentelemetry.io/collector/processor/processorhelper" +) + +const ( + stability = component.StabilityLevelBeta +) + +var ( + TypeStr, _ = component.NewType("kueueattributes") + processorCapabilities = consumer.Capabilities{MutatesData: true} +) + +func NewFactory() processor.Factory { + return processor.NewFactory( + TypeStr, + createDefaultConfig, + processor.WithMetrics(createMetricsProcessor, stability)) +} + +func createDefaultConfig() component.Config { + return &Config{} +} + +func createMetricsProcessor( + ctx context.Context, + set processor.CreateSettings, + cfg component.Config, + nextConsumer consumer.Metrics, +) (processor.Metrics, error) { + processorConfig, ok := cfg.(*Config) + if !ok { + return nil, fmt.Errorf("configuration parsing error") + } + + metricsProcessor := newKueueAttributesProcessor(processorConfig, set.Logger) + + return processorhelper.NewMetricsProcessor( + ctx, + set, + cfg, + nextConsumer, + metricsProcessor.processMetrics, + processorhelper.WithCapabilities(processorCapabilities)) +} diff --git a/plugins/processors/kueueattributes/factory_test.go b/plugins/processors/kueueattributes/factory_test.go new file mode 100644 index 0000000000..bc636e35f1 --- /dev/null +++ b/plugins/processors/kueueattributes/factory_test.go @@ -0,0 +1,45 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +package kueueattributes + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/component/componenttest" + "go.opentelemetry.io/collector/consumer/consumertest" + "go.opentelemetry.io/collector/processor/processortest" +) + +func TestCreateDefaultConfig(t *testing.T) { + factory := NewFactory() + require.NotNil(t, factory) + + cfg := factory.CreateDefaultConfig() + assert.NotNil(t, cfg, "failed to create default config") + assert.NoError(t, componenttest.CheckConfigStruct(cfg)) +} + +func TestCreateProcessor(t *testing.T) { + factory := NewFactory() + require.NotNil(t, factory) + + cfg := factory.CreateDefaultConfig() + setting := processortest.NewNopCreateSettings() + + tProcessor, err := factory.CreateTracesProcessor(context.Background(), setting, cfg, consumertest.NewNop()) + assert.Equal(t, err, component.ErrDataTypeIsNotSupported) + assert.Nil(t, tProcessor) + + mProcessor, err := factory.CreateMetricsProcessor(context.Background(), setting, cfg, consumertest.NewNop()) + assert.NoError(t, err) + assert.NotNil(t, mProcessor) + + lProcessor, err := factory.CreateLogsProcessor(context.Background(), setting, cfg, consumertest.NewNop()) + assert.Equal(t, err, component.ErrDataTypeIsNotSupported) + assert.Nil(t, lProcessor) +} diff --git a/plugins/processors/kueueattributes/processor.go b/plugins/processors/kueueattributes/processor.go new file mode 100644 index 0000000000..3bc7a5ebdb --- /dev/null +++ b/plugins/processors/kueueattributes/processor.go @@ -0,0 +1,94 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +package kueueattributes + +import ( + "context" + "strings" + + "go.opentelemetry.io/collector/pdata/pcommon" + "go.opentelemetry.io/collector/pdata/pmetric" + "go.uber.org/zap" + + "github.com/aws/amazon-cloudwatch-agent/internal/containerinsightscommon" +) + +const ( + kueueMetricsIdentifier = "kueue" +) + +var kueueLabelFilter = map[string]interface{}{ + containerinsightscommon.ClusterNameKey: nil, + containerinsightscommon.ClusterQueueNameKey: nil, + containerinsightscommon.ClusterQueueStatusKey: nil, + containerinsightscommon.ClusterQueueReasonKey: nil, + containerinsightscommon.ClusterQueueResourceKey: nil, + containerinsightscommon.Flavor: nil, +} + +type kueueAttributesProcessor struct { + *Config + logger *zap.Logger + labelFilter map[string]interface{} +} + +func newKueueAttributesProcessor(config *Config, logger *zap.Logger) *kueueAttributesProcessor { + d := &kueueAttributesProcessor{ + Config: config, + logger: logger, + labelFilter: kueueLabelFilter, + } + return d +} + +func (d *kueueAttributesProcessor) processMetrics(_ context.Context, md pmetric.Metrics) (pmetric.Metrics, error) { + rms := md.ResourceMetrics() + for i := 0; i < rms.Len(); i++ { + sms := rms.At(i).ScopeMetrics() + for j := 0; j < sms.Len(); j++ { + metrics := sms.At(j).Metrics() + for k := 0; k < metrics.Len(); k++ { + m := metrics.At(k) + d.processMetricAttributes(m) + } + } + } + return md, nil +} + +func (d *kueueAttributesProcessor) processMetricAttributes(m pmetric.Metric) { + + // only decorate kueue metrics + if !strings.HasPrefix(m.Name(), kueueMetricsIdentifier) { + return + } + + var dps pmetric.NumberDataPointSlice + switch m.Type() { + case pmetric.MetricTypeGauge: + dps = m.Gauge().DataPoints() + case pmetric.MetricTypeSum: + dps = m.Sum().DataPoints() + default: + d.logger.Debug("Ignore unknown metric type", zap.String(containerinsightscommon.MetricType, m.Type().String())) + } + + for i := 0; i < dps.Len(); i++ { + d.filterAttributes(dps.At(i).Attributes()) + } +} + +func (d *kueueAttributesProcessor) filterAttributes(attributes pcommon.Map) { + labels := d.labelFilter + if len(labels) == 0 { + return + } + // remove labels that are not in the keep list + attributes.RemoveIf(func(k string, _ pcommon.Value) bool { + if _, ok := labels[k]; ok { + return false + } + return true + }) +} diff --git a/plugins/processors/kueueattributes/processor_test.go b/plugins/processors/kueueattributes/processor_test.go new file mode 100644 index 0000000000..98f4378c55 --- /dev/null +++ b/plugins/processors/kueueattributes/processor_test.go @@ -0,0 +1,121 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +package kueueattributes + +import ( + "context" + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + "go.opentelemetry.io/collector/pdata/pmetric" + "go.uber.org/zap" +) + +func TestProcessMetricsForKueueMetrics(t *testing.T) { + logger, _ := zap.NewDevelopment() + kp := newKueueAttributesProcessor(createDefaultConfig().(*Config), logger) + ctx := context.Background() + + testcases := map[string]struct { + resource string + metrics pmetric.Metrics + wantMetriccnt int + want []map[string]string + }{ + "nonKueue": { + metrics: generateKueueMetrics("someOthermetric", []map[string]string{ + { + "ClusterName": "cluster", + }, + }), + wantMetriccnt: 1, + want: []map[string]string{ + { + "ClusterName": "cluster", + }, + }, + }, + "KeepAll": { + metrics: generateKueueMetrics("kueue_pending_workloads", []map[string]string{ + { + "ClusterName": "cluster", + "ClusterQueue": "production", + "Status": "active", + }, + { + "ClusterName": "cluster", + "ClusterQueue": "development", + "Status": "inadmissible", + }, + }), + wantMetriccnt: 1, + want: []map[string]string{ + { + "ClusterName": "cluster", + "ClusterQueue": "production", + "Status": "active", + }, + { + "ClusterName": "cluster", + "ClusterQueue": "development", + "Status": "inadmissible", + }, + }, + }, + "dropLabel": { + metrics: generateKueueMetrics("kueue_pending_workloads", []map[string]string{ + { + "ClusterName": "cluster", + "ClusterQueue": "production", + "Status": "active", + "Pod": "somepod", + }, + }), + wantMetriccnt: 1, + want: []map[string]string{ + { + "ClusterName": "cluster", + "ClusterQueue": "production", + "Status": "active", + }, + }, + }, + } + + for tname, tc := range testcases { + fmt.Printf("running %s\n", tname) + ms, _ := kp.processMetrics(ctx, tc.metrics) + assert.Equal(t, tc.wantMetriccnt, ms.MetricCount()) + if tc.wantMetriccnt > 0 { + dps := ms.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(0).Gauge().DataPoints() + assert.Equal(t, len(tc.want), dps.Len()) + for i, dim := range tc.want { + attrs := dps.At(i).Attributes() + assert.Equal(t, len(dim), attrs.Len()) + for k, v := range dim { + got, ok := attrs.Get(k) + assert.True(t, ok) + assert.Equal(t, v, got.Str()) + } + } + } + } + +} + +func generateKueueMetrics(metricName string, dimensions []map[string]string) pmetric.Metrics { + md := pmetric.NewMetrics() + ms := md.ResourceMetrics().AppendEmpty().ScopeMetrics().AppendEmpty().Metrics().AppendEmpty() + ms.SetName(metricName) + dps := ms.SetEmptyGauge().DataPoints() + for _, dim := range dimensions { + dp := dps.AppendEmpty() + dp.SetIntValue(10) + for k, v := range dim { + dp.Attributes().PutStr(k, v) + } + } + return md +} diff --git a/service/defaultcomponents/components.go b/service/defaultcomponents/components.go index 0645b40bc6..b11b8f9a1b 100644 --- a/service/defaultcomponents/components.go +++ b/service/defaultcomponents/components.go @@ -57,6 +57,7 @@ import ( "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsapplicationsignals" "github.com/aws/amazon-cloudwatch-agent/plugins/processors/ec2tagger" "github.com/aws/amazon-cloudwatch-agent/plugins/processors/gpuattributes" + "github.com/aws/amazon-cloudwatch-agent/plugins/processors/kueueattributes" "github.com/aws/amazon-cloudwatch-agent/processor/rollupprocessor" ) @@ -91,6 +92,7 @@ func Factories() (otelcol.Factories, error) { ec2tagger.NewFactory(), filterprocessor.NewFactory(), gpuattributes.NewFactory(), + kueueattributes.NewFactory(), groupbytraceprocessor.NewFactory(), k8sattributesprocessor.NewFactory(), memorylimiterprocessor.NewFactory(), diff --git a/service/defaultcomponents/components_test.go b/service/defaultcomponents/components_test.go index 68fd233249..52d8a9ca22 100644 --- a/service/defaultcomponents/components_test.go +++ b/service/defaultcomponents/components_test.go @@ -47,6 +47,7 @@ func TestComponents(t *testing.T) { "experimental_metricsgeneration", "filter", "gpuattributes", + "kueueattributes", "groupbytrace", "k8sattributes", "memory_limiter", diff --git a/translator/tocwconfig/sampleConfig/appsignals_and_eks_config.json b/translator/tocwconfig/sampleConfig/appsignals_and_eks_config.json index 0c44e570b4..c6d415b3c1 100644 --- a/translator/tocwconfig/sampleConfig/appsignals_and_eks_config.json +++ b/translator/tocwconfig/sampleConfig/appsignals_and_eks_config.json @@ -20,7 +20,8 @@ "metrics_collection_interval": 30, "disable_metric_extraction": true, "enhanced_container_insights": false, - "accelerated_compute_metrics": false + "accelerated_compute_metrics": false, + "kueue_container_insights": false } }, "force_flush_interval": 5, diff --git a/translator/tocwconfig/sampleConfig/appsignals_and_eks_config.yaml b/translator/tocwconfig/sampleConfig/appsignals_and_eks_config.yaml index 0993f8e186..17eeed4133 100644 --- a/translator/tocwconfig/sampleConfig/appsignals_and_eks_config.yaml +++ b/translator/tocwconfig/sampleConfig/appsignals_and_eks_config.yaml @@ -591,6 +591,7 @@ processors: receivers: awscontainerinsightreceiver: accelerated_compute_metrics: false + kueue_container_insights: false add_container_name_metric_label: false add_full_pod_name_metric_label: false add_service_as_attribute: true diff --git a/translator/tocwconfig/sampleConfig/appsignals_and_k8s_config.json b/translator/tocwconfig/sampleConfig/appsignals_and_k8s_config.json index 4ca852e3d4..7f823a760d 100644 --- a/translator/tocwconfig/sampleConfig/appsignals_and_k8s_config.json +++ b/translator/tocwconfig/sampleConfig/appsignals_and_k8s_config.json @@ -16,7 +16,8 @@ "metrics_collection_interval": 30, "disable_metric_extraction": true, "enhanced_container_insights": false, - "accelerated_compute_metrics": false + "accelerated_compute_metrics": false, + "kueue_container_insights": false } }, "force_flush_interval": 5, diff --git a/translator/tocwconfig/sampleConfig/appsignals_and_k8s_config.yaml b/translator/tocwconfig/sampleConfig/appsignals_and_k8s_config.yaml index d66ab2f1ba..acc2bac366 100644 --- a/translator/tocwconfig/sampleConfig/appsignals_and_k8s_config.yaml +++ b/translator/tocwconfig/sampleConfig/appsignals_and_k8s_config.yaml @@ -592,6 +592,7 @@ processors: receivers: awscontainerinsightreceiver: accelerated_compute_metrics: false + kueue_container_insights: false add_container_name_metric_label: false add_full_pod_name_metric_label: false add_service_as_attribute: true diff --git a/translator/tocwconfig/sampleConfig/appsignals_fallback_and_eks_config.json b/translator/tocwconfig/sampleConfig/appsignals_fallback_and_eks_config.json index 140b2d83ab..a51cb35bc3 100644 --- a/translator/tocwconfig/sampleConfig/appsignals_fallback_and_eks_config.json +++ b/translator/tocwconfig/sampleConfig/appsignals_fallback_and_eks_config.json @@ -20,7 +20,8 @@ "metrics_collection_interval": 30, "disable_metric_extraction": true, "enhanced_container_insights": false, - "accelerated_compute_metrics": false + "accelerated_compute_metrics": false, + "kueue_container_insights": false } }, "force_flush_interval": 5, diff --git a/translator/tocwconfig/sampleConfig/appsignals_fallback_and_eks_config.yaml b/translator/tocwconfig/sampleConfig/appsignals_fallback_and_eks_config.yaml index 0993f8e186..17eeed4133 100644 --- a/translator/tocwconfig/sampleConfig/appsignals_fallback_and_eks_config.yaml +++ b/translator/tocwconfig/sampleConfig/appsignals_fallback_and_eks_config.yaml @@ -591,6 +591,7 @@ processors: receivers: awscontainerinsightreceiver: accelerated_compute_metrics: false + kueue_container_insights: false add_container_name_metric_label: false add_full_pod_name_metric_label: false add_service_as_attribute: true diff --git a/translator/tocwconfig/sampleConfig/appsignals_over_fallback_config.json b/translator/tocwconfig/sampleConfig/appsignals_over_fallback_config.json index 371a6c9957..83e9d88019 100644 --- a/translator/tocwconfig/sampleConfig/appsignals_over_fallback_config.json +++ b/translator/tocwconfig/sampleConfig/appsignals_over_fallback_config.json @@ -31,7 +31,8 @@ "metrics_collection_interval": 30, "disable_metric_extraction": true, "enhanced_container_insights": false, - "accelerated_compute_metrics": false + "accelerated_compute_metrics": false, + "kueue_container_insights": false } }, "force_flush_interval": 5, diff --git a/translator/tocwconfig/sampleConfig/appsignals_over_fallback_config.yaml b/translator/tocwconfig/sampleConfig/appsignals_over_fallback_config.yaml index 0993f8e186..17eeed4133 100644 --- a/translator/tocwconfig/sampleConfig/appsignals_over_fallback_config.yaml +++ b/translator/tocwconfig/sampleConfig/appsignals_over_fallback_config.yaml @@ -591,6 +591,7 @@ processors: receivers: awscontainerinsightreceiver: accelerated_compute_metrics: false + kueue_container_insights: false add_container_name_metric_label: false add_full_pod_name_metric_label: false add_service_as_attribute: true diff --git a/translator/tocwconfig/sampleConfig/base_container_insights_config.json b/translator/tocwconfig/sampleConfig/base_container_insights_config.json index 510cb41463..45787f069a 100644 --- a/translator/tocwconfig/sampleConfig/base_container_insights_config.json +++ b/translator/tocwconfig/sampleConfig/base_container_insights_config.json @@ -11,7 +11,8 @@ "metrics_collection_interval": 30, "disable_metric_extraction": true, "prefer_full_pod_name": true, - "accelerated_compute_metrics": false + "accelerated_compute_metrics": false, + "kueue_container_insights": false } }, "force_flush_interval": 5, diff --git a/translator/tocwconfig/sampleConfig/base_container_insights_config.yaml b/translator/tocwconfig/sampleConfig/base_container_insights_config.yaml index 4db3578ebf..d961e759bf 100644 --- a/translator/tocwconfig/sampleConfig/base_container_insights_config.yaml +++ b/translator/tocwconfig/sampleConfig/base_container_insights_config.yaml @@ -162,6 +162,7 @@ processors: receivers: awscontainerinsightreceiver: accelerated_compute_metrics: false + kueue_container_insights: false add_container_name_metric_label: false add_full_pod_name_metric_label: false add_service_as_attribute: true diff --git a/translator/tocwconfig/sampleConfig/emf_and_kubernetes_config.json b/translator/tocwconfig/sampleConfig/emf_and_kubernetes_config.json index 5581444869..68aa347ee7 100644 --- a/translator/tocwconfig/sampleConfig/emf_and_kubernetes_config.json +++ b/translator/tocwconfig/sampleConfig/emf_and_kubernetes_config.json @@ -11,7 +11,8 @@ "metrics_collection_interval": 30, "disable_metric_extraction": true, "enhanced_container_insights": true, - "accelerated_compute_metrics": false + "accelerated_compute_metrics": false, + "kueue_container_insights": false } }, "force_flush_interval": 5, diff --git a/translator/tocwconfig/sampleConfig/emf_and_kubernetes_config.yaml b/translator/tocwconfig/sampleConfig/emf_and_kubernetes_config.yaml index 1f4fc27b70..e420e06c80 100644 --- a/translator/tocwconfig/sampleConfig/emf_and_kubernetes_config.yaml +++ b/translator/tocwconfig/sampleConfig/emf_and_kubernetes_config.yaml @@ -422,6 +422,7 @@ processors: receivers: awscontainerinsightreceiver: accelerated_compute_metrics: false + kueue_container_insights: false add_container_name_metric_label: true add_full_pod_name_metric_label: true add_service_as_attribute: true diff --git a/translator/tocwconfig/sampleConfig/emf_and_kubernetes_with_gpu_config.json b/translator/tocwconfig/sampleConfig/emf_and_kubernetes_with_gpu_config.json index 8e1ffdbbf9..b73013d933 100644 --- a/translator/tocwconfig/sampleConfig/emf_and_kubernetes_with_gpu_config.json +++ b/translator/tocwconfig/sampleConfig/emf_and_kubernetes_with_gpu_config.json @@ -10,7 +10,8 @@ "cluster_name": "TestCluster", "metrics_collection_interval": 30, "disable_metric_extraction": true, - "enhanced_container_insights": true + "enhanced_container_insights": true, + "kueue_container_insights": false } }, "force_flush_interval": 5, diff --git a/translator/tocwconfig/sampleConfig/emf_and_kubernetes_with_gpu_config.yaml b/translator/tocwconfig/sampleConfig/emf_and_kubernetes_with_gpu_config.yaml index 7cb2fe15f0..99f1b2868e 100644 --- a/translator/tocwconfig/sampleConfig/emf_and_kubernetes_with_gpu_config.yaml +++ b/translator/tocwconfig/sampleConfig/emf_and_kubernetes_with_gpu_config.yaml @@ -1094,6 +1094,7 @@ processors: receivers: awscontainerinsightreceiver: accelerated_compute_metrics: true + kueue_container_insights: false add_container_name_metric_label: true add_full_pod_name_metric_label: true add_service_as_attribute: true diff --git a/translator/tocwconfig/sampleConfig/emf_and_kubernetes_with_kueue_config.conf b/translator/tocwconfig/sampleConfig/emf_and_kubernetes_with_kueue_config.conf new file mode 100644 index 0000000000..007bb60efb --- /dev/null +++ b/translator/tocwconfig/sampleConfig/emf_and_kubernetes_with_kueue_config.conf @@ -0,0 +1,27 @@ +[agent] + collection_jitter = "0s" + debug = false + flush_interval = "1s" + flush_jitter = "0s" + hostname = "host_name_from_env" + interval = "60s" + logfile = "" + logtarget = "lumberjack" + metric_batch_size = 1000 + metric_buffer_limit = 10000 + omit_hostname = false + precision = "" + quiet = false + round_interval = false + +[inputs] + +[outputs] + + [[outputs.cloudwatchlogs]] + endpoint_override = "https://fake_endpoint" + force_flush_interval = "5s" + log_stream_name = "host_name_from_env" + region = "us-east-1" + +[processors] diff --git a/translator/tocwconfig/sampleConfig/emf_and_kubernetes_with_kueue_config.json b/translator/tocwconfig/sampleConfig/emf_and_kubernetes_with_kueue_config.json new file mode 100644 index 0000000000..7a9db60ddd --- /dev/null +++ b/translator/tocwconfig/sampleConfig/emf_and_kubernetes_with_kueue_config.json @@ -0,0 +1,21 @@ +{ + "agent": { + "region": "us-east-1" + }, + "logs": { + "metrics_collected": { + "emf": { + }, + "kubernetes": { + "cluster_name": "TestCluster", + "metrics_collection_interval": 30, + "disable_metric_extraction": true, + "enhanced_container_insights": true, + "accelerated_compute_metrics": false, + "kueue_container_insights": true + } + }, + "force_flush_interval": 5, + "endpoint_override":"https://fake_endpoint" + } +} diff --git a/translator/tocwconfig/sampleConfig/emf_and_kubernetes_with_kueue_config.yaml b/translator/tocwconfig/sampleConfig/emf_and_kubernetes_with_kueue_config.yaml new file mode 100644 index 0000000000..f9f4a755ea --- /dev/null +++ b/translator/tocwconfig/sampleConfig/emf_and_kubernetes_with_kueue_config.yaml @@ -0,0 +1,517 @@ +exporters: + awscloudwatchlogs/emf_logs: + certificate_file_path: "" + emf_only: true + endpoint: https://fake_endpoint + imds_retries: 2 + local_mode: true + log_group_name: emf/logs/default + log_retention: 0 + log_stream_name: host_name_from_env + max_retries: 2 + middleware: agenthealth/logs + no_verify_ssl: false + num_workers: 8 + profile: default + proxy_address: "" + raw_log: true + region: us-east-1 + request_timeout_seconds: 30 + resource_arn: "" + retry_on_failure: + enabled: true + initial_interval: 5s + max_elapsed_time: 5m0s + max_interval: 30s + multiplier: 1.5 + randomization_factor: 0.5 + role_arn: "" + sending_queue: + enabled: true + num_consumers: 1 + queue_size: 1000 + shared_credentials_file: + - /root/.aws/credentials + awsemf/containerinsights: + certificate_file_path: "" + detailed_metrics: false + dimension_rollup_option: NoDimensionRollup + disable_metric_extraction: true + eks_fargate_container_insights_enabled: false + endpoint: https://fake_endpoint + enhanced_container_insights: true + imds_retries: 2 + local_mode: true + log_group_name: /aws/containerinsights/{ClusterName}/performance + log_retention: 0 + log_stream_name: '{NodeName}' + max_retries: 2 + metric_declarations: + - dimensions: + - - ClusterName + - - ClusterName + - ContainerName + - FullPodName + - Namespace + - PodName + - - ClusterName + - ContainerName + - Namespace + - PodName + metric_name_selectors: + - container_cpu_utilization + - container_cpu_utilization_over_container_limit + - container_cpu_limit + - container_cpu_request + - container_memory_utilization + - container_memory_utilization_over_container_limit + - container_memory_failures_total + - container_memory_limit + - container_memory_request + - container_filesystem_usage + - container_filesystem_available + - container_filesystem_utilization + - dimensions: + - - ClusterName + - Namespace + - PodName + - - ClusterName + - - ClusterName + - Namespace + - Service + - - ClusterName + - Namespace + - - ClusterName + - FullPodName + - Namespace + - PodName + metric_name_selectors: + - pod_cpu_utilization + - pod_memory_utilization + - pod_network_rx_bytes + - pod_network_tx_bytes + - pod_cpu_utilization_over_pod_limit + - pod_memory_utilization_over_pod_limit + - dimensions: + - - ClusterName + - FullPodName + - Namespace + - PodName + - - ClusterName + - Namespace + - PodName + - - ClusterName + - Namespace + - - ClusterName + metric_name_selectors: + - pod_interface_network_rx_dropped + - pod_interface_network_tx_dropped + - dimensions: + - - ClusterName + - Namespace + - PodName + - - ClusterName + - - ClusterName + - FullPodName + - Namespace + - PodName + - - ClusterName + - Namespace + - Service + metric_name_selectors: + - pod_cpu_reserved_capacity + - pod_memory_reserved_capacity + - pod_number_of_container_restarts + - pod_number_of_containers + - pod_number_of_running_containers + - pod_status_ready + - pod_status_scheduled + - pod_status_running + - pod_status_pending + - pod_status_failed + - pod_status_unknown + - pod_status_succeeded + - pod_memory_request + - pod_memory_limit + - pod_cpu_limit + - pod_cpu_usage_total + - pod_memory_working_set + - pod_cpu_request + - pod_container_status_running + - pod_container_status_terminated + - pod_container_status_waiting + - pod_container_status_waiting_reason_crash_loop_back_off + - pod_container_status_waiting_reason_image_pull_error + - pod_container_status_waiting_reason_start_error + - pod_container_status_waiting_reason_create_container_error + - pod_container_status_waiting_reason_create_container_config_error + - pod_container_status_terminated_reason_oom_killed + - dimensions: + - - ClusterName + - InstanceId + - NodeName + - - ClusterName + metric_name_selectors: + - node_cpu_utilization + - node_memory_utilization + - node_network_total_bytes + - node_cpu_reserved_capacity + - node_memory_reserved_capacity + - node_number_of_running_pods + - node_number_of_running_containers + - node_cpu_usage_total + - node_cpu_limit + - node_memory_working_set + - node_memory_limit + - node_status_condition_ready + - node_status_condition_disk_pressure + - node_status_condition_memory_pressure + - node_status_condition_pid_pressure + - node_status_condition_network_unavailable + - node_status_condition_unknown + - node_status_capacity_pods + - node_status_allocatable_pods + - dimensions: + - - ClusterName + - InstanceId + - NodeName + - - ClusterName + metric_name_selectors: + - node_interface_network_rx_dropped + - node_interface_network_tx_dropped + - node_diskio_io_service_bytes_total + - node_diskio_io_serviced_total + - hyperpod_node_health_status_schedulable + - hyperpod_node_health_status_unschedulable_pending_replacement + - hyperpod_node_health_status_unschedulable_pending_reboot + - hyperpod_node_health_status_unschedulable + - dimensions: + - - ClusterName + - InstanceId + - NodeName + - - ClusterName + metric_name_selectors: + - node_filesystem_utilization + - node_filesystem_inodes + - node_filesystem_inodes_free + - dimensions: + - - ClusterName + - Namespace + - Service + - - ClusterName + metric_name_selectors: + - service_number_of_running_pods + - dimensions: + - - ClusterName + - Namespace + - PodName + - - ClusterName + metric_name_selectors: + - replicas_desired + - replicas_ready + - status_replicas_available + - status_replicas_unavailable + - dimensions: + - - ClusterName + - Namespace + - PodName + - - ClusterName + metric_name_selectors: + - daemonset_status_number_available + - daemonset_status_number_unavailable + - dimensions: + - - ClusterName + - Namespace + - - ClusterName + metric_name_selectors: + - namespace_number_of_running_pods + - dimensions: + - - ClusterName + metric_name_selectors: + - cluster_node_count + - cluster_failed_node_count + - cluster_number_of_running_pods + - dimensions: + - - ClusterName + - endpoint + - - ClusterName + metric_name_selectors: + - apiserver_storage_size_bytes + - apiserver_storage_db_total_size_in_bytes + - etcd_db_total_size_in_bytes + - dimensions: + - - ClusterName + - resource + - - ClusterName + metric_name_selectors: + - apiserver_storage_list_duration_seconds + - apiserver_longrunning_requests + - apiserver_storage_objects + - dimensions: + - - ClusterName + - verb + - - ClusterName + metric_name_selectors: + - apiserver_request_duration_seconds + - rest_client_request_duration_seconds + - dimensions: + - - ClusterName + - code + - verb + - - ClusterName + metric_name_selectors: + - apiserver_request_total + - apiserver_request_total_5xx + - dimensions: + - - ClusterName + - operation + - - ClusterName + metric_name_selectors: + - apiserver_admission_controller_admission_duration_seconds + - apiserver_admission_step_admission_duration_seconds + - etcd_request_duration_seconds + - dimensions: + - - ClusterName + - code + - method + - - ClusterName + metric_name_selectors: + - rest_client_requests_total + - dimensions: + - - ClusterName + - request_kind + - - ClusterName + metric_name_selectors: + - apiserver_current_inflight_requests + - apiserver_current_inqueue_requests + - dimensions: + - - ClusterName + - name + - - ClusterName + metric_name_selectors: + - apiserver_admission_webhook_admission_duration_seconds + - dimensions: + - - ClusterName + - group + - - ClusterName + metric_name_selectors: + - apiserver_requested_deprecated_apis + - dimensions: + - - ClusterName + - reason + - - ClusterName + metric_name_selectors: + - apiserver_flowcontrol_rejected_requests_total + - dimensions: + - - ClusterName + - priority_level + - - ClusterName + metric_name_selectors: + - apiserver_flowcontrol_request_concurrency_limit + metric_descriptors: + - metric_name: apiserver_admission_controller_admission_duration_seconds + overwrite: true + unit: Seconds + - metric_name: apiserver_admission_step_admission_duration_seconds + overwrite: true + unit: Seconds + - metric_name: apiserver_admission_webhook_admission_duration_seconds + overwrite: true + unit: Seconds + - metric_name: apiserver_current_inflight_requests + overwrite: true + unit: Count + - metric_name: apiserver_current_inqueue_requests + overwrite: true + unit: Count + - metric_name: apiserver_flowcontrol_rejected_requests_total + overwrite: true + unit: Count + - metric_name: apiserver_flowcontrol_request_concurrency_limit + overwrite: true + unit: Count + - metric_name: apiserver_longrunning_requests + overwrite: true + unit: Count + - metric_name: apiserver_request_duration_seconds + overwrite: true + unit: Seconds + - metric_name: apiserver_request_total + overwrite: true + unit: Count + - metric_name: apiserver_request_total_5xx + overwrite: true + unit: Count + - metric_name: apiserver_requested_deprecated_apis + overwrite: true + unit: Count + - metric_name: apiserver_storage_objects + overwrite: true + unit: Count + - metric_name: etcd_request_duration_seconds + overwrite: true + unit: Seconds + - metric_name: apiserver_storage_list_duration_seconds + overwrite: true + unit: Seconds + - metric_name: apiserver_storage_db_total_size_in_bytes + overwrite: true + unit: Bytes + - metric_name: apiserver_storage_size_bytes + overwrite: true + unit: Bytes + - metric_name: etcd_db_total_size_in_bytes + overwrite: true + unit: Bytes + - metric_name: rest_client_request_duration_seconds + overwrite: true + unit: Seconds + - metric_name: rest_client_requests_total + overwrite: true + unit: Count + middleware: agenthealth/logs + namespace: ContainerInsights + no_verify_ssl: false + num_workers: 8 + output_destination: cloudwatch + parse_json_encoded_attr_values: + - Sources + - kubernetes + profile: default + proxy_address: "" + region: us-east-1 + request_timeout_seconds: 30 + resource_arn: "" + resource_to_telemetry_conversion: + enabled: true + retain_initial_value_of_delta_metric: false + role_arn: "" + shared_credentials_file: + - /root/.aws/credentials + version: "0" +extensions: + agenthealth/logs: + is_usage_data_enabled: true + stats: + operations: + - PutLogEvents + usage_flags: + mode: OP + region_type: ACJ +processors: + batch/containerinsights: + metadata_cardinality_limit: 1000 + send_batch_max_size: 0 + send_batch_size: 8192 + timeout: 5s + batch/emf_logs: + metadata_cardinality_limit: 1000 + send_batch_max_size: 0 + send_batch_size: 8192 + timeout: 5s + kueueattributes/containerinsights: {} + metricstransform/containerinsights: + transforms: + - action: insert + aggregation_type: "" + experimental_match_labels: + code: ^5.* + include: apiserver_request_total + match_type: regexp + new_name: apiserver_request_total_5xx + submatch_case: "" +receivers: + awscontainerinsightreceiver: + accelerated_compute_metrics: false + kueue_container_insights: true + add_container_name_metric_label: true + add_full_pod_name_metric_label: true + add_service_as_attribute: true + certificate_file_path: "" + cluster_name: TestCluster + collection_interval: 30s + container_orchestrator: eks + enable_control_plane_metrics: true + endpoint: "" + host_ip: "" + host_name: "" + imds_retries: 2 + kube_config_path: "" + leader_lock_name: cwagent-clusterleader + leader_lock_using_config_map_only: true + local_mode: true + max_retries: 0 + no_verify_ssl: false + num_workers: 0 + prefer_full_pod_name: true + profile: default + proxy_address: "" + region: us-east-1 + request_timeout_seconds: 0 + resource_arn: "" + role_arn: "" + shared_credentials_file: + - /root/.aws/credentials + tcplog/emf_logs: + encoding: utf-8 + id: tcp_input + listen_address: 0.0.0.0:25888 + operators: [] + retry_on_failure: + enabled: false + initial_interval: 0s + max_elapsed_time: 0s + max_interval: 0s + type: tcp_input + udplog/emf_logs: + encoding: utf-8 + id: udp_input + listen_address: 0.0.0.0:25888 + multiline: + line_end_pattern: .^ + line_start_pattern: "" + omit_pattern: false + operators: [] + retry_on_failure: + enabled: false + initial_interval: 0s + max_elapsed_time: 0s + max_interval: 0s + type: udp_input +service: + extensions: + - agenthealth/logs + pipelines: + logs/emf_logs: + exporters: + - awscloudwatchlogs/emf_logs + processors: + - batch/emf_logs + receivers: + - tcplog/emf_logs + - udplog/emf_logs + metrics/containerinsights: + exporters: + - awsemf/containerinsights + processors: + - metricstransform/containerinsights + - kueueattributes/containerinsights + - batch/containerinsights + receivers: + - awscontainerinsightreceiver + telemetry: + logs: + development: false + disable_caller: false + disable_stacktrace: false + encoding: console + level: info + sampling: + enabled: true + initial: 2 + thereafter: 500 + tick: 10s + metrics: + address: "" + level: None + traces: {} diff --git a/translator/tocwconfig/sampleConfig/kubernetes_on_prem_config.json b/translator/tocwconfig/sampleConfig/kubernetes_on_prem_config.json index 1552a1b7b4..1c3894aac0 100644 --- a/translator/tocwconfig/sampleConfig/kubernetes_on_prem_config.json +++ b/translator/tocwconfig/sampleConfig/kubernetes_on_prem_config.json @@ -9,7 +9,8 @@ "metrics_collection_interval": 30, "disable_metric_extraction": true, "enhanced_container_insights": true, - "accelerated_compute_metrics": false + "accelerated_compute_metrics": false, + "kueue_container_insights": false } }, "force_flush_interval": 5, diff --git a/translator/tocwconfig/sampleConfig/kubernetes_on_prem_config.yaml b/translator/tocwconfig/sampleConfig/kubernetes_on_prem_config.yaml index c5f7ebc112..b8602ca3cc 100644 --- a/translator/tocwconfig/sampleConfig/kubernetes_on_prem_config.yaml +++ b/translator/tocwconfig/sampleConfig/kubernetes_on_prem_config.yaml @@ -384,6 +384,7 @@ processors: receivers: awscontainerinsightreceiver: accelerated_compute_metrics: false + kueue_container_insights: false add_container_name_metric_label: true add_full_pod_name_metric_label: true add_service_as_attribute: true diff --git a/translator/tocwconfig/sampleConfig/log_ecs_metric_only.yaml b/translator/tocwconfig/sampleConfig/log_ecs_metric_only.yaml index b08d51c861..835ec94cbf 100644 --- a/translator/tocwconfig/sampleConfig/log_ecs_metric_only.yaml +++ b/translator/tocwconfig/sampleConfig/log_ecs_metric_only.yaml @@ -111,6 +111,7 @@ processors: receivers: awscontainerinsightreceiver: accelerated_compute_metrics: true + kueue_container_insights: false add_container_name_metric_label: false add_full_pod_name_metric_label: false add_service_as_attribute: true diff --git a/translator/tocwconfig/sampleConfig/logs_and_kubernetes_config.json b/translator/tocwconfig/sampleConfig/logs_and_kubernetes_config.json index eed2cbb8ac..5c536cc48b 100644 --- a/translator/tocwconfig/sampleConfig/logs_and_kubernetes_config.json +++ b/translator/tocwconfig/sampleConfig/logs_and_kubernetes_config.json @@ -10,7 +10,8 @@ "cluster_name": "TestCluster", "metrics_collection_interval": 30, "enhanced_container_insights": true, - "accelerated_compute_metrics": false + "accelerated_compute_metrics": false, + "kueue_container_insights": false } }, "logs_collected": { diff --git a/translator/tocwconfig/sampleConfig/logs_and_kubernetes_config.yaml b/translator/tocwconfig/sampleConfig/logs_and_kubernetes_config.yaml index 6000b359a5..a6b56b0c5f 100644 --- a/translator/tocwconfig/sampleConfig/logs_and_kubernetes_config.yaml +++ b/translator/tocwconfig/sampleConfig/logs_and_kubernetes_config.yaml @@ -418,6 +418,7 @@ processors: receivers: awscontainerinsightreceiver: accelerated_compute_metrics: false + kueue_container_insights: false add_container_name_metric_label: true add_full_pod_name_metric_label: true add_service_as_attribute: true diff --git a/translator/tocwconfig/tocwconfig_test.go b/translator/tocwconfig/tocwconfig_test.go index 52755f6e93..eaee87eb37 100644 --- a/translator/tocwconfig/tocwconfig_test.go +++ b/translator/tocwconfig/tocwconfig_test.go @@ -191,6 +191,18 @@ func TestEmfAndKubernetesWithGpuConfig(t *testing.T) { checkTranslation(t, "emf_and_kubernetes_with_gpu_config", "darwin", nil, "") } +func TestEmfAndKubernetesWithKueueConfig(t *testing.T) { + resetContext(t) + readCommonConfig(t, "./sampleConfig/commonConfig/withCredentials.toml") + context.CurrentContext().SetRunInContainer(true) + context.CurrentContext().SetMode(config.ModeOnPremise) + t.Setenv(config.HOST_NAME, "host_name_from_env") + t.Setenv(config.HOST_IP, "127.0.0.1") + expectedEnvVars := map[string]string{} + checkTranslation(t, "emf_and_kubernetes_with_kueue_config", "linux", expectedEnvVars, "") + checkTranslation(t, "emf_and_kubernetes_with_kueue_config", "darwin", nil, "") +} + func TestKubernetesModeOnPremiseConfig(t *testing.T) { resetContext(t) context.CurrentContext().SetRunInContainer(true) diff --git a/translator/translate/otel/common/common.go b/translator/translate/otel/common/common.go index 51dab5bcda..5ef24c1eb1 100644 --- a/translator/translate/otel/common/common.go +++ b/translator/translate/otel/common/common.go @@ -57,6 +57,7 @@ const ( EnhancedContainerInsights = "enhanced_container_insights" PreferFullPodName = "prefer_full_pod_name" EnableAcceleratedComputeMetric = "accelerated_compute_metrics" + EnableKueueContainerInsights = "kueue_container_insights" AppendDimensionsKey = "append_dimensions" Console = "console" DiskKey = "disk" diff --git a/translator/translate/otel/pipeline/containerinsights/translator.go b/translator/translate/otel/pipeline/containerinsights/translator.go index 4e5169ffac..c3e940ada2 100644 --- a/translator/translate/otel/pipeline/containerinsights/translator.go +++ b/translator/translate/otel/pipeline/containerinsights/translator.go @@ -14,6 +14,7 @@ import ( "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/extension/agenthealth" "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/processor/batchprocessor" "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/processor/gpu" + "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/processor/kueue" "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/processor/metricstransformprocessor" "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/receiver/awscontainerinsight" ) @@ -56,6 +57,10 @@ func (t *translator) Translate(conf *confmap.Conf) (*common.ComponentTranslators if acceleratedComputeMetricsEnabled { processors.Set(gpu.NewTranslatorWithName(pipelineName)) } + KueueContainerInsightsEnabled := awscontainerinsight.KueueContainerInsightsEnabled(conf) + if KueueContainerInsightsEnabled { + processors.Set(kueue.NewTranslatorWithName(pipelineName)) + } processors.Set(batchprocessor.NewTranslatorWithNameAndSection(pipelineName, common.LogsKey)) return &common.ComponentTranslators{ Receivers: common.NewTranslatorMap(awscontainerinsight.NewTranslator()), diff --git a/translator/translate/otel/pipeline/containerinsights/translator_test.go b/translator/translate/otel/pipeline/containerinsights/translator_test.go index d3a4bb67a1..7069f20347 100644 --- a/translator/translate/otel/pipeline/containerinsights/translator_test.go +++ b/translator/translate/otel/pipeline/containerinsights/translator_test.go @@ -73,6 +73,7 @@ func TestTranslator(t *testing.T) { "metrics_collected": map[string]interface{}{ "kubernetes": map[string]interface{}{ "enhanced_container_insights": true, + "kueue_container_insights": true, "cluster_name": "TestCluster", }, }, @@ -81,9 +82,14 @@ func TestTranslator(t *testing.T) { want: &want{ pipelineType: "metrics/containerinsights", receivers: []string{"awscontainerinsightreceiver"}, - processors: []string{"metricstransform/containerinsights", "gpuattributes/containerinsights", "batch/containerinsights"}, - exporters: []string{"awsemf/containerinsights"}, - extensions: []string{"agenthealth/logs"}, + processors: []string{ + "metricstransform/containerinsights", + "gpuattributes/containerinsights", + "kueueattributes/containerinsights", + "batch/containerinsights", + }, + exporters: []string{"awsemf/containerinsights"}, + extensions: []string{"agenthealth/logs"}, }, }, } diff --git a/translator/translate/otel/processor/kueue/translator.go b/translator/translate/otel/processor/kueue/translator.go new file mode 100644 index 0000000000..d072144b4d --- /dev/null +++ b/translator/translate/otel/processor/kueue/translator.go @@ -0,0 +1,33 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +package kueue + +import ( + "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/confmap" + "go.opentelemetry.io/collector/processor" + + "github.com/aws/amazon-cloudwatch-agent/plugins/processors/kueueattributes" + "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/common" +) + +type translator struct { + name string + factory processor.Factory +} + +var _ common.Translator[component.Config] = (*translator)(nil) + +func NewTranslatorWithName(name string) common.Translator[component.Config] { + return &translator{name, kueueattributes.NewFactory()} +} + +func (t *translator) ID() component.ID { + return component.NewIDWithName(t.factory.Type(), t.name) +} + +func (t *translator) Translate(conf *confmap.Conf) (component.Config, error) { + cfg := t.factory.CreateDefaultConfig().(*kueueattributes.Config) + return cfg, nil +} diff --git a/translator/translate/otel/receiver/awscontainerinsight/translator.go b/translator/translate/otel/receiver/awscontainerinsight/translator.go index 193fdbe87b..32e8e96f9a 100644 --- a/translator/translate/otel/receiver/awscontainerinsight/translator.go +++ b/translator/translate/otel/receiver/awscontainerinsight/translator.go @@ -122,6 +122,7 @@ func (t *translator) Translate(conf *confmap.Conf) (component.Config, error) { cfg.PrefFullPodName = cfg.PrefFullPodName || common.GetOrDefaultBool(conf, common.ConfigKey(common.LogsKey, common.MetricsCollectedKey, common.KubernetesKey, common.PreferFullPodName), false) cfg.EnableAcceleratedComputeMetrics = cfg.EnableAcceleratedComputeMetrics || AcceleratedComputeMetricsEnabled(conf) + cfg.EnableKueueContainerInsights = cfg.EnableKueueContainerInsights || KueueContainerInsightsEnabled(conf) return cfg, nil } diff --git a/translator/translate/otel/receiver/awscontainerinsight/utils.go b/translator/translate/otel/receiver/awscontainerinsight/utils.go index 721951b056..ac0ae1976a 100644 --- a/translator/translate/otel/receiver/awscontainerinsight/utils.go +++ b/translator/translate/otel/receiver/awscontainerinsight/utils.go @@ -27,3 +27,7 @@ func EnhancedContainerInsightsEnabled(conf *confmap.Conf) bool { func AcceleratedComputeMetricsEnabled(conf *confmap.Conf) bool { return common.GetOrDefaultBool(conf, common.ConfigKey(common.LogsKey, common.MetricsCollectedKey, common.KubernetesKey, common.EnableAcceleratedComputeMetric), true) } + +func KueueContainerInsightsEnabled(conf *confmap.Conf) bool { + return common.GetOrDefaultBool(conf, common.ConfigKey(common.LogsKey, common.MetricsCollectedKey, common.KubernetesKey, common.EnableKueueContainerInsights), false) +} From 39ad36fdf7d965ab048ff0e9df22c381d1cfa9b6 Mon Sep 17 00:00:00 2001 From: Steven Tran <86030690+transtv@users.noreply.github.com> Date: Fri, 25 Oct 2024 11:42:59 -0700 Subject: [PATCH 02/32] Add Kueue Metrics Translator Definitions (#847) --- .../appsignals_and_eks_config.yaml | 1 + .../appsignals_and_k8s_config.yaml | 1 + .../appsignals_fallback_and_eks_config.yaml | 1 + .../appsignals_over_fallback_config.yaml | 1 + .../base_container_insights_config.yaml | 1 + .../emf_and_kubernetes_config.yaml | 3 +- .../emf_and_kubernetes_with_gpu_config.yaml | 1 + .../kubernetes_on_prem_config.yaml | 1 + .../kueue_container_insights_config.conf | 27 ++ .../kueue_container_insights_config.json | 21 ++ .../kueue_container_insights_config.yaml | 297 ++++++++++++++++++ .../sampleConfig/log_ecs_metric_only.yaml | 1 + .../logs_and_kubernetes_config.yaml | 1 + translator/tocwconfig/tocwconfig_test.go | 14 + .../otel/exporter/awsemf/kubernetes.go | 56 ++++ .../otel/exporter/awsemf/translator_test.go | 94 ++++++ 16 files changed, 520 insertions(+), 1 deletion(-) create mode 100644 translator/tocwconfig/sampleConfig/kueue_container_insights_config.conf create mode 100644 translator/tocwconfig/sampleConfig/kueue_container_insights_config.json create mode 100644 translator/tocwconfig/sampleConfig/kueue_container_insights_config.yaml diff --git a/translator/tocwconfig/sampleConfig/appsignals_and_eks_config.yaml b/translator/tocwconfig/sampleConfig/appsignals_and_eks_config.yaml index 17eeed4133..1042a3c968 100644 --- a/translator/tocwconfig/sampleConfig/appsignals_and_eks_config.yaml +++ b/translator/tocwconfig/sampleConfig/appsignals_and_eks_config.yaml @@ -605,6 +605,7 @@ receivers: host_name: "" imds_retries: 1 kube_config_path: "" + kueue_container_insights: false leader_lock_name: cwagent-clusterleader leader_lock_using_config_map_only: true local_mode: false diff --git a/translator/tocwconfig/sampleConfig/appsignals_and_k8s_config.yaml b/translator/tocwconfig/sampleConfig/appsignals_and_k8s_config.yaml index acc2bac366..792c8c3c15 100644 --- a/translator/tocwconfig/sampleConfig/appsignals_and_k8s_config.yaml +++ b/translator/tocwconfig/sampleConfig/appsignals_and_k8s_config.yaml @@ -606,6 +606,7 @@ receivers: host_name: "" imds_retries: 1 kube_config_path: "" + kueue_container_insights: false leader_lock_name: cwagent-clusterleader leader_lock_using_config_map_only: true local_mode: false diff --git a/translator/tocwconfig/sampleConfig/appsignals_fallback_and_eks_config.yaml b/translator/tocwconfig/sampleConfig/appsignals_fallback_and_eks_config.yaml index 17eeed4133..1042a3c968 100644 --- a/translator/tocwconfig/sampleConfig/appsignals_fallback_and_eks_config.yaml +++ b/translator/tocwconfig/sampleConfig/appsignals_fallback_and_eks_config.yaml @@ -605,6 +605,7 @@ receivers: host_name: "" imds_retries: 1 kube_config_path: "" + kueue_container_insights: false leader_lock_name: cwagent-clusterleader leader_lock_using_config_map_only: true local_mode: false diff --git a/translator/tocwconfig/sampleConfig/appsignals_over_fallback_config.yaml b/translator/tocwconfig/sampleConfig/appsignals_over_fallback_config.yaml index 17eeed4133..1042a3c968 100644 --- a/translator/tocwconfig/sampleConfig/appsignals_over_fallback_config.yaml +++ b/translator/tocwconfig/sampleConfig/appsignals_over_fallback_config.yaml @@ -605,6 +605,7 @@ receivers: host_name: "" imds_retries: 1 kube_config_path: "" + kueue_container_insights: false leader_lock_name: cwagent-clusterleader leader_lock_using_config_map_only: true local_mode: false diff --git a/translator/tocwconfig/sampleConfig/base_container_insights_config.yaml b/translator/tocwconfig/sampleConfig/base_container_insights_config.yaml index d961e759bf..de230cb3e5 100644 --- a/translator/tocwconfig/sampleConfig/base_container_insights_config.yaml +++ b/translator/tocwconfig/sampleConfig/base_container_insights_config.yaml @@ -176,6 +176,7 @@ receivers: host_name: "" imds_retries: 1 kube_config_path: "" + kueue_container_insights: false leader_lock_name: cwagent-clusterleader leader_lock_using_config_map_only: true local_mode: false diff --git a/translator/tocwconfig/sampleConfig/emf_and_kubernetes_config.yaml b/translator/tocwconfig/sampleConfig/emf_and_kubernetes_config.yaml index e420e06c80..2bae7a3e8d 100644 --- a/translator/tocwconfig/sampleConfig/emf_and_kubernetes_config.yaml +++ b/translator/tocwconfig/sampleConfig/emf_and_kubernetes_config.yaml @@ -134,9 +134,9 @@ exporters: - pod_memory_request - pod_memory_limit - pod_cpu_limit + - pod_cpu_request - pod_cpu_usage_total - pod_memory_working_set - - pod_cpu_request - pod_container_status_running - pod_container_status_terminated - pod_container_status_waiting @@ -436,6 +436,7 @@ receivers: host_name: "" imds_retries: 2 kube_config_path: "" + kueue_container_insights: false leader_lock_name: cwagent-clusterleader leader_lock_using_config_map_only: true local_mode: true diff --git a/translator/tocwconfig/sampleConfig/emf_and_kubernetes_with_gpu_config.yaml b/translator/tocwconfig/sampleConfig/emf_and_kubernetes_with_gpu_config.yaml index 99f1b2868e..bc6c588ca5 100644 --- a/translator/tocwconfig/sampleConfig/emf_and_kubernetes_with_gpu_config.yaml +++ b/translator/tocwconfig/sampleConfig/emf_and_kubernetes_with_gpu_config.yaml @@ -1108,6 +1108,7 @@ receivers: host_name: "" imds_retries: 2 kube_config_path: "" + kueue_container_insights: false leader_lock_name: cwagent-clusterleader leader_lock_using_config_map_only: true local_mode: true diff --git a/translator/tocwconfig/sampleConfig/kubernetes_on_prem_config.yaml b/translator/tocwconfig/sampleConfig/kubernetes_on_prem_config.yaml index b8602ca3cc..161d76df08 100644 --- a/translator/tocwconfig/sampleConfig/kubernetes_on_prem_config.yaml +++ b/translator/tocwconfig/sampleConfig/kubernetes_on_prem_config.yaml @@ -398,6 +398,7 @@ receivers: host_name: "" imds_retries: 1 kube_config_path: "" + kueue_container_insights: false leader_lock_name: cwagent-clusterleader leader_lock_using_config_map_only: true local_mode: true diff --git a/translator/tocwconfig/sampleConfig/kueue_container_insights_config.conf b/translator/tocwconfig/sampleConfig/kueue_container_insights_config.conf new file mode 100644 index 0000000000..007bb60efb --- /dev/null +++ b/translator/tocwconfig/sampleConfig/kueue_container_insights_config.conf @@ -0,0 +1,27 @@ +[agent] + collection_jitter = "0s" + debug = false + flush_interval = "1s" + flush_jitter = "0s" + hostname = "host_name_from_env" + interval = "60s" + logfile = "" + logtarget = "lumberjack" + metric_batch_size = 1000 + metric_buffer_limit = 10000 + omit_hostname = false + precision = "" + quiet = false + round_interval = false + +[inputs] + +[outputs] + + [[outputs.cloudwatchlogs]] + endpoint_override = "https://fake_endpoint" + force_flush_interval = "5s" + log_stream_name = "host_name_from_env" + region = "us-east-1" + +[processors] diff --git a/translator/tocwconfig/sampleConfig/kueue_container_insights_config.json b/translator/tocwconfig/sampleConfig/kueue_container_insights_config.json new file mode 100644 index 0000000000..cd2290daed --- /dev/null +++ b/translator/tocwconfig/sampleConfig/kueue_container_insights_config.json @@ -0,0 +1,21 @@ +{ + "agent": { + "region": "us-east-1" + }, + "logs": { + "metrics_collected": { + "emf": { + }, + "kubernetes": { + "cluster_name": "TestCluster", + "metrics_collection_interval": 30, + "disable_metric_extraction": true, + "prefer_full_pod_name": true, + "accelerated_compute_metrics": false, + "kueue_container_insights": true + } + }, + "force_flush_interval": 5, + "endpoint_override":"https://fake_endpoint" + } +} diff --git a/translator/tocwconfig/sampleConfig/kueue_container_insights_config.yaml b/translator/tocwconfig/sampleConfig/kueue_container_insights_config.yaml new file mode 100644 index 0000000000..e99304f3a1 --- /dev/null +++ b/translator/tocwconfig/sampleConfig/kueue_container_insights_config.yaml @@ -0,0 +1,297 @@ +exporters: + awscloudwatchlogs/emf_logs: + certificate_file_path: /etc/test/ca_bundle.pem + emf_only: true + endpoint: https://fake_endpoint + imds_retries: 1 + local_mode: false + log_group_name: emf/logs/default + log_retention: 0 + log_stream_name: host_name_from_env + max_retries: 2 + middleware: agenthealth/logs + no_verify_ssl: false + num_workers: 8 + profile: "" + proxy_address: "" + raw_log: true + region: us-east-1 + request_timeout_seconds: 30 + resource_arn: "" + retry_on_failure: + enabled: true + initial_interval: 5s + max_elapsed_time: 5m0s + max_interval: 30s + multiplier: 1.5 + randomization_factor: 0.5 + role_arn: "" + sending_queue: + enabled: true + num_consumers: 1 + queue_size: 1000 + awsemf/containerinsights: + certificate_file_path: /etc/test/ca_bundle.pem + detailed_metrics: false + dimension_rollup_option: NoDimensionRollup + disable_metric_extraction: true + eks_fargate_container_insights_enabled: false + endpoint: https://fake_endpoint + enhanced_container_insights: false + imds_retries: 1 + local_mode: false + log_group_name: /aws/containerinsights/{ClusterName}/performance + log_retention: 0 + log_stream_name: '{NodeName}' + max_retries: 2 + metric_declarations: + - dimensions: + - - ClusterName + - Namespace + - PodName + - - ClusterName + - - ClusterName + - Namespace + - Service + - - ClusterName + - Namespace + metric_name_selectors: + - pod_cpu_utilization + - pod_memory_utilization + - pod_network_rx_bytes + - pod_network_tx_bytes + - pod_cpu_utilization_over_pod_limit + - pod_memory_utilization_over_pod_limit + - dimensions: + - - ClusterName + - Namespace + - PodName + metric_name_selectors: + - pod_number_of_container_restarts + - dimensions: + - - ClusterName + - Namespace + - PodName + - - ClusterName + metric_name_selectors: + - pod_cpu_reserved_capacity + - pod_memory_reserved_capacity + - dimensions: + - - ClusterName + - InstanceId + - NodeName + - - ClusterName + metric_name_selectors: + - node_cpu_utilization + - node_memory_utilization + - node_network_total_bytes + - node_cpu_reserved_capacity + - node_memory_reserved_capacity + - node_number_of_running_pods + - node_number_of_running_containers + - dimensions: + - - ClusterName + metric_name_selectors: + - node_cpu_usage_total + - node_cpu_limit + - node_memory_working_set + - node_memory_limit + - dimensions: + - - ClusterName + - InstanceId + - NodeName + - - ClusterName + metric_name_selectors: + - node_filesystem_utilization + - dimensions: + - - ClusterName + - Namespace + - Service + - - ClusterName + metric_name_selectors: + - service_number_of_running_pods + - dimensions: + - - ClusterName + - Namespace + - - ClusterName + metric_name_selectors: + - namespace_number_of_running_pods + - dimensions: + - - ClusterName + metric_name_selectors: + - cluster_node_count + - cluster_failed_node_count + - dimensions: + - - ClusterName + - - ClusterName + - ClusterQueue + - - ClusterName + - ClusterQueue + - Status + - - ClusterName + - Status + metric_name_selectors: + - kueue_pending_workloads + - dimensions: + - - ClusterName + - - ClusterName + - ClusterQueue + - - ClusterName + - ClusterQueue + - Reason + - - ClusterName + - Reason + metric_name_selectors: + - kueue_evicted_workloads_total + - dimensions: + - - ClusterName + - - ClusterName + - ClusterQueue + metric_name_selectors: + - kueue_admitted_active_workloads + - dimensions: + - - ClusterName + - - ClusterName + - ClusterQueue + - - ClusterName + - ClusterQueue + - Resource + - - ClusterName + - ClusterQueue + - Flavor + - Resource + - - ClusterName + - ClusterQueue + - Flavor + metric_name_selectors: + - kueue_cluster_queue_resource_usage + - kueue_cluster_queue_nominal_quota + middleware: agenthealth/logs + namespace: ContainerInsights + no_verify_ssl: false + num_workers: 8 + output_destination: cloudwatch + parse_json_encoded_attr_values: + - Sources + - kubernetes + profile: "" + proxy_address: "" + region: us-east-1 + request_timeout_seconds: 30 + resource_arn: "" + resource_to_telemetry_conversion: + enabled: true + retain_initial_value_of_delta_metric: false + role_arn: "" + version: "0" +extensions: + agenthealth/logs: + is_usage_data_enabled: true + stats: + operations: + - PutLogEvents + usage_flags: + mode: EC2 + region_type: ACJ +processors: + batch/containerinsights: + metadata_cardinality_limit: 1000 + send_batch_max_size: 0 + send_batch_size: 8192 + timeout: 5s + batch/emf_logs: + metadata_cardinality_limit: 1000 + send_batch_max_size: 0 + send_batch_size: 8192 + timeout: 5s +receivers: + awscontainerinsightreceiver: + accelerated_compute_metrics: false + add_container_name_metric_label: false + add_full_pod_name_metric_label: false + add_service_as_attribute: true + certificate_file_path: "" + cluster_name: TestCluster + collection_interval: 30s + container_orchestrator: eks + enable_control_plane_metrics: false + endpoint: "" + host_ip: "" + host_name: "" + imds_retries: 1 + kube_config_path: "" + kueue_container_insights: true + leader_lock_name: cwagent-clusterleader + leader_lock_using_config_map_only: true + local_mode: false + max_retries: 0 + no_verify_ssl: false + num_workers: 0 + prefer_full_pod_name: true + profile: "" + proxy_address: "" + region: us-east-1 + request_timeout_seconds: 0 + resource_arn: "" + role_arn: "" + tcplog/emf_logs: + encoding: utf-8 + id: tcp_input + listen_address: 0.0.0.0:25888 + operators: [] + retry_on_failure: + enabled: false + initial_interval: 0s + max_elapsed_time: 0s + max_interval: 0s + type: tcp_input + udplog/emf_logs: + encoding: utf-8 + id: udp_input + listen_address: 0.0.0.0:25888 + multiline: + line_end_pattern: .^ + line_start_pattern: "" + omit_pattern: false + operators: [] + retry_on_failure: + enabled: false + initial_interval: 0s + max_elapsed_time: 0s + max_interval: 0s + type: udp_input +service: + extensions: + - agenthealth/logs + pipelines: + logs/emf_logs: + exporters: + - awscloudwatchlogs/emf_logs + processors: + - batch/emf_logs + receivers: + - tcplog/emf_logs + - udplog/emf_logs + metrics/containerinsights: + exporters: + - awsemf/containerinsights + processors: + - batch/containerinsights + receivers: + - awscontainerinsightreceiver + telemetry: + logs: + development: false + disable_caller: false + disable_stacktrace: false + encoding: console + level: info + sampling: + enabled: true + initial: 2 + thereafter: 500 + tick: 10s + metrics: + address: "" + level: None + traces: {} diff --git a/translator/tocwconfig/sampleConfig/log_ecs_metric_only.yaml b/translator/tocwconfig/sampleConfig/log_ecs_metric_only.yaml index 835ec94cbf..2304755f23 100644 --- a/translator/tocwconfig/sampleConfig/log_ecs_metric_only.yaml +++ b/translator/tocwconfig/sampleConfig/log_ecs_metric_only.yaml @@ -125,6 +125,7 @@ receivers: host_name: "" imds_retries: 1 kube_config_path: "" + kueue_container_insights: false leader_lock_name: otel-container-insight-clusterleader leader_lock_using_config_map_only: false local_mode: false diff --git a/translator/tocwconfig/sampleConfig/logs_and_kubernetes_config.yaml b/translator/tocwconfig/sampleConfig/logs_and_kubernetes_config.yaml index a6b56b0c5f..d9ec3480fa 100644 --- a/translator/tocwconfig/sampleConfig/logs_and_kubernetes_config.yaml +++ b/translator/tocwconfig/sampleConfig/logs_and_kubernetes_config.yaml @@ -432,6 +432,7 @@ receivers: host_name: "" imds_retries: 0 kube_config_path: "" + kueue_container_insights: false leader_lock_name: cwagent-clusterleader leader_lock_using_config_map_only: true local_mode: false diff --git a/translator/tocwconfig/tocwconfig_test.go b/translator/tocwconfig/tocwconfig_test.go index eaee87eb37..ba2c81f61c 100644 --- a/translator/tocwconfig/tocwconfig_test.go +++ b/translator/tocwconfig/tocwconfig_test.go @@ -213,6 +213,20 @@ func TestKubernetesModeOnPremiseConfig(t *testing.T) { checkTranslation(t, "kubernetes_on_prem_config", "linux", expectedEnvVars, "") } +func TestKueueContainerInsightsConfig(t *testing.T) { + resetContext(t) + context.CurrentContext().SetRunInContainer(true) + context.CurrentContext().SetMode(config.ModeEC2) + t.Setenv(config.HOST_NAME, "host_name_from_env") + t.Setenv(config.HOST_IP, "127.0.0.1") + t.Setenv(envconfig.AWS_CA_BUNDLE, "/etc/test/ca_bundle.pem") + expectedEnvVars := map[string]string{ + "AWS_CA_BUNDLE": "/etc/test/ca_bundle.pem", + } + checkTranslation(t, "kueue_container_insights_config", "linux", expectedEnvVars, "") + checkTranslation(t, "kueue_container_insights_config", "darwin", nil, "") +} + func TestLogsAndKubernetesConfig(t *testing.T) { resetContext(t) context.CurrentContext().SetRunInContainer(true) diff --git a/translator/translate/otel/exporter/awsemf/kubernetes.go b/translator/translate/otel/exporter/awsemf/kubernetes.go index f7bf4bfa1f..9b346816a9 100644 --- a/translator/translate/otel/exporter/awsemf/kubernetes.go +++ b/translator/translate/otel/exporter/awsemf/kubernetes.go @@ -54,6 +54,9 @@ func setKubernetesMetricDeclaration(conf *confmap.Conf, cfg *awsemfexporter.Conf kubernetesMetricDeclarations = append(kubernetesMetricDeclarations, getEFAMetricDeclarations(conf)...) + // Setup Kueue metrics + kubernetesMetricDeclarations = append(kubernetesMetricDeclarations, getKueueMetricDeclarations(conf)...) + cfg.MetricDeclarations = kubernetesMetricDeclarations cfg.MetricDescriptors = getControlPlaneMetricDescriptors(conf) @@ -644,3 +647,56 @@ func getEFAMetricDeclarations(conf *confmap.Conf) []*awsemfexporter.MetricDeclar } return metricDeclarations } + +func getKueueMetricDeclarations(conf *confmap.Conf) []*awsemfexporter.MetricDeclaration { + var metricDeclarations []*awsemfexporter.MetricDeclaration + if awscontainerinsight.KueueContainerInsightsEnabled(conf) { + metricDeclarations = []*awsemfexporter.MetricDeclaration{ + { + Dimensions: [][]string{ + {"ClusterName"}, + {"ClusterName", "ClusterQueue"}, + {"ClusterName", "ClusterQueue", "Status"}, + {"ClusterName", "Status"}, + }, + MetricNameSelectors: []string{ + "kueue_pending_workloads", + }, + }, + { + Dimensions: [][]string{ + {"ClusterName"}, + {"ClusterName", "ClusterQueue"}, + {"ClusterName", "ClusterQueue", "Reason"}, + {"ClusterName", "Reason"}, + }, + MetricNameSelectors: []string{ + "kueue_evicted_workloads_total", + }, + }, + { + Dimensions: [][]string{ + {"ClusterName"}, + {"ClusterName", "ClusterQueue"}, + }, + MetricNameSelectors: []string{ + "kueue_admitted_active_workloads", + }, + }, + { + Dimensions: [][]string{ + {"ClusterName"}, + {"ClusterName", "ClusterQueue"}, + {"ClusterName", "ClusterQueue", "Resource"}, + {"ClusterName", "ClusterQueue", "Resource", "Flavor"}, + {"ClusterName", "ClusterQueue", "Flavor"}, + }, + MetricNameSelectors: []string{ + "kueue_cluster_queue_resource_usage", + "kueue_cluster_queue_nominal_quota", + }, + }, + } + } + return metricDeclarations +} diff --git a/translator/translate/otel/exporter/awsemf/translator_test.go b/translator/translate/otel/exporter/awsemf/translator_test.go index 10e7d08280..0fcfe1cc63 100644 --- a/translator/translate/otel/exporter/awsemf/translator_test.go +++ b/translator/translate/otel/exporter/awsemf/translator_test.go @@ -186,6 +186,100 @@ func TestTranslator(t *testing.T) { "local_mode": false, }, }, + "GenerateAwsEmfExporterConfigKubernetesWithKueueMetrics": { + input: map[string]any{ + "logs": map[string]any{ + "metrics_collected": map[string]any{ + "kubernetes": map[string]any{ + "kueue_container_insights": true, + }, + }, + }, + }, + want: map[string]any{ + "namespace": "ContainerInsights", + "log_group_name": "/aws/containerinsights/{ClusterName}/performance", + "log_stream_name": "{NodeName}", + "dimension_rollup_option": "NoDimensionRollup", + "disable_metric_extraction": false, + "enhanced_container_insights": false, + "parse_json_encoded_attr_values": []string{"Sources", "kubernetes"}, + "output_destination": "cloudwatch", + "eks_fargate_container_insights_enabled": false, + "resource_to_telemetry_conversion": resourcetotelemetry.Settings{ + Enabled: true, + }, + "metric_declarations": []*awsemfexporter.MetricDeclaration{ + { + Dimensions: [][]string{{"PodName", "Namespace", "ClusterName"}, {"ClusterName"}, {"Service", "Namespace", "ClusterName"}, {"ClusterName", "Namespace"}}, + MetricNameSelectors: []string{"pod_cpu_utilization", "pod_memory_utilization", + "pod_network_rx_bytes", "pod_network_tx_bytes", "pod_cpu_utilization_over_pod_limit", + "pod_memory_utilization_over_pod_limit"}, + }, + { + Dimensions: [][]string{{"PodName", "Namespace", "ClusterName"}}, + MetricNameSelectors: []string{"pod_number_of_container_restarts"}, + }, + { + Dimensions: [][]string{{"PodName", "Namespace", "ClusterName"}, {"ClusterName"}}, + MetricNameSelectors: []string{"pod_cpu_reserved_capacity", "pod_memory_reserved_capacity"}, + }, + { + Dimensions: [][]string{{"NodeName", "InstanceId", "ClusterName"}, {"ClusterName"}}, + MetricNameSelectors: []string{"node_cpu_utilization", "node_memory_utilization", + "node_network_total_bytes", "node_cpu_reserved_capacity", + "node_memory_reserved_capacity", "node_number_of_running_pods", "node_number_of_running_containers"}, + }, + { + Dimensions: [][]string{{"ClusterName"}}, + MetricNameSelectors: []string{"node_cpu_usage_total", "node_cpu_limit", "node_memory_working_set", "node_memory_limit"}, + }, + { + Dimensions: [][]string{{"NodeName", "InstanceId", "ClusterName"}, {"ClusterName"}}, + MetricNameSelectors: []string{"node_filesystem_utilization"}, + }, + { + Dimensions: [][]string{{"Service", "Namespace", "ClusterName"}, {"ClusterName"}}, + MetricNameSelectors: []string{"service_number_of_running_pods"}, + }, + { + Dimensions: [][]string{{"Namespace", "ClusterName"}, {"ClusterName"}}, + MetricNameSelectors: []string{"namespace_number_of_running_pods"}, + }, + { + Dimensions: [][]string{{"ClusterName"}}, + MetricNameSelectors: []string{"cluster_node_count", "cluster_failed_node_count"}, + }, + { + Dimensions: [][]string{{"ClusterName"}, {"ClusterName", "ClusterQueue"}, {"ClusterName", "ClusterQueue", "Status"}, {"ClusterName", "Status"}}, + MetricNameSelectors: []string{ + "kueue_pending_workloads", + }, + }, + { + Dimensions: [][]string{{"ClusterName"}, {"ClusterName", "ClusterQueue"}, {"ClusterName", "ClusterQueue", "Reason"}, {"ClusterName", "Reason"}}, + MetricNameSelectors: []string{ + "kueue_evicted_workloads_total", + }, + }, + { + Dimensions: [][]string{{"ClusterName"}, {"ClusterName", "ClusterQueue"}}, + MetricNameSelectors: []string{ + "kueue_admitted_active_workloads", + }, + }, + { + Dimensions: [][]string{{"ClusterName"}, {"ClusterName", "ClusterQueue"}, {"ClusterName", "ClusterQueue", "Resource"}, {"ClusterName", "ClusterQueue", "Resource", "Flavor"}, {"ClusterName", "ClusterQueue", "Flavor"}}, + MetricNameSelectors: []string{ + "kueue_cluster_queue_resource_usage", + "kueue_cluster_queue_nominal_quota", + }, + }, + }, + "metric_descriptors": nilMetricDescriptorsSlice, + "local_mode": false, + }, + }, "GenerateAwsEmfExporterConfigKubernetesDisableMetricExtraction": { input: map[string]any{ "logs": map[string]any{ From 5242cb46045006a1daee6ef7b0658ec0b35d185f Mon Sep 17 00:00:00 2001 From: Rohan Gujarathi Date: Fri, 25 Oct 2024 13:04:02 -0700 Subject: [PATCH 03/32] add NodeName to kueueLabelFilter and fix unit tests (#862) Co-authored-by: Rohan Gujarathi --- .../processors/kueueattributes/processor.go | 1 + .../kueueattributes/processor_test.go | 2 + .../appsignals_and_eks_config.yaml | 1 - .../appsignals_and_k8s_config.yaml | 1 - .../appsignals_fallback_and_eks_config.yaml | 1 - .../appsignals_over_fallback_config.yaml | 1 - .../base_container_insights_config.yaml | 1 - .../emf_and_kubernetes_config.yaml | 1 - .../emf_and_kubernetes_with_gpu_config.yaml | 1 - .../emf_and_kubernetes_with_kueue_config.yaml | 45 +++++++++++++++++++ .../kubernetes_on_prem_config.yaml | 1 - .../sampleConfig/log_ecs_metric_only.yaml | 1 - .../logs_and_kubernetes_config.yaml | 1 - 13 files changed, 48 insertions(+), 10 deletions(-) diff --git a/plugins/processors/kueueattributes/processor.go b/plugins/processors/kueueattributes/processor.go index 3bc7a5ebdb..809c6a7e74 100644 --- a/plugins/processors/kueueattributes/processor.go +++ b/plugins/processors/kueueattributes/processor.go @@ -25,6 +25,7 @@ var kueueLabelFilter = map[string]interface{}{ containerinsightscommon.ClusterQueueReasonKey: nil, containerinsightscommon.ClusterQueueResourceKey: nil, containerinsightscommon.Flavor: nil, + containerinsightscommon.NodeNameKey: nil, } type kueueAttributesProcessor struct { diff --git a/plugins/processors/kueueattributes/processor_test.go b/plugins/processors/kueueattributes/processor_test.go index 98f4378c55..4d0676664f 100644 --- a/plugins/processors/kueueattributes/processor_test.go +++ b/plugins/processors/kueueattributes/processor_test.go @@ -48,6 +48,7 @@ func TestProcessMetricsForKueueMetrics(t *testing.T) { "ClusterName": "cluster", "ClusterQueue": "development", "Status": "inadmissible", + "NodeName": "kubernetes-kueue", }, }), wantMetriccnt: 1, @@ -61,6 +62,7 @@ func TestProcessMetricsForKueueMetrics(t *testing.T) { "ClusterName": "cluster", "ClusterQueue": "development", "Status": "inadmissible", + "NodeName": "kubernetes-kueue", }, }, }, diff --git a/translator/tocwconfig/sampleConfig/appsignals_and_eks_config.yaml b/translator/tocwconfig/sampleConfig/appsignals_and_eks_config.yaml index 1042a3c968..17eeed4133 100644 --- a/translator/tocwconfig/sampleConfig/appsignals_and_eks_config.yaml +++ b/translator/tocwconfig/sampleConfig/appsignals_and_eks_config.yaml @@ -605,7 +605,6 @@ receivers: host_name: "" imds_retries: 1 kube_config_path: "" - kueue_container_insights: false leader_lock_name: cwagent-clusterleader leader_lock_using_config_map_only: true local_mode: false diff --git a/translator/tocwconfig/sampleConfig/appsignals_and_k8s_config.yaml b/translator/tocwconfig/sampleConfig/appsignals_and_k8s_config.yaml index 792c8c3c15..acc2bac366 100644 --- a/translator/tocwconfig/sampleConfig/appsignals_and_k8s_config.yaml +++ b/translator/tocwconfig/sampleConfig/appsignals_and_k8s_config.yaml @@ -606,7 +606,6 @@ receivers: host_name: "" imds_retries: 1 kube_config_path: "" - kueue_container_insights: false leader_lock_name: cwagent-clusterleader leader_lock_using_config_map_only: true local_mode: false diff --git a/translator/tocwconfig/sampleConfig/appsignals_fallback_and_eks_config.yaml b/translator/tocwconfig/sampleConfig/appsignals_fallback_and_eks_config.yaml index 1042a3c968..17eeed4133 100644 --- a/translator/tocwconfig/sampleConfig/appsignals_fallback_and_eks_config.yaml +++ b/translator/tocwconfig/sampleConfig/appsignals_fallback_and_eks_config.yaml @@ -605,7 +605,6 @@ receivers: host_name: "" imds_retries: 1 kube_config_path: "" - kueue_container_insights: false leader_lock_name: cwagent-clusterleader leader_lock_using_config_map_only: true local_mode: false diff --git a/translator/tocwconfig/sampleConfig/appsignals_over_fallback_config.yaml b/translator/tocwconfig/sampleConfig/appsignals_over_fallback_config.yaml index 1042a3c968..17eeed4133 100644 --- a/translator/tocwconfig/sampleConfig/appsignals_over_fallback_config.yaml +++ b/translator/tocwconfig/sampleConfig/appsignals_over_fallback_config.yaml @@ -605,7 +605,6 @@ receivers: host_name: "" imds_retries: 1 kube_config_path: "" - kueue_container_insights: false leader_lock_name: cwagent-clusterleader leader_lock_using_config_map_only: true local_mode: false diff --git a/translator/tocwconfig/sampleConfig/base_container_insights_config.yaml b/translator/tocwconfig/sampleConfig/base_container_insights_config.yaml index de230cb3e5..d961e759bf 100644 --- a/translator/tocwconfig/sampleConfig/base_container_insights_config.yaml +++ b/translator/tocwconfig/sampleConfig/base_container_insights_config.yaml @@ -176,7 +176,6 @@ receivers: host_name: "" imds_retries: 1 kube_config_path: "" - kueue_container_insights: false leader_lock_name: cwagent-clusterleader leader_lock_using_config_map_only: true local_mode: false diff --git a/translator/tocwconfig/sampleConfig/emf_and_kubernetes_config.yaml b/translator/tocwconfig/sampleConfig/emf_and_kubernetes_config.yaml index 2bae7a3e8d..52deef3e10 100644 --- a/translator/tocwconfig/sampleConfig/emf_and_kubernetes_config.yaml +++ b/translator/tocwconfig/sampleConfig/emf_and_kubernetes_config.yaml @@ -436,7 +436,6 @@ receivers: host_name: "" imds_retries: 2 kube_config_path: "" - kueue_container_insights: false leader_lock_name: cwagent-clusterleader leader_lock_using_config_map_only: true local_mode: true diff --git a/translator/tocwconfig/sampleConfig/emf_and_kubernetes_with_gpu_config.yaml b/translator/tocwconfig/sampleConfig/emf_and_kubernetes_with_gpu_config.yaml index bc6c588ca5..99f1b2868e 100644 --- a/translator/tocwconfig/sampleConfig/emf_and_kubernetes_with_gpu_config.yaml +++ b/translator/tocwconfig/sampleConfig/emf_and_kubernetes_with_gpu_config.yaml @@ -1108,7 +1108,6 @@ receivers: host_name: "" imds_retries: 2 kube_config_path: "" - kueue_container_insights: false leader_lock_name: cwagent-clusterleader leader_lock_using_config_map_only: true local_mode: true diff --git a/translator/tocwconfig/sampleConfig/emf_and_kubernetes_with_kueue_config.yaml b/translator/tocwconfig/sampleConfig/emf_and_kubernetes_with_kueue_config.yaml index f9f4a755ea..cf42b02387 100644 --- a/translator/tocwconfig/sampleConfig/emf_and_kubernetes_with_kueue_config.yaml +++ b/translator/tocwconfig/sampleConfig/emf_and_kubernetes_with_kueue_config.yaml @@ -308,6 +308,51 @@ exporters: - - ClusterName metric_name_selectors: - apiserver_flowcontrol_request_concurrency_limit + - dimensions: + - - ClusterName + - - ClusterName + - ClusterQueue + - - ClusterName + - ClusterQueue + - Status + - - ClusterName + - Status + metric_name_selectors: + - kueue_pending_workloads + - dimensions: + - - ClusterName + - - ClusterName + - ClusterQueue + - - ClusterName + - ClusterQueue + - Reason + - - ClusterName + - Reason + metric_name_selectors: + - kueue_evicted_workloads_total + - dimensions: + - - ClusterName + - - ClusterName + - ClusterQueue + metric_name_selectors: + - kueue_admitted_active_workloads + - dimensions: + - - ClusterName + - - ClusterName + - ClusterQueue + - - ClusterName + - ClusterQueue + - Resource + - - ClusterName + - ClusterQueue + - Flavor + - Resource + - - ClusterName + - ClusterQueue + - Flavor + metric_name_selectors: + - kueue_cluster_queue_resource_usage + - kueue_cluster_queue_nominal_quota metric_descriptors: - metric_name: apiserver_admission_controller_admission_duration_seconds overwrite: true diff --git a/translator/tocwconfig/sampleConfig/kubernetes_on_prem_config.yaml b/translator/tocwconfig/sampleConfig/kubernetes_on_prem_config.yaml index 161d76df08..b8602ca3cc 100644 --- a/translator/tocwconfig/sampleConfig/kubernetes_on_prem_config.yaml +++ b/translator/tocwconfig/sampleConfig/kubernetes_on_prem_config.yaml @@ -398,7 +398,6 @@ receivers: host_name: "" imds_retries: 1 kube_config_path: "" - kueue_container_insights: false leader_lock_name: cwagent-clusterleader leader_lock_using_config_map_only: true local_mode: true diff --git a/translator/tocwconfig/sampleConfig/log_ecs_metric_only.yaml b/translator/tocwconfig/sampleConfig/log_ecs_metric_only.yaml index 2304755f23..835ec94cbf 100644 --- a/translator/tocwconfig/sampleConfig/log_ecs_metric_only.yaml +++ b/translator/tocwconfig/sampleConfig/log_ecs_metric_only.yaml @@ -125,7 +125,6 @@ receivers: host_name: "" imds_retries: 1 kube_config_path: "" - kueue_container_insights: false leader_lock_name: otel-container-insight-clusterleader leader_lock_using_config_map_only: false local_mode: false diff --git a/translator/tocwconfig/sampleConfig/logs_and_kubernetes_config.yaml b/translator/tocwconfig/sampleConfig/logs_and_kubernetes_config.yaml index d9ec3480fa..a6b56b0c5f 100644 --- a/translator/tocwconfig/sampleConfig/logs_and_kubernetes_config.yaml +++ b/translator/tocwconfig/sampleConfig/logs_and_kubernetes_config.yaml @@ -432,7 +432,6 @@ receivers: host_name: "" imds_retries: 0 kube_config_path: "" - kueue_container_insights: false leader_lock_name: cwagent-clusterleader leader_lock_using_config_map_only: true local_mode: false From 4c9b6b6035ff4e243f8d28672ab99e3d5d23099f Mon Sep 17 00:00:00 2001 From: Rohan Gujarathi Date: Wed, 30 Oct 2024 11:32:20 -0700 Subject: [PATCH 04/32] drop ResourceMetricAttributes for kueue (#866) Co-authored-by: Rohan Gujarathi --- plugins/processors/kueueattributes/processor.go | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/plugins/processors/kueueattributes/processor.go b/plugins/processors/kueueattributes/processor.go index 809c6a7e74..8769f797ff 100644 --- a/plugins/processors/kueueattributes/processor.go +++ b/plugins/processors/kueueattributes/processor.go @@ -46,7 +46,8 @@ func newKueueAttributesProcessor(config *Config, logger *zap.Logger) *kueueAttri func (d *kueueAttributesProcessor) processMetrics(_ context.Context, md pmetric.Metrics) (pmetric.Metrics, error) { rms := md.ResourceMetrics() for i := 0; i < rms.Len(); i++ { - sms := rms.At(i).ScopeMetrics() + rm := rms.At(i) + sms := rm.ScopeMetrics() for j := 0; j < sms.Len(); j++ { metrics := sms.At(j).Metrics() for k := 0; k < metrics.Len(); k++ { @@ -54,12 +55,12 @@ func (d *kueueAttributesProcessor) processMetrics(_ context.Context, md pmetric. d.processMetricAttributes(m) } } + d.dropResourceMetricAttributes(rm) } return md, nil } func (d *kueueAttributesProcessor) processMetricAttributes(m pmetric.Metric) { - // only decorate kueue metrics if !strings.HasPrefix(m.Name(), kueueMetricsIdentifier) { return @@ -93,3 +94,13 @@ func (d *kueueAttributesProcessor) filterAttributes(attributes pcommon.Map) { return true }) } + +func (d *kueueAttributesProcessor) dropResourceMetricAttributes(resourceMetric pmetric.ResourceMetrics) { + serviceNameKey := "service.name" + attributes := resourceMetric.Resource().Attributes() + serviceName, exists := attributes.Get(serviceNameKey) + + if exists && (serviceName.Str() == "containerInsightsKueueMetricsScraper") { + resourceMetric.Resource().Attributes().Clear() + } +} From 97581d46d9e4950d19f32301087cfb97c3797727 Mon Sep 17 00:00:00 2001 From: Rahul Sahu Date: Wed, 30 Oct 2024 19:44:55 +0000 Subject: [PATCH 05/32] change: separate kueue metrics into its own container insights pipeline --- .../pipeline/containerinsights/translator.go | 63 ++++++++++++------- .../containerinsights/translator_test.go | 49 ++++++++++++--- .../pipeline/containerinsights/translators.go | 21 +++++++ translator/translate/otel/translate_otel.go | 3 +- 4 files changed, 104 insertions(+), 32 deletions(-) create mode 100644 translator/translate/otel/pipeline/containerinsights/translators.go diff --git a/translator/translate/otel/pipeline/containerinsights/translator.go b/translator/translate/otel/pipeline/containerinsights/translator.go index c3e940ada2..5ef964ff52 100644 --- a/translator/translate/otel/pipeline/containerinsights/translator.go +++ b/translator/translate/otel/pipeline/containerinsights/translator.go @@ -20,7 +20,8 @@ import ( ) const ( - pipelineName = "containerinsights" + ciPipelineName = "containerinsights" + kueuePipelineName = "kueueContainerInsights" ) var ( @@ -30,16 +31,21 @@ var ( ) type translator struct { + pipelineName string } var _ common.Translator[*common.ComponentTranslators] = (*translator)(nil) func NewTranslator() common.Translator[*common.ComponentTranslators] { - return &translator{} + return NewTranslatorWithName(ciPipelineName) +} + +func NewTranslatorWithName(pipelineName string) common.Translator[*common.ComponentTranslators] { + return &translator{pipelineName: pipelineName} } func (t *translator) ID() component.ID { - return component.NewIDWithName(component.DataTypeMetrics, pipelineName) + return component.NewIDWithName(component.DataTypeMetrics, t.pipelineName) } // Translate creates a pipeline for container insights if the logs.metrics_collected.ecs or logs.metrics_collected.kubernetes @@ -49,31 +55,44 @@ func (t *translator) Translate(conf *confmap.Conf) (*common.ComponentTranslators return nil, &common.MissingKeyError{ID: t.ID(), JsonKey: fmt.Sprint(ecsKey, " or ", eksKey)} } - // Append the metricstransformprocessor only if enhanced container insights is enabled - enhancedContainerInsightsEnabled := awscontainerinsight.EnhancedContainerInsightsEnabled(conf) - if enhancedContainerInsightsEnabled { - processors := common.NewTranslatorMap(metricstransformprocessor.NewTranslatorWithName(pipelineName)) - acceleratedComputeMetricsEnabled := awscontainerinsight.AcceleratedComputeMetricsEnabled(conf) - if acceleratedComputeMetricsEnabled { - processors.Set(gpu.NewTranslatorWithName(pipelineName)) + // create processor map with default batch processor based on pipeline name + processors := common.NewTranslatorMap(batchprocessor.NewTranslatorWithNameAndSection(t.pipelineName, common.LogsKey)) + // create exporter map with default emf exporter based on pipeline name + exporters := common.NewTranslatorMap(awsemf.NewTranslatorWithName(t.pipelineName)) + // create extensions map based on pipeline name + extensions := common.NewTranslatorMap(agenthealth.NewTranslator(component.DataTypeLogs, []string{agenthealth.OperationPutLogEvents})) + // create variable for receivers, use switch block below to assign + var receivers common.TranslatorMap[component.Config] + + switch t.pipelineName { + case ciPipelineName: + // add aws container insights receiver + receivers = common.NewTranslatorMap(awscontainerinsight.NewTranslator()) + // Append the metricstransformprocessor only if enhanced container insights is enabled + enhancedContainerInsightsEnabled := awscontainerinsight.EnhancedContainerInsightsEnabled(conf) + if enhancedContainerInsightsEnabled { + // add metricstransformprocessor to processors for enhanced container insights + processors.Set(metricstransformprocessor.NewTranslatorWithName(t.pipelineName)) + acceleratedComputeMetricsEnabled := awscontainerinsight.AcceleratedComputeMetricsEnabled(conf) + if acceleratedComputeMetricsEnabled { + processors.Set(gpu.NewTranslatorWithName(t.pipelineName)) + } } + case kueuePipelineName: + // add prometheus receiver for kueue + receivers = common.NewTranslatorMap(awscontainerinsight.NewTranslator()) // TODO: replace w prometheus receiver KueueContainerInsightsEnabled := awscontainerinsight.KueueContainerInsightsEnabled(conf) if KueueContainerInsightsEnabled { - processors.Set(kueue.NewTranslatorWithName(pipelineName)) + processors.Set(kueue.NewTranslatorWithName(t.pipelineName)) } - processors.Set(batchprocessor.NewTranslatorWithNameAndSection(pipelineName, common.LogsKey)) - return &common.ComponentTranslators{ - Receivers: common.NewTranslatorMap(awscontainerinsight.NewTranslator()), - Processors: processors, // EKS & ECS CI sit under metrics_collected in "logs" - Exporters: common.NewTranslatorMap(awsemf.NewTranslatorWithName(pipelineName)), - Extensions: common.NewTranslatorMap(agenthealth.NewTranslator(component.DataTypeLogs, []string{agenthealth.OperationPutLogEvents})), - }, nil + default: + return nil, fmt.Errorf("unknown container insights pipeline name: %s", t.pipelineName) } return &common.ComponentTranslators{ - Receivers: common.NewTranslatorMap(awscontainerinsight.NewTranslator()), - Processors: common.NewTranslatorMap(batchprocessor.NewTranslatorWithNameAndSection(pipelineName, common.LogsKey)), // EKS & ECS CI sit under metrics_collected in "logs" - Exporters: common.NewTranslatorMap(awsemf.NewTranslatorWithName(pipelineName)), - Extensions: common.NewTranslatorMap(agenthealth.NewTranslator(component.DataTypeLogs, []string{agenthealth.OperationPutLogEvents})), + Receivers: receivers, + Processors: processors, // EKS & ECS CI sit under metrics_collected in "logs" + Exporters: exporters, + Extensions: extensions, }, nil } diff --git a/translator/translate/otel/pipeline/containerinsights/translator_test.go b/translator/translate/otel/pipeline/containerinsights/translator_test.go index 7069f20347..5e06cd273f 100644 --- a/translator/translate/otel/pipeline/containerinsights/translator_test.go +++ b/translator/translate/otel/pipeline/containerinsights/translator_test.go @@ -67,28 +67,59 @@ func TestTranslator(t *testing.T) { extensions: []string{"agenthealth/logs"}, }, }, + } + for name, testCase := range testCases { + t.Run(name, func(t *testing.T) { + conf := confmap.NewFromStringMap(testCase.input) + got, err := cit.Translate(conf) + require.Equal(t, testCase.wantErr, err) + if testCase.want == nil { + require.Nil(t, got) + } else { + require.NotNil(t, got) + assert.Equal(t, testCase.want.receivers, collections.MapSlice(got.Receivers.Keys(), component.ID.String)) + assert.Equal(t, testCase.want.processors, collections.MapSlice(got.Processors.Keys(), component.ID.String)) + assert.Equal(t, testCase.want.exporters, collections.MapSlice(got.Exporters.Keys(), component.ID.String)) + assert.Equal(t, testCase.want.extensions, collections.MapSlice(got.Extensions.Keys(), component.ID.String)) + } + }) + } +} + +func TestKueueTranslator(t *testing.T) { + type want struct { + pipelineType string + receivers []string + processors []string + exporters []string + extensions []string + } + cit := NewTranslatorWithName(kueuePipelineName) + require.EqualValues(t, "metrics/kueueContainerInsights", cit.ID().String()) + testCases := map[string]struct { + input map[string]interface{} + want *want + wantErr error + }{ "WithKubernetes/WithEnhancedContainerInsights": { input: map[string]interface{}{ "logs": map[string]interface{}{ "metrics_collected": map[string]interface{}{ "kubernetes": map[string]interface{}{ - "enhanced_container_insights": true, - "kueue_container_insights": true, - "cluster_name": "TestCluster", + "kueue_container_insights": true, + "cluster_name": "TestCluster", }, }, }, }, want: &want{ - pipelineType: "metrics/containerinsights", + pipelineType: "metrics/kueueContainerInsights", receivers: []string{"awscontainerinsightreceiver"}, processors: []string{ - "metricstransform/containerinsights", - "gpuattributes/containerinsights", - "kueueattributes/containerinsights", - "batch/containerinsights", + "batch/kueueContainerInsights", + "kueueattributes/kueueContainerInsights", }, - exporters: []string{"awsemf/containerinsights"}, + exporters: []string{"awsemf/kueueContainerInsights"}, extensions: []string{"agenthealth/logs"}, }, }, diff --git a/translator/translate/otel/pipeline/containerinsights/translators.go b/translator/translate/otel/pipeline/containerinsights/translators.go new file mode 100644 index 0000000000..573e7d4dc6 --- /dev/null +++ b/translator/translate/otel/pipeline/containerinsights/translators.go @@ -0,0 +1,21 @@ +package containerinsights + +import ( + "go.opentelemetry.io/collector/confmap" + + "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/common" + "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/pipeline" +) + +func NewTranslators(conf *confmap.Conf) (pipeline.TranslatorMap, error) { + translators := common.NewTranslatorMap[*common.ComponentTranslators]() + // create default container insights translator + ciTranslator := NewTranslatorWithName(ciPipelineName) + // create kueue container insights translator + kueueTranslator := NewTranslatorWithName(kueuePipelineName) + // add both to the translator map + translators.Set(ciTranslator) + translators.Set(kueueTranslator) + // return the translator map + return translators, nil +} diff --git a/translator/translate/otel/translate_otel.go b/translator/translate/otel/translate_otel.go index 4fa223394e..0be667d406 100644 --- a/translator/translate/otel/translate_otel.go +++ b/translator/translate/otel/translate_otel.go @@ -61,9 +61,10 @@ func Translate(jsonConfig interface{}, os string) (*otelcol.Config, error) { return nil, err } translators.Merge(logsHostTranslators) + containerInsightsTranslators, _ := containerinsights.NewTranslators(conf) + translators.Merge(containerInsightsTranslators) translators.Set(applicationsignals.NewTranslator(component.DataTypeTraces)) translators.Set(applicationsignals.NewTranslator(component.DataTypeMetrics)) - translators.Set(containerinsights.NewTranslator()) translators.Set(prometheus.NewTranslator()) translators.Set(emf_logs.NewTranslator()) translators.Set(xray.NewTranslator()) From 41f140313b4ceaa3744f69e8078ed8be8dae856c Mon Sep 17 00:00:00 2001 From: Rahul Sahu Date: Wed, 30 Oct 2024 19:46:56 +0000 Subject: [PATCH 06/32] change: add separate emf exporter config for kueue metrics --- .../awsemf_default_kubernetes_kueue.yaml | 10 + .../otel/exporter/awsemf/kubernetes.go | 56 ----- .../translate/otel/exporter/awsemf/kueue.go | 74 +++++++ .../otel/exporter/awsemf/translator.go | 45 +++- .../otel/exporter/awsemf/translator_test.go | 194 +++++++++--------- 5 files changed, 223 insertions(+), 156 deletions(-) create mode 100644 translator/translate/otel/exporter/awsemf/awsemf_default_kubernetes_kueue.yaml create mode 100644 translator/translate/otel/exporter/awsemf/kueue.go diff --git a/translator/translate/otel/exporter/awsemf/awsemf_default_kubernetes_kueue.yaml b/translator/translate/otel/exporter/awsemf/awsemf_default_kubernetes_kueue.yaml new file mode 100644 index 0000000000..9db149ac1d --- /dev/null +++ b/translator/translate/otel/exporter/awsemf/awsemf_default_kubernetes_kueue.yaml @@ -0,0 +1,10 @@ +namespace: ContainerInsights/Prometheus +log_group_name: '/aws/containerinsights/{ClusterName}/performance' +log_stream_name: 'kubernetes-kueue' # look into using {JobName} placeholder rather than hardcoding +detailed_metrics: false +dimension_rollup_option: NoDimensionRollup +version: "0" +retain_initial_value_of_delta_metric: false +parse_json_encoded_attr_values: [ Sources, kubernetes ] +resource_to_telemetry_conversion: + enabled: true \ No newline at end of file diff --git a/translator/translate/otel/exporter/awsemf/kubernetes.go b/translator/translate/otel/exporter/awsemf/kubernetes.go index 9b346816a9..f7bf4bfa1f 100644 --- a/translator/translate/otel/exporter/awsemf/kubernetes.go +++ b/translator/translate/otel/exporter/awsemf/kubernetes.go @@ -54,9 +54,6 @@ func setKubernetesMetricDeclaration(conf *confmap.Conf, cfg *awsemfexporter.Conf kubernetesMetricDeclarations = append(kubernetesMetricDeclarations, getEFAMetricDeclarations(conf)...) - // Setup Kueue metrics - kubernetesMetricDeclarations = append(kubernetesMetricDeclarations, getKueueMetricDeclarations(conf)...) - cfg.MetricDeclarations = kubernetesMetricDeclarations cfg.MetricDescriptors = getControlPlaneMetricDescriptors(conf) @@ -647,56 +644,3 @@ func getEFAMetricDeclarations(conf *confmap.Conf) []*awsemfexporter.MetricDeclar } return metricDeclarations } - -func getKueueMetricDeclarations(conf *confmap.Conf) []*awsemfexporter.MetricDeclaration { - var metricDeclarations []*awsemfexporter.MetricDeclaration - if awscontainerinsight.KueueContainerInsightsEnabled(conf) { - metricDeclarations = []*awsemfexporter.MetricDeclaration{ - { - Dimensions: [][]string{ - {"ClusterName"}, - {"ClusterName", "ClusterQueue"}, - {"ClusterName", "ClusterQueue", "Status"}, - {"ClusterName", "Status"}, - }, - MetricNameSelectors: []string{ - "kueue_pending_workloads", - }, - }, - { - Dimensions: [][]string{ - {"ClusterName"}, - {"ClusterName", "ClusterQueue"}, - {"ClusterName", "ClusterQueue", "Reason"}, - {"ClusterName", "Reason"}, - }, - MetricNameSelectors: []string{ - "kueue_evicted_workloads_total", - }, - }, - { - Dimensions: [][]string{ - {"ClusterName"}, - {"ClusterName", "ClusterQueue"}, - }, - MetricNameSelectors: []string{ - "kueue_admitted_active_workloads", - }, - }, - { - Dimensions: [][]string{ - {"ClusterName"}, - {"ClusterName", "ClusterQueue"}, - {"ClusterName", "ClusterQueue", "Resource"}, - {"ClusterName", "ClusterQueue", "Resource", "Flavor"}, - {"ClusterName", "ClusterQueue", "Flavor"}, - }, - MetricNameSelectors: []string{ - "kueue_cluster_queue_resource_usage", - "kueue_cluster_queue_nominal_quota", - }, - }, - } - } - return metricDeclarations -} diff --git a/translator/translate/otel/exporter/awsemf/kueue.go b/translator/translate/otel/exporter/awsemf/kueue.go new file mode 100644 index 0000000000..0ce162e443 --- /dev/null +++ b/translator/translate/otel/exporter/awsemf/kueue.go @@ -0,0 +1,74 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +package awsemf + +import ( + "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awsemfexporter" + "go.opentelemetry.io/collector/confmap" + + "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/receiver/awscontainerinsight" +) + +func setKubernetesKueueMetricDeclaration(conf *confmap.Conf, cfg *awsemfexporter.Config) error { + var kueueMetricsDeclarations []*awsemfexporter.MetricDeclaration + + kueueMetricsDeclarations = append(kueueMetricsDeclarations, getKueueMetricDeclarations(conf)...) + + cfg.MetricDeclarations = kueueMetricsDeclarations + + return nil +} + +func getKueueMetricDeclarations(conf *confmap.Conf) []*awsemfexporter.MetricDeclaration { + var metricDeclarations []*awsemfexporter.MetricDeclaration + if awscontainerinsight.KueueContainerInsightsEnabled(conf) { + metricDeclarations = []*awsemfexporter.MetricDeclaration{ + { + Dimensions: [][]string{ + {"ClusterName"}, + {"ClusterName", "ClusterQueue"}, + {"ClusterName", "ClusterQueue", "Status"}, + {"ClusterName", "Status"}, + }, + MetricNameSelectors: []string{ + "kueue_pending_workloads", + }, + }, + { + Dimensions: [][]string{ + {"ClusterName"}, + {"ClusterName", "ClusterQueue"}, + {"ClusterName", "ClusterQueue", "Reason"}, + {"ClusterName", "Reason"}, + }, + MetricNameSelectors: []string{ + "kueue_evicted_workloads_total", + }, + }, + { + Dimensions: [][]string{ + {"ClusterName"}, + {"ClusterName", "ClusterQueue"}, + }, + MetricNameSelectors: []string{ + "kueue_admitted_active_workloads", + }, + }, + { + Dimensions: [][]string{ + {"ClusterName"}, + {"ClusterName", "ClusterQueue"}, + {"ClusterName", "ClusterQueue", "Resource"}, + {"ClusterName", "ClusterQueue", "Resource", "Flavor"}, + {"ClusterName", "ClusterQueue", "Flavor"}, + }, + MetricNameSelectors: []string{ + "kueue_cluster_queue_resource_usage", + "kueue_cluster_queue_nominal_quota", + }, + }, + } + } + return metricDeclarations +} diff --git a/translator/translate/otel/exporter/awsemf/translator.go b/translator/translate/otel/exporter/awsemf/translator.go index 764fa3532d..be63479a0a 100644 --- a/translator/translate/otel/exporter/awsemf/translator.go +++ b/translator/translate/otel/exporter/awsemf/translator.go @@ -25,6 +25,10 @@ import ( "github.com/aws/amazon-cloudwatch-agent/translator/util/ecsutil" ) +const ( + kueuePipelineName = "kueueContainerInsights" +) + //go:embed awsemf_default_generic.yaml var defaultGenericConfig string @@ -34,6 +38,9 @@ var defaultEcsConfig string //go:embed awsemf_default_kubernetes.yaml var defaultKubernetesConfig string +//go:embed awsemf_default_kubernetes_kueue.yaml +var defaultKubernetesKueueConfig string + //go:embed awsemf_default_prometheus.yaml var defaultPrometheusConfig string @@ -47,12 +54,13 @@ var appSignalsConfigK8s string var appSignalsConfigGeneric string var ( - ecsBasePathKey = common.ConfigKey(common.LogsKey, common.MetricsCollectedKey, common.ECSKey) - kubernetesBasePathKey = common.ConfigKey(common.LogsKey, common.MetricsCollectedKey, common.KubernetesKey) - prometheusBasePathKey = common.ConfigKey(common.LogsKey, common.MetricsCollectedKey, common.PrometheusKey) - emfProcessorBasePathKey = common.ConfigKey(prometheusBasePathKey, common.EMFProcessorKey) - endpointOverrideKey = common.ConfigKey(common.LogsKey, common.EndpointOverrideKey) - roleARNPathKey = common.ConfigKey(common.LogsKey, common.CredentialsKey, common.RoleARNKey) + ecsBasePathKey = common.ConfigKey(common.LogsKey, common.MetricsCollectedKey, common.ECSKey) + kubernetesBasePathKey = common.ConfigKey(common.LogsKey, common.MetricsCollectedKey, common.KubernetesKey) + kubernetesKueueBasePathKey = common.ConfigKey(common.LogsKey, common.MetricsCollectedKey, common.KubernetesKey, common.EnableKueueContainerInsights) + prometheusBasePathKey = common.ConfigKey(common.LogsKey, common.MetricsCollectedKey, common.PrometheusKey) + emfProcessorBasePathKey = common.ConfigKey(prometheusBasePathKey, common.EMFProcessorKey) + endpointOverrideKey = common.ConfigKey(common.LogsKey, common.EndpointOverrideKey) + roleARNPathKey = common.ConfigKey(common.LogsKey, common.CredentialsKey, common.RoleARNKey) ) type translator struct { @@ -84,6 +92,8 @@ func (t *translator) Translate(c *confmap.Conf) (component.Config, error) { defaultConfig = getAppSignalsConfig() } else if isEcs(c) { defaultConfig = defaultEcsConfig + } else if isKubernetesKueue(c, t.name) { + defaultConfig = defaultKubernetesKueueConfig } else if isKubernetes(c) { defaultConfig = defaultKubernetesConfig } else if isPrometheus(c) { @@ -128,6 +138,10 @@ func (t *translator) Translate(c *confmap.Conf) (component.Config, error) { if err := setEcsFields(c, cfg); err != nil { return nil, err } + } else if isKubernetesKueue(c, t.name) { + if err := setKubernetesKueueFields(c, cfg); err != nil { + return nil, err + } } else if isKubernetes(c) { if err := setKubernetesFields(c, cfg); err != nil { return nil, err @@ -177,6 +191,11 @@ func isKubernetes(conf *confmap.Conf) bool { return conf.IsSet(kubernetesBasePathKey) } +// `kueue_container_insights` is a child of `kubernetes` in config spec. +func isKubernetesKueue(conf *confmap.Conf, pipelineName string) bool { + return isKubernetes(conf) && pipelineName == kueuePipelineName && conf.IsSet(kubernetesKueueBasePathKey) +} + func isPrometheus(conf *confmap.Conf) bool { return conf.IsSet(prometheusBasePathKey) } @@ -204,6 +223,20 @@ func setKubernetesFields(conf *confmap.Conf, cfg *awsemfexporter.Config) error { return nil } +func setKubernetesKueueFields(conf *confmap.Conf, cfg *awsemfexporter.Config) error { + setDisableMetricExtraction(kubernetesKueueBasePathKey, conf, cfg) + + if err := setKubernetesKueueMetricDeclaration(conf, cfg); err != nil { + return err + } + + if awscontainerinsight.KueueContainerInsightsEnabled(conf) { + cfg.KueueContainerInsights = true + } + + return nil +} + func setPrometheusFields(conf *confmap.Conf, cfg *awsemfexporter.Config) error { setDisableMetricExtraction(prometheusBasePathKey, conf, cfg) diff --git a/translator/translate/otel/exporter/awsemf/translator_test.go b/translator/translate/otel/exporter/awsemf/translator_test.go index 0fcfe1cc63..b0c779198a 100644 --- a/translator/translate/otel/exporter/awsemf/translator_test.go +++ b/translator/translate/otel/exporter/awsemf/translator_test.go @@ -186,100 +186,6 @@ func TestTranslator(t *testing.T) { "local_mode": false, }, }, - "GenerateAwsEmfExporterConfigKubernetesWithKueueMetrics": { - input: map[string]any{ - "logs": map[string]any{ - "metrics_collected": map[string]any{ - "kubernetes": map[string]any{ - "kueue_container_insights": true, - }, - }, - }, - }, - want: map[string]any{ - "namespace": "ContainerInsights", - "log_group_name": "/aws/containerinsights/{ClusterName}/performance", - "log_stream_name": "{NodeName}", - "dimension_rollup_option": "NoDimensionRollup", - "disable_metric_extraction": false, - "enhanced_container_insights": false, - "parse_json_encoded_attr_values": []string{"Sources", "kubernetes"}, - "output_destination": "cloudwatch", - "eks_fargate_container_insights_enabled": false, - "resource_to_telemetry_conversion": resourcetotelemetry.Settings{ - Enabled: true, - }, - "metric_declarations": []*awsemfexporter.MetricDeclaration{ - { - Dimensions: [][]string{{"PodName", "Namespace", "ClusterName"}, {"ClusterName"}, {"Service", "Namespace", "ClusterName"}, {"ClusterName", "Namespace"}}, - MetricNameSelectors: []string{"pod_cpu_utilization", "pod_memory_utilization", - "pod_network_rx_bytes", "pod_network_tx_bytes", "pod_cpu_utilization_over_pod_limit", - "pod_memory_utilization_over_pod_limit"}, - }, - { - Dimensions: [][]string{{"PodName", "Namespace", "ClusterName"}}, - MetricNameSelectors: []string{"pod_number_of_container_restarts"}, - }, - { - Dimensions: [][]string{{"PodName", "Namespace", "ClusterName"}, {"ClusterName"}}, - MetricNameSelectors: []string{"pod_cpu_reserved_capacity", "pod_memory_reserved_capacity"}, - }, - { - Dimensions: [][]string{{"NodeName", "InstanceId", "ClusterName"}, {"ClusterName"}}, - MetricNameSelectors: []string{"node_cpu_utilization", "node_memory_utilization", - "node_network_total_bytes", "node_cpu_reserved_capacity", - "node_memory_reserved_capacity", "node_number_of_running_pods", "node_number_of_running_containers"}, - }, - { - Dimensions: [][]string{{"ClusterName"}}, - MetricNameSelectors: []string{"node_cpu_usage_total", "node_cpu_limit", "node_memory_working_set", "node_memory_limit"}, - }, - { - Dimensions: [][]string{{"NodeName", "InstanceId", "ClusterName"}, {"ClusterName"}}, - MetricNameSelectors: []string{"node_filesystem_utilization"}, - }, - { - Dimensions: [][]string{{"Service", "Namespace", "ClusterName"}, {"ClusterName"}}, - MetricNameSelectors: []string{"service_number_of_running_pods"}, - }, - { - Dimensions: [][]string{{"Namespace", "ClusterName"}, {"ClusterName"}}, - MetricNameSelectors: []string{"namespace_number_of_running_pods"}, - }, - { - Dimensions: [][]string{{"ClusterName"}}, - MetricNameSelectors: []string{"cluster_node_count", "cluster_failed_node_count"}, - }, - { - Dimensions: [][]string{{"ClusterName"}, {"ClusterName", "ClusterQueue"}, {"ClusterName", "ClusterQueue", "Status"}, {"ClusterName", "Status"}}, - MetricNameSelectors: []string{ - "kueue_pending_workloads", - }, - }, - { - Dimensions: [][]string{{"ClusterName"}, {"ClusterName", "ClusterQueue"}, {"ClusterName", "ClusterQueue", "Reason"}, {"ClusterName", "Reason"}}, - MetricNameSelectors: []string{ - "kueue_evicted_workloads_total", - }, - }, - { - Dimensions: [][]string{{"ClusterName"}, {"ClusterName", "ClusterQueue"}}, - MetricNameSelectors: []string{ - "kueue_admitted_active_workloads", - }, - }, - { - Dimensions: [][]string{{"ClusterName"}, {"ClusterName", "ClusterQueue"}, {"ClusterName", "ClusterQueue", "Resource"}, {"ClusterName", "ClusterQueue", "Resource", "Flavor"}, {"ClusterName", "ClusterQueue", "Flavor"}}, - MetricNameSelectors: []string{ - "kueue_cluster_queue_resource_usage", - "kueue_cluster_queue_nominal_quota", - }, - }, - }, - "metric_descriptors": nilMetricDescriptorsSlice, - "local_mode": false, - }, - }, "GenerateAwsEmfExporterConfigKubernetesDisableMetricExtraction": { input: map[string]any{ "logs": map[string]any{ @@ -902,6 +808,106 @@ func TestTranslator(t *testing.T) { } } +func TestTranslatorForKueue(t *testing.T) { + t.Setenv(envconfig.AWS_CA_BUNDLE, "/ca/bundle") + agent.Global_Config.Region = "us-east-1" + agent.Global_Config.Role_arn = "global_arn" + tt := NewTranslatorWithName("kueueContainerInsights") + require.EqualValues(t, "awsemf/kueueContainerInsights", tt.ID().String()) + testCases := map[string]struct { + env map[string]string + input map[string]any + want map[string]any // Can't construct & use awsemfexporter.Config as it uses internal only types + wantErr error + }{ + "GenerateAwsEmfExporterConfigForKubernetesKueueMetrics": { + input: map[string]any{ + "logs": map[string]any{ + "metrics_collected": map[string]any{ + "kubernetes": map[string]any{ + "kueue_container_insights": true, + }, + }, + }, + }, + want: map[string]any{ + "namespace": "ContainerInsights/Prometheus", + "log_group_name": "/aws/containerinsights/{ClusterName}/performance", + "log_stream_name": "kubernetes-kueue", + "dimension_rollup_option": "NoDimensionRollup", + "disable_metric_extraction": false, + "enhanced_container_insights": false, + "parse_json_encoded_attr_values": []string{"Sources", "kubernetes"}, + "output_destination": "cloudwatch", + "eks_fargate_container_insights_enabled": false, + "resource_to_telemetry_conversion": resourcetotelemetry.Settings{ + Enabled: true, + }, + "metric_declarations": []*awsemfexporter.MetricDeclaration{ + { + Dimensions: [][]string{{"ClusterName"}, {"ClusterName", "ClusterQueue"}, {"ClusterName", "ClusterQueue", "Status"}, {"ClusterName", "Status"}}, + MetricNameSelectors: []string{ + "kueue_pending_workloads", + }, + }, + { + Dimensions: [][]string{{"ClusterName"}, {"ClusterName", "ClusterQueue"}, {"ClusterName", "ClusterQueue", "Reason"}, {"ClusterName", "Reason"}}, + MetricNameSelectors: []string{ + "kueue_evicted_workloads_total", + }, + }, + { + Dimensions: [][]string{{"ClusterName"}, {"ClusterName", "ClusterQueue"}}, + MetricNameSelectors: []string{ + "kueue_admitted_active_workloads", + }, + }, + { + Dimensions: [][]string{{"ClusterName"}, {"ClusterName", "ClusterQueue"}, {"ClusterName", "ClusterQueue", "Resource"}, {"ClusterName", "ClusterQueue", "Resource", "Flavor"}, {"ClusterName", "ClusterQueue", "Flavor"}}, + MetricNameSelectors: []string{ + "kueue_cluster_queue_resource_usage", + "kueue_cluster_queue_nominal_quota", + }, + }, + }, + "metric_descriptors": nilMetricDescriptorsSlice, + "local_mode": false, + }, + }, + } + for name, testCase := range testCases { + t.Run(name, func(t *testing.T) { + conf := confmap.NewFromStringMap(testCase.input) + got, err := tt.Translate(conf) + require.Equal(t, testCase.wantErr, err) + require.Truef(t, legacytranslator.IsTranslateSuccess(), "Error in legacy translation rules: %v", legacytranslator.ErrorMessages) + if err == nil { + require.NotNil(t, got) + gotCfg, ok := got.(*awsemfexporter.Config) + require.True(t, ok) + assert.Equal(t, testCase.want["namespace"], gotCfg.Namespace) + assert.Equal(t, testCase.want["log_group_name"], gotCfg.LogGroupName) + assert.Equal(t, testCase.want["log_stream_name"], gotCfg.LogStreamName) + assert.Equal(t, testCase.want["dimension_rollup_option"], gotCfg.DimensionRollupOption) + assert.Equal(t, testCase.want["disable_metric_extraction"], gotCfg.DisableMetricExtraction) + assert.Equal(t, testCase.want["enhanced_container_insights"], gotCfg.EnhancedContainerInsights) + assert.Equal(t, testCase.want["parse_json_encoded_attr_values"], gotCfg.ParseJSONEncodedAttributeValues) + assert.Equal(t, testCase.want["output_destination"], gotCfg.OutputDestination) + assert.Equal(t, testCase.want["eks_fargate_container_insights_enabled"], gotCfg.EKSFargateContainerInsightsEnabled) + assert.Equal(t, testCase.want["resource_to_telemetry_conversion"], gotCfg.ResourceToTelemetrySettings) + assert.ElementsMatch(t, testCase.want["metric_declarations"], gotCfg.MetricDeclarations) + assert.ElementsMatch(t, testCase.want["metric_descriptors"], gotCfg.MetricDescriptors) + assert.Equal(t, testCase.want["local_mode"], gotCfg.LocalMode) + assert.Equal(t, "/ca/bundle", gotCfg.CertificateFilePath) + assert.Equal(t, "global_arn", gotCfg.RoleARN) + assert.Equal(t, "us-east-1", gotCfg.Region) + assert.NotNil(t, gotCfg.MiddlewareID) + assert.Equal(t, "agenthealth/logs", gotCfg.MiddlewareID.String()) + } + }) + } +} + func TestTranslateAppSignals(t *testing.T) { t.Setenv(envconfig.AWS_CA_BUNDLE, "/ca/bundle") agent.Global_Config.Region = "us-east-1" From 1a79607a0d42a2698c34292ef525a86918625bb9 Mon Sep 17 00:00:00 2001 From: Rahul Sahu Date: Wed, 6 Nov 2024 18:48:53 +0000 Subject: [PATCH 07/32] refactor: remove unnecessary kueue config field from emf exporter --- translator/translate/otel/exporter/awsemf/translator.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/translator/translate/otel/exporter/awsemf/translator.go b/translator/translate/otel/exporter/awsemf/translator.go index be63479a0a..6d8b4c6e66 100644 --- a/translator/translate/otel/exporter/awsemf/translator.go +++ b/translator/translate/otel/exporter/awsemf/translator.go @@ -230,10 +230,6 @@ func setKubernetesKueueFields(conf *confmap.Conf, cfg *awsemfexporter.Config) er return err } - if awscontainerinsight.KueueContainerInsightsEnabled(conf) { - cfg.KueueContainerInsights = true - } - return nil } From edcf26bf9e76034ddff7c45518f16f580bd183b5 Mon Sep 17 00:00:00 2001 From: Rahul Sahu Date: Wed, 6 Nov 2024 18:51:12 +0000 Subject: [PATCH 08/32] change: add awscontainerinsightskueuereceiver as a new receiver type to CWA --- service/defaultcomponents/components.go | 2 + service/defaultcomponents/components_test.go | 1 + .../awscontainerinsightskueue/translator.go | 78 ++++++++++++++++++ .../translator_test.go | 79 +++++++++++++++++++ 4 files changed, 160 insertions(+) create mode 100644 translator/translate/otel/receiver/awscontainerinsightskueue/translator.go create mode 100644 translator/translate/otel/receiver/awscontainerinsightskueue/translator_test.go diff --git a/service/defaultcomponents/components.go b/service/defaultcomponents/components.go index b11b8f9a1b..c159bd406b 100644 --- a/service/defaultcomponents/components.go +++ b/service/defaultcomponents/components.go @@ -29,6 +29,7 @@ import ( "github.com/open-telemetry/opentelemetry-collector-contrib/processor/tailsamplingprocessor" "github.com/open-telemetry/opentelemetry-collector-contrib/processor/transformprocessor" "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver" + "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awscontainerinsightskueuereceiver" "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awsecscontainermetricsreceiver" "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awsxrayreceiver" "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/filelogreceiver" @@ -67,6 +68,7 @@ func Factories() (otelcol.Factories, error) { if factories.Receivers, err = receiver.MakeFactoryMap( awscontainerinsightreceiver.NewFactory(), + awscontainerinsightskueuereceiver.NewFactory(), awsecscontainermetricsreceiver.NewFactory(), awsxrayreceiver.NewFactory(), filelogreceiver.NewFactory(), diff --git a/service/defaultcomponents/components_test.go b/service/defaultcomponents/components_test.go index 52d8a9ca22..1163a94cc2 100644 --- a/service/defaultcomponents/components_test.go +++ b/service/defaultcomponents/components_test.go @@ -18,6 +18,7 @@ func TestComponents(t *testing.T) { assert.NoError(t, err) wantReceivers := []string{ "awscontainerinsightreceiver", + "awscontainerinsightskueuereceiver", "awsecscontainermetrics", "awsxray", "filelog", diff --git a/translator/translate/otel/receiver/awscontainerinsightskueue/translator.go b/translator/translate/otel/receiver/awscontainerinsightskueue/translator.go new file mode 100644 index 0000000000..f9df860d1b --- /dev/null +++ b/translator/translate/otel/receiver/awscontainerinsightskueue/translator.go @@ -0,0 +1,78 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +package awscontainerinsightskueue + +import ( + "errors" + "time" + + "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awscontainerinsightskueuereceiver" + "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/confmap" + "go.opentelemetry.io/collector/receiver" + + "github.com/aws/amazon-cloudwatch-agent/translator/translate/logs/util" + "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/common" +) + +const ( + defaultMetricsCollectionInterval = time.Minute +) + +type translator struct { + name string + factory receiver.Factory +} + +var _ common.Translator[component.Config] = (*translator)(nil) + +// NewTranslator creates a new aws container insight receiver translator. +func NewTranslator() common.Translator[component.Config] { + return NewTranslatorWithName("") +} + +func NewTranslatorWithName(name string) common.Translator[component.Config] { + return &translator{ + name: name, + factory: awscontainerinsightskueuereceiver.NewFactory(), + } +} + +func (t *translator) ID() component.ID { + return component.NewIDWithName(t.factory.Type(), t.name) +} + +// Translate creates an aws container insights receiver config if either +// of the sections defined in the services exist. +func (t *translator) Translate(conf *confmap.Conf) (component.Config, error) { + cfg := t.factory.CreateDefaultConfig().(*awscontainerinsightskueuereceiver.Config) + intervalKeyChain := []string{ + common.ConfigKey(common.AgentKey, common.MetricsCollectionIntervalKey), + } + cfg.CollectionInterval = common.GetOrDefaultDuration(conf, intervalKeyChain, defaultMetricsCollectionInterval) + + if err := t.setClusterName(conf, cfg); err != nil { + return nil, err + } + + return cfg, nil +} + +func (t *translator) setClusterName(conf *confmap.Conf, cfg *awscontainerinsightskueuereceiver.Config) error { + clusterNameKey := common.ConfigKey(common.LogsKey, common.MetricsCollectedKey, common.KubernetesKey, "cluster_name") + if clusterName, ok := common.GetString(conf, clusterNameKey); ok { + cfg.ClusterName = clusterName + } else { + cfg.ClusterName = util.GetClusterNameFromEc2Tagger() + } + + if cfg.ClusterName == "" { + return errors.New("cluster name is not provided and was not auto-detected from EC2 tags") + } + return nil +} + +func KueueContainerInsightsEnabled(conf *confmap.Conf) bool { + return common.GetOrDefaultBool(conf, common.ConfigKey(common.LogsKey, common.MetricsCollectedKey, common.KubernetesKey, common.EnableKueueContainerInsights), false) +} diff --git a/translator/translate/otel/receiver/awscontainerinsightskueue/translator_test.go b/translator/translate/otel/receiver/awscontainerinsightskueue/translator_test.go new file mode 100644 index 0000000000..1216605538 --- /dev/null +++ b/translator/translate/otel/receiver/awscontainerinsightskueue/translator_test.go @@ -0,0 +1,79 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +package awscontainerinsightskueue + +import ( + "testing" + "time" + + "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awscontainerinsightskueuereceiver" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/collector/confmap" + + "github.com/aws/amazon-cloudwatch-agent/translator/context" +) + +func TestTranslator(t *testing.T) { + acit := NewTranslator() + require.EqualValues(t, "awscontainerinsightskueuereceiver", acit.ID().String()) + testCases := map[string]struct { + input map[string]interface{} + isSystemd bool + want *awscontainerinsightskueuereceiver.Config + wantErr error + }{ + "WithClusterName": { + input: map[string]interface{}{ + "logs": map[string]interface{}{ + "metrics_collected": map[string]interface{}{ + "kubernetes": map[string]interface{}{ + "cluster_name": "TestCluster", + "kueue_container_insights": true, + }, + }, + }, + }, + isSystemd: true, + want: &awscontainerinsightskueuereceiver.Config{ + CollectionInterval: 60 * time.Second, + ClusterName: "TestCluster", + }, + }, + "WithClusterNameAndCollectionInterval": { + input: map[string]interface{}{ + "logs": map[string]interface{}{ + "metrics_collected": map[string]interface{}{ + "kubernetes": map[string]interface{}{ + "cluster_name": "TestCluster", + "kueue_container_insights": true, + }, + }, + }, + "agent": map[string]interface{}{ + "metrics_collection_interval": 30, + }, + }, + isSystemd: true, + want: &awscontainerinsightskueuereceiver.Config{ + CollectionInterval: 30 * time.Second, + ClusterName: "TestCluster", + }, + }, + } + for name, testCase := range testCases { + t.Run(name, func(t *testing.T) { + context.CurrentContext().SetRunInContainer(!testCase.isSystemd) + conf := confmap.NewFromStringMap(testCase.input) + got, err := acit.Translate(conf) + require.Equal(t, testCase.wantErr, err) + if err == nil { + require.NotNil(t, got) + gotCfg, ok := got.(*awscontainerinsightskueuereceiver.Config) + require.True(t, ok) + require.Equal(t, testCase.want.CollectionInterval, gotCfg.CollectionInterval) + require.Equal(t, testCase.want.ClusterName, gotCfg.ClusterName) + } + }) + } +} From 71d8f8f9bbf51c1c7e716fe854516264d456b7c4 Mon Sep 17 00:00:00 2001 From: Rahul Sahu Date: Wed, 6 Nov 2024 19:47:59 +0000 Subject: [PATCH 09/32] change: adjust kueue pipeline translator to use awscontainerinsightskueuereceiver --- .../translate/otel/pipeline/containerinsights/translator.go | 4 +++- .../otel/pipeline/containerinsights/translator_test.go | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/translator/translate/otel/pipeline/containerinsights/translator.go b/translator/translate/otel/pipeline/containerinsights/translator.go index 5ef964ff52..8cf02dc9ac 100644 --- a/translator/translate/otel/pipeline/containerinsights/translator.go +++ b/translator/translate/otel/pipeline/containerinsights/translator.go @@ -17,6 +17,7 @@ import ( "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/processor/kueue" "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/processor/metricstransformprocessor" "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/receiver/awscontainerinsight" + "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/receiver/awscontainerinsightskueue" ) const ( @@ -80,7 +81,8 @@ func (t *translator) Translate(conf *confmap.Conf) (*common.ComponentTranslators } case kueuePipelineName: // add prometheus receiver for kueue - receivers = common.NewTranslatorMap(awscontainerinsight.NewTranslator()) // TODO: replace w prometheus receiver + //receivers = common.NewTranslatorMap(awscontainerinsight.NewTranslator()) // TODO: replace w prometheus receiver + receivers = common.NewTranslatorMap((awscontainerinsightskueue.NewTranslator())) KueueContainerInsightsEnabled := awscontainerinsight.KueueContainerInsightsEnabled(conf) if KueueContainerInsightsEnabled { processors.Set(kueue.NewTranslatorWithName(t.pipelineName)) diff --git a/translator/translate/otel/pipeline/containerinsights/translator_test.go b/translator/translate/otel/pipeline/containerinsights/translator_test.go index 5e06cd273f..3fe281a167 100644 --- a/translator/translate/otel/pipeline/containerinsights/translator_test.go +++ b/translator/translate/otel/pipeline/containerinsights/translator_test.go @@ -101,7 +101,7 @@ func TestKueueTranslator(t *testing.T) { want *want wantErr error }{ - "WithKubernetes/WithEnhancedContainerInsights": { + "WithKueueContainerInsights": { input: map[string]interface{}{ "logs": map[string]interface{}{ "metrics_collected": map[string]interface{}{ @@ -114,7 +114,7 @@ func TestKueueTranslator(t *testing.T) { }, want: &want{ pipelineType: "metrics/kueueContainerInsights", - receivers: []string{"awscontainerinsightreceiver"}, + receivers: []string{"awscontainerinsightskueuereceiver"}, processors: []string{ "batch/kueueContainerInsights", "kueueattributes/kueueContainerInsights", From beb8e4fc78b37d2cc998f26b015f94986e920204 Mon Sep 17 00:00:00 2001 From: Rahul Sahu Date: Wed, 6 Nov 2024 20:21:25 +0000 Subject: [PATCH 10/32] refactor: only create kueue pipeline translator if kueue_container_insights is true test: add unit tests for ``awscontainerinsights/translator.go`` --- .../pipeline/containerinsights/translators.go | 15 ++- .../containerinsights/translators_test.go | 107 ++++++++++++++++++ 2 files changed, 118 insertions(+), 4 deletions(-) create mode 100644 translator/translate/otel/pipeline/containerinsights/translators_test.go diff --git a/translator/translate/otel/pipeline/containerinsights/translators.go b/translator/translate/otel/pipeline/containerinsights/translators.go index 573e7d4dc6..ef8acc3926 100644 --- a/translator/translate/otel/pipeline/containerinsights/translators.go +++ b/translator/translate/otel/pipeline/containerinsights/translators.go @@ -5,17 +5,24 @@ import ( "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/common" "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/pipeline" + "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/receiver/awscontainerinsight" +) + +var ( + LogsKey = common.ConfigKey(common.LogsKey, common.MetricsCollectedKey) ) func NewTranslators(conf *confmap.Conf) (pipeline.TranslatorMap, error) { translators := common.NewTranslatorMap[*common.ComponentTranslators]() // create default container insights translator ciTranslator := NewTranslatorWithName(ciPipelineName) - // create kueue container insights translator - kueueTranslator := NewTranslatorWithName(kueuePipelineName) - // add both to the translator map translators.Set(ciTranslator) - translators.Set(kueueTranslator) + // create kueue container insights translator + KueueContainerInsightsEnabled := awscontainerinsight.KueueContainerInsightsEnabled(conf) + if KueueContainerInsightsEnabled { + kueueTranslator := NewTranslatorWithName(kueuePipelineName) + translators.Set(kueueTranslator) + } // return the translator map return translators, nil } diff --git a/translator/translate/otel/pipeline/containerinsights/translators_test.go b/translator/translate/otel/pipeline/containerinsights/translators_test.go new file mode 100644 index 0000000000..cb07211475 --- /dev/null +++ b/translator/translate/otel/pipeline/containerinsights/translators_test.go @@ -0,0 +1,107 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +package containerinsights + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/confmap" + + "github.com/aws/amazon-cloudwatch-agent/internal/util/collections" + _ "github.com/aws/amazon-cloudwatch-agent/translator/registerrules" + "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/common" +) + +func TestTranslators(t *testing.T) { + type want struct { + receivers []string + exporters []string + } + testCases := map[string]struct { + input map[string]any + want map[string]want + }{ + "WithContainerInsights": { + input: map[string]interface{}{ + "logs": map[string]interface{}{ + "metrics_collected": map[string]interface{}{ + "kubernetes": map[string]interface{}{ + "cluster_name": "TestCluster", + }, + }, + }, + }, + want: map[string]want{ + "metrics/containerinsights": { + receivers: []string{"awscontainerinsightreceiver"}, + exporters: []string{"awsemf/containerinsights"}, + }, + }, + }, + "WithEnhancedContainerInsights": { + input: map[string]interface{}{ + "logs": map[string]interface{}{ + "metrics_collected": map[string]interface{}{ + "kubernetes": map[string]interface{}{ + "enhanced_container_insights": true, + "cluster_name": "TestCluster", + }, + }, + }, + }, + want: map[string]want{ + "metrics/containerinsights": { + receivers: []string{"awscontainerinsightreceiver"}, + exporters: []string{"awsemf/containerinsights"}, + }, + }, + }, + "WithContainerInsightsAndKueueMetrics": { + input: map[string]interface{}{ + "logs": map[string]interface{}{ + "metrics_collected": map[string]interface{}{ + "kubernetes": map[string]interface{}{ + "kueue_container_insights": true, + "cluster_name": "TestCluster", + }, + }, + }, + }, + want: map[string]want{ + "metrics/containerinsights": { + receivers: []string{"awscontainerinsightreceiver"}, + exporters: []string{"awsemf/containerinsights"}, + }, + "metrics/kueueContainerInsights": { + receivers: []string{"awscontainerinsightskueuereceiver"}, + exporters: []string{"awsemf/kueueContainerInsights"}, + }, + }, + }, + } + for name, testCase := range testCases { + t.Run(name, func(t *testing.T) { + conf := confmap.NewFromStringMap(testCase.input) + got, err := NewTranslators(conf) + require.NoError(t, err) + if testCase.want == nil { + require.Nil(t, got) + } else { + require.NotNil(t, got) + assert.Equal(t, len(testCase.want), got.Len()) + got.Range(func(tr common.Translator[*common.ComponentTranslators]) { + w, ok := testCase.want[tr.ID().String()] + require.True(t, ok) + g, err := tr.Translate(conf) + assert.NoError(t, err) + assert.Equal(t, w.receivers, collections.MapSlice(g.Receivers.Keys(), component.ID.String)) + assert.Equal(t, w.exporters, collections.MapSlice(g.Exporters.Keys(), component.ID.String)) + }) + } + }) + } +} From ea4c0f0f576e552a48b5bb35d2961261fca3575d Mon Sep 17 00:00:00 2001 From: Rahul Sahu Date: Wed, 6 Nov 2024 20:24:45 +0000 Subject: [PATCH 11/32] change: remove kueue flag from container insights receiver translator --- .../translate/otel/receiver/awscontainerinsight/translator.go | 1 - 1 file changed, 1 deletion(-) diff --git a/translator/translate/otel/receiver/awscontainerinsight/translator.go b/translator/translate/otel/receiver/awscontainerinsight/translator.go index 32e8e96f9a..193fdbe87b 100644 --- a/translator/translate/otel/receiver/awscontainerinsight/translator.go +++ b/translator/translate/otel/receiver/awscontainerinsight/translator.go @@ -122,7 +122,6 @@ func (t *translator) Translate(conf *confmap.Conf) (component.Config, error) { cfg.PrefFullPodName = cfg.PrefFullPodName || common.GetOrDefaultBool(conf, common.ConfigKey(common.LogsKey, common.MetricsCollectedKey, common.KubernetesKey, common.PreferFullPodName), false) cfg.EnableAcceleratedComputeMetrics = cfg.EnableAcceleratedComputeMetrics || AcceleratedComputeMetricsEnabled(conf) - cfg.EnableKueueContainerInsights = cfg.EnableKueueContainerInsights || KueueContainerInsightsEnabled(conf) return cfg, nil } From 024f0b3620afea36489c2ee67d78ea32024e2dd8 Mon Sep 17 00:00:00 2001 From: Rahul Sahu Date: Wed, 6 Nov 2024 20:25:35 +0000 Subject: [PATCH 12/32] change: add default emf config for kueue --- .../otel/exporter/awsemf/awsemf_default_kubernetes_kueue.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/translator/translate/otel/exporter/awsemf/awsemf_default_kubernetes_kueue.yaml b/translator/translate/otel/exporter/awsemf/awsemf_default_kubernetes_kueue.yaml index 9db149ac1d..2ee8d9b5f7 100644 --- a/translator/translate/otel/exporter/awsemf/awsemf_default_kubernetes_kueue.yaml +++ b/translator/translate/otel/exporter/awsemf/awsemf_default_kubernetes_kueue.yaml @@ -1,6 +1,6 @@ namespace: ContainerInsights/Prometheus log_group_name: '/aws/containerinsights/{ClusterName}/performance' -log_stream_name: 'kubernetes-kueue' # look into using {JobName} placeholder rather than hardcoding +log_stream_name: 'kubernetes-kueue' detailed_metrics: false dimension_rollup_option: NoDimensionRollup version: "0" From bf8e9231a9f0b99402bb8d67c5e7c92f492e52e0 Mon Sep 17 00:00:00 2001 From: Rahul Sahu Date: Wed, 6 Nov 2024 20:56:07 +0000 Subject: [PATCH 13/32] test: update tocw sample configs to reflect a separate receiver for kueue --- .../appsignals_and_eks_config.yaml | 1 - .../appsignals_and_k8s_config.yaml | 1 - .../appsignals_fallback_and_eks_config.yaml | 1 - .../appsignals_over_fallback_config.yaml | 1 - .../base_container_insights_config.yaml | 1 - .../emf_and_kubernetes_config.yaml | 1 - .../emf_and_kubernetes_with_gpu_config.yaml | 1 - .../emf_and_kubernetes_with_kueue_config.yaml | 149 ++++++++++++------ .../kubernetes_on_prem_config.yaml | 1 - .../kueue_container_insights_config.yaml | 53 ++++++- .../sampleConfig/log_ecs_metric_only.yaml | 1 - .../logs_and_kubernetes_config.yaml | 1 - 12 files changed, 150 insertions(+), 62 deletions(-) diff --git a/translator/tocwconfig/sampleConfig/appsignals_and_eks_config.yaml b/translator/tocwconfig/sampleConfig/appsignals_and_eks_config.yaml index 17eeed4133..0993f8e186 100644 --- a/translator/tocwconfig/sampleConfig/appsignals_and_eks_config.yaml +++ b/translator/tocwconfig/sampleConfig/appsignals_and_eks_config.yaml @@ -591,7 +591,6 @@ processors: receivers: awscontainerinsightreceiver: accelerated_compute_metrics: false - kueue_container_insights: false add_container_name_metric_label: false add_full_pod_name_metric_label: false add_service_as_attribute: true diff --git a/translator/tocwconfig/sampleConfig/appsignals_and_k8s_config.yaml b/translator/tocwconfig/sampleConfig/appsignals_and_k8s_config.yaml index acc2bac366..d66ab2f1ba 100644 --- a/translator/tocwconfig/sampleConfig/appsignals_and_k8s_config.yaml +++ b/translator/tocwconfig/sampleConfig/appsignals_and_k8s_config.yaml @@ -592,7 +592,6 @@ processors: receivers: awscontainerinsightreceiver: accelerated_compute_metrics: false - kueue_container_insights: false add_container_name_metric_label: false add_full_pod_name_metric_label: false add_service_as_attribute: true diff --git a/translator/tocwconfig/sampleConfig/appsignals_fallback_and_eks_config.yaml b/translator/tocwconfig/sampleConfig/appsignals_fallback_and_eks_config.yaml index 17eeed4133..0993f8e186 100644 --- a/translator/tocwconfig/sampleConfig/appsignals_fallback_and_eks_config.yaml +++ b/translator/tocwconfig/sampleConfig/appsignals_fallback_and_eks_config.yaml @@ -591,7 +591,6 @@ processors: receivers: awscontainerinsightreceiver: accelerated_compute_metrics: false - kueue_container_insights: false add_container_name_metric_label: false add_full_pod_name_metric_label: false add_service_as_attribute: true diff --git a/translator/tocwconfig/sampleConfig/appsignals_over_fallback_config.yaml b/translator/tocwconfig/sampleConfig/appsignals_over_fallback_config.yaml index 17eeed4133..0993f8e186 100644 --- a/translator/tocwconfig/sampleConfig/appsignals_over_fallback_config.yaml +++ b/translator/tocwconfig/sampleConfig/appsignals_over_fallback_config.yaml @@ -591,7 +591,6 @@ processors: receivers: awscontainerinsightreceiver: accelerated_compute_metrics: false - kueue_container_insights: false add_container_name_metric_label: false add_full_pod_name_metric_label: false add_service_as_attribute: true diff --git a/translator/tocwconfig/sampleConfig/base_container_insights_config.yaml b/translator/tocwconfig/sampleConfig/base_container_insights_config.yaml index d961e759bf..4db3578ebf 100644 --- a/translator/tocwconfig/sampleConfig/base_container_insights_config.yaml +++ b/translator/tocwconfig/sampleConfig/base_container_insights_config.yaml @@ -162,7 +162,6 @@ processors: receivers: awscontainerinsightreceiver: accelerated_compute_metrics: false - kueue_container_insights: false add_container_name_metric_label: false add_full_pod_name_metric_label: false add_service_as_attribute: true diff --git a/translator/tocwconfig/sampleConfig/emf_and_kubernetes_config.yaml b/translator/tocwconfig/sampleConfig/emf_and_kubernetes_config.yaml index 52deef3e10..3dc8048000 100644 --- a/translator/tocwconfig/sampleConfig/emf_and_kubernetes_config.yaml +++ b/translator/tocwconfig/sampleConfig/emf_and_kubernetes_config.yaml @@ -422,7 +422,6 @@ processors: receivers: awscontainerinsightreceiver: accelerated_compute_metrics: false - kueue_container_insights: false add_container_name_metric_label: true add_full_pod_name_metric_label: true add_service_as_attribute: true diff --git a/translator/tocwconfig/sampleConfig/emf_and_kubernetes_with_gpu_config.yaml b/translator/tocwconfig/sampleConfig/emf_and_kubernetes_with_gpu_config.yaml index 99f1b2868e..7cb2fe15f0 100644 --- a/translator/tocwconfig/sampleConfig/emf_and_kubernetes_with_gpu_config.yaml +++ b/translator/tocwconfig/sampleConfig/emf_and_kubernetes_with_gpu_config.yaml @@ -1094,7 +1094,6 @@ processors: receivers: awscontainerinsightreceiver: accelerated_compute_metrics: true - kueue_container_insights: false add_container_name_metric_label: true add_full_pod_name_metric_label: true add_service_as_attribute: true diff --git a/translator/tocwconfig/sampleConfig/emf_and_kubernetes_with_kueue_config.yaml b/translator/tocwconfig/sampleConfig/emf_and_kubernetes_with_kueue_config.yaml index cf42b02387..e205d9525a 100644 --- a/translator/tocwconfig/sampleConfig/emf_and_kubernetes_with_kueue_config.yaml +++ b/translator/tocwconfig/sampleConfig/emf_and_kubernetes_with_kueue_config.yaml @@ -134,9 +134,9 @@ exporters: - pod_memory_request - pod_memory_limit - pod_cpu_limit + - pod_cpu_request - pod_cpu_usage_total - pod_memory_working_set - - pod_cpu_request - pod_container_status_running - pod_container_status_terminated - pod_container_status_waiting @@ -308,51 +308,6 @@ exporters: - - ClusterName metric_name_selectors: - apiserver_flowcontrol_request_concurrency_limit - - dimensions: - - - ClusterName - - - ClusterName - - ClusterQueue - - - ClusterName - - ClusterQueue - - Status - - - ClusterName - - Status - metric_name_selectors: - - kueue_pending_workloads - - dimensions: - - - ClusterName - - - ClusterName - - ClusterQueue - - - ClusterName - - ClusterQueue - - Reason - - - ClusterName - - Reason - metric_name_selectors: - - kueue_evicted_workloads_total - - dimensions: - - - ClusterName - - - ClusterName - - ClusterQueue - metric_name_selectors: - - kueue_admitted_active_workloads - - dimensions: - - - ClusterName - - - ClusterName - - ClusterQueue - - - ClusterName - - ClusterQueue - - Resource - - - ClusterName - - ClusterQueue - - Flavor - - Resource - - - ClusterName - - ClusterQueue - - Flavor - metric_name_selectors: - - kueue_cluster_queue_resource_usage - - kueue_cluster_queue_nominal_quota metric_descriptors: - metric_name: apiserver_admission_controller_admission_duration_seconds overwrite: true @@ -434,6 +389,86 @@ exporters: shared_credentials_file: - /root/.aws/credentials version: "0" + awsemf/kueueContainerInsights: + certificate_file_path: "" + detailed_metrics: false + dimension_rollup_option: NoDimensionRollup + disable_metric_extraction: false + eks_fargate_container_insights_enabled: false + endpoint: https://fake_endpoint + enhanced_container_insights: false + imds_retries: 2 + local_mode: true + log_group_name: /aws/containerinsights/{ClusterName}/performance + log_retention: 0 + log_stream_name: kubernetes-kueue + max_retries: 2 + metric_declarations: + - dimensions: + - - ClusterName + - - ClusterName + - ClusterQueue + - - ClusterName + - ClusterQueue + - Status + - - ClusterName + - Status + metric_name_selectors: + - kueue_pending_workloads + - dimensions: + - - ClusterName + - - ClusterName + - ClusterQueue + - - ClusterName + - ClusterQueue + - Reason + - - ClusterName + - Reason + metric_name_selectors: + - kueue_evicted_workloads_total + - dimensions: + - - ClusterName + - - ClusterName + - ClusterQueue + metric_name_selectors: + - kueue_admitted_active_workloads + - dimensions: + - - ClusterName + - - ClusterName + - ClusterQueue + - - ClusterName + - ClusterQueue + - Resource + - - ClusterName + - ClusterQueue + - Flavor + - Resource + - - ClusterName + - ClusterQueue + - Flavor + metric_name_selectors: + - kueue_cluster_queue_resource_usage + - kueue_cluster_queue_nominal_quota + middleware: agenthealth/logs + namespace: ContainerInsights/Prometheus + no_verify_ssl: false + num_workers: 8 + output_destination: cloudwatch + parse_json_encoded_attr_values: + - Sources + - kubernetes + profile: default + proxy_address: "" + region: us-east-1 + request_timeout_seconds: 30 + resource_arn: "" + resource_to_telemetry_conversion: + enabled: true + retain_initial_value_of_delta_metric: false + role_arn: "" + shared_credentials_file: + - /root/.aws/credentials + version: "0" extensions: agenthealth/logs: is_usage_data_enabled: true @@ -454,7 +489,12 @@ processors: send_batch_max_size: 0 send_batch_size: 8192 timeout: 5s - kueueattributes/containerinsights: {} + batch/kueueContainerInsights: + metadata_cardinality_limit: 1000 + send_batch_max_size: 0 + send_batch_size: 8192 + timeout: 5s + kueueattributes/kueueContainerInsights: {} metricstransform/containerinsights: transforms: - action: insert @@ -468,7 +508,6 @@ processors: receivers: awscontainerinsightreceiver: accelerated_compute_metrics: false - kueue_container_insights: true add_container_name_metric_label: true add_full_pod_name_metric_label: true add_service_as_attribute: true @@ -497,6 +536,9 @@ receivers: role_arn: "" shared_credentials_file: - /root/.aws/credentials + awscontainerinsightskueuereceiver: + cluster_name: TestCluster + collection_interval: 1m0s tcplog/emf_logs: encoding: utf-8 id: tcp_input @@ -539,11 +581,18 @@ service: exporters: - awsemf/containerinsights processors: - - metricstransform/containerinsights - - kueueattributes/containerinsights - batch/containerinsights + - metricstransform/containerinsights receivers: - awscontainerinsightreceiver + metrics/kueueContainerInsights: + exporters: + - awsemf/kueueContainerInsights + processors: + - batch/kueueContainerInsights + - kueueattributes/kueueContainerInsights + receivers: + - awscontainerinsightskueuereceiver telemetry: logs: development: false diff --git a/translator/tocwconfig/sampleConfig/kubernetes_on_prem_config.yaml b/translator/tocwconfig/sampleConfig/kubernetes_on_prem_config.yaml index b8602ca3cc..c5f7ebc112 100644 --- a/translator/tocwconfig/sampleConfig/kubernetes_on_prem_config.yaml +++ b/translator/tocwconfig/sampleConfig/kubernetes_on_prem_config.yaml @@ -384,7 +384,6 @@ processors: receivers: awscontainerinsightreceiver: accelerated_compute_metrics: false - kueue_container_insights: false add_container_name_metric_label: true add_full_pod_name_metric_label: true add_service_as_attribute: true diff --git a/translator/tocwconfig/sampleConfig/kueue_container_insights_config.yaml b/translator/tocwconfig/sampleConfig/kueue_container_insights_config.yaml index e99304f3a1..27c7190bd5 100644 --- a/translator/tocwconfig/sampleConfig/kueue_container_insights_config.yaml +++ b/translator/tocwconfig/sampleConfig/kueue_container_insights_config.yaml @@ -121,6 +121,39 @@ exporters: metric_name_selectors: - cluster_node_count - cluster_failed_node_count + middleware: agenthealth/logs + namespace: ContainerInsights + no_verify_ssl: false + num_workers: 8 + output_destination: cloudwatch + parse_json_encoded_attr_values: + - Sources + - kubernetes + profile: "" + proxy_address: "" + region: us-east-1 + request_timeout_seconds: 30 + resource_arn: "" + resource_to_telemetry_conversion: + enabled: true + retain_initial_value_of_delta_metric: false + role_arn: "" + version: "0" + awsemf/kueueContainerInsights: + certificate_file_path: /etc/test/ca_bundle.pem + detailed_metrics: false + dimension_rollup_option: NoDimensionRollup + disable_metric_extraction: false + eks_fargate_container_insights_enabled: false + endpoint: https://fake_endpoint + enhanced_container_insights: false + imds_retries: 1 + local_mode: false + log_group_name: /aws/containerinsights/{ClusterName}/performance + log_retention: 0 + log_stream_name: kubernetes-kueue + max_retries: 2 + metric_declarations: - dimensions: - - ClusterName - - ClusterName @@ -167,7 +200,7 @@ exporters: - kueue_cluster_queue_resource_usage - kueue_cluster_queue_nominal_quota middleware: agenthealth/logs - namespace: ContainerInsights + namespace: ContainerInsights/Prometheus no_verify_ssl: false num_workers: 8 output_destination: cloudwatch @@ -204,6 +237,12 @@ processors: send_batch_max_size: 0 send_batch_size: 8192 timeout: 5s + batch/kueueContainerInsights: + metadata_cardinality_limit: 1000 + send_batch_max_size: 0 + send_batch_size: 8192 + timeout: 5s + kueueattributes/kueueContainerInsights: {} receivers: awscontainerinsightreceiver: accelerated_compute_metrics: false @@ -220,7 +259,6 @@ receivers: host_name: "" imds_retries: 1 kube_config_path: "" - kueue_container_insights: true leader_lock_name: cwagent-clusterleader leader_lock_using_config_map_only: true local_mode: false @@ -234,6 +272,9 @@ receivers: request_timeout_seconds: 0 resource_arn: "" role_arn: "" + awscontainerinsightskueuereceiver: + cluster_name: TestCluster + collection_interval: 1m0s tcplog/emf_logs: encoding: utf-8 id: tcp_input @@ -279,6 +320,14 @@ service: - batch/containerinsights receivers: - awscontainerinsightreceiver + metrics/kueueContainerInsights: + exporters: + - awsemf/kueueContainerInsights + processors: + - batch/kueueContainerInsights + - kueueattributes/kueueContainerInsights + receivers: + - awscontainerinsightskueuereceiver telemetry: logs: development: false diff --git a/translator/tocwconfig/sampleConfig/log_ecs_metric_only.yaml b/translator/tocwconfig/sampleConfig/log_ecs_metric_only.yaml index 835ec94cbf..b08d51c861 100644 --- a/translator/tocwconfig/sampleConfig/log_ecs_metric_only.yaml +++ b/translator/tocwconfig/sampleConfig/log_ecs_metric_only.yaml @@ -111,7 +111,6 @@ processors: receivers: awscontainerinsightreceiver: accelerated_compute_metrics: true - kueue_container_insights: false add_container_name_metric_label: false add_full_pod_name_metric_label: false add_service_as_attribute: true diff --git a/translator/tocwconfig/sampleConfig/logs_and_kubernetes_config.yaml b/translator/tocwconfig/sampleConfig/logs_and_kubernetes_config.yaml index a6b56b0c5f..6000b359a5 100644 --- a/translator/tocwconfig/sampleConfig/logs_and_kubernetes_config.yaml +++ b/translator/tocwconfig/sampleConfig/logs_and_kubernetes_config.yaml @@ -418,7 +418,6 @@ processors: receivers: awscontainerinsightreceiver: accelerated_compute_metrics: false - kueue_container_insights: false add_container_name_metric_label: true add_full_pod_name_metric_label: true add_service_as_attribute: true From 10492d65baabb4267f34558503ad92102a0a4658 Mon Sep 17 00:00:00 2001 From: Rahul Sahu Date: Wed, 6 Nov 2024 20:58:56 +0000 Subject: [PATCH 14/32] change: add placeholder version for new receiver this will need to be replaced with a real version when merged into public repos --- go.mod | 1 + 1 file changed, 1 insertion(+) diff --git a/go.mod b/go.mod index 2033cc6e46..85355f02d4 100644 --- a/go.mod +++ b/go.mod @@ -37,6 +37,7 @@ replace ( replace github.com/open-telemetry/opentelemetry-collector-contrib/processor/resourcedetectionprocessor => github.com/amazon-contributing/opentelemetry-collector-contrib/processor/resourcedetectionprocessor v0.0.0-20241004163757-90ec962f8f20 replace ( + github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awscontainerinsightskueuereceiver => ../opentelemetry-collector-contrib/receiver/awscontainerinsightskueuereceiver github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver => github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver v0.0.0-20241004163757-90ec962f8f20 github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awsxrayreceiver => github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awsxrayreceiver v0.0.0-20241004163757-90ec962f8f20 github.com/open-telemetry/opentelemetry-collector-contrib/receiver/jmxreceiver => github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/jmxreceiver v0.0.0-20241004163757-90ec962f8f20 From 84f5d9f91d8b647e53a52acb247bf0acc82fa4cd Mon Sep 17 00:00:00 2001 From: Rahul Sahu Date: Wed, 6 Nov 2024 21:26:25 +0000 Subject: [PATCH 15/32] doc: add licence to containerinsights/translators.go --- .../translate/otel/pipeline/containerinsights/translators.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/translator/translate/otel/pipeline/containerinsights/translators.go b/translator/translate/otel/pipeline/containerinsights/translators.go index ef8acc3926..728085d406 100644 --- a/translator/translate/otel/pipeline/containerinsights/translators.go +++ b/translator/translate/otel/pipeline/containerinsights/translators.go @@ -1,3 +1,6 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + package containerinsights import ( From b5b05119bf58a9da898ebde2652ecfbd2561bd35 Mon Sep 17 00:00:00 2001 From: Rahul Sahu Date: Wed, 6 Nov 2024 21:31:54 +0000 Subject: [PATCH 16/32] fix: add another placeholder for new awscontainerinsightskueuereceiver to go.mod --- go.mod | 2 ++ 1 file changed, 2 insertions(+) diff --git a/go.mod b/go.mod index 85355f02d4..b2924043c9 100644 --- a/go.mod +++ b/go.mod @@ -201,6 +201,8 @@ require ( k8s.io/klog/v2 v2.120.1 ) +require github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awscontainerinsightskueuereceiver v0.0.0-00010101000000-000000000000 + require ( cloud.google.com/go v0.112.1 // indirect cloud.google.com/go/compute/metadata v0.3.0 // indirect From 47db1d82b45dba0a73d0907cc9adcefd9953cc21 Mon Sep 17 00:00:00 2001 From: Rahul Sahu Date: Thu, 7 Nov 2024 00:16:55 +0000 Subject: [PATCH 17/32] refactor: simplify emf exporter helper functions --- translator/translate/otel/exporter/awsemf/kueue.go | 7 +------ .../translate/otel/exporter/awsemf/translator.go | 12 ++++-------- 2 files changed, 5 insertions(+), 14 deletions(-) diff --git a/translator/translate/otel/exporter/awsemf/kueue.go b/translator/translate/otel/exporter/awsemf/kueue.go index 0ce162e443..610be4d22e 100644 --- a/translator/translate/otel/exporter/awsemf/kueue.go +++ b/translator/translate/otel/exporter/awsemf/kueue.go @@ -11,12 +11,7 @@ import ( ) func setKubernetesKueueMetricDeclaration(conf *confmap.Conf, cfg *awsemfexporter.Config) error { - var kueueMetricsDeclarations []*awsemfexporter.MetricDeclaration - - kueueMetricsDeclarations = append(kueueMetricsDeclarations, getKueueMetricDeclarations(conf)...) - - cfg.MetricDeclarations = kueueMetricsDeclarations - + cfg.MetricDeclarations = getKueueMetricDeclarations(conf) return nil } diff --git a/translator/translate/otel/exporter/awsemf/translator.go b/translator/translate/otel/exporter/awsemf/translator.go index 6d8b4c6e66..f5585fd62d 100644 --- a/translator/translate/otel/exporter/awsemf/translator.go +++ b/translator/translate/otel/exporter/awsemf/translator.go @@ -25,10 +25,6 @@ import ( "github.com/aws/amazon-cloudwatch-agent/translator/util/ecsutil" ) -const ( - kueuePipelineName = "kueueContainerInsights" -) - //go:embed awsemf_default_generic.yaml var defaultGenericConfig string @@ -92,7 +88,7 @@ func (t *translator) Translate(c *confmap.Conf) (component.Config, error) { defaultConfig = getAppSignalsConfig() } else if isEcs(c) { defaultConfig = defaultEcsConfig - } else if isKubernetesKueue(c, t.name) { + } else if isKubernetesKueue(c) { defaultConfig = defaultKubernetesKueueConfig } else if isKubernetes(c) { defaultConfig = defaultKubernetesConfig @@ -138,7 +134,7 @@ func (t *translator) Translate(c *confmap.Conf) (component.Config, error) { if err := setEcsFields(c, cfg); err != nil { return nil, err } - } else if isKubernetesKueue(c, t.name) { + } else if isKubernetesKueue(c) { if err := setKubernetesKueueFields(c, cfg); err != nil { return nil, err } @@ -192,8 +188,8 @@ func isKubernetes(conf *confmap.Conf) bool { } // `kueue_container_insights` is a child of `kubernetes` in config spec. -func isKubernetesKueue(conf *confmap.Conf, pipelineName string) bool { - return isKubernetes(conf) && pipelineName == kueuePipelineName && conf.IsSet(kubernetesKueueBasePathKey) +func isKubernetesKueue(conf *confmap.Conf) bool { + return isKubernetes(conf) && common.GetOrDefaultBool(conf, kubernetesKueueBasePathKey, false) } func isPrometheus(conf *confmap.Conf) bool { From fef26a5fac4f17d5583a6bc045a02ffed259943f Mon Sep 17 00:00:00 2001 From: Rahul Sahu Date: Thu, 7 Nov 2024 00:18:28 +0000 Subject: [PATCH 18/32] refactor: no need to always return nil error from containerinsights/translators.go --- .../translate/otel/pipeline/containerinsights/translators.go | 4 ++-- .../otel/pipeline/containerinsights/translators_test.go | 3 +-- translator/translate/otel/translate_otel.go | 2 +- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/translator/translate/otel/pipeline/containerinsights/translators.go b/translator/translate/otel/pipeline/containerinsights/translators.go index 728085d406..8243f45fe2 100644 --- a/translator/translate/otel/pipeline/containerinsights/translators.go +++ b/translator/translate/otel/pipeline/containerinsights/translators.go @@ -15,7 +15,7 @@ var ( LogsKey = common.ConfigKey(common.LogsKey, common.MetricsCollectedKey) ) -func NewTranslators(conf *confmap.Conf) (pipeline.TranslatorMap, error) { +func NewTranslators(conf *confmap.Conf) pipeline.TranslatorMap { translators := common.NewTranslatorMap[*common.ComponentTranslators]() // create default container insights translator ciTranslator := NewTranslatorWithName(ciPipelineName) @@ -27,5 +27,5 @@ func NewTranslators(conf *confmap.Conf) (pipeline.TranslatorMap, error) { translators.Set(kueueTranslator) } // return the translator map - return translators, nil + return translators } diff --git a/translator/translate/otel/pipeline/containerinsights/translators_test.go b/translator/translate/otel/pipeline/containerinsights/translators_test.go index cb07211475..178a0159b3 100644 --- a/translator/translate/otel/pipeline/containerinsights/translators_test.go +++ b/translator/translate/otel/pipeline/containerinsights/translators_test.go @@ -86,8 +86,7 @@ func TestTranslators(t *testing.T) { for name, testCase := range testCases { t.Run(name, func(t *testing.T) { conf := confmap.NewFromStringMap(testCase.input) - got, err := NewTranslators(conf) - require.NoError(t, err) + got := NewTranslators(conf) if testCase.want == nil { require.Nil(t, got) } else { diff --git a/translator/translate/otel/translate_otel.go b/translator/translate/otel/translate_otel.go index 0be667d406..c8d3f3f98c 100644 --- a/translator/translate/otel/translate_otel.go +++ b/translator/translate/otel/translate_otel.go @@ -61,7 +61,7 @@ func Translate(jsonConfig interface{}, os string) (*otelcol.Config, error) { return nil, err } translators.Merge(logsHostTranslators) - containerInsightsTranslators, _ := containerinsights.NewTranslators(conf) + containerInsightsTranslators := containerinsights.NewTranslators(conf) translators.Merge(containerInsightsTranslators) translators.Set(applicationsignals.NewTranslator(component.DataTypeTraces)) translators.Set(applicationsignals.NewTranslator(component.DataTypeMetrics)) From 481e9e9ba2a2297315cb4a55d77acd272af14b6e Mon Sep 17 00:00:00 2001 From: Rahul Sahu Date: Thu, 7 Nov 2024 00:23:53 +0000 Subject: [PATCH 19/32] refactor: move KueueContainerInsightsEnabled to containerinsights package --- .../translate/otel/pipeline/containerinsights/translator.go | 6 +++++- .../otel/pipeline/containerinsights/translators.go | 3 +-- .../translate/otel/receiver/awscontainerinsight/utils.go | 4 ---- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/translator/translate/otel/pipeline/containerinsights/translator.go b/translator/translate/otel/pipeline/containerinsights/translator.go index 8cf02dc9ac..8833e3f3b1 100644 --- a/translator/translate/otel/pipeline/containerinsights/translator.go +++ b/translator/translate/otel/pipeline/containerinsights/translator.go @@ -83,7 +83,7 @@ func (t *translator) Translate(conf *confmap.Conf) (*common.ComponentTranslators // add prometheus receiver for kueue //receivers = common.NewTranslatorMap(awscontainerinsight.NewTranslator()) // TODO: replace w prometheus receiver receivers = common.NewTranslatorMap((awscontainerinsightskueue.NewTranslator())) - KueueContainerInsightsEnabled := awscontainerinsight.KueueContainerInsightsEnabled(conf) + KueueContainerInsightsEnabled := KueueContainerInsightsEnabled(conf) if KueueContainerInsightsEnabled { processors.Set(kueue.NewTranslatorWithName(t.pipelineName)) } @@ -98,3 +98,7 @@ func (t *translator) Translate(conf *confmap.Conf) (*common.ComponentTranslators Extensions: extensions, }, nil } + +func KueueContainerInsightsEnabled(conf *confmap.Conf) bool { + return common.GetOrDefaultBool(conf, common.ConfigKey(common.LogsKey, common.MetricsCollectedKey, common.KubernetesKey, common.EnableKueueContainerInsights), false) +} diff --git a/translator/translate/otel/pipeline/containerinsights/translators.go b/translator/translate/otel/pipeline/containerinsights/translators.go index 8243f45fe2..f5637812e1 100644 --- a/translator/translate/otel/pipeline/containerinsights/translators.go +++ b/translator/translate/otel/pipeline/containerinsights/translators.go @@ -8,7 +8,6 @@ import ( "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/common" "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/pipeline" - "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/receiver/awscontainerinsight" ) var ( @@ -21,7 +20,7 @@ func NewTranslators(conf *confmap.Conf) pipeline.TranslatorMap { ciTranslator := NewTranslatorWithName(ciPipelineName) translators.Set(ciTranslator) // create kueue container insights translator - KueueContainerInsightsEnabled := awscontainerinsight.KueueContainerInsightsEnabled(conf) + KueueContainerInsightsEnabled := KueueContainerInsightsEnabled(conf) if KueueContainerInsightsEnabled { kueueTranslator := NewTranslatorWithName(kueuePipelineName) translators.Set(kueueTranslator) diff --git a/translator/translate/otel/receiver/awscontainerinsight/utils.go b/translator/translate/otel/receiver/awscontainerinsight/utils.go index ac0ae1976a..721951b056 100644 --- a/translator/translate/otel/receiver/awscontainerinsight/utils.go +++ b/translator/translate/otel/receiver/awscontainerinsight/utils.go @@ -27,7 +27,3 @@ func EnhancedContainerInsightsEnabled(conf *confmap.Conf) bool { func AcceleratedComputeMetricsEnabled(conf *confmap.Conf) bool { return common.GetOrDefaultBool(conf, common.ConfigKey(common.LogsKey, common.MetricsCollectedKey, common.KubernetesKey, common.EnableAcceleratedComputeMetric), true) } - -func KueueContainerInsightsEnabled(conf *confmap.Conf) bool { - return common.GetOrDefaultBool(conf, common.ConfigKey(common.LogsKey, common.MetricsCollectedKey, common.KubernetesKey, common.EnableKueueContainerInsights), false) -} From 7e54555117bd3173bb7cc28d3d9569987df823c6 Mon Sep 17 00:00:00 2001 From: Rahul Sahu Date: Thu, 7 Nov 2024 00:24:41 +0000 Subject: [PATCH 20/32] doc: remove commented code --- .../translate/otel/pipeline/containerinsights/translator.go | 1 - 1 file changed, 1 deletion(-) diff --git a/translator/translate/otel/pipeline/containerinsights/translator.go b/translator/translate/otel/pipeline/containerinsights/translator.go index 8833e3f3b1..ab307c59cc 100644 --- a/translator/translate/otel/pipeline/containerinsights/translator.go +++ b/translator/translate/otel/pipeline/containerinsights/translator.go @@ -81,7 +81,6 @@ func (t *translator) Translate(conf *confmap.Conf) (*common.ComponentTranslators } case kueuePipelineName: // add prometheus receiver for kueue - //receivers = common.NewTranslatorMap(awscontainerinsight.NewTranslator()) // TODO: replace w prometheus receiver receivers = common.NewTranslatorMap((awscontainerinsightskueue.NewTranslator())) KueueContainerInsightsEnabled := KueueContainerInsightsEnabled(conf) if KueueContainerInsightsEnabled { From f38780505996c07d8938e320876d1a0001d39a63 Mon Sep 17 00:00:00 2001 From: Rahul Sahu Date: Thu, 7 Nov 2024 00:26:00 +0000 Subject: [PATCH 21/32] change: remove redundant helper function --- .../otel/receiver/awscontainerinsightskueue/translator.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/translator/translate/otel/receiver/awscontainerinsightskueue/translator.go b/translator/translate/otel/receiver/awscontainerinsightskueue/translator.go index f9df860d1b..0e3582f6ff 100644 --- a/translator/translate/otel/receiver/awscontainerinsightskueue/translator.go +++ b/translator/translate/otel/receiver/awscontainerinsightskueue/translator.go @@ -72,7 +72,3 @@ func (t *translator) setClusterName(conf *confmap.Conf, cfg *awscontainerinsight } return nil } - -func KueueContainerInsightsEnabled(conf *confmap.Conf) bool { - return common.GetOrDefaultBool(conf, common.ConfigKey(common.LogsKey, common.MetricsCollectedKey, common.KubernetesKey, common.EnableKueueContainerInsights), false) -} From 204f0a246108e8bae8ec0669c13a2d6561da237e Mon Sep 17 00:00:00 2001 From: Rahul Sahu Date: Thu, 7 Nov 2024 00:46:12 +0000 Subject: [PATCH 22/32] fix: move KueueContainerInsightsEnabled to common package --- translator/translate/otel/common/common.go | 4 ++++ translator/translate/otel/exporter/awsemf/kueue.go | 5 ++--- .../translate/otel/pipeline/containerinsights/translator.go | 6 +----- .../otel/pipeline/containerinsights/translators.go | 2 +- 4 files changed, 8 insertions(+), 9 deletions(-) diff --git a/translator/translate/otel/common/common.go b/translator/translate/otel/common/common.go index 5ef24c1eb1..b575783ed3 100644 --- a/translator/translate/otel/common/common.go +++ b/translator/translate/otel/common/common.go @@ -414,3 +414,7 @@ func IsAnySet(conf *confmap.Conf, keys []string) bool { } return false } + +func KueueContainerInsightsEnabled(conf *confmap.Conf) bool { + return GetOrDefaultBool(conf, ConfigKey(LogsKey, MetricsCollectedKey, KubernetesKey, EnableKueueContainerInsights), false) +} diff --git a/translator/translate/otel/exporter/awsemf/kueue.go b/translator/translate/otel/exporter/awsemf/kueue.go index 610be4d22e..f9b6b4a7ec 100644 --- a/translator/translate/otel/exporter/awsemf/kueue.go +++ b/translator/translate/otel/exporter/awsemf/kueue.go @@ -4,10 +4,9 @@ package awsemf import ( + "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/common" "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awsemfexporter" "go.opentelemetry.io/collector/confmap" - - "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/receiver/awscontainerinsight" ) func setKubernetesKueueMetricDeclaration(conf *confmap.Conf, cfg *awsemfexporter.Config) error { @@ -17,7 +16,7 @@ func setKubernetesKueueMetricDeclaration(conf *confmap.Conf, cfg *awsemfexporter func getKueueMetricDeclarations(conf *confmap.Conf) []*awsemfexporter.MetricDeclaration { var metricDeclarations []*awsemfexporter.MetricDeclaration - if awscontainerinsight.KueueContainerInsightsEnabled(conf) { + if common.KueueContainerInsightsEnabled(conf) { metricDeclarations = []*awsemfexporter.MetricDeclaration{ { Dimensions: [][]string{ diff --git a/translator/translate/otel/pipeline/containerinsights/translator.go b/translator/translate/otel/pipeline/containerinsights/translator.go index ab307c59cc..a03bdb3f44 100644 --- a/translator/translate/otel/pipeline/containerinsights/translator.go +++ b/translator/translate/otel/pipeline/containerinsights/translator.go @@ -82,7 +82,7 @@ func (t *translator) Translate(conf *confmap.Conf) (*common.ComponentTranslators case kueuePipelineName: // add prometheus receiver for kueue receivers = common.NewTranslatorMap((awscontainerinsightskueue.NewTranslator())) - KueueContainerInsightsEnabled := KueueContainerInsightsEnabled(conf) + KueueContainerInsightsEnabled := common.KueueContainerInsightsEnabled(conf) if KueueContainerInsightsEnabled { processors.Set(kueue.NewTranslatorWithName(t.pipelineName)) } @@ -97,7 +97,3 @@ func (t *translator) Translate(conf *confmap.Conf) (*common.ComponentTranslators Extensions: extensions, }, nil } - -func KueueContainerInsightsEnabled(conf *confmap.Conf) bool { - return common.GetOrDefaultBool(conf, common.ConfigKey(common.LogsKey, common.MetricsCollectedKey, common.KubernetesKey, common.EnableKueueContainerInsights), false) -} diff --git a/translator/translate/otel/pipeline/containerinsights/translators.go b/translator/translate/otel/pipeline/containerinsights/translators.go index f5637812e1..ae986cc10e 100644 --- a/translator/translate/otel/pipeline/containerinsights/translators.go +++ b/translator/translate/otel/pipeline/containerinsights/translators.go @@ -20,7 +20,7 @@ func NewTranslators(conf *confmap.Conf) pipeline.TranslatorMap { ciTranslator := NewTranslatorWithName(ciPipelineName) translators.Set(ciTranslator) // create kueue container insights translator - KueueContainerInsightsEnabled := KueueContainerInsightsEnabled(conf) + KueueContainerInsightsEnabled := common.KueueContainerInsightsEnabled(conf) if KueueContainerInsightsEnabled { kueueTranslator := NewTranslatorWithName(kueuePipelineName) translators.Set(kueueTranslator) From db141b6ffd387687fcece9f3635cdf986fba6c2d Mon Sep 17 00:00:00 2001 From: Rahul Sahu Date: Thu, 7 Nov 2024 00:47:08 +0000 Subject: [PATCH 23/32] refactor: remove unneeded KueueContainerInsightsCall condition is guaranteed to be true in that block of the code --- .../translate/otel/pipeline/containerinsights/translator.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/translator/translate/otel/pipeline/containerinsights/translator.go b/translator/translate/otel/pipeline/containerinsights/translator.go index a03bdb3f44..e31bf42524 100644 --- a/translator/translate/otel/pipeline/containerinsights/translator.go +++ b/translator/translate/otel/pipeline/containerinsights/translator.go @@ -82,10 +82,7 @@ func (t *translator) Translate(conf *confmap.Conf) (*common.ComponentTranslators case kueuePipelineName: // add prometheus receiver for kueue receivers = common.NewTranslatorMap((awscontainerinsightskueue.NewTranslator())) - KueueContainerInsightsEnabled := common.KueueContainerInsightsEnabled(conf) - if KueueContainerInsightsEnabled { - processors.Set(kueue.NewTranslatorWithName(t.pipelineName)) - } + processors.Set(kueue.NewTranslatorWithName(t.pipelineName)) default: return nil, fmt.Errorf("unknown container insights pipeline name: %s", t.pipelineName) } From 85153086b553308888dbb66be61c78c181bcac2d Mon Sep 17 00:00:00 2001 From: Rahul Sahu Date: Thu, 7 Nov 2024 17:59:33 +0000 Subject: [PATCH 24/32] doc: lint fix to awsemf/kueue.go --- translator/translate/otel/exporter/awsemf/kueue.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/translator/translate/otel/exporter/awsemf/kueue.go b/translator/translate/otel/exporter/awsemf/kueue.go index f9b6b4a7ec..10dd880aca 100644 --- a/translator/translate/otel/exporter/awsemf/kueue.go +++ b/translator/translate/otel/exporter/awsemf/kueue.go @@ -4,9 +4,10 @@ package awsemf import ( - "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/common" "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awsemfexporter" "go.opentelemetry.io/collector/confmap" + + "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/common" ) func setKubernetesKueueMetricDeclaration(conf *confmap.Conf, cfg *awsemfexporter.Config) error { From dfd17ce16ea8948cf194bc4db2705217f2b33450 Mon Sep 17 00:00:00 2001 From: Rahul Sahu Date: Tue, 12 Nov 2024 18:38:21 +0000 Subject: [PATCH 25/32] build: fill placeholder for kueue receiver in go.mod --- go.mod | 58 ++++++++++++++--------------- go.sum | 114 +++++++++++++++++++++++++++++---------------------------- 2 files changed, 87 insertions(+), 85 deletions(-) diff --git a/go.mod b/go.mod index ac2e6856e5..bcb0170724 100644 --- a/go.mod +++ b/go.mod @@ -7,41 +7,41 @@ replace github.com/influxdata/telegraf => github.com/aws/telegraf v0.10.2-0.2024 // Replace with https://github.com/amazon-contributing/opentelemetry-collector-contrib, there are no requirements for all receivers/processors/exporters // to be all replaced since there are some changes that will always be from upstream replace ( - github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awscloudwatchlogsexporter => github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awscloudwatchlogsexporter v0.0.0-20241024160840-0c7076070f12 + github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awscloudwatchlogsexporter => github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awscloudwatchlogsexporter v0.0.0-20241112181539-bc8278558352 github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awsemfexporter => github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awsemfexporter v0.0.0-20241024185216-f0b80d5c67a5 - github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awsxrayexporter => github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awsxrayexporter v0.0.0-20241024160840-0c7076070f12 + github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awsxrayexporter => github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awsxrayexporter v0.0.0-20241112181539-bc8278558352 ) -replace github.com/open-telemetry/opentelemetry-collector-contrib/extension/awsproxy => github.com/amazon-contributing/opentelemetry-collector-contrib/extension/awsproxy v0.0.0-20241024160840-0c7076070f12 +replace github.com/open-telemetry/opentelemetry-collector-contrib/extension/awsproxy => github.com/amazon-contributing/opentelemetry-collector-contrib/extension/awsproxy v0.0.0-20241112181539-bc8278558352 replace ( - github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/awsutil => github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/awsutil v0.0.0-20241024160840-0c7076070f12 - github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/containerinsight => github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/containerinsight v0.0.0-20241024160840-0c7076070f12 - github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/cwlogs => github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/cwlogs v0.0.0-20241024160840-0c7076070f12 - github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/k8s => github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/k8s v0.0.0-20241024160840-0c7076070f12 - github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/proxy => github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/proxy v0.0.0-20241024160840-0c7076070f12 - github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/xray => github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/xray v0.0.0-20241024160840-0c7076070f12 - github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal => github.com/amazon-contributing/opentelemetry-collector-contrib/internal/coreinternal v0.0.0-20241024160840-0c7076070f12 - github.com/open-telemetry/opentelemetry-collector-contrib/internal/k8sconfig => github.com/amazon-contributing/opentelemetry-collector-contrib/internal/k8sconfig v0.0.0-20241024160840-0c7076070f12 - github.com/open-telemetry/opentelemetry-collector-contrib/internal/kubelet => github.com/amazon-contributing/opentelemetry-collector-contrib/internal/kubelet v0.0.0-20241024160840-0c7076070f12 + github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/awsutil => github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/awsutil v0.0.0-20241112181539-bc8278558352 + github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/containerinsight => github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/containerinsight v0.0.0-20241112181539-bc8278558352 + github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/cwlogs => github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/cwlogs v0.0.0-20241112181539-bc8278558352 + github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/k8s => github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/k8s v0.0.0-20241112181539-bc8278558352 + github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/proxy => github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/proxy v0.0.0-20241112181539-bc8278558352 + github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/xray => github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/xray v0.0.0-20241112181539-bc8278558352 + github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal => github.com/amazon-contributing/opentelemetry-collector-contrib/internal/coreinternal v0.0.0-20241112181539-bc8278558352 + github.com/open-telemetry/opentelemetry-collector-contrib/internal/k8sconfig => github.com/amazon-contributing/opentelemetry-collector-contrib/internal/k8sconfig v0.0.0-20241112181539-bc8278558352 + github.com/open-telemetry/opentelemetry-collector-contrib/internal/kubelet => github.com/amazon-contributing/opentelemetry-collector-contrib/internal/kubelet v0.0.0-20241112181539-bc8278558352 ) replace ( // For clear resource attributes after copy functionality https://github.com/amazon-contributing/opentelemetry-collector-contrib/pull/148 - github.com/open-telemetry/opentelemetry-collector-contrib/pkg/resourcetotelemetry => github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/resourcetotelemetry v0.0.0-20241024160840-0c7076070f12 - github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza => github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/stanza v0.0.0-20241024160840-0c7076070f12 + github.com/open-telemetry/opentelemetry-collector-contrib/pkg/resourcetotelemetry => github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/resourcetotelemetry v0.0.0-20241112181539-bc8278558352 + github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza => github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/stanza v0.0.0-20241112181539-bc8278558352 // Replace with contrib to revert upstream change https://github.com/open-telemetry/opentelemetry-collector-contrib/pull/20519 - github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/prometheus => github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/translator/prometheus v0.0.0-20241024160840-0c7076070f12 + github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/prometheus => github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/translator/prometheus v0.0.0-20241112181539-bc8278558352 ) -replace github.com/open-telemetry/opentelemetry-collector-contrib/processor/resourcedetectionprocessor => github.com/amazon-contributing/opentelemetry-collector-contrib/processor/resourcedetectionprocessor v0.0.0-20241024160840-0c7076070f12 +replace github.com/open-telemetry/opentelemetry-collector-contrib/processor/resourcedetectionprocessor => github.com/amazon-contributing/opentelemetry-collector-contrib/processor/resourcedetectionprocessor v0.0.0-20241112181539-bc8278558352 replace ( - github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awscontainerinsightskueuereceiver => ../opentelemetry-collector-contrib/receiver/awscontainerinsightskueuereceiver - github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver => github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver v0.0.0-20241024160840-0c7076070f12 - github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awsxrayreceiver => github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awsxrayreceiver v0.0.0-20241024160840-0c7076070f12 - github.com/open-telemetry/opentelemetry-collector-contrib/receiver/jmxreceiver => github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/jmxreceiver v0.0.0-20241024160840-0c7076070f12 - github.com/open-telemetry/opentelemetry-collector-contrib/receiver/prometheusreceiver => github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/prometheusreceiver v0.0.0-20241024160840-0c7076070f12 + github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver => github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver v0.0.0-20241112181539-bc8278558352 + github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awscontainerinsightskueuereceiver => github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awscontainerinsightskueuereceiver v0.0.0-20241112181539-bc8278558352 + github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awsxrayreceiver => github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awsxrayreceiver v0.0.0-20241112181539-bc8278558352 + github.com/open-telemetry/opentelemetry-collector-contrib/receiver/jmxreceiver => github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/jmxreceiver v0.0.0-20241112181539-bc8278558352 + github.com/open-telemetry/opentelemetry-collector-contrib/receiver/prometheusreceiver => github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/prometheusreceiver v0.0.0-20241112181539-bc8278558352 ) // Temporary fix, pending PR https://github.com/shirou/gopsutil/pull/957 @@ -92,7 +92,7 @@ replace github.com/aws/aws-sdk-go => github.com/aws/aws-sdk-go v1.48.6 require ( github.com/BurntSushi/toml v1.3.2 github.com/Jeffail/gabs v1.4.0 - github.com/amazon-contributing/opentelemetry-collector-contrib/extension/awsmiddleware v0.0.0-20241024160840-0c7076070f12 + github.com/amazon-contributing/opentelemetry-collector-contrib/extension/awsmiddleware v0.0.0-20241112181539-bc8278558352 github.com/aws/aws-sdk-go v1.53.11 github.com/aws/aws-sdk-go-v2/service/cloudwatch v1.30.2 github.com/bigkevmcd/go-configparser v0.0.0-20200217161103-d137835d2579 @@ -232,7 +232,7 @@ require ( github.com/alecthomas/participle v0.4.1 // indirect github.com/alecthomas/participle/v2 v2.1.1 // indirect github.com/alecthomas/units v0.0.0-20231202071711-9a357b53e9c9 // indirect - github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20241024160840-0c7076070f12 // indirect + github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20241112181539-bc8278558352 // indirect github.com/antchfx/jsonquery v1.1.5 // indirect github.com/antchfx/xmlquery v1.3.9 // indirect github.com/antchfx/xpath v1.2.0 // indirect @@ -270,7 +270,7 @@ require ( github.com/containerd/cgroups/v3 v3.0.3 // indirect github.com/containerd/console v1.0.3 // indirect github.com/containerd/errdefs v0.1.0 // indirect - github.com/containerd/ttrpc v1.2.2 // indirect + github.com/containerd/ttrpc v1.2.4 // indirect github.com/coreos/go-semver v0.3.0 // indirect github.com/coreos/go-systemd/v22 v22.5.0 // indirect github.com/cyphar/filepath-securejoin v0.2.4 // indirect @@ -278,7 +278,7 @@ require ( github.com/dennwc/varint v1.0.0 // indirect github.com/digitalocean/godo v1.109.0 // indirect github.com/distribution/reference v0.6.0 // indirect - github.com/docker/docker v25.0.6+incompatible // indirect + github.com/docker/docker v26.1.4+incompatible // indirect github.com/docker/go-connections v0.5.0 // indirect github.com/docker/go-units v0.5.0 // indirect github.com/doclambda/protobufquery v0.0.0-20210317203640-88ffabe06a60 // indirect @@ -316,7 +316,7 @@ require ( github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/golang/snappy v0.0.4 // indirect - github.com/google/cadvisor v0.49.0 // indirect + github.com/google/cadvisor v0.49.1-0.20240628164550-89f779d86055 // indirect github.com/google/gnostic-models v0.6.8 // indirect github.com/google/go-querystring v1.1.0 // indirect github.com/google/gofuzz v1.2.0 // indirect @@ -380,9 +380,9 @@ require ( github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible // indirect github.com/mitchellh/copystructure v1.2.0 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect - github.com/mitchellh/hashstructure/v2 v2.0.2 // indirect github.com/mitchellh/reflectwalk v1.0.2 // indirect - github.com/moby/sys/mountinfo v0.6.2 // indirect + github.com/moby/docker-image-spec v1.3.1 // indirect + github.com/moby/sys/mountinfo v0.7.1 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/mostynb/go-grpc-compression v1.2.3 // indirect @@ -419,7 +419,7 @@ require ( github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.0 // indirect github.com/opencontainers/runc v1.1.14 // indirect - github.com/opencontainers/runtime-spec v1.1.0 // indirect + github.com/opencontainers/runtime-spec v1.2.0 // indirect github.com/opencontainers/selinux v1.11.0 // indirect github.com/openshift/api v3.9.0+incompatible // indirect github.com/openshift/client-go v0.0.0-20210521082421-73d9475a9142 // indirect diff --git a/go.sum b/go.sum index bab30f7f73..345238075c 100644 --- a/go.sum +++ b/go.sum @@ -180,52 +180,54 @@ github.com/alecthomas/units v0.0.0-20231202071711-9a357b53e9c9 h1:ez/4by2iGztzR4 github.com/alecthomas/units v0.0.0-20231202071711-9a357b53e9c9/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= github.com/aliyun/alibaba-cloud-sdk-go v1.61.1483 h1:J8HaD+Zpfi1gcel3HCKpoHHEsrcuRrZlSnx7R9SCf5I= github.com/aliyun/alibaba-cloud-sdk-go v1.61.1483/go.mod h1:RcDobYh8k5VP6TNybz9m++gL3ijVI5wueVr0EM10VsU= -github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awscloudwatchlogsexporter v0.0.0-20241024160840-0c7076070f12 h1:+USPP+QlDqsDjsMG1RO2lc1UURGCXwejqC1Z/HAkVQ4= -github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awscloudwatchlogsexporter v0.0.0-20241024160840-0c7076070f12/go.mod h1:f9JwM/LQdKH8ZbYoH9TO35rmjM6WoTNlhWtYJ3YZucc= +github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awscloudwatchlogsexporter v0.0.0-20241112181539-bc8278558352 h1:L6zMEb+SbLm4ac8zG94ZLCeQtLvjmdua58ayIzaVpAE= +github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awscloudwatchlogsexporter v0.0.0-20241112181539-bc8278558352/go.mod h1:f9JwM/LQdKH8ZbYoH9TO35rmjM6WoTNlhWtYJ3YZucc= github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awsemfexporter v0.0.0-20241024185216-f0b80d5c67a5 h1:JOp9f4a1pP1zb5U+JPTezTdTZ1LCkcJSTcCd/AAqknM= github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awsemfexporter v0.0.0-20241024185216-f0b80d5c67a5/go.mod h1:LPWBVdTSNbZkk80v6aCUthS59cnR4VauVRdAIE3ifaY= -github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awsxrayexporter v0.0.0-20241024160840-0c7076070f12 h1:9E5WNd4oR52CDE74trrIA6/hlG+FRUoo3NdtdKsnprw= -github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awsxrayexporter v0.0.0-20241024160840-0c7076070f12/go.mod h1:GNeNylfr5KMt55XowzSdgbP7z8CkIAfIHtWSd+xxtws= -github.com/amazon-contributing/opentelemetry-collector-contrib/extension/awsmiddleware v0.0.0-20241024160840-0c7076070f12 h1:4T8EO8AhjBwFK3hcocbAqOZEK5YU/6hmDYsVLlDTBEo= -github.com/amazon-contributing/opentelemetry-collector-contrib/extension/awsmiddleware v0.0.0-20241024160840-0c7076070f12/go.mod h1:/RaNSxxO06niapGT00snMdgFfjjjW/kV3TZGX8kHuwM= -github.com/amazon-contributing/opentelemetry-collector-contrib/extension/awsproxy v0.0.0-20241024160840-0c7076070f12 h1:DOu5IxGFsk6GuWyeijlYDEifHL62dQAEPHol/tasFrE= -github.com/amazon-contributing/opentelemetry-collector-contrib/extension/awsproxy v0.0.0-20241024160840-0c7076070f12/go.mod h1:hRZt1DsvoLDIYBwjFvjwg/9IkaBXeCPG0QI57wbj98Q= -github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/awsutil v0.0.0-20241024160840-0c7076070f12 h1:5UfduJNrjayxXhOLhHGmLUTf47TnBeJpe6nv7DRW6ZY= -github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/awsutil v0.0.0-20241024160840-0c7076070f12/go.mod h1:YL1Y62PxJ7dem1ZBUqCfvbnePaGr5p7DTSyOXSCi6O4= -github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/containerinsight v0.0.0-20241024160840-0c7076070f12 h1:UvHsL0P1OAf2quIOFVk8A9NyGRuypN03b6+0Cu1Kb0M= -github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/containerinsight v0.0.0-20241024160840-0c7076070f12/go.mod h1:LT+qAyMutoADv2qezO+vkm/BkxR88qEfXdF2d13mV+E= -github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/cwlogs v0.0.0-20241024160840-0c7076070f12 h1:rnqocKDo3A82NkHIivOhLekhZXmLQP0S/M6kE7RAMjs= -github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/cwlogs v0.0.0-20241024160840-0c7076070f12/go.mod h1:SkscNdWANcuDJ7PkjS5wurSTAuY69nqP0I+cEVY9Ryw= -github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/k8s v0.0.0-20241024160840-0c7076070f12 h1:fd/eSG33sDKHduSwWHsLgbnkA89eXnAH49PI3/HNH1o= -github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/k8s v0.0.0-20241024160840-0c7076070f12/go.mod h1:/TOECDME2jYRPY21CrpTX2eMADJdkmBFBXc1lV/nRZA= -github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/proxy v0.0.0-20241024160840-0c7076070f12 h1:Tq6ET7zbGOlzID7XO1Ox3SN+xBpwyXA7lJ+9E7j2+nM= -github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/proxy v0.0.0-20241024160840-0c7076070f12/go.mod h1:J08A2gx8VFQfuoBiEfZ6uHIkMtVLd0OuRe5pP88b3I0= -github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/xray v0.0.0-20241024160840-0c7076070f12 h1:4n0Tg7jjGspPA76ZlI+cITJaDrDp9V79SgaZ2leLYNQ= -github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/xray v0.0.0-20241024160840-0c7076070f12/go.mod h1:+w647+1nLYvZWdk24gZWvdl/kFowbe2iDyISXLDYdmQ= -github.com/amazon-contributing/opentelemetry-collector-contrib/internal/coreinternal v0.0.0-20241024160840-0c7076070f12 h1:bNkLVrj3s8+P1/Xgth/4E4ffGci8jO29v/N5i3OnzvY= -github.com/amazon-contributing/opentelemetry-collector-contrib/internal/coreinternal v0.0.0-20241024160840-0c7076070f12/go.mod h1:WkLIVRD7DNz7zNXQ6CyTajsJFhsV52hgNovjr6fVm1g= -github.com/amazon-contributing/opentelemetry-collector-contrib/internal/k8sconfig v0.0.0-20241024160840-0c7076070f12 h1:7Ri8ritBlBB1hTnFoPE1Upuwqy/uQd8fHeiOUaBWZZw= -github.com/amazon-contributing/opentelemetry-collector-contrib/internal/k8sconfig v0.0.0-20241024160840-0c7076070f12/go.mod h1:VS66oUydCMwiWl1BFmLs7iNy4lGsfVYsriXr/d1fpAk= -github.com/amazon-contributing/opentelemetry-collector-contrib/internal/kubelet v0.0.0-20241024160840-0c7076070f12 h1:PHz4IjbJHsNEXO7266BPMs5nzYqhFQ5e0CNTcizqmPA= -github.com/amazon-contributing/opentelemetry-collector-contrib/internal/kubelet v0.0.0-20241024160840-0c7076070f12/go.mod h1:4qvmHiXPOkOXJdpmmxMqprb2BXxOGPgOG45BwLdipUM= -github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20241024160840-0c7076070f12 h1:bzbj2sMwlS7m2pf3pBL4ReBF5e/My8CXo8Uk03yUWBY= -github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20241024160840-0c7076070f12/go.mod h1:t/hYoRTnlPuRjh8y0BwVGgNvNIXpU2QJME5YVppUUHQ= -github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/resourcetotelemetry v0.0.0-20241024160840-0c7076070f12 h1:sIPPJ1tzrLbNQocY/X0scGuZEtbhbuxi6Tmhcw/6zgg= -github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/resourcetotelemetry v0.0.0-20241024160840-0c7076070f12/go.mod h1:Rr5b3hr6Jy9w/zTjsOl3vcyDDusc90P+iGdOd0UCYOo= -github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/stanza v0.0.0-20241024160840-0c7076070f12 h1:rmZ7B/y85fARC0BUXh/ZY/2usWtnpUKPrWwzGZi6EtQ= -github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/stanza v0.0.0-20241024160840-0c7076070f12/go.mod h1:2NSghK+mafMGxM8c4Gff8qcprdMD3YQebZtD9UAdB3E= -github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/translator/prometheus v0.0.0-20241024160840-0c7076070f12 h1:ShcpTZVJvFUt3PnNjkcrq065L6sy8IQaElnM6PHJpzU= -github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/translator/prometheus v0.0.0-20241024160840-0c7076070f12/go.mod h1:21nuEQl7YYeLkVrGGvxPXkljqjR40teBCG5trGZ5LxM= -github.com/amazon-contributing/opentelemetry-collector-contrib/processor/resourcedetectionprocessor v0.0.0-20241024160840-0c7076070f12 h1:gBq9hHmpfUjJIi4mpHrwoRrcpnH5czcBP//FWvf43RU= -github.com/amazon-contributing/opentelemetry-collector-contrib/processor/resourcedetectionprocessor v0.0.0-20241024160840-0c7076070f12/go.mod h1:TgRVPbdaFu8pWg4H5yCqVznsRVe8wQJRfEeUpduTKT8= -github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver v0.0.0-20241024160840-0c7076070f12 h1:BaDnRWWeZrdpRLXAo3DRRLcWoyKuC1U3uAnj8gs8PWE= -github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver v0.0.0-20241024160840-0c7076070f12/go.mod h1:tvoacZuBVf9lftcH2O6rKjumRCFAy+ycjKk3QJFaFXQ= -github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awsxrayreceiver v0.0.0-20241024160840-0c7076070f12 h1:Qx7pmI91E9i48x7EliCSH3q0ExL/hVzFBuoOEGR8R+Q= -github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awsxrayreceiver v0.0.0-20241024160840-0c7076070f12/go.mod h1:igQaQJt7eA/y3dZ2VLXVql+6k/ZXBgrAa2y9FrMMIKQ= -github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/jmxreceiver v0.0.0-20241024160840-0c7076070f12 h1:KNu8pcXJ5KlOA93JRXo7xenIo7/ACNpfa4RuulOvwRc= -github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/jmxreceiver v0.0.0-20241024160840-0c7076070f12/go.mod h1:NfNPpD1AYb1Z8C/0iNtViQUmI8Nzxnr3Q4RY9EreG5c= -github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/prometheusreceiver v0.0.0-20241024160840-0c7076070f12 h1:RMWINfQxfIPPSi3pdMMHcOGppCKD4uFygMc93znXiVo= -github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/prometheusreceiver v0.0.0-20241024160840-0c7076070f12/go.mod h1:FMPwht01yV4UvBAWkPFO/5jAkGFtfsmz1TRaoYgWeqU= +github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awsxrayexporter v0.0.0-20241112181539-bc8278558352 h1:dPHS08raR7CTTcOBvwpWZj6FVsQTg5JaliJj7eROQEk= +github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awsxrayexporter v0.0.0-20241112181539-bc8278558352/go.mod h1:GNeNylfr5KMt55XowzSdgbP7z8CkIAfIHtWSd+xxtws= +github.com/amazon-contributing/opentelemetry-collector-contrib/extension/awsmiddleware v0.0.0-20241112181539-bc8278558352 h1:hI3ivhujC1tziwnFVPZnH2CroqMRB2fMQPncoYS9Ib4= +github.com/amazon-contributing/opentelemetry-collector-contrib/extension/awsmiddleware v0.0.0-20241112181539-bc8278558352/go.mod h1:/RaNSxxO06niapGT00snMdgFfjjjW/kV3TZGX8kHuwM= +github.com/amazon-contributing/opentelemetry-collector-contrib/extension/awsproxy v0.0.0-20241112181539-bc8278558352 h1:gJ2z3/a7RQ5bZGVhPf9x16SsSqJT8h+BSWS2STTvpgA= +github.com/amazon-contributing/opentelemetry-collector-contrib/extension/awsproxy v0.0.0-20241112181539-bc8278558352/go.mod h1:hRZt1DsvoLDIYBwjFvjwg/9IkaBXeCPG0QI57wbj98Q= +github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/awsutil v0.0.0-20241112181539-bc8278558352 h1:UV6V2Kob3u1FHATO5CQWwvE3XE7jWUgc2MsSSm3OyKg= +github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/awsutil v0.0.0-20241112181539-bc8278558352/go.mod h1:YL1Y62PxJ7dem1ZBUqCfvbnePaGr5p7DTSyOXSCi6O4= +github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/containerinsight v0.0.0-20241112181539-bc8278558352 h1:zvSI5fXzdJUxF9J/uPQYcjuLpj5Baqx662Up1TX/eOY= +github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/containerinsight v0.0.0-20241112181539-bc8278558352/go.mod h1:LT+qAyMutoADv2qezO+vkm/BkxR88qEfXdF2d13mV+E= +github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/cwlogs v0.0.0-20241112181539-bc8278558352 h1:hiXgYOLSRMdjb8B3rp5j4Ltr+GdFPErkVKNm7c7NWnQ= +github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/cwlogs v0.0.0-20241112181539-bc8278558352/go.mod h1:SkscNdWANcuDJ7PkjS5wurSTAuY69nqP0I+cEVY9Ryw= +github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/k8s v0.0.0-20241112181539-bc8278558352 h1:LcoT/OfU0xKJ3J2hnRRIsn7riptwtxwmFFoxPpK7oGU= +github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/k8s v0.0.0-20241112181539-bc8278558352/go.mod h1:/TOECDME2jYRPY21CrpTX2eMADJdkmBFBXc1lV/nRZA= +github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/proxy v0.0.0-20241112181539-bc8278558352 h1:RbOMRfKDveIAJjyYhgFfg18EuFteJSI9GZhmvN028xQ= +github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/proxy v0.0.0-20241112181539-bc8278558352/go.mod h1:J08A2gx8VFQfuoBiEfZ6uHIkMtVLd0OuRe5pP88b3I0= +github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/xray v0.0.0-20241112181539-bc8278558352 h1:qId35MkW3ULtkiSNrX2O2ZsZquGUyDZqsHRwWaIjS0Q= +github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/xray v0.0.0-20241112181539-bc8278558352/go.mod h1:+w647+1nLYvZWdk24gZWvdl/kFowbe2iDyISXLDYdmQ= +github.com/amazon-contributing/opentelemetry-collector-contrib/internal/coreinternal v0.0.0-20241112181539-bc8278558352 h1:Ox8Nd5N/AnqvSWjVGQqmRpEg19keiZ8J+/wc1HpyMp4= +github.com/amazon-contributing/opentelemetry-collector-contrib/internal/coreinternal v0.0.0-20241112181539-bc8278558352/go.mod h1:Ai4BsM7C05bEQYO2O272S1LmsyIhO5r0iLmXF5NN5so= +github.com/amazon-contributing/opentelemetry-collector-contrib/internal/k8sconfig v0.0.0-20241112181539-bc8278558352 h1:JGlVGaPJVmdDn+RXqSVikzHnfL4er9IdSZs4UxBgOWI= +github.com/amazon-contributing/opentelemetry-collector-contrib/internal/k8sconfig v0.0.0-20241112181539-bc8278558352/go.mod h1:VS66oUydCMwiWl1BFmLs7iNy4lGsfVYsriXr/d1fpAk= +github.com/amazon-contributing/opentelemetry-collector-contrib/internal/kubelet v0.0.0-20241112181539-bc8278558352 h1:TL7D/0nD6lme6grzw6iY8Gz9O0p9QZMzoaTuE3xl/m8= +github.com/amazon-contributing/opentelemetry-collector-contrib/internal/kubelet v0.0.0-20241112181539-bc8278558352/go.mod h1:4qvmHiXPOkOXJdpmmxMqprb2BXxOGPgOG45BwLdipUM= +github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20241112181539-bc8278558352 h1:e+flG75ArSgcGygvbol5MnSHji+qG4NxeVvK3Cwm/KM= +github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20241112181539-bc8278558352/go.mod h1:t/hYoRTnlPuRjh8y0BwVGgNvNIXpU2QJME5YVppUUHQ= +github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/resourcetotelemetry v0.0.0-20241112181539-bc8278558352 h1:Pkkq9i9d38HZp8s54NEl+gNZzcRNsQ//rt8i8q4TuBU= +github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/resourcetotelemetry v0.0.0-20241112181539-bc8278558352/go.mod h1:Rr5b3hr6Jy9w/zTjsOl3vcyDDusc90P+iGdOd0UCYOo= +github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/stanza v0.0.0-20241112181539-bc8278558352 h1:ufBV8GDkrdxIRhAfa8ZeY9sDdgkprHnOJVVciuXzdbg= +github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/stanza v0.0.0-20241112181539-bc8278558352/go.mod h1:2NSghK+mafMGxM8c4Gff8qcprdMD3YQebZtD9UAdB3E= +github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/translator/prometheus v0.0.0-20241112181539-bc8278558352 h1:IX7BX1mROOXnomalw5r0VCAqlRPh8n+zPPioJCkXNoQ= +github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/translator/prometheus v0.0.0-20241112181539-bc8278558352/go.mod h1:21nuEQl7YYeLkVrGGvxPXkljqjR40teBCG5trGZ5LxM= +github.com/amazon-contributing/opentelemetry-collector-contrib/processor/resourcedetectionprocessor v0.0.0-20241112181539-bc8278558352 h1:O1LjHvuQ6oA9E9shKNb1h0rc7LW8lbh0Yk2Ly7W2CiU= +github.com/amazon-contributing/opentelemetry-collector-contrib/processor/resourcedetectionprocessor v0.0.0-20241112181539-bc8278558352/go.mod h1:uzpU7Y6+oL6RdOv8IWi6fjT8LNV6FYX6CN6NATLJOiQ= +github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver v0.0.0-20241112181539-bc8278558352 h1:uE9s7HKNZAsT2nrqeZTDr2GFh/X819PeubgWtOVit7U= +github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver v0.0.0-20241112181539-bc8278558352/go.mod h1:StgsMi0cNUydO2N/7WbLYPUBXzvp9wIMcWp9P8x/Vck= +github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awscontainerinsightskueuereceiver v0.0.0-20241112181539-bc8278558352 h1:WqkB8yOj8YCSxWzVUVCeVx263UWnmBUpMquetzwmntQ= +github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awscontainerinsightskueuereceiver v0.0.0-20241112181539-bc8278558352/go.mod h1://9Xy+KG8K9KvujBh6sZXIYPDvbu8xsiU1l8StFHjMA= +github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awsxrayreceiver v0.0.0-20241112181539-bc8278558352 h1:I+2D1klGQqSSWLNVN8cd9cwTjtLphTDvsxG55Q0dVgA= +github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awsxrayreceiver v0.0.0-20241112181539-bc8278558352/go.mod h1:igQaQJt7eA/y3dZ2VLXVql+6k/ZXBgrAa2y9FrMMIKQ= +github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/jmxreceiver v0.0.0-20241112181539-bc8278558352 h1:qXpJ3DMe0h0H0tVo1i9w/V3RP1f24MgVnDCm2N/Hm6M= +github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/jmxreceiver v0.0.0-20241112181539-bc8278558352/go.mod h1:hRUrYatVP/GFNxHn2yW1gJcnPyGtdlTXyebpzzzjZeU= +github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/prometheusreceiver v0.0.0-20241112181539-bc8278558352 h1:X6PoMvfAifaZh7qaQsmhC/THtzLFMn0oGVcsOnTuw04= +github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/prometheusreceiver v0.0.0-20241112181539-bc8278558352/go.mod h1:TTwtk1IMiqmyGm66w9aIq/TInicOm2Y6DwcJzRIpv1U= github.com/amir/raidman v0.0.0-20170415203553-1ccc43bfb9c9 h1:FXrPTd8Rdlc94dKccl7KPmdmIbVh/OjelJ8/vgMRzcQ= github.com/amir/raidman v0.0.0-20170415203553-1ccc43bfb9c9/go.mod h1:eliMa/PW+RDr2QLWRmLH1R1ZA4RInpmvOzDDXtaIZkc= github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs= @@ -400,8 +402,8 @@ github.com/containerd/errdefs v0.1.0 h1:m0wCRBiu1WJT/Fr+iOoQHMQS/eP5myQ8lCv4Dz5Z github.com/containerd/errdefs v0.1.0/go.mod h1:YgWiiHtLmSeBrvpw+UfPijzbLaB77mEG1WwJTDETIV0= github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= -github.com/containerd/ttrpc v1.2.2 h1:9vqZr0pxwOF5koz6N0N3kJ0zDHokrcPxIR/ZR2YFtOs= -github.com/containerd/ttrpc v1.2.2/go.mod h1:sIT6l32Ph/H9cvnJsfXM5drIVzTr5A2flTf1G5tYZak= +github.com/containerd/ttrpc v1.2.4 h1:eQCQK4h9dxDmpOb9QOOMh2NHTfzroH1IkmHiKZi05Oo= +github.com/containerd/ttrpc v1.2.4/go.mod h1:ojvb8SJBSch0XkqNO0L0YX/5NxR3UnVk2LzFKBK0upc= github.com/containerd/typeurl v1.0.2 h1:Chlt8zIieDbzQFzXzAeBEF92KhExuE4p9p92/QmY7aY= github.com/containerd/typeurl v1.0.2/go.mod h1:9trJWW2sRlGub4wZJRTW83VtbOLS6hwcDZXTn6oPz9s= github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= @@ -445,8 +447,8 @@ github.com/djherbis/times v1.5.0 h1:79myA211VwPhFTqUk8xehWrsEO+zcIZj0zT8mXPVARU= github.com/djherbis/times v1.5.0/go.mod h1:5q7FDLvbNg1L/KaBmPcWlVR9NmoKo3+ucqUA3ijQhA0= github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI= github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ= -github.com/docker/docker v25.0.6+incompatible h1:5cPwbwriIcsua2REJe8HqQV+6WlWc1byg2QSXzBxBGg= -github.com/docker/docker v25.0.6+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v26.1.4+incompatible h1:vuTpXDuoga+Z38m1OZHzl7NKisKWaWlhjQk7IDPSLsU= +github.com/docker/docker v26.1.4+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c= github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= @@ -669,8 +671,8 @@ github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Z github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.1 h1:gK4Kx5IaGY9CD5sPJ36FHiBJ6ZXl0kilRiiCj+jdYp4= github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= -github.com/google/cadvisor v0.49.0 h1:1PYeiORXmcFYi609M4Qvq5IzcvcVaWgYxDt78uH8jYA= -github.com/google/cadvisor v0.49.0/go.mod h1:s6Fqwb2KiWG6leCegVhw4KW40tf9f7m+SF1aXiE8Wsk= +github.com/google/cadvisor v0.49.1-0.20240628164550-89f779d86055 h1:LH//TOiNPpCn3a48Flz5sNUmuMW5V/sTKheuXAPowUA= +github.com/google/cadvisor v0.49.1-0.20240628164550-89f779d86055/go.mod h1:koGHrZgL6RdZ29g/iXWT4kUIawssUi88XCYDlyPt5RI= github.com/google/flatbuffers v23.5.26+incompatible h1:M9dgRyhJemaM4Sw8+66GHBu8ioaQmyPLg1b8VwK5WJg= github.com/google/flatbuffers v23.5.26+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= @@ -1059,8 +1061,6 @@ github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eI github.com/mitchellh/go-testing-interface v1.14.1 h1:jrgshOhYAUVNMAJiKbEu7EqAwgJJ2JqpQmpLJOu07cU= github.com/mitchellh/go-testing-interface v1.14.1/go.mod h1:gfgS7OtZj6MA4U1UrDRp04twqAjfvlZyCfX3sDjEym8= github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo= -github.com/mitchellh/hashstructure/v2 v2.0.2 h1:vGKWl0YJqUNxE8d+h8f6NJLcCJrgbhC4NcD46KavDd4= -github.com/mitchellh/hashstructure/v2 v2.0.2/go.mod h1:MG3aRVU/N29oo/V/IhBX8GR/zz4kQkprJgF2EVszyDE= github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.4.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= @@ -1070,13 +1070,15 @@ github.com/mitchellh/mapstructure v1.5.1-0.20231216201459-8508981c8b6c/go.mod h1 github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= +github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= +github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= github.com/moby/ipvs v1.0.1 h1:aoZ7fhLTXgDbzVrAnvV+XbKOU8kOET7B3+xULDF/1o0= github.com/moby/ipvs v1.0.1/go.mod h1:2pngiyseZbIKXNv7hsKj3O9UEz30c53MT9005gt2hxQ= github.com/moby/patternmatcher v0.6.0 h1:GmP9lR19aU5GqSSFko+5pRqHi+Ohk1O69aFiKkVGiPk= github.com/moby/patternmatcher v0.6.0/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc= github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= -github.com/moby/sys/mountinfo v0.6.2 h1:BzJjoreD5BMFNmD9Rus6gdd1pLuecOFPt8wC+Vygl78= -github.com/moby/sys/mountinfo v0.6.2/go.mod h1:IJb6JQeOklcdMU9F5xQ8ZALD+CUr5VlGpwtX+VE0rpI= +github.com/moby/sys/mountinfo v0.7.1 h1:/tTvQaSJRr2FshkhXiIpux6fQ2Zvc4j7tAhMTStAG2g= +github.com/moby/sys/mountinfo v0.7.1/go.mod h1:IJb6JQeOklcdMU9F5xQ8ZALD+CUr5VlGpwtX+VE0rpI= github.com/moby/sys/sequential v0.5.0 h1:OPvI35Lzn9K04PBbCLW0g4LcFAJgHsvXsRyewg5lXtc= github.com/moby/sys/sequential v0.5.0/go.mod h1:tH2cOOs5V9MlPiXcQzRC+eEyab644PWKGRYaaV5ZZlo= github.com/moby/sys/user v0.3.0 h1:9ni5DlcW5an3SvRSx4MouotOygvzaXbaSrc/wGDFWPo= @@ -1261,8 +1263,8 @@ github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQ github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM= github.com/opencontainers/runc v1.1.14 h1:rgSuzbmgz5DUJjeSnw337TxDbRuqjs6iqQck/2weR6w= github.com/opencontainers/runc v1.1.14/go.mod h1:E4C2z+7BxR7GHXp0hAY53mek+x49X1LjPNeMTfRGvOA= -github.com/opencontainers/runtime-spec v1.1.0 h1:HHUyrt9mwHUjtasSbXSMvs4cyFxh+Bll4AjJ9odEGpg= -github.com/opencontainers/runtime-spec v1.1.0/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= +github.com/opencontainers/runtime-spec v1.2.0 h1:z97+pHb3uELt/yiAWD691HNHQIF07bE7dzrbT927iTk= +github.com/opencontainers/runtime-spec v1.2.0/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/opencontainers/selinux v1.11.0 h1:+5Zbo97w3Lbmb3PeqQtpmTkMwsW5nRI3YaLpt7tQ7oU= github.com/opencontainers/selinux v1.11.0/go.mod h1:E5dMC3VPuVvVHDYmi78qvhJp8+M586T4DlDRYpFkyec= github.com/openshift/api v0.0.0-20180801171038-322a19404e37 h1:05irGU4HK4IauGGDbsk+ZHrm1wOzMLYjMlfaiqMrBYc= From 2e1e1f3be8522ab48bdeb2e1bde32805ac800afb Mon Sep 17 00:00:00 2001 From: Rahul Sahu Date: Tue, 12 Nov 2024 19:14:26 +0000 Subject: [PATCH 26/32] fix: merge conflict for AWS EMF exporter translator --- translator/translate/otel/exporter/awsemf/translator.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/translator/translate/otel/exporter/awsemf/translator.go b/translator/translate/otel/exporter/awsemf/translator.go index 73b370fb87..21e28bc12c 100644 --- a/translator/translate/otel/exporter/awsemf/translator.go +++ b/translator/translate/otel/exporter/awsemf/translator.go @@ -231,11 +231,9 @@ func setKubernetesFields(conf *confmap.Conf, cfg *awsemfexporter.Config) error { return nil } -<<<<<<< HEAD func setCiJmxFields() error { return nil } -======= func setKubernetesKueueFields(conf *confmap.Conf, cfg *awsemfexporter.Config) error { setDisableMetricExtraction(kubernetesKueueBasePathKey, conf, cfg) @@ -246,7 +244,6 @@ func setKubernetesKueueFields(conf *confmap.Conf, cfg *awsemfexporter.Config) er return nil } ->>>>>>> main-kueue func setPrometheusFields(conf *confmap.Conf, cfg *awsemfexporter.Config) error { setDisableMetricExtraction(prometheusBasePathKey, conf, cfg) From 0d24e7d97c20dc97b652b425c2bbf79f97b13eb0 Mon Sep 17 00:00:00 2001 From: Rahul Sahu Date: Thu, 14 Nov 2024 19:57:09 +0000 Subject: [PATCH 27/32] doc: move kueue receiver dependency to require block with other receivers --- go.mod | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 2288cfa31a..ae94ce54c7 100644 --- a/go.mod +++ b/go.mod @@ -144,6 +144,7 @@ require ( github.com/open-telemetry/opentelemetry-collector-contrib/processor/tailsamplingprocessor v0.103.0 github.com/open-telemetry/opentelemetry-collector-contrib/processor/transformprocessor v0.103.0 github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver v0.103.0 + github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awscontainerinsightskueuereceiver v0.0.0-20241112181539-bc8278558352 github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awsecscontainermetricsreceiver v0.103.0 github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awsxrayreceiver v0.103.0 github.com/open-telemetry/opentelemetry-collector-contrib/receiver/filelogreceiver v0.103.0 @@ -209,8 +210,6 @@ require ( k8s.io/klog/v2 v2.120.1 ) -require github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awscontainerinsightskueuereceiver v0.0.0-20241112181539-bc8278558352 - require ( cloud.google.com/go v0.112.1 // indirect cloud.google.com/go/compute/metadata v0.3.0 // indirect From 6ab1cea39caa2f340990a7e54a1f76f8e937b7e5 Mon Sep 17 00:00:00 2001 From: Rahul Sahu Date: Thu, 14 Nov 2024 20:46:35 +0000 Subject: [PATCH 28/32] build: update versions for otel dependencies --- go.mod | 48 +++++++++++++++--------------- go.sum | 92 +++++++++++++++++++++++++++++----------------------------- 2 files changed, 70 insertions(+), 70 deletions(-) diff --git a/go.mod b/go.mod index ae94ce54c7..df5112a84b 100644 --- a/go.mod +++ b/go.mod @@ -7,41 +7,41 @@ replace github.com/influxdata/telegraf => github.com/aws/telegraf v0.10.2-0.2024 // Replace with https://github.com/amazon-contributing/opentelemetry-collector-contrib, there are no requirements for all receivers/processors/exporters // to be all replaced since there are some changes that will always be from upstream replace ( - github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awscloudwatchlogsexporter => github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awscloudwatchlogsexporter v0.0.0-20241104203805-20919412150d + github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awscloudwatchlogsexporter => github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awscloudwatchlogsexporter v0.0.0-20241114192544-cb251fe99cbb github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awsemfexporter => github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awsemfexporter v0.0.0-20241024185216-f0b80d5c67a5 - github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awsxrayexporter => github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awsxrayexporter v0.0.0-20241104203805-20919412150d + github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awsxrayexporter => github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awsxrayexporter v0.0.0-20241114192544-cb251fe99cbb ) -replace github.com/open-telemetry/opentelemetry-collector-contrib/extension/awsproxy => github.com/amazon-contributing/opentelemetry-collector-contrib/extension/awsproxy v0.0.0-20241104203805-20919412150d +replace github.com/open-telemetry/opentelemetry-collector-contrib/extension/awsproxy => github.com/amazon-contributing/opentelemetry-collector-contrib/extension/awsproxy v0.0.0-20241114192544-cb251fe99cbb replace ( - github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/awsutil => github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/awsutil v0.0.0-20241104203805-20919412150d - github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/containerinsight => github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/containerinsight v0.0.0-20241104203805-20919412150d - github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/cwlogs => github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/cwlogs v0.0.0-20241104203805-20919412150d - github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/k8s => github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/k8s v0.0.0-20241104203805-20919412150d - github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/proxy => github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/proxy v0.0.0-20241104203805-20919412150d - github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/xray => github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/xray v0.0.0-20241104203805-20919412150d - github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal => github.com/amazon-contributing/opentelemetry-collector-contrib/internal/coreinternal v0.0.0-20241104203805-20919412150d - github.com/open-telemetry/opentelemetry-collector-contrib/internal/k8sconfig => github.com/amazon-contributing/opentelemetry-collector-contrib/internal/k8sconfig v0.0.0-20241104203805-20919412150d - github.com/open-telemetry/opentelemetry-collector-contrib/internal/kubelet => github.com/amazon-contributing/opentelemetry-collector-contrib/internal/kubelet v0.0.0-20241104203805-20919412150d + github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/awsutil => github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/awsutil v0.0.0-20241114192544-cb251fe99cbb + github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/containerinsight => github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/containerinsight v0.0.0-20241114192544-cb251fe99cbb + github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/cwlogs => github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/cwlogs v0.0.0-20241114192544-cb251fe99cbb + github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/k8s => github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/k8s v0.0.0-20241114192544-cb251fe99cbb + github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/proxy => github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/proxy v0.0.0-20241114192544-cb251fe99cbb + github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/xray => github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/xray v0.0.0-20241114192544-cb251fe99cbb + github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal => github.com/amazon-contributing/opentelemetry-collector-contrib/internal/coreinternal v0.0.0-20241114192544-cb251fe99cbb + github.com/open-telemetry/opentelemetry-collector-contrib/internal/k8sconfig => github.com/amazon-contributing/opentelemetry-collector-contrib/internal/k8sconfig v0.0.0-20241114192544-cb251fe99cbb + github.com/open-telemetry/opentelemetry-collector-contrib/internal/kubelet => github.com/amazon-contributing/opentelemetry-collector-contrib/internal/kubelet v0.0.0-20241114192544-cb251fe99cbb ) replace ( // For clear resource attributes after copy functionality https://github.com/amazon-contributing/opentelemetry-collector-contrib/pull/148 - github.com/open-telemetry/opentelemetry-collector-contrib/pkg/resourcetotelemetry => github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/resourcetotelemetry v0.0.0-20241104203805-20919412150d - github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza => github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/stanza v0.0.0-20241104203805-20919412150d + github.com/open-telemetry/opentelemetry-collector-contrib/pkg/resourcetotelemetry => github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/resourcetotelemetry v0.0.0-20241114192544-cb251fe99cbb + github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza => github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/stanza v0.0.0-20241114192544-cb251fe99cbb // Replace with contrib to revert upstream change https://github.com/open-telemetry/opentelemetry-collector-contrib/pull/20519 - github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/prometheus => github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/translator/prometheus v0.0.0-20241104203805-20919412150d + github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/prometheus => github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/translator/prometheus v0.0.0-20241114192544-cb251fe99cbb ) -replace github.com/open-telemetry/opentelemetry-collector-contrib/processor/resourcedetectionprocessor => github.com/amazon-contributing/opentelemetry-collector-contrib/processor/resourcedetectionprocessor v0.0.0-20241104203805-20919412150d +replace github.com/open-telemetry/opentelemetry-collector-contrib/processor/resourcedetectionprocessor => github.com/amazon-contributing/opentelemetry-collector-contrib/processor/resourcedetectionprocessor v0.0.0-20241114192544-cb251fe99cbb replace ( - github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver => github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver v0.0.0-20241104203805-20919412150d - github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awscontainerinsightskueuereceiver => github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awscontainerinsightskueuereceiver v0.0.0-20241112181539-bc8278558352 - github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awsxrayreceiver => github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awsxrayreceiver v0.0.0-20241104203805-20919412150d - github.com/open-telemetry/opentelemetry-collector-contrib/receiver/jmxreceiver => github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/jmxreceiver v0.0.0-20241104203805-20919412150d - github.com/open-telemetry/opentelemetry-collector-contrib/receiver/prometheusreceiver => github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/prometheusreceiver v0.0.0-20241104203805-20919412150d + github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver => github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver v0.0.0-20241114192544-cb251fe99cbb + github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awscontainerinsightskueuereceiver => github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awscontainerinsightskueuereceiver v0.0.0-20241114192544-cb251fe99cbb + github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awsxrayreceiver => github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awsxrayreceiver v0.0.0-20241114192544-cb251fe99cbb + github.com/open-telemetry/opentelemetry-collector-contrib/receiver/jmxreceiver => github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/jmxreceiver v0.0.0-20241114192544-cb251fe99cbb + github.com/open-telemetry/opentelemetry-collector-contrib/receiver/prometheusreceiver => github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/prometheusreceiver v0.0.0-20241114192544-cb251fe99cbb ) // Temporary fix, pending PR https://github.com/shirou/gopsutil/pull/957 @@ -92,7 +92,7 @@ replace github.com/aws/aws-sdk-go => github.com/aws/aws-sdk-go v1.48.6 require ( github.com/BurntSushi/toml v1.3.2 github.com/Jeffail/gabs v1.4.0 - github.com/amazon-contributing/opentelemetry-collector-contrib/extension/awsmiddleware v0.0.0-20241104203805-20919412150d + github.com/amazon-contributing/opentelemetry-collector-contrib/extension/awsmiddleware v0.0.0-20241114192544-cb251fe99cbb github.com/aws/aws-sdk-go v1.53.11 github.com/aws/aws-sdk-go-v2/service/cloudwatch v1.30.2 github.com/bigkevmcd/go-configparser v0.0.0-20200217161103-d137835d2579 @@ -144,7 +144,7 @@ require ( github.com/open-telemetry/opentelemetry-collector-contrib/processor/tailsamplingprocessor v0.103.0 github.com/open-telemetry/opentelemetry-collector-contrib/processor/transformprocessor v0.103.0 github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver v0.103.0 - github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awscontainerinsightskueuereceiver v0.0.0-20241112181539-bc8278558352 + github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awscontainerinsightskueuereceiver v0.0.0-20241114192544-cb251fe99cbb github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awsecscontainermetricsreceiver v0.103.0 github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awsxrayreceiver v0.103.0 github.com/open-telemetry/opentelemetry-collector-contrib/receiver/filelogreceiver v0.103.0 @@ -232,7 +232,7 @@ require ( github.com/alecthomas/participle v0.4.1 // indirect github.com/alecthomas/participle/v2 v2.1.1 // indirect github.com/alecthomas/units v0.0.0-20231202071711-9a357b53e9c9 // indirect - github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20241104203805-20919412150d // indirect + github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20241114192544-cb251fe99cbb // indirect github.com/antchfx/jsonquery v1.1.5 // indirect github.com/antchfx/xmlquery v1.3.9 // indirect github.com/antchfx/xpath v1.2.0 // indirect diff --git a/go.sum b/go.sum index ee97d7b4a6..274c2df246 100644 --- a/go.sum +++ b/go.sum @@ -180,54 +180,54 @@ github.com/alecthomas/units v0.0.0-20231202071711-9a357b53e9c9 h1:ez/4by2iGztzR4 github.com/alecthomas/units v0.0.0-20231202071711-9a357b53e9c9/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= github.com/aliyun/alibaba-cloud-sdk-go v1.61.1483 h1:J8HaD+Zpfi1gcel3HCKpoHHEsrcuRrZlSnx7R9SCf5I= github.com/aliyun/alibaba-cloud-sdk-go v1.61.1483/go.mod h1:RcDobYh8k5VP6TNybz9m++gL3ijVI5wueVr0EM10VsU= -github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awscloudwatchlogsexporter v0.0.0-20241104203805-20919412150d h1:1nlENK4aucOEP+jSMFTgMfJaL8Rl/h6q2LXKiS8kFkM= -github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awscloudwatchlogsexporter v0.0.0-20241104203805-20919412150d/go.mod h1:f9JwM/LQdKH8ZbYoH9TO35rmjM6WoTNlhWtYJ3YZucc= +github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awscloudwatchlogsexporter v0.0.0-20241114192544-cb251fe99cbb h1:UePw4/QUtalit5jG7K301PPxtq/tRgYeXiPLaBgA71I= +github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awscloudwatchlogsexporter v0.0.0-20241114192544-cb251fe99cbb/go.mod h1:f9JwM/LQdKH8ZbYoH9TO35rmjM6WoTNlhWtYJ3YZucc= github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awsemfexporter v0.0.0-20241024185216-f0b80d5c67a5 h1:JOp9f4a1pP1zb5U+JPTezTdTZ1LCkcJSTcCd/AAqknM= github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awsemfexporter v0.0.0-20241024185216-f0b80d5c67a5/go.mod h1:LPWBVdTSNbZkk80v6aCUthS59cnR4VauVRdAIE3ifaY= -github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awsxrayexporter v0.0.0-20241104203805-20919412150d h1:OaeHJsqrLqlgsIG24uYZvDkZ41nFr9LRhKPtX8f4O+Q= -github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awsxrayexporter v0.0.0-20241104203805-20919412150d/go.mod h1:GNeNylfr5KMt55XowzSdgbP7z8CkIAfIHtWSd+xxtws= -github.com/amazon-contributing/opentelemetry-collector-contrib/extension/awsmiddleware v0.0.0-20241104203805-20919412150d h1:eXEVMyh0ESwB4HGuPmDH4DDTlEq8kSJ4o+Hr0tdy+RI= -github.com/amazon-contributing/opentelemetry-collector-contrib/extension/awsmiddleware v0.0.0-20241104203805-20919412150d/go.mod h1:/RaNSxxO06niapGT00snMdgFfjjjW/kV3TZGX8kHuwM= -github.com/amazon-contributing/opentelemetry-collector-contrib/extension/awsproxy v0.0.0-20241104203805-20919412150d h1:f13PqXJBa78lokYPhoL1b96csNXfroC8nlK7R7zpN7U= -github.com/amazon-contributing/opentelemetry-collector-contrib/extension/awsproxy v0.0.0-20241104203805-20919412150d/go.mod h1:hRZt1DsvoLDIYBwjFvjwg/9IkaBXeCPG0QI57wbj98Q= -github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/awsutil v0.0.0-20241104203805-20919412150d h1:Oz6LLJwcpPD5IYHBzsb0mWZUt0OOgUpDth35qCAF+1Q= -github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/awsutil v0.0.0-20241104203805-20919412150d/go.mod h1:YL1Y62PxJ7dem1ZBUqCfvbnePaGr5p7DTSyOXSCi6O4= -github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/containerinsight v0.0.0-20241104203805-20919412150d h1:SDkwRzHzXO7HfmhCCvamdQmPH2fKK7PYMXZKhbrpHCk= -github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/containerinsight v0.0.0-20241104203805-20919412150d/go.mod h1:LT+qAyMutoADv2qezO+vkm/BkxR88qEfXdF2d13mV+E= -github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/cwlogs v0.0.0-20241104203805-20919412150d h1:1iitRjrfd7JjmfJbe1iM7XENgzjJ6L2THTKtv6V/qmU= -github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/cwlogs v0.0.0-20241104203805-20919412150d/go.mod h1:SkscNdWANcuDJ7PkjS5wurSTAuY69nqP0I+cEVY9Ryw= -github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/k8s v0.0.0-20241104203805-20919412150d h1:JEGyD9HBLCRaw7wWq1H+1oarZ0Aui3a/27V1t0kcnXg= -github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/k8s v0.0.0-20241104203805-20919412150d/go.mod h1:/TOECDME2jYRPY21CrpTX2eMADJdkmBFBXc1lV/nRZA= -github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/proxy v0.0.0-20241104203805-20919412150d h1:/ogt2A7O9rOZRCgb0kK4ps0T1b9gCnIiF6XfgaOLuVo= -github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/proxy v0.0.0-20241104203805-20919412150d/go.mod h1:J08A2gx8VFQfuoBiEfZ6uHIkMtVLd0OuRe5pP88b3I0= -github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/xray v0.0.0-20241104203805-20919412150d h1:kSwmEzswg2jdB/WpHhj54u4XL2gLRlwC3dBxO4u9WPY= -github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/xray v0.0.0-20241104203805-20919412150d/go.mod h1:+w647+1nLYvZWdk24gZWvdl/kFowbe2iDyISXLDYdmQ= -github.com/amazon-contributing/opentelemetry-collector-contrib/internal/coreinternal v0.0.0-20241104203805-20919412150d h1:1thAqvybJZ8Uvelq1rwUKbvdHG3+tmHXYru2j3PkGaQ= -github.com/amazon-contributing/opentelemetry-collector-contrib/internal/coreinternal v0.0.0-20241104203805-20919412150d/go.mod h1:Ai4BsM7C05bEQYO2O272S1LmsyIhO5r0iLmXF5NN5so= -github.com/amazon-contributing/opentelemetry-collector-contrib/internal/k8sconfig v0.0.0-20241104203805-20919412150d h1:Gdw6doVzDuXBheQS2pdA2b8nEEyUCBIadcHRwPceDB4= -github.com/amazon-contributing/opentelemetry-collector-contrib/internal/k8sconfig v0.0.0-20241104203805-20919412150d/go.mod h1:VS66oUydCMwiWl1BFmLs7iNy4lGsfVYsriXr/d1fpAk= -github.com/amazon-contributing/opentelemetry-collector-contrib/internal/kubelet v0.0.0-20241104203805-20919412150d h1:IeExpZfRsSLKsRW7ee7IriYHW2k+to26bQ7NxTvYygw= -github.com/amazon-contributing/opentelemetry-collector-contrib/internal/kubelet v0.0.0-20241104203805-20919412150d/go.mod h1:4qvmHiXPOkOXJdpmmxMqprb2BXxOGPgOG45BwLdipUM= -github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20241104203805-20919412150d h1:JM0ZVUKL4vNXQcGOpB3PALHrpe9Uq7YrD3zyOdDXleQ= -github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20241104203805-20919412150d/go.mod h1:t/hYoRTnlPuRjh8y0BwVGgNvNIXpU2QJME5YVppUUHQ= -github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/resourcetotelemetry v0.0.0-20241104203805-20919412150d h1:IyQzj0JRHEU00FfiwE/mMX6CepK4I8Dhz3XYYetRaaM= -github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/resourcetotelemetry v0.0.0-20241104203805-20919412150d/go.mod h1:Rr5b3hr6Jy9w/zTjsOl3vcyDDusc90P+iGdOd0UCYOo= -github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/stanza v0.0.0-20241104203805-20919412150d h1:l+U6eNiVjhUjsCXD9fFtTssXjRgO2oboalpcCtz2FdM= -github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/stanza v0.0.0-20241104203805-20919412150d/go.mod h1:2NSghK+mafMGxM8c4Gff8qcprdMD3YQebZtD9UAdB3E= -github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/translator/prometheus v0.0.0-20241104203805-20919412150d h1:5310F/G6U5YcjKots1ovIJzf+4+NMJ/2UAHfVN2JAVE= -github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/translator/prometheus v0.0.0-20241104203805-20919412150d/go.mod h1:21nuEQl7YYeLkVrGGvxPXkljqjR40teBCG5trGZ5LxM= -github.com/amazon-contributing/opentelemetry-collector-contrib/processor/resourcedetectionprocessor v0.0.0-20241104203805-20919412150d h1:2LmndCpP8KMMn7l9zyI5zeiu6OD18vIBMcrGuFRHdfs= -github.com/amazon-contributing/opentelemetry-collector-contrib/processor/resourcedetectionprocessor v0.0.0-20241104203805-20919412150d/go.mod h1:uzpU7Y6+oL6RdOv8IWi6fjT8LNV6FYX6CN6NATLJOiQ= -github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver v0.0.0-20241104203805-20919412150d h1:LvHBMj+2Kh+SAESWTVoLbJ8bD4Xq3toB4SU66hYjg0M= -github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver v0.0.0-20241104203805-20919412150d/go.mod h1:StgsMi0cNUydO2N/7WbLYPUBXzvp9wIMcWp9P8x/Vck= -github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awscontainerinsightskueuereceiver v0.0.0-20241112181539-bc8278558352 h1:WqkB8yOj8YCSxWzVUVCeVx263UWnmBUpMquetzwmntQ= -github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awscontainerinsightskueuereceiver v0.0.0-20241112181539-bc8278558352/go.mod h1://9Xy+KG8K9KvujBh6sZXIYPDvbu8xsiU1l8StFHjMA= -github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awsxrayreceiver v0.0.0-20241104203805-20919412150d h1:eQsOVRzXk32sVDkwJALfN0A4kZEMC2NNG8jBjZAZ7UM= -github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awsxrayreceiver v0.0.0-20241104203805-20919412150d/go.mod h1:igQaQJt7eA/y3dZ2VLXVql+6k/ZXBgrAa2y9FrMMIKQ= -github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/jmxreceiver v0.0.0-20241104203805-20919412150d h1:Cld9lc7zzU/EV70Lv8EXylsx4ATjz9jHfsObEYfjyzQ= -github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/jmxreceiver v0.0.0-20241104203805-20919412150d/go.mod h1:hRUrYatVP/GFNxHn2yW1gJcnPyGtdlTXyebpzzzjZeU= -github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/prometheusreceiver v0.0.0-20241104203805-20919412150d h1:awQLc9eG7SbW5vZCt1lxyRc+ZwTpjXwsH/AhtPF51uE= -github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/prometheusreceiver v0.0.0-20241104203805-20919412150d/go.mod h1:TTwtk1IMiqmyGm66w9aIq/TInicOm2Y6DwcJzRIpv1U= +github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awsxrayexporter v0.0.0-20241114192544-cb251fe99cbb h1:ViZMK5wtqDwNkKIMW/HxtogaKouIn5ZMjuobz+5ui9g= +github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awsxrayexporter v0.0.0-20241114192544-cb251fe99cbb/go.mod h1:GNeNylfr5KMt55XowzSdgbP7z8CkIAfIHtWSd+xxtws= +github.com/amazon-contributing/opentelemetry-collector-contrib/extension/awsmiddleware v0.0.0-20241114192544-cb251fe99cbb h1:NVIpIv4JrYgZ/rowSOqHaNpwGhs9cAbU1T7MNqF6QPw= +github.com/amazon-contributing/opentelemetry-collector-contrib/extension/awsmiddleware v0.0.0-20241114192544-cb251fe99cbb/go.mod h1:/RaNSxxO06niapGT00snMdgFfjjjW/kV3TZGX8kHuwM= +github.com/amazon-contributing/opentelemetry-collector-contrib/extension/awsproxy v0.0.0-20241114192544-cb251fe99cbb h1:Gk9EIxKayGfo1WV20/1elx8vRlO4YgXmM2JRgnB497I= +github.com/amazon-contributing/opentelemetry-collector-contrib/extension/awsproxy v0.0.0-20241114192544-cb251fe99cbb/go.mod h1:hRZt1DsvoLDIYBwjFvjwg/9IkaBXeCPG0QI57wbj98Q= +github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/awsutil v0.0.0-20241114192544-cb251fe99cbb h1:nyR+UQJeHa4qxaLujo7bEBLb5aeTYbgVXf3AYuZegcI= +github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/awsutil v0.0.0-20241114192544-cb251fe99cbb/go.mod h1:YL1Y62PxJ7dem1ZBUqCfvbnePaGr5p7DTSyOXSCi6O4= +github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/containerinsight v0.0.0-20241114192544-cb251fe99cbb h1:xbEI9BrUR5CPVAmjG5+XoAisV+FQQQzs28lgXHli8dY= +github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/containerinsight v0.0.0-20241114192544-cb251fe99cbb/go.mod h1:LT+qAyMutoADv2qezO+vkm/BkxR88qEfXdF2d13mV+E= +github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/cwlogs v0.0.0-20241114192544-cb251fe99cbb h1:LtkELbLo6ini93/ww8QOFcpSbjeI3QloupqSYYl7M/I= +github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/cwlogs v0.0.0-20241114192544-cb251fe99cbb/go.mod h1:SkscNdWANcuDJ7PkjS5wurSTAuY69nqP0I+cEVY9Ryw= +github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/k8s v0.0.0-20241114192544-cb251fe99cbb h1:61ZIUSIemzpU9fSPfcF2x9MX6z/bRnML2iiOo2TKj64= +github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/k8s v0.0.0-20241114192544-cb251fe99cbb/go.mod h1:/TOECDME2jYRPY21CrpTX2eMADJdkmBFBXc1lV/nRZA= +github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/proxy v0.0.0-20241114192544-cb251fe99cbb h1:c5BAAF/nC9A9+7VcXoujR/5MPiPPMu85hJRTq1W1Ckg= +github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/proxy v0.0.0-20241114192544-cb251fe99cbb/go.mod h1:J08A2gx8VFQfuoBiEfZ6uHIkMtVLd0OuRe5pP88b3I0= +github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/xray v0.0.0-20241114192544-cb251fe99cbb h1:FhnG36f47Xr6Qt4/zCd4yT/rwAswizH7dQ5tVHeFVP8= +github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/xray v0.0.0-20241114192544-cb251fe99cbb/go.mod h1:+w647+1nLYvZWdk24gZWvdl/kFowbe2iDyISXLDYdmQ= +github.com/amazon-contributing/opentelemetry-collector-contrib/internal/coreinternal v0.0.0-20241114192544-cb251fe99cbb h1:FhAGXTeK6GBDrdwYZ8xlgMVC7yWdnf3wU2wEu3LAgsM= +github.com/amazon-contributing/opentelemetry-collector-contrib/internal/coreinternal v0.0.0-20241114192544-cb251fe99cbb/go.mod h1:Ai4BsM7C05bEQYO2O272S1LmsyIhO5r0iLmXF5NN5so= +github.com/amazon-contributing/opentelemetry-collector-contrib/internal/k8sconfig v0.0.0-20241114192544-cb251fe99cbb h1:s7JP9zgrIzwuN0X7dD5qbmQYkl+hLwnDb6spXnc9H+8= +github.com/amazon-contributing/opentelemetry-collector-contrib/internal/k8sconfig v0.0.0-20241114192544-cb251fe99cbb/go.mod h1:VS66oUydCMwiWl1BFmLs7iNy4lGsfVYsriXr/d1fpAk= +github.com/amazon-contributing/opentelemetry-collector-contrib/internal/kubelet v0.0.0-20241114192544-cb251fe99cbb h1:YUf64mMnt76wV2zAfR97/h4trbrecyoXabN69ioGQZY= +github.com/amazon-contributing/opentelemetry-collector-contrib/internal/kubelet v0.0.0-20241114192544-cb251fe99cbb/go.mod h1:4qvmHiXPOkOXJdpmmxMqprb2BXxOGPgOG45BwLdipUM= +github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20241114192544-cb251fe99cbb h1:D0Pj3U4Tu8aOuVSYVS+uwKMfQ+V1tiMEgSYJa+3VklM= +github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20241114192544-cb251fe99cbb/go.mod h1:t/hYoRTnlPuRjh8y0BwVGgNvNIXpU2QJME5YVppUUHQ= +github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/resourcetotelemetry v0.0.0-20241114192544-cb251fe99cbb h1:yP5shDdkxuoYPX06Ae1W9WibvLc0cUXtsYD4504Dk7U= +github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/resourcetotelemetry v0.0.0-20241114192544-cb251fe99cbb/go.mod h1:Rr5b3hr6Jy9w/zTjsOl3vcyDDusc90P+iGdOd0UCYOo= +github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/stanza v0.0.0-20241114192544-cb251fe99cbb h1:2MuvnSSZM/oO1jaxe100FvI5lYjARtKLII6NVkkb1Rw= +github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/stanza v0.0.0-20241114192544-cb251fe99cbb/go.mod h1:2NSghK+mafMGxM8c4Gff8qcprdMD3YQebZtD9UAdB3E= +github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/translator/prometheus v0.0.0-20241114192544-cb251fe99cbb h1:U7GKcASLD5rwnjxhU/eAsHJziB2ay9ILcG3fcoiRtkQ= +github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/translator/prometheus v0.0.0-20241114192544-cb251fe99cbb/go.mod h1:21nuEQl7YYeLkVrGGvxPXkljqjR40teBCG5trGZ5LxM= +github.com/amazon-contributing/opentelemetry-collector-contrib/processor/resourcedetectionprocessor v0.0.0-20241114192544-cb251fe99cbb h1:yKACQrKRT2ug6LEKuJWG471UgGPQat5OFONjruCr+JI= +github.com/amazon-contributing/opentelemetry-collector-contrib/processor/resourcedetectionprocessor v0.0.0-20241114192544-cb251fe99cbb/go.mod h1:uzpU7Y6+oL6RdOv8IWi6fjT8LNV6FYX6CN6NATLJOiQ= +github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver v0.0.0-20241114192544-cb251fe99cbb h1:Tm0tYYa7SHmOGlyjUOSAGaR8/MnIZUlWiFmPup4N7t0= +github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver v0.0.0-20241114192544-cb251fe99cbb/go.mod h1:StgsMi0cNUydO2N/7WbLYPUBXzvp9wIMcWp9P8x/Vck= +github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awscontainerinsightskueuereceiver v0.0.0-20241114192544-cb251fe99cbb h1:im6grEA5ApFxNWSDiOLk5GZvFq3/xc6awZME10zP3Jc= +github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awscontainerinsightskueuereceiver v0.0.0-20241114192544-cb251fe99cbb/go.mod h1://9Xy+KG8K9KvujBh6sZXIYPDvbu8xsiU1l8StFHjMA= +github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awsxrayreceiver v0.0.0-20241114192544-cb251fe99cbb h1:Nlebx6xeywLO195gSd2E96jiPTydTfJbkr0+jvYjwn0= +github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awsxrayreceiver v0.0.0-20241114192544-cb251fe99cbb/go.mod h1:igQaQJt7eA/y3dZ2VLXVql+6k/ZXBgrAa2y9FrMMIKQ= +github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/jmxreceiver v0.0.0-20241114192544-cb251fe99cbb h1:rgCu3nHj3jh26HjIMUu03+VQxAU+4FIdhPXn0KEhqzM= +github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/jmxreceiver v0.0.0-20241114192544-cb251fe99cbb/go.mod h1:hRUrYatVP/GFNxHn2yW1gJcnPyGtdlTXyebpzzzjZeU= +github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/prometheusreceiver v0.0.0-20241114192544-cb251fe99cbb h1:pnXdRMiah9knvHazdQKUvyCbUmPxixfRpPtKarnKX6U= +github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/prometheusreceiver v0.0.0-20241114192544-cb251fe99cbb/go.mod h1:TTwtk1IMiqmyGm66w9aIq/TInicOm2Y6DwcJzRIpv1U= github.com/amir/raidman v0.0.0-20170415203553-1ccc43bfb9c9 h1:FXrPTd8Rdlc94dKccl7KPmdmIbVh/OjelJ8/vgMRzcQ= github.com/amir/raidman v0.0.0-20170415203553-1ccc43bfb9c9/go.mod h1:eliMa/PW+RDr2QLWRmLH1R1ZA4RInpmvOzDDXtaIZkc= github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs= From f6fad062e23fba3bc284120ef978902dc92f62d6 Mon Sep 17 00:00:00 2001 From: Rahul Sahu Date: Thu, 14 Nov 2024 20:50:57 +0000 Subject: [PATCH 29/32] build: re-enable github PR build actions --- .github/workflows/PR-build.yml | 68 ++++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) diff --git a/.github/workflows/PR-build.yml b/.github/workflows/PR-build.yml index 75c71b1b1a..32b1315def 100644 --- a/.github/workflows/PR-build.yml +++ b/.github/workflows/PR-build.yml @@ -68,3 +68,71 @@ jobs: - name: Check license and imports if: needs.changes.outputs.lint == 'true' run: make simple-lint + + build: + needs: [lint, changes] + name: Build ${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ ubuntu-latest, windows-2019, windows-latest, macos-12] + include: + - os: ubuntu-latest + family: linux + cache-path: | + ~/.cache/go-build + ~/go/pkg/mod + - os: macos-12 + family: darwin + cache-path: | + ~/Library/Caches/go-build + ~/go/pkg/mod + - os: windows-2019 + family: windows + cache-path: | + ~\AppData\Local\go-build + ~\go\pkg\mod + - os: windows-latest + family: windows + cache-path: | + ~\AppData\Local\go-build + ~\go\pkg\mod + steps: + - name: Set up Go 1.x + if: needs.changes.outputs.build == 'true' + uses: actions/setup-go@v4 + with: + go-version: ~1.22.2 + cache: false + + - name: Check out code + if: needs.changes.outputs.build == 'true' + uses: actions/checkout@v3 + + - name: Cache binaries + id: cached_binaries + if: needs.changes.outputs.build == 'true' + uses: actions/cache@v3 + with: + key: "cached-binaries-${{ matrix.os }}-${{ github.sha }}" + path: go.mod + + - name: Cache build output + if: steps.cached_binaries.outputs.cache-hit != 'true' && needs.changes.outputs.build == 'true' + uses: actions/cache@v3 + with: + path: ${{ matrix.cache-path }} + key: v1-go-pkg-mod-${{ matrix.os }}-${{ hashFiles('**/go.sum') }} + + - name: Install make + if: matrix.family == 'windows' && steps.cached_binaries.outputs.cache-hit != 'true' && needs.changes.outputs.build == 'true' + run: choco install make + + - name: Unit Test + if: steps.cached_binaries.outputs.cache-hit != 'true' && needs.changes.outputs.build == 'true' + run: make test + + - name: Build + if: steps.cached_binaries.outputs.cache-hit != 'true' && needs.changes.outputs.build == 'true' + run: make amazon-cloudwatch-agent-${{ matrix.family }} From ffaebc5dd5ffcdcbe654e01b81617cc6d9207b31 Mon Sep 17 00:00:00 2001 From: Rahul Sahu Date: Thu, 14 Nov 2024 21:49:15 +0000 Subject: [PATCH 30/32] fix: add pipeline name back as a parameter --- .../translate/otel/exporter/awsemf/translator.go | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/translator/translate/otel/exporter/awsemf/translator.go b/translator/translate/otel/exporter/awsemf/translator.go index 21e28bc12c..18ed408bed 100644 --- a/translator/translate/otel/exporter/awsemf/translator.go +++ b/translator/translate/otel/exporter/awsemf/translator.go @@ -25,6 +25,10 @@ import ( "github.com/aws/amazon-cloudwatch-agent/translator/util/ecsutil" ) +const ( + kueuePipelineName = "kueueContainerInsights" +) + //go:embed awsemf_default_generic.yaml var defaultGenericConfig string @@ -93,7 +97,7 @@ func (t *translator) Translate(c *confmap.Conf) (component.Config, error) { defaultConfig = defaultJmxConfig } else if isEcs(c) { defaultConfig = defaultEcsConfig - } else if isKubernetesKueue(c) { + } else if isKubernetesKueue(c, t.name) { defaultConfig = defaultKubernetesKueueConfig } else if isKubernetes(c) { defaultConfig = defaultKubernetesConfig @@ -143,7 +147,7 @@ func (t *translator) Translate(c *confmap.Conf) (component.Config, error) { if err := setEcsFields(c, cfg); err != nil { return nil, err } - } else if isKubernetesKueue(c) { + } else if isKubernetesKueue(c, t.name) { if err := setKubernetesKueueFields(c, cfg); err != nil { return nil, err } @@ -200,8 +204,8 @@ func isKubernetes(conf *confmap.Conf) bool { } // `kueue_container_insights` is a child of `kubernetes` in config spec. -func isKubernetesKueue(conf *confmap.Conf) bool { - return isKubernetes(conf) && common.GetOrDefaultBool(conf, kubernetesKueueBasePathKey, false) +func isKubernetesKueue(conf *confmap.Conf, pipelineName string) bool { + return isKubernetes(conf) && pipelineName == kueuePipelineName && common.GetOrDefaultBool(conf, kubernetesKueueBasePathKey, false) } func isPrometheus(conf *confmap.Conf) bool { From 8cf5193f0b777ba2327b85af4b330c050c9ba326 Mon Sep 17 00:00:00 2001 From: Rahul Sahu Date: Thu, 14 Nov 2024 21:50:13 +0000 Subject: [PATCH 31/32] fix: add new entitystore extension to kueue translated yamls --- .../sampleConfig/emf_and_kubernetes_with_kueue_config.yaml | 6 ++++++ .../sampleConfig/kueue_container_insights_config.yaml | 4 ++++ 2 files changed, 10 insertions(+) diff --git a/translator/tocwconfig/sampleConfig/emf_and_kubernetes_with_kueue_config.yaml b/translator/tocwconfig/sampleConfig/emf_and_kubernetes_with_kueue_config.yaml index e205d9525a..36d9202ca7 100644 --- a/translator/tocwconfig/sampleConfig/emf_and_kubernetes_with_kueue_config.yaml +++ b/translator/tocwconfig/sampleConfig/emf_and_kubernetes_with_kueue_config.yaml @@ -478,6 +478,11 @@ extensions: usage_flags: mode: OP region_type: ACJ + entitystore: + mode: onPremise + profile: default + region: us-east-1 + shared_credential_file: /root/.aws/credentials processors: batch/containerinsights: metadata_cardinality_limit: 1000 @@ -568,6 +573,7 @@ receivers: service: extensions: - agenthealth/logs + - entitystore pipelines: logs/emf_logs: exporters: diff --git a/translator/tocwconfig/sampleConfig/kueue_container_insights_config.yaml b/translator/tocwconfig/sampleConfig/kueue_container_insights_config.yaml index 27c7190bd5..7d2fa024b0 100644 --- a/translator/tocwconfig/sampleConfig/kueue_container_insights_config.yaml +++ b/translator/tocwconfig/sampleConfig/kueue_container_insights_config.yaml @@ -226,6 +226,9 @@ extensions: usage_flags: mode: EC2 region_type: ACJ + entitystore: + mode: ec2 + region: us-east-1 processors: batch/containerinsights: metadata_cardinality_limit: 1000 @@ -304,6 +307,7 @@ receivers: service: extensions: - agenthealth/logs + - entitystore pipelines: logs/emf_logs: exporters: From fa60db615746de616ca5786dadd539636c0223ae Mon Sep 17 00:00:00 2001 From: Rahul Sahu Date: Thu, 14 Nov 2024 22:00:15 +0000 Subject: [PATCH 32/32] build: update version for awsemfexporter --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index df5112a84b..62a0d39b19 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ replace github.com/influxdata/telegraf => github.com/aws/telegraf v0.10.2-0.2024 // to be all replaced since there are some changes that will always be from upstream replace ( github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awscloudwatchlogsexporter => github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awscloudwatchlogsexporter v0.0.0-20241114192544-cb251fe99cbb - github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awsemfexporter => github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awsemfexporter v0.0.0-20241024185216-f0b80d5c67a5 + github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awsemfexporter => github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awsemfexporter v0.0.0-20241114192544-cb251fe99cbb github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awsxrayexporter => github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awsxrayexporter v0.0.0-20241114192544-cb251fe99cbb ) diff --git a/go.sum b/go.sum index 274c2df246..0434a49d74 100644 --- a/go.sum +++ b/go.sum @@ -182,8 +182,8 @@ github.com/aliyun/alibaba-cloud-sdk-go v1.61.1483 h1:J8HaD+Zpfi1gcel3HCKpoHHEsrc github.com/aliyun/alibaba-cloud-sdk-go v1.61.1483/go.mod h1:RcDobYh8k5VP6TNybz9m++gL3ijVI5wueVr0EM10VsU= github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awscloudwatchlogsexporter v0.0.0-20241114192544-cb251fe99cbb h1:UePw4/QUtalit5jG7K301PPxtq/tRgYeXiPLaBgA71I= github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awscloudwatchlogsexporter v0.0.0-20241114192544-cb251fe99cbb/go.mod h1:f9JwM/LQdKH8ZbYoH9TO35rmjM6WoTNlhWtYJ3YZucc= -github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awsemfexporter v0.0.0-20241024185216-f0b80d5c67a5 h1:JOp9f4a1pP1zb5U+JPTezTdTZ1LCkcJSTcCd/AAqknM= -github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awsemfexporter v0.0.0-20241024185216-f0b80d5c67a5/go.mod h1:LPWBVdTSNbZkk80v6aCUthS59cnR4VauVRdAIE3ifaY= +github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awsemfexporter v0.0.0-20241114192544-cb251fe99cbb h1:JUMGOjiVxW7uKAqqCYYP/S6Jv2TRDlFno3KHeMDva2c= +github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awsemfexporter v0.0.0-20241114192544-cb251fe99cbb/go.mod h1:LPWBVdTSNbZkk80v6aCUthS59cnR4VauVRdAIE3ifaY= github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awsxrayexporter v0.0.0-20241114192544-cb251fe99cbb h1:ViZMK5wtqDwNkKIMW/HxtogaKouIn5ZMjuobz+5ui9g= github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awsxrayexporter v0.0.0-20241114192544-cb251fe99cbb/go.mod h1:GNeNylfr5KMt55XowzSdgbP7z8CkIAfIHtWSd+xxtws= github.com/amazon-contributing/opentelemetry-collector-contrib/extension/awsmiddleware v0.0.0-20241114192544-cb251fe99cbb h1:NVIpIv4JrYgZ/rowSOqHaNpwGhs9cAbU1T7MNqF6QPw=