Add Azure Resource Logs translator

.chloggen/azureeventhub_otelsc.yaml

@@ -0,0 +1,27 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: new_component
+
+# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
+component: azurelogs_translater
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Adds a new translater that converts Azure EventHub logs to OpenTelemetry logs used by the Azure Events Hub receiver.
+
+# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists.
+issues: [39704]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext:
+
+# If your change doesn't affect end users or the exported elements of any package,
+# you should instead start your pull request title with [chore] or use the "Skip Changelog" label.
+# Optional: The change log or logs in which this entry should be included.
+# e.g. '[user]' or '[user, api]'
+# Include 'user' if the change is relevant to end users.
+# Include 'api' if there is a change to a library API.
+# Default: '[user]'
+change_logs: [user]

.github/CODEOWNERS

@@ -156,6 +156,7 @@ pkg/sampling/                                                       @open-teleme
 pkg/stanza/                                                         @open-telemetry/collector-contrib-approvers @djaglowski
 pkg/stanza/fileconsumer/                                            @open-telemetry/collector-contrib-approvers @djaglowski
 pkg/translator/azure/                                               @open-telemetry/collector-contrib-approvers @open-telemetry/collector-approvers @atoulme @cparkins
+pkg/translator/azurelogs/                                           @open-telemetry/collector-contrib-approvers @atoulme @cparkins @MikeGoldsmith
 pkg/translator/jaeger/                                              @open-telemetry/collector-contrib-approvers @open-telemetry/collector-approvers @frzifus
 pkg/translator/loki/                                                @open-telemetry/collector-contrib-approvers @gouthamve @jpkrohling @mar4uk
 pkg/translator/opencensus/                                          @open-telemetry/collector-contrib-approvers @open-telemetry/collector-approvers

.github/ISSUE_TEMPLATE/bug_report.yaml

@@ -152,6 +152,7 @@ body:
       - pkg/stanza
       - pkg/stanza/fileconsumer
       - pkg/translator/azure
+      - pkg/translator/azurelogs
       - pkg/translator/jaeger
       - pkg/translator/loki
       - pkg/translator/opencensus

.github/ISSUE_TEMPLATE/feature_request.yaml

@@ -146,6 +146,7 @@ body:
       - pkg/stanza
       - pkg/stanza/fileconsumer
       - pkg/translator/azure
+      - pkg/translator/azurelogs
       - pkg/translator/jaeger
       - pkg/translator/loki
       - pkg/translator/opencensus

.github/ISSUE_TEMPLATE/other.yaml

@@ -146,6 +146,7 @@ body:
       - pkg/stanza
       - pkg/stanza/fileconsumer
       - pkg/translator/azure
+      - pkg/translator/azurelogs
       - pkg/translator/jaeger
       - pkg/translator/loki
       - pkg/translator/opencensus

.github/ISSUE_TEMPLATE/unmaintained.yaml

@@ -151,6 +151,7 @@ body:
       - pkg/stanza
       - pkg/stanza/fileconsumer
       - pkg/translator/azure
+      - pkg/translator/azurelogs
       - pkg/translator/jaeger
       - pkg/translator/loki
       - pkg/translator/opencensus

cmd/otelcontribcol/builder-config.yaml

@@ -468,6 +468,7 @@ replaces:
   - github.com/openshift/api v3.9.0+incompatible => github.com/openshift/api v0.0.0-20180801171038-322a19404e37
   - github.com/outcaste-io/ristretto v0.2.0 => github.com/outcaste-io/ristretto v0.2.1
   - github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/azure => ../../pkg/translator/azure
+  - github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/azurelogs => ../../pkg/translator/azurelogs
   - github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/skywalking => ../../pkg/translator/skywalking
   - github.com/open-telemetry/opentelemetry-collector-contrib/internal/collectd => ../../internal/collectd
   - github.com/open-telemetry/opentelemetry-collector-contrib/extension/encoding => ../../extension/encoding

cmd/otelcontribcol/go.mod

@@ -662,6 +662,7 @@ require (
 	github.com/open-telemetry/opentelemetry-collector-contrib/pkg/resourcetotelemetry v0.109.0 // indirect
 	github.com/open-telemetry/opentelemetry-collector-contrib/pkg/sampling v0.109.0 // indirect
 	github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/azure v0.109.0 // indirect
+	github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/azurelogs v0.109.0 // indirect
 	github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/jaeger v0.109.0 // indirect
 	github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/loki v0.109.0 // indirect
 	github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/opencensus v0.109.0 // indirect
@@ -1320,6 +1321,8 @@ replace github.com/outcaste-io/ristretto v0.2.0 => github.com/outcaste-io/ristre
 
 replace github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/azure => ../../pkg/translator/azure
 
+replace github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/azurelogs => ../../pkg/translator/azurelogs
+
 replace github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/skywalking => ../../pkg/translator/skywalking
 
 replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/collectd => ../../internal/collectd

pkg/translator/azurelogs/Makefile

@@ -0,0 +1 @@
+include ../../../Makefile.Common

pkg/translator/azurelogs/complex_conversions.go

@@ -0,0 +1,105 @@
+// Copyright The OpenTelemetry Authors
+// SPDX-License-Identifier: Apache-2.0
+
+package azurelogs // import "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/azurelogs"
+
+import (
+	"fmt"
+	"strconv"
+	"strings"
+)
+
+type ComplexConversion func(string, any, map[string]any) bool
+type TypeConversion func(string, any, map[string]any, string) bool
+
+var conversions = map[string]ComplexConversion{
+	"AzureCdnAccessLog:SecurityProtocol":               azureCdnAccessLogSecurityProtocol,
+	"FrontDoorAccessLog:securityProtocol":              azureCdnAccessLogSecurityProtocol,
+	"AppServiceHTTPLogs:Protocol":                      appServiceHTTPLogsProtocol,
+	"AppServiceHTTPLogs:TimeTaken":                     appServiceHTTPLogTimeTakenMilliseconds,
+	"FrontDoorHealthProbeLog:DNSLatencyMicroseconds":   frontDoorHealthProbeLogDNSLatencyMicroseconds,
+	"FrontDoorHealthProbeLog:totalLatencyMilliseconds": frontDoorHealthProbeLogTotalLatencyMilliseconds,
+}
+
+// Splits the "TLS 1.2" value into "TLS" and "1.2" and sets as "network.protocol.name" and "network.protocol.version"
+func azureCdnAccessLogSecurityProtocol(_ string, value any, attrs map[string]any) bool {
+	if str, ok := value.(string); ok {
+		if parts := strings.SplitN(str, " ", 2); len(parts) == 2 {
+			attrs["tls.protocol.name"] = strings.ToLower(parts[0])
+			attrs["tls.protocol.version"] = parts[1]
+			return true
+		}
+	}
+	return false
+}
+
+// Splits the "HTTP/1.1" value into "HTTP" and "1.1" and sets as "network.protocol.name" and "network.protocol.version"
+func appServiceHTTPLogsProtocol(_ string, value any, attrs map[string]any) bool {
+	if str, ok := value.(string); ok {
+		if parts := strings.SplitN(str, "/", 2); len(parts) == 2 {
+			attrs["network.protocol.name"] = strings.ToLower(parts[0])
+			attrs["network.protocol.version"] = parts[1]
+			return true
+		}
+	}
+	return false
+}
+
+// Converts Microseconds value to Seconds and sets as "dns.lookup.duration"
+func frontDoorHealthProbeLogDNSLatencyMicroseconds(_ string, value any, attrs map[string]any) bool {
+	microseconds, ok := tryParseFloat64(value)
+	if !ok {
+		return false
+	}
+	seconds := microseconds / 1_000_000
+	attrs["dns.lookup.duration"] = seconds
+	return true
+}
+
+// Converts Milliseconds value to Seconds and sets as "http.client.request.duration"
+func frontDoorHealthProbeLogTotalLatencyMilliseconds(_ string, value any, attrs map[string]any) bool {
+	milliseconds, ok := tryParseFloat64(value)
+	if !ok {
+		return false
+	}
+	seconds := milliseconds / 1_000
+	attrs["http.request.duration"] = seconds
+	return true
+}
+
+// Converts Milliseconds value to Seconds and sets as "http.server.request.duration"
+func appServiceHTTPLogTimeTakenMilliseconds(_ string, value any, attrs map[string]any) bool {
+	milliseconds, ok := tryParseFloat64(value)
+	if !ok {
+		return false
+	}
+	seconds := milliseconds / 1_000
+	attrs["http.server.request.duration"] = seconds
+	return true
+}
+
+func tryParseFloat64(value any) (float64, bool) {
+	switch v := value.(type) {
+	case float32:
+		return float64(v), true
+	case float64:
+		return v, true
+	case int:
+		return float64(v), true
+	case int32:
+		return float64(v), true
+	case int64:
+		return float64(v), true
+	case string:
+		f, err := strconv.ParseFloat(v, 64)
+		return f, err == nil
+	default:
+		return 0, false
+	}
+}
+
+func tryGetComplexConversion(category string, propertyName string) (ComplexConversion, bool) {
+	key := fmt.Sprintf("%s:%s", category, propertyName)
+	conversion, ok := conversions[key]
+	return conversion, ok
+}

pkg/translator/azurelogs/complex_conversions_test.go

@@ -0,0 +1,76 @@
+// Copyright The OpenTelemetry Authors
+// SPDX-License-Identifier: Apache-2.0
+
+package azurelogs // import "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/azurelogs"
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestFrontDoorAccessLogSecurityProtocol(t *testing.T) {
+	f, ok := tryGetComplexConversion("FrontDoorAccessLog", "securityProtocol")
+	assert.True(t, ok)
+	attrs := map[string]any{}
+	ok = f("securityProtocol", "TLS 1.2", attrs)
+	assert.True(t, ok)
+	protocolName, ok := attrs["tls.protocol.name"]
+	assert.True(t, ok)
+	// Protocol name is normalized to lower case
+	assert.Equal(t, "tls", protocolName)
+	protocolVersion, ok := attrs["tls.protocol.version"]
+	assert.True(t, ok)
+	assert.Equal(t, "1.2", protocolVersion)
+}
+
+func TestAzureCDNAccessLogSecurityProtocol(t *testing.T) {
+	f, ok := tryGetComplexConversion("AzureCdnAccessLog", "SecurityProtocol")
+	assert.True(t, ok)
+	attrs := map[string]any{}
+	ok = f("SecurityProtocol", "TLS 1.2", attrs)
+	assert.True(t, ok)
+	protocolName, ok := attrs["tls.protocol.name"]
+	assert.True(t, ok)
+	// Protocol name is normalized to lower case
+	assert.Equal(t, "tls", protocolName)
+	protocolVersion, ok := attrs["tls.protocol.version"]
+	assert.True(t, ok)
+	assert.Equal(t, "1.2", protocolVersion)
+}
+
+func TestAppServiceHTTPLogsProtocol(t *testing.T) {
+	f, ok := tryGetComplexConversion("AppServiceHTTPLogs", "Protocol")
+	assert.True(t, ok)
+	attrs := map[string]any{}
+	ok = f("Protocol", "HTTP/1.1", attrs)
+	assert.True(t, ok)
+	protocolName, ok := attrs["network.protocol.name"]
+	assert.True(t, ok)
+	assert.Equal(t, "http", protocolName)
+	protocolVersion, ok := attrs["network.protocol.version"]
+	assert.True(t, ok)
+	assert.Equal(t, "1.1", protocolVersion)
+}
+
+func TestFrontDoorHealthProbeLogDNSLatencyMicroseconds(t *testing.T) {
+	f, ok := tryGetComplexConversion("FrontDoorHealthProbeLog", "DNSLatencyMicroseconds")
+	assert.True(t, ok)
+	attrs := map[string]any{}
+	ok = f("DNSLatencyMicroseconds", 123456, attrs)
+	assert.True(t, ok)
+	duration, ok := attrs["dns.lookup.duration"].(float64)
+	assert.True(t, ok)
+	assert.Equal(t, 0.123456, duration)
+}
+
+func TestFrontDoorHealthProbeLogTotalLatencyMilliseconds(t *testing.T) {
+	f, ok := tryGetComplexConversion("FrontDoorHealthProbeLog", "totalLatencyMilliseconds")
+	assert.True(t, ok)
+	attrs := map[string]any{}
+	ok = f("totalLatencyMilliseconds", 123, attrs)
+	assert.True(t, ok)
+	duration, ok := attrs["http.request.duration"].(float64)
+	assert.True(t, ok)
+	assert.Equal(t, 0.123, duration)
+}

pkg/translator/azurelogs/go.mod

@@ -0,0 +1,44 @@
+module github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/azurelogs
+
+go 1.22.0
+
+require (
+	github.com/json-iterator/go v1.1.12
+	github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatatest v0.99.0
+	github.com/relvacode/iso8601 v1.4.0
+	github.com/stretchr/testify v1.9.0
+	go.opentelemetry.io/collector/component v0.109.0
+	go.opentelemetry.io/collector/pdata v1.15.0
+	go.opentelemetry.io/collector/semconv v0.109.0
+	go.uber.org/goleak v1.3.0
+	go.uber.org/zap v1.27.0
+	golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f
+)
+
+require (
+	github.com/cespare/xxhash/v2 v2.3.0 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatautil v0.109.0 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	go.opentelemetry.io/collector/config/configtelemetry v0.109.0 // indirect
+	go.opentelemetry.io/otel v1.29.0 // indirect
+	go.opentelemetry.io/otel/metric v1.29.0 // indirect
+	go.opentelemetry.io/otel/trace v1.29.0 // indirect
+	go.uber.org/multierr v1.11.0 // indirect
+	golang.org/x/net v0.27.0 // indirect
+	golang.org/x/sys v0.24.0 // indirect
+	golang.org/x/text v0.16.0 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20240604185151-ef581f913117 // indirect
+	google.golang.org/grpc v1.66.0 // indirect
+	google.golang.org/protobuf v1.34.2 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+)
+
+replace github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatautil => ../../pdatautil
+
+replace github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatatest => ../../pdatatest
+
+replace github.com/open-telemetry/opentelemetry-collector-contrib/pkg/golden => ../../golden