Integrate telemetry exporter (nginx#1656)

Problem:
Integrate the exporter library
https://github.com/nginxinc/telemetry-exporter so that it is possible
to send product telemetry data to an endpoint.

Solution:
- Integrate the exporter library.
- Update existing telemetry data struct to use the common data struct
  defined in the exporter library.
- Refactor existing telemetry data structs to adhere to the exporter
  library requirements.
- Generate scheme and Attributes for data structs.
- Allow configuring telemetry endpoint params via build flags. If the
  telemetry endpoint is not specified, NGF will log data points to the
  debug log (existing behavior).
- Add root CA certs to NGF image so that NGF can verify the cert of
  the telemetry service.

Testing:
- Unit tests
- Manual testing of build flags validation.
- Manual testing that NGF sends data to an OTel collector.
- Manual testing that NGF sends data to a dev F5 telemetry service.

CLOSES -- nginx#1377

Author: pleshakov

Diff for: .goreleaser.yml

@@ -15,7 +15,7 @@ builds:
     asmflags:
       - all=-trimpath={{.Env.GOPATH}}
     ldflags:
-      - -s -w -X main.version={{.Version}} -X main.commit={{.Commit}} -X main.date={{.Date}} -X main.telemetryReportPeriod=24h
+      - -s -w -X main.version={{.Version}} -X main.commit={{.Commit}} -X main.date={{.Date}} -X main.telemetryReportPeriod=24h -X main.telemetryEndpointInsecure=false
     main: ./cmd/gateway/
     binary: gateway
 

Diff for: .pre-commit-config.yaml

@@ -5,7 +5,7 @@ repos:
     rev: v4.5.0
     hooks:
       - id: trailing-whitespace
-        exclude: (^tests/results/)
+        exclude: (^tests/results/|\.avdl$|_generated.go$)
       - id: end-of-file-fixer
       - id: check-yaml
         args: [--allow-multiple-documents]

Diff for: Makefile

@@ -9,12 +9,19 @@ NJS_DIR = internal/mode/static/nginx/modules/src
 NGINX_DOCKER_BUILD_PLUS_ARGS = --secret id=nginx-repo.crt,src=nginx-repo.crt --secret id=nginx-repo.key,src=nginx-repo.key
 BUILD_AGENT=local
 TELEMETRY_REPORT_PERIOD = 24h # also configured in goreleaser.yml
+
+# FIXME(pleshakov) - TELEMETRY_ENDPOINT will have the default value of F5 telemetry service once we're ready
+# to report. https://github.com/nginxinc/nginx-gateway-fabric/issues/1563
+# Also, we will need to set it in goreleaser.yml
+TELEMETRY_ENDPOINT =# if empty, NGF will report telemetry in its logs at debug level.
+
+TELEMETRY_ENDPOINT_INSECURE = false # also configured in goreleaser.yml
 GW_API_VERSION = 1.0.0
 INSTALL_WEBHOOK = false
 NODE_VERSION = $(shell cat .nvmrc)
 
 # go build flags - should not be overridden by the user
-GO_LINKER_FlAGS_VARS = -X main.version=${VERSION} -X main.commit=${GIT_COMMIT} -X main.date=${DATE} -X main.telemetryReportPeriod=${TELEMETRY_REPORT_PERIOD}
+GO_LINKER_FlAGS_VARS = -X main.version=${VERSION} -X main.commit=${GIT_COMMIT} -X main.date=${DATE} -X main.telemetryReportPeriod=${TELEMETRY_REPORT_PERIOD} -X main.telemetryEndpoint=${TELEMETRY_ENDPOINT} -X main.telemetryEndpointInsecure=${TELEMETRY_ENDPOINT_INSECURE}
 GO_LINKER_FLAGS_OPTIMIZATIONS = -s -w
 GO_LINKER_FLAGS = $(GO_LINKER_FLAGS_OPTIMIZATIONS) $(GO_LINKER_FlAGS_VARS)
 

Diff for: build/Dockerfile

@@ -26,6 +26,9 @@ COPY dist/gateway_linux_$TARGETARCH*/gateway /usr/bin/
 RUN setcap 'cap_kill=+ep' /usr/bin/gateway
 
 FROM scratch as common
+# CA certs are needed for telemetry report and NGINX Plus usage report features, so that
+# NGF can verify the server's certificate.
+COPY --from=builder --link /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
 USER 102:1001
 ARG BUILD_AGENT
 ENV BUILD_AGENT=${BUILD_AGENT}

Diff for: cmd/gateway/commands.go

@@ -4,6 +4,7 @@ import (
 	"errors"
 	"fmt"
 	"os"
+	"strconv"
 	"time"
 
 	"github.com/spf13/cobra"
@@ -160,6 +161,17 @@ func createStaticModeCommand() *cobra.Command {
 				return fmt.Errorf("error parsing telemetry report period: %w", err)
 			}
 
+			if telemetryEndpoint != "" {
+				if err := validateEndpoint(telemetryEndpoint); err != nil {
+					return fmt.Errorf("error validating telemetry endpoint: %w", err)
+				}
+			}
+
+			telemetryEndpointInsecure, err := strconv.ParseBool(telemetryEndpointInsecure)
+			if err != nil {
+				return fmt.Errorf("error parsing telemetry endpoint insecure: %w", err)
+			}
+
 			var gwNsName *types.NamespacedName
 			if cmd.Flags().Changed(gatewayFlag) {
 				gwNsName = &gateway.value
@@ -211,8 +223,10 @@ func createStaticModeCommand() *cobra.Command {
 				},
 				UsageReportConfig: usageReportConfig,
 				ProductTelemetryConfig: config.ProductTelemetryConfig{
-					TelemetryReportPeriod: period,
-					Enabled:               !disableProductTelemetry,
+					ReportPeriod:     period,
+					Enabled:          !disableProductTelemetry,
+					Endpoint:         telemetryEndpoint,
+					EndpointInsecure: telemetryEndpointInsecure,
 				},
 				Plus:                 plus,
 				Version:              version,

Diff for: cmd/gateway/main.go

@@ -13,6 +13,10 @@ var (
 
 	// telemetryReportPeriod is the period at which telemetry reports are sent.
 	telemetryReportPeriod string
+	// telemetryEndpoint is the endpoint to which telemetry reports are sent.
+	telemetryEndpoint string
+	// telemetryEndpointInsecure controls whether TLS should be used when sending telemetry reports.
+	telemetryEndpointInsecure string
 )
 
 func main() {

Diff for: cmd/gateway/validation.go

@@ -6,6 +6,7 @@ import (
 	"net"
 	"net/url"
 	"regexp"
+	"strconv"
 	"strings"
 
 	"k8s.io/apimachinery/pkg/types"
@@ -133,6 +134,35 @@ func validateIP(ip string) error {
 	return nil
 }
 
+// validateEndpoint validates an endpoint, which is <host>:<port> where host is either a hostname or an IP address.
+func validateEndpoint(endpoint string) error {
+	host, port, err := net.SplitHostPort(endpoint)
+	if err != nil {
+		return fmt.Errorf("%q must be in the format <host>:<port>: %w", endpoint, err)
+	}
+
+	portVal, err := strconv.ParseInt(port, 10, 16)
+	if err != nil {
+		return fmt.Errorf("port must be a valid number: %w", err)
+	}
+
+	if portVal < 1 || portVal > 65535 {
+		return fmt.Errorf("port outside of valid port range [1 - 65535]: %v", port)
+	}
+
+	if err := validateIP(host); err == nil {
+		return nil
+	}
+
+	if errs := validation.IsDNS1123Subdomain(host); len(errs) == 0 {
+		return nil
+	}
+
+	// we don't know if the user intended to use a hostname or an IP address,
+	// so we return a generic error message
+	return fmt.Errorf("%q must be in the format <host>:<port>", endpoint)
+}
+
 // validatePort makes sure a given port is inside the valid port range for its usage
 func validatePort(port int) error {
 	if port < 1024 || port > 65535 {

Diff for: cmd/gateway/validation_test.go

@@ -419,6 +419,73 @@ func TestValidateIP(t *testing.T) {
 	}
 }
 
+func TestValidateEndpoint(t *testing.T) {
+	tests := []struct {
+		name   string
+		endp   string
+		expErr bool
+	}{
+		{
+			name:   "valid endpoint with hostname",
+			endp:   "localhost:8080",
+			expErr: false,
+		},
+		{
+			name:   "valid endpoint with IPv4",
+			endp:   "1.2.3.4:8080",
+			expErr: false,
+		},
+		{
+			name:   "valid endpoint with IPv6",
+			endp:   "[::1]:8080",
+			expErr: false,
+		},
+		{
+			name:   "invalid port - 1",
+			endp:   "localhost:0",
+			expErr: true,
+		},
+		{
+			name:   "invalid port - 2",
+			endp:   "localhost:65536",
+			expErr: true,
+		},
+		{
+			name:   "missing port with hostname",
+			endp:   "localhost",
+			expErr: true,
+		},
+		{
+			name:   "missing port with IPv4",
+			endp:   "1.2.3.4",
+			expErr: true,
+		},
+		{
+			name:   "missing port with IPv6",
+			endp:   "[::1]",
+			expErr: true,
+		},
+		{
+			name:   "invalid hostname or IP",
+			endp:   "loc@lhost:8080",
+			expErr: true,
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			g := NewWithT(t)
+
+			err := validateEndpoint(tc.endp)
+			if !tc.expErr {
+				g.Expect(err).ToNot(HaveOccurred())
+			} else {
+				g.Expect(err).To(HaveOccurred())
+			}
+		})
+	}
+}
+
 func TestValidatePort(t *testing.T) {
 	tests := []struct {
 		name   string

Diff for: docs/developer/implementing-a-feature.md

@@ -59,7 +59,11 @@ practices to ensure a successful feature development process.
     different reviewer in mind, you can request them as well. Refer to
     the [pull request](/docs/developer/pull-request.md) documentation for expectations and guidelines.
 14. **Obtain the necessary approvals**: Work with code reviewers to maintain the required number of approvals.
-15. **Squash and merge**: Squash your commits locally, or use the GitHub UI to squash and merge. Only one commit per
+15. **Ensure the product telemetry works**. If you made any changes to the product telemetry data points, it is
+    necessary to push the generated scheme (`.avdl`, generated in Step 12) to the scheme registry. After that, manually
+    verify that the product telemetry data is successfully pushed to the telemetry service by confirming that the data
+    has been received.
+16. **Squash and merge**: Squash your commits locally, or use the GitHub UI to squash and merge. Only one commit per
     pull request should be merged. Make sure the first line of the final commit message includes the pull request
     number. For example, Fix supported gateway conditions in compatibility doc (#674).
     > **Note**:

Diff for: docs/developer/quickstart.md

@@ -214,14 +214,16 @@ Run the following make command from the project's root directory to lint the Hel
 make lint-helm
 ```
 
-## Run go generate
+## Run Code Generation
 
 To ensure all the generated code is up to date, run the following make command from the project's root directory:
 
 ```shell
 make generate
 ```
 
+That command also will generate the avro scheme (`.avdl`) for product telemetry data points.
+
 ## Update Generated Manifests
 
 To update the generated manifests, run the following make command from the project's root directory:

Diff for: go.mod

@@ -12,13 +12,16 @@ require (
 	github.com/maxbrunsfeld/counterfeiter/v6 v6.8.1
 	github.com/nginxinc/nginx-plus-go-client v1.2.0
 	github.com/nginxinc/nginx-prometheus-exporter v1.1.0
+	github.com/nginxinc/telemetry-exporter v0.0.0-20240307135433-a5ecce59bddf
 	github.com/onsi/ginkgo/v2 v2.16.0
 	github.com/onsi/gomega v1.31.1
 	github.com/prometheus/client_golang v1.19.0
 	github.com/prometheus/common v0.50.0
 	github.com/spf13/cobra v1.8.0
 	github.com/spf13/pflag v1.0.5
 	github.com/tsenart/vegeta/v12 v12.11.1
+	go.opentelemetry.io/otel v1.24.0
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.24.0
 	go.uber.org/zap v1.27.0
 	k8s.io/api v0.29.2
 	k8s.io/apiextensions-apiserver v0.29.2
@@ -32,6 +35,7 @@ require (
 
 require (
 	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/cenkalti/backoff/v4 v4.2.1 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/emicklei/go-restful/v3 v3.11.0 // indirect
@@ -40,6 +44,7 @@ require (
 	github.com/fatih/color v1.16.0 // indirect
 	github.com/fsnotify/fsnotify v1.7.0 // indirect
 	github.com/go-logfmt/logfmt v0.5.1 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-logr/zapr v1.3.0 // indirect
 	github.com/go-openapi/jsonpointer v0.20.0 // indirect
 	github.com/go-openapi/jsonreference v0.20.2 // indirect
@@ -52,8 +57,9 @@ require (
 	github.com/google/gnostic-models v0.6.8 // indirect
 	github.com/google/gofuzz v1.2.0 // indirect
 	github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 // indirect
-	github.com/google/uuid v1.3.1 // indirect
+	github.com/google/uuid v1.6.0 // indirect
 	github.com/gorilla/websocket v1.5.0 // indirect
+	github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.0 // indirect
 	github.com/imdario/mergo v0.3.16 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/influxdata/tdigest v0.0.1 // indirect
@@ -73,19 +79,27 @@ require (
 	github.com/prometheus/procfs v0.12.0 // indirect
 	github.com/rs/dnscache v0.0.0-20211102005908-e0241e321417 // indirect
 	github.com/stretchr/testify v1.8.4 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.24.0 // indirect
+	go.opentelemetry.io/otel/metric v1.24.0 // indirect
+	go.opentelemetry.io/otel/sdk v1.24.0 // indirect
+	go.opentelemetry.io/otel/trace v1.24.0 // indirect
+	go.opentelemetry.io/proto/otlp v1.1.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 	golang.org/x/exp v0.0.0-20231006140011-7918f672742d // indirect
-	golang.org/x/mod v0.14.0 // indirect
+	golang.org/x/mod v0.16.0 // indirect
 	golang.org/x/net v0.22.0 // indirect
 	golang.org/x/oauth2 v0.18.0 // indirect
 	golang.org/x/sync v0.6.0 // indirect
 	golang.org/x/sys v0.18.0 // indirect
 	golang.org/x/term v0.18.0 // indirect
 	golang.org/x/text v0.14.0 // indirect
 	golang.org/x/time v0.3.0 // indirect
-	golang.org/x/tools v0.17.0 // indirect
+	golang.org/x/tools v0.19.0 // indirect
 	gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect
 	google.golang.org/appengine v1.6.8 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20240102182953-50ed04b92917 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20240102182953-50ed04b92917 // indirect
+	google.golang.org/grpc v1.61.1 // indirect
 	google.golang.org/protobuf v1.33.0 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect