Integrate NGINX prometheus exporter and enable metrics server

Makefile

@@ -135,7 +135,7 @@ build-nkg-debug-image: debug-build build-nkg-image ## Build NKG image with debug
 generate-manifests: ## Generate manifests using Helm.
  cp $(CHART_DIR)/crds/* $(MANIFEST_DIR)/crds/
  helm template nginx-gateway $(CHART_DIR) $(HELM_TEMPLATE_COMMON_ARGS) $(HELM_TEMPLATE_EXTRA_ARGS_FOR_ALL_MANIFESTS_FILE) -n nginx-gateway | cat $(strip $(MANIFEST_DIR))/namespace.yaml - > $(strip $(MANIFEST_DIR))/nginx-gateway.yaml
- helm template nginx-gateway $(CHART_DIR) $(HELM_TEMPLATE_COMMON_ARGS) -n nginx-gateway -s templates/deployment.yaml > conformance/provisioner/static-deployment.yaml
+ helm template nginx-gateway $(CHART_DIR) $(HELM_TEMPLATE_COMMON_ARGS) --set metrics.disable=true -n nginx-gateway -s templates/deployment.yaml > conformance/provisioner/static-deployment.yaml
  helm template nginx-gateway $(CHART_DIR) $(HELM_TEMPLATE_COMMON_ARGS) -n nginx-gateway -s templates/service.yaml > $(strip $(MANIFEST_DIR))/service/loadbalancer.yaml
  helm template nginx-gateway $(CHART_DIR) $(HELM_TEMPLATE_COMMON_ARGS) --set service.annotations.'service\.beta\.kubernetes\.io\/aws-load-balancer-type'="nlb" -n nginx-gateway -s templates/service.yaml > $(strip $(MANIFEST_DIR))/service/loadbalancer-aws-nlb.yaml
  helm template nginx-gateway $(CHART_DIR) $(HELM_TEMPLATE_COMMON_ARGS) --set service.type=NodePort --set service.externalTrafficPolicy="" -n nginx-gateway -s templates/service.yaml > $(strip $(MANIFEST_DIR))/service/nodeport.yaml

cmd/gateway/commands.go

@@ -24,6 +24,7 @@ const (
  gatewayCtrlNameFlag = "gateway-ctlr-name"
  gatewayCtrlNameUsageFmt = `The name of the Gateway controller. ` +
  `The controller name must be of the form: DOMAIN/PATH. The controller's domain is '%s'`
+ gatewayFlag = "gateway"
 )
 
 var (
@@ -36,58 +37,21 @@ var (
  gatewayClassName = stringValidatingValue{
  validator: validateResourceName,
  }
-)
 
-// stringValidatingValue is a string flag value with custom validation logic.
-// it implements the pflag.Value interface.
-type stringValidatingValue struct {
- validator func(v string) error
- value string
-}
+ // Backing values for static subcommand cli flags.
+ updateGCStatus bool
+ disableMetrics bool
+ metricsSecure bool
 
-func (v *stringValidatingValue) String() string {
- return v.value
-}
-
-func (v *stringValidatingValue) Set(param string) error {
- if err := v.validator(param); err != nil {
- return err
+ metricsListenPort = intValidatingValue{
+ validator: validatePort,
+ value: 9113,
  }
- v.value = param
- return nil
-}
-
-func (v *stringValidatingValue) Type() string {
- return "string"
-}
-
-// namespacedNameValue is a string flag value that represents a namespaced name.
-// it implements the pflag.Value interface.
-type namespacedNameValue struct {
- value types.NamespacedName
-}
-
-func (v *namespacedNameValue) String() string {
- if (v.value == types.NamespacedName{}) {
- // if we don't do that, the default value in the help message will be printed as "/"
- return ""
- }
- return v.value.String()
-}
-
-func (v *namespacedNameValue) Set(param string) error {
- nsname, err := parseNamespacedResourceName(param)
- if err != nil {
- return err
+ gateway = namespacedNameValue{}
+ configName = stringValidatingValue{
+ validator: validateResourceName,
  }
-
- v.value = nsname
- return nil
-}
-
-func (v *namespacedNameValue) Type() string {
- return "string"
-}
+)
 
 func createRootCommand() *cobra.Command {
  rootCmd := &cobra.Command{
@@ -115,15 +79,6 @@ func createRootCommand() *cobra.Command {
 }
 
 func createStaticModeCommand() *cobra.Command {
- const gatewayFlag = "gateway"
-
- // flag values
- gateway := namespacedNameValue{}
- var updateGCStatus bool
- configName := stringValidatingValue{
- validator: validateResourceName,
- }
-
  cmd := &cobra.Command{
  Use: "static-mode",
  Short: "Configure NGINX in the scope of a single Gateway resource",
@@ -153,6 +108,13 @@ func createStaticModeCommand() *cobra.Command {
  gwNsName = &gateway.value
  }
 
+ metricsConfig := config.MetricsConfig{}
+ if !disableMetrics {
+ metricsConfig.Enabled = true
+ metricsConfig.Port = metricsListenPort.value
+ metricsConfig.Secure = metricsSecure
+ }
+
  conf := config.Config{
  GatewayCtlrName: gatewayCtlrName.value,
  ConfigName: configName.String(),
@@ -163,6 +125,7 @@ func createStaticModeCommand() *cobra.Command {
  Namespace: namespace,
  GatewayNsName: gwNsName,
  UpdateGatewayClassStatus: updateGCStatus,
+ MetricsConfig: metricsConfig,
  }
 
  if err := static.StartManager(conf); err != nil {
@@ -198,6 +161,27 @@ func createStaticModeCommand() *cobra.Command {
  "Update the status of the GatewayClass resource.",
  )
 
+ cmd.Flags().BoolVar(
+ &disableMetrics,
+ "metrics-disable",
+ false,
+ "Disable exposing metrics in the Prometheus format.",
+ )
+
+ cmd.Flags().Var(
+ &metricsListenPort,
+ "metrics-port",
+ "Set the port where the metrics are exposed. Format: [1023 - 65535]",
+ )
+
+ cmd.Flags().BoolVar(
+ &metricsSecure,
+ "metrics-secure-serving",
+ false,
+ "Enable serving metrics via https. By default metrics are served via http."+
+ " Please note that this endpoint will be secured with a self-signed certificate.",
+ )
+
  return cmd
 }
 

cmd/gateway/commands_test.go

@@ -118,6 +118,9 @@ func TestStaticModeCmdFlagValidation(t *testing.T) {
  "--gateway=nginx-gateway/nginx",
  "--config=nginx-gateway-config",
  "--update-gatewayclass-status=true",
+ "--metrics-port=9114",
+ "--metrics-disable",
+ "--metrics-secure-serving",
  },
  wantErr: false,
  },
@@ -175,6 +178,42 @@ func TestStaticModeCmdFlagValidation(t *testing.T) {
  wantErr: true,
  expectedErrPrefix: `invalid argument "invalid" for "--update-gatewayclass-status" flag: strconv.ParseBool`,
  },
+ {
+ name: "metrics-port is invalid type",
+ args: []string{
+ "--metrics-port=invalid", // not an int
+ },
+ wantErr: true,
+ expectedErrPrefix: `invalid argument "invalid" for "--metrics-port" flag: failed to parse int value:` +
+ ` strconv.ParseInt: parsing "invalid": invalid syntax`,
+ },
+ {
+ name: "metrics-port is outside of range",
+ args: []string{
+ "--metrics-port=999", // outside of range
+ },
+ wantErr: true,
+ expectedErrPrefix: `invalid argument "999" for "--metrics-port" flag:` +
+ ` port outside of valid port range [1024 - 65535]: 999`,
+ },
+ {
+ name: "metrics-disable is not a bool",
+ args: []string{
+ "--metrics-disable=999", // not a bool
+ },
+ wantErr: true,
+ expectedErrPrefix: `invalid argument "999" for "--metrics-disable" flag: strconv.ParseBool:` +
+ ` parsing "999": invalid syntax`,
+ },
+ {
+ name: "metrics-secure-serving is not a bool",
+ args: []string{
+ "--metrics-secure-serving=999", // not a bool
+ },
+ wantErr: true,
+ expectedErrPrefix: `invalid argument "999" for "--metrics-secure-serving" flag: strconv.ParseBool:` +
+ ` parsing "999": invalid syntax`,
+ },
  }
 
  for _, test := range tests {

cmd/gateway/validating_types.go

@@ -0,0 +1,86 @@
+package main
+
+import (
+ "fmt"
+ "strconv"
+
+ "k8s.io/apimachinery/pkg/types"
+)
+
+// stringValidatingValue is a string flag value with custom validation logic.
+// it implements the pflag.Value interface.
+type stringValidatingValue struct {
+ validator func(v string) error
+ value string
+}
+
+func (v *stringValidatingValue) String() string {
+ return v.value
+}
+
+func (v *stringValidatingValue) Set(param string) error {
+ if err := v.validator(param); err != nil {
+ return err
+ }
+ v.value = param
+ return nil
+}
+
+func (v *stringValidatingValue) Type() string {
+ return "string"
+}
+
+type intValidatingValue struct {
+ validator func(v int) error
+ value int
+}
+
+func (v *intValidatingValue) String() string {
+ return strconv.Itoa(v.value)
+}
+
+func (v *intValidatingValue) Set(param string) error {
+ intVal, err := strconv.ParseInt(param, 10, 32)
+ if err != nil {
+ return fmt.Errorf("failed to parse int value: %w", err)
+ }
+
+ if err := v.validator(int(intVal)); err != nil {
+ return err
+ }
+
+ v.value = int(intVal)
+ return nil
+}
+
+func (v *intValidatingValue) Type() string {
+ return "int"
+}
+
+// namespacedNameValue is a string flag value that represents a namespaced name.
+// it implements the pflag.Value interface.
+type namespacedNameValue struct {
+ value types.NamespacedName
+}
+
+func (v *namespacedNameValue) String() string {
+ if (v.value == types.NamespacedName{}) {
+ // if we don't do that, the default value in the help message will be printed as "/"
+ return ""
+ }
+ return v.value.String()
+}
+
+func (v *namespacedNameValue) Set(param string) error {
+ nsname, err := parseNamespacedResourceName(param)
+ if err != nil {
+ return err
+ }
+
+ v.value = nsname
+ return nil
+}
+
+func (v *namespacedNameValue) Type() string {
+ return "string"
+}

cmd/gateway/validation.go

@@ -99,3 +99,11 @@ func validateIP(ip string) error {
 
  return nil
 }
+
+// validatePort makes sure a given port is inside the valid port range for its usage
+func validatePort(port int) error {
+ if port < 1024 || port > 65535 {
+ return fmt.Errorf("port outside of valid port range [1024 - 65535]: %v", port)
+ }
+ return nil
+}

cmd/gateway/validation_test.go

@@ -294,3 +294,40 @@ func TestValidateIP(t *testing.T) {
  })
  }
 }
+
+func TestValidatePort(t *testing.T) {
+ tests := []struct {
+ name string
+ port int
+ expErr bool
+ }{
+ {
+ name: "port under minimum allowed value",
+ port: 1023,
+ expErr: true,
+ },
+ {
+ name: "port over maximum allowed value",
+ port: 65536,
+ expErr: true,
+ },
+ {
+ name: "valid port",
+ port: 9113,
+ expErr: false,
+ },
+ }
+
+ for _, tc := range tests {
+ t.Run(tc.name, func(t *testing.T) {
+ g := NewGomegaWithT(t)
+
+ err := validatePort(tc.port)
+ if !tc.expErr {
+ g.Expect(err).ToNot(HaveOccurred())
+ } else {
+ g.Expect(err).To(HaveOccurred())
+ }
+ })
+ }
+}

conformance/provisioner/static-deployment.yaml

@@ -28,6 +28,7 @@ spec:
  - --gateway-ctlr-name=gateway.nginx.org/nginx-gateway-controller
  - --gatewayclass=nginx
  - --config=nginx-gateway-config
+ - --metrics-disable
  env:
  - name: POD_IP
  valueFrom:

deploy/helm-chart/README.md

@@ -142,3 +142,6 @@ The following tables lists the configurable parameters of the NGINX Kubernetes G
 | `service.externalTrafficPolicy` | The `externalTrafficPolicy` of the service. The value `Local` preserves the client source IP. | Local |
 | `service.annotations` | The `annotations` of the NGINX Kubernetes Gateway service. | {} |
 | `service.ports` | A list of ports to expose through the NGINX Kubernetes Gateway service. Update it to match the listener ports from your Gateway resource. Follows the conventional Kubernetes yaml syntax for service ports. | [ port: 80, targetPort: 80, protocol: TCP, name: http; port: 443, targetPort: 443, protocol: TCP, name: https ] |
+| `metrics.disable` | Disable exposing metrics in the Prometheus format. |false |
+| `metrics.port` | Set the port where the Prometheus metrics are exposed. Format: [1024 - 65535] |9113 |
+| `metrics.secure` | Enable serving metrics via https. By default metrics are served via http. Please note that this endpoint will be secured with a self-signed certificate. |false |