From 2b765b3ff4b778c2d1e108d50d27f1b680457d75 Mon Sep 17 00:00:00 2001 From: Eoin O'Shaughnessy Date: Fri, 23 Feb 2024 16:28:46 +0000 Subject: [PATCH 01/16] initial agent configuration --- charts/nginx-ingress/templates/_helpers.tpl | 17 ++++- .../templates/controller-configmap.yaml | 64 ++++++++++++++++++- charts/nginx-ingress/values.schema.json | 54 ++++++++++++++++ charts/nginx-ingress/values.yaml | 27 +++++++- cmd/nginx-ingress/flags.go | 3 + cmd/nginx-ingress/main.go | 27 +++++++- internal/nginx/fake_manager.go | 15 +++++ internal/nginx/manager.go | 45 ++++++++++++- 8 files changed, 245 insertions(+), 7 deletions(-) diff --git a/charts/nginx-ingress/templates/_helpers.tpl b/charts/nginx-ingress/templates/_helpers.tpl index 3f13513dc9..8dd61ebc48 100644 --- a/charts/nginx-ingress/templates/_helpers.tpl +++ b/charts/nginx-ingress/templates/_helpers.tpl @@ -83,6 +83,17 @@ Expand the name of the configmap. {{- end -}} {{- end -}} +{{/* +Expand the name of the configmap used for NGINX Agent. +*/}} +{{- define "nginx-ingress.agentConfigName" -}} +{{- if .Values.nginxAgent.customConfigMap -}} +{{ .Values.nginxAgent.customConfigMap }} +{{- else -}} +{{- printf "%s-agent-config" (include "nginx-ingress.fullname" . | trunc 49 | trimSuffix "-") -}} +{{- end -}} +{{- end -}} + {{/* Expand leader election lock name. */}} @@ -162,7 +173,7 @@ Build the args for the service binary. - --continue {{- end }} - -- -{{- end }} +{{- end -}} - -nginx-plus={{ .Values.controller.nginxplus }} - -nginx-reload-timeout={{ .Values.controller.nginxReloadTimeout }} - -enable-app-protect={{ .Values.controller.appprotect.enable }} @@ -249,4 +260,8 @@ Build the args for the service binary. - -enable-latency-metrics={{ .Values.controller.enableLatencyMetrics }} - -ssl-dynamic-reload={{ .Values.controller.enableSSLDynamicReload }} - -enable-telemetry-reporting={{ .Values.controller.enableTelemetryReporting}} +{{- if .Values.nginxAgent.enable }} +- -agent=true +- -agent-instance-group={{ default (include "nginx-ingress.controller.fullname" .) .Values.nginxAgent.instanceGroup }} +{{- end }} {{- end -}} diff --git a/charts/nginx-ingress/templates/controller-configmap.yaml b/charts/nginx-ingress/templates/controller-configmap.yaml index fd11991865..1321d6f43a 100644 --- a/charts/nginx-ingress/templates/controller-configmap.yaml +++ b/charts/nginx-ingress/templates/controller-configmap.yaml @@ -11,7 +11,67 @@ metadata: {{ toYaml .Values.controller.config.annotations | indent 4 }} {{- end }} data: -{{- if .Values.controller.config.entries }} -{{ toYaml .Values.controller.config.entries | indent 2 }} +{{ toYaml (default dict .Values.controller.config.entries) | indent 2 }} {{- end }} +--- +{{- if and .Values.nginxAgent.enable (not .Values.nginxAgent.customConfigMap) }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "nginx-ingress.agentConfigName" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "nginx-ingress.labels" . | nindent 4 }} +{{- if .Values.controller.config.annotations }} + annotations: +{{ toYaml .Values.controller.config.annotations | indent 4 }} +{{- end }} +data: + nginx-agent.conf: |- + instance_group: nic-deployment + log: + level: {{ .Values.nginxAgent.logLevel }} + path: "" + server: + host: {{ .Values.nginxAgent.instanceManager.host }} + grpcPort: {{ .Values.nginxAgent.instanceManager.grpcPort }} + {{- if .Values.nginxAgent.instanceManager.tls.enable }} + tls: + # enable tls in the nginx-agent setup for grpcs + # default to enable to connect with tls connection but without client cert for mtls + enable: true + skip_verify: {{ .Values.nginxAgent.instanceManager.tls.skipVerify }} + # path to ca certificate file used for server cert validation + ca: "" + # path to cert and key files for mTLS + cert: "" + key: "" + {{- end }} + features: + - registration + - nginx-counting + - metrics-sender + - metrics-collection + - dataplane-status + extensions: + - nginx-app-protect + - nap-monitoring + - advanced-metrics + nginx_app_protect: + report_interval: 15s + precompiled_publication: true + nap_monitoring: + collector_buffer_size: 50000 + processor_buffer_size: 50000 + syslog_ip: "127.0.0.1" + syslog_port: 1514 + advanced_metrics: + socket_path: /var/run/nginx-agent/advanced-metrics.sock + aggregation_period: 1s + publishing_period: 3s + table_sizes_limits: + staging_table_max_size: 1000 + staging_table_threshold: 1000 + priority_table_max_size: 1000 + priority_table_threshold: 1000 {{- end }} diff --git a/charts/nginx-ingress/values.schema.json b/charts/nginx-ingress/values.schema.json index 2a1569c66b..1cabd2fe61 100644 --- a/charts/nginx-ingress/values.schema.json +++ b/charts/nginx-ingress/values.schema.json @@ -1803,6 +1803,60 @@ "enableEgress": false } ] + }, + "nginxAgent": { + "type": "object", + "default": { + "enable": false + }, + "title": "Configuration for NGINX Agent.", + "required": [ + "enable" + ], + "properties": { + "enable": { + "type": "boolean", + "default": false, + "title": "Enable NGINX Agent", + "examples": [ + false + ] + }, + "instanceGroup": { + "type": "string", + "default": "", + "title": "Set the --instance-group argument for NGINX Agent", + "examples": [ + "my-instance-group" + ] + }, + "instanceManager": { + "type": "object", + "default": {}, + "title": "Configuration for the connection to NGINX Instance Manager", + "examples": [], + "required": [ + "host" + ], + "properties": { + "host": { + "type": "string", + "title": "FQDN or IP for connecting to Ingress Controller", + "examples": [ + "nim.example.com" + ] + }, + "grpcPort": { + "type": "integer", + "title": "Port for connecting to Ingress Controller", + "default": 443, + "examples": [ + 443 + ] + } + } + } + } } }, "examples": [ diff --git a/charts/nginx-ingress/values.yaml b/charts/nginx-ingress/values.yaml index 6c2cabb95d..f2f9182d20 100644 --- a/charts/nginx-ingress/values.yaml +++ b/charts/nginx-ingress/values.yaml @@ -173,7 +173,8 @@ controller: type: RuntimeDefault ## The security context for the Ingress Controller containers. - securityContext: {} # Remove curly brackets before adding values + securityContext: + {} # Remove curly brackets before adding values # allowPrivilegeEscalation: true # readOnlyRootFilesystem: true # runAsUser: 101 #nginx @@ -551,3 +552,27 @@ nginxServiceMesh: ## Enables NGINX Service Mesh workload to route egress traffic through the Ingress Controller. ## Requires nginxServiceMesh.enable enableEgress: false + +nginxAgent: + ## Enables NGINX Agent. + enable: false +## If nginxAgent.instanceGroup is not set the value of nginx-ingress.controller.fullname will be used +# instanceGroup: "" +# logLevel: "error" +# syslog: +# host: "127.0.0.1" +# port: 1514 +# instanceManager: +# host: +# grpcPort: 443 +# tls: +# enabled: false +# skipVerify: true +# ## The secret with a TLS certificate and key for the default HTTPS server. +# ## The value must follow the following format: `/`. +# ## Used as an alternative to specifying a certificate and key using `controller.defaultTLS.cert` and `controller.defaultTLS.key` parameters. +# ## Note: Alternatively, omitting the default server secret completely will configure NGINX to reject TLS connections to the default server. +# ## Format: / +# secret: "" +# ## The custom ConfigMap to use instead of the one provided by default +# customConfigMap: "" diff --git a/cmd/nginx-ingress/flags.go b/cmd/nginx-ingress/flags.go index 2d162b5e3c..1fc4efdd6b 100644 --- a/cmd/nginx-ingress/flags.go +++ b/cmd/nginx-ingress/flags.go @@ -66,6 +66,9 @@ var ( appProtectDosMaxWorkers = flag.Int("app-protect-dos-max-workers", 0, "Max number of nginx processes to support. Requires -nginx-plus and -enable-app-protect-dos.") appProtectDosMemory = flag.Int("app-protect-dos-memory", 0, "RAM memory size to consume in MB. Requires -nginx-plus and -enable-app-protect-dos.") + agent = flag.Bool("agent", false, "Enable NGINX Agent") + agentInstanceGroup = flag.String("agent-instance-group", "nginx-ingress-controller", "Grouping used to associate NGINX Ingress Controller instances") + ingressClass = flag.String("ingress-class", "nginx", `A class of the Ingress Controller. diff --git a/cmd/nginx-ingress/main.go b/cmd/nginx-ingress/main.go index 57bb70426d..6f8c2e0bbf 100644 --- a/cmd/nginx-ingress/main.go +++ b/cmd/nginx-ingress/main.go @@ -49,6 +49,7 @@ const ( nginxVersionLabel = "app.nginx.org/version" versionLabel = "app.kubernetes.io/version" appProtectVersionLabel = "appprotect.f5.com/version" + agentVersionLabel = "app.nginx.org/agent-version" appProtectVersionPath = "/opt/app_protect/VERSION" ) @@ -81,7 +82,12 @@ func main() { appProtectVersion = getAppProtectVersionInfo() } - go updateSelfWithVersionInfo(kubeClient, version, appProtectVersion, nginxVersion, 10, time.Second*5) + var agentVersion string + if *agent { + agentVersion = getAgentVersionInfo(nginxManager) + } + + go updateSelfWithVersionInfo(kubeClient, version, appProtectVersion, agentVersion, nginxVersion, 10, time.Second*5) templateExecutor, templateExecutorV2 := createTemplateExecutors() @@ -421,12 +427,18 @@ func getAppProtectVersionInfo() string { return version } +func getAgentVersionInfo(nginxManager nginx.Manager) string { + return nginxManager.AgentVersion() +} + type childProcessConfig struct { nginxDone chan error aPPluginEnable bool aPPluginDone chan error aPDosEnable bool aPDosDone chan error + agentEnable bool + agentDone chan error } func startChildProcesses(nginxManager nginx.Manager) childProcessConfig { @@ -447,12 +459,20 @@ func startChildProcesses(nginxManager nginx.Manager) childProcessConfig { nginxDone := make(chan error, 1) nginxManager.Start(nginxDone) + var agentDone chan error + if *agent { + agentDone = make(chan error, 1) + nginxManager.AgentStart(agentDone, *agentInstanceGroup) + } + return childProcessConfig{ nginxDone: nginxDone, aPPluginEnable: *appProtect, aPPluginDone: aPPluginDone, aPDosEnable: *appProtectDos, aPDosDone: aPPDosAgentDone, + agentEnable: *agent, + agentDone: agentDone, } } @@ -772,7 +792,7 @@ func processConfigMaps(kubeClient *kubernetes.Clientset, cfgParams *configs.Conf return cfgParams } -func updateSelfWithVersionInfo(kubeClient *kubernetes.Clientset, version, appProtectVersion string, nginxVersion nginx.Version, maxRetries int, waitTime time.Duration) { +func updateSelfWithVersionInfo(kubeClient *kubernetes.Clientset, version, appProtectVersion, agentVersion string, nginxVersion nginx.Version, maxRetries int, waitTime time.Duration) { podUpdated := false for i := 0; (i < maxRetries || maxRetries == 0) && !podUpdated; i++ { @@ -797,6 +817,9 @@ func updateSelfWithVersionInfo(kubeClient *kubernetes.Clientset, version, appPro if appProtectVersion != "" { labels[appProtectVersionLabel] = appProtectVersion } + if agentVersion != "" { + labels[agentVersionLabel] = agentVersion + } newPod.ObjectMeta.Labels = labels _, err = kubeClient.CoreV1().Pods(newPod.ObjectMeta.Namespace).Update(context.TODO(), newPod, meta_v1.UpdateOptions{}) diff --git a/internal/nginx/fake_manager.go b/internal/nginx/fake_manager.go index 65ccbcc578..b52e3d2f65 100644 --- a/internal/nginx/fake_manager.go +++ b/internal/nginx/fake_manager.go @@ -174,6 +174,21 @@ func (*FakeManager) AppProtectDosAgentStart(_ chan error, _ bool, _ int, _ int, glog.V(3).Infof("Starting FakeAppProtectDosAgent") } +// AgentQuit is a fake implementation AppProtectAgentQuit +func (*FakeManager) AgentQuit() { + glog.V(3).Infof("Quitting FakeAgent") +} + +// AgentStart is a fake implementation of AppProtectAgentStart +func (*FakeManager) AgentStart(_ chan error, _ string) { + glog.V(3).Infof("Starting FakeAgent") +} + +// AgentVersion is a fake implementation of AppProtectAgentStart +func (*FakeManager) AgentVersion() string { + return "v0.00.0-00000000" +} + // GetSecretsDir is a fake implementation func (fm *FakeManager) GetSecretsDir() string { return fm.secretsPath diff --git a/internal/nginx/manager.go b/internal/nginx/manager.go index f440386ef5..2a9117e077 100644 --- a/internal/nginx/manager.go +++ b/internal/nginx/manager.go @@ -39,6 +39,8 @@ const ( appProtectPluginStartCmd = "/usr/share/ts/bin/bd-socket-plugin" appProtectLogLevelCmd = "/opt/app_protect/bin/set_log_level" + agentPath = "/usr/bin/nginx-agent" + // appPluginParams is the configuration of App-Protect plugin appPluginParams = "tmm_count 4 proc_cpuinfo_cpu_mhz 2000000 total_xml_memory 471859200 total_umu_max_size 3129344 sys_max_account_id 1024 no_static_config" @@ -90,6 +92,9 @@ type Manager interface { AppProtectPluginQuit() AppProtectDosAgentStart(apdaDone chan error, debug bool, maxDaemon int, maxWorkers int, memory int) AppProtectDosAgentQuit() + AgentStart(agentDone chan error, logLevel string) + AgentQuit() + AgentVersion() string GetSecretsDir() string } @@ -113,6 +118,7 @@ type LocalManager struct { OpenTracing bool appProtectPluginPid int appProtectDosAgentPid int + agentPid int } // NewLocalManager creates a LocalManager. @@ -481,7 +487,7 @@ func (lm *LocalManager) AppProtectPluginStart(appDone chan error, logLevel strin glog.V(3).Info("Starting AppProtect Plugin") startupParams := strings.Fields(appPluginParams) - cmd := exec.Command(appProtectPluginStartCmd, startupParams...) + cmd := exec.Command(appProtectPluginStartCmd, startupParams...) // #nosec G204 cmd.Stdout = os.Stdout cmd.Stderr = os.Stdout @@ -558,6 +564,43 @@ func (lm *LocalManager) AppProtectDosAgentStart(apdaDone chan error, debug bool, }() } +// AgentStart starts the AppProtect plugin and sets AppProtect log level. +func (lm *LocalManager) AgentStart(agentDone chan error, instanceGroup string) { + glog.V(3).Info("Starting Agent") + args := []string{"--instance-group", instanceGroup} + cmd := exec.Command(agentPath, args...) // #nosec G204 + + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stdout + cmd.Env = os.Environ() + + if err := cmd.Start(); err != nil { + glog.Fatalf("Failed to start Agent: %v", err) + } + lm.agentPid = cmd.Process.Pid + go func() { + agentDone <- cmd.Wait() + }() +} + +// AgentQuit gracefully ends AppProtect Agent. +func (lm *LocalManager) AgentQuit() { + glog.V(3).Info("Quitting Agent") + killcmd := fmt.Sprintf("kill %d", lm.agentPid) + if err := shellOut(killcmd); err != nil { + glog.Fatalf("Failed to quit Agent: %v", err) + } +} + +// AgentVersion returns NGINX Agent version +func (lm *LocalManager) AgentVersion() string { + out, err := exec.Command(agentPath, "-v").CombinedOutput() + if err != nil { + glog.Fatalf("Failed to get nginx agent version: %v", err) + } + return strings.TrimSpace(strings.TrimPrefix(string(out), "nginx-agent version ")) +} + func getBinaryFileName(debug bool) string { if debug { return nginxBinaryPathDebug From 7559193e0def81c26b201f093f12c4f769b1dfa5 Mon Sep 17 00:00:00 2001 From: Eoin O'Shaughnessy Date: Wed, 28 Feb 2024 11:44:26 +0000 Subject: [PATCH 02/16] add initial tls configuration --- charts/nginx-ingress/templates/_helpers.tpl | 51 ++++++++++++++++++- .../templates/controller-configmap.yaml | 49 +----------------- charts/nginx-ingress/values.schema.json | 27 ++++++++++ charts/nginx-ingress/values.yaml | 41 +++++++-------- 4 files changed, 100 insertions(+), 68 deletions(-) diff --git a/charts/nginx-ingress/templates/_helpers.tpl b/charts/nginx-ingress/templates/_helpers.tpl index 35558abcc4..e2fb7d9d56 100644 --- a/charts/nginx-ingress/templates/_helpers.tpl +++ b/charts/nginx-ingress/templates/_helpers.tpl @@ -58,6 +58,9 @@ helm.sh/chart: {{ include "nginx-ingress.chart" . }} app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} {{- end }} app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- if and .Values.nginxAgent.enable (eq (.Values.nginxAgent.customConfigMap | default "") "") }} +agent-configuration-revision-hash: {{ include "nginx-ingress.agentConfiguration" . | sha1sum | trunc 8 | quote }} +{{- end }} {{- end }} {{/* @@ -87,7 +90,7 @@ Expand the name of the configmap. Expand the name of the configmap used for NGINX Agent. */}} {{- define "nginx-ingress.agentConfigName" -}} -{{- if .Values.nginxAgent.customConfigMap -}} +{{- if ne (.Values.nginxAgent.customConfigMap | default "") "" -}} {{ .Values.nginxAgent.customConfigMap }} {{- else -}} {{- printf "%s-agent-config" (include "nginx-ingress.fullname" . | trunc 49 | trimSuffix "-") -}} @@ -309,3 +312,49 @@ volumeMounts: {{ toYaml .Values.controller.volumeMounts }} {{- end }} {{- end -}} + +{{- define "nginx-ingress.agentConfiguration" -}} +log: + level: {{ .Values.nginxAgent.logLevel }} + path: "" +server: + host: {{ .Values.nginxAgent.instanceManager.host }} + grpcPort: {{ .Values.nginxAgent.instanceManager.grpcPort }} +{{- if .Values.nginxAgent.instanceManager.tls }} +tls: + enable: {{ .Values.nginxAgent.instanceManager.tls.enable | default false }} + skip_verify: {{ .Values.nginxAgent.instanceManager.tls.skipVerify | default false }} + {{- if ne .Values.nginxAgent.instanceManager.tls.caSecret "" -}} + ca: "" + {{- end }} + cert: "" + key: "" +{{- end }} +features: + - registration + - nginx-counting + - metrics-sender + - metrics-collection + - dataplane-status +extensions: + - nginx-app-protect + - nap-monitoring + - advanced-metrics +nginx_app_protect: + report_interval: 15s + precompiled_publication: true +nap_monitoring: + collector_buffer_size: 50000 + processor_buffer_size: 50000 + syslog_ip: "127.0.0.1" + syslog_port: 1514 +advanced_metrics: + socket_path: /var/run/nginx-agent/advanced-metrics.sock + aggregation_period: 1s + publishing_period: 3s + table_sizes_limits: + staging_table_max_size: 1000 + staging_table_threshold: 1000 + priority_table_max_size: 1000 + priority_table_threshold: 1000 +{{- end -}} diff --git a/charts/nginx-ingress/templates/controller-configmap.yaml b/charts/nginx-ingress/templates/controller-configmap.yaml index 1321d6f43a..8f1d3e47bb 100644 --- a/charts/nginx-ingress/templates/controller-configmap.yaml +++ b/charts/nginx-ingress/templates/controller-configmap.yaml @@ -14,7 +14,7 @@ data: {{ toYaml (default dict .Values.controller.config.entries) | indent 2 }} {{- end }} --- -{{- if and .Values.nginxAgent.enable (not .Values.nginxAgent.customConfigMap) }} +{{- if and .Values.nginxAgent.enable (eq (.Values.nginxAgent.customConfigMap | default "") "") }} apiVersion: v1 kind: ConfigMap metadata: @@ -28,50 +28,5 @@ metadata: {{- end }} data: nginx-agent.conf: |- - instance_group: nic-deployment - log: - level: {{ .Values.nginxAgent.logLevel }} - path: "" - server: - host: {{ .Values.nginxAgent.instanceManager.host }} - grpcPort: {{ .Values.nginxAgent.instanceManager.grpcPort }} - {{- if .Values.nginxAgent.instanceManager.tls.enable }} - tls: - # enable tls in the nginx-agent setup for grpcs - # default to enable to connect with tls connection but without client cert for mtls - enable: true - skip_verify: {{ .Values.nginxAgent.instanceManager.tls.skipVerify }} - # path to ca certificate file used for server cert validation - ca: "" - # path to cert and key files for mTLS - cert: "" - key: "" - {{- end }} - features: - - registration - - nginx-counting - - metrics-sender - - metrics-collection - - dataplane-status - extensions: - - nginx-app-protect - - nap-monitoring - - advanced-metrics - nginx_app_protect: - report_interval: 15s - precompiled_publication: true - nap_monitoring: - collector_buffer_size: 50000 - processor_buffer_size: 50000 - syslog_ip: "127.0.0.1" - syslog_port: 1514 - advanced_metrics: - socket_path: /var/run/nginx-agent/advanced-metrics.sock - aggregation_period: 1s - publishing_period: 3s - table_sizes_limits: - staging_table_max_size: 1000 - staging_table_threshold: 1000 - priority_table_max_size: 1000 - priority_table_threshold: 1000 +{{ include "nginx-ingress.agentConfiguration" . | indent 4 }} {{- end }} diff --git a/charts/nginx-ingress/values.schema.json b/charts/nginx-ingress/values.schema.json index 7a175ff1d2..09256c0d07 100644 --- a/charts/nginx-ingress/values.schema.json +++ b/charts/nginx-ingress/values.schema.json @@ -1853,6 +1853,33 @@ "examples": [ 443 ] + }, + "tls": { + "type": "object", + "default": {}, + "title": "TLS configuration for connection between NGINX Agent and Instance Manager", + "properties": { + "enable": { + "type": "boolean", + "default": "false", + "title": "enable TLS for Instance Manager connection" + }, + "secret": { + "type": "string", + "default": "", + "title": "Name of the kubernetes.io/tls secret used for TLS connection to Instance Manager" + }, + "caSecret": { + "type": "string", + "default": "", + "title": "Name of the nginx.org/ca secret used for verification" + }, + "skipVerify": { + "type": "boolean", + "default": "false", + "title": "skip certificate verification" + } + } } } } diff --git a/charts/nginx-ingress/values.yaml b/charts/nginx-ingress/values.yaml index 5124e151c4..3b4b08e434 100644 --- a/charts/nginx-ingress/values.yaml +++ b/charts/nginx-ingress/values.yaml @@ -556,23 +556,24 @@ nginxServiceMesh: nginxAgent: ## Enables NGINX Agent. enable: false -## If nginxAgent.instanceGroup is not set the value of nginx-ingress.controller.fullname will be used -# instanceGroup: "" -# logLevel: "error" -# syslog: -# host: "127.0.0.1" -# port: 1514 -# instanceManager: -# host: -# grpcPort: 443 -# tls: -# enabled: false -# skipVerify: true -# ## The secret with a TLS certificate and key for the default HTTPS server. -# ## The value must follow the following format: `/`. -# ## Used as an alternative to specifying a certificate and key using `controller.defaultTLS.cert` and `controller.defaultTLS.key` parameters. -# ## Note: Alternatively, omitting the default server secret completely will configure NGINX to reject TLS connections to the default server. -# ## Format: / -# secret: "" -# ## The custom ConfigMap to use instead of the one provided by default -# customConfigMap: "" + ## If nginxAgent.instanceGroup is not set the value of nginx-ingress.controller.fullname will be used + instanceGroup: "" + logLevel: "error" + syslog: + host: "127.0.0.1" + port: 1514 + instanceManager: + host: "" + grpcPort: 443 + tls: + enabled: false + skipVerify: false + ## The secret with a TLS certificate and key for the default HTTPS server. + ## The value must follow the following format: `/`. + ## Used as an alternative to specifying a certificate and key using `controller.defaultTLS.cert` and `controller.defaultTLS.key` parameters. + ## Note: Alternatively, omitting the default server secret completely will configure NGINX to reject TLS connections to the default server. + ## Format: / + secret: "" + caSecret: "" + ## The custom ConfigMap to use instead of the one provided by default + customConfigMap: "" From aff4846adfe06d8b321bc77bf7970b2568338141 Mon Sep 17 00:00:00 2001 From: Eoin O'Shaughnessy Date: Thu, 29 Feb 2024 16:53:23 +0000 Subject: [PATCH 03/16] helper updates for agent secrets --- charts/nginx-ingress/templates/_helpers.tpl | 71 +++++++++++++++++---- 1 file changed, 57 insertions(+), 14 deletions(-) diff --git a/charts/nginx-ingress/templates/_helpers.tpl b/charts/nginx-ingress/templates/_helpers.tpl index 219004ea64..cf13d42db2 100644 --- a/charts/nginx-ingress/templates/_helpers.tpl +++ b/charts/nginx-ingress/templates/_helpers.tpl @@ -58,9 +58,6 @@ helm.sh/chart: {{ include "nginx-ingress.chart" . }} app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} {{- end }} app.kubernetes.io/managed-by: {{ .Release.Service }} -{{- if and .Values.nginxAgent.enable (eq (.Values.nginxAgent.customConfigMap | default "") "") }} -agent-configuration-revision-hash: {{ include "nginx-ingress.agentConfiguration" . | sha1sum | trunc 8 | quote }} -{{- end }} {{- end }} {{/* @@ -73,6 +70,9 @@ nsm.nginx.com/enable-ingress: "true" nsm.nginx.com/enable-egress: "{{ .Values.nginxServiceMesh.enableEgress }}" nsm.nginx.com/{{ .Values.controller.kind }}: {{ include "nginx-ingress.controller.fullname" . }} {{- end }} +{{- if and .Values.nginxAgent.enable (eq (.Values.nginxAgent.customConfigMap | default "") "") }} +agent-configuration-revision-hash: {{ include "nginx-ingress.agentConfiguration" . | sha1sum | trunc 8 | quote }} +{{- end }} {{- if .Values.controller.pod.extraLabels }} {{ toYaml .Values.controller.pod.extraLabels }} {{- end }} @@ -288,10 +288,10 @@ Build the args for the service binary. Volumes for controller. */}} {{- define "nginx-ingress.volumes" -}} -{{- if or (eq (include "nginx-ingress.readOnlyRootFilesystem" .) "true" ) .Values.controller.volumes }} +{{- $volumesSet := "false" }} volumes: -{{- end }} {{- if eq (include "nginx-ingress.readOnlyRootFilesystem" .) "true" }} +{{- $volumesSet = "true" }} - name: nginx-etc emptyDir: {} - name: nginx-cache @@ -302,18 +302,41 @@ volumes: emptyDir: {} {{- end }} {{- if .Values.controller.volumes }} +{{- $volumesSet = "true" }} {{ toYaml .Values.controller.volumes }} {{- end }} +{{- if .Values.nginxAgent.enable -}} +{{- $volumesSet = "true" }} +- name: agent-conf + configMap: + name: {{ include "nginx-ingress.agentConfigName" . }} +- name: agent-dynamic + emptyDir: {} +{{- if and .Values.nginxAgent.instanceManager.tls (ne (.Values.nginxAgent.instanceManager.tls.secret | default "") "") }} +- name: nginx-agent-tls + projected: + sources: + - secret: + name: {{ .Values.nginxAgent.instanceManager.tls.secret }} +{{- if .Values.nginxAgent.instanceManager.tls.caSecret }} + - secret: + name: {{ .Values.nginxAgent.instanceManager.tls.caSecret }} +{{- end }} +{{- end }} +{{- end -}} +{{- if eq $volumesSet "false" -}} +{{ toYaml list | printf " %s" }} +{{- end -}} {{- end -}} {{/* Volume mounts for controller. */}} {{- define "nginx-ingress.volumeMounts" -}} -{{- if or ( eq (include "nginx-ingress.readOnlyRootFilesystem" .) "true" ) .Values.controller.volumeMounts }} +{{- $volumeMountSet := "false" }} volumeMounts: -{{- end }} {{- if eq (include "nginx-ingress.readOnlyRootFilesystem" .) "true" }} +{{- $volumeMountSet = "true" }} - mountPath: /etc/nginx name: nginx-etc - mountPath: /var/cache/nginx @@ -323,9 +346,26 @@ volumeMounts: - mountPath: /var/log/nginx name: nginx-log {{- end }} -{{- if .Values.controller.volumeMounts}} +{{- if .Values.controller.volumeMounts }} +{{- $volumeMountSet = "true" }} {{ toYaml .Values.controller.volumeMounts }} {{- end }} +{{- if .Values.nginxAgent.enable -}} +{{- $volumeMountSet = "true" }} +- name: agent-conf + mountPath: /etc/nginx-agent/nginx-agent.conf + subPath: nginx-agent.conf +- name: agent-dynamic + mountPath: /var/lib/nginx-agent +{{- if and .Values.nginxAgent.instanceManager.tls .Values.nginxAgent.instanceManager.tls.secret }} +- name: nginx-agent-tls + mountPath: /etc/ssl/nms + readOnly: true +{{- end }} +{{- end -}} +{{- if eq $volumeMountSet "false" -}} +{{ toYaml list | printf " %s" }} +{{- end -}} {{- end -}} {{- define "nginx-ingress.agentConfiguration" -}} @@ -338,12 +378,14 @@ server: {{- if .Values.nginxAgent.instanceManager.tls }} tls: enable: {{ .Values.nginxAgent.instanceManager.tls.enable | default false }} - skip_verify: {{ .Values.nginxAgent.instanceManager.tls.skipVerify | default false }} - {{- if ne .Values.nginxAgent.instanceManager.tls.caSecret "" -}} - ca: "" + skip_verify: {{ .Values.nginxAgent.instanceManager.tls.skipVerify | default false }} + {{- if ne .Values.nginxAgent.instanceManager.tls.caSecret "" }} + ca: "/etc/ssl/nms/ca.crt" + {{- end }} + {{- if ne .Values.nginxAgent.instanceManager.tls.secret "" }} + cert: "/etc/ssl/nms/tls.crt" + key: "/etc/ssl/nms/tls.key" {{- end }} - cert: "" - key: "" {{- end }} features: - registration @@ -372,4 +414,5 @@ advanced_metrics: staging_table_threshold: 1000 priority_table_max_size: 1000 priority_table_threshold: 1000 -{{- end -}} + +{{ end -}} From 19ab58cb73e026e6056b234fae5c65296c643c53 Mon Sep 17 00:00:00 2001 From: Eoin O'Shaughnessy Date: Fri, 8 Mar 2024 11:05:56 +0000 Subject: [PATCH 04/16] handle missing agent --- build/Dockerfile | 5 +++-- cmd/nginx-ingress/flags.go | 4 ++++ internal/nginx/manager.go | 2 +- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/build/Dockerfile b/build/Dockerfile index 513af86db1..64b169a880 100644 --- a/build/Dockerfile +++ b/build/Dockerfile @@ -366,8 +366,9 @@ RUN --mount=type=bind,target=/tmp mkdir -p /var/lib/nginx /etc/nginx/secrets /et && setcap -v 'cap_net_bind_service=+eip' /usr/sbin/nginx 'cap_net_bind_service=+eip' /usr/sbin/nginx-debug \ && [ -z "${BUILD_OS##*plus*}" ] && PLUS=-plus; cp -a /tmp/internal/configs/version1/nginx$PLUS.ingress.tmpl /tmp/internal/configs/version1/nginx$PLUS.tmpl \ /tmp/internal/configs/version2/nginx$PLUS.virtualserver.tmpl /tmp/internal/configs/version2/nginx$PLUS.transportserver.tmpl / \ - && chown -R 101:0 /etc/nginx /var/cache/nginx /var/lib/nginx /var/log/nginx /*.tmpl \ - && chmod -R g=u /etc/nginx /var/cache/nginx /var/lib/nginx /var/log/nginx /*.tmpl \ + && mkdir -p /etc/ssl/nms \ + && chown -R 101:0 /etc/nginx /var/cache/nginx /var/lib/nginx /var/log/nginx /*.tmpl /etc/ssl/nms \ + && chmod -R g=u /etc/nginx /var/cache/nginx /var/lib/nginx /var/log/nginx /*.tmpl /etc/ssl/nms \ && rm -f /etc/nginx/conf.d/* # Uncomment the line below if you would like to add the default.pem to the image diff --git a/cmd/nginx-ingress/flags.go b/cmd/nginx-ingress/flags.go index 1fc4efdd6b..257409ea75 100644 --- a/cmd/nginx-ingress/flags.go +++ b/cmd/nginx-ingress/flags.go @@ -283,6 +283,10 @@ func parseFlags() { if *ingressLink != "" && *externalService != "" { glog.Fatal("ingresslink and external-service cannot both be set") } + + if *agent && !*appProtect { + glog.Fatal("NGINX Agent is used to enable the Security Monitoring dashboard and requires NGINX App Protect to be enabled") + } } func initialChecks() { diff --git a/internal/nginx/manager.go b/internal/nginx/manager.go index a5b5719cd0..877fa4af63 100644 --- a/internal/nginx/manager.go +++ b/internal/nginx/manager.go @@ -599,7 +599,7 @@ func (lm *LocalManager) AgentQuit() { func (lm *LocalManager) AgentVersion() string { out, err := exec.Command(agentPath, "-v").CombinedOutput() if err != nil { - glog.Fatalf("Failed to get nginx agent version: %v", err) + glog.Fatalf("Failed to get nginx-agent version: %v", err) } return strings.TrimSpace(strings.TrimPrefix(string(out), "nginx-agent version ")) } From 6953ddf571f4fc593deaedbdbefd1796cb599953 Mon Sep 17 00:00:00 2001 From: Eoin O'Shaughnessy Date: Fri, 8 Mar 2024 12:10:51 +0000 Subject: [PATCH 05/16] docs for new values and helper cleanup --- charts/nginx-ingress/README.md | 13 +++++ charts/nginx-ingress/templates/_helpers.tpl | 58 +++++++++++---------- charts/nginx-ingress/values.schema.json | 56 ++++++++++++++++++-- charts/nginx-ingress/values.yaml | 13 +++-- 4 files changed, 102 insertions(+), 38 deletions(-) diff --git a/charts/nginx-ingress/README.md b/charts/nginx-ingress/README.md index 403c5efd1c..c274a9ef0f 100644 --- a/charts/nginx-ingress/README.md +++ b/charts/nginx-ingress/README.md @@ -489,6 +489,19 @@ The following tables lists the configurable parameters of the NGINX Ingress Cont |`serviceNameOverride` | Used to prevent cloud load balancers from being replaced due to service name change during helm upgrades. | "" | |`nginxServiceMesh.enable` | Enable integration with NGINX Service Mesh. See the NGINX Service Mesh [docs](https://docs.nginx.com/nginx-service-mesh/tutorials/kic/deploy-with-kic/) for more details. Requires `controller.nginxplus`. | false | |`nginxServiceMesh.enableEgress` | Enable NGINX Service Mesh workloads to route egress traffic through the Ingress Controller. See the NGINX Service Mesh [docs](https://docs.nginx.com/nginx-service-mesh/tutorials/kic/deploy-with-kic/#enabling-egress) for more details. Requires `nginxServiceMesh.enable`. | false | +|`nginxAgent.enable` | Enable NGINX Agent to enable Security Monitoring integration with App Protect WAF module. Requires `controller.appprotect.enable`. | false | +|`nginxAgent.instanceGroup` | Set a custom Instance Group name which the deployment will be shown as when connected to NGINX Instance Manager. `nginx-ingress.controller.fullname` will be used if not set. | "" | +|`nginxAgent.logLevel` | Log level for NGINX Agent. | "error | +|`nginxAgent.instanceManager.host` | FQDN or IP for connecting to Ingress Controller. Required when `nginxAgent.enable` is set to `true` | "" | +|`nginxAgent.instanceManager.grpcPort` | Port for connecting to Ingress Controller. | 443 | +|`nginxAgent.instanceManager.sni` | Server Name Indication for Instance Manager. See the NGINX Agent [docs](https://docs.nginx.com/nginx-agent/configuration/encrypt-communication/) for more details. | "" | +|`nginxAgent.instanceManager.tls.enable` | Enable TLS for Instance Manager connection. | true | +|`nginxAgent.instanceManager.tls.skipVerify` | Skip certification verification for Instance Manager connection. | false | +|`nginxAgent.instanceManager.tls.caSecret` | Name of `nginx.org/ca` secret used for verification of Instance Manager TLS. | "" | +|`nginxAgent.instanceManager.tls.secret` | Name of `kubernetes.io/tls` secret with a TLS certificate and key for using mTLS between NGINX Agent and Instance Manager. See the NGINX Instance Manager [docs](https://docs.nginx.com/nginx-management-suite/admin-guides/configuration/secure-traffic/#mutual-client-certificate-auth-setup-mtls) and the NGINX Agent [docs](https://docs.nginx.com/nginx-agent/configuration/encrypt-communication/) for more details. | "" | +|`nginxAgent.syslog.host` | Address for NGINX Agent to run syslog listener. | 127.0.0.1 | +|`nginxAgent.syslog.port` | Port for NGINX Agent to run syslog listener. | 1514 | +|`nginxAgent.customConfigMap` | The name of a custom ConfigMap to use instead of the one provided by default. | "" | ## Notes diff --git a/charts/nginx-ingress/templates/_helpers.tpl b/charts/nginx-ingress/templates/_helpers.tpl index cf13d42db2..d9fd31f2f5 100644 --- a/charts/nginx-ingress/templates/_helpers.tpl +++ b/charts/nginx-ingress/templates/_helpers.tpl @@ -290,8 +290,18 @@ Volumes for controller. {{- define "nginx-ingress.volumes" -}} {{- $volumesSet := "false" }} volumes: +{{- if eq (include "nginx-ingress.volumeEntries" .) "" -}} +{{ toYaml list | printf " %s" }} +{{- else }} +{{ include "nginx-ingress.volumeEntries" . }} +{{- end -}} +{{- end -}} + +{{/* +List of volumes for controller. +*/}} +{{- define "nginx-ingress.volumeEntries" -}} {{- if eq (include "nginx-ingress.readOnlyRootFilesystem" .) "true" }} -{{- $volumesSet = "true" }} - name: nginx-etc emptyDir: {} - name: nginx-cache @@ -302,41 +312,45 @@ volumes: emptyDir: {} {{- end }} {{- if .Values.controller.volumes }} -{{- $volumesSet = "true" }} {{ toYaml .Values.controller.volumes }} {{- end }} {{- if .Values.nginxAgent.enable -}} -{{- $volumesSet = "true" }} - name: agent-conf configMap: name: {{ include "nginx-ingress.agentConfigName" . }} - name: agent-dynamic emptyDir: {} -{{- if and .Values.nginxAgent.instanceManager.tls (ne (.Values.nginxAgent.instanceManager.tls.secret | default "") "") }} +{{- if and .Values.nginxAgent.instanceManager.tls (or (ne (.Values.nginxAgent.instanceManager.tls.secret | default "") "") (ne (.Values.nginxAgent.instanceManager.tls.caSecret | default "") "")) }} - name: nginx-agent-tls projected: sources: +{{- if ne .Values.nginxAgent.instanceManager.tls.secret "" }} - secret: name: {{ .Values.nginxAgent.instanceManager.tls.secret }} -{{- if .Values.nginxAgent.instanceManager.tls.caSecret }} +{{- end }} +{{- if ne .Values.nginxAgent.instanceManager.tls.caSecret "" }} - secret: name: {{ .Values.nginxAgent.instanceManager.tls.caSecret }} {{- end }} {{- end }} {{- end -}} -{{- if eq $volumesSet "false" -}} -{{ toYaml list | printf " %s" }} -{{- end -}} {{- end -}} {{/* Volume mounts for controller. */}} {{- define "nginx-ingress.volumeMounts" -}} -{{- $volumeMountSet := "false" }} +{{- $volumesSet := "false" }} volumeMounts: +{{- if eq (include "nginx-ingress.volumeMountEntries" .) "" -}} +{{ toYaml list | printf " %s" }} +{{- else }} +{{ include "nginx-ingress.volumeMountEntries" . }} +{{- end -}} +{{- end -}} + +{{- define "nginx-ingress.volumeMountEntries" -}} {{- if eq (include "nginx-ingress.readOnlyRootFilesystem" .) "true" }} -{{- $volumeMountSet = "true" }} - mountPath: /etc/nginx name: nginx-etc - mountPath: /var/cache/nginx @@ -347,25 +361,20 @@ volumeMounts: name: nginx-log {{- end }} {{- if .Values.controller.volumeMounts }} -{{- $volumeMountSet = "true" }} {{ toYaml .Values.controller.volumeMounts }} {{- end }} {{- if .Values.nginxAgent.enable -}} -{{- $volumeMountSet = "true" }} - name: agent-conf mountPath: /etc/nginx-agent/nginx-agent.conf subPath: nginx-agent.conf - name: agent-dynamic mountPath: /var/lib/nginx-agent -{{- if and .Values.nginxAgent.instanceManager.tls .Values.nginxAgent.instanceManager.tls.secret }} +{{- if and .Values.nginxAgent.instanceManager.tls (or (ne (.Values.nginxAgent.instanceManager.tls.secret | default "") "") (ne (.Values.nginxAgent.instanceManager.tls.caSecret | default "") "")) }} - name: nginx-agent-tls mountPath: /etc/ssl/nms readOnly: true {{- end }} {{- end -}} -{{- if eq $volumeMountSet "false" -}} -{{ toYaml list | printf " %s" }} -{{- end -}} {{- end -}} {{- define "nginx-ingress.agentConfiguration" -}} @@ -373,8 +382,12 @@ log: level: {{ .Values.nginxAgent.logLevel }} path: "" server: - host: {{ .Values.nginxAgent.instanceManager.host }} + host: {{ required ".Values.nginxAgent.instanceManager.host is required when setting .Values.nginxAgent.enable to true" .Values.nginxAgent.instanceManager.host }} grpcPort: {{ .Values.nginxAgent.instanceManager.grpcPort }} +{{- if ne (.Values.nginxAgent.instanceManager.sni | default "") "" }} + metrics: {{ .Values.nginxAgent.instanceManager.sni }} + command: {{ .Values.nginxAgent.instanceManager.sni }} +{{- end }} {{- if .Values.nginxAgent.instanceManager.tls }} tls: enable: {{ .Values.nginxAgent.instanceManager.tls.enable | default false }} @@ -391,12 +404,10 @@ features: - registration - nginx-counting - metrics-sender - - metrics-collection - dataplane-status extensions: - nginx-app-protect - nap-monitoring - - advanced-metrics nginx_app_protect: report_interval: 15s precompiled_publication: true @@ -405,14 +416,5 @@ nap_monitoring: processor_buffer_size: 50000 syslog_ip: "127.0.0.1" syslog_port: 1514 -advanced_metrics: - socket_path: /var/run/nginx-agent/advanced-metrics.sock - aggregation_period: 1s - publishing_period: 3s - table_sizes_limits: - staging_table_max_size: 1000 - staging_table_threshold: 1000 - priority_table_max_size: 1000 - priority_table_threshold: 1000 {{ end -}} diff --git a/charts/nginx-ingress/values.schema.json b/charts/nginx-ingress/values.schema.json index 09256c0d07..b210f36cc4 100644 --- a/charts/nginx-ingress/values.schema.json +++ b/charts/nginx-ingress/values.schema.json @@ -1830,6 +1830,14 @@ "my-instance-group" ] }, + "logLevel": { + "type": "string", + "default": "error", + "title": "Log level for NGINX Agent", + "examples": [ + "error" + ] + }, "instanceManager": { "type": "object", "default": {}, @@ -1854,6 +1862,14 @@ 443 ] }, + "sni": { + "type": "string", + "title": "Server Name Indication for Instance Manager", + "default": "", + "examples": [ + "nim.example.com" + ] + }, "tls": { "type": "object", "default": {}, @@ -1861,18 +1877,18 @@ "properties": { "enable": { "type": "boolean", - "default": "false", + "default": "true", "title": "enable TLS for Instance Manager connection" }, "secret": { "type": "string", "default": "", - "title": "Name of the kubernetes.io/tls secret used for TLS connection to Instance Manager" + "title": "kubernetes.io/tls secret with a TLS certificate and key for using mTLS between NGINX Agent and Instance Manager" }, "caSecret": { "type": "string", "default": "", - "title": "Name of the nginx.org/ca secret used for verification" + "title": "nginx.org/ca secret for verification of Instance Manager TLS" }, "skipVerify": { "type": "boolean", @@ -1882,6 +1898,40 @@ } } } + }, + "syslog": { + "type": "object", + "default": { + "host": "127.0.0.1", + "port": 1514 + }, + "title": "Syslog listener which NGINX Agent uses to accept messages from App Protect WAF", + "properties": { + "host": { + "type": "string", + "title": "Address for NGINX Agent to run syslog listener", + "default": "127.0.0.1", + "examples": [ + "127.0.0.1" + ] + }, + "port": { + "type": "integer", + "title": "Port for NGINX Agent to run syslog listener", + "default": 1514, + "examples": [ + 1514 + ] + } + } + }, + "customConfigMap": { + "type": "string", + "title": "The name of a custom ConfigMap to use instead of the one provided by default", + "default": "", + "examples": [ + "my-custom-configmap" + ] } } } diff --git a/charts/nginx-ingress/values.yaml b/charts/nginx-ingress/values.yaml index 3b4b08e434..6eef2f7479 100644 --- a/charts/nginx-ingress/values.yaml +++ b/charts/nginx-ingress/values.yaml @@ -559,21 +559,20 @@ nginxAgent: ## If nginxAgent.instanceGroup is not set the value of nginx-ingress.controller.fullname will be used instanceGroup: "" logLevel: "error" + ## Syslog listener which NGINX Agent uses to accept messages from App Protect WAF syslog: host: "127.0.0.1" port: 1514 instanceManager: host: "" grpcPort: 443 + sni: "" tls: - enabled: false + enabled: true skipVerify: false - ## The secret with a TLS certificate and key for the default HTTPS server. - ## The value must follow the following format: `/`. - ## Used as an alternative to specifying a certificate and key using `controller.defaultTLS.cert` and `controller.defaultTLS.key` parameters. - ## Note: Alternatively, omitting the default server secret completely will configure NGINX to reject TLS connections to the default server. - ## Format: / + ## kubernetes.io/tls secret with a TLS certificate and key for using mTLS between NGINX Agent and Instance Manager secret: "" + ## nginx.org/ca secret for verification of Instance Manager TLS caSecret: "" - ## The custom ConfigMap to use instead of the one provided by default + ## The name of a custom ConfigMap to use instead of the one provided by default customConfigMap: "" From 89fba2d3a20cfd5700fc700dc1f89fe3dc5b32d4 Mon Sep 17 00:00:00 2001 From: Eoin O'Shaughnessy Date: Fri, 8 Mar 2024 12:21:00 +0000 Subject: [PATCH 06/16] add new helm args to docs --- .../installing-nic/installation-with-helm.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/docs/content/installation/installing-nic/installation-with-helm.md b/docs/content/installation/installing-nic/installation-with-helm.md index 7eda35bb3b..fd18adcc0c 100644 --- a/docs/content/installation/installing-nic/installation-with-helm.md +++ b/docs/content/installation/installing-nic/installation-with-helm.md @@ -451,6 +451,19 @@ The following tables lists the configurable parameters of the NGINX Ingress Cont | **serviceNameOverride** | Used to prevent cloud load balancers from being replaced due to service name change during helm upgrades. | "" | | **nginxServiceMesh.enable** | Enable integration with NGINX Service Mesh. See the NGINX Service Mesh [docs](https://docs.nginx.com/nginx-service-mesh/tutorials/kic/deploy-with-kic/) for more details. Requires `controller.nginxplus`. | false | | **nginxServiceMesh.enableEgress** | Enable NGINX Service Mesh workloads to route egress traffic through the Ingress Controller. See the NGINX Service Mesh [docs](https://docs.nginx.com/nginx-service-mesh/tutorials/kic/deploy-with-kic/#enabling-egress) for more details. Requires `nginxServiceMesh.enable`. | false | +|**nginxAgent.enable** | Enable NGINX Agent to enable Security Monitoring integration with App Protect WAF module. Requires `controller.appprotect.enable`. | false | +|**nginxAgent.instanceGroup** | Set a custom Instance Group name which the deployment will be shown as when connected to NGINX Instance Manager. `nginx-ingress.controller.fullname` will be used if not set. | "" | +|**nginxAgent.logLevel** | Log level for NGINX Agent. | "error | +|**nginxAgent.instanceManager.host** | FQDN or IP for connecting to Ingress Controller. Required when `nginxAgent.enable` is set to `true` | "" | +|**nginxAgent.instanceManager.grpcPort** | Port for connecting to Ingress Controller. | 443 | +|**nginxAgent.instanceManager.sni** | Server Name Indication for Instance Manager. See the NGINX Agent [docs](https://docs.nginx.com/nginx-agent/configuration/encrypt-communication/) for more details. | "" | +|**nginxAgent.instanceManager.tls.enable** | Enable TLS for Instance Manager connection. | true | +|**nginxAgent.instanceManager.tls.skipVerify** | Skip certification verification for Instance Manager connection. | false | +|**nginxAgent.instanceManager.tls.caSecret** | Name of `nginx.org/ca` secret used for verification of Instance Manager TLS. | "" | +|**nginxAgent.instanceManager.tls.secret** | Name of `kubernetes.io/tls` secret with a TLS certificate and key for using mTLS between NGINX Agent and Instance Manager. See the NGINX Instance Manager [docs](https://docs.nginx.com/nginx-management-suite/admin-guides/configuration/secure-traffic/#mutual-client-certificate-auth-setup-mtls) and the NGINX Agent [docs](https://docs.nginx.com/nginx-agent/configuration/encrypt-communication/) for more details. | "" | +|**nginxAgent.syslog.host** | Address for NGINX Agent to run syslog listener. | 127.0.0.1 | +|**nginxAgent.syslog.port** | Port for NGINX Agent to run syslog listener. | 1514 | +|**nginxAgent.customConfigMap** | The name of a custom ConfigMap to use instead of the one provided by default. | "" | {{}} {{< /table >}} From 17f1d24993bd929f07bb40f4c22b908cbf62b57d Mon Sep 17 00:00:00 2001 From: Eoin O'Shaughnessy Date: Tue, 12 Mar 2024 12:05:05 +0000 Subject: [PATCH 07/16] CLI docs and helper blank line fix --- charts/nginx-ingress/templates/_helpers.tpl | 4 ++-- .../global-configuration/command-line-arguments.md | 14 ++++++++++++++ internal/nginx/manager.go | 5 ++++- 3 files changed, 20 insertions(+), 3 deletions(-) diff --git a/charts/nginx-ingress/templates/_helpers.tpl b/charts/nginx-ingress/templates/_helpers.tpl index d9fd31f2f5..85c13c3c91 100644 --- a/charts/nginx-ingress/templates/_helpers.tpl +++ b/charts/nginx-ingress/templates/_helpers.tpl @@ -314,7 +314,7 @@ List of volumes for controller. {{- if .Values.controller.volumes }} {{ toYaml .Values.controller.volumes }} {{- end }} -{{- if .Values.nginxAgent.enable -}} +{{- if .Values.nginxAgent.enable }} - name: agent-conf configMap: name: {{ include "nginx-ingress.agentConfigName" . }} @@ -363,7 +363,7 @@ volumeMounts: {{- if .Values.controller.volumeMounts }} {{ toYaml .Values.controller.volumeMounts }} {{- end }} -{{- if .Values.nginxAgent.enable -}} +{{- if .Values.nginxAgent.enable }} - name: agent-conf mountPath: /etc/nginx-agent/nginx-agent.conf subPath: nginx-agent.conf diff --git a/docs/content/configuration/global-configuration/command-line-arguments.md b/docs/content/configuration/global-configuration/command-line-arguments.md index 93e14eea02..de3f47cb4f 100644 --- a/docs/content/configuration/global-configuration/command-line-arguments.md +++ b/docs/content/configuration/global-configuration/command-line-arguments.md @@ -535,3 +535,17 @@ Used to activate or deactivate lazy loading for SSL Certificates. The default value is `true`. + +### -agent + +Enable NGINX Agent which can used with `-enable-app-protect` to send events to Security Monitoring. + +The default value is `false`. + + + +### -agent-instance-group + +Specify the instance group name to use for the NGINX Ingress Controller deployment when using `-agent`. + + diff --git a/internal/nginx/manager.go b/internal/nginx/manager.go index 877fa4af63..f23918856c 100644 --- a/internal/nginx/manager.go +++ b/internal/nginx/manager.go @@ -570,7 +570,10 @@ func (lm *LocalManager) AppProtectDosAgentStart(apdaDone chan error, debug bool, // AgentStart starts the AppProtect plugin and sets AppProtect log level. func (lm *LocalManager) AgentStart(agentDone chan error, instanceGroup string) { glog.V(3).Info("Starting Agent") - args := []string{"--instance-group", instanceGroup} + args := []string{} + if len(instanceGroup) > 0 { + args = append(args, "--instance-group", instanceGroup) + } cmd := exec.Command(agentPath, args...) // #nosec G204 cmd.Stdout = os.Stdout From 87c50d3b861e51243d4326c18faaee5a35d14703 Mon Sep 17 00:00:00 2001 From: Eoin O'Shaughnessy Date: Tue, 12 Mar 2024 13:43:06 +0000 Subject: [PATCH 08/16] add examples for security monitoring --- .../security-monitoring/README.md | 105 ++++++++++++++++ .../security-monitoring/ap-apple-uds.yaml | 18 +++ .../ap-dataguard-alarm-policy.yaml | 34 ++++++ .../security-monitoring/ap-logconf.yaml | 16 +++ .../security-monitoring/virtual-server.yaml | 16 +++ .../security-monitoring/waf.yaml | 12 ++ .../security-monitoring/webapp.yaml | 32 +++++ .../security-monitoring/README.md | 114 ++++++++++++++++++ .../security-monitoring/ap-apple-uds.yaml | 18 +++ .../ap-dataguard-alarm-policy.yaml | 34 ++++++ .../security-monitoring/ap-logconf.yaml | 16 +++ .../security-monitoring/cafe-ingress.yaml | 34 ++++++ .../security-monitoring/cafe-secret.yaml | 8 ++ .../security-monitoring/cafe.yaml | 66 ++++++++++ 14 files changed, 523 insertions(+) create mode 100644 examples/custom-resources/security-monitoring/README.md create mode 100644 examples/custom-resources/security-monitoring/ap-apple-uds.yaml create mode 100644 examples/custom-resources/security-monitoring/ap-dataguard-alarm-policy.yaml create mode 100644 examples/custom-resources/security-monitoring/ap-logconf.yaml create mode 100644 examples/custom-resources/security-monitoring/virtual-server.yaml create mode 100644 examples/custom-resources/security-monitoring/waf.yaml create mode 100644 examples/custom-resources/security-monitoring/webapp.yaml create mode 100644 examples/ingress-resources/security-monitoring/README.md create mode 100644 examples/ingress-resources/security-monitoring/ap-apple-uds.yaml create mode 100644 examples/ingress-resources/security-monitoring/ap-dataguard-alarm-policy.yaml create mode 100644 examples/ingress-resources/security-monitoring/ap-logconf.yaml create mode 100644 examples/ingress-resources/security-monitoring/cafe-ingress.yaml create mode 100644 examples/ingress-resources/security-monitoring/cafe-secret.yaml create mode 100644 examples/ingress-resources/security-monitoring/cafe.yaml diff --git a/examples/custom-resources/security-monitoring/README.md b/examples/custom-resources/security-monitoring/README.md new file mode 100644 index 0000000000..6f7d53aaa8 --- /dev/null +++ b/examples/custom-resources/security-monitoring/README.md @@ -0,0 +1,105 @@ +# WAF + +In this example we deploy the NGINX Plus Ingress Controller with [NGINX App +Protect](https://www.nginx.com/products/nginx-app-protect/) and [NGINX Agent](https://docs.nginx.com/nginx-agent/overview/) in order to integrate with [NGINX Management Suite Security Monitoring](https://docs.nginx.com/nginx-management-suite/security/). We also deploy a simple web application and then configure load balancing and WAF protection for that application using the VirtualServer resource. We configure logging for NGINX App Protect to send logs to the NGINX Agent syslog listener which is then sent to the Security Monitoring dashboard in NGINX Instance Manager. + +## Prerequisites + +1. Follow the installation [instructions](https://docs.nginx.com/nginx-ingress-controller/installation) to deploy the + Ingress Controller with NGINX App Protect and NGINX Agent. Configure NGINX Agent to connect to a deployment of NGINX Instance Manager with Security Monitoring, and verify that your NGINX Ingress Controller deployment is online in NGINX Instance Manager. + +1. Save the public IP address of the Ingress Controller into a shell variable: + + ```console + IC_IP=XXX.YYY.ZZZ.III + ``` + +1. Save the HTTP port of the Ingress Controller into a shell variable: + + ```console + IC_HTTP_PORT= + ``` + +## Step 1. Deploy a Web Application + +Create the application deployment and service: + +```console +kubectl apply -f webapp.yaml +``` + +## Step 2 - Deploy the AP Policy + +1. Create the User Defined Signature, App Protect policy and log configuration: + + ```console + kubectl apply -f ap-apple-uds.yaml + kubectl apply -f ap-dataguard-alarm-policy.yaml + kubectl apply -f ap-logconf.yaml + ``` + +Note the log configuration in `ap-logconf.yaml` is a specific format required by NGINX Agent for integration with Security Monitoring. + +## Step 3 - Deploy the WAF Policy + +1. Create the WAF policy + + ```console + kubectl apply -f waf.yaml + ``` + +Note the App Protect configuration settings in the Policy resource. They enable WAF protection by configuring App +Protect with the policy and log configuration created in the previous step. + +## Step 4 - Configure Load Balancing + +1. Create the VirtualServer Resource: + + ```console + kubectl apply -f virtual-server.yaml + ``` + +Note that the VirtualServer references the policy `waf-policy` created in Step 3. + +## Step 5 - Test the Application + +To access the application, curl the coffee and the tea services. We'll use the --resolve option to set the Host header +of a request with `webapp.example.com` + +1. Send a request to the application: + + ```console + curl --resolve webapp.example.com:$IC_HTTP_PORT:$IC_IP http://webapp.example.com:$IC_HTTP_PORT/ + ``` + + ```text + Server address: 10.12.0.18:80 + Server name: webapp-7586895968-r26zn + ... + ``` + +1. Now, let's try to send a request with a suspicious URL: + + ```console + curl --resolve webapp.example.com:$IC_HTTP_PORT:$IC_IP "http://webapp.example.com:$IC_HTTP_PORT/