Skip to content

Commit 2dd6e3d

Browse files
sjbermansjberman
committed
Support NginxProxy CRD and global tracing settings
Problem: As a user of NGF I want to set the collection point for my traces for my installation of NGF So that I can ensure all my traces are sent to the same collection platform. Solution: Implement the NginxProxy CRD which contains the fields required to configure the collection point for tracing. This resource is attached to the GatewayClass. If the resource is not found, a condition will be set on the GatewayClass to indicate this. The GatewayClass will continue to be Accepted even if the parametersRef is invalid. This configuration sets the `http` context-level otel directives. The otel module is loaded conditionally based on the existence of this configuration. Note: tracing is not enabled by this configuration, this only sets high level options. #1828 is required to actually enable tracing on a per-route basis.
1 parent e082924 commit 2dd6e3d

38 files changed

+770
-40
lines changed

‎build/Dockerfile.nginx

+8-1
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,18 @@
11
# syntax=docker/dockerfile:1.6
2+
FROM scratch as nginx-files
3+
4+
# the following links can be replaced with local files if needed, i.e. ADD --chown=101:1001 <local_file> <container_file>
5+
ADD --link --chown=101:1001 https://cs.nginx.com/static/keys/nginx_signing.rsa.pub nginx_signing.rsa.pub
6+
27
FROM nginx:1.25.5-alpine
38

49
ARG NJS_DIR
510
ARG NGINX_CONF_DIR
611
ARG BUILD_AGENT
712

8-
RUN apk add --no-cache libcap \
13+
RUN --mount=type=bind,from=nginx-files,src=nginx_signing.rsa.pub,target=/etc/apk/keys/nginx_signing.rsa.pub \
14+
printf "%s\n" "http://nginx.org/packages/mainline/alpine/v$(egrep -o '^[0-9]+\.[0-9]+' /etc/alpine-release)/main" >> /etc/apk/repositories \
15+
&& apk add --no-cache libcap nginx-module-otel \
916
&& mkdir -p /var/lib/nginx /usr/lib/nginx/modules \
1017
&& setcap 'cap_net_bind_service=+ep' /usr/sbin/nginx \
1118
&& setcap -v 'cap_net_bind_service=+ep' /usr/sbin/nginx \

‎build/Dockerfile.nginxplus

+1-1
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ RUN --mount=type=secret,id=nginx-repo.crt,dst=/etc/apk/cert.pem,mode=0644 \
1818
addgroup -g 1001 -S nginx \
1919
&& adduser -S -D -H -u 101 -h /var/cache/nginx -s /sbin/nologin -G nginx -g nginx nginx \
2020
&& printf "%s\n" "https://pkgs.nginx.com/plus/${NGINX_PLUS_VERSION}/alpine/v$(grep -E -o '^[0-9]+\.[0-9]+' /etc/alpine-release)/main" >> /etc/apk/repositories \
21-
&& apk add --no-cache nginx-plus nginx-plus-module-njs libcap \
21+
&& apk add --no-cache nginx-plus nginx-plus-module-njs nginx-plus-module-otel libcap \
2222
&& mkdir -p /var/lib/nginx /usr/lib/nginx/modules \
2323
&& setcap 'cap_net_bind_service=+ep' /usr/sbin/nginx \
2424
&& setcap -v 'cap_net_bind_service=+ep' /usr/sbin/nginx \

‎charts/nginx-gateway-fabric/README.md

+1-1
Original file line numberDiff line numberDiff line change
@@ -224,7 +224,7 @@ To uninstall/delete the release `ngf`:
224224
```shell
225225
helm uninstall ngf -n nginx-gateway
226226
kubectl delete ns nginx-gateway
227-
kubectl delete crd nginxgateways.gateway.nginx.org
227+
for crd in `kubectl get crds -oname | grep gateway.nginx.org | awk -F / '{ print $2 }'`; do kubectl delete crd $crd; done
228228
```
229229

230230
These commands remove all the Kubernetes components associated with the release and deletes the release.

‎charts/nginx-gateway-fabric/templates/deployment.yaml

+6
Original file line numberDiff line numberDiff line change
@@ -118,6 +118,8 @@ spec:
118118
volumeMounts:
119119
- name: nginx-conf
120120
mountPath: /etc/nginx/conf.d
121+
- name: nginx-includes
122+
mountPath: /etc/nginx/includes
121123
- name: nginx-secrets
122124
mountPath: /etc/nginx/secrets
123125
- name: nginx-run
@@ -149,6 +151,8 @@ spec:
149151
volumeMounts:
150152
- name: nginx-conf
151153
mountPath: /etc/nginx/conf.d
154+
- name: nginx-includes
155+
mountPath: /etc/nginx/includes
152156
- name: nginx-secrets
153157
mountPath: /etc/nginx/secrets
154158
- name: nginx-run
@@ -181,6 +185,8 @@ spec:
181185
volumes:
182186
- name: nginx-conf
183187
emptyDir: {}
188+
- name: nginx-includes
189+
emptyDir: {}
184190
- name: nginx-secrets
185191
emptyDir: {}
186192
- name: nginx-run

‎charts/nginx-gateway-fabric/templates/rbac.yaml

+2-1
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ metadata:
1010
{{- if or .Values.serviceAccount.imagePullSecret .Values.serviceAccount.imagePullSecrets }}
1111
imagePullSecrets:
1212
{{- if .Values.serviceAccount.imagePullSecret }}
13-
- name: {{ .Values.serviceAccount.imagePullSecret}}
13+
- name: {{ .Values.serviceAccount.imagePullSecret }}
1414
{{- end }}
1515
{{- if .Values.serviceAccount.imagePullSecrets }}
1616
{{- range .Values.serviceAccount.imagePullSecrets }}
@@ -111,6 +111,7 @@ rules:
111111
- gateway.nginx.org
112112
resources:
113113
- nginxgateways
114+
- nginxproxies
114115
verbs:
115116
- get
116117
- list

‎conformance/provisioner/static-deployment.yaml

+6
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,8 @@ spec:
7070
volumeMounts:
7171
- name: nginx-conf
7272
mountPath: /etc/nginx/conf.d
73+
- name: nginx-includes
74+
mountPath: /etc/nginx/includes
7375
- name: nginx-secrets
7476
mountPath: /etc/nginx/secrets
7577
- name: nginx-run
@@ -94,6 +96,8 @@ spec:
9496
volumeMounts:
9597
- name: nginx-conf
9698
mountPath: /etc/nginx/conf.d
99+
- name: nginx-includes
100+
mountPath: /etc/nginx/includes
97101
- name: nginx-secrets
98102
mountPath: /etc/nginx/secrets
99103
- name: nginx-run
@@ -111,6 +115,8 @@ spec:
111115
volumes:
112116
- name: nginx-conf
113117
emptyDir: {}
118+
- name: nginx-includes
119+
emptyDir: {}
114120
- name: nginx-secrets
115121
emptyDir: {}
116122
- name: nginx-run

‎deploy/manifests/nginx-gateway-experimental.yaml

+7
Original file line numberDiff line numberDiff line change
@@ -93,6 +93,7 @@ rules:
9393
- gateway.nginx.org
9494
resources:
9595
- nginxgateways
96+
- nginxproxies
9697
verbs:
9798
- get
9899
- list
@@ -213,6 +214,8 @@ spec:
213214
volumeMounts:
214215
- name: nginx-conf
215216
mountPath: /etc/nginx/conf.d
217+
- name: nginx-includes
218+
mountPath: /etc/nginx/includes
216219
- name: nginx-secrets
217220
mountPath: /etc/nginx/secrets
218221
- name: nginx-run
@@ -237,6 +240,8 @@ spec:
237240
volumeMounts:
238241
- name: nginx-conf
239242
mountPath: /etc/nginx/conf.d
243+
- name: nginx-includes
244+
mountPath: /etc/nginx/includes
240245
- name: nginx-secrets
241246
mountPath: /etc/nginx/secrets
242247
- name: nginx-run
@@ -254,6 +259,8 @@ spec:
254259
volumes:
255260
- name: nginx-conf
256261
emptyDir: {}
262+
- name: nginx-includes
263+
emptyDir: {}
257264
- name: nginx-secrets
258265
emptyDir: {}
259266
- name: nginx-run

‎deploy/manifests/nginx-gateway.yaml

+7
Original file line numberDiff line numberDiff line change
@@ -90,6 +90,7 @@ rules:
9090
- gateway.nginx.org
9191
resources:
9292
- nginxgateways
93+
- nginxproxies
9394
verbs:
9495
- get
9596
- list
@@ -209,6 +210,8 @@ spec:
209210
volumeMounts:
210211
- name: nginx-conf
211212
mountPath: /etc/nginx/conf.d
213+
- name: nginx-includes
214+
mountPath: /etc/nginx/includes
212215
- name: nginx-secrets
213216
mountPath: /etc/nginx/secrets
214217
- name: nginx-run
@@ -233,6 +236,8 @@ spec:
233236
volumeMounts:
234237
- name: nginx-conf
235238
mountPath: /etc/nginx/conf.d
239+
- name: nginx-includes
240+
mountPath: /etc/nginx/includes
236241
- name: nginx-secrets
237242
mountPath: /etc/nginx/secrets
238243
- name: nginx-run
@@ -250,6 +255,8 @@ spec:
250255
volumes:
251256
- name: nginx-conf
252257
emptyDir: {}
258+
- name: nginx-includes
259+
emptyDir: {}
253260
- name: nginx-secrets
254261
emptyDir: {}
255262
- name: nginx-run

‎deploy/manifests/nginx-plus-gateway-experimental.yaml

+7
Original file line numberDiff line numberDiff line change
@@ -99,6 +99,7 @@ rules:
9999
- gateway.nginx.org
100100
resources:
101101
- nginxgateways
102+
- nginxproxies
102103
verbs:
103104
- get
104105
- list
@@ -220,6 +221,8 @@ spec:
220221
volumeMounts:
221222
- name: nginx-conf
222223
mountPath: /etc/nginx/conf.d
224+
- name: nginx-includes
225+
mountPath: /etc/nginx/includes
223226
- name: nginx-secrets
224227
mountPath: /etc/nginx/secrets
225228
- name: nginx-run
@@ -244,6 +247,8 @@ spec:
244247
volumeMounts:
245248
- name: nginx-conf
246249
mountPath: /etc/nginx/conf.d
250+
- name: nginx-includes
251+
mountPath: /etc/nginx/includes
247252
- name: nginx-secrets
248253
mountPath: /etc/nginx/secrets
249254
- name: nginx-run
@@ -261,6 +266,8 @@ spec:
261266
volumes:
262267
- name: nginx-conf
263268
emptyDir: {}
269+
- name: nginx-includes
270+
emptyDir: {}
264271
- name: nginx-secrets
265272
emptyDir: {}
266273
- name: nginx-run

‎deploy/manifests/nginx-plus-gateway.yaml

+7
Original file line numberDiff line numberDiff line change
@@ -96,6 +96,7 @@ rules:
9696
- gateway.nginx.org
9797
resources:
9898
- nginxgateways
99+
- nginxproxies
99100
verbs:
100101
- get
101102
- list
@@ -216,6 +217,8 @@ spec:
216217
volumeMounts:
217218
- name: nginx-conf
218219
mountPath: /etc/nginx/conf.d
220+
- name: nginx-includes
221+
mountPath: /etc/nginx/includes
219222
- name: nginx-secrets
220223
mountPath: /etc/nginx/secrets
221224
- name: nginx-run
@@ -240,6 +243,8 @@ spec:
240243
volumeMounts:
241244
- name: nginx-conf
242245
mountPath: /etc/nginx/conf.d
246+
- name: nginx-includes
247+
mountPath: /etc/nginx/includes
243248
- name: nginx-secrets
244249
mountPath: /etc/nginx/secrets
245250
- name: nginx-run
@@ -257,6 +262,8 @@ spec:
257262
volumes:
258263
- name: nginx-conf
259264
emptyDir: {}
265+
- name: nginx-includes
266+
emptyDir: {}
260267
- name: nginx-secrets
261268
emptyDir: {}
262269
- name: nginx-run

‎docs/proposals/gateway-settings.md

+11-2
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
# Enhancement Proposal-1775: Gateway Settings
22

33
- Issue: https://github.com/nginxinc/nginx-gateway-fabric/issues/1775
4-
- Status: Implementable
4+
- Status: Completed
55

66
## Summary
77

@@ -93,7 +93,7 @@ type Telemetry struct {
9393
// SpanAttributes are custom key/value attributes that are added to each span.
9494
//
9595
// +optional
96-
SpanAttributes map[string]string `json:"spanAttributes,omitempty"`
96+
SpanAttributes []SpanAttribute `json:"spanAttributes,omitempty"`
9797
}
9898

9999
// TelemetryExporter specifies OpenTelemetry export parameters.
@@ -122,6 +122,15 @@ type TelemetryExporter struct {
122122
// The format is a subset of the syntax parsed by Golang time.ParseDuration.
123123
// Examples: 1h, 12m, 30s, 150ms.
124124
type Duration string
125+
126+
// SpanAttribute is a key value pair to be added to a tracing span.
127+
type SpanAttribute struct {
128+
// Key is the key for a span attribute.
129+
Key string `json:"key"`
130+
131+
// Value is the value for a span attribute.
132+
Value string `json:"value"`
133+
}
125134
```
126135

127136
### Status

‎internal/mode/static/manager.go

+7
Original file line numberDiff line numberDiff line change
@@ -414,6 +414,12 @@ func registerControllers(
414414
),
415415
},
416416
},
417+
{
418+
objectType: &ngfAPI.NginxProxy{},
419+
options: []controller.Option{
420+
controller.WithK8sPredicate(k8spredicate.GenerationChangedPredicate{}),
421+
},
422+
},
417423
}
418424

419425
if cfg.ExperimentalFeatures {
@@ -592,6 +598,7 @@ func prepareFirstEventBatchPreparerArgs(
592598
&discoveryV1.EndpointSliceList{},
593599
&gatewayv1.HTTPRouteList{},
594600
&gatewayv1beta1.ReferenceGrantList{},
601+
&ngfAPI.NginxProxyList{},
595602
partialObjectMetadataList,
596603
}
597604

‎internal/mode/static/manager_test.go

+4
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ import (
1616
gatewayv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2"
1717
gatewayv1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1"
1818

19+
ngfAPI "github.com/nginxinc/nginx-gateway-fabric/apis/v1alpha1"
1920
"github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/config"
2021
)
2122

@@ -52,6 +53,7 @@ func TestPrepareFirstEventBatchPreparerArgs(t *testing.T) {
5253
&gatewayv1.HTTPRouteList{},
5354
&gatewayv1.GatewayList{},
5455
&gatewayv1beta1.ReferenceGrantList{},
56+
&ngfAPI.NginxProxyList{},
5557
partialObjectMetadataList,
5658
},
5759
},
@@ -72,6 +74,7 @@ func TestPrepareFirstEventBatchPreparerArgs(t *testing.T) {
7274
&discoveryV1.EndpointSliceList{},
7375
&gatewayv1.HTTPRouteList{},
7476
&gatewayv1beta1.ReferenceGrantList{},
77+
&ngfAPI.NginxProxyList{},
7578
partialObjectMetadataList,
7679
},
7780
},
@@ -93,6 +96,7 @@ func TestPrepareFirstEventBatchPreparerArgs(t *testing.T) {
9396
&discoveryV1.EndpointSliceList{},
9497
&gatewayv1.HTTPRouteList{},
9598
&gatewayv1beta1.ReferenceGrantList{},
99+
&ngfAPI.NginxProxyList{},
96100
partialObjectMetadataList,
97101
&gatewayv1alpha2.BackendTLSPolicyList{},
98102
},

‎internal/mode/static/nginx/conf/nginx-plus.conf

+1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
load_module /usr/lib/nginx/modules/ngx_http_js_module.so;
2+
include /etc/nginx/includes/*.conf;
23

34
worker_processes auto;
45

‎internal/mode/static/nginx/conf/nginx.conf

+1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
load_module /usr/lib/nginx/modules/ngx_http_js_module.so;
2+
include /etc/nginx/includes/*.conf;
23

34
worker_processes auto;
45

‎internal/mode/static/nginx/config/generator.go

+20
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ const (
1515

1616
// httpFolder is the folder where NGINX HTTP configuration files are stored.
1717
httpFolder = configFolder + "/conf.d"
18+
1819
// secretsFolder is the folder where secrets (like TLS certs/keys) are stored.
1920
secretsFolder = configFolder + "/secrets"
2021

@@ -26,6 +27,9 @@ const (
2627

2728
// httpMatchVarsFile is the path to the http_match pairs configuration file.
2829
httpMatchVarsFile = httpFolder + "/matches.json"
30+
31+
// loadModulesFile is the path to the file containing any load_module directives.
32+
loadModulesFile = configFolder + "/includes/load_modules.conf"
2933
)
3034

3135
// ConfigFolders is a list of folders where NGINX configuration files are stored.
@@ -82,6 +86,8 @@ func (g GeneratorImpl) Generate(conf dataplane.Configuration) []file.File {
8286
files = append(files, generateCertBundle(id, bundle))
8387
}
8488

89+
files = append(files, generateLoadModulesConf(conf))
90+
8591
return files
8692
}
8793

@@ -142,6 +148,7 @@ func (g GeneratorImpl) getExecuteFuncs() []executeFunc {
142148
g.executeUpstreams,
143149
executeSplitClients,
144150
executeMaps,
151+
executeTelemetry,
145152
}
146153
}
147154

@@ -155,3 +162,16 @@ func generateConfigVersion(configVersion int) file.File {
155162
Type: file.TypeRegular,
156163
}
157164
}
165+
166+
func generateLoadModulesConf(conf dataplane.Configuration) file.File {
167+
var c []byte
168+
if conf.Telemetry.Endpoint != "" {
169+
c = []byte("load_module modules/ngx_otel_module.so;")
170+
}
171+
172+
return file.File{
173+
Content: c,
174+
Path: loadModulesFile,
175+
Type: file.TypeRegular,
176+
}
177+
}

0 commit comments

Comments
 (0)