From 3eae59f98d71e6c8960efbb1d453017af9264575 Mon Sep 17 00:00:00 2001 From: Ben Hardill Date: Thu, 24 Oct 2024 09:56:35 +0100 Subject: [PATCH 01/15] stash work in progress --- helm/flowforge/templates/configmap.yaml | 4 + helm/flowforge/templates/emqx.yaml | 228 ++++++++++++++++++++++++ helm/flowforge/values.yaml | 5 + 3 files changed, 237 insertions(+) create mode 100644 helm/flowforge/templates/emqx.yaml diff --git a/helm/flowforge/templates/configmap.yaml b/helm/flowforge/templates/configmap.yaml index 5b9c437e..1205e69c 100644 --- a/helm/flowforge/templates/configmap.yaml +++ b/helm/flowforge/templates/configmap.yaml @@ -160,7 +160,11 @@ data: {{ if .Values.forge.broker.url -}} url: {{ .Values.forge.broker.url }} {{ else -}} + {{ if .Values.forge.broker.emqx }} + url: mqtt://emqx-listeners.{{ .Release.Namespace }}:1883 + {{ else -}} url: mqtt://flowforge-broker.{{ .Release.Namespace }}:1883 + {{end -}} {{ end -}} {{ if .Values.forge.broker.public_url -}} public_url: {{ .Values.forge.broker.public_url }} diff --git a/helm/flowforge/templates/emqx.yaml b/helm/flowforge/templates/emqx.yaml new file mode 100644 index 00000000..e25d3fbd --- /dev/null +++ b/helm/flowforge/templates/emqx.yaml @@ -0,0 +1,228 @@ +{{- if .Values.forge.broker.emqx.enabled -}} +apiVersion: apps.emqx.io/v2beta1 +kind: EMQX +metadata: + name: emqx +spec: + image: emqx:5 + imagePullPolicy: IfNotPresent + coreTemplate: + spec: + volumeClaimTemplates: + {{ if .Values.forge.broker.emqx.storageClassName}} + storageClassName: {{ .Values.forge.broker.emqx.storageClassName }} + {{ end}} + resources: + requests: + storage: 10Gi + accessModes: + - ReadWriteOnce + replicaCount: 1 + config: + data: | + authentication = [ + { + backend = http + body = { + clientId = "${clientid}" + username = "${username}" + password = "${password}" + } + enable = true + connect_timeout = "15s" + enable_pipelining = 100 + headers { + content-type = "application/json" + } + mechanism = password_based + method = post + pool_size = 8 + request_timeout = "8s" + ssl { + enable = false + } + url = "http://forge.default/api/comms/v2/auth" + } + ] + authorization { + cache { + enable = true + excludes = [] + max_size = 32 + ttl = "1m" + } + deny_action = ignore + no_match = allow + sources = [ + { + enable = true + enable_pipelining = 100 + connect_timeout = "15s" + request_timeout = "30s" + pool_size = 8 + body { + action = "${action}" + topic = "${topic}" + username = "${username}" + } + headers { + content-type = "application/json" + } + method = post + type = http + ssl { + enable = false + } + url = "http://forge.default/api/comms/v2/acls" + } + ] + } + listeners { + tcp { + default { + bind = "0.0.0.0:1883" + access_rules = [ + "allow all" + ] + enable = true + enable_authn = true + mountpoint = "${client_attrs.team}" + max_connections = infinity + acceptors = 16 + proxy_protocol = false + proxy_protocol_timeout = 3s + tcp_options { + backlog = 1024 + send_timeout = 15s + recbuf = 2KB + sndbuf = 4KB + buffer = 4KB + high_watermark = 1MB + nodelay = true + reuseaddr = true + keepalive = "none" + } + } + } + ssl { + default { + enable = false + } + } + wss { + default { + enable = false + } + } + ws { + default { + bind = "0.0.0.0:8080" + access_rules = [ + "allow all" + ] + enable = true + enable_authn = true + mountpoint = "${client_attrs.team}" + max_connections = infinity + proxy_protocol = false + proxy_protocol_timeout = 3s + tcp_options { + backlog = 1024 + send_timeout = 15s + recbuf = 2KB + sndbuf = 4KB + buffer = 4KB + high_watermark = 1MB + nodelay = true + reuseaddr = true + keepalive = "none" + } + websocket { + mqtt_path = "/" + allow_origin_absence = true + check_origin_enable = false + fail_if_no_subprotocol = true + supported_subprotocols = "mqtt, mqtt-v3, mqtt-v3.1.1 mqtt-v5" + mqtt_piggyback = multiple + compress = false + idle_timeout = 7200s + max_frame_size = infinity + proxy_address_header = "x-forwarded-for" + proxy_port_header = "x-forwarded-port" + } + } + } + } + dashboard { + default_password = topSecret + } + api_key { + bootstrap_file = "/mounted/config/api-keys" + } + coreTemplate: + spec: + volumeClaimTemaples: + resources: + requests: + storage: 5Gi + accessModes: + - ReadWriteOnce + extraVolumes: + - name: config + secret: + name: emqx-config + extraVolumeMounts: + - name: config + mountPath: /mounted/config + listenersServiceTemplate: + spec: + type: ClusterIP + dashboardServiceTemplate: + spec: + type: ClusterIP +--- +apiVersion: v1 +kind: Secret +metadata: + name: emqx-config + namespace: {{ .Release.Namespace }} +stringData: + api-keys: | + flowforge:verySecret:administrator + +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: flowforge-broker + labels: + {{- include "forge.brokerSelectorLabels" . | nindent 4 }} + annotations: + {{- if .Values.ingress.certManagerIssuer }} + cert-manager.io/cluster-issuer: {{ $.Values.ingress.certManagerIssuer }} + {{- end }} + {{- if and .Values.forge.broker.enabled .Values.forge.broker.ingress (hasKey .Values.forge.broker.ingress "annotations") }} +{{ toYaml .Values.forge.broker.ingress.annotations | replace "{{ instanceHost }}" "{{ include forge.brokerDomain . }}" | replace "{{ serviceName }}" "flowforge-broker" | indent 4 }} + {{- end }} +spec: + {{- if $.Values.ingress.className }} + ingressClassName: {{ $.Values.ingress.className }} + {{- end }} + rules: + - host: {{ include "forge.brokerDomain" . }} + http: + paths: + - pathType: Prefix + path: / + backend: + service: + name: emqx-listeners + port: + number: 8080 + {{- if .Values.ingress.certManagerIssuer }} + tls: + - hosts: + - {{ include "forge.brokerDomain" . }} + secretName: {{ include "forge.brokerDomain" . }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/helm/flowforge/values.yaml b/helm/flowforge/values.yaml index dd53181c..b6031bf7 100644 --- a/helm/flowforge/values.yaml +++ b/helm/flowforge/values.yaml @@ -47,6 +47,11 @@ forge: podLabels: {} tolerations: [] ingress.annotations: {} + emqx: + enabled: false + persistence: + storageClassName: '' + persistentStorage: enabled: false From 22f969474e0128c430c064ac5fd36568e732bd3a Mon Sep 17 00:00:00 2001 From: Ben Hardill Date: Fri, 25 Oct 2024 14:32:48 +0100 Subject: [PATCH 02/15] stash changes --- helm/flowforge/templates/configmap.yaml | 2 +- helm/flowforge/templates/emqx.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/helm/flowforge/templates/configmap.yaml b/helm/flowforge/templates/configmap.yaml index 1205e69c..19e17338 100644 --- a/helm/flowforge/templates/configmap.yaml +++ b/helm/flowforge/templates/configmap.yaml @@ -160,7 +160,7 @@ data: {{ if .Values.forge.broker.url -}} url: {{ .Values.forge.broker.url }} {{ else -}} - {{ if .Values.forge.broker.emqx }} + {{ if .Values.forge.broker.teamBroker }} url: mqtt://emqx-listeners.{{ .Release.Namespace }}:1883 {{ else -}} url: mqtt://flowforge-broker.{{ .Release.Namespace }}:1883 diff --git a/helm/flowforge/templates/emqx.yaml b/helm/flowforge/templates/emqx.yaml index e25d3fbd..947b21e7 100644 --- a/helm/flowforge/templates/emqx.yaml +++ b/helm/flowforge/templates/emqx.yaml @@ -1,4 +1,4 @@ -{{- if .Values.forge.broker.emqx.enabled -}} +{{- if .Values.forge.broker.teamBroker.enabled -}} apiVersion: apps.emqx.io/v2beta1 kind: EMQX metadata: From c0e473828c460c824c541d650ca8c851439415cb Mon Sep 17 00:00:00 2001 From: Ben Hardill Date: Fri, 25 Oct 2024 16:07:17 +0100 Subject: [PATCH 03/15] Getting closer --- helm/flowforge/templates/broker-config.yaml | 2 +- helm/flowforge/templates/broker-ingress.yaml | 2 +- helm/flowforge/templates/broker.yaml | 2 +- helm/flowforge/templates/emqx.yaml | 43 +++++------ test/customizations-teambroker.yml | 81 ++++++++++++++++++++ 5 files changed, 105 insertions(+), 25 deletions(-) create mode 100644 test/customizations-teambroker.yml diff --git a/helm/flowforge/templates/broker-config.yaml b/helm/flowforge/templates/broker-config.yaml index 66cc10c5..e9871870 100644 --- a/helm/flowforge/templates/broker-config.yaml +++ b/helm/flowforge/templates/broker-config.yaml @@ -1,4 +1,4 @@ -{{- if .Values.forge.broker.enabled -}} +{{- if and ( eq .Values.forge.broker.enabled true) ( eq .Values.forge.broker.teamBroker.enabled false ) -}} {{- $metricsUser := "metrics_reader" }} apiVersion: v1 kind: ConfigMap diff --git a/helm/flowforge/templates/broker-ingress.yaml b/helm/flowforge/templates/broker-ingress.yaml index 1b3490eb..d86e9c99 100644 --- a/helm/flowforge/templates/broker-ingress.yaml +++ b/helm/flowforge/templates/broker-ingress.yaml @@ -1,4 +1,4 @@ -{{- if .Values.forge.broker.enabled -}} +{{- if and ( eq .Values.forge.broker.enabled true) ( eq .Values.forge.broker.teamBroker.enabled false ) -}} {{- $brokerHostname := (printf "%s%s" "mqtt." .Values.forge.domain) -}} apiVersion: networking.k8s.io/v1 kind: Ingress diff --git a/helm/flowforge/templates/broker.yaml b/helm/flowforge/templates/broker.yaml index de41e6da..6961b218 100644 --- a/helm/flowforge/templates/broker.yaml +++ b/helm/flowforge/templates/broker.yaml @@ -1,4 +1,4 @@ -{{- if .Values.forge.broker.enabled -}} +{{- if and ( eq .Values.forge.broker.enabled true) ( eq .Values.forge.broker.teamBroker.enabled false ) -}} apiVersion: apps/v1 kind: Deployment metadata: diff --git a/helm/flowforge/templates/emqx.yaml b/helm/flowforge/templates/emqx.yaml index 947b21e7..1036a0f4 100644 --- a/helm/flowforge/templates/emqx.yaml +++ b/helm/flowforge/templates/emqx.yaml @@ -1,4 +1,4 @@ -{{- if .Values.forge.broker.teamBroker.enabled -}} +{{- if and ( eq .Values.forge.broker.enabled true) ( eq .Values.forge.broker.teamBroker.enabled true ) -}} apiVersion: apps.emqx.io/v2beta1 kind: EMQX metadata: @@ -6,18 +6,6 @@ metadata: spec: image: emqx:5 imagePullPolicy: IfNotPresent - coreTemplate: - spec: - volumeClaimTemplates: - {{ if .Values.forge.broker.emqx.storageClassName}} - storageClassName: {{ .Values.forge.broker.emqx.storageClassName }} - {{ end}} - resources: - requests: - storage: 10Gi - accessModes: - - ReadWriteOnce - replicaCount: 1 config: data: | authentication = [ @@ -41,7 +29,7 @@ spec: ssl { enable = false } - url = "http://forge.default/api/comms/v2/auth" + url = "http://forge.{{ .Release.Namespace }}/api/comms/v2/auth" } ] authorization { @@ -73,7 +61,7 @@ spec: ssl { enable = false } - url = "http://forge.default/api/comms/v2/acls" + url = "http://forge.{{ .Release.Namespace }}/api/comms/v2/acls" } ] } @@ -161,25 +149,36 @@ spec: } coreTemplate: spec: - volumeClaimTemaples: + volumeClaimTemplates: + {{- if .Values.forge.broker.teamBroker.storageClassName }} + storageClassName: {{ .Values.forge.broker.teamBroker.storageClassName }} + {{- end}} resources: requests: storage: 5Gi accessModes: - ReadWriteOnce extraVolumes: - - name: config - secret: - name: emqx-config + - name: config + secret: + secretName: emqx-config extraVolumeMounts: - - name: config - mountPath: /mounted/config + - name: config + mountPath: /mounted/config listenersServiceTemplate: + {{- if .Values.forge.broker.teamBroker.listenersServiceTemplate }} +{{ toYaml .Values.forge.broker.teamBroker.listenersServiceTemplate | indent 8 }} + {{ else }} spec: type: ClusterIP + {{- end }} dashboardServiceTemplate: + {{- if .Values.forge.broker.teamBroker.dashboardServiceTemplate }} +{{ toYaml .Values.forge.broker.teamBroker.dashboardServiceTemplate | indent 8 }} + {{ else }} spec: type: ClusterIP + {{- end }} --- apiVersion: v1 kind: Secret @@ -225,4 +224,4 @@ spec: - {{ include "forge.brokerDomain" . }} secretName: {{ include "forge.brokerDomain" . }} {{- end }} -{{- end }} \ No newline at end of file +{{- end }} diff --git a/test/customizations-teambroker.yml b/test/customizations-teambroker.yml new file mode 100644 index 00000000..42511960 --- /dev/null +++ b/test/customizations-teambroker.yml @@ -0,0 +1,81 @@ +forge: + domain: example.com + entryPoint: app.example + https: true + localPostgresql: false + cloudProvider: aws + managementSelector: + role: management + projectSelector: + role: projects + aws: + IAMRole: arn:aws:iam::xxxxxxxxxxxxxxxxxxx:role/flowforge_service_account_role + email: + from: "\"FlowForge\" " + ses: + region: eu-west-1 + broker: + enabled: true + url: mqtt://flowforge-broker.default:1883 + public_url: wss://mqtt.example.com + teamBroker: + enabled: true + ee: + billing: + stripe: + key: sk_live_dfadfsajflsadfafsafsajfdsfdsflfdladjfjf + wh_secret: whsec_fkjdflksajflgljfajfdlahfdkhflksahfhf + team_price: price_123456 + team_product: prod_123456 + project_price: price_1123456 + project_product: prod_123456 + device_price: price_8888 + device_product: prod_8888 + new_customer_free_credit: 1500 + teams: + starter: + price: price_123456 + product: prod_123456 + userCost: 0 + telemetry: + enabled: true + posthog: + capture_pageview: false + apikey: phc_fdlksajfdfadfsafsaf + sentry: + production_mode: false + frontend_dsn: 'https://sentry.io/flowforge/flowforge-frontend' + backend_dsn: 'https://sentry.io/flowforge/flowforge-backend' + backend: + prometheus: + enabled: true + support: + enabled: true + hubspot: 12345678 + fileStore: + enabled: true + type: s3 + options: + bucket: flowforge-production-files + forcePathStyle: true + region: eu-west-1 + credentials: + accessKeyId: ACCESSKEY + secretAccessKey: SECRETKEY + context: + type: sequelize + options: + type: postgres + branding: + account: + signUpTopBanner: HelloWorld + rate_limits: + enabled: true + +postgresql: + host: flowforge-postgresql + auth: + username: forge + password: password + database: flowforge + postgresPassword: postgres-password From 6d2fa77ea6552975c63737b0e5d7dbb5bf7c4458 Mon Sep 17 00:00:00 2001 From: Ben Hardill Date: Fri, 25 Oct 2024 16:20:43 +0100 Subject: [PATCH 04/15] Add to default and schema --- helm/flowforge/values.schema.json | 9 +++++++++ helm/flowforge/values.yaml | 8 +++----- test/customizations-teambroker.yml | 21 +++++++++++++++++++++ 3 files changed, 33 insertions(+), 5 deletions(-) diff --git a/helm/flowforge/values.schema.json b/helm/flowforge/values.schema.json index 79a4250c..ecb88bdd 100644 --- a/helm/flowforge/values.schema.json +++ b/helm/flowforge/values.schema.json @@ -230,6 +230,15 @@ "properties": { "enabled": { "type": "boolean" + }, + "storageClass": { + "type": "string" + }, + "listenersServiceTemplate": { + "type": "object" + }, + "dashboardServiceTemplate": { + "type": "object" } } }, diff --git a/helm/flowforge/values.yaml b/helm/flowforge/values.yaml index b6031bf7..80daf33f 100644 --- a/helm/flowforge/values.yaml +++ b/helm/flowforge/values.yaml @@ -21,6 +21,9 @@ forge: enabled: false teamBroker: enabled: false + storageClassName: '' + listenersServiceTemplate: '' + dashboardServiceTemplate: '' createMetricsUser: false podSecurityContext: runAsUser: 1000 @@ -47,11 +50,6 @@ forge: podLabels: {} tolerations: [] ingress.annotations: {} - emqx: - enabled: false - persistence: - storageClassName: '' - persistentStorage: enabled: false diff --git a/test/customizations-teambroker.yml b/test/customizations-teambroker.yml index 42511960..ded6cf38 100644 --- a/test/customizations-teambroker.yml +++ b/test/customizations-teambroker.yml @@ -20,6 +20,27 @@ forge: public_url: wss://mqtt.example.com teamBroker: enabled: true + listenersServiceTemplate: + metadata: + annotations: + service.beta.kubernetes.io/aws-load-balancer-scheme: internet-facing + service.beta.kubernetes.io/aws-load-balancer-backend-protocol: tcp + service.beta.kubernetes.io/aws-load-balancer-type: nlb + service.beta.kubernetes.io/aws-load-balancer-ssl-cert: arn:aws:acm:eu-west-1:287220081737:certificate/4d561d45-35e4-4bef-9aac-07bfb13e5fdd + service.beta.kubernetes.io/aws-load-balancer-ssl-ports: "8883,443" + service.beta.kubernetes.io/aws-load-balancer-eip-allocations: eipalloc-0e44de88ccacc13be,eipalloc-0e9e5f05c5d860ff9,eipalloc-08a3321a6b27e6d26 + spec: + type: LoadBalancer + ports: + - name: wss-wrap + protocol: TCP + port: 443 + targetPort: 8080 + - name: tls-wrap + protocol: TCP + port: 8883 + targetPort: 1883 + ee: billing: stripe: From 4d104cb9ebf982e3dff6d08cd47df6ab93760e9c Mon Sep 17 00:00:00 2001 From: Ben Hardill Date: Fri, 25 Oct 2024 16:39:53 +0100 Subject: [PATCH 05/15] Update README.md --- helm/flowforge/README.md | 3 +++ helm/flowforge/templates/emqx.yaml | 21 +++++++++++++++++---- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/helm/flowforge/README.md b/helm/flowforge/README.md index 8596224a..0584e0c5 100644 --- a/helm/flowforge/README.md +++ b/helm/flowforge/README.md @@ -92,6 +92,9 @@ To use STMP to send email - `forge.broker.public_url` URL to access the broker from outside the cluster (default `ws://mqtt.[forge.domain]`, uses `wss://` if `forge.https` is `true`) - `forge.broker.hostname` the custom Fully Qualified Domain Name (FQDN) where the broker will be hosted (default `mqtt.[forge.domain]`) - `forge.broker.teamBroker.enabled` Enables Team Broker feature (default `false`) + - `forge.broker.teamBroker.storageClass` the StorageClass to use for the teamBroker persistent Storage + - `forge.broker.teamBroker.listenersServiceTemplate` Service spec for the MQTT listeners + - `forge.broker.teamBroker.dashboardServiceTemplate` Service spec for the teamBroker admin console - `forge.broker.createMetricsUser` defines if a dedicated MQTT user with broker metrics collection permissions should be created (default `true`) - `forge.broker.affinity` allows to configure [affinity or anti-affinity](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity) for the broker pod - `forge.broker.resources` allows to configure [resources](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/) for the broker container diff --git a/helm/flowforge/templates/emqx.yaml b/helm/flowforge/templates/emqx.yaml index 1036a0f4..15b7f1e6 100644 --- a/helm/flowforge/templates/emqx.yaml +++ b/helm/flowforge/templates/emqx.yaml @@ -149,6 +149,12 @@ spec: } coreTemplate: spec: + {{- if .Values.forge.registrySecrets }} + imagePullSecrets: + {{- range .Values.forge.registrySecrets }} + - name: {{ . }} + {{- end }} + {{- end }} volumeClaimTemplates: {{- if .Values.forge.broker.teamBroker.storageClassName }} storageClassName: {{ .Values.forge.broker.teamBroker.storageClassName }} @@ -165,18 +171,25 @@ spec: extraVolumeMounts: - name: config mountPath: /mounted/config + {{- if .Values.forge.broker.affinity }} + affinity: {{ toYaml .Values.forge.broker.affinity | indent 12 }} + {{- end }} + {{- if .Values.forge.broker.tolerations}} + tolerations: + {{ toYaml .Values.forge.broker.tolerations | nindent 12 }} + {{- end }} listenersServiceTemplate: + spec: {{- if .Values.forge.broker.teamBroker.listenersServiceTemplate }} -{{ toYaml .Values.forge.broker.teamBroker.listenersServiceTemplate | indent 8 }} +{{ toYaml .Values.forge.broker.teamBroker.listenersServiceTemplate | indent 12 }} {{ else }} - spec: type: ClusterIP {{- end }} dashboardServiceTemplate: + spec: {{- if .Values.forge.broker.teamBroker.dashboardServiceTemplate }} -{{ toYaml .Values.forge.broker.teamBroker.dashboardServiceTemplate | indent 8 }} +{{ toYaml .Values.forge.broker.teamBroker.dashboardServiceTemplate | indent 12 }} {{ else }} - spec: type: ClusterIP {{- end }} --- From 12a8341bcd12e17a2028cf0cbf4c5ab01822d248 Mon Sep 17 00:00:00 2001 From: Ben Hardill Date: Fri, 25 Oct 2024 16:41:52 +0100 Subject: [PATCH 06/15] Fix values vs schema --- helm/flowforge/values.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/helm/flowforge/values.yaml b/helm/flowforge/values.yaml index 80daf33f..b03ee78a 100644 --- a/helm/flowforge/values.yaml +++ b/helm/flowforge/values.yaml @@ -22,8 +22,8 @@ forge: teamBroker: enabled: false storageClassName: '' - listenersServiceTemplate: '' - dashboardServiceTemplate: '' + listenersServiceTemplate: {} + dashboardServiceTemplate: {} createMetricsUser: false podSecurityContext: runAsUser: 1000 From e11aa27f2c18012af96c5501148480e03e933073 Mon Sep 17 00:00:00 2001 From: Ben Hardill Date: Fri, 25 Oct 2024 17:26:27 +0100 Subject: [PATCH 07/15] Update customizations-teambroker.yml --- test/customizations-teambroker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/customizations-teambroker.yml b/test/customizations-teambroker.yml index ded6cf38..c776fec9 100644 --- a/test/customizations-teambroker.yml +++ b/test/customizations-teambroker.yml @@ -26,7 +26,7 @@ forge: service.beta.kubernetes.io/aws-load-balancer-scheme: internet-facing service.beta.kubernetes.io/aws-load-balancer-backend-protocol: tcp service.beta.kubernetes.io/aws-load-balancer-type: nlb - service.beta.kubernetes.io/aws-load-balancer-ssl-cert: arn:aws:acm:eu-west-1:287220081737:certificate/4d561d45-35e4-4bef-9aac-07bfb13e5fdd + service.beta.kubernetes.io/aws-load-balancer-ssl-cert: arn:aws:acm:eu-west-1:xxxxxxxxxx:certificate/4d561d45-35e4-4bef-9aac-07bfb13e5fdd service.beta.kubernetes.io/aws-load-balancer-ssl-ports: "8883,443" service.beta.kubernetes.io/aws-load-balancer-eip-allocations: eipalloc-0e44de88ccacc13be,eipalloc-0e9e5f05c5d860ff9,eipalloc-08a3321a6b27e6d26 spec: From f9bf6dd79d26700409f40d7fbf31bb9b266e8944 Mon Sep 17 00:00:00 2001 From: Ben Hardill Date: Mon, 28 Oct 2024 11:04:22 +0000 Subject: [PATCH 08/15] Move to 1 secret And also test for EMQX Operator --- helm/flowforge/templates/emqx.yaml | 32 ++++++++++++++++++++---------- 1 file changed, 21 insertions(+), 11 deletions(-) diff --git a/helm/flowforge/templates/emqx.yaml b/helm/flowforge/templates/emqx.yaml index 15b7f1e6..ffb569a6 100644 --- a/helm/flowforge/templates/emqx.yaml +++ b/helm/flowforge/templates/emqx.yaml @@ -1,4 +1,5 @@ -{{- if and ( eq .Values.forge.broker.enabled true) ( eq .Values.forge.broker.teamBroker.enabled true ) -}} +{{- if and ( eq .Values.forge.broker.enabled true) ( eq .Values.forge.broker.teamBroker.enabled true ) -}} +{{- if .Capabilities.APIVersions.Has "apps.emqx.io/v2beta1" }} apiVersion: apps.emqx.io/v2beta1 kind: EMQX metadata: @@ -141,9 +142,6 @@ spec: } } } - dashboard { - default_password = topSecret - } api_key { bootstrap_file = "/mounted/config/api-keys" } @@ -152,9 +150,15 @@ spec: {{- if .Values.forge.registrySecrets }} imagePullSecrets: {{- range .Values.forge.registrySecrets }} - - name: {{ . }} + - name: {{ . }} {{- end }} {{- end }} + env: + - name: EMQX_DASHBOARD__DEFAULT_PASSWORD + valueFrom: + secretKeyRef: + name: emqx-config-secrets + key: EMQX_DASHBOARD__DEFAULT_PASSWORD volumeClaimTemplates: {{- if .Values.forge.broker.teamBroker.storageClassName }} storageClassName: {{ .Values.forge.broker.teamBroker.storageClassName }} @@ -167,10 +171,11 @@ spec: extraVolumes: - name: config secret: - secretName: emqx-config + secretName: emqx-config-secrets extraVolumeMounts: - name: config - mountPath: /mounted/config + mountPath: /mounted/config/api-keys + subPath: api-keys {{- if .Values.forge.broker.affinity }} affinity: {{ toYaml .Values.forge.broker.affinity | indent 12 }} {{- end }} @@ -196,11 +201,13 @@ spec: apiVersion: v1 kind: Secret metadata: - name: emqx-config - namespace: {{ .Release.Namespace }} + name: emqx-config-secrets + namespace: {{ .Release.Namespace }} +type: Opaque stringData: - api-keys: | - flowforge:verySecret:administrator + EMQX_DASHBOARD__DEFAULT_PASSWORD: topSecret + api-keys: | + flowfuse:verySecret:administrator --- apiVersion: networking.k8s.io/v1 @@ -238,3 +245,6 @@ spec: secretName: {{ include "forge.brokerDomain" . }} {{- end }} {{- end }} +{{- else }} + {{- fail "EMQX Operator not installed" }} +{{- end }} \ No newline at end of file From af936ea77e075752f364a1e4bd719d58f20f76c4 Mon Sep 17 00:00:00 2001 From: Ben Hardill Date: Mon, 28 Oct 2024 11:07:59 +0000 Subject: [PATCH 09/15] Fix if/else/end order --- helm/flowforge/templates/emqx.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helm/flowforge/templates/emqx.yaml b/helm/flowforge/templates/emqx.yaml index ffb569a6..016f9449 100644 --- a/helm/flowforge/templates/emqx.yaml +++ b/helm/flowforge/templates/emqx.yaml @@ -244,7 +244,7 @@ spec: - {{ include "forge.brokerDomain" . }} secretName: {{ include "forge.brokerDomain" . }} {{- end }} -{{- end }} {{- else }} {{- fail "EMQX Operator not installed" }} +{{- end }} {{- end }} \ No newline at end of file From 0d8c28a077dd3d19f0638080d59dfd551a3cc0e2 Mon Sep 17 00:00:00 2001 From: Ben Hardill Date: Mon, 28 Oct 2024 13:48:30 +0000 Subject: [PATCH 10/15] Move Team Broker settings --- helm/flowforge/README.md | 9 ++++--- helm/flowforge/templates/emqx.yaml | 12 ++++----- helm/flowforge/values.schema.json | 23 +++++++++------- helm/flowforge/values.yaml | 8 +++--- test/customizations-teambroker.yml | 42 ++++++++++++++++-------------- 5 files changed, 53 insertions(+), 41 deletions(-) diff --git a/helm/flowforge/README.md b/helm/flowforge/README.md index 0584e0c5..9ffc73e6 100644 --- a/helm/flowforge/README.md +++ b/helm/flowforge/README.md @@ -92,9 +92,6 @@ To use STMP to send email - `forge.broker.public_url` URL to access the broker from outside the cluster (default `ws://mqtt.[forge.domain]`, uses `wss://` if `forge.https` is `true`) - `forge.broker.hostname` the custom Fully Qualified Domain Name (FQDN) where the broker will be hosted (default `mqtt.[forge.domain]`) - `forge.broker.teamBroker.enabled` Enables Team Broker feature (default `false`) - - `forge.broker.teamBroker.storageClass` the StorageClass to use for the teamBroker persistent Storage - - `forge.broker.teamBroker.listenersServiceTemplate` Service spec for the MQTT listeners - - `forge.broker.teamBroker.dashboardServiceTemplate` Service spec for the teamBroker admin console - `forge.broker.createMetricsUser` defines if a dedicated MQTT user with broker metrics collection permissions should be created (default `true`) - `forge.broker.affinity` allows to configure [affinity or anti-affinity](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity) for the broker pod - `forge.broker.resources` allows to configure [resources](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/) for the broker container @@ -115,6 +112,12 @@ To use STMP to send email `forge.broker.createMetricsUser` parameter controlls if a dedicated MQTT user with broker metrics collection permissions should be created. This user can by used by the tools like [Mosquitto Exporter](https://github.com/sapcc/mosquitto-exporter) to expose broker's metrics for Prometheus scrapper. +### Team Broker + + - `broker.storageClassName` the StorageClass to use for the teamBroker persistent Storage + - `broker.listenersServiceTemplate` Service spec for the MQTT listeners + - `broker.dashboardServiceTemplate` Service spec for the teamBroker admin console + ### Telemetry Enables FlowForge Telemetry diff --git a/helm/flowforge/templates/emqx.yaml b/helm/flowforge/templates/emqx.yaml index 016f9449..9aad8770 100644 --- a/helm/flowforge/templates/emqx.yaml +++ b/helm/flowforge/templates/emqx.yaml @@ -160,8 +160,8 @@ spec: name: emqx-config-secrets key: EMQX_DASHBOARD__DEFAULT_PASSWORD volumeClaimTemplates: - {{- if .Values.forge.broker.teamBroker.storageClassName }} - storageClassName: {{ .Values.forge.broker.teamBroker.storageClassName }} + {{- if .Values.broker.storageClassName }} + storageClassName: {{ .Values.broker.storageClassName }} {{- end}} resources: requests: @@ -185,15 +185,15 @@ spec: {{- end }} listenersServiceTemplate: spec: - {{- if .Values.forge.broker.teamBroker.listenersServiceTemplate }} -{{ toYaml .Values.forge.broker.teamBroker.listenersServiceTemplate | indent 12 }} + {{- if .Values.broker.listenersServiceTemplate }} +{{ toYaml .Values.broker.listenersServiceTemplate | indent 12 }} {{ else }} type: ClusterIP {{- end }} dashboardServiceTemplate: spec: - {{- if .Values.forge.broker.teamBroker.dashboardServiceTemplate }} -{{ toYaml .Values.forge.broker.teamBroker.dashboardServiceTemplate | indent 12 }} + {{- if .Values.broker.dashboardServiceTemplate }} +{{ toYaml .Values.broker.dashboardServiceTemplate | indent 12 }} {{ else }} type: ClusterIP {{- end }} diff --git a/helm/flowforge/values.schema.json b/helm/flowforge/values.schema.json index ecb88bdd..5d227268 100644 --- a/helm/flowforge/values.schema.json +++ b/helm/flowforge/values.schema.json @@ -230,15 +230,6 @@ "properties": { "enabled": { "type": "boolean" - }, - "storageClass": { - "type": "string" - }, - "listenersServiceTemplate": { - "type": "object" - }, - "dashboardServiceTemplate": { - "type": "object" } } }, @@ -972,6 +963,20 @@ "required": ["username", "password", "database"] } } + }, + "broker": { + "type": "object", + "properties": { + "storageClassName": { + "type": "string" + }, + "listenersServiceTemplate": { + "type": "object" + }, + "dashboardServiceTemplate": { + "type": "object" + } + } } }, "if": { diff --git a/helm/flowforge/values.yaml b/helm/flowforge/values.yaml index b03ee78a..75480061 100644 --- a/helm/flowforge/values.yaml +++ b/helm/flowforge/values.yaml @@ -21,9 +21,6 @@ forge: enabled: false teamBroker: enabled: false - storageClassName: '' - listenersServiceTemplate: {} - dashboardServiceTemplate: {} createMetricsUser: false podSecurityContext: runAsUser: 1000 @@ -154,3 +151,8 @@ editors: create: true annotations: {} name: editors + +broker: + storageClassName: '' + listenersServiceTemplate: {} + dashboardServiceTemplate: {} \ No newline at end of file diff --git a/test/customizations-teambroker.yml b/test/customizations-teambroker.yml index c776fec9..3ebb2c2e 100644 --- a/test/customizations-teambroker.yml +++ b/test/customizations-teambroker.yml @@ -20,26 +20,6 @@ forge: public_url: wss://mqtt.example.com teamBroker: enabled: true - listenersServiceTemplate: - metadata: - annotations: - service.beta.kubernetes.io/aws-load-balancer-scheme: internet-facing - service.beta.kubernetes.io/aws-load-balancer-backend-protocol: tcp - service.beta.kubernetes.io/aws-load-balancer-type: nlb - service.beta.kubernetes.io/aws-load-balancer-ssl-cert: arn:aws:acm:eu-west-1:xxxxxxxxxx:certificate/4d561d45-35e4-4bef-9aac-07bfb13e5fdd - service.beta.kubernetes.io/aws-load-balancer-ssl-ports: "8883,443" - service.beta.kubernetes.io/aws-load-balancer-eip-allocations: eipalloc-0e44de88ccacc13be,eipalloc-0e9e5f05c5d860ff9,eipalloc-08a3321a6b27e6d26 - spec: - type: LoadBalancer - ports: - - name: wss-wrap - protocol: TCP - port: 443 - targetPort: 8080 - - name: tls-wrap - protocol: TCP - port: 8883 - targetPort: 1883 ee: billing: @@ -100,3 +80,25 @@ postgresql: password: password database: flowforge postgresPassword: postgres-password + +broker: + listenersServiceTemplate: + metadata: + annotations: + service.beta.kubernetes.io/aws-load-balancer-scheme: internet-facing + service.beta.kubernetes.io/aws-load-balancer-backend-protocol: tcp + service.beta.kubernetes.io/aws-load-balancer-type: nlb + service.beta.kubernetes.io/aws-load-balancer-ssl-cert: arn:aws:acm:eu-west-1:xxxxxxxxxx:certificate/4d561d45-35e4-4bef-9aac-07bfb13e5fdd + service.beta.kubernetes.io/aws-load-balancer-ssl-ports: "8883,443" + service.beta.kubernetes.io/aws-load-balancer-eip-allocations: eipalloc-0e44de88ccacc13be,eipalloc-0e9e5f05c5d860ff9,eipalloc-08a3321a6b27e6d26 + spec: + type: LoadBalancer + ports: + - name: wss-wrap + protocol: TCP + port: 443 + targetPort: 8080 + - name: tls-wrap + protocol: TCP + port: 8883 + targetPort: 1883 From 31889a49112d8c50957d63bab97c2bcf1b46fb00 Mon Sep 17 00:00:00 2001 From: Ben Hardill Date: Mon, 28 Oct 2024 13:50:15 +0000 Subject: [PATCH 11/15] Add empty line for lint --- helm/flowforge/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helm/flowforge/values.yaml b/helm/flowforge/values.yaml index 75480061..fc7af947 100644 --- a/helm/flowforge/values.yaml +++ b/helm/flowforge/values.yaml @@ -155,4 +155,4 @@ editors: broker: storageClassName: '' listenersServiceTemplate: {} - dashboardServiceTemplate: {} \ No newline at end of file + dashboardServiceTemplate: {} From a8840c029e14e3beb37373d0554f136b8e9a6dba Mon Sep 17 00:00:00 2001 From: Ben Hardill Date: Mon, 28 Oct 2024 15:25:04 +0000 Subject: [PATCH 12/15] Update helm/flowforge/templates/emqx.yaml Co-authored-by: PPawlowski --- helm/flowforge/templates/emqx.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/helm/flowforge/templates/emqx.yaml b/helm/flowforge/templates/emqx.yaml index 9aad8770..b7fb871f 100644 --- a/helm/flowforge/templates/emqx.yaml +++ b/helm/flowforge/templates/emqx.yaml @@ -204,7 +204,10 @@ metadata: name: emqx-config-secrets namespace: {{ .Release.Namespace }} type: Opaque -stringData: +data: + EMQX_DASHBOARD__DEFAULT_PASSWORD: {{ "topSecret" | b64enc | quote }} + api-keys: | + {{ "flowfuse:verySecret:administrator" | b64enc | quote }} EMQX_DASHBOARD__DEFAULT_PASSWORD: topSecret api-keys: | flowfuse:verySecret:administrator From 4b5733e748fa0dee334bfdf18de1b40aee593930 Mon Sep 17 00:00:00 2001 From: Ben Hardill Date: Mon, 28 Oct 2024 15:26:06 +0000 Subject: [PATCH 13/15] Update helm/flowforge/templates/emqx.yaml --- helm/flowforge/templates/emqx.yaml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/helm/flowforge/templates/emqx.yaml b/helm/flowforge/templates/emqx.yaml index b7fb871f..0c0fc546 100644 --- a/helm/flowforge/templates/emqx.yaml +++ b/helm/flowforge/templates/emqx.yaml @@ -206,11 +206,8 @@ metadata: type: Opaque data: EMQX_DASHBOARD__DEFAULT_PASSWORD: {{ "topSecret" | b64enc | quote }} - api-keys: | - {{ "flowfuse:verySecret:administrator" | b64enc | quote }} - EMQX_DASHBOARD__DEFAULT_PASSWORD: topSecret api-keys: | - flowfuse:verySecret:administrator + {{ "flowfuse:verySecret:administrator" | b64enc | quote }} --- apiVersion: networking.k8s.io/v1 From 8609dc906356f1ea2bb95129e84fca62422b72b2 Mon Sep 17 00:00:00 2001 From: Ben Hardill Date: Mon, 28 Oct 2024 16:25:43 +0000 Subject: [PATCH 14/15] Add existing secret for emqx dashboard --- helm/flowforge/README.md | 1 + helm/flowforge/templates/emqx.yaml | 11 ++++++++++- helm/flowforge/values.schema.json | 3 +++ helm/flowforge/values.yaml | 1 + 4 files changed, 15 insertions(+), 1 deletion(-) diff --git a/helm/flowforge/README.md b/helm/flowforge/README.md index 9ffc73e6..37559a7b 100644 --- a/helm/flowforge/README.md +++ b/helm/flowforge/README.md @@ -117,6 +117,7 @@ To use STMP to send email - `broker.storageClassName` the StorageClass to use for the teamBroker persistent Storage - `broker.listenersServiceTemplate` Service spec for the MQTT listeners - `broker.dashboardServiceTemplate` Service spec for the teamBroker admin console + - `broker.existingSecret` name of existing Secret holding dashboard admin password and API key ### Telemetry diff --git a/helm/flowforge/templates/emqx.yaml b/helm/flowforge/templates/emqx.yaml index 0c0fc546..a7c8d068 100644 --- a/helm/flowforge/templates/emqx.yaml +++ b/helm/flowforge/templates/emqx.yaml @@ -157,7 +157,11 @@ spec: - name: EMQX_DASHBOARD__DEFAULT_PASSWORD valueFrom: secretKeyRef: + {{- if .Values.broker.exisitingSecret }} + name: {{ .Values.broker.exisitingSecret }} + {{- else }} name: emqx-config-secrets + {{- end }} key: EMQX_DASHBOARD__DEFAULT_PASSWORD volumeClaimTemplates: {{- if .Values.broker.storageClassName }} @@ -171,7 +175,11 @@ spec: extraVolumes: - name: config secret: + {{- if .Values.broker.exisitingSecret }} + secretName: {{ .Values.broker.exisitingSecret }} + {{- else }} secretName: emqx-config-secrets + {{- end }} extraVolumeMounts: - name: config mountPath: /mounted/config/api-keys @@ -198,6 +206,7 @@ spec: type: ClusterIP {{- end }} --- +{{- if not .Values.broker.exisitingSecret }} apiVersion: v1 kind: Secret metadata: @@ -208,8 +217,8 @@ data: EMQX_DASHBOARD__DEFAULT_PASSWORD: {{ "topSecret" | b64enc | quote }} api-keys: | {{ "flowfuse:verySecret:administrator" | b64enc | quote }} - --- +{{- end }} apiVersion: networking.k8s.io/v1 kind: Ingress metadata: diff --git a/helm/flowforge/values.schema.json b/helm/flowforge/values.schema.json index 5d227268..870b8887 100644 --- a/helm/flowforge/values.schema.json +++ b/helm/flowforge/values.schema.json @@ -975,6 +975,9 @@ }, "dashboardServiceTemplate": { "type": "object" + }, + "existingSecret": { + "type": "string" } } } diff --git a/helm/flowforge/values.yaml b/helm/flowforge/values.yaml index fc7af947..2b0ec9d5 100644 --- a/helm/flowforge/values.yaml +++ b/helm/flowforge/values.yaml @@ -156,3 +156,4 @@ broker: storageClassName: '' listenersServiceTemplate: {} dashboardServiceTemplate: {} + existingSecret: '' From 0b2d19189bea9f8a3c8e5767681f454a663871e6 Mon Sep 17 00:00:00 2001 From: Ben Hardill Date: Thu, 31 Oct 2024 09:35:35 +0000 Subject: [PATCH 15/15] Add flag to test teamBroker UI --- helm/flowforge/templates/configmap.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helm/flowforge/templates/configmap.yaml b/helm/flowforge/templates/configmap.yaml index 19e17338..7f4fa496 100644 --- a/helm/flowforge/templates/configmap.yaml +++ b/helm/flowforge/templates/configmap.yaml @@ -171,7 +171,7 @@ data: {{ else -}} public_url: ws{{- if .Values.forge.https -}}s{{- end -}}://{{ include "forge.brokerDomain" . }} {{ end -}} - {{ if .Values.forge.broker.teamBroker.enabled }} + {{ if or .Values.forge.broker.teamBroker.enabled .Values.forge.broker.teamBroker.uiOnly }} teamBroker: enabled: true {{ end -}}