diff --git a/helm/flowforge/README.md b/helm/flowforge/README.md index 1e7f9456..f94756fd 100644 --- a/helm/flowforge/README.md +++ b/helm/flowforge/README.md @@ -113,6 +113,13 @@ To use STMP to send email `forge.broker.createMetricsUser` parameter controlls if a dedicated MQTT user with broker metrics collection permissions should be created. This user can by used by the tools like [Mosquitto Exporter](https://github.com/sapcc/mosquitto-exporter) to expose broker's metrics for Prometheus scrapper. +### Team Broker + + - `broker.storageClassName` the StorageClass to use for the teamBroker persistent Storage + - `broker.listenersServiceTemplate` Service spec for the MQTT listeners + - `broker.dashboardServiceTemplate` Service spec for the teamBroker admin console + - `broker.existingSecret` name of existing Secret holding dashboard admin password and API key + ### Telemetry Enables FlowForge Telemetry diff --git a/helm/flowforge/templates/broker-config.yaml b/helm/flowforge/templates/broker-config.yaml index 66cc10c5..e9871870 100644 --- a/helm/flowforge/templates/broker-config.yaml +++ b/helm/flowforge/templates/broker-config.yaml @@ -1,4 +1,4 @@ -{{- if .Values.forge.broker.enabled -}} +{{- if and ( eq .Values.forge.broker.enabled true) ( eq .Values.forge.broker.teamBroker.enabled false ) -}} {{- $metricsUser := "metrics_reader" }} apiVersion: v1 kind: ConfigMap diff --git a/helm/flowforge/templates/broker-ingress.yaml b/helm/flowforge/templates/broker-ingress.yaml index 1b3490eb..d86e9c99 100644 --- a/helm/flowforge/templates/broker-ingress.yaml +++ b/helm/flowforge/templates/broker-ingress.yaml @@ -1,4 +1,4 @@ -{{- if .Values.forge.broker.enabled -}} +{{- if and ( eq .Values.forge.broker.enabled true) ( eq .Values.forge.broker.teamBroker.enabled false ) -}} {{- $brokerHostname := (printf "%s%s" "mqtt." .Values.forge.domain) -}} apiVersion: networking.k8s.io/v1 kind: Ingress diff --git a/helm/flowforge/templates/broker.yaml b/helm/flowforge/templates/broker.yaml index af434102..7f8febb4 100644 --- a/helm/flowforge/templates/broker.yaml +++ b/helm/flowforge/templates/broker.yaml @@ -1,4 +1,4 @@ -{{- if .Values.forge.broker.enabled -}} +{{- if and ( eq .Values.forge.broker.enabled true) ( eq .Values.forge.broker.teamBroker.enabled false ) -}} apiVersion: apps/v1 kind: Deployment metadata: diff --git a/helm/flowforge/templates/configmap.yaml b/helm/flowforge/templates/configmap.yaml index 5b9c437e..7f4fa496 100644 --- a/helm/flowforge/templates/configmap.yaml +++ b/helm/flowforge/templates/configmap.yaml @@ -160,14 +160,18 @@ data: {{ if .Values.forge.broker.url -}} url: {{ .Values.forge.broker.url }} {{ else -}} + {{ if .Values.forge.broker.teamBroker }} + url: mqtt://emqx-listeners.{{ .Release.Namespace }}:1883 + {{ else -}} url: mqtt://flowforge-broker.{{ .Release.Namespace }}:1883 + {{end -}} {{ end -}} {{ if .Values.forge.broker.public_url -}} public_url: {{ .Values.forge.broker.public_url }} {{ else -}} public_url: ws{{- if .Values.forge.https -}}s{{- end -}}://{{ include "forge.brokerDomain" . }} {{ end -}} - {{ if .Values.forge.broker.teamBroker.enabled }} + {{ if or .Values.forge.broker.teamBroker.enabled .Values.forge.broker.teamBroker.uiOnly }} teamBroker: enabled: true {{ end -}} diff --git a/helm/flowforge/templates/emqx.yaml b/helm/flowforge/templates/emqx.yaml new file mode 100644 index 00000000..a7c8d068 --- /dev/null +++ b/helm/flowforge/templates/emqx.yaml @@ -0,0 +1,259 @@ +{{- if and ( eq .Values.forge.broker.enabled true) ( eq .Values.forge.broker.teamBroker.enabled true ) -}} +{{- if .Capabilities.APIVersions.Has "apps.emqx.io/v2beta1" }} +apiVersion: apps.emqx.io/v2beta1 +kind: EMQX +metadata: + name: emqx +spec: + image: emqx:5 + imagePullPolicy: IfNotPresent + config: + data: | + authentication = [ + { + backend = http + body = { + clientId = "${clientid}" + username = "${username}" + password = "${password}" + } + enable = true + connect_timeout = "15s" + enable_pipelining = 100 + headers { + content-type = "application/json" + } + mechanism = password_based + method = post + pool_size = 8 + request_timeout = "8s" + ssl { + enable = false + } + url = "http://forge.{{ .Release.Namespace }}/api/comms/v2/auth" + } + ] + authorization { + cache { + enable = true + excludes = [] + max_size = 32 + ttl = "1m" + } + deny_action = ignore + no_match = allow + sources = [ + { + enable = true + enable_pipelining = 100 + connect_timeout = "15s" + request_timeout = "30s" + pool_size = 8 + body { + action = "${action}" + topic = "${topic}" + username = "${username}" + } + headers { + content-type = "application/json" + } + method = post + type = http + ssl { + enable = false + } + url = "http://forge.{{ .Release.Namespace }}/api/comms/v2/acls" + } + ] + } + listeners { + tcp { + default { + bind = "0.0.0.0:1883" + access_rules = [ + "allow all" + ] + enable = true + enable_authn = true + mountpoint = "${client_attrs.team}" + max_connections = infinity + acceptors = 16 + proxy_protocol = false + proxy_protocol_timeout = 3s + tcp_options { + backlog = 1024 + send_timeout = 15s + recbuf = 2KB + sndbuf = 4KB + buffer = 4KB + high_watermark = 1MB + nodelay = true + reuseaddr = true + keepalive = "none" + } + } + } + ssl { + default { + enable = false + } + } + wss { + default { + enable = false + } + } + ws { + default { + bind = "0.0.0.0:8080" + access_rules = [ + "allow all" + ] + enable = true + enable_authn = true + mountpoint = "${client_attrs.team}" + max_connections = infinity + proxy_protocol = false + proxy_protocol_timeout = 3s + tcp_options { + backlog = 1024 + send_timeout = 15s + recbuf = 2KB + sndbuf = 4KB + buffer = 4KB + high_watermark = 1MB + nodelay = true + reuseaddr = true + keepalive = "none" + } + websocket { + mqtt_path = "/" + allow_origin_absence = true + check_origin_enable = false + fail_if_no_subprotocol = true + supported_subprotocols = "mqtt, mqtt-v3, mqtt-v3.1.1 mqtt-v5" + mqtt_piggyback = multiple + compress = false + idle_timeout = 7200s + max_frame_size = infinity + proxy_address_header = "x-forwarded-for" + proxy_port_header = "x-forwarded-port" + } + } + } + } + api_key { + bootstrap_file = "/mounted/config/api-keys" + } + coreTemplate: + spec: + {{- if .Values.forge.registrySecrets }} + imagePullSecrets: + {{- range .Values.forge.registrySecrets }} + - name: {{ . }} + {{- end }} + {{- end }} + env: + - name: EMQX_DASHBOARD__DEFAULT_PASSWORD + valueFrom: + secretKeyRef: + {{- if .Values.broker.exisitingSecret }} + name: {{ .Values.broker.exisitingSecret }} + {{- else }} + name: emqx-config-secrets + {{- end }} + key: EMQX_DASHBOARD__DEFAULT_PASSWORD + volumeClaimTemplates: + {{- if .Values.broker.storageClassName }} + storageClassName: {{ .Values.broker.storageClassName }} + {{- end}} + resources: + requests: + storage: 5Gi + accessModes: + - ReadWriteOnce + extraVolumes: + - name: config + secret: + {{- if .Values.broker.exisitingSecret }} + secretName: {{ .Values.broker.exisitingSecret }} + {{- else }} + secretName: emqx-config-secrets + {{- end }} + extraVolumeMounts: + - name: config + mountPath: /mounted/config/api-keys + subPath: api-keys + {{- if .Values.forge.broker.affinity }} + affinity: {{ toYaml .Values.forge.broker.affinity | indent 12 }} + {{- end }} + {{- if .Values.forge.broker.tolerations}} + tolerations: + {{ toYaml .Values.forge.broker.tolerations | nindent 12 }} + {{- end }} + listenersServiceTemplate: + spec: + {{- if .Values.broker.listenersServiceTemplate }} +{{ toYaml .Values.broker.listenersServiceTemplate | indent 12 }} + {{ else }} + type: ClusterIP + {{- end }} + dashboardServiceTemplate: + spec: + {{- if .Values.broker.dashboardServiceTemplate }} +{{ toYaml .Values.broker.dashboardServiceTemplate | indent 12 }} + {{ else }} + type: ClusterIP + {{- end }} +--- +{{- if not .Values.broker.exisitingSecret }} +apiVersion: v1 +kind: Secret +metadata: + name: emqx-config-secrets + namespace: {{ .Release.Namespace }} +type: Opaque +data: + EMQX_DASHBOARD__DEFAULT_PASSWORD: {{ "topSecret" | b64enc | quote }} + api-keys: | + {{ "flowfuse:verySecret:administrator" | b64enc | quote }} +--- +{{- end }} +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: flowforge-broker + labels: + {{- include "forge.brokerSelectorLabels" . | nindent 4 }} + annotations: + {{- if .Values.ingress.certManagerIssuer }} + cert-manager.io/cluster-issuer: {{ $.Values.ingress.certManagerIssuer }} + {{- end }} + {{- if and .Values.forge.broker.enabled .Values.forge.broker.ingress (hasKey .Values.forge.broker.ingress "annotations") }} +{{ toYaml .Values.forge.broker.ingress.annotations | replace "{{ instanceHost }}" "{{ include forge.brokerDomain . }}" | replace "{{ serviceName }}" "flowforge-broker" | indent 4 }} + {{- end }} +spec: + {{- if $.Values.ingress.className }} + ingressClassName: {{ $.Values.ingress.className }} + {{- end }} + rules: + - host: {{ include "forge.brokerDomain" . }} + http: + paths: + - pathType: Prefix + path: / + backend: + service: + name: emqx-listeners + port: + number: 8080 + {{- if .Values.ingress.certManagerIssuer }} + tls: + - hosts: + - {{ include "forge.brokerDomain" . }} + secretName: {{ include "forge.brokerDomain" . }} + {{- end }} +{{- else }} + {{- fail "EMQX Operator not installed" }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/helm/flowforge/values.schema.json b/helm/flowforge/values.schema.json index 4ec4808d..1d72195b 100644 --- a/helm/flowforge/values.schema.json +++ b/helm/flowforge/values.schema.json @@ -966,6 +966,23 @@ "required": ["username", "password", "database"] } } + }, + "broker": { + "type": "object", + "properties": { + "storageClassName": { + "type": "string" + }, + "listenersServiceTemplate": { + "type": "object" + }, + "dashboardServiceTemplate": { + "type": "object" + }, + "existingSecret": { + "type": "string" + } + } } }, "if": { diff --git a/helm/flowforge/values.yaml b/helm/flowforge/values.yaml index 8797fab5..0fce2fbb 100644 --- a/helm/flowforge/values.yaml +++ b/helm/flowforge/values.yaml @@ -153,3 +153,9 @@ editors: create: true annotations: {} name: editors + +broker: + storageClassName: '' + listenersServiceTemplate: {} + dashboardServiceTemplate: {} + existingSecret: '' diff --git a/test/customizations-teambroker.yml b/test/customizations-teambroker.yml new file mode 100644 index 00000000..3ebb2c2e --- /dev/null +++ b/test/customizations-teambroker.yml @@ -0,0 +1,104 @@ +forge: + domain: example.com + entryPoint: app.example + https: true + localPostgresql: false + cloudProvider: aws + managementSelector: + role: management + projectSelector: + role: projects + aws: + IAMRole: arn:aws:iam::xxxxxxxxxxxxxxxxxxx:role/flowforge_service_account_role + email: + from: "\"FlowForge\" " + ses: + region: eu-west-1 + broker: + enabled: true + url: mqtt://flowforge-broker.default:1883 + public_url: wss://mqtt.example.com + teamBroker: + enabled: true + + ee: + billing: + stripe: + key: sk_live_dfadfsajflsadfafsafsajfdsfdsflfdladjfjf + wh_secret: whsec_fkjdflksajflgljfajfdlahfdkhflksahfhf + team_price: price_123456 + team_product: prod_123456 + project_price: price_1123456 + project_product: prod_123456 + device_price: price_8888 + device_product: prod_8888 + new_customer_free_credit: 1500 + teams: + starter: + price: price_123456 + product: prod_123456 + userCost: 0 + telemetry: + enabled: true + posthog: + capture_pageview: false + apikey: phc_fdlksajfdfadfsafsaf + sentry: + production_mode: false + frontend_dsn: 'https://sentry.io/flowforge/flowforge-frontend' + backend_dsn: 'https://sentry.io/flowforge/flowforge-backend' + backend: + prometheus: + enabled: true + support: + enabled: true + hubspot: 12345678 + fileStore: + enabled: true + type: s3 + options: + bucket: flowforge-production-files + forcePathStyle: true + region: eu-west-1 + credentials: + accessKeyId: ACCESSKEY + secretAccessKey: SECRETKEY + context: + type: sequelize + options: + type: postgres + branding: + account: + signUpTopBanner: HelloWorld + rate_limits: + enabled: true + +postgresql: + host: flowforge-postgresql + auth: + username: forge + password: password + database: flowforge + postgresPassword: postgres-password + +broker: + listenersServiceTemplate: + metadata: + annotations: + service.beta.kubernetes.io/aws-load-balancer-scheme: internet-facing + service.beta.kubernetes.io/aws-load-balancer-backend-protocol: tcp + service.beta.kubernetes.io/aws-load-balancer-type: nlb + service.beta.kubernetes.io/aws-load-balancer-ssl-cert: arn:aws:acm:eu-west-1:xxxxxxxxxx:certificate/4d561d45-35e4-4bef-9aac-07bfb13e5fdd + service.beta.kubernetes.io/aws-load-balancer-ssl-ports: "8883,443" + service.beta.kubernetes.io/aws-load-balancer-eip-allocations: eipalloc-0e44de88ccacc13be,eipalloc-0e9e5f05c5d860ff9,eipalloc-08a3321a6b27e6d26 + spec: + type: LoadBalancer + ports: + - name: wss-wrap + protocol: TCP + port: 443 + targetPort: 8080 + - name: tls-wrap + protocol: TCP + port: 8883 + targetPort: 1883