diff --git a/Makefile b/Makefile index 5ff4eda..7eb6256 100644 --- a/Makefile +++ b/Makefile @@ -131,3 +131,10 @@ update-velero: @helm repo add $(patsubst update-%,%,$@) https://vmware-tanzu.github.io/helm-charts @helm pull -d external --untar $(patsubst update-%,%,$@)/$(patsubst update-%,%,$@) @echo "" + +update-nifi: + @echo "Updating NiFi" + @rm -rf external/$(patsubst update-%,%,$@) + @helm repo add $(patsubst update-%,%,$@) https://cetic.github.io/helm-charts + @helm pull -d external --untar $(patsubst update-%,%,$@)/$(patsubst update-%,%,$@) + @echo "" \ No newline at end of file diff --git a/external/nifi/.helmignore b/external/nifi/.helmignore new file mode 100644 index 0000000..9d488a5 --- /dev/null +++ b/external/nifi/.helmignore @@ -0,0 +1,29 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store + +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ + +# Common backup files +*.swp +*.bak +*.tmp +*~ + +# Various IDEs +.project +.idea/ +*.tmproj + +.circleci/ + +doc/ +.github \ No newline at end of file diff --git a/external/nifi/Chart.yaml b/external/nifi/Chart.yaml new file mode 100644 index 0000000..c3b7ef5 --- /dev/null +++ b/external/nifi/Chart.yaml @@ -0,0 +1,36 @@ +--- +apiVersion: v2 +name: nifi +version: 1.2.1 +appVersion: 1.23.2 +description: Apache NiFi is a software project from the Apache Software Foundation designed to automate the flow of data between software systems. +keywords: + - nifi + - data + - process +home: https://nifi.apache.org/ +icon: https://svn.apache.org/repos/asf/nifi/site/trunk/images/niFi-logo-horizontal-scaled.png +source: + - https://github.com/cetic/helm-nifi +maintainers: + - name: Heet + email: heet.sankesara@kcl.ac.uk + url: https://github.com/Hsankesara + +dependencies: + - name: zookeeper + version: 9.2.7 + repository: https://charts.bitnami.com/bitnami + condition: zookeeper.enabled + - name: nifi-registry + alias: registry + version: 1.0.0 + repository: https://dysnix.github.io/charts/ + condition: registry.enabled + - name: ca + version: 1.0.1 + condition: ca.enabled + - name: openldap + version: ~1.2.4 + repository: https://charts.helm.sh/stable + condition: openldap.enabled diff --git a/external/nifi/LICENSE b/external/nifi/LICENSE new file mode 100644 index 0000000..261eeb9 --- /dev/null +++ b/external/nifi/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/external/nifi/README.md b/external/nifi/README.md new file mode 100644 index 0000000..0c91a6d --- /dev/null +++ b/external/nifi/README.md @@ -0,0 +1,323 @@ +# Helm Chart for Apache NiFi +## Introduction + +This [Helm](https://helm.sh/) chart installs [Apache NiFi](https://nifi.apache.org/) 1.23.2 in a [Kubernetes](https://kubernetes.io/) cluster. + +## Prerequisites + +- Kubernetes cluster 1.10+ +- Helm 3.0.0+ +- [Persistent Volumes (PV)](https://kubernetes.io/docs/concepts/storage/persistent-volumes/) provisioner support in the underlying infrastructure. + +## Installation + +### Add Helm repository + +```bash +helm repo add cetic https://cetic.github.io/helm-charts +helm repo update +``` + +### Configure the chart + +The following items can be set via `--set` flag during installation or configured by editing the [`values.yaml`](values.yaml) file directly (need to download the chart first). + +#### Configure how to expose nifi service + +- **Ingress**: The ingress controller must be installed in the Kubernetes cluster. +- **ClusterIP**: Exposes the service on a cluster-internal IP. Choosing this value makes the service only reachable from within the cluster. +- **NodePort**: Exposes the service on each Node’s IP at a static port (the NodePort). You’ll be able to contact the NodePort service, from outside the cluster, by requesting `NodeIP:NodePort`. +- **LoadBalancer**: Exposes the service externally using a cloud provider’s load balancer. + +#### Configure how to persist data + +- **Disable(default)**: The data does not survive the termination of a pod. +- **Persistent Volume Claim**: Enable persistence so that data survives termination of the pod. There is the choice of using one large persistent volume (using subPath) or seven separate persistent volumes for config, data, logs, repos, etc. + A default `StorageClass` is needed in the Kubernetes cluster to dynamically provision the volumes. Specify another StorageClass in the `persistence.storageClass` setting. + +#### Configure authentication + +- By default, the authentication is a `Single-User` authentication. You can optionally enable `ldap` or `oidc` to provide an external authentication. See the [configuration section](README.md#configuration) or [doc](doc/) folder for more details. + +#### Use custom processors + +To add [custom processors](https://cwiki.apache.org/confluence/display/NIFI/Maven+Projects+for+Extensions#MavenProjectsforExtensions-MavenProcessorArchetype), the `values.yaml` file `nifi` section should contain the following options, where `CUSTOM_LIB_FOLDER` should be replaced by the path where the libs are: + +```yaml + extraVolumeMounts: + - name: mycustomlibs + mountPath: /opt/configuration_resources/custom_lib + extraVolumes: # this will create the volume from the directory + - name: mycustomlibs + hostPath: + path: "CUSTOM_LIB_FOLDER" + properties: + customLibPath: "/opt/configuration_resources/custom_lib" +``` + +#### Configure prometheus monitoring + +- You first need monitoring to be enabled which can be accomplished by enabling the appropriate metrics flag (`metrics.prometheus.enabled` to true). +To enable the creation of prometheus metrics within Nifi we need to create a *Reporting Task*. Login to the Nifi UI and go to the Hamburger menu on the top right corner, click *Controller Settings* --> *Reporting Tasks* After that use the + icon to add a task. Click on the *Reporting* in the wordcloud on the left and select *PrometheusReportingTask* --> change *Send JVM metrics* to `true` and click on the play button to enable this task. + +If you plan to use Grafana for the visualization of the metrics data [the following dashboard](https://grafana.com/grafana/dashboards/12314) is compatible with the exposed metrics. + +### Install the chart + +Install the nifi helm chart with a release name `my-release`: + +```bash +helm install my-release cetic/nifi +``` + +### Install from local clone + +You will find how to perform an installation from a local clone on this [page](doc/INSTALLATION.md). + +## Uninstallation + +To uninstall/delete the `my-release` deployment: + +```bash +helm uninstall my-release +``` + +## Configuration + +The following table lists the configurable parameters of the nifi chart and the default values. + +| Parameter | Description | Default | +| --------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------| ------------------------------- | +| **ReplicaCount** | +| `replicaCount` | Number of nifi nodes | `1` | +| **Image** | +| `image.repository` | nifi Image name | `apache/nifi` | +| `image.tag` | nifi Image tag | `1.23.2` | +| `image.pullPolicy` | nifi Image pull policy | `IfNotPresent` | +| `image.pullSecret` | nifi Image pull secret | `nil` | +| **SecurityContext** | +| `securityContext.runAsUser` | nifi Docker User | `1000` | +| `securityContext.fsGroup` | nifi Docker Group | `1000` | +| **sts** | +| `sts.useHostNetwork` | If true, use the host's network | `nil` | +| `sts.serviceAccount.create` | If true, a service account will be created and used by the statefulset | `false` | +| `sts.serviceAccount.name` | When set, the set name will be used as the service account name. If a value is not provided a name will be generated based on Chart options | `nil` | +| `sts.serviceAccount.annotations` | Service account annotations | `{}` | +| `sts.podManagementPolicy` | Parallel podManagementPolicy | `Parallel` | +| `sts.AntiAffinity` | Affinity for pod assignment | `soft` | +| `sts.pod.annotations` | Pod template annotations | `security.alpha.kubernetes.io/sysctls: net.ipv4.ip_local_port_range=10000 65000` | +| `sts.hostAliases ` | Add entries to Pod /etc/hosts | `[]` | +| `sts.startupProbe.enabled` | enable [Startup Probe](https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/#define-startup-probes) on Nifi server container | `false` | +| `sts.startupProbe.failureThreshold` | sets Startup Probe failureThreshold field value | `60` | +| `sts.startupProbe.periodSeconds` | sets Startup Probe periodSeconds field value | `10` | +| **secrets** +| `secrets` | Pass any secrets to the nifi pods. The secret can also be mounted to a specific path if required. | `nil` | +| **configmaps** +| `configmaps` | Pass any configmaps to the nifi pods. The configmap can also be mounted to a specific path if required. | `nil` | +| **nifi properties** | +| `properties.algorithm` | [Encryption method](https://nifi.apache.org/docs/nifi-docs/html/administration-guide.html#nifi_sensitive_props_key) | `NIFI_PBKDF2_AES_GCM_256` | +| `properties.sensitiveKey` | [Encryption password](https://nifi.apache.org/docs/nifi-docs/html/administration-guide.html#nifi_sensitive_props_key) (at least 12 characters) | `changeMechangeMe` | +| `properties.sensitiveKeySetFile` | [Update Sensitive Properties Key](https://nifi.apache.org/docs/nifi-docs/html/administration-guide.html#updating-the-sensitive-properties-key) if this file does not exist, and then create it. | `nil` | +| `properties.sensitiveKeyPrior` | Prior `sensitiveKey` when updating via `sensitiveKeySetFile` mechanism | `nil` | +| `properties.externalSecure` | externalSecure for when inbound SSL | `false` | +| `properties.isNode` | cluster node properties (only configure for cluster nodes) | `false` | +| `properties.httpPort` | web properties HTTP port | `8080` | +| `properties.httpsPort` | web properties HTTPS port | `null` | +| `properties.clusterPort` | cluster node port | `6007` | +| `properties.clusterNodeConnectionTimeout` | cluster node connection timeout | `5 sec` | +| `properties.clusterNodeReadTimeout` | cluster node read timeout | `5 sec` | +| `properties.zookeeperConnectTimeout` | zookeeper connect timeout | `3 secs` | +| `properties.zookeeperSessionTimeout` | zookeeper session timeout | `3 secs` | +| `properties.archiveMaxRetentionPeriod` | nifi content repository archive max retention period | `3 days` | +| `properties.archiveMaxUsagePercentage` | nifi content repository archive max usage | `85%` | +| `properties.provenanceStorage` | nifi provenance repository max storage size | `8 GB` | +| `properties.provenanceMaxStorageTime` | nifi provenance repository max storage time | `10 days` | +| `properties.flowArchiveMaxTime` | nifi flow archive max time | `30 days` | +| `properties.flowArchiveMaxStorage` | nifi flow archive max storage | `500 MB` | +| `properties.siteToSite.secure` | Site to Site properties Secure mode | `false` | +| `properties.siteToSite.port` | Site to Site properties Secure port | `10000` | +| `properties.safetyValve` | Map of explicit 'property: value' pairs that overwrite other configuration | `nil` | +| `properties.customLibPath` | Path of the custom libraries folder | `nil` | +| `properties.webProxyHost` | Proxy to access to Nifi through the cluster ip address | `Port:30236` +| **[Authentication](/doc/USERMANAGEMENT.md)** | +| **Single-user authentication** | Automatically disabled if Client Certificate, OIDC, or LDAP enabled +| `auth. admin` | Default admin identity. It will overwrite the LDAP Bind DN for this purpose, when both is filled | ` CN=admin, OU=NIFI` | +| `auth.singleUser.username` | Single user identity | `username` | +| `auth.singleUser.password` | Single user password | `changemechangeme` | +| **Client Certificate authentication** | +| `auth.clientAuth.enabled` | Enable User auth via Client Certificates | `false` +| **Ldap authentication** | +| `auth.ldap.admin` | Default admin identity and LDAP Bind DN | | +| `auth.ldap.enabled` | Enable User auth via ldap | `false` | +| `auth.ldap.host` | ldap hostname | `ldap://:` | +| `auth.ldap.searchBase` | ldap searchBase | `CN=Users,DC=example,DC=com` | +| `auth.ldap.searchFilter` | ldap searchFilter | `CN=john` | +| `auth.ldap.userSearchScope` | ldap userSearchScope | `ONE_LEVEL` | +| `auth.ldap.groupSearchScope` | ldap groupSearchScope | `ONE_LEVEL` | +| **Oidc authentication** +| `auth.oidc.enabled` | Enable User auth via oidc | `false` | +| `auth.oidc.discoveryUrl` | oidc discover url | `https:///.well-known/openid-configuration` | +| `auth.oidc.clientId` | oidc clientId | `nil` | +| `auth.oidc.clientSecret` | oidc clientSecret | `nil` | +| `auth.oidc.claimIdentifyingUser` | oidc claimIdentifyingUser | `email` | +| `auth.oidc.preferredJwsAlgorithm` | The preferred algorithm for validating identity tokens. If this value is blank, it will default to RS256 which is required to be supported by the OpenID Connect Provider according to the specification. If this value is HS256, HS384, or HS512, NiFi will attempt to validate HMAC protected tokens using the specified client secret. If this value is none, NiFi will attempt to validate unsecured/plain tokens. | `nil` | +| `auth.oidc.admin` | Default OIDC admin identity | `nifi@example.com` | +| Note that OIDC authentication to a multi-NiFi-node cluster requires Ingress sticky sessions | See [background](https://community.cloudera.com/t5/Support-Questions/OIDC-With-Azure-AD/m-p/232324#M194163) | Also [how](https://kubernetes.github.io/ingress-nginx/examples/affinity/cookie/) | +| **postStart** | +| `postStart` | Include additional libraries in the Nifi containers by using the postStart handler | `nil` | +| **Headless Service** | +| `headless.type` | Type of the headless service for nifi | `ClusterIP` | +| `headless.annotations` | Headless Service annotations | `service.alpha.kubernetes.io/tolerate-unready-endpoints: "true"`| +| **UI Service** | +| `service.type` | Type of the UI service for nifi | `NodePort` | +| `service.httpPort` | Port to expose service | `8080` | +| `service.httpsPort` | Port to expose service in tls | `443` | +| `service.annotations` | Service annotations | `{}` | +| `service.loadBalancerIP` | LoadBalancerIP if service type is `LoadBalancer` | `nil` | +| `service.loadBalancerSourceRanges` | Address that are allowed when svc is `LoadBalancer` | `[]` | +| `service.processors.enabled` | Enables additional port/ports to nifi service for internal processors | `false` | +| `service.processors.ports` | Specify "name/port/targetPort/nodePort" for processors sockets | `[]` | +| **ContainerPorts** | | +| `containerPorts` | Additional containerPorts for the nifi-container. Example is given in values.yaml | `[]` +| **Ingress** | +| `ingress.enabled` | Enables Ingress | `false` | +| `ingress.className` | Ingress controller Class | `nginx` | +| `ingress.annotations` | Ingress annotations | `{}` | +| `ingress.path` | Path to access frontend (See issue [#22](https://github.com/cetic/helm-nifi/issues/22)) | `/` | +| `ingress.hosts` | Ingress hosts | `[]` | +| `ingress.tls` | Ingress TLS configuration | `[]` | +| **Persistence** | +| `persistence.enabled` | Use persistent volume to store data | `false` | +| `persistence.storageClass` | Storage class name of PVCs (use the default type if unset) | `nil` | +| `persistence.accessMode` | ReadWriteOnce or ReadOnly | `[ReadWriteOnce]` | +| `persistence.subPath.enabled` | Use only one persistent volume with subPath instead of seven separate persistent volumes | `false` | +| `persistence.subPath.name` | Name of the one persistent volume claim when using subPath | `data` | +| `persistence.subPath.size` | Size of the one persistent volume claim when using subPath | `36Gi` | +| `persistence.configStorage.size` | Size of persistent volume claim | `100Mi` | +| `persistence.authconfStorage.size` | Size of persistent volume claim | `100Mi` | +| `persistence.dataStorage.size` | Size of persistent volume claim | `1Gi` | +| `persistence.flowfileRepoStorage.size` | Size of persistent volume claim | `10Gi` | +| `persistence.contentRepoStorage.size` | Size of persistent volume claim | `10Gi` | +| `persistence.provenanceRepoStorage.size` | Size of persistent volume claim | `10Gi` | +| `persistence.logStorage.size` | Size of persistent volume claim | `5Gi` | +| **jvmMemory** | +| `jvmMemory` | bootstrap jvm size | `2g` | +| **SideCar** | +| `sidecar.image` | Separate image for tailing each log separately and checking zookeeper connectivity | `busybox` | +| `sidecar.tag` | Image tag | `1.32.0` | +| `sidecar.imagePullPolicy` | Image imagePullPolicy | `IfNotPresent` | +| **Resources** | +| `resources` | Pod resource requests and limits for logs | `{}` | +| **logResources** | +| `logresources.` | Pod resource requests and limits | `{}` | +| **affinity** | +| `affinity` | Pod affinity scheduling rules | `{}` | +| **nodeSelector** | +| `nodeSelector` | Node labels for pod assignment | `{}` | +| **terminationGracePeriodSeconds** | +| `terminationGracePeriodSeconds` | Number of seconds the pod needs to terminate gracefully. For clean scale down of the nifi-cluster the default is set to 60, opposed to k8s-default 30. | `60` | +| **tolerations** | +| `tolerations` | Tolerations for pod assignment | `[]` | +| **initContainers** | +| `initContainers` | Container definition that will be added to the pod as [initContainers](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#container-v1-core) | `[]` | +| **extraVolumes** | +| `extraVolumes` | Additional Volumes available within the pod (see [spec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#volume-v1-core) for format) | `[]` | +| **extraVolumeMounts** | +| `extraVolumeMounts` | VolumeMounts for the nifi-server container (see [spec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#volumemount-v1-core) for details) | `[]` | +| **env** | +| `env` | Additional environment variables for the nifi-container (see [spec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#envvar-v1-core) for details) | `[]` | +| `envFrom` | Additional environment variables for the nifi-container from config-maps or secrets (see [spec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#envfromsource-v1-core) for details) | `[]` | +| **extraOptions** | +| `extraOptions` | Additional bootstrap.conf properties (see [properties](https://nifi.apache.org/docs/nifi-docs/html/administration-guide.html#bootstrap_properties) for details) | `[]` | +| **extraContainers** | +| `extraContainers` | Additional container-specifications that should run within the pod (see [spec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#container-v1-core) for details) | `[]` | +| **extraLabels** | +| `extraLabels` | Additional labels for the nifi pod | `nil` | +| **openshift** | +| `openshift.scc.enabled` | If true, a openshift security context will be created permitting to run the statefulset as AnyUID | `false` | +| `openshift.route.enabled` | If true, a openshift route will be created. This option cannot be used together with Ingress as a route object replaces the Ingress. The property `properties.externalSecure` will configure the route in edge termination mode, the default is passthrough. The property `properties.httpsPort` has to be set if the cluster is intended to work with SSL termination | `false` | +| `openshift.route.host` | The hostname intended to be used in order to access NiFi web interface | `nil` | +| `openshift.route.path` | Path to access frontend, works the same way as the ingress path option | `nil` | +| **zookeeper** | +| `zookeeper.enabled` | If true, deploy Zookeeper | `true` | +| `zookeeper.url` | If the Zookeeper Chart is disabled a URL and port are required to connect | `nil` | +| `zookeeper.port` | If the Zookeeper Chart is disabled a URL and port are required to connect | `2181` | +| **registry** | +| `registry.enabled` | If true, deploy [Nifi Registry](https://nifi.apache.org/registry.html) | `false` | +| `registry.url` | If the Nifi Registry Chart is disabled a URL and port are required to connect | `nil` | +| `registry.port` | If the Nifi Registry Chart is disabled a URL and port are required to connect | `80` | +| **ca** | +| `ca.enabled` | If true, deploy Nifi Toolkit as CA | `false` | +| `ca.server` | CA server dns name | `nil` | +| `ca.port` | CA server port number | `9090` | +| `ca.token` | The token to use to prevent MITM | `80` | +| `ca.admin.cn` | CN for admin certificate | `admin` | +| `ca.serviceAccount.create` | If true, a service account will be created and used by the deployment | `false` | +| `ca.serviceAccount.name` |When set, the set name will be used as the service account name. If a value is not provided a name will be generated based on Chart options | `nil` | +| `ca.openshift.scc.enabled` | If true, an openshift security context will be created permitting to run the deployment as AnyUID | `false` | +| **certManager** | +| `certManager.enabled` | If true, use [cert-manager](https://cert-manager.io/) to create and rotate intra-NiFi-cluster TLS keys (note that cert-manager is a Kubernetes cluster-wide resource, so is not installed automatically by this chart) | `false` | +| `certManager.clusterDomain` | Kubernetes cluster top level domain, to generate fully qualified domain names for certificate Common Names | `cluster.local` | +| `certManager.keystorePasswd` | Java Key Store password for NiFi keystore | `changeme` | +| `certManager.truststorePasswd` | Java Key Store password for NiFi truststore | `changeme` | +| `certManager.additionalDnsNames` | Additional DNS names to incorporate into TLS certificates (e.g. where users point browsers to access the NiFi UI) | `[ localhost ]` | +| `certManager.caSecrets` | Names of Kubernetes secrets containing `ca.crt` keys to add to the NiFi truststore | `[ ]` | +| `certManager.refreshSeconds` | How often the sidecar refreshes the NiFi keystore (not truststore) from the cert-manager Kubernetes secrets | `300` | +| `certManager.resources` | Memory and CPU resources for the node certificate refresh sidecar | 100m CPU, 128MiB RAM | +| `certManager.replaceDefaultTrustStore` | Use the certManager truststore, not the default Java trusted CA collection (for [e.g.] private OIDC provider) | `false` | +| `certManager.certDuration` | NiFi node certificate lifetime (90 days) | `2160h` | +| `certManager.caDuration` | Certificate Authority certificate lifetime (10 years) | `87660h` | +| **metrics** | +| `metrics.prometheus.enabled` | Enable prometheus to access nifi metrics endpoint | `false` | +| `metrics.prometheus.port` | Port where Nifi server will expose Prometheus metrics | `9092` | +| `metrics.prometheus.serviceMonitor.enabled` | If `true`, creates a Prometheus Operator ServiceMonitor (also requires `metrics.prometheus.enabled` to be `true`) | `false` | +| `metrics.prometheus.serviceMonitor.namespace` | In which namespace the ServiceMonitor should be created | +| `metrics.prometheus.serviceMonitor.labels` | Additional labels for the ServiceMonitor | `nil` | +| **customFlow** | | | +| `customFlow` | Use this file (uncompressed XML; [possibly from a configmap](tests/06-site-to-site.bash)) as the Flow definition | `nil` | + +## Troubleshooting + +Before [filing a bug report](https://github.com/cetic/helm-nifi/issues/new/choose), you may want to: + +* check the [FAQ](/doc/FAQ.md) +* check that [persistent storage](https://kubernetes.io/docs/concepts/storage/persistent-volumes/) is configured on your cluster +* keep in mind that a first installation may take a significant amount of time on a home internet connection +* check if a pod is in error: +```bash +kubectl get pod +NAME READY STATUS RESTARTS AGE +myrelease-nifi-0 3/4 Failed 1 56m +myrelease-nifi-registry-0 1/1 Running 0 56m +myrelease-nifi-zookeeper-0 1/1 Running 0 56m +myrelease-nifi-zookeeper-1 1/1 Running 0 56m +myrelease-nifi-zookeeper-2 1/1 Running 0 56m +``` + +Inspect the pod, check the "Events" section at the end for anything suspicious. + +```bash +kubectl describe pod myrelease-nifi-0 +``` + +Get logs on a failed container inside the pod (here the `server` one): + +```bash +kubectl logs myrelease-nifi-0 server +``` + +## Credits + +Initially inspired from https://github.com/YolandaMDavis/apache-nifi. + +TLS work/inspiration from https://github.com/sushilkm/nifi-chart.git. + +## Contributing + +Feel free to contribute by making a [pull request](https://github.com/cetic/helm-nifi/pull/new/master). + +Please read the official [Helm Contribution Guide](https://github.com/helm/charts/blob/master/CONTRIBUTING.md) from Helm for more information on how you can contribute to this Chart. + +## License + +[Apache License 2.0](/LICENSE) diff --git a/external/nifi/charts/ca/.helmignore b/external/nifi/charts/ca/.helmignore new file mode 100644 index 0000000..0e8a0eb --- /dev/null +++ b/external/nifi/charts/ca/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/external/nifi/charts/ca/Chart.yaml b/external/nifi/charts/ca/Chart.yaml new file mode 100644 index 0000000..3d17b7f --- /dev/null +++ b/external/nifi/charts/ca/Chart.yaml @@ -0,0 +1,12 @@ +apiVersion: v2 +name: ca +version: 1.0.1 +# We are using the nifi version as appVersion +appVersion: 1.11.4 +description: A Helm chart to deploy ca server to generate self-signed certificates using nifi-toolkit. +keywords: + - nifi-toolkit + - ca + - self-signed certs +home: https://nifi.apache.org/ +icon: https://svn.apache.org/repos/asf/nifi/site/trunk/images/niFi-logo-horizontal-scaled.png diff --git a/external/nifi/charts/ca/templates/NOTES.txt b/external/nifi/charts/ca/templates/NOTES.txt new file mode 100644 index 0000000..9957122 --- /dev/null +++ b/external/nifi/charts/ca/templates/NOTES.txt @@ -0,0 +1,6 @@ +The release is named {{ .Release.Name }}. + +To learn more about the release, try: + + helm -n {{ .Release.Namespace }} status {{ .Release.Name }} + helm -n {{ .Release.Namespace }} get all {{ .Release.Name }} diff --git a/external/nifi/charts/ca/templates/_helpers.tpl b/external/nifi/charts/ca/templates/_helpers.tpl new file mode 100644 index 0000000..f89d79d --- /dev/null +++ b/external/nifi/charts/ca/templates/_helpers.tpl @@ -0,0 +1,36 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "ca.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "ca.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Set the service account name +*/}} +{{- define "ca.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "ca.fullname" .) .Values.serviceAccount.name }}-sa +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/external/nifi/charts/ca/templates/deployment.yaml b/external/nifi/charts/ca/templates/deployment.yaml new file mode 100644 index 0000000..0af662f --- /dev/null +++ b/external/nifi/charts/ca/templates/deployment.yaml @@ -0,0 +1,84 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "ca.fullname" . }} + labels: + app: {{ template "ca.name" . }}-ca + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + release: {{ .Release.Name | quote }} + heritage: {{ .Release.Service | quote }} +spec: + replicas: 1 + selector: + matchLabels: + app: {{ template "ca.name" . }}-ca + release: {{ .Release.Name }} + template: + metadata: + labels: + app: {{ template "ca.name" . }}-ca + release: {{ .Release.Name }} + spec: + serviceAccountName: {{ include "ca.serviceAccountName" . }} + containers: + - name: ca + imagePullPolicy: {{ .Values.image.pullPolicy | quote }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + command: + - sh + - -c + - | + if [ -f config.json ]; then + OPTIONS="--configJson config.json --useConfigJson" + fi + exec ${NIFI_TOOLKIT_HOME}/bin/tls-toolkit.sh server -c "{{ template "ca.fullname" . }}" -t {{ .Values.token }} -p {{ .Values.app_port }} ${OPTIONS} + resources: +{{ toYaml .Values.resources | indent 10 }} + ports: + - containerPort: {{ .Values.app_port }} + livenessProbe: + initialDelaySeconds: 90 + periodSeconds: 60 + tcpSocket: + port: {{ .Values.app_port }} +{{- if .Values.persistence.enabled }} + volumeMounts: + - mountPath: /ca + name: nifi-ca + subPath: ca + workingDir: /ca +{{- end }} + terminationGracePeriodSeconds: 0 + securityContext: +{{ toYaml .Values.securityContext | indent 8 }} +{{- if .Values.persistence.enabled }} + initContainers: + - command: + - sh + - -c + - chown -R 1000:1000 /ca + image: busybox + imagePullPolicy: IfNotPresent + name: volume-permissions + resources: {} + securityContext: + runAsUser: 0 + allowPrivilegeEscalation: false + capabilities: {} + privileged: false + readOnlyRootFilesystem: false + runAsNonRoot: false + stdin: true + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + tty: true + volumeMounts: + - mountPath: /ca + name: nifi-ca + subPath: ca + volumes: + - name: nifi-ca + persistentVolumeClaim: + claimName: nifi-ca +{{- end }} diff --git a/external/nifi/charts/ca/templates/scc.yaml b/external/nifi/charts/ca/templates/scc.yaml new file mode 100644 index 0000000..41279dc --- /dev/null +++ b/external/nifi/charts/ca/templates/scc.yaml @@ -0,0 +1,45 @@ +{{- if .Values.openshift.scc.enabled -}} +apiVersion: security.openshift.io/v1 +kind: SecurityContextConstraints +metadata: + labels: + app: {{ template "ca.name" . }}-ca + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + release: {{ .Release.Name | quote }} + heritage: {{ .Release.Service | quote }} + annotations: + kubernetes.io/description: nifi provides all features of the restricted SCC but + allows users to run with any UID and any GID. + name: {{ template "ca.fullname" . }}-scc +allowHostDirVolumePlugin: false +allowHostIPC: false +allowHostNetwork: false +allowHostPID: false +allowHostPorts: false +allowPrivilegeEscalation: true +allowPrivilegedContainer: false +allowedCapabilities: null +defaultAddCapabilities: null +fsGroup: + type: RunAsAny +groups: [] +priority: 10 +readOnlyRootFilesystem: false +requiredDropCapabilities: +- MKNOD +runAsUser: + type: RunAsAny +seLinuxContext: + type: MustRunAs +supplementalGroups: + type: RunAsAny +users: +- system:serviceaccount:{{ .Release.Namespace }}:{{ include "ca.serviceAccountName" . }} +volumes: +- configMap +- downwardAPI +- emptyDir +- persistentVolumeClaim +- projected +- secret +{{- end }} \ No newline at end of file diff --git a/external/nifi/charts/ca/templates/service.yaml b/external/nifi/charts/ca/templates/service.yaml new file mode 100644 index 0000000..ff2c380 --- /dev/null +++ b/external/nifi/charts/ca/templates/service.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ template "ca.fullname" . }} + labels: + app: {{ template "ca.name" . }}-ca + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + release: {{ .Release.Name | quote }} + heritage: {{ .Release.Service | quote }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + targetPort: {{ .Values.app_port }} + name: ca-server + selector: + app: {{ template "ca.name" . }}-ca + release: {{ .Release.Name }} diff --git a/external/nifi/charts/ca/templates/serviceaccount.yaml b/external/nifi/charts/ca/templates/serviceaccount.yaml new file mode 100644 index 0000000..b9527ec --- /dev/null +++ b/external/nifi/charts/ca/templates/serviceaccount.yaml @@ -0,0 +1,11 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + app: {{ template "ca.name" . }}-ca + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + release: {{ .Release.Name | quote }} + heritage: {{ .Release.Service | quote }} + name: {{ include "ca.serviceAccountName" . }} +{{- end }} \ No newline at end of file diff --git a/external/nifi/charts/ca/templates/volume.yaml b/external/nifi/charts/ca/templates/volume.yaml new file mode 100644 index 0000000..d046bfe --- /dev/null +++ b/external/nifi/charts/ca/templates/volume.yaml @@ -0,0 +1,15 @@ +{{- if .Values.persistence.enabled }} +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: nifi-ca +spec: + accessModes: + {{- range .Values.persistence.accessModes }} + - {{ . | quote }} + {{- end }} + storageClassName: {{ .Values.persistence.storageClass | quote }} + resources: + requests: + storage: {{ .Values.persistence.caStorage.size }} +{{- end }} \ No newline at end of file diff --git a/external/nifi/charts/ca/values.yaml b/external/nifi/charts/ca/values.yaml new file mode 100644 index 0000000..936e270 --- /dev/null +++ b/external/nifi/charts/ca/values.yaml @@ -0,0 +1,61 @@ +# Default values for ca. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +replicaCount: 1 + +image: + repository: apache/nifi-toolkit + pullPolicy: IfNotPresent + tag: "1.12.1" + +service: + type: ClusterIP + port: 9090 + +## Enable persistence using Persistent Volume Claims +## ref: http://kubernetes.io/docs/user-guide/persistent-volumes/ +## +persistence: + enabled: false + + # When creating persistent storage, the NiFi helm chart can either reference an already-defined + # storage class by name, such as "standard" or can define a custom storage class by specifying + # customStorageClass: true and providing the "storageClass", "storageProvisioner" and "storageType". + # For example, to use SSD storage on Google Compute Engine see values-gcp.yaml + # + # To use a storage class that already exists on the Kubernetes cluster, we can simply reference it by name. + # For example: + # storageClass: standard + # + # The default storage class is used if this variable is not set. + + accessModes: [ReadWriteOnce] + + ## Storage Capacities for persistent volumes + # Storage capacity for the 'data' directory, which is used to hold things such as the flow.xml.gz, configuration, state, etc. + caStorage: + size: 1Gi + +resources: + requests: + memory: "250Mi" + cpu: "0.1" + +app_port: 9090 + +token: sixteenCharacters + +securityContext: + fsGroup: 1000 + runAsUser: 1000 + +serviceAccount: + create: false + #name: nifi-ca + +## Openshift support +## Use the following varables in order to enable Route and Security Context Constraint creation +openshift: + scc: + enabled: false diff --git a/external/nifi/configs/authorizers.xml b/external/nifi/configs/authorizers.xml new file mode 100644 index 0000000..58c1b32 --- /dev/null +++ b/external/nifi/configs/authorizers.xml @@ -0,0 +1,282 @@ +{{- $replicas := int .Values.replicaCount }} +{{- $chart := .Chart.Name }} +{{- $release := .Release.Name }} +{{- $fullname := include "apache-nifi.fullname" . }} +{{- $namespace := .Release.Namespace }} + + + + + + + file-user-group-provider + org.apache.nifi.authorization.FileUserGroupProvider + ./auth-conf/users.xml + + {{- range $i := until $replicas }} + CN={{ $fullname }}-{{ $i }}.{{ $fullname }}-headless.{{ $namespace }}.svc.cluster.local, OU=NIFI + {{- end }} + {{- if and .Values.auth.ldap.enabled (not .Values.auth.admin) }} + {{.Values.auth.ldap.admin}} + {{- else }} + {{ .Values.auth.admin }} + {{- end}} + + + {{- if .Values.auth.ldap.enabled}} + + ldap-user-group-provider + org.apache.nifi.ldap.tenants.LdapUserGroupProvider + SIMPLE + {{.Values.auth.ldap.admin}} + {{.Values.auth.ldap.pass}} + /opt/nifi/nifi-current/conf/{{.Release.Name}}-nifi-0.{{.Release.Name}}-nifi-headless.{{.Values.properties.namespace}}.svc.cluster.local/keystore.jks + {{.Values.auth.SSL.keystorePasswd}} + jks + /opt/nifi/nifi-current/conf/{{.Release.Name}}-nifi-0.{{.Release.Name}}-nifi-headless.{{.Values.properties.namespace}}.svc.cluster.local/truststore.jks + {{.Values.auth.SSL.truststorePasswd}} + JKS + NONE + TLS + false + IGNORE + 10 secs + 10 secs + {{.Values.auth.ldap.host}} + + 30 mins + {{.Values.auth.ldap.searchBase}} + person + {{.Values.auth.ldap.userSearchScope}} + {{.Values.auth.ldap.searchFilter}} + {{.Values.auth.ldap.UserIdentityAttribute}} + + + + group + {{.Values.auth.ldap.groupSearchScope}} + + + + + + {{- end}} + + + {{- if .Values.auth.ldap.enabled}} + + composite-configurable-user-group-provider + org.apache.nifi.authorization.CompositeConfigurableUserGroupProvider + file-user-group-provider + ldap-user-group-provider + + {{- end}} + + + + + + + file-access-policy-provider + org.apache.nifi.authorization.FileAccessPolicyProvider + file-user-group-provider + ./auth-conf/authorizations.xml + {{- if and .Values.auth.ldap.enabled (not .Values.auth.admin) }} + {{.Values.auth.ldap.admin}} + {{- else }} + {{ .Values.auth.admin }} + {{- end}} + + {{- range $i := until $replicas }} + CN={{ $fullname }}-{{ $i }}.{{ $fullname }}-headless.{{ $namespace }}.svc.cluster.local, OU=NIFI + {{- end }} + + + + + managed-authorizer + org.apache.nifi.authorization.StandardManagedAuthorizer + file-access-policy-provider + + + {{- if .Values.auth.ldap.enabled}} + + file-provider + org.apache.nifi.authorization.FileAuthorizer + ./auth-conf/authorizations.xml + ./auth-conf/users.xml + {{.Values.auth.ldap.admin}} + + + + {{- end}} + diff --git a/external/nifi/configs/bootstrap-notification-services.xml b/external/nifi/configs/bootstrap-notification-services.xml new file mode 100644 index 0000000..b2600b0 --- /dev/null +++ b/external/nifi/configs/bootstrap-notification-services.xml @@ -0,0 +1,53 @@ + + + + + + + + + diff --git a/external/nifi/configs/bootstrap.conf b/external/nifi/configs/bootstrap.conf new file mode 100644 index 0000000..a9183e6 --- /dev/null +++ b/external/nifi/configs/bootstrap.conf @@ -0,0 +1,88 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Java command to use when running NiFi +java=java + +# Username to use when running NiFi. This value will be ignored on Windows. +run.as= + +# Configure where NiFi's lib and conf directories live +lib.dir=./lib +conf.dir=./conf + +# How long to wait after telling NiFi to shutdown before explicitly killing the Process +graceful.shutdown.seconds=20 + +# Disable JSR 199 so that we can use JSP's without running a JDK +java.arg.1=-Dorg.apache.jasper.compiler.disablejsr199=true + +# JVM memory settings +java.arg.2=-Xms{{.Values.jvmMemory}} +java.arg.3=-Xmx{{.Values.jvmMemory}} + +# Enable Remote Debugging +#java.arg.debug=-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=8000 + +java.arg.4=-Djava.net.preferIPv4Stack=true + +# allowRestrictedHeaders is required for Cluster/Node communications to work properly +java.arg.5=-Dsun.net.http.allowRestrictedHeaders=true +java.arg.6=-Djava.protocol.handler.pkgs=sun.net.www.protocol + +{{ if .Values.certManager.replaceDefaultTrustStore }} +java.arg.7=-Djavax.net.ssl.trustStore=/opt/nifi/nifi-current/tls/truststore.jks +{{/* if .Values.certManager.replaceDefaultTrustStore */}}{{ end }} + +# The G1GC is still considered experimental but has proven to be very advantageous in providing great +# performance without significant "stop-the-world" delays. +#java.arg.13=-XX:+UseG1GC + +#Set headless mode by default +java.arg.14=-Djava.awt.headless=true + +# Master key in hexadecimal format for encrypted sensitive configuration values +nifi.bootstrap.sensitive.key= + +# Sets the provider of SecureRandom to /dev/urandom to prevent blocking on VMs +java.arg.15=-Djava.security.egd=file:/dev/urandom + +### +# Notification Services for notifying interested parties when NiFi is stopped, started, dies +### + +# XML File that contains the definitions of the notification services +notification.services.file=./conf/bootstrap-notification-services.xml + +# In the case that we are unable to send a notification for an event, how many times should we retry? +notification.max.attempts=5 + +# Comma-separated list of identifiers that are present in the notification.services.file; which services should be used to notify when NiFi is started? +#nifi.start.notification.services=email-notification + +# Comma-separated list of identifiers that are present in the notification.services.file; which services should be used to notify when NiFi is stopped? +#nifi.stop.notification.services=email-notification + +# Comma-separated list of identifiers that are present in the notification.services.file; which services should be used to notify when NiFi dies? +#nifi.dead.notification.services=email-notification + + +# Extra bootstrap options +{{- range .Values.extraOptions }} +{{ .name }}={{ .value }} +{{- end }} + diff --git a/external/nifi/configs/flow.xml b/external/nifi/configs/flow.xml new file mode 100644 index 0000000..cc878a0 --- /dev/null +++ b/external/nifi/configs/flow.xml @@ -0,0 +1,21 @@ + + + 10 + 5 + + + {{ default uuidv4 }} + default + {{ template "registry.url" . }} + + + + + {{ default uuidv4 }} + Nifi Flow + + + + + + diff --git a/external/nifi/configs/login-identity-providers-ldap.xml b/external/nifi/configs/login-identity-providers-ldap.xml new file mode 100644 index 0000000..d34e5f7 --- /dev/null +++ b/external/nifi/configs/login-identity-providers-ldap.xml @@ -0,0 +1,90 @@ + + + + + + + ldap-provider + org.apache.nifi.ldap.LdapProvider + {{.Values.auth.ldap.authStrategy}} + {{.Values.auth.ldap.admin}} + {{.Values.auth.ldap.pass}} + /opt/nifi/nifi-current/conf/{{.Release.Name}}-nifi-0.{{.Release.Name}}-nifi-headless.{{.Release.Namespace}}.svc.cluster.local/keystore.jks + {{.Values.auth.keystorePasswd}} + JKS + /opt/nifi/nifi-current/conf/{{.Release.Name}}-nifi-0.{{.Release.Name}}-nifi-headless.{{.Release.Namespace}}.svc.cluster.local/truststore.jks + {{.Values.auth.truststorePasswd}} + JKS + NONE + TLS + false + + FOLLOW + 10 secs + 10 secs + {{.Values.auth.ldap.host}} + {{.Values.auth.ldap.searchBase}} + ({{.Values.auth.ldap.userIdentityAttribute}}={0}) + {{.Values.auth.ldap.IdentityStrategy}} + {{.Values.auth.ldap.authExpiration}} + + diff --git a/external/nifi/configs/nifi.properties b/external/nifi/configs/nifi.properties new file mode 100644 index 0000000..35caceb --- /dev/null +++ b/external/nifi/configs/nifi.properties @@ -0,0 +1,271 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Core Properties # +nifi.flow.configuration.file=../data/flow.xml.gz +nifi.flow.configuration.archive.enabled=true +nifi.flow.configuration.archive.dir=../data/archive/ +nifi.flow.configuration.archive.max.time={{.Values.properties.flowArchiveMaxTime}} +nifi.flow.configuration.archive.max.storage={{.Values.properties.flowArchiveMaxStorage}} +nifi.flow.configuration.archive.max.count= +nifi.flowcontroller.autoResumeState=true +nifi.flowcontroller.graceful.shutdown.period=10 sec +nifi.flowservice.writedelay.interval=500 ms +nifi.administrative.yield.duration=30 sec +# If a component has no work to do (is "bored"), how long should we wait before checking again for work? +nifi.bored.yield.duration=10 millis + +nifi.authorizer.configuration.file=./conf/authorizers.xml +nifi.login.identity.provider.configuration.file=./conf/login-identity-providers.xml +nifi.templates.directory=../data/templates +nifi.ui.banner.text= +nifi.ui.autorefresh.interval=30 sec +nifi.nar.library.directory=./lib +nifi.nar.library.directory.custom={{.Values.properties.customLibPath}} +nifi.nar.library.autoload.directory=./extensions +nifi.nar.working.directory=./work/nar/ +nifi.documentation.working.directory=./work/docs/components + +#################### +# State Management # +#################### +nifi.state.management.configuration.file=./conf/state-management.xml +# The ID of the local state provider +nifi.state.management.provider.local=local-provider +# The ID of the cluster-wide state provider. This will be ignored if NiFi is not clustered but must be populated if running in a cluster. +nifi.state.management.provider.cluster=zk-provider +# Specifies whether or not this instance of NiFi should run an embedded ZooKeeper server +nifi.state.management.embedded.zookeeper.start=false +# Properties file that provides the ZooKeeper properties to use if is set to true +nifi.state.management.embedded.zookeeper.properties=./conf/zookeeper.properties + + +# H2 Settings +nifi.database.directory=../data/database_repository +nifi.h2.url.append=;LOCK_TIMEOUT=25000;WRITE_DELAY=0;AUTO_SERVER=FALSE + +# FlowFile Repository +nifi.flowfile.repository.implementation=org.apache.nifi.controller.repository.WriteAheadFlowFileRepository +nifi.flowfile.repository.directory=../flowfile_repository +nifi.flowfile.repository.partitions=256 +nifi.flowfile.repository.checkpoint.interval=2 mins +nifi.flowfile.repository.always.sync=false + +nifi.swap.manager.implementation=org.apache.nifi.controller.FileSystemSwapManager +nifi.queue.swap.threshold=20000 +nifi.swap.in.period=5 sec +nifi.swap.in.threads=1 +nifi.swap.out.period=5 sec +nifi.swap.out.threads=4 + +# Content Repository +nifi.content.repository.implementation=org.apache.nifi.controller.repository.FileSystemRepository +nifi.content.claim.max.appendable.size=1 MB +nifi.content.claim.max.flow.files=100 +nifi.content.repository.directory.default=../content_repository +nifi.content.repository.archive.max.retention.period={{.Values.properties.archiveMaxRetentionPeriod}} +nifi.content.repository.archive.max.usage.percentage={{.Values.properties.archiveMaxUsagePercentage}} +nifi.content.repository.archive.enabled=true +nifi.content.repository.always.sync=false +nifi.content.viewer.url=/nifi-content-viewer/ + +# Provenance Repository Properties +nifi.provenance.repository.implementation=org.apache.nifi.provenance.WriteAheadProvenanceRepository +nifi.provenance.repository.debug.frequency=1000000 +nifi.provenance.repository.encryption.key.provider.implementation= +nifi.provenance.repository.encryption.key.provider.location= +nifi.provenance.repository.encryption.key.id= +nifi.provenance.repository.encryption.key= + +# Persistent Provenance Repository Properties +nifi.provenance.repository.directory.default=../provenance_repository +nifi.provenance.repository.max.storage.time={{.Values.properties.provenanceMaxStorageTime}} +nifi.provenance.repository.max.storage.size={{.Values.properties.provenanceStorage}} +nifi.provenance.repository.rollover.time=30 secs +nifi.provenance.repository.rollover.size=100 MB +nifi.provenance.repository.query.threads=2 +nifi.provenance.repository.index.threads=2 +nifi.provenance.repository.compress.on.rollover=true +nifi.provenance.repository.always.sync=false +nifi.provenance.repository.journal.count=16 +# Comma-separated list of fields. Fields that are not indexed will not be searchable. Valid fields are: +# EventType, FlowFileUUID, Filename, TransitURI, ProcessorID, AlternateIdentifierURI, Relationship, Details +nifi.provenance.repository.indexed.fields=EventType, FlowFileUUID, Filename, ProcessorID, Relationship +# FlowFile Attributes that should be indexed and made searchable. Some examples to consider are filename, uuid, mime.type +nifi.provenance.repository.indexed.attributes= +# Large values for the shard size will result in more Java heap usage when searching the Provenance Repository +# but should provide better performance +nifi.provenance.repository.index.shard.size=500 MB +# Indicates the maximum length that a FlowFile attribute can be when retrieving a Provenance Event from +# the repository. If the length of any attribute exceeds this value, it will be truncated when the event is retrieved. +nifi.provenance.repository.max.attribute.length=65536 + +# Volatile Provenance Respository Properties +nifi.provenance.repository.buffer.size=100000 + +# Component Status Repository +nifi.components.status.repository.implementation=org.apache.nifi.controller.status.history.VolatileComponentStatusRepository +nifi.components.status.repository.buffer.size=1440 +nifi.components.status.snapshot.frequency=1 min + +# Site to Site properties +nifi.remote.input.host= +nifi.remote.input.secure=true +nifi.remote.input.socket.port={{.Values.properties.siteToSite.port}} +nifi.remote.input.http.enabled=true +nifi.remote.input.http.transaction.ttl=30 sec +nifi.remote.contents.cache.expiration=30 secs + +# web properties # +nifi.web.war.directory=./lib +nifi.web.proxy.host={{.Values.properties.webProxyHost}} +nifi.web.https.port={{.Values.properties.httpsPort}} +nifi.web.http.host= +nifi.web.http.network.interface.default= +nifi.web.https.host={{.Values.properties.webHttpsHost}} +nifi.web.https.network.interface.default= +nifi.web.jetty.working.directory=./work/jetty +nifi.web.jetty.threads=200 +# nifi.web.proxy.context.path= + +# security properties # +nifi.sensitive.props.key={{.Values.properties.sensitiveKey}} +nifi.sensitive.props.key.protected= +nifi.sensitive.props.algorithm={{ .Values.properties.algorithm }} +nifi.sensitive.props.provider=BC +nifi.sensitive.props.additional.keys= + +{{if .Values.auth.clientAuth.enabled}} +nifi.security.keystore=/opt/nifi/nifi-current/config-data/certs/keystore.jks +nifi.security.keystoreType=jks +nifi.security.keystorePasswd={{.Values.auth.SSL.keystorePasswd}} +nifi.security.keyPasswd={{.Values.auth.SSL.keystorePasswd}} +nifi.security.truststore=/opt/nifi/nifi-current/config-data/certs/truststore.jks +nifi.security.truststoreType=jks +nifi.security.truststorePasswd={{.Values.auth.SSL.truststorePasswd}} +nifi.security.user.authorizer=managed-authorizer +nifi.security.user.login.identity.provider= +{{else if .Values.auth.ldap.enabled}} +nifi.security.keystore=/opt/nifi/nifi-current/conf/{{.Release.Name}}-nifi-0.{{.Release.Name}}-nifi-headless.{{.Release.Namespace}}.svc.cluster.local/keystore.jks +nifi.security.keystoreType=jks +nifi.security.keystorePasswd={{.Values.auth.SSL.keystorePasswd}} +nifi.security.keyPasswd={{.Values.auth.SSL.keystorePasswd}} +nifi.security.truststore=/opt/nifi/nifi-current/conf/{{.Release.Name}}-nifi-0.{{.Release.Name}}-nifi-headless.{{.Release.Namespace}}.svc.cluster.local/truststore.jks +nifi.security.truststoreType=jks +nifi.security.truststorePasswd={{.Values.auth.SSL.truststorePasswd}} +proxiedEntity={{.Values.auth.ldap.admin}} +nifi.security.user.authorizer=file-provider +nifi.security.user.login.identity.provider=ldap-provider +{{else if .Values.auth.oidc.enabled}} +nifi.security.keystore=/opt/nifi/nifi-current/conf/keystore.p12 +nifi.security.keystoreType=PKCS12 +nifi.security.keystorePasswd= +nifi.security.keyPasswd= +nifi.security.truststore=/opt/nifi/nifi-current/conf/truststore.p12 +nifi.security.truststoreType=PKCS12 +nifi.security.truststorePasswd= +nifi.security.user.authorizer=managed-authorizer +{{ else }} +nifi.security.keystore=./conf/keystore.p12 +nifi.security.keystoreType=PKCS12 +nifi.security.keystorePasswd= +nifi.security.keyPasswd= +nifi.security.truststore=./conf/truststore.p12 +nifi.security.truststoreType=PKCS12 +nifi.security.truststorePasswd= +nifi.security.user.login.identity.provider=single-user-provider +nifi.security.user.authorizer=single-user-authorizer +{{end}} +nifi.security.needClientAuth={{.Values.properties.needClientAuth}} + +{{if .Values.auth.oidc.enabled}} +# OpenId Connect SSO Properties # +nifi.security.user.oidc.discovery.url={{.Values.auth.oidc.discoveryUrl}} +nifi.security.user.oidc.connect.timeout=5 secs +nifi.security.user.oidc.read.timeout=5 secs +nifi.security.user.oidc.client.id={{.Values.auth.oidc.clientId}} +nifi.security.user.oidc.client.secret={{.Values.auth.oidc.clientSecret}} +nifi.security.user.oidc.preferred.jwsalgorithm={{.Values.auth.oidc.preferredJwsAlgorithm}} +nifi.security.user.oidc.claim.identifying.user={{.Values.auth.oidc.claimIdentifyingUser}} +nifi.security.user.oidc.additional.scopes={{.Values.auth.oidc.additionalScopes}} +{{end}} + + +# Apache Knox SSO Properties # +nifi.security.user.knox.url= +nifi.security.user.knox.publicKey= +nifi.security.user.knox.cookieName=hadoop-jwt +nifi.security.user.knox.audiences= + +# Identity Mapping Properties # +# These properties allow normalizing user identities such that identities coming from different identity providers +# (certificates, LDAP, Kerberos) can be treated the same internally in NiFi. The following example demonstrates normalizing +# DNs from certificates and principals from Kerberos into a common identity string: +# +# nifi.security.identity.mapping.pattern.dn=^CN=(.*?), OU=(.*?), O=(.*?), L=(.*?), ST=(.*?), C=(.*?)$ +# nifi.security.identity.mapping.value.dn=$1@$2 +# nifi.security.identity.mapping.pattern.kerb=^(.*?)/instance@(.*?)$ +# nifi.security.identity.mapping.value.kerb=$1@$2 + +# cluster common properties (all nodes must have same values) # +nifi.cluster.protocol.heartbeat.interval=5 sec +nifi.cluster.protocol.is.secure=true + +# cluster node properties (only configure for cluster nodes) # +nifi.cluster.is.node={{.Values.properties.isNode}} +nifi.cluster.node.address= +nifi.cluster.node.protocol.port={{.Values.properties.clusterPort}} +nifi.cluster.node.protocol.threads=10 +nifi.cluster.node.protocol.max.threads=50 +nifi.cluster.node.event.history.size=25 +nifi.cluster.node.connection.timeout={{.Values.properties.clusterNodeConnectionTimeout}} +nifi.cluster.node.read.timeout={{.Values.properties.clusterNodeReadTimeout}} +nifi.cluster.node.max.concurrent.requests=100 +nifi.cluster.firewall.file= +nifi.cluster.flow.election.max.wait.time=1 mins +nifi.cluster.flow.election.max.candidates= + +# zookeeper properties, used for cluster management # +nifi.zookeeper.client.ensembleTracker={{.Values.properties.zkClientEnsembleTraker}} +nifi.zookeeper.connect.string= +nifi.zookeeper.connect.timeout={{.Values.properties.zookeeperConnectTimeout}} +nifi.zookeeper.session.timeout={{.Values.properties.zookeeperSessionTimeout}} +nifi.zookeeper.root.node=/nifi + +# Zookeeper properties for the authentication scheme used when creating acls on znodes used for cluster management +# Values supported for nifi.zookeeper.auth.type are "default", which will apply world/anyone rights on znodes +# and "sasl" which will give rights to the sasl/kerberos identity used to authenticate the nifi node +# The identity is determined using the value in nifi.kerberos.service.principal and the removeHostFromPrincipal +# and removeRealmFromPrincipal values (which should align with the kerberos.removeHostFromPrincipal and kerberos.removeRealmFromPrincipal +# values configured on the zookeeper server). +nifi.zookeeper.auth.type= +nifi.zookeeper.kerberos.removeHostFromPrincipal= +nifi.zookeeper.kerberos.removeRealmFromPrincipal= + +# kerberos # +nifi.kerberos.krb5.file= + +# kerberos service principal # +nifi.kerberos.service.principal= +nifi.kerberos.service.keytab.location= + +# kerberos spnego principal # +nifi.kerberos.spnego.principal= +nifi.kerberos.spnego.keytab.location= +nifi.kerberos.spnego.authentication.expiration=12 hours + +# external properties files for variable registry +# supports a comma delimited list of file locations +nifi.variable.registry.properties= diff --git a/external/nifi/configs/state-management.xml b/external/nifi/configs/state-management.xml new file mode 100644 index 0000000..111b516 --- /dev/null +++ b/external/nifi/configs/state-management.xml @@ -0,0 +1,19 @@ + + + + local-provider + org.apache.nifi.controller.state.providers.local.WriteAheadLocalStateProvider + ./state/local + false + 16 + 2 mins + + + zk-provider + org.apache.nifi.controller.state.providers.zookeeper.ZooKeeperStateProvider + {{ template "zookeeper.url" . }} + /nifi + 10 seconds + Open + + diff --git a/external/nifi/configs/zookeeper.properties b/external/nifi/configs/zookeeper.properties new file mode 100644 index 0000000..f0c45ab --- /dev/null +++ b/external/nifi/configs/zookeeper.properties @@ -0,0 +1,45 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# +# + +initLimit=10 +autopurge.purgeInterval=24 +syncLimit=5 +tickTime=2000 +dataDir=./state/zookeeper +autopurge.snapRetainCount=30 + +# +# Specifies the servers that are part of this zookeeper ensemble. For +# every NiFi instance running an embedded zookeeper, there needs to be +# a server entry below. For instance: +# +# server.1=nifi-node1-hostname:2888:3888;2181 +# server.2=nifi-node2-hostname:2888:3888;2181 +# server.3=nifi-node3-hostname:2888:3888;2181 +# +# The index of the server corresponds to the myid file that gets created +# in the dataDir of each node running an embedded zookeeper. See the +# administration guide for more details. +# + +server.1= + diff --git a/external/nifi/doc/FAQ.md b/external/nifi/doc/FAQ.md new file mode 100644 index 0000000..f36a05e --- /dev/null +++ b/external/nifi/doc/FAQ.md @@ -0,0 +1,17 @@ +FAQ - Frequently Asked Questions +================================ + +Readyness probe fails +--------------------- + +When encountering errors like `Readiness probe failed: Node not found with CONNECTED state` or `Multi-Attach error for volume "pvc-xxxxxx-xxx-xxx-xxxx-xxxxxxxxx" Volume is already exclusively attached to one node and can't be attached to another`, it means Kubernetes can't provide the pod access to the persistent data it wants. + +When this happens, reach out to your Kubernetes cluster administrators to find and fix the problem manually. + +For more background, see https://blog.mayadata.io/recover-from-volume-multi-attach-error-in-on-prem-kubernetes-clusters + +(see https://github.com/cetic/helm-nifi/issues/47#issuecomment-1122702262) + +## Session Afffinity + +As mentioned in the official NIFI document regarding [session affinity](https://nifi.apache.org/docs/nifi-docs/html/administration-guide.html#session_affinity), it's required to implement this feature for your ingress. Please refer to the ingress controller your are using for how to implement it. One example for GKE is with [issue #271](https://github.com/cetic/helm-nifi/issues/271). If NIFI cluster has more than one node, the session affinity has to be there due to the stateful implementation of each node. diff --git a/external/nifi/doc/INSTALLATION.md b/external/nifi/doc/INSTALLATION.md new file mode 100644 index 0000000..8f23097 --- /dev/null +++ b/external/nifi/doc/INSTALLATION.md @@ -0,0 +1,47 @@ +Installation +============= + + +### Install from local clone + +1. **Clone the repo** + +```bash +git clone https://github.com/cetic/helm-nifi.git +cd helm-nifi +helm repo add bitnami https://charts.bitnami.com/bitnami +helm repo add dysnix https://dysnix.github.io/charts/ +helm repo update +helm dep up +``` +2. **Set a sensitiveKey** + +In 1.23.2 version, Nifi needs a sensitiveKey to encrypt sensitive information. This key can be setted in the `values.yaml` file: + +```` +properties: + sensitiveKey: changeMechangeMe +```` + +3. **Configure a user authentication** + +This helm chart provides three types of authentication: Single User, LDAP and OIDC. + +You can find how to configure these authentications on this [page](doc/USERMANAGER.md). + +4. **Install Nifi** + +To install Nifi, run this command: + +```bash +helm install nifi . +``` +5. **Access Nifi** + +If you let the Nifi service in ClusterIP mode, you cannot reach Nifi from the outside of the cluster. To fix that, you have to make a port forwarding to access Nifi from the localhost. To do that, run the command below: + +```` +kubectl port-forward service/nifi 8443:8443 +```` + +Now you can access to Nifi with a browser by typing the address: `https://localhost:8443` diff --git a/external/nifi/doc/KEYCLOAK.md b/external/nifi/doc/KEYCLOAK.md new file mode 100644 index 0000000..0ab5943 --- /dev/null +++ b/external/nifi/doc/KEYCLOAK.md @@ -0,0 +1,109 @@ +KeyCloak +============= + +Example OIDC configuration with Keycloak provider + +## 0. Pull, extract and install the Keycloak helm chart +``` +helm repo add bitnami https://charts.bitnami.com/bitnami +helm repo update +helm pull bitnami/keycloak +tar -xzvf keycloak-*.tgz +cd keycloak/ +``` +Keycloak is first deployed with one admin user that we need to configure before deployment, to do so we need to pass the username/password in the `values.yaml` file like follows: + +``` +auth: + adminUser: admin + adminPassword: password +``` +Launching installation + + +``` +helm dep up +helm install my-release bitnami/keycloak +``` +## 1. Configure keycloak + + Keycloak logo + +> "Keycloak is an open source Identity and Access Management solution aimed at modern applications and services. It makes it easy to secure applications and services with little to no code." + +Once deployed we can access the user interface and log in: + +Keycloak login + +We click on **administration console** then we log in using the credentials we configured above. + +Keycloak home + +## Create realm + +When you log in to the admin console, you work in a realm, which is a space where you manage objects. Two types of realms exist: + +* **Master realm** - This realm was created for you when you first started Keycloak. It contains the admin account you created at the first login. You use this realm only to create other realms. + +* **Other realms** - These realms are created by the admin in the master realm. In these realms, administrators create users and applications. The applications are owned by the users. + +Keycloak realms + +To create a realm, head to the Master menu, click `Add Realm`. When you are logged in to the master realm, this menu lists all other realms, then type for example `devops` in the Name field to name our new realm devops. + +Keycloak add realm + +When we click `Create`, the main admin console page opens with realm set to devops, now we can switch between managing the master realm and the realm we just created by clicking entries in the `Select realm` drop-down list. + +Keycloak select realm + + +## Create Nifi client + +To create clients we first click **Clients** in the **left side menu** to open the Clients page. + +Keycloak create client + +On the right side, we click `Create` and then on the `Add Client` dialog, we create a client called `Nifi` by filling the fields as follows: + +* Client ID: `Nifi` +* Root URL: `\`, for this example our Nifi adress is https://localhost:8443 (don't forget the "s" of "https") + +Keycloak create client + +Once the client is created, we open the client configuration and change the **access type** to **confidential** from public, and complete the rest of the fields as shown below assuming our Nifi address is https://localhost:8443, then we **Save the config**. + + +Keycloak Nifi client created + + +Now we open the client Nifi again, go to **credentials tag** and copy the `client id` and `secret` because we are going to need them to configure Nifi later. + +Keycloak get Nifi credentials + +## Create user + + +In the devops realm, we need to create a new user and a temporary password for that new user, we head to the left menu, click Users to open the user list page. + +Create user + + +On the right side of the empty user list, click Add User to open the Add user page. + +Create user + + +we enter a name in the Username field (this is the only required field), then we flip the Email Verified switch to On and click Save. + +Create user + + + +The management page for the new user opens, we Click the **Credentials tab** to set a temporary password for the new user, we type a new password and confirm it. + +Create user + + +then we Click **Set Password** to set the user password to the new one we specified. + diff --git a/external/nifi/doc/README.md b/external/nifi/doc/README.md new file mode 100644 index 0000000..dbb9af2 --- /dev/null +++ b/external/nifi/doc/README.md @@ -0,0 +1,6 @@ +Nifi installation & configuration +========== + +* [Nifi installation](INSTALLATION.md) - install Nifi locally to update the chart +* [Users management](USERMANAGEMENT.md) - user identification and authorization (Single-User, OIDC, LDAP) +* [Keycloak](Keycloak.md) - example OIDC configuration with Keycloak provider diff --git a/external/nifi/doc/USERMANAGEMENT.md b/external/nifi/doc/USERMANAGEMENT.md new file mode 100644 index 0000000..09a298c --- /dev/null +++ b/external/nifi/doc/USERMANAGEMENT.md @@ -0,0 +1,122 @@ +# User Authentication + +This helm chart provides four types of authentication: Single User, Client Certificate, LDAP, and OIDC. These four authentication types can be managed essentialy from the `values.yaml` file. + +The parameter `admin` will set the initial admin username. If used in conjunction with an enabled LDAP configuration, this value will get used instead of the LDAP Bind DN for the admin username. + +## 1. Single User + +The Single User authentication is the default authentication in this helm chart. To login like a single user, the values below must be set in `values.yaml` file: + +``` +singleUser: + username: username + password: changemechangeme +``` + +## 2. Client Certificate + +Client Certificate authentication assumes a central Certificate Authority (CA) will issue a Client PKI Certificate and Server Certificate for the Nifi server. + +Add keystore files to a Kubernetes secret: + +``` +kubectl create secret generic mysecrets \ +--from-file=keystore.jks=/path/to/keystore.jks \ +--from-file=truststore.jks=/path/to/truststore.jks +``` + +Make the Kubernetes secret available to the Nifi server. Update `values.yaml`: + +``` +secrets: +- name: mysecrets + keys: + - keystore.jks + - truststore.jks + mountPath: /opt/nifi/nifi-current/config-data/certs/ +``` + +Enable the Nifi server to prompt for client certificates: + +``` +properties: + needClientAuth: true +``` + +Indicate Client Authentication mode configurations should be applied and set SSL values: + +``` +auth: + SSL: + keystorePasswd: + truststorePasswd: + clientAuth: + enabled: true +``` + +For cluster deployments, the example below illustrates how to create a 3 replica cluster with unique keystores. + +Create the secret: + +``` +kubectl create secret generic mysecrets \ +--from-file=.jks=/path/to/.jks \ +--from-file=.jks=/path/to/.jks \ +--from-file=.jks=/path/to/.jks \ +--from-file=truststore.jks=/path/to/truststore.jks +``` + +Make the secret available to the replicas: + +``` +secrets: +- name: mysecrets + keys: + - .jks + - .jks + - .jks + - truststore.jks + mountPath: /opt/nifi/nifi-current/config-data/certs/ +``` + +Add a safetyValve entry to align the container with the associated keystore: + +``` +properties: + safetyValve: + nifi.security.keystore: ${NIFI_HOME}/config-data/certs/${FQDN}.jks +``` + +## 3. OIDC + +OpenID Connect (OIDC) is an open authentication protocol that profiles and extends OAuth 2.0 to add an identity layer. It can be used by an external identity provider to make authentication. + +To enable OIDC user authentication, the values below must be set in `values.yaml` file: + +``` +oidc: + enabled: true + discoveryUrl: http://:/auth/realms//.well-known/openid-configuration + clientId: + clientSecret: + claimIdentifyingUser: email + admin: nifi@example.com +``` + +There are a lot of ID providers that can be used to perform an OIDC authentication. In our case, we have tested that with Keycloak. You will find an example of Keycloak config on this [page](doc/KEYCLOAK.md). + +## 4. LDAP + +Like OIDC, LDAP (Lightweight Directory Access Protocol) provide an external authentication. If you have your own LDAP, you can use it. If not, set `openldap.enabled` to `true` in `values.yaml` file to deploy a local instance of OpenLDAP. + +To enable authentication through LDAP, set the values below in `values.yaml` file: + +``` +ldap: + enabled: true + host: ldap://: + searchBase: CN=Users,DC=example,DC=com + admin: cn=admin,dc=example,dc=be + pass: changeMe +``` diff --git a/external/nifi/doc/images/installation/add-realm.png b/external/nifi/doc/images/installation/add-realm.png new file mode 100644 index 0000000..db27d1b Binary files /dev/null and b/external/nifi/doc/images/installation/add-realm.png differ diff --git a/external/nifi/doc/images/installation/add-user.png b/external/nifi/doc/images/installation/add-user.png new file mode 100644 index 0000000..920db35 Binary files /dev/null and b/external/nifi/doc/images/installation/add-user.png differ diff --git a/external/nifi/doc/images/installation/change-pass.png b/external/nifi/doc/images/installation/change-pass.png new file mode 100644 index 0000000..80b18cc Binary files /dev/null and b/external/nifi/doc/images/installation/change-pass.png differ diff --git a/external/nifi/doc/images/installation/client-nifi-created.PNG b/external/nifi/doc/images/installation/client-nifi-created.PNG new file mode 100644 index 0000000..f858e85 Binary files /dev/null and b/external/nifi/doc/images/installation/client-nifi-created.PNG differ diff --git a/external/nifi/doc/images/installation/client-nifi.png b/external/nifi/doc/images/installation/client-nifi.png new file mode 100644 index 0000000..0422e96 Binary files /dev/null and b/external/nifi/doc/images/installation/client-nifi.png differ diff --git a/external/nifi/doc/images/installation/devops-realm.png b/external/nifi/doc/images/installation/devops-realm.png new file mode 100644 index 0000000..6862256 Binary files /dev/null and b/external/nifi/doc/images/installation/devops-realm.png differ diff --git a/external/nifi/doc/images/installation/grafana-keycloak-auth.png b/external/nifi/doc/images/installation/grafana-keycloak-auth.png new file mode 100644 index 0000000..28652e0 Binary files /dev/null and b/external/nifi/doc/images/installation/grafana-keycloak-auth.png differ diff --git a/external/nifi/doc/images/installation/john-doe.png b/external/nifi/doc/images/installation/john-doe.png new file mode 100644 index 0000000..54f548e Binary files /dev/null and b/external/nifi/doc/images/installation/john-doe.png differ diff --git a/external/nifi/doc/images/installation/keycloak-clients.png b/external/nifi/doc/images/installation/keycloak-clients.png new file mode 100644 index 0000000..9c70ab6 Binary files /dev/null and b/external/nifi/doc/images/installation/keycloak-clients.png differ diff --git a/external/nifi/doc/images/installation/keycloak-first-screen.png b/external/nifi/doc/images/installation/keycloak-first-screen.png new file mode 100644 index 0000000..f945238 Binary files /dev/null and b/external/nifi/doc/images/installation/keycloak-first-screen.png differ diff --git a/external/nifi/doc/images/installation/keycloak-realms.png b/external/nifi/doc/images/installation/keycloak-realms.png new file mode 100644 index 0000000..f0581ad Binary files /dev/null and b/external/nifi/doc/images/installation/keycloak-realms.png differ diff --git a/external/nifi/doc/images/installation/keycloak-ui.png b/external/nifi/doc/images/installation/keycloak-ui.png new file mode 100644 index 0000000..da9f531 Binary files /dev/null and b/external/nifi/doc/images/installation/keycloak-ui.png differ diff --git a/external/nifi/doc/images/installation/nifi-credentials.PNG b/external/nifi/doc/images/installation/nifi-credentials.PNG new file mode 100644 index 0000000..dbb350f Binary files /dev/null and b/external/nifi/doc/images/installation/nifi-credentials.PNG differ diff --git a/external/nifi/doc/images/installation/users-page.png b/external/nifi/doc/images/installation/users-page.png new file mode 100644 index 0000000..9435fa5 Binary files /dev/null and b/external/nifi/doc/images/installation/users-page.png differ diff --git a/external/nifi/doc/images/logos/cetic.png b/external/nifi/doc/images/logos/cetic.png new file mode 100644 index 0000000..c66290f Binary files /dev/null and b/external/nifi/doc/images/logos/cetic.png differ diff --git a/external/nifi/doc/images/logos/helm.png b/external/nifi/doc/images/logos/helm.png new file mode 100644 index 0000000..7c83a7b Binary files /dev/null and b/external/nifi/doc/images/logos/helm.png differ diff --git a/external/nifi/doc/images/logos/keycloak-logo.png b/external/nifi/doc/images/logos/keycloak-logo.png new file mode 100644 index 0000000..c3439f4 Binary files /dev/null and b/external/nifi/doc/images/logos/keycloak-logo.png differ diff --git a/external/nifi/doc/images/logos/nifi.png b/external/nifi/doc/images/logos/nifi.png new file mode 100644 index 0000000..6677cd9 Binary files /dev/null and b/external/nifi/doc/images/logos/nifi.png differ diff --git a/external/nifi/templates/NOTES.txt b/external/nifi/templates/NOTES.txt new file mode 100644 index 0000000..e1ddf03 --- /dev/null +++ b/external/nifi/templates/NOTES.txt @@ -0,0 +1,6 @@ +To access the NiFi UI via kubectl port forwarding, +wait until the cluster is ready and then run: + +kubectl port-forward -n {{ .Release.Namespace}} svc/{{ .Release.Name }} {{ .Values.service.httpsPort }}:{{ .Values.service.httpsPort }} + +...and point your web browser to https://localhost:{{ .Values.service.httpsPort }}/nifi/ diff --git a/external/nifi/templates/_helpers.tpl b/external/nifi/templates/_helpers.tpl new file mode 100644 index 0000000..530c6f0 --- /dev/null +++ b/external/nifi/templates/_helpers.tpl @@ -0,0 +1,92 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "apache-nifi.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "apache-nifi.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "apache-nifi.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Form the Zookeeper Server part of the URL. If zookeeper is installed as part of this chart, use k8s service discovery, +else use user-provided server name +*/}} +{{- define "zookeeper.server" }} +{{- if .Values.zookeeper.enabled -}} +{{- printf "%s-zookeeper" .Release.Name }} +{{- else -}} +{{- printf "%s" .Values.zookeeper.url }} +{{- end -}} +{{- end -}} + +{{/* +Form the Zookeeper URL and port. If zookeeper is installed as part of this chart, use k8s service discovery, +else use user-provided name and port +*/}} +{{- define "zookeeper.url" }} +{{- $port := .Values.zookeeper.port | toString }} +{{- if .Values.zookeeper.enabled -}} +{{- printf "%s-zookeeper:%s" .Release.Name $port }} +{{- else -}} +{{- printf "%s:%s" .Values.zookeeper.url $port }} +{{- end -}} +{{- end -}} + +{{/* +Form the Nifi Registry URL and port. If nifi-registry is installed as part of this chart, use k8s service discovery, +else use user-provided name and port +*/}} +{{- define "registry.url" }} +{{- $port := .Values.registry.port | toString }} +{{- if .Values.registry.enabled -}} +{{- printf "http://%s-registry:%s" .Release.Name $port }} +{{- else -}} +{{- printf "http://%s:%s" .Values.registry.url $port }} +{{- end -}} +{{- end -}} + +{{/* +Create ca.server +*/}} +{{- define "ca.server" }} +{{- if .Values.ca.enabled -}} +{{- printf "%s-ca" .Release.Name }} +{{- else -}} +{{- printf "%s" .Values.ca.server }} +{{- end -}} +{{- end -}} + +{{/* +Set the service account name +*/}} +{{- define "apache-nifi.serviceAccountName" -}} +{{- if .Values.sts.serviceAccount.create }} +{{- default (include "apache-nifi.fullname" .) .Values.sts.serviceAccount.name }}-sa +{{- else }} +{{- default "default" .Values.sts.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/external/nifi/templates/cert-manager.yaml b/external/nifi/templates/cert-manager.yaml new file mode 100644 index 0000000..4e8f547 --- /dev/null +++ b/external/nifi/templates/cert-manager.yaml @@ -0,0 +1,157 @@ +{{ if .Values.certManager.enabled }} +--- +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: {{ template "apache-nifi.fullname" $ }} + namespace: {{ .Release.Namespace }} +spec: + selfSigned: {} +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: {{ template "apache-nifi.fullname" $ }}-ca + namespace: {{ .Release.Namespace }} +spec: + isCA: true + duration: {{ $.Values.certManager.caDuration }} + renewBefore: 5m1s + commonName: {{ template "apache-nifi.fullname" $ }}-ca.{{ $.Release.Namespace }}.svc.{{ $.Values.certManager.clusterDomain }} + subject: + organizationalUnits: + - NIFI + secretName: {{ template "apache-nifi.fullname" $ }}-ca + privateKey: + algorithm: RSA + size: 2048 + rotationPolicy: Always + issuerRef: + name: {{ template "apache-nifi.fullname" $ }} + kind: Issuer + group: cert-manager.io +--- +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: {{ template "apache-nifi.fullname" $ }}-ca + namespace: {{ .Release.Namespace }} +spec: + ca: + secretName: {{ template "apache-nifi.fullname" $ }}-ca +--- +{{ range untilStep 0 (int .Values.replicaCount) 1 }} +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: {{ template "apache-nifi.fullname" $ }}-{{ . }} + namespace: {{ $.Release.Namespace }} +spec: + duration: {{ $.Values.certManager.certDuration }} + commonName: {{ template "apache-nifi.fullname" $ }}-{{ . }}.{{ template "apache-nifi.fullname" $ }}-headless.{{ $.Release.Namespace }}.svc.{{ $.Values.certManager.clusterDomain }} + subject: + organizationalUnits: + - NIFI + secretName: {{ template "apache-nifi.fullname" $ }}-{{ . }} + privateKey: + rotationPolicy: Always + usages: + - digital signature + - content commitment + - key encipherment + - data encipherment + - key agreement + - server auth + - client auth + dnsNames: +{{ toYaml $.Values.certManager.additionalDnsNames | indent 4 }} + - {{ template "apache-nifi.fullname" $ }}.{{ $.Release.Namespace }}.svc + - {{ template "apache-nifi.fullname" $ }}.{{ $.Release.Namespace }}.svc.{{ $.Values.certManager.clusterDomain }} + - {{ template "apache-nifi.fullname" $ }}-{{ . }}.{{ template "apache-nifi.fullname" $ }}-headless.{{ $.Release.Namespace }}.svc + - {{ template "apache-nifi.fullname" $ }}-{{ . }}.{{ template "apache-nifi.fullname" $ }}-headless.{{ $.Release.Namespace }}.svc.{{ $.Values.certManager.clusterDomain }} + issuerRef: + name: {{ template "apache-nifi.fullname" $ }}-ca + kind: Issuer +--- +{{/* range untilStep 0 (int .Values.replicaCount ) 1 */}}{{ end }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "apache-nifi.fullname" $ }}-secret-reader + namespace: {{ $.Release.Namespace }} +automountServiceAccountToken: false +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ template "apache-nifi.fullname" $ }}-secret-reader-token + namespace: {{ $.Release.Namespace }} + annotations: + kubernetes.io/service-account.name: {{ template "apache-nifi.fullname" $ }}-secret-reader +type: kubernetes.io/service-account-token +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ template "apache-nifi.fullname" $ }}-secret-reader + namespace: {{ $.Release.Namespace }} +rules: +- apiGroups: [""] + resources: ["secrets"] + verbs: ["get"] +{{/* + SECURITY NOTE: This Role can read any secret in the namespace, which + at first can seem excessive. However, in mitigation, + this role's secret is only exposed to the cert-manager + sidecar. That sidecar runs a very simple script that + (1) can be audited quickly for vulnerabilities and + (2) has no attack surface exposure to user input. + + ALTERNATIVES + CONSIDERED: (A) Use https://github.com/spoditor/spoditor to provide + a mutating webhook that would only mount a given + pod's secret in that pod. + + Downside: Requires installing spoditor with + full cluster-wide privileges. + + (B) Write a chart-specific mutating webhook to perform + the same action, perhaps running under the + https://olm.operatorframework.io/ to limit the + webhook to the namespace. + + Downside: Again requires installation of OLM, an + operator with full cluster-wide privileges, plus + writing the mutating webhook and either building + a container to host it or injecting it into + (say) python:latest with a configMap mount, and + then maintaining that webhook code over time. + + (C) Modify the statefulset definition to have precisely + 1 replica, emitting individual pods for each NiFi node + through a large Helm loop. + + Downside: Ew. + + (D) Create a separate role for each pod and limit that + role to a single secret, associating the role based + on (say) pod name label. + + Downside: Not possible. k8s does not allow label-based + selection of resources in RBAC, c.f.: + https://github.com/kubernetes/kubernetes/issues/44703 +*/}} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ template "apache-nifi.fullname" $ }}-secret-reader + namespace: {{ $.Release.Namespace }} +subjects: +- kind: ServiceAccount + name: {{ template "apache-nifi.fullname" $ }}-secret-reader +roleRef: + kind: Role + name: {{ template "apache-nifi.fullname" $ }}-secret-reader + apiGroup: rbac.authorization.k8s.io +{{/* if .Values.certManager.enabled */}}{{ end }} diff --git a/external/nifi/templates/configmap.yaml b/external/nifi/templates/configmap.yaml new file mode 100644 index 0000000..f74ffb8 --- /dev/null +++ b/external/nifi/templates/configmap.yaml @@ -0,0 +1,12 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "apache-nifi.fullname" . }}-config + labels: + app: {{ include "apache-nifi.name" . | quote }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" + release: {{ .Release.Name | quote }} + heritage: {{ .Release.Service | quote }} +data: +{{ (tpl (.Files.Glob "configs/*").AsConfig . ) | indent 2 }} diff --git a/external/nifi/templates/ingress.yaml b/external/nifi/templates/ingress.yaml new file mode 100644 index 0000000..e398fa7 --- /dev/null +++ b/external/nifi/templates/ingress.yaml @@ -0,0 +1,47 @@ +--- +{{- if .Values.ingress.enabled -}} +{{- $fullName := include "apache-nifi.fullname" $ -}} +{{- $ingressPath := .Values.ingress.path -}} +{{- $ingressHttpsPort := .Values.service.httpsPort -}} +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: {{ template "apache-nifi.fullname" $ }}-ingress + namespace: {{ .Release.Namespace }} + labels: + app: {{ include "apache-nifi.name" . | quote }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" + release: {{ .Release.Name | quote }} + heritage: {{ .Release.Service | quote }} +{{- with .Values.ingress.annotations }} + annotations: +{{ toYaml . | indent 4 }} +{{- end }} +spec: +{{- if .Values.ingress.className }} + ingressClassName: {{ .Values.ingress.className | quote }} +{{- end }} +{{- if .Values.ingress.tls }} + tls: + {{- range .Values.ingress.tls }} + - hosts: + {{- range .hosts }} + - {{ . | quote }} + {{- end }} + secretName: {{ .secretName }} + {{- end }} +{{- end }} + rules: + {{- range .Values.ingress.hosts }} + - host: {{ . }} + http: + paths: + - path: {{ $ingressPath }} + pathType: Prefix + backend: + service: + name: {{ $fullName }} + port: + number: {{ $ingressHttpsPort }} + {{- end }} +{{- end }} diff --git a/external/nifi/templates/route.yaml b/external/nifi/templates/route.yaml new file mode 100644 index 0000000..d0b3ae6 --- /dev/null +++ b/external/nifi/templates/route.yaml @@ -0,0 +1,33 @@ +{{- if .Values.openshift.route.enabled -}} +{{- $fullName := include "apache-nifi.fullname" . -}} +{{- $ingressPath := .Values.openshift.route.path -}} +kind: Route +apiVersion: route.openshift.io/v1 +metadata: + labels: + app: {{ include "apache-nifi.name" . | quote }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" + release: {{ .Release.Name | quote }} + heritage: {{ .Release.Service | quote }} + name: {{ template "apache-nifi.fullname" . }} +spec: + {{- if .Values.openshift.route.host }} + host: {{ .Values.openshift.route.host }} + {{- end }} + {{- if .Values.openshift.route.host }} + path: {{ $ingressPath }} + {{- end }} + to: + kind: Service + name: {{ $fullName }} + weight: 100 + port: + targetPort: https + tls: + {{- if .Values.properties.externalSecure }} + termination: edge + {{- else }} + termination: passthrough + {{- end }} + insecureEdgeTerminationPolicy: Redirect +{{- end }} \ No newline at end of file diff --git a/external/nifi/templates/scc.yaml b/external/nifi/templates/scc.yaml new file mode 100644 index 0000000..e781cc8 --- /dev/null +++ b/external/nifi/templates/scc.yaml @@ -0,0 +1,45 @@ +{{- if .Values.openshift.scc.enabled -}} +apiVersion: security.openshift.io/v1 +kind: SecurityContextConstraints +metadata: + labels: + app: {{ include "apache-nifi.name" . | quote }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" + release: {{ .Release.Name | quote }} + heritage: {{ .Release.Service | quote }} + annotations: + kubernetes.io/description: {{ template "apache-nifi.fullname" . }}-scc provides all features of the restricted SCC but + allows users to run with any UID and any GID. + name: {{ template "apache-nifi.fullname" . }}-scc +allowHostDirVolumePlugin: false +allowHostIPC: false +allowHostNetwork: false +allowHostPID: false +allowHostPorts: false +allowPrivilegeEscalation: true +allowPrivilegedContainer: false +allowedCapabilities: null +defaultAddCapabilities: null +fsGroup: + type: RunAsAny +groups: [] +priority: 10 +readOnlyRootFilesystem: false +requiredDropCapabilities: +- MKNOD +runAsUser: + type: RunAsAny +seLinuxContext: + type: MustRunAs +supplementalGroups: + type: RunAsAny +users: +- system:serviceaccount:{{ .Release.Namespace }}:{{ include "apache-nifi.serviceAccountName" . }} +volumes: +- configMap +- downwardAPI +- emptyDir +- persistentVolumeClaim +- projected +- secret +{{- end }} \ No newline at end of file diff --git a/external/nifi/templates/service.yaml b/external/nifi/templates/service.yaml new file mode 100644 index 0000000..5de375b --- /dev/null +++ b/external/nifi/templates/service.yaml @@ -0,0 +1,109 @@ +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "apache-nifi.fullname" . }}-headless + labels: + app: {{ include "apache-nifi.name" . | quote }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" + release: {{ .Release.Name | quote }} + heritage: {{ .Release.Service | quote }} +{{- if .Values.headless.annotations }} + annotations: +{{ toYaml .Values.headless.annotations | indent 4 }} +{{- end }} +spec: + type: {{ .Values.headless.type }} + clusterIP: None + ports: + - port: {{ .Values.properties.httpsPort }} + name: https + - port: {{ .Values.properties.clusterPort }} + name: cluster + - port: {{ .Values.properties.siteToSite.port }} + name: site-to-site + selector: + app: {{ include "apache-nifi.name" . | quote }} + release: {{ .Release.Name | quote }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "apache-nifi.fullname" . }} + labels: + app: {{ include "apache-nifi.name" . | quote }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" + release: {{ .Release.Name | quote }} + heritage: {{ .Release.Service | quote }} +{{- if .Values.service.annotations }} + annotations: +{{ toYaml .Values.service.annotations | indent 4 }} +{{- end }} +spec: + type: {{ .Values.service.type }} + {{- if .Values.service.loadBalancerIP }} + loadBalancerIP: {{ .Values.service.loadBalancerIP }} + {{- end }} + {{- if .Values.service.loadBalancerSourceRanges }} + loadBalancerSourceRanges: +{{ with .Values.service.loadBalancerSourceRanges }} +{{ toYaml . | indent 4 }} + {{- end }} + {{- end }} + {{- if .Values.service.sessionAffinity }} + sessionAffinity: {{ .Values.service.sessionAffinity }} + {{- end }} + {{- if .Values.service.sessionAffinityConfig }} + sessionAffinityConfig: +{{- with .Values.service.sessionAffinityConfig }} +{{ toYaml . | indent 4 }} + {{- end }} + {{- end }} + ports: + - port: {{ .Values.service.httpsPort }} + name: https + targetPort: {{ .Values.properties.httpsPort }} + nodePort: {{ .Values.service.nodePort }} + - port: {{ .Values.properties.siteToSite.port }} + name: site-to-site + targetPort: {{ .Values.properties.siteToSite.port }} + {{- if .Values.properties.siteToSite.nodePort }} + nodePort: {{ .Values.properties.siteToSite.nodePort }} + {{- end }} +{{- if .Values.service.processors.enabled }} +{{- with .Values.service.processors.ports }} +{{- range . }} + - port: {{ .port }} + name: {{ .name }} + targetPort: {{ .targetPort }} + {{- if eq $.Values.service.type "NodePort" }} + nodePort: {{ .nodePort }} + {{- end }} +{{- end }} +{{- end }} +{{- end }} + selector: + app: {{ template "apache-nifi.name" . }} + release: {{ .Release.Name }} +{{- if .Values.metrics.prometheus.enabled }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "apache-nifi.fullname" . }}-metrics + labels: + app: {{ include "apache-nifi.name" . | quote }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" + release: {{ .Release.Name | quote }} + heritage: {{ .Release.Service | quote }} +spec: + ports: + - name: metrics + port: {{ .Values.metrics.prometheus.port }} + protocol: TCP + targetPort: metrics + selector: + app: {{ template "apache-nifi.name" . }} + release: {{ .Release.Name }} + type: ClusterIP + {{- end }} \ No newline at end of file diff --git a/external/nifi/templates/serviceaccount.yaml b/external/nifi/templates/serviceaccount.yaml new file mode 100644 index 0000000..a66d6ef --- /dev/null +++ b/external/nifi/templates/serviceaccount.yaml @@ -0,0 +1,15 @@ +{{- if .Values.sts.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + app: {{ include "apache-nifi.name" . | quote }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" + release: {{ .Release.Name | quote }} + heritage: {{ .Release.Service | quote }} + name: {{ include "apache-nifi.serviceAccountName" . }} + {{- with .Values.sts.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/external/nifi/templates/servicemonitor.yaml b/external/nifi/templates/servicemonitor.yaml new file mode 100644 index 0000000..4f3f18f --- /dev/null +++ b/external/nifi/templates/servicemonitor.yaml @@ -0,0 +1,40 @@ +{{- if .Values.metrics.prometheus.serviceMonitor.enabled }} +--- +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "apache-nifi.fullname" . }} +{{- with .Values.metrics.prometheus.serviceMonitor.namespace }} + namespace: {{ . }} +{{- end }} + labels: + app: {{ include "apache-nifi.name" . | quote }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" + release: {{ .Release.Name | quote }} + heritage: {{ .Release.Service | quote }} +{{- if .Values.metrics.prometheus.serviceMonitor.labels }} +{{ .Values.metrics.prometheus.serviceMonitor.labels | toYaml | indent 4 }} +{{- end }} +spec: + endpoints: + - port: metrics + honorLabels: true +{{- if .Values.metrics.prometheus.serviceMonitor.interval }} + interval: {{ .Values.metrics.prometheus.serviceMonitor.interval }} +{{- end }} +{{- if .Values.metrics.prometheus.serviceMonitor.metricRelabelings }} + metricRelabelings: +{{ tpl (toYaml .Values.metrics.prometheus.serviceMonitor.metricRelabelings | indent 6) . }} +{{- end }} +{{- if .Values.metrics.prometheus.serviceMonitor.relabelings }} + relabelings: +{{ toYaml .Values.metrics.prometheus.serviceMonitor.relabelings | indent 6 }} +{{- end }} + selector: + matchLabels: + app: {{ template "apache-nifi.name" . }} + release: {{ .Release.Name }} + namespaceSelector: + matchNames: + - {{ .Release.Namespace | quote }} +{{- end }} diff --git a/external/nifi/templates/statefulset.yaml b/external/nifi/templates/statefulset.yaml new file mode 100644 index 0000000..97d4b27 --- /dev/null +++ b/external/nifi/templates/statefulset.yaml @@ -0,0 +1,932 @@ +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ template "apache-nifi.fullname" . }} + labels: + app: {{ include "apache-nifi.name" . | quote }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" + release: {{ .Release.Name | quote }} + heritage: {{ .Release.Service | quote }} +spec: + podManagementPolicy: {{ .Values.sts.podManagementPolicy }} + serviceName: {{ template "apache-nifi.fullname" . }}-headless + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + app: {{ template "apache-nifi.name" . }} + release: {{ .Release.Name }} + template: + metadata: + annotations: +{{- if .Values.sts.pod.annotations }} +{{ toYaml .Values.sts.pod.annotations | indent 8 }} +{{- else }} + security.alpha.kubernetes.io/sysctls: net.ipv4.ip_local_port_range=10000 65000 +{{- end }} + labels: + app: {{ include "apache-nifi.name" . | quote }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" + release: {{ .Release.Name | quote }} + heritage: {{ .Release.Service | quote }} +{{- if .Values.extraLabels }} +{{ toYaml .Values.extraLabels | indent 8 }} +{{- end }} + spec: +{{- if .Values.sts.useHostNetwork }} + hostNetwork: {{ .Values.sts.useHostNetwork }} + dnsPolicy: ClusterFirstWithHostNet +{{- end }} +{{- if .Values.sts.hostAliases }} + hostAliases: +{{ toYaml .Values.sts.hostAliases | indent 8 }} +{{- end }} + serviceAccountName: {{ include "apache-nifi.serviceAccountName" . }} + {{- if eq .Values.sts.AntiAffinity "hard" }} + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: "app" + operator: In + values: + - {{ include "apache-nifi.name" . | quote }} + topologyKey: "kubernetes.io/hostname" + {{- else if eq .Values.sts.AntiAffinity "soft" }} + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 1 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: "app" + operator: In + values: + - {{ include "apache-nifi.name" . | quote }} + topologyKey: "kubernetes.io/hostname" + {{- end }} +{{- if and .Values.affinity (and (ne .Values.sts.AntiAffinity "hard") (ne .Values.sts.AntiAffinity "soft")) }} + affinity: +{{ toYaml .Values.affinity | indent 8 }} +{{- end }} +{{- if .Values.terminationGracePeriodSeconds }} + terminationGracePeriodSeconds: {{ .Values.terminationGracePeriodSeconds }} +{{- end }} +{{- if .Values.tolerations }} + tolerations: +{{ toYaml .Values.tolerations | indent 8 }} +{{- end }} +{{- if .Values.nodeSelector }} + nodeSelector: +{{ toYaml .Values.nodeSelector | indent 8 }} +{{- end }} +{{- if .Values.securityContext }} + securityContext: +{{ toYaml .Values.securityContext | indent 8 }} +{{- end }} + initContainers: +{{- if .Values.properties.isNode }} + - name: zookeeper + imagePullPolicy: {{ .Values.sidecar.imagePullPolicy | default "Always" | quote }} + image: "{{ .Values.sidecar.image }}:{{ .Values.sidecar.tag }}" + command: + - sh + - -c + - | + echo trying to contact {{ template "zookeeper.server" . }} {{ .Values.zookeeper.port }} + until nc -vzw 1 {{ template "zookeeper.server" . }} {{ .Values.zookeeper.port }}; do + echo "waiting for zookeeper..." + sleep 2 + done +{{- end }} +{{- range $key, $value := .Values.initContainers }} + - name: {{ $key }} +{{ toYaml $value | indent 8 }} +{{- end }} + {{- if .Values.image.pullSecret }} + imagePullSecrets: + - name: {{ .Values.image.pullSecret }} + {{- end }} + containers: +{{- if .Values.extraContainers }} +{{- toYaml .Values.extraContainers | nindent 6 }} +{{- end }} + - name: server + imagePullPolicy: {{ .Values.image.pullPolicy | quote }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + command: + - bash + - -ce + - | + prop_replace () { + target_file=${NIFI_HOME}/conf/${3:-nifi.properties} + echo "updating ${1} in ${target_file}" + if egrep "^${1}=" ${target_file} &> /dev/null; then + sed -i -e "s|^$1=.*$|$1=$2|" ${target_file} + else + echo ${1}=${2} >> ${target_file} + fi + } + mkdir -p ${NIFI_HOME}/config-data/conf +{{- if .Values.sts.useHostNetwork }} + FQDN="0.0.0.0" +{{- else }} + FQDN=$(hostname -f) +{{- end }} + + cat "${NIFI_HOME}/conf/nifi.temp" > "${NIFI_HOME}/conf/nifi.properties" + +{{- if .Values.auth.ldap.enabled }} + cat "${NIFI_HOME}/conf/authorizers.temp" > "${NIFI_HOME}/conf/authorizers.xml" + cat "${NIFI_HOME}/conf/login-identity-providers-ldap.xml" > "${NIFI_HOME}/conf/login-identity-providers.xml" +{{- else if .Values.auth.oidc.enabled }} + prop_replace nifi.security.user.login.identity.provider '' + prop_replace nifi.security.user.authorizer managed-authorizer + prop_replace nifi.security.user.oidc.discovery.url {{ .Values.auth.oidc.discoveryUrl }} + prop_replace nifi.security.user.oidc.client.id {{ .Values.auth.oidc.clientId }} + prop_replace nifi.security.user.oidc.client.secret {{ .Values.auth.oidc.clientSecret }} + prop_replace nifi.security.user.oidc.claim.identifying.user {{ .Values.auth.oidc.claimIdentifyingUser }} + xmlstarlet ed --inplace --delete "//authorizers/authorizer[identifier='single-user-authorizer']" "${NIFI_HOME}/conf/authorizers.xml" + xmlstarlet ed --inplace --update "//authorizers/userGroupProvider/property[@name='Users File']" -v './auth-conf/users.xml' "${NIFI_HOME}/conf/authorizers.xml" + xmlstarlet ed --inplace --delete "//authorizers/userGroupProvider/property[@name='Initial User Identity 1']" "${NIFI_HOME}/conf/authorizers.xml" + xmlstarlet ed --inplace \ + --subnode "authorizers/userGroupProvider" --type 'elem' -n 'property' \ + --value {{ .Values.auth.oidc.admin | quote }} \ + --insert "authorizers/userGroupProvider/property[not(@name)]" --type attr -n name \ + --value "Initial User Identity {{ .Values.replicaCount }}" \ + "${NIFI_HOME}/conf/authorizers.xml" + xmlstarlet ed --inplace --update "//authorizers/accessPolicyProvider/property[@name='Initial Admin Identity']" -v {{ .Values.auth.oidc.admin | quote }} "${NIFI_HOME}/conf/authorizers.xml" + xmlstarlet ed --inplace --update "//authorizers/accessPolicyProvider/property[@name='Authorizations File']" -v './auth-conf/authorizations.xml' "${NIFI_HOME}/conf/authorizers.xml" + {{- if .Values.properties.isNode }} + xmlstarlet ed --inplace --delete "authorizers/accessPolicyProvider/property[@name='Node Identity 1']" "${NIFI_HOME}/conf/authorizers.xml" + {{ range untilStep 0 (int .Values.replicaCount) 1 }} + xmlstarlet ed --inplace \ + --subnode "authorizers/accessPolicyProvider" --type 'elem' -n 'property' \ + --value "CN={{ template "apache-nifi.fullname" $ }}-{{ . }}.{{ template "apache-nifi.fullname" $ }}-headless.{{ $.Release.Namespace }}.svc.{{ $.Values.certManager.clusterDomain }}, OU=NIFI" \ + --insert "authorizers/accessPolicyProvider/property[not(@name)]" --type attr -n name \ + --value "Node Identity {{ . }}" \ + "${NIFI_HOME}/conf/authorizers.xml" + xmlstarlet ed --inplace \ + --subnode "authorizers/userGroupProvider" --type 'elem' -n 'property' \ + --value "CN={{ template "apache-nifi.fullname" $ }}-{{ . }}.{{ template "apache-nifi.fullname" $ }}-headless.{{ $.Release.Namespace }}.svc.{{ $.Values.certManager.clusterDomain }}, OU=NIFI" \ + --insert "authorizers/userGroupProvider/property[not(@name)]" --type attr -n name \ + --value "Initial User Identity {{ . }}" \ + "${NIFI_HOME}/conf/authorizers.xml" + {{/* range untilStep 0 (int .Values.replicaCount ) 1 */}}{{ end }} + {{- end }} +{{- else if .Values.auth.clientAuth.enabled }} + cat "${NIFI_HOME}/conf/authorizers.temp" > "${NIFI_HOME}/conf/authorizers.xml" + xmlstarlet ed --inplace --delete "//authorizers/authorizer[identifier='single-user-authorizer']" "${NIFI_HOME}/conf/authorizers.xml" +{{- else if .Values.auth.singleUser.username }} + bin/nifi.sh set-single-user-credentials {{ .Values.auth.singleUser.username }} {{ .Values.auth.singleUser.password }} +{{- end }} + +{{ if .Values.customFlow }} + gzip < {{ .Values.customFlow }} > /opt/nifi/data/flow.xml.gz +{{ else }} + if ! test -f /opt/nifi/data/flow.xml.gz && test -f /opt/nifi/data/flow.xml; then + gzip /opt/nifi/data/flow.xml + fi +{{ end }} + + prop_replace nifi.ui.banner.text $(hostname -s) + prop_replace nifi.remote.input.host ${FQDN} + prop_replace nifi.cluster.node.address ${FQDN} + prop_replace nifi.zookeeper.connect.string ${NIFI_ZOOKEEPER_CONNECT_STRING} + prop_replace nifi.web.http.host ${FQDN} + +{{- if .Values.properties.webProxyHost }} + # Update nifi.properties for web ui proxy hostname + prop_replace nifi.web.proxy.host {{ .Values.properties.webProxyHost }} +{{- else }} + prop_replace nifi.web.proxy.host {{ template "apache-nifi.fullname" $ }}.{{ .Release.Namespace }}.svc +{{- end }} + +{{- if .Values.certManager.enabled }} + xmlstarlet ed --inplace --delete "authorizers/accessPolicyProvider/property[@name='Node Identity 1']" "${NIFI_HOME}/conf/authorizers.xml" +{{ range untilStep 0 (int .Values.replicaCount) 1 }} + xmlstarlet ed --inplace \ + --subnode "authorizers/accessPolicyProvider" --type 'elem' -n 'property' \ + --value "CN={{ template "apache-nifi.fullname" $ }}-{{ . }}.{{ template "apache-nifi.fullname" $ }}-headless.{{ $.Release.Namespace }}.svc.{{ $.Values.certManager.clusterDomain }}, OU=NIFI" \ + --insert "authorizers/accessPolicyProvider/property[not(@name)]" --type attr -n name \ + --value "Node Identity {{ . }}" \ + "${NIFI_HOME}/conf/authorizers.xml" + xmlstarlet ed --inplace \ + --subnode "authorizers/userGroupProvider" --type 'elem' -n 'property' \ + --value "CN={{ template "apache-nifi.fullname" $ }}-{{ . }}.{{ template "apache-nifi.fullname" $ }}-headless.{{ $.Release.Namespace }}.svc.{{ $.Values.certManager.clusterDomain }}, OU=NIFI" \ + --insert "authorizers/userGroupProvider/property[not(@name)]" --type attr -n name \ + --value "Initial User Identity {{ . }}" \ + "${NIFI_HOME}/conf/authorizers.xml" +{{/* range untilStep 0 (int .Values.replicaCount ) 1 */}}{{ end }} + + prop_replace nifi.security.keystore "${NIFI_HOME}/tls/keystore.jks" + prop_replace nifi.security.keystoreType JKS + prop_replace nifi.security.keystorePasswd "{{ .Values.certManager.keystorePasswd }}" + prop_replace nifi.security.truststore "${NIFI_HOME}/tls/truststore.jks" + prop_replace nifi.security.truststoreType JKS + prop_replace nifi.security.truststorePasswd "{{ .Values.certManager.truststorePasswd }}" + + prop_replace nifi.web.https.host "$(hostname).{{ template "apache-nifi.fullname" $ }}-headless.{{ $.Release.Namespace }}.svc.{{ $.Values.certManager.clusterDomain }}" + prop_replace nifi.cluster.node.address "$(hostname).{{ template "apache-nifi.fullname" $ }}-headless.{{ $.Release.Namespace }}.svc.{{ $.Values.certManager.clusterDomain }}" + prop_replace nifi.web.https.network.interface.default eth0 + prop_replace nifi.web.https.network.interface.lo lo + prop_replace nifi.web.http.host "" + prop_replace nifi.web.http.port "" + + prop_replace nifi.security.autoreload.enabled true + prop_replace nifi.security.autoreload.interval "{{ .Values.certManager.refreshSeconds }} secs" + + while [ ! -r "${NIFI_HOME}/tls/truststore.jks" ] + do + echo "${NIFI_HOME}/tls/truststore.jks" is not readable! Waiting for cert-manager sidecar to populate it. + sleep 2 + done + + while [ ! -r "${NIFI_HOME}/tls/keystore.jks" ] + do + echo "${NIFI_HOME}/tls/keystore.jks" is not readable! Waiting for cert-manager sidecar to populate it. + sleep 2 + done + +{{- /* if .Values.certManager.enabled */}}{{ else }} + + if [ ! -r "${NIFI_HOME}/conf/nifi-cert.pem" ] + then + /opt/nifi/nifi-toolkit-current/bin/tls-toolkit.sh standalone \ + -n '{{.Release.Name}}-nifi-0.{{.Release.Name}}-nifi-headless.{{.Release.Namespace}}.svc.cluster.local' \ + -C '{{.Values.auth.admin}}' \ + -o "${NIFI_HOME}/conf/" \ + -P {{.Values.auth.SSL.truststorePasswd}} \ + -S {{.Values.auth.SSL.keystorePasswd}} \ + --nifiPropertiesFile /opt/nifi/nifi-current/conf/nifi.properties + fi + +{{- /* if .Values.certManager.enabled */}}{{ end }} + +{{- if .Values.properties.safetyValve }} + {{- range $prop, $val := .Values.properties.safetyValve }} + prop_replace {{ $prop }} "{{ $val }}" nifi.properties + {{- end }} +{{- end }} + +{{- if .Values.properties.sensitiveKeySetFile }} + if [ ! -r {{ .Values.properties.sensitiveKeySetFile | quote }} ] + then +{{- if .Values.properties.sensitiveKeyPrior }} + prop_replace nifi.sensitive.props.key {{ .Values.properties.sensitiveKeyPrior | quote }} +{{- /* if .Values.properties.sensitiveKeyPrior */}}{{ else }} + prop_replace nifi.sensitive.props.key "" +{{- /* if .Values.properties.sensitiveKeyPrior */}}{{ end }} + bin/nifi.sh set-sensitive-properties-key {{ .Values.properties.sensitiveKey | quote }} + touch {{ .Values.properties.sensitiveKeySetFile | quote }} + fi +{{- /* if .Values.properties.sensitiveKeySetFile */}}{{ end }} + + for f in "${NIFI_HOME}/conf/authorizers.xml" "${NIFI_HOME}/conf/login-identity-providers.xml" ${NIFI_HOME}/conf/nifi.properties + do + echo === $f === + cat $f + done + echo === end of files === + + function prop () { + target_file=${NIFI_HOME}/conf/nifi.properties + egrep "^${1}=" ${target_file} | cut -d'=' -f2 + } + + function offloadNode() { + FQDN=$(hostname -f) + echo "disconnecting node '$FQDN'" + baseUrl=https://${FQDN}:{{ .Values.properties.httpsPort }} + + echo "keystoreType=$(prop nifi.security.keystoreType)" > secure.properties + echo "keystore=$(prop nifi.security.keystore)" >> secure.properties + echo "keystorePasswd=$(prop nifi.security.keystorePasswd)" >> secure.properties + echo "truststoreType=$(prop nifi.security.truststoreType)" >> secure.properties + echo "truststore=$(prop nifi.security.truststore)" >> secure.properties + echo "truststorePasswd=$(prop nifi.security.truststorePasswd)" >> secure.properties + echo "proxiedEntity={{ .Values.auth.admin }}" >> secure.properties + + secureArgs="-p secure.properties" + + echo baseUrl ${baseUrl} + echo "gracefully disconnecting node '$FQDN' from cluster" + ${NIFI_TOOLKIT_HOME}/bin/cli.sh nifi get-nodes -ot json -u ${baseUrl} ${secureArgs} > nodes.json + nnid=$(jq --arg FQDN "$FQDN" '.cluster.nodes[] | select(.address==$FQDN) | .nodeId' nodes.json) + echo "disconnecting node ${nnid}" + ${NIFI_TOOLKIT_HOME}/bin/cli.sh nifi disconnect-node -nnid $nnid -u ${baseUrl} ${secureArgs} + echo "" + echo "get a connected node" + connectedNode=$(jq -r 'first(.cluster.nodes|=sort_by(.address)| .cluster.nodes[] | select(.status=="CONNECTED")) | .address' nodes.json) + baseUrl=https://${connectedNode}:{{ .Values.properties.httpsPort }} + echo baseUrl ${baseUrl} + echo "" + echo "wait until node has state 'DISCONNECTED'" + while [[ "${node_state}" != "DISCONNECTED" ]]; do + sleep 1 + ${NIFI_TOOLKIT_HOME}/bin/cli.sh nifi get-nodes -ot json -u ${baseUrl} ${secureArgs} > nodes.json + node_state=$(jq -r --arg FQDN "$FQDN" '.cluster.nodes[] | select(.address==$FQDN) | .status' nodes.json) + echo "state is '${node_state}'" + done + echo "" + echo "node '${nnid}' was disconnected" + echo "offloading node" + ${NIFI_TOOLKIT_HOME}/bin/cli.sh nifi offload-node -nnid $nnid -u ${baseUrl} ${secureArgs} + echo "" + echo "wait until node has state 'OFFLOADED'" + while [[ "${node_state}" != "OFFLOADED" ]]; do + sleep 1 + ${NIFI_TOOLKIT_HOME}/bin/cli.sh nifi get-nodes -ot json -u ${baseUrl} ${secureArgs} > nodes.json + node_state=$(jq -r --arg FQDN "$FQDN" '.cluster.nodes[] | select(.address==$FQDN) | .status' nodes.json) + echo "state is '${node_state}'" + done + } + + deleteNode() { + echo "deleting node" + ${NIFI_TOOLKIT_HOME}/bin/cli.sh nifi delete-node -nnid ${nnid} -u ${baseUrl} ${secureArgs} + echo "node deleted" + } + + executeTrap() { + echo Received trapped signal, beginning shutdown...; +{{- if .Values.properties.isNode }} + offloadNode; +{{- end }} + ./bin/nifi.sh stop; +{{- if .Values.properties.isNode }} + deleteNode; +{{- end }} + exit 0; + } + + trap executeTrap TERM HUP INT; + trap ":" EXIT + + exec bin/nifi.sh run & nifi_pid="$!" + echo NiFi running with PID ${nifi_pid}. + wait ${nifi_pid} + + resources: +{{ toYaml .Values.resources | indent 10 }} + ports: +{{- if .Values.metrics.prometheus.enabled }} + - containerPort: {{ .Values.metrics.prometheus.port }} + name: metrics + protocol: TCP +{{- end }} + - containerPort: {{ .Values.properties.httpsPort }} +{{- if .Values.sts.hostPort }} + hostPort: {{ .Values.sts.hostPort }} +{{- end }} + name: https + protocol: TCP + - containerPort: {{ .Values.properties.clusterPort }} + name: cluster + protocol: TCP +{{- if .Values.containerPorts }} +{{ toYaml .Values.containerPorts | indent 8 }} +{{- end }} + env: + - name: NIFI_ZOOKEEPER_CONNECT_STRING + value: {{ template "zookeeper.url" . }} +{{- if not (or (.Values.auth.ldap.enabled) (.Values.auth.oidc.enabled)) }} + - name: NIFI_WEB_HTTPS_HOST + value: 0.0.0.0 +{{- end }} +{{- if .Values.env }} +{{ toYaml .Values.env | indent 8 }} +{{- end }} +{{- if .Values.envFrom }} + envFrom: +{{ toYaml .Values.envFrom | indent 8 }} +{{- end }} +{{- if .Values.postStart }} + lifecycle: + postStart: + exec: + command: ["/bin/sh", "-c", {{ .Values.postStart | quote }}] +{{- end }} +{{- if .Values.properties.isNode }} + readinessProbe: +{{- if not .Values.sts.startupProbe.enabled }} + initialDelaySeconds: 60 +{{- end }} + periodSeconds: 20 + tcpSocket: + port: {{ .Values.properties.httpsPort }} +# exec: +# command: +# - bash +# - -c +# - | +# {{- if .Values.properties.httpsPort }} +# curl -kv \ +# --cert ${NIFI_HOME}/config-data/certs/admin/crt.pem --cert-type PEM \ +# --key ${NIFI_HOME}/config-data/certs/admin/key.pem --key-type PEM \ +# https://$(hostname -f):{{ .Values.properties.httpsPort }}/nifi-api/controller/cluster > $NIFI_BASE_DIR/data/cluster.state +# {{- else }} +# curl -kv \ +# http://$(hostname -f):{{ .Values.properties.httpPort }}/nifi-api/controller/cluster > $NIFI_BASE_DIR/data/cluster.state +# {{- end }} +# STATUS=$(jq -r ".cluster.nodes[] | select((.address==\"$(hostname -f)\") or .address==\"localhost\") | .status" $NIFI_BASE_DIR/data/cluster.state) + +# if [[ ! $STATUS = "CONNECTED" ]]; then +# echo "Node not found with CONNECTED state. Full cluster state:" +# jq . $NIFI_BASE_DIR/data/cluster.state +# exit 1 +# fi +{{- end }} +{{- if .Values.sts.startupProbe.enabled }} + startupProbe: + failureThreshold: {{ .Values.sts.startupProbe.failureThreshold }} + periodSeconds: {{ .Values.sts.startupProbe.periodSeconds }} + tcpSocket: + port: {{ .Values.properties.httpsPort }} +{{- end }} + livenessProbe: +{{- if not .Values.sts.startupProbe.enabled }} + initialDelaySeconds: 90 +{{- end }} + periodSeconds: 60 + tcpSocket: + port: {{ .Values.properties.httpsPort }} + volumeMounts: + - mountPath: /opt/nifi/nifi-current/logs + {{- if and .Values.persistence.enabled .Values.persistence.subPath.enabled }} + name: {{ .Values.persistence.subPath.name }} + subPath: logs + {{- else }} + name: "logs" + {{- end }} + - mountPath: /opt/nifi/data + {{- if and .Values.persistence.enabled .Values.persistence.subPath.enabled }} + name: {{ .Values.persistence.subPath.name }} + subPath: data + {{- else }} + name: "data" + {{- end }} + - mountPath: /opt/nifi/nifi-current/auth-conf/ + {{- if and .Values.persistence.enabled .Values.persistence.subPath.enabled }} + name: {{ .Values.persistence.subPath.name }} + subPath: auth-conf + {{- else }} + name: "auth-conf" + {{- end }} + - mountPath: /opt/nifi/nifi-current/config-data + {{- if and .Values.persistence.enabled .Values.persistence.subPath.enabled }} + name: {{ .Values.persistence.subPath.name }} + subPath: config-data + {{- else }} + name: "config-data" + {{- end }} + - mountPath: /opt/nifi/flowfile_repository + {{- if and .Values.persistence.enabled .Values.persistence.subPath.enabled }} + name: {{ .Values.persistence.subPath.name }} + subPath: flowfile-repository + {{- else }} + name: "flowfile-repository" + {{- end }} + - mountPath: /opt/nifi/content_repository + {{- if and .Values.persistence.enabled .Values.persistence.subPath.enabled }} + name: {{ .Values.persistence.subPath.name }} + subPath: content-repository + {{- else }} + name: "content-repository" + {{- end }} + - mountPath: /opt/nifi/provenance_repository + {{- if and .Values.persistence.enabled .Values.persistence.subPath.enabled }} + name: {{ .Values.persistence.subPath.name }} + subPath: provenance-repository + {{- else }} + name: "provenance-repository" + {{- end }} + - name: "bootstrap-conf" + mountPath: /opt/nifi/nifi-current/conf/bootstrap.conf + subPath: "bootstrap.conf" + - name: "nifi-properties" + mountPath: /opt/nifi/nifi-current/conf/nifi.temp + subPath: "nifi.temp" + - name: "authorizers-temp" + mountPath: /opt/nifi/nifi-current/conf/authorizers.temp + subPath: "authorizers.temp" + - name: "bootstrap-notification-services-xml" + mountPath: /opt/nifi/nifi-current/conf/bootstrap-notification-services.xml + subPath: "bootstrap-notification-services.xml" + - name: "login-identity-providers-ldap-xml" + mountPath: /opt/nifi/nifi-current/conf/login-identity-providers-ldap.xml + subPath: "login-identity-providers-ldap.xml" + - name: "state-management-xml" + mountPath: /opt/nifi/nifi-current/conf/state-management.xml + subPath: "state-management.xml" + - name: "zookeeper-properties" + mountPath: /opt/nifi/nifi-current/conf/zookeeper.properties + subPath: "zookeeper.properties" + - name: "flow-content" + mountPath: /opt/nifi/data/flow.xml + subPath: "flow.xml" + {{- range $secret := .Values.secrets }} + {{- if $secret.mountPath }} + {{- if $secret.keys }} + {{- range $key := $secret.keys }} + - name: {{ include "apache-nifi.fullname" $ }}-{{ $secret.name }} + mountPath: {{ $secret.mountPath }}/{{ $key }} + subPath: {{ $key }} + readOnly: true + {{- end }} + {{- else }} + - name: {{ include "apache-nifi.fullname" $ }}-{{ $secret.name }} + mountPath: {{ $secret.mountPath }} + readOnly: true + {{- end }} + {{- end }} + {{- end }} + {{- range $configmap := .Values.configmaps }} + {{- if $configmap.mountPath }} + {{- if $configmap.keys }} + {{- range $key := $configmap.keys }} + - name: {{ include "apache-nifi.fullname" $ }}-{{ $configmap.name }} + mountPath: {{ $configmap.mountPath }}/{{ $key }} + subPath: {{ $key }} + readOnly: true + {{- end }} + {{- else }} + - name: {{ include "apache-nifi.fullname" $ }}-{{ $configmap.name }} + mountPath: {{ $configmap.mountPath }} + readOnly: true + {{- end }} + {{- end }} + {{- end }} +{{- if .Values.certManager.enabled }} + - name: "tls" + mountPath: /opt/nifi/nifi-current/tls + readOnly: true +{{- /* if .Values.certManager.enabled */}}{{ end }} + {{- if .Values.extraVolumeMounts }} +{{ toYaml .Values.extraVolumeMounts | indent 10 }} + {{- end }} + - name: app-log + imagePullPolicy: {{ .Values.sidecar.imagePullPolicy | default "Always" | quote }} + image: "{{ .Values.sidecar.image }}:{{ .Values.sidecar.tag }}" + args: + - /bin/sh + - -c + - trap "exit 0" TERM; tail -n+1 -F /var/log/nifi-app.log & wait $! + resources: +{{ toYaml .Values.logresources | indent 10 }} + volumeMounts: + - mountPath: /var/log + {{- if and .Values.persistence.enabled .Values.persistence.subPath.enabled }} + name: {{ .Values.persistence.subPath.name }} + subPath: logs + {{- else }} + name: "logs" + {{- end }} + - name: bootstrap-log + imagePullPolicy: {{ .Values.sidecar.imagePullPolicy | default "Always" | quote }} + image: "{{ .Values.sidecar.image }}:{{ .Values.sidecar.tag }}" + args: + - /bin/sh + - -c + - trap "exit 0" TERM; tail -n+1 -F /var/log/nifi-bootstrap.log & wait $! + resources: +{{ toYaml .Values.logresources | indent 10 }} + volumeMounts: + - mountPath: /var/log + {{- if and .Values.persistence.enabled .Values.persistence.subPath.enabled }} + name: {{ .Values.persistence.subPath.name }} + subPath: logs + {{- else }} + name: "logs" + {{- end }} + - name: user-log + imagePullPolicy: {{ .Values.sidecar.imagePullPolicy | default "Always" | quote }} + image: "{{ .Values.sidecar.image }}:{{ .Values.sidecar.tag }}" + args: + - /bin/sh + - -c + - trap "exit 0" TERM; tail -n+1 -F /var/log/nifi-user.log & wait $! + resources: +{{ toYaml .Values.logresources | indent 10 }} + volumeMounts: + - mountPath: /var/log + {{- if and .Values.persistence.enabled .Values.persistence.subPath.enabled }} + name: {{ .Values.persistence.subPath.name }} + subPath: logs + {{- else }} + name: "logs" + {{- end }} +{{- if .Values.certManager.enabled }} + - name: cert-manager + imagePullPolicy: {{ .Values.image.pullPolicy | quote }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + command: + - bash + - -ce + - | + trap "exit 0" TERM + + NODENAME=$(hostname) + mkdir -p "${NIFI_HOME}"/tls/cert-manager + + # Note opportunity here to inject additional trusted certs named ca.crt in other + # subdirectories of /opt/nifi/nifi-current/tls/, using custom secrets and/or configmaps. + # If any of those trusted certs expire then you will need to restart the pod to pick + # them up, as the truststore is only created at pod startup and Kubernetes won't + # update secrets mounted as subPaths anyway. + # c.f. https://kubernetes.io/docs/concepts/storage/volumes/#secret + + function pullNodeSecretData() { + rm -f /tmp/secret.json /tmp/secret-data.json + curl --cacert /run/secrets/kubernetes.io/secret-reader-token/ca.crt \ + -H "Authorization: Bearer $(cat /run/secrets/kubernetes.io/secret-reader-token/token)" \ + https://kubernetes.default.svc/api/v1/namespaces/$(cat /run/secrets/kubernetes.io/secret-reader-token/namespace)/secrets/$(hostname) \ + --output /tmp/secret.json + jq .data < /tmp/secret.json > /tmp/secret-data.json + rm -f /tmp/secret.json + } + + pullNodeSecretData + jq -r '."ca.crt"' < /tmp/secret-data.json | base64 -d > "${NIFI_HOME}"/tls/cert-manager/ca.crt + touch /tmp/tls.crt.old + + rm -f "${NIFI_HOME}/tls/truststore-new.jks" + for ca in "${NIFI_HOME}"/tls/*/ca.crt + do + ALIAS=$(echo $ca | awk -F "/" '{print $(NF-1) }' -) + keytool -import \ + -noprompt \ + -trustcacerts \ + -alias $ALIAS \ + -storetype JKS \ + -file $ca \ + -keystore "${NIFI_HOME}/tls/truststore-new.jks" \ + -storepass "{{ .Values.certManager.truststorePasswd }}" + done + keytool -list -keystore "${NIFI_HOME}/tls/truststore-new.jks" \ + -storepass "{{ .Values.certManager.truststorePasswd }}" + mv "${NIFI_HOME}/tls/truststore-new.jks" "${NIFI_HOME}/tls/truststore.jks" + + while : + do + pullNodeSecretData + jq -r '."tls.crt"' < /tmp/secret-data.json | base64 -d > /tmp/tls.crt + jq -r '."tls.key"' < /tmp/secret-data.json | base64 -d > /tmp/tls.key + if ! diff /tmp/tls.crt.old /tmp/tls.crt > /dev/null + then + openssl pkcs12 -export -in "/tmp/tls.crt" \ + -inkey "/tmp/tls.key" \ + -name "${NODENAME}" \ + -out "/tmp/tls.p12" \ + -passout "pass:{{ .Values.certManager.keystorePasswd }}" + rm -f "${NIFI_HOME}/tls/keystore-new.jks" + keytool -importkeystore \ + -noprompt \ + -destkeystore "${NIFI_HOME}/tls/keystore-new.jks" \ + -srckeystore "/tmp/tls.p12" \ + -srcstoretype PKCS12 \ + -deststoretype JKS \ + -srcstorepass "{{ .Values.certManager.keystorePasswd }}" \ + -deststorepass "{{ .Values.certManager.keystorePasswd }}" + mv "${NIFI_HOME}/tls/keystore-new.jks" "${NIFI_HOME}/tls/keystore.jks" + mv /tmp/tls.crt /tmp/tls.crt.old + fi + + ls -l "${NIFI_HOME}/tls/keystore.jks" + keytool -list -keystore "${NIFI_HOME}/tls/keystore.jks" \ + -storepass "{{ .Values.certManager.keystorePasswd }}" + + echo Starting to sleep for {{ .Values.certManager.refreshSeconds }} seconds at $(date) + sleep {{ .Values.certManager.refreshSeconds }} + done + volumeMounts: + - name: "tls" + mountPath: /opt/nifi/nifi-current/tls + - name: secret-reader-token + mountPath: /run/secrets/kubernetes.io/secret-reader-token + {{- range $secret := .Values.secrets }} + {{- if $secret.mountPath }} + {{- if $secret.keys }} + {{- range $key := $secret.keys }} + - name: {{ include "apache-nifi.fullname" $ }}-{{ $secret.name }} + mountPath: {{ $secret.mountPath }}/{{ $key }} + subPath: {{ $key }} + readOnly: true + {{- end }} + {{- else }} + - name: {{ include "apache-nifi.fullname" $ }}-{{ $secret.name }} + mountPath: {{ $secret.mountPath }} + readOnly: true + {{- end }} + {{- end }} + {{- end }} + {{ range $secret := .Values.certManager.caSecrets }} + - name: {{ include "apache-nifi.fullname" $ }}-{{ $secret }} + mountPath: /opt/nifi/nifi-current/tls/{{ $secret }} + readOnly: true + {{ end }} + {{- range $configmap := .Values.configmaps }} + {{- if $configmap.mountPath }} + {{- if $configmap.keys }} + {{- range $key := $configmap.keys }} + - name: {{ include "apache-nifi.fullname" $ }}-{{ $configmap.name }} + mountPath: {{ $configmap.mountPath }}/{{ $key }} + subPath: {{ $key }} + readOnly: true + {{- end }} + {{- else }} + - name: {{ include "apache-nifi.fullname" $ }}-{{ $configmap.name }} + mountPath: {{ $configmap.mountPath }} + readOnly: true + {{- end }} + {{- end }} + {{- end }} + {{- if .Values.extraVolumeMounts }} +{{ toYaml .Values.extraVolumeMounts | indent 10 }} + {{/* if .Values.extraVolumeMounts */}}{{ end }} + resources: +{{ toYaml .Values.certManager.resources | indent 10 }} +{{- /* if .Values.certManager.enabled */}}{{ end }} + volumes: + - name: "bootstrap-conf" + configMap: + name: {{ template "apache-nifi.fullname" . }}-config + items: + - key: "bootstrap.conf" + path: "bootstrap.conf" + - name: "nifi-properties" + configMap: + name: {{ template "apache-nifi.fullname" . }}-config + items: + - key: "nifi.properties" + path: "nifi.temp" + - name: "authorizers-temp" + configMap: + name: {{ template "apache-nifi.fullname" . }}-config + items: + - key: "authorizers.xml" + path: "authorizers.temp" + - name: "bootstrap-notification-services-xml" + configMap: + name: {{ template "apache-nifi.fullname" . }}-config + items: + - key: "bootstrap-notification-services.xml" + path: "bootstrap-notification-services.xml" + - name: "login-identity-providers-ldap-xml" + configMap: + name: {{ template "apache-nifi.fullname" . }}-config + items: + - key: "login-identity-providers-ldap.xml" + path: "login-identity-providers-ldap.xml" + - name: "state-management-xml" + configMap: + name: {{ template "apache-nifi.fullname" . }}-config + items: + - key: "state-management.xml" + path: "state-management.xml" + - name: "zookeeper-properties" + configMap: + name: {{ template "apache-nifi.fullname" . }}-config + items: + - key: "zookeeper.properties" + path: "zookeeper.properties" + - name: "flow-content" + configMap: + name: {{ template "apache-nifi.fullname" . }}-config + items: + - key: "flow.xml" + path: "flow.xml" +{{- if .Values.certManager.enabled }} + - name: secret-reader-token + secret: + secretName: {{ template "apache-nifi.fullname" $ }}-secret-reader-token + - name: tls + emptyDir: {} + {{- range .Values.certManager.caSecrets }} + - name: {{ include "apache-nifi.fullname" $ }}-{{ . }} + secret: + secretName: {{ . }} + {{- end }} +{{- /* if .Values.certManager.enabled */}}{{ end }} + {{- range .Values.secrets }} + - name: {{ include "apache-nifi.fullname" $ }}-{{ .name }} + secret: + secretName: {{ .name }} + {{- end }} + {{- range .Values.configmaps }} + - name: {{ include "apache-nifi.fullname" $ }}-{{ .name }} + configMap: + name: {{ .name }} + {{- end }} +{{- if not .Values.persistence.enabled }} + - name: config-data + emptyDir: {} + - name: auth-conf + emptyDir: {} + - name: data + emptyDir: {} + - name: flowfile-repository + emptyDir: {} + - name: content-repository + emptyDir: {} + - name: provenance-repository + emptyDir: {} + - name: logs + emptyDir: {} +{{- end }} +{{- if .Values.extraVolumes }} +{{ toYaml .Values.extraVolumes | indent 6 }} +{{- end }} +{{- if and .Values.persistence.enabled .Values.persistence.subPath.enabled }} + volumeClaimTemplates: + - metadata: + name: {{ .Values.persistence.subPath.name }} + spec: + accessModes: + {{- range .Values.persistence.accessModes }} + - {{ . | quote }} + {{- end }} + storageClassName: {{ .Values.persistence.storageClass | quote }} + resources: + requests: + storage: {{ .Values.persistence.subPath.size }} +{{- end }} +{{- if and .Values.persistence.enabled (not .Values.persistence.subPath.enabled) }} + volumeClaimTemplates: + - metadata: + name: logs + spec: + accessModes: + {{- range .Values.persistence.accessModes }} + - {{ . | quote }} + {{- end }} + storageClassName: {{ .Values.persistence.storageClass | quote }} + resources: + requests: + storage: {{ .Values.persistence.logStorage.size }} + - metadata: + name: "config-data" + spec: + accessModes: ["ReadWriteOnce"] + storageClassName: {{ .Values.persistence.storageClass | quote }} + resources: + requests: + storage: {{ .Values.persistence.configStorage.size }} + - metadata: + name: data + spec: + accessModes: + {{- range .Values.persistence.accessModes }} + - {{ . | quote }} + {{- end }} + storageClassName: {{ .Values.persistence.storageClass | quote }} + resources: + requests: + storage: {{ .Values.persistence.dataStorage.size }} + - metadata: + name: flowfile-repository + spec: + accessModes: + {{- range .Values.persistence.accessModes }} + - {{ . | quote }} + {{- end }} + storageClassName: {{ .Values.persistence.storageClass | quote }} + resources: + requests: + storage: {{ .Values.persistence.flowfileRepoStorage.size }} + - metadata: + name: content-repository + spec: + accessModes: + {{- range .Values.persistence.accessModes }} + - {{ . | quote }} + {{- end }} + storageClassName: {{ .Values.persistence.storageClass | quote }} + resources: + requests: + storage: {{ .Values.persistence.contentRepoStorage.size }} + - metadata: + name: provenance-repository + spec: + accessModes: + {{- range .Values.persistence.accessModes }} + - {{ . | quote }} + {{- end }} + storageClassName: {{ .Values.persistence.storageClass | quote }} + resources: + requests: + storage: {{ .Values.persistence.provenanceRepoStorage.size }} + - metadata: + name: auth-conf + spec: + accessModes: + {{- range .Values.persistence.accessModes }} + - {{ . | quote }} + {{- end }} + storageClassName: {{ .Values.persistence.storageClass | quote }} + resources: + requests: + storage: {{ .Values.persistence.authconfStorage.size }} +{{- end }} diff --git a/external/nifi/templates/storageclass.yaml b/external/nifi/templates/storageclass.yaml new file mode 100644 index 0000000..5a48269 --- /dev/null +++ b/external/nifi/templates/storageclass.yaml @@ -0,0 +1,15 @@ +--- +{{- if .Values.customStorageClass -}} +kind: StorageClass +apiVersion: storage.k8s.io/v1 +metadata: + name: {{ .Values.storageClass }} + labels: + app: {{ include "apache-nifi.name" . | quote }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" + release: {{ .Release.Name | quote }} + heritage: {{ .Release.Service | quote }} +provisioner: {{ .Values.storageProvisioner | quote }} +parameters: + type: {{ .Values.storageType | quote }} +{{- end -}} \ No newline at end of file diff --git a/external/nifi/tests/01-safetyValve-values.yaml b/external/nifi/tests/01-safetyValve-values.yaml new file mode 100644 index 0000000..ffca4bc --- /dev/null +++ b/external/nifi/tests/01-safetyValve-values.yaml @@ -0,0 +1,3 @@ +properties: + safetyValve: + nifi.content.claim.max.appendable.size: "1 B" diff --git a/external/nifi/tests/02-persistence-disabled-values.yaml b/external/nifi/tests/02-persistence-disabled-values.yaml new file mode 100644 index 0000000..6b53a01 --- /dev/null +++ b/external/nifi/tests/02-persistence-disabled-values.yaml @@ -0,0 +1,8 @@ +persistence: + enabled: false + +zookeeper: + enabled: false + +registry: + enabled: false diff --git a/external/nifi/tests/02-persistence-enabled-values.yaml b/external/nifi/tests/02-persistence-enabled-values.yaml new file mode 100644 index 0000000..a434bf0 --- /dev/null +++ b/external/nifi/tests/02-persistence-enabled-values.yaml @@ -0,0 +1,2 @@ +persistence: + enabled: true diff --git a/external/nifi/tests/03-ldap-values.yaml b/external/nifi/tests/03-ldap-values.yaml new file mode 100644 index 0000000..ecf5b33 --- /dev/null +++ b/external/nifi/tests/03-ldap-values.yaml @@ -0,0 +1,14 @@ +zookeeper: + enabled: false + +registry: + enabled: false + +auth: + ldap: + enabled: true + host: ldap://openldap:389 + searchBase: ou=users,dc=example,dc=org + admin: cn=user1,ou=users,dc=example,dc=org + pass: password1 + authStrategy: SIMPLE diff --git a/external/nifi/tests/03-ldap/deployment.yaml b/external/nifi/tests/03-ldap/deployment.yaml new file mode 100644 index 0000000..462a697 --- /dev/null +++ b/external/nifi/tests/03-ldap/deployment.yaml @@ -0,0 +1,45 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: openldap + labels: + app.kubernetes.io/name: openldap +spec: + selector: + matchLabels: + app.kubernetes.io/name: openldap + replicas: 1 + template: + metadata: + labels: + app.kubernetes.io/name: openldap + spec: + containers: +# - name: netshoot +# image: docker.io/nicolaka/netshoot:latest +# imagePullPolicy: "Always" +# command: [ 'tail', '-f', '/dev/null' ] + - name: openldap + image: docker.io/bitnami/openldap:latest + imagePullPolicy: "Always" + env: + - name: LDAP_ADMIN_USERNAME + value: "admin" + - name: LDAP_ADMIN_PASSWORD + valueFrom: + secretKeyRef: + key: adminpassword + name: openldap + - name: LDAP_USERS + valueFrom: + secretKeyRef: + key: users + name: openldap + - name: LDAP_PASSWORDS + valueFrom: + secretKeyRef: + key: passwords + name: openldap + ports: + - name: tcp-ldap + containerPort: 1389 diff --git a/external/nifi/tests/03-ldap/secret.yaml b/external/nifi/tests/03-ldap/secret.yaml new file mode 100644 index 0000000..992ef5a --- /dev/null +++ b/external/nifi/tests/03-ldap/secret.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: Secret +metadata: + name: openldap +stringData: + adminpassword: admin + passwords: password1,password2 + users: user1,user2 diff --git a/external/nifi/tests/03-ldap/service.yaml b/external/nifi/tests/03-ldap/service.yaml new file mode 100644 index 0000000..bf6b56e --- /dev/null +++ b/external/nifi/tests/03-ldap/service.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: Service +metadata: + name: openldap + labels: + app.kubernetes.io/name: openldap +spec: + type: ClusterIP + ports: + - name: tcp-ldap + port: 389 + targetPort: tcp-ldap + selector: + app.kubernetes.io/name: openldap diff --git a/external/nifi/tests/04-oidc-keycloak-setup.bash b/external/nifi/tests/04-oidc-keycloak-setup.bash new file mode 100755 index 0000000..b57f205 --- /dev/null +++ b/external/nifi/tests/04-oidc-keycloak-setup.bash @@ -0,0 +1,87 @@ +#!/bin/bash -x +# +# Create a realm and user to verify NiFi OIDC support works correctly + +# Find the SOCKS5 Proxy into the cluster + +S5IP=$(kubectl get node -o json | jq -r '.items[0].status.addresses[] | select(.type=="InternalIP") | .address') +S5PORT=$(kubectl get service socks5 -o json | jq -r '.spec.ports[0].nodePort') + +CURL="curl -s --socks5-hostname $S5IP:$S5PORT" + +# NOTE: Strictly speaking the SOCKS5 cluster could be removed from the test harness +# by running curl through a kubectl exec in (say) the NiFi server container or +# directly through a kubectl run using the hub.docker.io/curlimages/curl image. +# Either way curl would be able to use the fully qualified domain name (FQDN). +# +# But doing it this way also allows someone debugging OIDC to set their +# workstation browser to use the SOCKS5 proxy (using the IP address and port +# discovered through the above kubectl commands, and with remote DNS resolution) +# to access the NiFi UI as https://nifi.default.svc.cluster.local:8843/nifi/ and +# confirm it all works--including the FQDN-based redirects from NiFi to Keycloak and +# back again. And it's very useful if (when, really) it doesn't work to have the full +# desktop browser debugging and tracing capabilities available. +# +# Also, while writing the tests it was sure nice having the full browser available +# to spelunk through the DOM of the Keycloak and NiFi pages to zero in on what +# to have puppeteer interact with, both in terms of sending text/clicks and +# scraping results. + +KCURL=http://keycloak.default.svc.cluster.local:8080/auth + +# Get a KeyCloak admin token + +KCAT=$($CURL \ + -d username=admin \ + -d password=admin \ + -d client_id=admin-cli \ + -d grant_type=password \ + $KCURL/realms/master/protocol/openid-connect/token | \ + jq --raw-output .access_token ) + +# Create the NiFi Realm + +$CURL \ + --request POST $KCURL/admin/realms/ \ + --header "Authorization: Bearer $KCAT" \ + --header "Content-Type: application/json" \ + --data-raw '{ + "realm":"nifi", + "displayName":"NiFi", + "enabled":"true", + "ssoSessionIdleTimeout":7200, + "accessTokenLifespan":3600 + }' + +# Create the NiFi User + +$CURL \ + --request POST $KCURL/admin/realms/nifi/users \ + --header "Authorization: Bearer $KCAT" \ + --header "Content-Type: application/json" \ + --data-raw '{ + "firstName":"NiFi", + "lastName":"User", + "username":"nifi", + "enabled":"true", + "email":"nifi@example.com", + "credentials":[ + { + "type":"password", + "value":"reallychangeme", + "temporary":"false" + } + ] + }' + +$CURL \ + --request POST $KCURL/admin/realms/nifi/clients \ + --header "Authorization: Bearer $KCAT" \ + --header "Content-Type: application/json" \ + --data-raw '{ + "clientId":"nifi", + "enabled":"true", + "redirectUris": [ "https://nifi.default.svc.cluster.local:8443/*", "https://ingress-nginx-controller.ingress-nginx.svc.cluster.local:443/*" ], + "publicClient": "false", + "secret":"CZhA1IOePlXHz3PWqVwYoVAcYIUHTcDK" + }' \ No newline at end of file diff --git a/external/nifi/tests/04-oidc-login-test.js b/external/nifi/tests/04-oidc-login-test.js new file mode 100644 index 0000000..579d244 --- /dev/null +++ b/external/nifi/tests/04-oidc-login-test.js @@ -0,0 +1,84 @@ +const puppeteer = require ('puppeteer-core') +const expect = require('chai').expect + +describe('NiFi Login via OIDC', () => { + let browser + let page + + before(async () => { + browser = await puppeteer.connect({ + browserWSEndpoint: 'ws://'+process.env.K8SNODEIP+':'+process.env.K8SPORT, + ignoreHTTPSErrors: true + }) + page = await browser.newPage() + }) + + it('NiFi redirects to KeyCloak login page', async () => { + let gotoSucceeded = false + for ( let i = 0; ( i < 60 ) && ( ! gotoSucceeded ); i++ ) { + try { + await Promise.all([ + page.goto(process.env.NIFIURL), + page.waitForNavigation(), + page.waitForNetworkIdle() + ]) + gotoSucceeded = true + } + catch(err) { + console.log(" Connection to "+process.env.NIFIURL+"failed: "+err.message+" ( try "+i.toString()+")") + await page.waitForTimeout(5000) + } + } + const pageTitle = await page.waitForSelector('h1[id="kc-page-title"]') + const titleContent = await pageTitle.evaluate(el => el.textContent) + expect(titleContent).to.include('Sign in to your account') + }).timeout(30000) + + it('Get screenshot of Keycloak login page', async () => { + await page.screenshot({ + path: process.env.HOME+"/screenshots/01-keycloak-redirect.png", + fullPage: true + }) + }) + + it('nifi@example.com shown as logged in user', async () => { + await page.type('input[id="username"]','nifi') + await page.type('input[id="password"]','reallychangeme') + await Promise.all([ + page.click('input[id="kc-login"]'), + page.waitForNavigation(), + page.waitForNetworkIdle() + ]) + let messageContentSelector = await page.waitForSelector('div[id="message-content"]') + let messageContent = await messageContentSelector.evaluate(el => el.textContent) + // loop 30 times unless messageContent is empty + for (let i = 0; ( i < 30 ) && ( messageContent != "" ); i++ ) { + await Promise.all([ + page.reload(), + page.waitForNavigation(), + page.waitForNetworkIdle(), + page.waitForTimeout(5000) + ]) + console.log(" Message Content: "+messageContent+" ( try "+i.toString()+")") + messageContentSelector = await page.waitForSelector('div[id="message-content"]') + messageContent = await messageContentSelector.evaluate(el => el.textContent) + if ( messageContent != "") { + console.log(" Message Content: "+messageContent) + } + } + const currentUserElement = await page.waitForSelector('div[id="current-user"]') + const userName = await currentUserElement.evaluate(el => el.textContent) + expect(userName).to.equal('nifi@example.com') + }).timeout(300000) + + it('Get screenshot of logged in user', async () => { + await page.screenshot({ + path: process.env.HOME+"/screenshots/02-logged-in-user.png", + fullPage: true + }) + }) + + after(async () => { + await browser.close() + }) +}) diff --git a/external/nifi/tests/04-oidc-test-framework/browserless-service.yaml b/external/nifi/tests/04-oidc-test-framework/browserless-service.yaml new file mode 100644 index 0000000..7ee96fe --- /dev/null +++ b/external/nifi/tests/04-oidc-test-framework/browserless-service.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: Service +metadata: + name: browserless + labels: + app.kubernetes.io/name: browserless +spec: + type: NodePort + ports: + - name: tcp-browserless + port: 3000 + targetPort: tcp-browserless + selector: + app.kubernetes.io/name: browserless diff --git a/external/nifi/tests/04-oidc-test-framework/browserless-statefulset.yaml b/external/nifi/tests/04-oidc-test-framework/browserless-statefulset.yaml new file mode 100644 index 0000000..e1796fe --- /dev/null +++ b/external/nifi/tests/04-oidc-test-framework/browserless-statefulset.yaml @@ -0,0 +1,31 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: browserless + labels: + app.kubernetes.io/name: browserless +spec: + selector: + matchLabels: + app.kubernetes.io/name: browserless + serviceName: browserless + replicas: 1 + template: + metadata: + labels: + app.kubernetes.io/name: browserless + spec: + containers: + - name: browserless + image: browserless/chrome + imagePullPolicy: "Always" + ports: + - name: tcp-browserless + containerPort: 3000 + env: + - name: CONNECTION_TIMEOUT + value: "-1" + # - name: netshoot + # command: [ 'tail', '-f', '/dev/null' ] + # image: docker.io/nicolaka/netshoot:latest + # imagePullPolicy: "Always" diff --git a/external/nifi/tests/04-oidc-test-framework/keycloak-secret.yaml b/external/nifi/tests/04-oidc-test-framework/keycloak-secret.yaml new file mode 100644 index 0000000..b2e56e3 --- /dev/null +++ b/external/nifi/tests/04-oidc-test-framework/keycloak-secret.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +kind: Secret +metadata: + name: keycloak +stringData: + adminpassword: admin diff --git a/external/nifi/tests/04-oidc-test-framework/keycloak-service.yaml b/external/nifi/tests/04-oidc-test-framework/keycloak-service.yaml new file mode 100644 index 0000000..03aa64f --- /dev/null +++ b/external/nifi/tests/04-oidc-test-framework/keycloak-service.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: Service +metadata: + name: keycloak + labels: + app.kubernetes.io/name: keycloak +spec: + type: ClusterIP + ports: + - name: tcp-keycloak + port: 8080 + targetPort: tcp-keycloak + selector: + app.kubernetes.io/name: keycloak diff --git a/external/nifi/tests/04-oidc-test-framework/keycloak-statefulset.yaml b/external/nifi/tests/04-oidc-test-framework/keycloak-statefulset.yaml new file mode 100644 index 0000000..f40c8b0 --- /dev/null +++ b/external/nifi/tests/04-oidc-test-framework/keycloak-statefulset.yaml @@ -0,0 +1,45 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: keycloak + labels: + app.kubernetes.io/name: keycloak +spec: + selector: + matchLabels: + app.kubernetes.io/name: keycloak + serviceName: keycloak + replicas: 1 + template: + metadata: + labels: + app.kubernetes.io/name: keycloak + spec: + containers: + - name: keycloak + image: quay.io/keycloak/keycloak:16.1.0 + imagePullPolicy: "Always" + env: + - name: KEYCLOAK_USER + value: "admin" + - name: KEYCLOAK_PASSWORD + valueFrom: + secretKeyRef: + key: adminpassword + name: keycloak + ports: + - name: tcp-keycloak + containerPort: 8080 + startupProbe: + exec: + command: + - /bin/sh + - -x + - -c + - curl -s http://localhost:8080 | grep 'If you are not redirected automatically, follow this' + failureThreshold: 30 + periodSeconds: 10 +# - name: netshoot +# command: [ 'tail', '-f', '/dev/null' ] +# image: docker.io/nicolaka/netshoot:latest +# imagePullPolicy: "Always" diff --git a/external/nifi/tests/04-oidc-test-framework/socks5.yaml b/external/nifi/tests/04-oidc-test-framework/socks5.yaml new file mode 100644 index 0000000..213b8bf --- /dev/null +++ b/external/nifi/tests/04-oidc-test-framework/socks5.yaml @@ -0,0 +1,43 @@ +apiVersion: v1 +kind: Service +metadata: + name: socks5 + labels: + app.kubernetes.io/name: socks5 +spec: + type: NodePort + ports: + - name: tcp-socks5 + port: 1080 + targetPort: tcp-socks5 + selector: + app.kubernetes.io/name: socks5 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: socks5 + labels: + app.kubernetes.io/name: socks5 +spec: + selector: + matchLabels: + app.kubernetes.io/name: socks5 + serviceName: socks5 + replicas: 1 + template: + metadata: + labels: + app.kubernetes.io/name: socks5 + spec: + containers: + - name: socks5 + image: serjs/go-socks5-proxy:latest + imagePullPolicy: "Always" + ports: + - name: tcp-socks5 + containerPort: 1080 + # - name: netshoot + # command: [ 'tail', '-f', '/dev/null' ] + # image: docker.io/nicolaka/netshoot:latest + # imagePullPolicy: "Always" diff --git a/external/nifi/tests/04-oidc-values.yaml b/external/nifi/tests/04-oidc-values.yaml new file mode 100644 index 0000000..0f0bccc --- /dev/null +++ b/external/nifi/tests/04-oidc-values.yaml @@ -0,0 +1,21 @@ +zookeeper: + enabled: false + +registry: + enabled: false + +auth: + oidc: + enabled: true + discoveryUrl: http://keycloak.default.svc.cluster.local:8080/auth/realms/nifi/.well-known/openid-configuration + clientId: nifi + clientSecret: CZhA1IOePlXHz3PWqVwYoVAcYIUHTcDK + admin: nifi@example.com + claimIdentifyingUser: email + +properties: + webProxyHost: nifi.default.svc.cluster.local:8443 + +sts: + startupProbe: + enabled: true diff --git a/external/nifi/tests/05-install-cert-manager.bash b/external/nifi/tests/05-install-cert-manager.bash new file mode 100755 index 0000000..d30add6 --- /dev/null +++ b/external/nifi/tests/05-install-cert-manager.bash @@ -0,0 +1,17 @@ +#!/bin/bash -x + +# Install cmctl per https://cert-manager.io/docs/usage/cmctl/#installation + +sudo apt-get update +sudo apt-get install -y golang-go + +OS=$(go env GOOS) +ARCH=$(go env GOARCH) + +/bin/rm -rf /tmp/cmctl-install +mkdir -p /tmp/cmctl-install + +curl -L -o /tmp/cmctl-install/cmctl.tar.gz https://github.com/jetstack/cert-manager/releases/latest/download/cmctl-$OS-$ARCH.tar.gz +(cd /tmp/cmctl-install ; tar xvzf cmctl.tar.gz ; sudo mv cmctl /usr/local/bin) + +cmctl experimental install diff --git a/external/nifi/tests/05-secure-cluster-values.yaml b/external/nifi/tests/05-secure-cluster-values.yaml new file mode 100644 index 0000000..899ca3f --- /dev/null +++ b/external/nifi/tests/05-secure-cluster-values.yaml @@ -0,0 +1,14 @@ +zookeeper: + enabled: true + +properties: + isNode: true + webProxyHost: nifi.default.svc.cluster.local:8443 + +replicaCount: 2 + +registry: + enabled: false + +certManager: + enabled: true diff --git a/external/nifi/tests/06-alpha.flow.xml b/external/nifi/tests/06-alpha.flow.xml new file mode 100644 index 0000000..3119c1b --- /dev/null +++ b/external/nifi/tests/06-alpha.flow.xml @@ -0,0 +1,255 @@ + + + 10 + 5 + + + + 85928fc5-336a-4a64-b887-6869cd41eda9 + ALPHA + + + UNBOUNDED + STREAM_WHEN_AVAILABLE + 0 sec + 10000 + 1 GB + + c1202f8e-017e-1000-a733-50c5cb8784b1 + GenerateFlowFile + + + + org.apache.nifi.processors.standard.GenerateFlowFile + + org.apache.nifi + nifi-standard-nar + 1.23.2 + + 1 + 1 sec + 30 sec + 1 sec + WARN + false + RUNNING + TIMER_DRIVEN + ALL + 0 + + File Size + 0B + + + Batch Size + 1 + + + Data Format + Text + + + Unique FlowFiles + false + + + generate-ff-custom-text + + + character-set + UTF-8 + + + mime-type + + + + c11f5fe0-017e-1000-2f63-a9a78031e0c0 + BRAVO + + + https://nifi.bravo.svc.cluster.local:8443/nifi + https://nifi.bravo.svc.cluster.local:8443/nifi + 30 sec + 10 sec + true + RAW + + + + eccfb75d-417c-38d5-be12-cdffc1189223 + BravoInput + + + RUNNING + c11a3c30-017e-1000-f2f0-17232c042b30 + 1 + false + + + + c120776b-017e-1000-6cdc-3639cebe805e + + + 1 + 0 + c1202f8e-017e-1000-a733-50c5cb8784b1 + 85928fc5-336a-4a64-b887-6869cd41eda9 + PROCESSOR + eccfb75d-417c-38d5-be12-cdffc1189223 + c11f5fe0-017e-1000-2f63-a9a78031e0c0 + REMOTE_INPUT_PORT + success + 10000 + 1 GB + 0 sec + DO_NOT_LOAD_BALANCE + + DO_NOT_COMPRESS + + + + + c135c2ce-017e-1000-d88a-347b945f505f + StandardRestrictedSSLContextService + + org.apache.nifi.ssl.StandardRestrictedSSLContextService + + org.apache.nifi + nifi-ssl-context-service-nar + 1.23.2 + + true + + Keystore Filename + /opt/nifi/nifi-current/tls/keystore.jks + + + Keystore Password + enc{10df1c866feaaa5fc2ed0918dde6710eb521e863e6e3f0ed75717c10dfaa33375f74daef53ae4236} + + + key-password + + + Keystore Type + JKS + + + Truststore Filename + /opt/nifi/nifi-current/tls/truststore.jks + + + Truststore Password + enc{00b997b8ad2a4d299d351dde723d6cae5274414096d3db4ed225a45ed216090a3dfb9de123741a72} + + + Truststore Type + JKS + + + SSL Protocol + TLS + + + + + + c131bf78-017e-1000-dbc1-a9b6b34a798b + SiteToSiteProvenanceReportingTask + + org.apache.nifi.reporting.SiteToSiteProvenanceReportingTask + + org.apache.nifi + nifi-site-to-site-reporting-nar + 1.23.2 + + 5 sec + RUNNING + TIMER_DRIVEN + + Destination URL + https://nifi.bravo.svc.cluster.local:8443/nifi + + + Input Port Name + ProvInput + + + SSL Context Service + c135c2ce-017e-1000-d88a-347b945f505f + + + Instance URL + http://${hostname(true)}:8080/nifi + + + Compress Events + true + + + Communications Timeout + 30 secs + + + Batch Size + 1000 + + + s2s-transport-protocol + RAW + + + s2s-http-proxy-hostname + + + s2s-http-proxy-port + + + s2s-http-proxy-username + + + s2s-http-proxy-password + + + record-writer + + + include-null-values + false + + + Platform + nifi + + + s2s-prov-task-event-filter + + + s2s-prov-task-event-filter-exclude + + + s2s-prov-task-type-filter + + + s2s-prov-task-type-filter-exclude + + + s2s-prov-task-id-filter + + + s2s-prov-task-id-filter-exclude + + + s2s-prov-task-name-filter + + + s2s-prov-task-name-filter-exclude + + + start-position + beginning-of-stream + + + + diff --git a/external/nifi/tests/06-bravo.flow.xml b/external/nifi/tests/06-bravo.flow.xml new file mode 100644 index 0000000..5e5a3ec --- /dev/null +++ b/external/nifi/tests/06-bravo.flow.xml @@ -0,0 +1,192 @@ + + + 10 + 5 + + + + 356a3caa-e4c2-492d-9c1d-e040d8a3b5be + BRAVO + + + UNBOUNDED + STREAM_WHEN_AVAILABLE + 0 sec + 10000 + 1 GB + + c130053e-017e-1000-e581-48dcfc185038 + UpdateAttribute + + + + org.apache.nifi.processors.attributes.UpdateAttribute + + org.apache.nifi + nifi-update-attribute-nar + 1.15.2 + + 1 + 0 sec + 30 sec + 1 sec + WARN + false + RUNNING + TIMER_DRIVEN + ALL + 0 + + Delete Attributes Expression + + + Store State + Do not store state + + + Stateful Variables Initial Value + + + canonical-value-lookup-cache-size + 100 + + success + + + c11a6e7f-017e-1000-cdd8-2d3e424d7225 + UpdateAttribute + + + + org.apache.nifi.processors.attributes.UpdateAttribute + + org.apache.nifi + nifi-update-attribute-nar + 1.15.2 + + 1 + 0 sec + 30 sec + 1 sec + WARN + false + RUNNING + TIMER_DRIVEN + ALL + 0 + + Delete Attributes Expression + + + Store State + Do not store state + + + Stateful Variables Initial Value + + + canonical-value-lookup-cache-size + 100 + + success + + + c12f987d-017e-1000-de92-17bdff9f8b21 + ProvInput + + + RUNNING + 1 + true + + + c11a3c30-017e-1000-f2f0-17232c042b30 + BravoInput + + + RUNNING + 1 + true + + + c11aa559-017e-1000-c24b-fb6ccb999e60 + + + 1 + 0 + c11a3c30-017e-1000-f2f0-17232c042b30 + 356a3caa-e4c2-492d-9c1d-e040d8a3b5be + INPUT_PORT + c11a6e7f-017e-1000-cdd8-2d3e424d7225 + 356a3caa-e4c2-492d-9c1d-e040d8a3b5be + PROCESSOR + + 10000 + 1 GB + 0 sec + DO_NOT_LOAD_BALANCE + + DO_NOT_COMPRESS + + + c13023be-017e-1000-d1fd-b1eaa0ad742d + + + 1 + 0 + c12f987d-017e-1000-de92-17bdff9f8b21 + 356a3caa-e4c2-492d-9c1d-e040d8a3b5be + INPUT_PORT + c130053e-017e-1000-e581-48dcfc185038 + 356a3caa-e4c2-492d-9c1d-e040d8a3b5be + PROCESSOR + + 10000 + 1 GB + 0 sec + DO_NOT_LOAD_BALANCE + + DO_NOT_COMPRESS + + + + + + c2b0c85e-017e-1000-0f6d-53ed5df3eb43 + PrometheusReportingTask + + org.apache.nifi.reporting.prometheus.PrometheusReportingTask + + org.apache.nifi + nifi-prometheus-nar + 1.15.2 + + 5 sec + RUNNING + TIMER_DRIVEN + + prometheus-reporting-task-metrics-endpoint-port + 9092 + + + prometheus-reporting-task-instance-id + ${hostname(true)} + + + prometheus-reporting-task-metrics-strategy + All Components + + + prometheus-reporting-task-metrics-send-jvm + false + + + prometheus-reporting-task-ssl-context + + + prometheus-reporting-task-client-auth + No Authentication + + + + diff --git a/external/nifi/tests/06-site-to-site.bash b/external/nifi/tests/06-site-to-site.bash new file mode 100755 index 0000000..84d9522 --- /dev/null +++ b/external/nifi/tests/06-site-to-site.bash @@ -0,0 +1,59 @@ +#!/bin/bash -x + +# Create a pair of namespaces and secure NiFi instances + +for ns in alpha bravo; +do + sed -e 's/^ //' << ENDOFYAML | kubectl apply -f - + --- + apiVersion: v1 + kind: Namespace + metadata: + name: $ns + --- +ENDOFYAML + kubectl -n $ns create --dry-run=client configmap flow-xml --from-file=tests/06-$ns.flow.xml -o yaml | kubectl apply -f - +done + +# Install NiFi expecting secrets: + +helm -n alpha install nifi . \ + --set zookeeper.enabled=false \ + --set properties.isNode=false \ + --set properties.webProxyHost=nifi.alpha.svc.cluster.local:8443 \ + --set replicaCount=1 \ + --set registry.enabled=false \ + --set certManager.enabled=true \ + --set configmaps[0].name=flow-xml \ + --set configmaps[0].mountPath=/opt/nifi/flow-xml \ + --set customFlow=/opt/nifi/flow-xml/06-alpha.flow.xml \ + --set certManager.caDuration=1h \ + --set certManager.refreshSeconds=30 \ + --set 'certManager.caSecrets[0]=bravo-ca' + +helm -n bravo install nifi . \ + --set zookeeper.enabled=false \ + --set properties.isNode=false \ + --set properties.webProxyHost=nifi.bravo.svc.cluster.local:8443 \ + --set replicaCount=1 \ + --set registry.enabled=false \ + --set certManager.enabled=true \ + --set configmaps[0].name=flow-xml \ + --set configmaps[0].mountPath=/opt/nifi/flow-xml \ + --set customFlow=/opt/nifi/flow-xml/06-bravo.flow.xml \ + --set certManager.caDuration=1h \ + --set certManager.refreshSeconds=30 \ + --set 'certManager.caSecrets[0]=alpha-ca' + +# Copy certificate authorities from one namespace to the other + +kubectl -n alpha wait --for=condition=Ready=true certificate/nifi-ca --timeout=60s +kubectl -n alpha get secret nifi-ca -o json | \ + jq 'del(.metadata)|del(.data."tls.crt")|del(.data."tls.key") + { metadata: { name: "alpha-ca" } } + { type: "kubernetes.io/generic" }' | \ + kubectl -n bravo apply -f - + +kubectl -n bravo wait --for=condition=Ready=true certificate/nifi-ca --timeout=60s +kubectl -n bravo get secret nifi-ca -o json | \ + jq 'del(.metadata)|del(.data."tls.crt")|del(.data."tls.key") + { metadata: { name: "bravo-ca" } } + { type: "kubernetes.io/generic" }' | \ + kubectl -n alpha apply -f - + diff --git a/external/nifi/tests/07-increase-webhook-timeout.yaml b/external/nifi/tests/07-increase-webhook-timeout.yaml new file mode 100644 index 0000000..cd81051 --- /dev/null +++ b/external/nifi/tests/07-increase-webhook-timeout.yaml @@ -0,0 +1,17 @@ +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + name: ingress-nginx-admission + namespace: nginx-ingress +webhooks: +- name: validate.nginx.ingress.kubernetes.io + admissionReviewVersions: + - v1 + clientConfig: + service: + name: ingress-nginx-controller-admission + namespace: ingress-nginx + path: /networking/v1/ingresses + port: 443 + sideEffects: None + timeoutSeconds: 30 diff --git a/external/nifi/tests/07-oidc-cluster-login-test.js b/external/nifi/tests/07-oidc-cluster-login-test.js new file mode 100644 index 0000000..4fc9dcc --- /dev/null +++ b/external/nifi/tests/07-oidc-cluster-login-test.js @@ -0,0 +1,114 @@ +const { it } = require('mocha') +const puppeteer = require ('puppeteer-core') +const expect = require('chai').expect + +describe('NiFi Login via OIDC', () => { + let browser + let page + + before(async () => { + browser = await puppeteer.connect({ + browserWSEndpoint: 'ws://'+process.env.K8SNODEIP+':'+process.env.K8SPORT, + ignoreHTTPSErrors: true + }) + page = await browser.newPage() + }) + + it('NiFi redirects to KeyCloak login page', async () => { + let gotoSucceeded = false + for ( let i = 0; ( i < 60 ) && ( ! gotoSucceeded ); i++ ) { + try { + await Promise.all([ + page.goto(process.env.NIFIURL), + page.waitForNavigation(), + page.waitForNetworkIdle() + ]) + gotoSucceeded = true + } + catch(err) { + console.log(" Connection to "+process.env.NIFIURL+"failed: "+err.message+" ( try "+i.toString()+")") + await page.waitForTimeout(5000) + i++ + } + } + const pageTitle = await page.waitForSelector('h1[id="kc-page-title"]') + const titleContent = await pageTitle.evaluate(el => el.textContent) + expect(titleContent).to.include('Sign in to your account') + }).timeout(30000) + + it('Get screenshot of Keycloak login page', async () => { + await page.screenshot({ + path: process.env.HOME+"/screenshots/01-keycloak-redirect.png", + fullPage: true + }) + }) + + it('nifi@example.com shown as logged in user', async () => { + await page.type('input[id="username"]','nifi') + await page.type('input[id="password"]','reallychangeme') + await Promise.all([ + page.click('input[id="kc-login"]'), + page.waitForNavigation(), + page.waitForNetworkIdle() + ]) + let messageContentSelector = await page.waitForSelector('div[id="message-content"]') + let messageContent = await messageContentSelector.evaluate(el => el.textContent) + // loop 30 times unless messageContent is empty + for (let i = 0; ( i < 30 ) && ( messageContent != "" ); i++ ) { + await Promise.all([ + page.reload(), + page.waitForNavigation(), + page.waitForNetworkIdle(), + page.waitForTimeout(2000) + ]) + messageContentSelector = await page.waitForSelector('div[id="message-content"]') + messageContent = await messageContentSelector.evaluate(el => el.textContent) + if ( messageContent != "") { + console.log(" Message Content: "+messageContent+" ( try "+i.toString()+")") + } + } + const currentUserElement = await page.waitForSelector('div[id="current-user"]') + const userName = await currentUserElement.evaluate(el => el.textContent) + expect(userName).to.equal('nifi@example.com') + }).timeout(300000) + + it('Get screenshot of logged in user', async () => { + await page.screenshot({ + path: process.env.HOME+"/screenshots/02-logged-in-user.png", + fullPage: true + }) + }) + + it('All nodes connected', async () => { + connectedNodesUserElement = await page.waitForSelector('span[id="connected-nodes-count"]') + connectedNodesCount = await connectedNodesUserElement.evaluate(el => el.textContent ) + // loop 30 times unless connectedNodesCount isn't "3 / 3" + for (let i = 0; ( i < 30 ) && ( connectedNodesCount != "3 / 3" ); i++ ) { + console.log(" Connected Nodes Count: "+connectedNodesCount+" ( try "+i.toString()+")") + await Promise.all([ + page.reload(), + page.waitForNavigation(), + page.waitForNetworkIdle(), + page.waitForTimeout(2000) + ]) + try { + connectedNodesUserElement = await page.waitForSelector('span[id="connected-nodes-count"]', { timeout: 1000 }) + connectedNodesCount = await connectedNodesUserElement.evaluate(el => el.textContent ) + } + catch(err) { + connectedNodesCount = err.message + } + } + expect(connectedNodesCount).to.equal('3 / 3') + }).timeout(300000) + + it('Get screenshot of all nodes connected', async () => { + await page.screenshot({ + path: process.env.HOME+"/screenshots/03-all-nodes-connected.png", + fullPage: true + }) + }) + after(async () => { + await browser.close() + }) +}) diff --git a/external/nifi/tests/07-oidc-cluster-values.yaml b/external/nifi/tests/07-oidc-cluster-values.yaml new file mode 100644 index 0000000..54f8aa5 --- /dev/null +++ b/external/nifi/tests/07-oidc-cluster-values.yaml @@ -0,0 +1,40 @@ +zookeeper: + enabled: true + +registry: + enabled: false + +certManager: + enabled: true + +persistence: + enabled: true + +sts: + startupProbe: + enabled: true + +replicaCount: 3 + +properties: + webProxyHost: ingress-nginx-controller.ingress-nginx.svc.cluster.local + isNode: true + +auth: + oidc: + enabled: true + discoveryUrl: http://keycloak.default.svc.cluster.local:8080/auth/realms/nifi/.well-known/openid-configuration + clientId: nifi + clientSecret: CZhA1IOePlXHz3PWqVwYoVAcYIUHTcDK + admin: nifi@example.com + claimIdentifyingUser: email + +ingress: + enabled: true + annotations: + nginx.ingress.kubernetes.io/backend-protocol: HTTPS + nginx.ingress.kubernetes.io/affinity: cookie + nginx.ingress.kubernetes.io/affinity-mode: persistent + path: / + hosts: + - ingress-nginx-controller.ingress-nginx.svc.cluster.local diff --git a/external/nifi/values.yaml b/external/nifi/values.yaml new file mode 100644 index 0000000..5daec4f --- /dev/null +++ b/external/nifi/values.yaml @@ -0,0 +1,456 @@ +--- +# Number of nifi nodes +replicaCount: 1 + +## Set default image, imageTag, and imagePullPolicy. +## ref: https://hub.docker.com/r/apache/nifi/ +## +image: + repository: apache/nifi + tag: "1.23.2" + pullPolicy: "IfNotPresent" + + ## Optionally specify an imagePullSecret. + ## Secret must be manually created in the namespace. + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + ## + # pullSecret: myRegistrKeySecretName + +securityContext: + runAsUser: 1000 + fsGroup: 1000 + +## @param useHostNetwork - boolean - optional +## Bind ports on the hostNetwork. Useful for CNI networking where hostPort might +## not be supported. The ports need to be available on all hosts. It can be +## used for custom metrics instead of a service endpoint. +## +## WARNING: Make sure that hosts using this are properly firewalled otherwise +## metrics and traces are accepted from any host able to connect to this host. +# + +sts: + # Parallel podManagementPolicy for faster bootstrap and teardown. Default is OrderedReady. + podManagementPolicy: Parallel + AntiAffinity: soft + useHostNetwork: null + hostPort: null + pod: + annotations: + security.alpha.kubernetes.io/sysctls: net.ipv4.ip_local_port_range=10000 65000 + #prometheus.io/scrape: "true" + serviceAccount: + create: false + #name: nifi + annotations: {} + hostAliases: [] +# - ip: "1.2.3.4" +# hostnames: +# - example.com +# - example + + startupProbe: + enabled: false + failureThreshold: 60 + periodSeconds: 10 + +## Useful if using any custom secrets +## Pass in some secrets to use (if required) +# secrets: +# - name: myNifiSecret +# keys: +# - key1 +# - key2 +# mountPath: /opt/nifi/secret + +## Useful if using any custom configmaps +## Pass in some configmaps to use (if required) +# configmaps: +# - name: myNifiConf +# keys: +# - myconf.conf +# mountPath: /opt/nifi/custom-config + + +properties: + # https://nifi.apache.org/docs/nifi-docs/html/administration-guide.html#nifi_sensitive_props_key + sensitiveKey: changeMechangeMe # Must have at least 12 characters + # NiFi assumes conf/nifi.properties is persistent but this helm chart + # recreates it every time. Setting the Sensitive Properties Key + # (nifi.sensitive.props.key) is supposed to happen at the same time + # /opt/nifi/data/flow.xml.gz sensitive properties are encrypted. If that + # doesn't happen then NiFi won't start because decryption fails. + # So if sensitiveKeySetFile is configured but doesn't exist, assume + # /opt/nifi/flow.xml.gz hasn't been encrypted and follow the procedure + # https://nifi.apache.org/docs/nifi-docs/html/administration-guide.html#updating-the-sensitive-properties-key + # to simultaneously encrypt it and set nifi.sensitive.props.key. + # sensitiveKeySetFile: /opt/nifi/data/sensitive-props-key-applied + # If sensitiveKey was already set, then pass in sensitiveKeyPrior with the old key. + # sensitiveKeyPrior: OldPasswordToChangeFrom + algorithm: NIFI_PBKDF2_AES_GCM_256 + # use externalSecure for when inbound SSL is provided by nginx-ingress or other external mechanism + externalSecure: false + isNode: false + httpsPort: 8443 + webProxyHost: # : (If Nifi service is NodePort or LoadBalancer) + clusterPort: 6007 + zkClientEnsembleTraker: false # https://issues.apache.org/jira/browse/NIFI-10481 + clusterNodeConnectionTimeout: '5 sec' + clusterNodeReadTimeout: '5 sec' + zookeeperConnectTimeout: '3 secs' + zookeeperSessionTimeout: '3 secs' + archiveMaxRetentionPeriod: "3 days" + archiveMaxUsagePercentage: "85%" + provenanceStorage: "8 GB" + provenanceMaxStorageTime: "10 days" + flowArchiveMaxTime: "30 days" + flowArchiveMaxStorage: "500 MB" + siteToSite: + port: 10000 + # use properties.safetyValve to pass explicit 'key: value' pairs that overwrite other configuration + safetyValve: + #nifi.variable.registry.properties: "${NIFI_HOME}/example1.properties, ${NIFI_HOME}/example2.properties" + nifi.web.http.network.interface.default: eth0 + # listen to loopback interface so "kubectl port-forward ..." works + nifi.web.http.network.interface.lo: lo + + ## Include aditional processors + # customLibPath: "/opt/configuration_resources/custom_lib" + +## Include additional libraries in the Nifi containers by using the postStart handler +## ref: https://kubernetes.io/docs/tasks/configure-pod-container/attach-handler-lifecycle-event/ +# postStart: /opt/nifi/psql; wget -P /opt/nifi/psql https://jdbc.postgresql.org/download/postgresql-42.2.6.jar + +# Nifi User Authentication +auth: + # If set while LDAP is enabled, this value will be used for the initial admin and not the ldap bind dn / admin + admin: CN=admin, OU=NIFI + SSL: + keystorePasswd: changeMe + truststorePasswd: changeMe + + # Automaticaly disabled if OIDC or LDAP enabled + singleUser: + username: username + password: changemechangeme # Must to have at least 12 characters + + clientAuth: + enabled: false + + ldap: + enabled: false + host: #ldap://: + searchBase: #CN=Users,DC=ldap,DC=example,DC=be + admin: #cn=admin,dc=ldap,dc=example,dc=be + pass: #ChangeMe + searchFilter: (objectClass=*) + userIdentityAttribute: cn + authStrategy: SIMPLE # How the connection to the LDAP server is authenticated. Possible values are ANONYMOUS, SIMPLE, LDAPS, or START_TLS. + identityStrategy: USE_DN + authExpiration: 12 hours + userSearchScope: ONE_LEVEL # Search scope for searching users (ONE_LEVEL, OBJECT, or SUBTREE). Required if searching users. + groupSearchScope: ONE_LEVEL # Search scope for searching groups (ONE_LEVEL, OBJECT, or SUBTREE). Required if searching groups. + + oidc: + enabled: false + discoveryUrl: #http://:/auth/realms//.well-known/openid-configuration + clientId: # + clientSecret: # + claimIdentifyingUser: email + admin: nifi@example.com + preferredJwsAlgorithm: + ## Request additional scopes, for example profile + additionalScopes: + +openldap: + enabled: false + persistence: + enabled: true + env: + LDAP_ORGANISATION: # name of your organization e.g. "Example" + LDAP_DOMAIN: # your domain e.g. "ldap.example.be" + LDAP_BACKEND: "hdb" + LDAP_TLS: "true" + LDAP_TLS_ENFORCE: "false" + LDAP_REMOVE_CONFIG_AFTER_SETUP: "false" + adminPassword: #ChengeMe + configPassword: #ChangeMe + customLdifFiles: + 1-default-users.ldif: |- + # You can find an example ldif file at https://github.com/cetic/fadi/blob/master/examples/basic/example.ldif +## Expose the nifi service to be accessed from outside the cluster (LoadBalancer service). +## or access it from within the cluster (ClusterIP service). Set the service type and the port to serve it. +## ref: http://kubernetes.io/docs/user-guide/services/ +## + +# headless service +headless: + type: ClusterIP + annotations: + service.alpha.kubernetes.io/tolerate-unready-endpoints: "true" + +# ui service +service: + type: ClusterIP + httpsPort: 8443 + # nodePort: 30236 + annotations: {} + # loadBalancerIP: + ## Load Balancer sources + ## https://kubernetes.io/docs/tasks/access-application-cluster/configure-cloud-provider-firewall/#restrict-access-for-loadbalancer-service + ## + # loadBalancerSourceRanges: + # - 10.10.10.0/24 + ## OIDC authentication requires "sticky" session on the LoadBalancer for JWT to work properly...but AWS doesn't like it on creation + # sessionAffinity: ClientIP + # sessionAffinityConfig: + # clientIP: + # timeoutSeconds: 10800 + + # Enables additional port/ports to nifi service for internal processors + processors: + enabled: false + ports: + - name: processor01 + port: 7001 + targetPort: 7001 + #nodePort: 30701 + - name: processor02 + port: 7002 + targetPort: 7002 + #nodePort: 30702 +## Configure containerPorts section with following attributes: name, containerport and protocol. +containerPorts: [] +# - name: example +# containerPort: 1111 +# protocol: TCP + +## Configure Ingress based on the documentation here: https://kubernetes.io/docs/concepts/services-networking/ingress/ +## +ingress: + enabled: false + # className: nginx + annotations: {} + tls: [] + hosts: [] + path: / + # If you want to change the default path, see this issue https://github.com/cetic/helm-nifi/issues/22 + +# Amount of memory to give the NiFi java heap +jvmMemory: 2g + +# Separate image for tailing each log separately and checking zookeeper connectivity +sidecar: + image: busybox + tag: "1.32.0" + imagePullPolicy: "IfNotPresent" + +## Enable persistence using Persistent Volume Claims +## ref: http://kubernetes.io/docs/user-guide/persistent-volumes/ +## +persistence: + enabled: false + + # When creating persistent storage, the NiFi helm chart can either reference an already-defined + # storage class by name, such as "standard" or can define a custom storage class by specifying + # customStorageClass: true and providing the "storageClass", "storageProvisioner" and "storageType". + # For example, to use SSD storage on Google Compute Engine see values-gcp.yaml + # + # To use a storage class that already exists on the Kubernetes cluster, we can simply reference it by name. + # For example: + # storageClass: standard + # + # The default storage class is used if this variable is not set. + + accessModes: [ReadWriteOnce] + + ## Use subPath and have 1 persistent volume instead of 7 volumes - use when your k8s nodes have limited volume slots, to limit waste of space, + ## or your available volume sizes are quite large + # The one disk will have a directory folder for each volumeMount, but this is hidden. Run 'mount' to view each mount. + subPath: + enabled: false + name: data + size: 30Gi + + ## Storage Capacities for persistent volumes (these are ignored if using one volume with subPath) + configStorage: + size: 100Mi + authconfStorage: + size: 100Mi + # Storage capacity for the 'data' directory, which is used to hold things such as the flow.xml.gz, configuration, state, etc. + dataStorage: + size: 1Gi + # Storage capacity for the FlowFile repository + flowfileRepoStorage: + size: 10Gi + # Storage capacity for the Content repository + contentRepoStorage: + size: 10Gi + # Storage capacity for the Provenance repository. When changing this, one should also change the properties.provenanceStorage value above, also. + provenanceRepoStorage: + size: 10Gi + # Storage capacity for nifi logs + logStorage: + size: 5Gi + +## Configure resource requests and limits +## ref: http://kubernetes.io/docs/user-guide/compute-resources/ +## +resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + +logresources: + requests: + cpu: 10m + memory: 10Mi + limits: + cpu: 50m + memory: 50Mi + +## Enables setting your own affinity. Mutually exclusive with sts.AntiAffinity +## You need to set the value of sts.AntiAffinity other than "soft" and "hard" +affinity: {} + +nodeSelector: {} + +tolerations: [] + +initContainers: {} + # foo-init: # <- will be used as container name + # image: "busybox:1.30.1" + # imagePullPolicy: "IfNotPresent" + # command: ['sh', '-c', 'echo this is an initContainer'] + # volumeMounts: + # - mountPath: /tmp/foo + # name: foo + +extraVolumeMounts: [] + +extraVolumes: [] + +## Extra containers +extraContainers: [] + +terminationGracePeriodSeconds: 30 + +## Extra environment variables that will be pass onto deployment pods +env: [] + +## Extra environment variables from secrets and config maps +envFrom: [] + +## Extra options to add to the bootstrap.conf file +extraOptions: [] + +# envFrom: +# - configMapRef: +# name: config-name +# - secretRef: +# name: mysecret + +## Openshift support +## Use the following varables in order to enable Route and Security Context Constraint creation +openshift: + scc: + enabled: false + route: + enabled: false + #host: www.test.com + #path: /nifi + +# ca server details +# Setting this true would create a nifi-toolkit based ca server +# The ca server will be used to generate self-signed certificates required setting up secured cluster +ca: + ## If true, enable the nifi-toolkit certificate authority + enabled: false + persistence: + enabled: true + server: "" + service: + port: 9090 + token: sixteenCharacters + admin: + cn: admin + serviceAccount: + create: false + #name: nifi-ca + openshift: + scc: + enabled: false + +# cert-manager support +# Setting this true will have cert-manager create a private CA for the cluster +# as well as the certificates for each cluster node. +certManager: + enabled: false + clusterDomain: cluster.local + keystorePasswd: changeme + truststorePasswd: changeme + replaceDefaultTrustStore: false + additionalDnsNames: + - localhost + refreshSeconds: 300 + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 100m + memory: 128Mi + # cert-manager takes care of rotating the node certificates, so default + # their lifetime to 90 days. But when the CA expires you may need to + # 'helm delete' the cluster, delete all the node certificates and secrets, + # and then 'helm install' the NiFi cluster again. If a site-to-site trusted + # CA or a NiFi Registry CA certificate expires, you'll need to restart all + # pods to pick up the new version of the CA certificate. So default the CA + # lifetime to 10 years to avoid that happening very often. + # c.f. https://github.com/cert-manager/cert-manager/issues/2478#issuecomment-1095545529 + certDuration: 2160h + caDuration: 87660h + +# ------------------------------------------------------------------------------ +# Zookeeper: +# ------------------------------------------------------------------------------ +zookeeper: + ## If true, install the Zookeeper chart + ## ref: https://github.com/bitnami/charts/blob/master/bitnami/zookeeper/values.yaml + enabled: true + ## If the Zookeeper Chart is disabled a URL and port are required to connect + url: "" + port: 2181 + replicaCount: 3 + +# ------------------------------------------------------------------------------ +# Nifi registry: +# ------------------------------------------------------------------------------ +registry: + ## If true, install the Nifi registry + enabled: false + url: "" + port: 80 + ## Add values for the nifi-registry here + ## ref: https://github.com/dysnix/charts/blob/main/dysnix/nifi-registry/values.yaml + +# Configure metrics +metrics: + prometheus: + # Enable Prometheus metrics + enabled: false + # Port used to expose Prometheus metrics + port: 9092 + serviceMonitor: + # Enable deployment of Prometheus Operator ServiceMonitor resource + enabled: false + # namespace: monitoring + # Additional labels for the ServiceMonitor + labels: {}