Merge branch 'develop' into master

cetic · Nov 26, 2020 · ade7693 · ade7693
2 parents 8a6d7d7 + 107f9f9
commit ade7693
Show file tree

Hide file tree

Showing 8 changed files with 130 additions and 79 deletions.
diff --git a/Chart.yaml b/Chart.yaml
@@ -1,7 +1,7 @@
 ---
 apiVersion: v2
 name: nifi
-version: 0.6.0
+version: 0.6.1
 appVersion: 1.12.1
 description: Apache NiFi is a software project from the Apache Software Foundation designed to automate the flow of data between software systems.
 keywords:

diff --git a/configs/authorizers.xml b/configs/authorizers.xml
@@ -47,11 +47,14 @@
         <class>org.apache.nifi.authorization.FileUserGroupProvider</class>
         <property name="Users File">./auth-conf/users.xml</property>
         <property name="Legacy Authorized Users File"></property>
-{{- range $i := until $replicas }}
+        {{- range $i := until $replicas }}
         <property name="Initial User Identity {{ $i }}">CN={{ $fullname }}-{{ $i }}.{{ $fullname }}-headless.{{ $namespace }}.svc.cluster.local, OU=NIFI</property>
-{{- end }}
-        <property name="Initial User Identity ca admin">CN={{ .Values.ca.admin.cn }}, OU=NIFI</property>
+        {{- end }}
+        {{- if .Values.auth.ldap.enabled}}
+        <property name="Initial User Identity admin">{{.Values.auth.ldap.admin}}</property>
+        {{- else }}
         <property name="Initial User Identity admin">{{ .Values.auth.admin }}</property>
+        {{- end}}
     </userGroupProvider>
     <!--
         The LdapUserGroupProvider will retrieve users and groups from an LDAP server. The users and groups
@@ -114,33 +117,33 @@
         NOTE: Any identity mapping rules specified in nifi.properties will also be applied to the user identities.
             Group names are not mapped.
     -->
-    <!-- To enable the ldap-user-group-provider remove 2 lines. This is 1 of 2.
+    {{- if .Values.auth.ldap.enabled}}
     <userGroupProvider>
         <identifier>ldap-user-group-provider</identifier>
         <class>org.apache.nifi.ldap.tenants.LdapUserGroupProvider</class>
-        <property name="Authentication Strategy">START_TLS</property>
-        <property name="Manager DN"></property>
-        <property name="Manager Password"></property>
-        <property name="TLS - Keystore"></property>
-        <property name="TLS - Keystore Password"></property>
-        <property name="TLS - Keystore Type"></property>
-        <property name="TLS - Truststore"></property>
-        <property name="TLS - Truststore Password"></property>
-        <property name="TLS - Truststore Type"></property>
-        <property name="TLS - Client Auth"></property>
-        <property name="TLS - Protocol"></property>
-        <property name="TLS - Shutdown Gracefully"></property>
-        <property name="Referral Strategy">FOLLOW</property>
+        <property name="Authentication Strategy">SIMPLE</property>
+        <property name="Manager DN">{{.Values.auth.ldap.admin}}</property>
+        <property name="Manager Password">{{.Values.auth.ldap.pass}}</property>
+        <property name="TLS - Keystore">/opt/nifi/nifi-current/conf/{{.Release.Name}}-nifi-0.{{.Release.Name}}-nifi-headless.{{.Values.properties.namespace}}.svc.cluster.local/keystore.jks</property>
+        <property name="TLS - Keystore Password">{{.Values.auth.SSL.keystorePasswd}}</property>
+        <property name="TLS - Keystore Type">jks</property>
+        <property name="TLS - Truststore">/opt/nifi/nifi-current/conf/{{.Release.Name}}-nifi-0.{{.Release.Name}}-nifi-headless.{{.Values.properties.namespace}}.svc.cluster.local/truststore.jks</property>
+        <property name="TLS - Truststore Password">{{.Values.auth.SSL.truststorePasswd}}</property>
+        <property name="TLS - Truststore Type">JKS</property>
+        <property name="TLS - Client Auth">NONE</property>
+        <property name="TLS - Protocol">TLS</property>
+        <property name="TLS - Shutdown Gracefully">false</property>
+        <property name="Referral Strategy">IGNORE</property>
         <property name="Connect Timeout">10 secs</property>
         <property name="Read Timeout">10 secs</property>
-        <property name="Url"></property>
+        <property name="Url">{{.Values.auth.ldap.host}}</property>
         <property name="Page Size"></property>
         <property name="Sync Interval">30 mins</property>
-        <property name="User Search Base"></property>
+        <property name="User Search Base">{{.Values.auth.ldap.searchBase}}</property>
         <property name="User Object Class">person</property>
         <property name="User Search Scope">ONE_LEVEL</property>
-        <property name="User Search Filter"></property>
-        <property name="User Identity Attribute"></property>
+        <property name="User Search Filter">{{.Values.auth.ldap.searchFilter}}</property>
+        <property name="User Identity Attribute">{{.Values.auth.ldap.UserIdentityAttribute}}</property>
         <property name="User Group Name Attribute"></property>
         <property name="User Group Name Attribute - Referenced Group Attribute"></property>
         <property name="Group Search Base"></property>
@@ -151,7 +154,7 @@
         <property name="Group Member Attribute"></property>
         <property name="Group Member Attribute - Referenced User Attribute"></property>
     </userGroupProvider>
-    To enable the ldap-user-group-provider remove 2 lines. This is 2 of 2. -->
+    {{- end}}
 
     <!--
         The CompositeUserGroupProvider will provide support for retrieving users and groups from multiple sources.
@@ -161,13 +164,14 @@
             NOTE: Any identity mapping rules specified in nifi.properties are not applied in this implementation. This behavior
             would need to be applied by the base implementation.
     -->
-    <!-- To enable the composite-user-group-provider remove 2 lines. This is 1 of 2.
+    {{- if .Values.auth.ldap.enabled}}
     <userGroupProvider>
-        <identifier>composite-user-group-provider</identifier>
-        <class>org.apache.nifi.authorization.CompositeUserGroupProvider</class>
-        <property name="User Group Provider 1"></property>
+        <identifier>composite-configurable-user-group-provider</identifier>
+        <class>org.apache.nifi.authorization.CompositeConfigurableUserGroupProvider</class>
+        <property name="Configurable User Group Provider">file-user-group-provider</property>
+        <property name="User Group Provider 1">ldap-user-group-provider</property>
     </userGroupProvider>
-    To enable the composite-user-group-provider remove 2 lines. This is 2 of 2. -->
+    {{- end}}
 
     <!--
         The CompositeConfigurableUserGroupProvider will provide support for retrieving users and groups from multiple sources.
@@ -218,11 +222,16 @@
         <class>org.apache.nifi.authorization.FileAccessPolicyProvider</class>
         <property name="User Group Provider">file-user-group-provider</property>
         <property name="Authorizations File">./auth-conf/authorizations.xml</property>
+        {{- if .Values.auth.ldap.enabled}}
+        <property name="Initial Admin Identity">{{.Values.auth.ldap.admin}}</property>
+        {{- else }}
         <property name="Initial Admin Identity">{{ .Values.auth.admin }}</property>
+        {{- end}}
         <property name="Legacy Authorized Users File"></property>
-{{- range $i := until $replicas }}
+        {{- range $i := until $replicas }}
         <property name="Node Identity {{ $i }}">CN={{ $fullname }}-{{ $i }}.{{ $fullname }}-headless.{{ $namespace }}.svc.cluster.local, OU=NIFI</property>
-{{- end }}
+        {{- end }}
+        <property name="Node Identity"></property>
     </accessPolicyProvider>
      <!--
         The StandardManagedAuthorizer. This authorizer implementation must be configured with the
@@ -259,14 +268,15 @@
             NOTE: Any identity mapping rules specified in nifi.properties will also be applied to the node identities,
             so the values should be the unmapped identities (i.e. full DN from a certificate).
     -->
-    <!-- <authorizer>
+    {{- if .Values.auth.ldap.enabled}}
+    <authorizer>
         <identifier>file-provider</identifier>
         <class>org.apache.nifi.authorization.FileAuthorizer</class>
         <property name="Authorizations File">./conf/authorizations.xml</property>
         <property name="Users File">./conf/users.xml</property>
-        <property name="Initial Admin Identity"></property>
+        <property name="Initial Admin Identity">{{.Values.auth.ldap.admin}}</property>
         <property name="Legacy Authorized Users File"></property>
-        <property name="Node Identity 1"></property>
+
     </authorizer>
-    -->
+    {{- end}}
 </authorizers>
diff --git a/configs/login-identity-providers.xml b/configs/login-identity-providers.xml
@@ -66,26 +66,26 @@
     <provider>
         <identifier>ldap-provider</identifier>
         <class>org.apache.nifi.ldap.LdapProvider</class>
-        <property name="Authentication Strategy">START_TLS</property>
-        <property name="Manager DN"></property>
-        <property name="Manager Password"></property>
-        <property name="TLS - Keystore"></property>
-        <property name="TLS - Keystore Password"></property>
-        <property name="TLS - Keystore Type"></property>
-        <property name="TLS - Truststore"></property>
-        <property name="TLS - Truststore Password"></property>
-        <property name="TLS - Truststore Type"></property>
-        <property name="TLS - Client Auth"></property>
-        <property name="TLS - Protocol"></property>
-        <property name="TLS - Shutdown Gracefully"></property>
-        
+        <property name="Authentication Strategy">SIMPLE</property>
+        <property name="Manager DN">{{.Values.auth.ldap.admin}}</property>
+        <property name="Manager Password">{{.Values.auth.ldap.pass}}</property>
+        <property name="TLS - Keystore">/opt/nifi/nifi-current/conf/{{.Release.Name}}-nifi-0.{{.Release.Name}}-nifi-headless.{{.Release.Namespace}}.svc.cluster.local/keystore.jks</property>
+        <property name="TLS - Keystore Password">{{.Values.auth.keystorePasswd}}</property>
+        <property name="TLS - Keystore Type">JKS</property>
+        <property name="TLS - Truststore">/opt/nifi/nifi-current/conf/{{.Release.Name}}-nifi-0.{{.Release.Name}}-nifi-headless.{{.Release.Namespace}}.svc.cluster.local/truststore.jks</property>
+        <property name="TLS - Truststore Password">{{.Values.auth.truststorePasswd}}</property>
+        <property name="TLS - Truststore Type">JKS</property>
+        <property name="TLS - Client Auth">NONE</property>
+        <property name="TLS - Protocol">TLS</property>
+        <property name="TLS - Shutdown Gracefully">false</property>
+
         <property name="Referral Strategy">FOLLOW</property>
         <property name="Connect Timeout">10 secs</property>
         <property name="Read Timeout">10 secs</property>
         <property name="Url">{{.Values.auth.ldap.host}}</property>
         <property name="User Search Base">{{.Values.auth.ldap.searchBase}}</property>
-        <property name="User Search Filter">{{.Values.auth.ldap.searchFilter}}</property>
-        <property name="Identity Strategy">USE_DN</property>
+        <property name="User Search Filter">(cn={0})</property>
+        <property name="Identity Strategy">{{.Values.auth.ldap.IdentityStrategy}}</property>
         <property name="Authentication Expiration">12 hours</property>
     </provider>
     {{end}}

diff --git a/configs/nifi.properties b/configs/nifi.properties
@@ -100,7 +100,7 @@ nifi.provenance.repository.index.threads=2
 nifi.provenance.repository.compress.on.rollover=true
 nifi.provenance.repository.always.sync=false
 nifi.provenance.repository.journal.count=16
-# Comma-separated list of fields. Fields that are not indexed will not be searchable. Valid fields are: 
+# Comma-separated list of fields. Fields that are not indexed will not be searchable. Valid fields are:
 # EventType, FlowFileUUID, Filename, TransitURI, ProcessorID, AlternateIdentifierURI, Relationship, Details
 nifi.provenance.repository.indexed.fields=EventType, FlowFileUUID, Filename, ProcessorID, Relationship
 # FlowFile Attributes that should be indexed and made searchable.  Some examples to consider are filename, uuid, mime.type
@@ -121,28 +121,30 @@ nifi.components.status.repository.buffer.size=1440
 nifi.components.status.snapshot.frequency=1 min
 
 # Site to Site properties
-nifi.remote.input.host= # set by server contianer in statefulset.yaml
+nifi.remote.input.host=
 nifi.remote.input.secure={{.Values.properties.clusterSecure}}
 nifi.remote.input.socket.port={{.Values.properties.siteToSite.port}}
 nifi.remote.input.http.enabled=true
 nifi.remote.input.http.transaction.ttl=30 sec
+nifi.remote.contents.cache.expiration=30 secs
 
 # web properties #
 nifi.web.war.directory=./lib
 nifi.web.proxy.host={{.Values.properties.webProxyHost}}
-nifi.web.http.host=  # set by server contianer in statefulset.yaml
-nifi.web.https.host= # set by server contianer in statefulset.yaml
     {{if .Values.properties.clusterSecure}}
 nifi.web.http.port=
 nifi.web.https.port={{.Values.properties.httpsPort}}
     {{else}}
 nifi.web.http.port={{.Values.properties.httpPort}}
 nifi.web.https.port=
     {{end}}
+nifi.web.http.host=
 nifi.web.http.network.interface.default=
+nifi.web.https.host={{.Values.properties.webHttpsHost}}
 nifi.web.https.network.interface.default=
 nifi.web.jetty.working.directory=./work/jetty
 nifi.web.jetty.threads=200
+# nifi.web.proxy.context.path=
 
 # security properties #
 nifi.sensitive.props.key=
@@ -151,6 +153,18 @@ nifi.sensitive.props.algorithm=PBEWITHMD5AND256BITAES-CBC-OPENSSL
 nifi.sensitive.props.provider=BC
 nifi.sensitive.props.additional.keys=
 
+{{if .Values.auth.ldap.enabled}}
+nifi.security.keystore=/opt/nifi/nifi-current/conf/{{.Release.Name}}-nifi-0.{{.Release.Name}}-nifi-headless.{{.Release.Namespace}}.svc.cluster.local/keystore.jks
+nifi.security.keystoreType=jks
+nifi.security.keystorePasswd={{.Values.auth.SSL.keystorePasswd}}
+nifi.security.keyPasswd={{.Values.auth.SSL.keystorePasswd}}
+nifi.security.truststore=/opt/nifi/nifi-current/conf/{{.Release.Name}}-nifi-0.{{.Release.Name}}-nifi-headless.{{.Release.Namespace}}.svc.cluster.local/truststore.jks
+nifi.security.truststoreType=jks
+nifi.security.truststorePasswd={{.Values.auth.SSL.truststorePasswd}}
+proxiedEntity={{.Values.auth.ldap.admin}}
+nifi.security.user.authorizer=file-provider
+nifi.security.needClientAuth={{.Values.properties.needClientAuth}}
+{{else}}
 nifi.security.keystore=
 nifi.security.keystoreType=
 nifi.security.keystorePasswd=
@@ -160,11 +174,12 @@ nifi.security.truststoreType=
 nifi.security.truststorePasswd=
 nifi.security.needClientAuth={{.Values.properties.needClientAuth}}
 nifi.security.user.authorizer={{.Values.properties.authorizer}}
-    {{if .Values.auth.ldap.enabled}}
+{{end}}
+{{if .Values.auth.ldap.enabled}}
 nifi.security.user.login.identity.provider=ldap-provider
-    {{else}}
+{{else}}
 nifi.security.user.login.identity.provider=
-    {{end}}
+{{end}}
 nifi.security.ocsp.responder.url=
 nifi.security.ocsp.responder.certificate=
 
@@ -177,8 +192,10 @@ nifi.security.user.oidc.client.id={{.Values.auth.oidc.clientId}}
 nifi.security.user.oidc.client.secret={{.Values.auth.oidc.clientSecret}}
 nifi.security.user.oidc.preferred.jwsalgorithm=
 nifi.security.user.oidc.claim.identifying.user={{.Values.auth.oidc.claimIdentifyingUser}}
+nifi.security.user.oidc.additional.scopes={{.Values.auth.oidc.additionalScopes}}
     {{end}}
 
+
 # Apache Knox SSO Properties #
 nifi.security.user.knox.url=
 nifi.security.user.knox.publicKey=
@@ -201,7 +218,7 @@ nifi.cluster.protocol.is.secure={{.Values.properties.clusterSecure}}
 
 # cluster node properties (only configure for cluster nodes) #
 nifi.cluster.is.node={{.Values.properties.isNode}}
-nifi.cluster.node.address= # set by server contianer in statefulset.yaml
+nifi.cluster.node.address=
 nifi.cluster.node.protocol.port={{.Values.properties.clusterPort}}
 nifi.cluster.node.protocol.threads=10
 nifi.cluster.node.protocol.max.threads=50

diff --git a/templates/ingress.yaml b/templates/ingress.yaml
@@ -52,4 +52,4 @@ spec:
               servicePort: {{ $ingressHttpPort }}
   {{- end }}
 {{- end }}
-{{- end }}
+{{- end }}
diff --git a/templates/service.yaml b/templates/service.yaml
@@ -67,6 +67,7 @@ spec:
     - port: {{ .Values.service.httpsPort }}
       name: https
       targetPort: {{ .Values.properties.httpsPort }}
+      nodePort: {{ .Values.service.nodePort }}
 {{- else }}
     - port: {{ .Values.service.httpPort }}
       name: http

diff --git a/templates/statefulset.yaml b/templates/statefulset.yaml
@@ -137,6 +137,7 @@ spec:
 
           openssl pkcs12 -in "keystore.pkcs12" -out "key.pem" -nocerts -nodes -password "env:PASS"
           openssl pkcs12 -in "keystore.pkcs12" -out "crt.pem" -clcerts -nokeys -password "env:PASS"
+          openssl pkcs12 -in "keystore.pkcs12" -out "keystore.jks" -clcerts -nokeys -password "env:PASS"
         volumeMounts:
           - name: "config-data"
             mountPath: /data/config-data
@@ -189,6 +190,7 @@ spec:
           prop_replace nifi.cluster.node.address ${FQDN}
           prop_replace nifi.zookeeper.connect.string ${NIFI_ZOOKEEPER_CONNECT_STRING}
 
+{{- if .Values.auth.oidc.enabled }}
 {{- if .Values.properties.clusterSecure }}
           # Update nifi.properties for security properties
           prop_replace nifi.web.https.host ${FQDN}
@@ -216,6 +218,8 @@ spec:
           prop_replace {{ $prop }} "{{ $val }}" nifi.properties
   {{- end }}
 {{- end }}
+{{- end }}
+
 
           exec bin/nifi.sh run & nifi_pid="$!"
 
@@ -275,6 +279,10 @@ spec:
 
           echo NiFi running with PID ${nifi_pid}.
           wait ${nifi_pid}
+          {{ if .Values.auth.ldap }}
+          /opt/nifi/nifi-toolkit-current/bin/tls-toolkit.sh standalone -n '{{.Release.Name}}-nifi-0.{{.Release.Name}}-nifi-headless.{{.Release.Namespace}}.svc.cluster.local' -C '{{.Values.auth.ldap.admin}}' -o '/opt/nifi/nifi-current/conf/' -P {{.Values.auth.SSL.truststorePasswd}}  -S {{.Values.auth.SSL.keystorePasswd}}  --nifiPropertiesFile /opt/nifi/nifi-current/conf/nifi.properties
+          exec bin/nifi.sh run
+          {{ end }}
         resources:
 {{ toYaml .Values.resources | indent 10 }}
         ports: