Merge branch 'apache:main' into NIFI-12300

NOTICE

@@ -134,4 +134,4 @@ This includes derived works from Spring Framework available under Apache Softwar
     The derived work is adapted from
       https://github.com/spring-projects/spring-framework/blob/main/spring-web/src/main/java/org/springframework/web/util/UriComponentsBuilder.java
     and can be found in
-      nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/UriUtils.java
+      nifi-api/src/main/java/org/apache/nifi/processor/util/URLValidator.java

minifi/minifi-commons/minifi-commons-framework/src/main/java/org/apache/nifi/minifi/commons/service/StandardFlowPropertyEncryptor.java

@@ -93,7 +93,9 @@ private Map<String, Set<String>> flowProvidedSensitiveProperties(VersionedDatafl
     }
 
     private Map<String, Set<String>> runtimeManifestSensitiveProperties() {
-        return ofNullable(runTimeManifest.getBundles()).orElse(List.of())
+        return ofNullable(runTimeManifest)
+            .map(RuntimeManifest::getBundles)
+            .orElse(List.of())
             .stream()
             .flatMap(bundle -> Stream.of(
                 ofNullable(bundle.getComponentManifest().getProcessors()).orElse(List.of()),

minifi/minifi-toolkit/minifi-toolkit-assembly/README.md

@@ -64,7 +64,7 @@ It's not guaranteed in all circumstances that the migration will result in a cor
 # <a id="encrypt-sensitive-properties-in-bootstrapconf" href="#encrypt-sensitive-properties-in-bootstrapconf">Encrypting Sensitive Properties in bootstrap.conf</a>
 
 ## MiNiFi Encrypt-Config Tool
-The encrypt-config command line tool (invoked in minifi-toolkit as ./bin/encrypt-config.sh or bin\encrypt-config.bat) reads from a bootstrap.conf file with plaintext sensitive configuration values and encrypts each value using a random encryption key. It replaces the plain values with the protected value in the same file, or writes to a new bootstrap.conf file if specified.
+The encrypt-config command line tool (invoked in minifi-toolkit as ./bin/encrypt-config.sh or bin\encrypt-config.bat) reads from a bootstrap.conf file with plaintext sensitive configuration values and encrypts each value using a random encryption key. It replaces the plain values with the protected value in the same file, or writes to a new bootstrap.conf file if specified. Additionally it can be used to encrypt the unencrypted sensitive properties (if any) in the flow.json.raw. For using this functionality `nifi.minifi.sensitive.props.key` and `nifi.minifi.sensitive.props.algorithm` has to be provided in bootstrap.conf. 
 
 The supported encryption algorithm utilized is AES/GCM 256-bit.
 
@@ -78,9 +78,12 @@ To show help:
 The following are the available options:
 * -b, --bootstrapConf <bootstrapConfPath> Path to file containing Bootstrap Configuration [bootstrap.conf]
 * -B, --outputBootstrapConf <outputBootstrapConf> Path to output file for Bootstrap Configuration [bootstrap.conf] with root key configured
+* -x, --encryptRawFlowJsonOnly Process Raw Flow Configuration [flow.json.raw] sensitive property values without modifying other configuration files
+* -f, --rawFlowJson <flowConfigurationPath> Path to file containing Raw Flow Configuration [flow.json.raw] that will be updated unless the output argument is provided
+* -g, --outputRawFlowJson <outputFlowConfigurationPath> ath to output file for Raw Flow Configuration [flow.json.raw] with property protection applied
 * -h, --help          Show help message and exit.
 
-### Example
+### Example 1
 As an example of how the tool works with the following existing values in the bootstrap.conf file:
 ```
 nifi.sensitive.props.key=thisIsABadSensitiveKeyPassword
@@ -141,6 +144,18 @@ Sensitive configuration values are encrypted by the tool by default, however you
 
 If the bootstrap.conf file already has valid protected values, those property values are not modified by the tool.
 
+### Example 2
+An example to encrypt non encrypted sensitive properties in flow.json.raw
+```
+nifi.sensitive.props.key=sensitivePropsKey
+nifi.sensitive.props.algorithm=NIFI_PBKDF2_AES_GCM_256
+```
+Enter the following arguments when using the tool:
+```
+encrypt-config.sh -x -f flow.json.raw
+```
+As a result, the flow.json.raw file is overwritten with encrypted sensitive properties
+The algorithm uses the property descriptors in the flow.json.raw to determine if a property is sensitive or not. If that information is missing, no properties will be encrypted even if it is defined sensitive in the agent manifest.
 
 ## Getting Help
 If you have questions, you can reach out to our mailing list: dev@nifi.apache.org

minifi/minifi-toolkit/minifi-toolkit-encrypt-config/pom.xml

@@ -52,6 +52,18 @@
         <dependency>
             <groupId>org.apache.nifi.minifi</groupId>
             <artifactId>minifi-commons-utils</artifactId>
+            <version>2.0.0-SNAPSHOT</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.nifi.minifi</groupId>
+            <artifactId>minifi-commons-framework</artifactId>
+            <version>2.0.0-SNAPSHOT</version>
+            <exclusions>
+                <exclusion>
+                    <groupId>org.apache.nifi</groupId>
+                    <artifactId>nifi-framework-core</artifactId>
+                </exclusion>
+            </exclusions>
         </dependency>
     </dependencies>
 </project>

minifi/minifi-toolkit/minifi-toolkit-encrypt-config/src/main/java/org/apache/nifi/minifi/toolkit/config/command/MiNiFiEncryptConfig.java

@@ -14,8 +14,16 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.nifi.minifi.toolkit.config.command;
 
+import static java.nio.file.Files.readAllBytes;
+import static java.nio.file.Files.write;
+import static java.util.Optional.ofNullable;
+import static org.apache.commons.lang3.StringUtils.isAnyBlank;
+import static org.apache.nifi.minifi.commons.api.MiNiFiProperties.NIFI_MINIFI_SENSITIVE_PROPS_ALGORITHM;
+import static org.apache.nifi.minifi.commons.api.MiNiFiProperties.NIFI_MINIFI_SENSITIVE_PROPS_KEY;
+
 import java.io.IOException;
 import java.io.UncheckedIOException;
 import java.nio.file.Files;
@@ -24,8 +32,13 @@
 import java.security.SecureRandom;
 import java.util.HashSet;
 import java.util.HexFormat;
-import java.util.Optional;
 import java.util.Set;
+import org.apache.nifi.controller.flow.VersionedDataflow;
+import org.apache.nifi.encrypt.PropertyEncryptorBuilder;
+import org.apache.nifi.minifi.commons.service.FlowPropertyEncryptor;
+import org.apache.nifi.minifi.commons.service.FlowSerDeService;
+import org.apache.nifi.minifi.commons.service.StandardFlowPropertyEncryptor;
+import org.apache.nifi.minifi.commons.service.StandardFlowSerDeService;
 import org.apache.nifi.minifi.properties.BootstrapProperties;
 import org.apache.nifi.minifi.properties.BootstrapPropertiesLoader;
 import org.apache.nifi.minifi.properties.ProtectedBootstrapProperties;
@@ -40,54 +53,76 @@
 import picocli.CommandLine.Option;
 
 /**
- * Shared Encrypt Configuration for NiFi and NiFi Registry
+ * Encrypt Configuration for MiNiFi
  */
 @Command(
-        name = "encrypt-config",
-        sortOptions = false,
-        mixinStandardHelpOptions = true,
-        usageHelpWidth = 160,
-        separator = " ",
-        version = {
-                "Java ${java.version} (${java.vendor} ${java.vm.name} ${java.vm.version})"
-        },
-        descriptionHeading = "Description: ",
-        description = {
-                "encrypt-config supports protection of sensitive values in Apache MiNiFi"
-        }
+    name = "encrypt-config",
+    sortOptions = false,
+    mixinStandardHelpOptions = true,
+    usageHelpWidth = 160,
+    separator = " ",
+    version = {
+        "Java ${java.version} (${java.vendor} ${java.vm.name} ${java.vm.version})"
+    },
+    descriptionHeading = "Description: ",
+    description = {
+        "encrypt-config supports protection of sensitive values in Apache MiNiFi"
+    }
 )
-public class MiNiFiEncryptConfig implements Runnable{
+public class MiNiFiEncryptConfig implements Runnable {
 
     static final String BOOTSTRAP_ROOT_KEY_PROPERTY = "minifi.bootstrap.sensitive.key";
 
     private static final String WORKING_FILE_NAME_FORMAT = "%s.%d.working";
     private static final int KEY_LENGTH = 32;
 
     @Option(
-            names = {"-b", "--bootstrapConf"},
-            description = "Path to file containing Bootstrap Configuration [bootstrap.conf] for optional root key and property protection scheme settings"
+        names = {"-b", "--bootstrapConf"},
+        description = "Path to file containing Bootstrap Configuration [bootstrap.conf] for optional root key and property protection scheme settings"
     )
     Path bootstrapConfPath;
 
     @Option(
-            names = {"-B", "--outputBootstrapConf"},
-            description = "Path to output file for Bootstrap Configuration [bootstrap.conf] with root key configured"
+        names = {"-B", "--outputBootstrapConf"},
+        description = "Path to output file for Bootstrap Configuration [bootstrap.conf] with root key configured"
     )
     Path outputBootstrapConf;
 
+    @Option(
+        names = {"-x", "--encryptRawFlowJsonOnly"},
+        description = "Process Raw Flow Configuration [flow.json.raw] sensitive property values without modifying other configuration files"
+    )
+    boolean flowConfigurationRequested;
+
+    @Option(
+        names = {"-f", "--rawFlowJson"},
+        description = "Path to file containing Raw Flow Configuration [flow.json.raw] that will be updated unless the output argument is provided"
+    )
+    Path flowConfigurationPath;
+
+    @Option(
+        names = {"-g", "--outputRawFlowJson"},
+        description = "Path to output file for Raw Flow Configuration [flow.json.raw] with property protection applied"
+    )
+    Path outputFlowConfigurationPath;
+
     protected final Logger logger = LoggerFactory.getLogger(getClass());
 
     @Override
     public void run() {
-        processBootstrapConf();
+        BootstrapProperties unprotectedProperties = BootstrapPropertiesLoader.load(bootstrapConfPath.toFile());
+        processBootstrapConf(unprotectedProperties);
+        processFlowConfiguration(unprotectedProperties);
     }
 
     /**
      * Process bootstrap.conf writing new Root Key to specified Root Key Property when bootstrap.conf is specified
-     *
      */
-    protected void processBootstrapConf() {
-        BootstrapProperties unprotectedProperties = BootstrapPropertiesLoader.load(bootstrapConfPath.toFile());
+    protected void processBootstrapConf(BootstrapProperties unprotectedProperties) {
+        if (flowConfigurationRequested) {
+            logger.info("Bootstrap Configuration [bootstrap.conf] not modified based on provided arguments");
+            return;
+        }
 
         logger.info("Started processing Bootstrap Configuration [{}]", bootstrapConfPath);
 
@@ -98,7 +133,7 @@ protected void processBootstrapConf() {
         runFileTransformer(fileTransformer2, bootstrapConfPath, outputBootstrapConf);
 
         FileTransformer fileTransformer = new BootstrapConfigurationFileTransformer(BOOTSTRAP_ROOT_KEY_PROPERTY, newRootKey);
-        runFileTransformer(fileTransformer, Optional.ofNullable(outputBootstrapConf).orElse(bootstrapConfPath), outputBootstrapConf);
+        runFileTransformer(fileTransformer, ofNullable(outputBootstrapConf).orElse(bootstrapConfPath), outputBootstrapConf);
         logger.info("Completed processing Bootstrap Configuration [{}]", bootstrapConfPath);
     }
 
@@ -113,8 +148,8 @@ private String getRootKey() {
      * Run File Transformer using working path based on output path
      *
      * @param fileTransformer File Transformer to be invoked
-     * @param inputPath Input path of file to be transformed
-     * @param outputPath Output path for transformed file that defaults to the input path when not specified
+     * @param inputPath       Input path of file to be transformed
+     * @param outputPath      Output path for transformed file that defaults to the input path when not specified
      */
     protected void runFileTransformer(FileTransformer fileTransformer, Path inputPath, Path outputPath) {
         Path configuredOutputPath = outputPath == null ? inputPath : outputPath;
@@ -143,4 +178,45 @@ private Path getWorkingPath(Path resourcePath) {
         String workingFileName = String.format(WORKING_FILE_NAME_FORMAT, fileName, System.currentTimeMillis());
         return resourcePath.resolveSibling(workingFileName);
     }
+
+    private void processFlowConfiguration(BootstrapProperties unprotectedProperties) {
+        if (flowConfigurationPath == null) {
+            logger.info("Flow Configuration not specified");
+            return;
+        }
+        String sensitivePropertiesKey = unprotectedProperties.getProperty(NIFI_MINIFI_SENSITIVE_PROPS_KEY.getKey());
+        String sensitivePropertiesAlgorithm = unprotectedProperties.getProperty(NIFI_MINIFI_SENSITIVE_PROPS_ALGORITHM.getKey());
+        if (isAnyBlank(sensitivePropertiesKey, sensitivePropertiesAlgorithm)) {
+            logger.info("Sensitive Properties Key or Sensitive Properties Algorithm is not provided");
+            return;
+        }
+
+        logger.info("Started processing Flow Configuration [{}]", flowConfigurationPath);
+
+        byte[] flowAsBytes;
+        try {
+            flowAsBytes = readAllBytes(flowConfigurationPath);
+        } catch (IOException e) {
+            logger.error("Unable to load Flow Configuration [{}]", flowConfigurationPath);
+            return;
+        }
+
+        FlowSerDeService flowSerDeService = StandardFlowSerDeService.defaultInstance();
+        FlowPropertyEncryptor flowPropertyEncryptor = new StandardFlowPropertyEncryptor(
+            new PropertyEncryptorBuilder(sensitivePropertiesKey).setAlgorithm(sensitivePropertiesAlgorithm).build(), null);
+
+        VersionedDataflow flow = flowSerDeService.deserialize(flowAsBytes);
+        VersionedDataflow encryptedFlow = flowPropertyEncryptor.encryptSensitiveProperties(flow);
+        byte[] encryptedFlowAsBytes = flowSerDeService.serialize(encryptedFlow);
+
+        Path targetPath = ofNullable(outputFlowConfigurationPath).orElse(flowConfigurationPath);
+        try {
+            write(targetPath, encryptedFlowAsBytes);
+        } catch (IOException e) {
+            logger.error("Unable to write Flow Configuration [{}]", targetPath);
+            return;
+        }
+
+        logger.info("Completed processing Flow Configuration [{}]", flowConfigurationPath);
+    }
 }

nifi-assembly/NOTICE

@@ -2040,11 +2040,6 @@ The following binary components are provided under the Apache Software License v
            the terms of a BSD style license.
            The original software and related information is available
            at http://www.jcraft.com/jsch/.
-
-    (ASLv2) DataStax Java Driver for Apache Cassandra - Core
-        The following NOTICE information applies:
-          DataStax Java Driver for Apache Cassandra - Core
-          Copyright (C) 2012-2017 DataStax Inc.
     (ASLv2) bytebuffer-collections
         The following NOTICE information applies:
           bytebuffer-collections

nifi-assembly/pom.xml

@@ -530,24 +530,6 @@ language governing permissions and limitations under the License. -->
             <version>2.0.0-SNAPSHOT</version>
             <type>nar</type>
         </dependency>
-        <dependency>
-            <groupId>org.apache.nifi</groupId>
-            <artifactId>nifi-cassandra-nar</artifactId>
-            <version>2.0.0-SNAPSHOT</version>
-            <type>nar</type>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.nifi</groupId>
-            <artifactId>nifi-cassandra-services-api-nar</artifactId>
-            <version>2.0.0-SNAPSHOT</version>
-            <type>nar</type>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.nifi</groupId>
-            <artifactId>nifi-cassandra-services-nar</artifactId>
-            <version>2.0.0-SNAPSHOT</version>
-            <type>nar</type>
-        </dependency>
         <dependency>
             <groupId>org.apache.nifi</groupId>
             <artifactId>nifi-registry-nar</artifactId>

nifi-code-coverage/pom.xml

@@ -31,6 +31,7 @@
         <calcite.avatica.version>1.6.0</calcite.avatica.version>
         <avatica.version>1.24.0</avatica.version>
         <org.apache.sshd.version>2.12.0</org.apache.sshd.version>
+        <mime4j.version>0.8.11</mime4j.version>
     </properties>
 
     <!-- Managed Dependency Versions for referenced modules required based on different parent bundle project -->
@@ -125,6 +126,12 @@
                 <artifactId>sshd-osgi</artifactId>
                 <version>${org.apache.sshd.version}</version>
             </dependency>
+            <!-- MIME4J from Tika Parsers in media-processors -->
+            <dependency>
+                <groupId>org.apache.james</groupId>
+                <artifactId>apache-mime4j-core</artifactId>
+                <version>${mime4j.version}</version>
+            </dependency>
         </dependencies>
     </dependencyManagement>
 
@@ -838,26 +845,6 @@
             <artifactId>nifi-box-services-api</artifactId>
             <version>2.0.0-SNAPSHOT</version>
         </dependency>
-        <dependency>
-            <groupId>org.apache.nifi</groupId>
-            <artifactId>nifi-cassandra-distributedmapcache-service</artifactId>
-            <version>2.0.0-SNAPSHOT</version>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.nifi</groupId>
-            <artifactId>nifi-cassandra-processors</artifactId>
-            <version>2.0.0-SNAPSHOT</version>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.nifi</groupId>
-            <artifactId>nifi-cassandra-services</artifactId>
-            <version>2.0.0-SNAPSHOT</version>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.nifi</groupId>
-            <artifactId>nifi-cassandra-services-api</artifactId>
-            <version>2.0.0-SNAPSHOT</version>
-        </dependency>
         <dependency>
             <groupId>org.apache.nifi</groupId>
             <artifactId>nifi-cdc-api</artifactId>

nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/CompiledExpression.java

@@ -44,6 +44,7 @@ public Tree getTree() {
         return tree;
     }
 
+    @Override
     public String getExpression() {
         return expression;
     }

nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/EmptyPreparedQuery.java

@@ -20,6 +20,7 @@
 import org.apache.nifi.processor.exception.ProcessException;
 
 import java.util.Collections;
+import java.util.List;
 import java.util.Set;
 
 public class EmptyPreparedQuery implements PreparedQuery {
@@ -49,4 +50,9 @@ public VariableImpact getVariableImpact() {
     public Set<String> getExplicitlyReferencedAttributes() {
         return Collections.emptySet();
     }
+
+    @Override
+    public List<Expression> getExpressions() {
+        return List.of();
+    }
 }

nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/Expression.java

@@ -28,4 +28,9 @@ public interface Expression {
      * @return the evaluated value
      */
     String evaluate(EvaluationContext evaluationContext, AttributeValueDecorator decorator);
+
+    /**
+     * @return the expression as a String
+     */
+    String getExpression();
 }