NIFI-13872: extend manifest generation to add dependencies for proper…

…ty descriptors in case of python processors (apache#9390)

nifi-framework-bundle/nifi-framework-extensions/nifi-py4j-framework-bundle/nifi-python-extension-api/src/main/python/src/nifiapi/documentation.py

@@ -95,6 +95,27 @@ def getConfigurations(self):
     def __str__(self):
         return f"MultiProcessorUseCaseDetails[description={self.description}]"
 
+class PropertyDependency:
+    class Java:
+            implements = ['org.apache.nifi.python.processor.documentation.PropertyDependency']
+
+    def __init__(self,
+                      name: str,
+                      display_name: str,
+                      dependent_values: list[str]):
+             self.name = name
+             self.display_name = display_name
+             self.dependent_values = dependent_values
+
+    def getName(self):
+        return self.name
+
+    def getDisplayName(self):
+        return self.display_name
+
+    def getDependentValues(self):
+        return ArrayList(self.dependent_values)
+
 class PropertyDescription:
     class Java:
         implements = ['org.apache.nifi.python.processor.documentation.PropertyDescription']
@@ -108,7 +129,8 @@ def __init__(self,
                  default_value: str = None,
                  expression_language_scope: str = None,
                  controller_service_definition: str = None,
-                 allowable_values: list[str] = None):
+                 allowable_values: list[str] = None,
+                 dependencies: list[PropertyDependency] = None):
         self.name = name
         self.description = description
         self.display_name = display_name
@@ -118,6 +140,7 @@ def __init__(self,
         self.expression_language_scope = expression_language_scope
         self.controller_service_definition = controller_service_definition
         self.allowable_values = allowable_values if allowable_values is not None else []
+        self.dependencies = dependencies if dependencies is not None else []
 
     def getName(self):
         return self.name
@@ -145,3 +168,6 @@ def getControllerServiceDefinition(self):
 
     def getAllowableValues(self):
         return ArrayList(self.allowable_values)
+
+    def getDependencies(self):
+        return ArrayList(self.dependencies)

nifi-framework-bundle/nifi-framework-extensions/nifi-py4j-framework-bundle/nifi-python-framework-api/src/main/java/org/apache/nifi/python/processor/documentation/PropertyDependency.java

@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.python.processor.documentation;
+
+import java.util.List;
+
+import org.apache.nifi.python.PythonObjectProxy;
+
+public interface PropertyDependency extends PythonObjectProxy {
+    String getName();
+
+    String getDisplayName();
+    List<String> getDependentValues();
+}

nifi-framework-bundle/nifi-framework-extensions/nifi-py4j-framework-bundle/nifi-python-framework-api/src/main/java/org/apache/nifi/python/processor/documentation/PropertyDescription.java

@@ -38,4 +38,6 @@ public interface PropertyDescription extends PythonObjectProxy {
     String getControllerServiceDefinition();
 
     List<String> getAllowableValues();
+
+    List<PropertyDependency> getDependencies();
 }

nifi-framework-bundle/nifi-framework-extensions/nifi-py4j-framework-bundle/nifi-python-framework/src/main/python/framework/ProcessorInspection.py

@@ -18,7 +18,7 @@
 import textwrap
 import os
 import BundleCoordinate
-from nifiapi.documentation import UseCaseDetails, MultiProcessorUseCaseDetails, ProcessorConfiguration, PropertyDescription
+from nifiapi.documentation import UseCaseDetails, MultiProcessorUseCaseDetails, ProcessorConfiguration, PropertyDescription, PropertyDependency
 
 import ExtensionDetails
 
@@ -47,6 +47,71 @@ def visit_Assign(self, node):
                             self.string_assignments.append[variable_name] = string_value
         self.generic_visit(node)
 
+class CollectPropertyDescriptorVisitors(ast.NodeVisitor):
+
+    def __init__(self, module_string_constants, processor_name):
+        self.module_string_constants = module_string_constants
+        self.discovered_property_descriptors = {}
+        self.processor_name = processor_name
+        self.logger = logging.getLogger("python.CollectPropertyDescriptorVisitors")
+
+    def resolve_dependencies(self, node: ast.AST):
+        resolved_dependencies = []
+        for dependency in node.elts:
+            variable_name = dependency.args[0].id
+            if not self.discovered_property_descriptors[variable_name]:
+                self.logger.error(f"Not able to find actual property descriptor for {variable_name}, so not able to resolve property dependencies in {self.processor_name}.")
+            else:
+                actual_property = self.discovered_property_descriptors[variable_name]
+                dependent_values = []
+                for dependent_value in dependency.args[1:]:
+                    dependent_values.append(get_constant_values(dependent_value, self.module_string_constants))
+                resolved_dependencies.append(PropertyDependency(name = actual_property.name,
+                                                                display_name = actual_property.display_name,
+                                                                dependent_values = dependent_values))
+        return resolved_dependencies
+
+    def resolve_property_descriptor_name_in_code(self, node: ast.AST):
+        if isinstance(node.targets[0], ast.Name):
+            return node.targets[0].id
+        elif isinstance(node.targets[0], ast.Attribute):
+            return node.targets[0].attr
+        else:
+            raise Exception("Unable to determine name from source code")
+
+    def visit_Assign(self, node: ast.AST):
+        if self.assignment_is_property_descriton(node):
+            property_descriptor_name_in_code = self.resolve_property_descriptor_name_in_code(node)
+            self.logger.debug(f"Found PropertyDescriptor in the following assignment {property_descriptor_name_in_code}")
+            if not node.value.keywords:
+                self.logger.error(f"Not able to parse {property_descriptor_name_in_code} PropertyDescriptor as no keywords assignments used.")
+            else:
+                descriptor_info = {}
+                for keyword in node.value.keywords:
+                    key = keyword.arg
+                    if key == 'dependencies':
+                        self.logger.debug(f"Resolving dependencies for {property_descriptor_name_in_code}.")
+                        value = self.resolve_dependencies(keyword.value)
+                    else:
+                        value = get_constant_values(keyword.value, self.module_string_constants)
+                    descriptor_info[key] = value
+
+                self.discovered_property_descriptors[property_descriptor_name_in_code] = PropertyDescription(name=descriptor_info.get('name'),
+                                                                                        description=descriptor_info.get('description'),
+                                                                                         display_name=replace_null(descriptor_info.get('display_name'), descriptor_info.get('name')),
+                                                                                         required=replace_null(descriptor_info.get('required'), False),
+                                                                                         sensitive=replace_null(descriptor_info.get('sensitive'), False),
+                                                                                         default_value=descriptor_info.get('default_value'),
+                                                                                         expression_language_scope=replace_null(descriptor_info.get('expression_language_scope'), 'NONE'),
+                                                                                            controller_service_definition=descriptor_info.get('controller_service_definition'),
+                                                                                          allowable_values = descriptor_info.get('allowable_values'),
+                                                                                          dependencies = descriptor_info.get('dependencies'))
+        self.generic_visit(node)
+
+
+    def assignment_is_property_descriton(self, node: ast.AST):
+        return isinstance(node.value, ast.Call) and isinstance(node.value.func, ast.Name) and node.value.func.id == 'PropertyDescriptor'
+
 
 def get_module_string_constants(module_file: str) -> dict:
     with open(module_file) as file:
@@ -219,36 +284,9 @@ def get_processor_configurations(constructor_calls: ast.List) -> list:
 
 
 def get_property_descriptions(class_node, module_string_constants):
-    descriptions = []
-
-    for element in class_node.body:
-        if not isinstance(element, ast.Assign) or not element.value:
-            continue
-        if not isinstance(element.value, ast.Call):
-            continue
-        if element.value.func.id != 'PropertyDescriptor':
-            continue
-        if not element.value.keywords:
-            continue
-
-        descriptor_info = {}
-        for keyword in element.value.keywords:
-            key = keyword.arg
-            value = get_constant_values(keyword.value, module_string_constants)
-            descriptor_info[key] = value
-
-        description = PropertyDescription(name=descriptor_info.get('name'),
-                                        description=descriptor_info.get('description'),
-                                        display_name=replace_null(descriptor_info.get('display_name'), descriptor_info.get('name')),
-                                        required=replace_null(descriptor_info.get('required'), False),
-                                        sensitive=replace_null(descriptor_info.get('sensitive'), False),
-                                        default_value=descriptor_info.get('default_value'),
-                                        expression_language_scope=replace_null(descriptor_info.get('expression_language_scope'), 'NONE'),
-                                        controller_service_definition=descriptor_info.get('controller_service_definition'),
-                                        allowable_values = descriptor_info.get('allowable_values'))
-        descriptions.append(description)
-
-    return descriptions
+    visitor = CollectPropertyDescriptorVisitors(module_string_constants, class_node.name)
+    visitor.visit(class_node)
+    return visitor.discovered_property_descriptors.values()
 
 
 def replace_null(val: any, replacement: any):

nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/manifest/StandardRuntimeManifestService.java

@@ -23,6 +23,8 @@
 import org.apache.nifi.c2.protocol.component.api.RuntimeManifest;
 import org.apache.nifi.extension.manifest.AllowableValue;
 import org.apache.nifi.extension.manifest.ControllerServiceDefinition;
+import org.apache.nifi.extension.manifest.Dependency;
+import org.apache.nifi.extension.manifest.DependentValues;
 import org.apache.nifi.extension.manifest.ExpressionLanguageScope;
 import org.apache.nifi.extension.manifest.Extension;
 import org.apache.nifi.extension.manifest.ExtensionManifest;
@@ -240,9 +242,28 @@ private static Property createManifestProperty(final PropertyDescription propert
         property.setControllerServiceDefinition(getManifestControllerServiceDefinition(propertyDescription.getControllerServiceDefinition()));
         property.setAllowableValues(getAllowableValues(propertyDescription));
 
+        property.setDependencies(getDependencies(propertyDescription));
+
         return property;
     }
 
+    private static List<Dependency> getDependencies(org.apache.nifi.python.processor.documentation.PropertyDescription propertyDescription) {
+        return Optional.ofNullable(propertyDescription.getDependencies()).orElse(List.of())
+                .stream()
+                .map(value -> {
+                    DependentValues dependentValues = new DependentValues();
+                    dependentValues.setValues(value.getDependentValues());
+
+                    Dependency dependency = new Dependency();
+                    dependency.setPropertyName(value.getName());
+                    dependency.setPropertyDisplayName(value.getDisplayName());
+                    dependency.setDependentValues(dependentValues);
+
+                    return dependency;
+                })
+                .toList();
+    }
+
     private static ControllerServiceDefinition getManifestControllerServiceDefinition(final String controllerServiceClassName) {
         if (controllerServiceClassName == null) {
             return null;