Merge pull request #4721 from lcpopa/master

OLS - Add path from vertex properties on DataFile nodes
odpi · Feb 23, 2021 · cb981db · cb981db
2 parents 591899a + c3dc327
commit cb981db
Show file tree

Hide file tree

Showing 2 changed files with 27 additions and 1 deletion.
diff --git a/...monconnectors/openlineageconnectors/janusconnector/graph/LineageGraphConnectorHelper.java b/...monconnectors/openlineageconnectors/janusconnector/graph/LineageGraphConnectorHelper.java
@@ -79,10 +79,11 @@
 import static org.odpi.openmetadata.openconnectors.governancedaemonconnectors.openlineageconnectors.janusconnector.utils.GraphConstants.NODE_LABEL_CONDENSED;
 import static org.odpi.openmetadata.openconnectors.governancedaemonconnectors.openlineageconnectors.janusconnector.utils.GraphConstants.NODE_LABEL_SUB_PROCESS;
 import static org.odpi.openmetadata.openconnectors.governancedaemonconnectors.openlineageconnectors.janusconnector.utils.GraphConstants.PROPERTY_KEY_ADDITIONAL_PROPERTIES;
-import static org.odpi.openmetadata.openconnectors.governancedaemonconnectors.openlineageconnectors.janusconnector.utils.GraphConstants.PROPERTY_KEY_EXTENDED_PROPERTIES;
 import static org.odpi.openmetadata.openconnectors.governancedaemonconnectors.openlineageconnectors.janusconnector.utils.GraphConstants.PROPERTY_KEY_DISPLAY_NAME;
 import static org.odpi.openmetadata.openconnectors.governancedaemonconnectors.openlineageconnectors.janusconnector.utils.GraphConstants.PROPERTY_KEY_ENTITY_GUID;
+import static org.odpi.openmetadata.openconnectors.governancedaemonconnectors.openlineageconnectors.janusconnector.utils.GraphConstants.PROPERTY_KEY_EXTENDED_PROPERTIES;
 import static org.odpi.openmetadata.openconnectors.governancedaemonconnectors.openlineageconnectors.janusconnector.utils.GraphConstants.PROPERTY_KEY_INSTANCEPROP_DISPLAY_NAME;
+import static org.odpi.openmetadata.openconnectors.governancedaemonconnectors.openlineageconnectors.janusconnector.utils.GraphConstants.PROPERTY_KEY_INSTANCE_PROP_ADDITIONAL_PROPERTIES;
 import static org.odpi.openmetadata.openconnectors.governancedaemonconnectors.openlineageconnectors.janusconnector.utils.GraphConstants.PROPERTY_KEY_LABEL;
 import static org.odpi.openmetadata.openconnectors.governancedaemonconnectors.openlineageconnectors.janusconnector.utils.GraphConstants.PROPERTY_KEY_PREFIX_ELEMENT;
 import static org.odpi.openmetadata.openconnectors.governancedaemonconnectors.openlineageconnectors.janusconnector.utils.GraphConstants.PROPERTY_KEY_PREFIX_VERTEX_INSTANCE_PROPERTY;
@@ -847,6 +848,15 @@ private Map<String, String> getRelationalTableProperties(GraphTraversalSource g,
  }
 
  private Map<String, String> getDataFileProperties(GraphTraversalSource g, Object vertexId) {
+ Map<String, String> properties = extractPropertiesFromNeighborhood(g, vertexId);
+ if(!properties.containsKey(FILE_FOLDER_KEY)){
+ Optional<String> path = extractPathFromVertexProperties(g, vertexId);
+ path.ifPresent(s -> properties.put(FILE_FOLDER_KEY, "/" + s.trim()));
+ }
+ return properties;
+ }
+
+ private Map<String, String> extractPropertiesFromNeighborhood(GraphTraversalSource g, Object vertexId) {
  Map<String, String> properties = new HashMap<>();
 
  List<Vertex> folderVertices = getFolderVertices(g, vertexId);
@@ -863,7 +873,21 @@ private Map<String, String> getDataFileProperties(GraphTraversalSource g, Object
  }
 
  return properties;
+ }
+
+ private Optional<String> extractPathFromVertexProperties(GraphTraversalSource g, Object vertexId) {
+
+ VertexProperty<String> additionalProperties =
+ g.V(vertexId).next().property(PROPERTY_KEY_INSTANCE_PROP_ADDITIONAL_PROPERTIES);
+ if(!additionalProperties.isPresent()){
+ return Optional.empty();
+ }
 
+ String additionalPropertiesValue = additionalProperties.value();
+ return Arrays.stream(additionalPropertiesValue.split(","))
+ .filter(s -> s.trim().startsWith("path"))
+ .map(s -> s.split(":")[1])
+ .findFirst();
  }
 
  private Map<String, String> getProcessProperties(GraphTraversalSource g, Object vertexId) {

diff --git a/...governancedaemonconnectors/openlineageconnectors/janusconnector/utils/GraphConstants.java b/...governancedaemonconnectors/openlineageconnectors/janusconnector/utils/GraphConstants.java
@@ -114,6 +114,8 @@ private GraphConstants() {
  public static final String PROPERTY_KEY_CONNECTION_NAME = PROPERTY_KEY_PREFIX_ELEMENT + PROPERTY_NAME_CONNECTION;
  public static final String PROPERTY_KEY_PATH = PROPERTY_KEY_PREFIX_ELEMENT + PROPERTY_NAME_PATH;
  public static final String PROPERTY_KEY_PROCESS_LINEAGE_COMPLETED_FLAG = PROPERTY_KEY_PREFIX_VERTEX_INSTANCE_PROPERTY + PROPERTY_NAME_PROCESS_LINEAGE_COMPLETED_FLAG;
+ public static final String PROPERTY_KEY_INSTANCE_PROP_ADDITIONAL_PROPERTIES =
+ PROPERTY_KEY_PREFIX_VERTEX_INSTANCE_PROPERTY + PROPERTY_KEY_ADDITIONAL_PROPERTIES;
 
  public static final String PROPERTY_KEY_RELATIONSHIP_GUID = PROPERTY_KEY_PREFIX_RELATIONSHIP + PROPERTY_NAME_GUID;
  public static final String PROPERTY_KEY_RELATIONSHIP_VERSION = PROPERTY_KEY_PREFIX_RELATIONSHIP + PROPERTY_NAME_VERSION;