Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

OLS - Add path from vertex properties on DataFile nodes #4721

Merged
merged 3 commits into from
Feb 23, 2021
Merged

OLS - Add path from vertex properties on DataFile nodes #4721

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
693dce5
OLS - Add path from vertex properties on DataFile nodes
lcpopa Feb 22, 2021
9fd8c8d
OLS - Add path from vertex properties on DataFile nodes
lcpopa Feb 23, 2021
c3dc327
OLS - Add path from vertex properties on DataFile nodes
lcpopa Feb 23, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 22 additions & 2 deletions ...monconnectors/openlineageconnectors/janusconnector/graph/LineageGraphConnectorHelper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -79,10 +79,11 @@
import static org.odpi.openmetadata.openconnectors.governancedaemonconnectors.openlineageconnectors.janusconnector.utils.GraphConstants.NODE_LABEL_CONDENSED;
import static org.odpi.openmetadata.openconnectors.governancedaemonconnectors.openlineageconnectors.janusconnector.utils.GraphConstants.NODE_LABEL_SUB_PROCESS;
import static org.odpi.openmetadata.openconnectors.governancedaemonconnectors.openlineageconnectors.janusconnector.utils.GraphConstants.PROPERTY_KEY_ADDITIONAL_PROPERTIES;
import static org.odpi.openmetadata.openconnectors.governancedaemonconnectors.openlineageconnectors.janusconnector.utils.GraphConstants.PROPERTY_KEY_EXTENDED_PROPERTIES;
import static org.odpi.openmetadata.openconnectors.governancedaemonconnectors.openlineageconnectors.janusconnector.utils.GraphConstants.PROPERTY_KEY_DISPLAY_NAME;
import static org.odpi.openmetadata.openconnectors.governancedaemonconnectors.openlineageconnectors.janusconnector.utils.GraphConstants.PROPERTY_KEY_ENTITY_GUID;
import static org.odpi.openmetadata.openconnectors.governancedaemonconnectors.openlineageconnectors.janusconnector.utils.GraphConstants.PROPERTY_KEY_EXTENDED_PROPERTIES;
import static org.odpi.openmetadata.openconnectors.governancedaemonconnectors.openlineageconnectors.janusconnector.utils.GraphConstants.PROPERTY_KEY_INSTANCEPROP_DISPLAY_NAME;
import static org.odpi.openmetadata.openconnectors.governancedaemonconnectors.openlineageconnectors.janusconnector.utils.GraphConstants.PROPERTY_KEY_INSTANCE_PROP_ADDITIONAL_PROPERTIES;
import static org.odpi.openmetadata.openconnectors.governancedaemonconnectors.openlineageconnectors.janusconnector.utils.GraphConstants.PROPERTY_KEY_LABEL;
import static org.odpi.openmetadata.openconnectors.governancedaemonconnectors.openlineageconnectors.janusconnector.utils.GraphConstants.PROPERTY_KEY_PREFIX_ELEMENT;
import static org.odpi.openmetadata.openconnectors.governancedaemonconnectors.openlineageconnectors.janusconnector.utils.GraphConstants.PROPERTY_KEY_PREFIX_VERTEX_INSTANCE_PROPERTY;
Expand Down Expand Up @@ -847,6 +848,15 @@ private Map<String, String> getRelationalTableProperties(GraphTraversalSource g,
}

private Map<String, String> getDataFileProperties(GraphTraversalSource g, Object vertexId) {
Map<String, String> properties = extractPropertiesFromNeighborhood(g, vertexId);
if(!properties.containsKey(FILE_FOLDER_KEY)){
String path = extractPathFromVertexProperties(g, vertexId).trim();
properties.put(FILE_FOLDER_KEY, "/" + path);
lcpopa marked this conversation as resolved.
Show resolved Hide resolved
}
return properties;
}

private Map<String, String> extractPropertiesFromNeighborhood(GraphTraversalSource g, Object vertexId) {
Map<String, String> properties = new HashMap<>();

List<Vertex> folderVertices = getFolderVertices(g, vertexId);
Expand All @@ -863,10 +873,20 @@ private Map<String, String> getDataFileProperties(GraphTraversalSource g, Object
}

return properties;
}

private String extractPathFromVertexProperties(GraphTraversalSource g, Object vertexId) {

String additionalProperties = g.V(vertexId).next().property(PROPERTY_KEY_INSTANCE_PROP_ADDITIONAL_PROPERTIES).value().toString();
lcpopa marked this conversation as resolved.
Show resolved Hide resolved
Optional<String> path = Arrays.stream(additionalProperties.split(","))
.filter(s -> s.trim().startsWith("path"))
.map(s -> s.split(":")[1])
.findFirst();

return path.orElse("<unknown>");
lcpopa marked this conversation as resolved.
Show resolved Hide resolved
}

private Map<String, String> getProcessProperties(GraphTraversalSource g, Object vertexId) {
private Map<String, String> getProcessProperties(GraphTraversalSource g, Object vertexId) {
Map<String, String> properties = new HashMap<>();

GraphTraversal<Vertex, ? extends Property<Object>> qualifiedNameTraversal = g.V(vertexId).properties(PROPERTY_NAME_INSTANCEPROP_QUALIFIED_NAME);
Expand Down
2 changes: 2 additions & 0 deletions ...governancedaemonconnectors/openlineageconnectors/janusconnector/utils/GraphConstants.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -114,6 +114,8 @@ private GraphConstants() {
public static final String PROPERTY_KEY_CONNECTION_NAME = PROPERTY_KEY_PREFIX_ELEMENT + PROPERTY_NAME_CONNECTION;
public static final String PROPERTY_KEY_PATH = PROPERTY_KEY_PREFIX_ELEMENT + PROPERTY_NAME_PATH;
public static final String PROPERTY_KEY_PROCESS_LINEAGE_COMPLETED_FLAG = PROPERTY_KEY_PREFIX_VERTEX_INSTANCE_PROPERTY + PROPERTY_NAME_PROCESS_LINEAGE_COMPLETED_FLAG;
public static final String PROPERTY_KEY_INSTANCE_PROP_ADDITIONAL_PROPERTIES =
PROPERTY_KEY_PREFIX_VERTEX_INSTANCE_PROPERTY + PROPERTY_KEY_ADDITIONAL_PROPERTIES;

public static final String PROPERTY_KEY_RELATIONSHIP_GUID = PROPERTY_KEY_PREFIX_RELATIONSHIP + PROPERTY_NAME_GUID;
public static final String PROPERTY_KEY_RELATIONSHIP_VERSION = PROPERTY_KEY_PREFIX_RELATIONSHIP + PROPERTY_NAME_VERSION;
Expand Down