Skip to content
This repository has been archived by the owner on Dec 13, 2023. It is now read-only.

feature support nested jsonpath expressions #3764

Merged
merged 2 commits into from
Sep 16, 2023
Merged

feature support nested jsonpath expressions #3764

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
1480ac3
feature support nested jsonpath expressions
Sep 6, 2023
449241f
Merge branch 'main' into feature/nestedJsonpathExpressions
ryanmurf Sep 12, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
124 changes: 80 additions & 44 deletions core/src/main/java/com/netflix/conductor/core/utils/ParametersUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,8 @@
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Objects;
import java.util.regex.Pattern;

import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
Expand All @@ -44,6 +46,9 @@
public class ParametersUtils {

private static final Logger LOGGER = LoggerFactory.getLogger(ParametersUtils.class);
private static final Pattern PATTERN =
Pattern.compile(
"(?=(?<!\\$)\\$\\{)(?:(?=.*?\\{(?!.*?\\1)(.*\\}(?!.*\\2).*))(?=.*?\\}(?!.*?\\2)(.*)).)+?.*?(?=\\1)[^{]*(?=\\2$)");

private final ObjectMapper objectMapper;
private final TypeReference<Map<String, Object>> map = new TypeReference<>() {};
Expand Down Expand Up @@ -221,56 +226,58 @@ private Object replaceList(List<?> values, String taskId, DocumentContext io) {

private Object replaceVariables(
String paramString, DocumentContext documentContext, String taskId) {
String[] values = paramString.split("(?=(?<!\\$)\\$\\{)|(?<=})");
Object[] convertedValues = new Object[values.length];
for (int i = 0; i < values.length; i++) {
convertedValues[i] = values[i];
if (values[i].startsWith("${") && values[i].endsWith("}")) {
String paramPath = values[i].substring(2, values[i].length() - 1);
// if the paramPath is blank, meaning no value in between ${ and }
// like ${}, ${ } etc, set the value to empty string
if (StringUtils.isBlank(paramPath)) {
convertedValues[i] = "";
continue;
}
if (EnvUtils.isEnvironmentVariable(paramPath)) {
String sysValue = EnvUtils.getSystemParametersValue(paramPath, taskId);
if (sysValue != null) {
convertedValues[i] = sysValue;
}
return replaceVariables(paramString, documentContext, taskId, 0);
}

} else {
try {
convertedValues[i] = documentContext.read(paramPath);
} catch (Exception e) {
LOGGER.warn(
"Error reading documentContext for paramPath: {}. Exception: {}",
paramPath,
e);
convertedValues[i] = null;
}
}
} else if (values[i].contains("$${")) {
convertedValues[i] = values[i].replaceAll("\\$\\$\\{", "\\${");
private Object replaceVariables(
String paramString, DocumentContext documentContext, String taskId, int depth) {
var matcher = PATTERN.matcher(paramString);
var replacements = new LinkedList<Replacement>();
while (matcher.find()) {
var start = matcher.start();
var end = matcher.end();
var match = paramString.substring(start, end);
String paramPath = match.substring(2, match.length() - 1);
paramPath = replaceVariables(paramPath, documentContext, taskId, depth + 1).toString();
// if the paramPath is blank, meaning no value in between ${ and }
// like ${}, ${ } etc, set the value to empty string
if (StringUtils.isBlank(paramPath)) {
replacements.add(new Replacement("", start, end));
continue;
}
}

Object retObj = convertedValues[0];
// If the parameter String was "v1 v2 v3" then make sure to stitch it back
if (convertedValues.length > 1) {
for (int i = 0; i < convertedValues.length; i++) {
Object val = convertedValues[i];
if (val == null) {
val = "";
if (EnvUtils.isEnvironmentVariable(paramPath)) {
String sysValue = EnvUtils.getSystemParametersValue(paramPath, taskId);
if (sysValue != null) {
replacements.add(new Replacement(sysValue, start, end));
}
if (i == 0) {
retObj = val;
} else {
retObj = retObj + "" + val.toString();
} else {
try {
replacements.add(new Replacement(documentContext.read(paramPath), start, end));
} catch (Exception e) {
LOGGER.warn(
"Error reading documentContext for paramPath: {}. Exception: {}",
paramPath,
e);
replacements.add(new Replacement(null, start, end));
}
}
}
return retObj;
if (replacements.size() == 1
&& replacements.getFirst().getStartIndex() == 0
&& replacements.getFirst().getEndIndex() == paramString.length()
&& depth == 0) {
return replacements.get(0).getReplacement();
}
Collections.sort(replacements);
var builder = new StringBuilder(paramString);
for (int i = replacements.size() - 1; i >= 0; i--) {
var replacement = replacements.get(i);
builder.replace(
replacement.getStartIndex(),
replacement.getEndIndex(),
Objects.toString(replacement.getReplacement()));
}
return builder.toString().replaceAll("\\$\\$\\{", "\\${");
}

@Deprecated
Expand Down Expand Up @@ -321,4 +328,33 @@ public Map<String, Object> getWorkflowInput(
}
return inputParams;
}

private static class Replacement implements Comparable<Replacement> {
private final int startIndex;
private final int endIndex;
private final Object replacement;

public Replacement(Object replacement, int startIndex, int endIndex) {
this.replacement = replacement;
this.startIndex = startIndex;
this.endIndex = endIndex;
}

public Object getReplacement() {
return replacement;
}

public int getStartIndex() {
return startIndex;
}

public int getEndIndex() {
return endIndex;
}

@Override
public int compareTo(Replacement o) {
return Long.compare(startIndex, o.startIndex);
}
}
}
24 changes: 24 additions & 0 deletions core/src/test/java/com/netflix/conductor/core/utils/ParametersUtilsTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -233,6 +233,30 @@ public void testReplaceInputWithMapAndList() throws Exception {
assertEquals("${version}", inputList.get(1));
}

@Test
public void testNestedPathExpressions() throws Exception {
Map<String, Object> map = new HashMap<>();
map.put("name", "conductor");
map.put("index", 1);
map.put("mapValue", "a");
map.put("recordIds", List.of(1, 2, 3));
map.put("map", Map.of("a", List.of(1, 2, 3), "b", List.of(2, 4, 5), "c", List.of(3, 7, 8)));

Map<String, Object> input = new HashMap<>();
input.put("k1", "${recordIds[${index}]}");
input.put("k2", "${map.${mapValue}[${index}]}");
input.put("k3", "${map.b[${map.${mapValue}[${index}]}]}");

Object jsonObj = objectMapper.readValue(objectMapper.writeValueAsString(map), Object.class);

Map<String, Object> replaced = parametersUtils.replace(input, jsonObj);
assertNotNull(replaced);

assertEquals(2, replaced.get("k1"));
assertEquals(2, replaced.get("k2"));
assertEquals(5, replaced.get("k3"));
}

@Test
public void testReplaceWithEscapedTags() throws Exception {
Map<String, Object> map = new HashMap<>();
Expand Down