Skip to content

[Backport 2.19-dev] Support spath with dynamic fields (#5058) #5073

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
LantaoJin merged 2 commits into from
Jan 27, 2026
Merged

[Backport 2.19-dev] Support spath with dynamic fields (#5058) #5073

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
5f67a8a
Support spath with dynamic fields (#5058)
ykmr1224 Jan 26, 2026
6eb8623
Fix for Java 11
ykmr1224 Jan 26, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 1 addition & 4 deletions common/src/main/java/org/opensearch/sql/common/utils/DebugUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,6 @@
import java.util.Collection;
import java.util.Map;
import java.util.stream.Collectors;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;

/**
* Utility class for debugging operations. This class is only for debugging purpose, and not
Expand All @@ -18,7 +16,6 @@
public class DebugUtils {
// Update this to true while you are debugging. (Safe guard to avoid usage in production code. )
private static final boolean IS_DEBUG = false;
private static final Logger logger = LogManager.getLogger(DebugUtils.class);

public static <T> T debug(T obj, String message) {
verifyDebug();
Expand All @@ -39,7 +36,7 @@ private static void verifyDebug() {
}

private static void print(String format, Object... args) {
logger.info(String.format(format, args));
System.out.println(String.format(format, args));
}

private static String getCalledFrom(int pos) {
Expand Down
29 changes: 21 additions & 8 deletions core/src/main/java/org/opensearch/sql/ast/analysis/FieldResolutionResult.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,17 +28,17 @@ public class FieldResolutionResult {
@NonNull private final Set<String> regularFields;
@NonNull private final Wildcard wildcard;

public FieldResolutionResult(Set<String> regularFields) {
public FieldResolutionResult(Collection<String> regularFields) {
this.regularFields = new HashSet<>(regularFields);
this.wildcard = NULL_WILDCARD;
}

public FieldResolutionResult(Set<String> regularFields, Wildcard wildcard) {
public FieldResolutionResult(Collection<String> regularFields, Wildcard wildcard) {
this.regularFields = new HashSet<>(regularFields);
this.wildcard = wildcard;
}

public FieldResolutionResult(Set<String> regularFields, String wildcard) {
public FieldResolutionResult(Collection<String> regularFields, String wildcard) {
this.regularFields = new HashSet<>(regularFields);
this.wildcard = getWildcard(wildcard);
}
Expand All @@ -53,12 +53,12 @@ private static Wildcard getWildcard(String wildcard) {
}
}

public FieldResolutionResult(Set<String> regularFields, Set<String> wildcards) {
public FieldResolutionResult(Collection<String> regularFields, Collection<String> wildcards) {
this.regularFields = new HashSet<>(regularFields);
this.wildcard = createOrWildcard(wildcards);
}

private static Wildcard createOrWildcard(Set<String> patterns) {
private static Wildcard createOrWildcard(Collection<String> patterns) {
if (patterns == null || patterns.isEmpty()) {
return NULL_WILDCARD;
}
Expand All @@ -70,38 +70,50 @@ private static Wildcard createOrWildcard(Set<String> patterns) {
return new OrWildcard(wildcards);
}

/** Returns unmodifiable view of regular fields. */
public Set<String> getRegularFieldsUnmodifiable() {
return Collections.unmodifiableSet(regularFields);
}

/** Checks if result contains any wildcard patterns. */
public boolean hasWildcards() {
return wildcard != NULL_WILDCARD;
}

/** Checks if result contains partial wildcard patterns (not '*'). */
public boolean hasPartialWildcards() {
return wildcard != NULL_WILDCARD && wildcard != ANY_WILDCARD;
}

/** Checks if result contains regular fields. */
public boolean hasRegularFields() {
return !regularFields.isEmpty();
}

/** Creates new result excluding specified fields. */
public FieldResolutionResult exclude(Collection<String> fields) {
Set<String> combinedFields = new HashSet<>(this.regularFields);
combinedFields.removeAll(fields);
return new FieldResolutionResult(combinedFields, this.wildcard);
}

public FieldResolutionResult or(Set<String> fields) {
/** Creates new result combining this result with additional fields (union). */
public FieldResolutionResult or(Collection<String> fields) {
Set<String> combinedFields = new HashSet<>(this.regularFields);
combinedFields.addAll(fields);
return new FieldResolutionResult(combinedFields, this.wildcard);
}

private Set<String> and(Set<String> fields) {
private Set<String> and(Collection<String> fields) {
return fields.stream()
.filter(field -> this.getRegularFields().contains(field) || this.wildcard.matches(field))
.collect(Collectors.toSet());
}

/** Creates new result intersecting this result with another (intersection). */
public FieldResolutionResult and(FieldResolutionResult other) {
Set<String> combinedFields = this.and(other.regularFields);
Set<String> combinedFields = new HashSet<>();
combinedFields.addAll(this.and(other.regularFields));
combinedFields.addAll(other.and(this.regularFields));

Wildcard combinedWildcard = this.wildcard.and(other.wildcard);
Expand All @@ -111,6 +123,7 @@ public FieldResolutionResult and(FieldResolutionResult other) {

/** Interface for wildcard pattern matching. */
public interface Wildcard {
/** Checks if field name matches wildcard pattern. */
boolean matches(String fieldName);

default Wildcard and(Wildcard other) {
Expand Down
54 changes: 37 additions & 17 deletions core/src/main/java/org/opensearch/sql/ast/analysis/FieldResolutionVisitor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,8 @@
*/
public class FieldResolutionVisitor extends AbstractNodeVisitor<Node, FieldResolutionContext> {

private static final String ALL_FIELDS = "*";

/**
* Analyzes PPL query plan to determine required fields at each node.
*
Expand Down Expand Up @@ -110,10 +112,10 @@ private void acceptAndVerifyNodeVisited(Node node, FieldResolutionContext contex

@Override
public Node visitProject(Project node, FieldResolutionContext context) {
boolean isSelectAll =
node.getProjectList().stream().anyMatch(expr -> expr instanceof AllFields);
boolean isSingleSelectAll =
node.getProjectList().size() == 1 && node.getProjectList().get(0) instanceof AllFields;

if (isSelectAll) {
if (isSingleSelectAll) {
visitChildren(node, context);
} else {
Set<String> projectFields = new HashSet<>();
Expand Down Expand Up @@ -179,15 +181,14 @@ public Node visitSpath(SPath node, FieldResolutionContext context) {
return visitEval(node.rewriteAsEval(), context);
} else {
// set requirements for spath command;
context.setResult(node, context.getCurrentRequirements());
FieldResolutionResult requirements = context.getCurrentRequirements();
if (requirements.hasWildcards()) {
context.setResult(node, requirements);
if (requirements.hasPartialWildcards()) {
throw new IllegalArgumentException(
"Spath command cannot extract arbitrary fields. Please project fields explicitly by"
+ " fields command without wildcard or stats command.");
"Spath command cannot be used with partial wildcard such as `prefix*`.");
}

context.pushRequirements(context.getCurrentRequirements().or(Set.of(node.getInField())));
context.pushRequirements(requirements.or(Set.of(node.getInField())));
visitChildren(node, context);
context.popRequirements();
return node;
Expand Down Expand Up @@ -238,6 +239,8 @@ private Set<String> extractFieldsFromExpression(UnresolvedExpression expr) {
if (expr instanceof Field) {
Field field = (Field) expr;
fields.add(field.getField().toString());
} else if (expr instanceof AllFields) {
fields.add(ALL_FIELDS);
} else if (expr instanceof QualifiedName) {
QualifiedName name = (QualifiedName) expr;
fields.add(name.toString());
Expand Down Expand Up @@ -496,43 +499,56 @@ public Node visitStreamWindow(StreamWindow node, FieldResolutionContext context)

@Override
public Node visitFillNull(FillNull node, FieldResolutionContext context) {
if (node.isAgainstAllFields()) {
throw new IllegalArgumentException("Fields need to be specified with fillnull command");
}
Set<String> fields = new HashSet<>();
node.getFields().forEach(field -> fields.addAll(extractFieldsFromExpression(field)));

context.pushRequirements(context.getCurrentRequirements().or(fields));
visitChildren(node, context);
context.popRequirements();
return node;
}

@Override
public Node visitAppendCol(AppendCol node, FieldResolutionContext context) {
visitChildren(node, context);
return node;
throw new IllegalArgumentException(
"AppendCol command cannot be used together with spath command");
}

@Override
public Node visitAppend(Append node, FieldResolutionContext context) {
// dispatch requirements to subsearch and main
acceptAndVerifyNodeVisited(node.getSubSearch(), context);
visitChildren(node, context);
return node;
}

@Override
public Node visitMultisearch(Multisearch node, FieldResolutionContext context) {
visitChildren(node, context);
return node;
throw new IllegalArgumentException(
"Multisearch command cannot be used together with spath command");
}

@Override
public Node visitLookup(Lookup node, FieldResolutionContext context) {
visitChildren(node, context);
return node;
throw new IllegalArgumentException("Lookup command cannot be used together with spath command");
}

@Override
public Node visitValues(Values node, FieldResolutionContext context) {
visitChildren(node, context);
return node;
throw new IllegalArgumentException("Values command cannot be used together with spath command");
}

@Override
public Node visitReplace(Replace node, FieldResolutionContext context) {
Set<String> fields = new HashSet<>();
node.getFieldList().forEach(field -> fields.addAll(extractFieldsFromExpression(field)));

context.pushRequirements(context.getCurrentRequirements().or(fields));
visitChildren(node, context);
context.popRequirements();
return node;
}

Expand Down Expand Up @@ -627,6 +643,10 @@ private Set<String> extractFieldsFromAggregation(UnresolvedExpression expr) {
}
}
}
return fields;
return excludeAllFieldsWildcard(fields);
}

private Set<String> excludeAllFieldsWildcard(Set<String> fields) {
return fields.stream().filter(f -> !f.equals(ALL_FIELDS)).collect(Collectors.toSet());
}
}
4 changes: 4 additions & 0 deletions core/src/main/java/org/opensearch/sql/ast/tree/FillNull.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,10 @@ public List<Field> getFields() {
return getReplacementPairs().stream().map(Pair::getLeft).collect(Collectors.toList());
}

public boolean isAgainstAllFields() {
return !replacementForAll.isEmpty() && getReplacementPairs().isEmpty();
}

@Override
public FillNull attach(UnresolvedPlan child) {
this.child = child;
Expand Down
9 changes: 9 additions & 0 deletions core/src/main/java/org/opensearch/sql/ast/tree/Join.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
import org.opensearch.sql.ast.AbstractNodeVisitor;
import org.opensearch.sql.ast.expression.Argument;
import org.opensearch.sql.ast.expression.Field;
import org.opensearch.sql.ast.expression.Literal;
import org.opensearch.sql.ast.expression.UnresolvedExpression;

@ToString
Expand Down Expand Up @@ -90,6 +91,14 @@ public <T, C> T accept(AbstractNodeVisitor<T, C> nodeVisitor, C context) {
return nodeVisitor.visitJoin(this, context);
}

/**
* @return `overwrite` option value in argumentMap
*/
public boolean isOverwrite() {
return getArgumentMap().get("overwrite") == null // 'overwrite' default value is true
|| getArgumentMap().get("overwrite").equals(Literal.TRUE);
}

public enum JoinType {
INNER,
LEFT,
Expand Down
Loading
Loading