Skip to content

Commit

Permalink
EQL: Fix cidrMatch function fails to match when used in scripts (#56246)
Browse files Browse the repository at this point in the history
Addresses #55709
  • Loading branch information
aleksmaus authored May 14, 2020
1 parent cf40c96 commit 4b47e84
Show file tree
Hide file tree
Showing 15 changed files with 747 additions and 51 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,43 @@ query = '''
file where between(file_path, "dev", ".json", true) == "\\TestLogs\\something"
'''

[[queries]]
expected_event_ids = [75304, 75305]
query = '''
network where cidrMatch(source_address, "10.6.48.157/8") == true
'''

[[queries]]
expected_event_ids = [75304, 75305]
query = '''
network where string(cidrMatch(source_address, "10.6.48.157/8")) == "true"
'''

[[queries]]
expected_event_ids = [75304, 75305]
query = '''
network where true == cidrMatch(source_address, "10.6.48.157/8")
'''

[[queries]]
expected_event_ids = []
query = '''
network where cidrMatch(source_address, "192.168.0.0/16") == true
'''

[[queries]]
expected_event_ids = [75304, 75305]
query = '''
network where cidrMatch(source_address, "192.168.0.0/16", "10.6.48.157/8") == true
'''

[[queries]]
expected_event_ids = [75304, 75305]
query = '''
network where cidrMatch(source_address, "0.0.0.0/0") == true
'''


[[queries]]
description = "test string concatenation. update test to avoid case-sensitivity issues"
query = '''
Expand Down Expand Up @@ -41,7 +78,6 @@ query = 'process where serial_event_id < 5 and concat(process_name, null, null)
expected_event_ids = [1, 2, 3, 4]



[[queries]]
query = 'process where serial_event_id < 5 and concat(parent_process_name, null) == null'
expected_event_ids = [1, 2, 3, 4]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,58 +6,55 @@

package org.elasticsearch.xpack.eql.expression.function.scalar.string;

import org.elasticsearch.common.logging.LoggerMessageFormat;
import org.elasticsearch.xpack.ql.expression.Expression;
import org.elasticsearch.xpack.ql.expression.Expressions;
import org.elasticsearch.xpack.ql.expression.Expressions.ParamOrdinal;
import org.elasticsearch.xpack.ql.expression.function.scalar.BaseSurrogateFunction;
import org.elasticsearch.xpack.ql.expression.FieldAttribute;
import org.elasticsearch.xpack.ql.expression.function.scalar.ScalarFunction;
import org.elasticsearch.xpack.ql.expression.predicate.logical.Or;
import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.Equals;
import org.elasticsearch.xpack.ql.expression.gen.pipeline.Pipe;
import org.elasticsearch.xpack.ql.expression.gen.script.ScriptTemplate;
import org.elasticsearch.xpack.ql.expression.gen.script.Scripts;
import org.elasticsearch.xpack.ql.tree.NodeInfo;
import org.elasticsearch.xpack.ql.tree.Source;
import org.elasticsearch.xpack.ql.type.DataType;
import org.elasticsearch.xpack.ql.type.DataTypes;
import org.elasticsearch.xpack.ql.util.CollectionUtils;

import java.util.ArrayList;
import java.util.LinkedHashSet;
import java.util.List;

import static java.util.Collections.emptyList;
import static java.util.Collections.singletonList;
import static org.elasticsearch.xpack.eql.expression.function.scalar.string.CIDRMatchFunctionProcessor.doProcess;
import static org.elasticsearch.xpack.ql.expression.TypeResolutions.isFoldable;
import static org.elasticsearch.xpack.ql.expression.TypeResolutions.isIPAndExact;
import static org.elasticsearch.xpack.ql.expression.TypeResolutions.isStringAndExact;
import static org.elasticsearch.xpack.ql.expression.gen.script.ParamsBuilder.paramsBuilder;

/**
* EQL specific cidrMatch function
* Returns true if the source address matches any of the provided CIDR blocks.
* Refer to: https://eql.readthedocs.io/en/latest/query-guide/functions.html#cidrMatch
*/
public class CIDRMatch extends BaseSurrogateFunction {
public class CIDRMatch extends ScalarFunction {

private final Expression field;
private final List<Expression> addresses;

public CIDRMatch(Source source, Expression field, List<Expression> addresses) {
super(source, CollectionUtils.combine(singletonList(field), addresses));
super(source, CollectionUtils.combine(singletonList(field), addresses == null ? emptyList() : addresses));
this.field = field;
this.addresses = addresses;
this.addresses = addresses == null ? emptyList() : addresses;
}

@Override
protected NodeInfo<? extends Expression> info() {
return NodeInfo.create(this, CIDRMatch::new, field, addresses);
public Expression field() {
return field;
}

@Override
public Expression replaceChildren(List<Expression> newChildren) {
if (newChildren.size() < 2) {
throw new IllegalArgumentException("expected at least [2] children but received [" + newChildren.size() + "]");
}
return new CIDRMatch(source(), newChildren.get(0), newChildren.subList(1, newChildren.size()));
}

@Override
public DataType dataType() {
return DataTypes.BOOLEAN;
public List<Expression> addresses() {
return addresses;
}

@Override
Expand All @@ -71,15 +68,6 @@ protected TypeResolution resolveType() {
return resolution;
}

for (Expression addr : addresses) {
// Currently we have limited enum for ordinal numbers
// So just using default here for error messaging
resolution = isStringAndExact(addr, sourceText(), ParamOrdinal.DEFAULT);
if (resolution.unresolved()) {
return resolution;
}
}

int index = 1;

for (Expression addr : addresses) {
Expand All @@ -101,14 +89,60 @@ protected TypeResolution resolveType() {
}

@Override
public ScalarFunction makeSubstitute() {
ScalarFunction func = null;

protected Pipe makePipe() {
ArrayList<Pipe> arr = new ArrayList<>(addresses.size());
for (Expression address : addresses) {
final Equals eq = new Equals(source(), field, address);
func = (func == null) ? eq : new Or(source(), func, eq);
arr.add(Expressions.pipe(address));
}
return new CIDRMatchFunctionPipe(source(), this, Expressions.pipe(field), arr);
}

@Override
public boolean foldable() {
return field.foldable() && Expressions.foldable(addresses);
}

return func;
@Override
public Object fold() {
return doProcess(field.fold(), Expressions.fold(addresses));
}

@Override
protected NodeInfo<? extends Expression> info() {
return NodeInfo.create(this, CIDRMatch::new, field, addresses);
}

@Override
public ScriptTemplate asScript() {
ScriptTemplate leftScript = asScript(field);

List<Object> values = new ArrayList<>(new LinkedHashSet<>(Expressions.fold(addresses)));
return new ScriptTemplate(
formatTemplate(LoggerMessageFormat.format("{eql}.","cidrMatch({}, {})", leftScript.template())),
paramsBuilder()
.script(leftScript.params())
.variable(values)
.build(),
dataType());
}

@Override
public ScriptTemplate scriptWithField(FieldAttribute field) {
return new ScriptTemplate(processScript(Scripts.DOC_VALUE),
paramsBuilder().variable(field.exactAttribute().name()).build(),
dataType());
}

@Override
public DataType dataType() {
return DataTypes.BOOLEAN;
}

@Override
public Expression replaceChildren(List<Expression> newChildren) {
if (newChildren.size() < 2) {
throw new IllegalArgumentException("expected at least [2] children but received [" + newChildren.size() + "]");
}
return new CIDRMatch(source(), newChildren.get(0), newChildren.subList(1, newChildren.size()));
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,136 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.eql.expression.function.scalar.string;

import org.elasticsearch.xpack.ql.execution.search.QlSourceBuilder;
import org.elasticsearch.xpack.ql.expression.Expression;
import org.elasticsearch.xpack.ql.expression.gen.pipeline.Pipe;
import org.elasticsearch.xpack.ql.expression.gen.processor.Processor;
import org.elasticsearch.xpack.ql.tree.NodeInfo;
import org.elasticsearch.xpack.ql.tree.Source;
import org.elasticsearch.xpack.ql.util.CollectionUtils;

import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Objects;

public class CIDRMatchFunctionPipe extends Pipe {

private final Pipe source;
private final List<Pipe> addresses;

public CIDRMatchFunctionPipe(Source source, Expression expression, Pipe src, List<Pipe> addresses) {
super(source, expression, CollectionUtils.combine(Collections.singletonList(src), addresses));
this.source = src;
this.addresses = addresses;
}

@Override
public final Pipe replaceChildren(List<Pipe> newChildren) {
if (newChildren.size() < 2) {
throw new IllegalArgumentException("expected at least [2] children but received [" + newChildren.size() + "]");
}
return replaceChildren(newChildren.get(0), newChildren.subList(1, newChildren.size()));
}

@Override
public final Pipe resolveAttributes(AttributeResolver resolver) {
Pipe newSource = source.resolveAttributes(resolver);
boolean same = (newSource == source);

ArrayList<Pipe> newAddresses = new ArrayList<Pipe>(addresses.size());
for (Pipe address : addresses) {
Pipe newAddress = address.resolveAttributes(resolver);
newAddresses.add(newAddress);
same = same && (address == newAddress);
}
if (same) {
return this;
}
return replaceChildren(newSource, newAddresses);
}

@Override
public boolean supportedByAggsOnlyQuery() {
if (source.supportedByAggsOnlyQuery() == false) {
return false;
}
for (Pipe address : addresses) {
if (address.supportedByAggsOnlyQuery() == false) {
return false;
}
}
return true;
}

@Override
public boolean resolved() {
if (source.resolved() == false) {
return false;
}
for (Pipe address : addresses) {
if (address.resolved() == false) {
return false;
}
}
return true;
}

protected Pipe replaceChildren(Pipe newSource, List<Pipe> newAddresses) {
return new CIDRMatchFunctionPipe(source(), expression(), newSource, newAddresses);
}

@Override
public final void collectFields(QlSourceBuilder sourceBuilder) {
source.collectFields(sourceBuilder);
for (Pipe address : addresses) {
address.collectFields(sourceBuilder);
}
}

@Override
protected NodeInfo<CIDRMatchFunctionPipe> info() {
return NodeInfo.create(this, CIDRMatchFunctionPipe::new, expression(), source, addresses);
}

@Override
public CIDRMatchFunctionProcessor asProcessor() {
ArrayList<Processor> processors = new ArrayList<>(addresses.size());
for (Pipe address: addresses) {
processors.add(address.asProcessor());
}
return new CIDRMatchFunctionProcessor(source.asProcessor(), processors);
}

public Pipe src() {
return source;
}

public List<Pipe> addresses() {
return addresses;
}

@Override
public int hashCode() {
return Objects.hash(source(), addresses());
}

@Override
public boolean equals(Object obj) {
if (this == obj) {
return true;
}

if (obj == null || getClass() != obj.getClass()) {
return false;
}

CIDRMatchFunctionPipe other = (CIDRMatchFunctionPipe) obj;
return Objects.equals(source(), other.source())
&& Objects.equals(addresses(), other.addresses());
}
}
Loading

0 comments on commit 4b47e84

Please sign in to comment.