Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add long flavored script field #59721

Merged
merged 4 commits into from
Jul 17, 2020
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -701,23 +701,7 @@ public List<Field> createFields(String name, Number value,
LONG("long", NumericType.LONG) {
@Override
public Long parse(Object value, boolean coerce) {
if (value instanceof Long) {
return (Long)value;
}

double doubleValue = objectToDouble(value);
// this check does not guarantee that value is inside MIN_VALUE/MAX_VALUE because values up to 9223372036854776832 will
// be equal to Long.MAX_VALUE after conversion to double. More checks ahead.
if (doubleValue < Long.MIN_VALUE || doubleValue > Long.MAX_VALUE) {
throw new IllegalArgumentException("Value [" + value + "] is out of range for a long");
}
if (!coerce && doubleValue % 1 != 0) {
throw new IllegalArgumentException("Value [" + value + "] has a decimal part");
}

// longs need special handling so we don't lose precision while parsing
String stringValue = (value instanceof BytesRef) ? ((BytesRef) value).utf8ToString() : value.toString();
return Numbers.toLong(stringValue, coerce);
return objectToLong(value, coerce);
}

@Override
Expand Down Expand Up @@ -854,7 +838,7 @@ Number valueForSearch(Number value) {
/**
* Returns true if the object is a number and has a decimal part
*/
boolean hasDecimalPart(Object number) {
public static boolean hasDecimalPart(Object number) {
if (number instanceof Number) {
double doubleValue = ((Number) number).doubleValue();
return doubleValue % 1 != 0;
Expand Down Expand Up @@ -898,6 +882,30 @@ private static double objectToDouble(Object value) {

return doubleValue;
}

/**
* Converts and Object to a {@code long} by checking it against known
* types and checking its range.
*/
public static long objectToLong(Object value, boolean coerce) {
if (value instanceof Long) {
return (Long)value;
}

double doubleValue = objectToDouble(value);
// this check does not guarantee that value is inside MIN_VALUE/MAX_VALUE because values up to 9223372036854776832 will
// be equal to Long.MAX_VALUE after conversion to double. More checks ahead.
if (doubleValue < Long.MIN_VALUE || doubleValue > Long.MAX_VALUE) {
throw new IllegalArgumentException("Value [" + value + "] is out of range for a long");
}
if (!coerce && doubleValue % 1 != 0) {
throw new IllegalArgumentException("Value [" + value + "] has a decimal part");
}

// longs need special handling so we don't lose precision while parsing
String stringValue = (value instanceof BytesRef) ? ((BytesRef) value).utf8ToString() : value.toString();
return Numbers.toLong(stringValue, coerce);
}
}

public static final class NumberFieldType extends SimpleMappedFieldType {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
package org.elasticsearch.xpack.runtimefields;

import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.util.ArrayUtil;
import org.elasticsearch.painless.spi.Whitelist;
import org.elasticsearch.painless.spi.WhitelistLoader;
import org.elasticsearch.script.ScriptContext;
Expand All @@ -16,10 +17,9 @@
import java.io.IOException;
import java.util.List;
import java.util.Map;
import java.util.function.LongConsumer;

public abstract class LongScriptFieldScript extends AbstractScriptFieldScript {
static final ScriptContext<Factory> CONTEXT = new ScriptContext<>("long_script_field", Factory.class);
public static final ScriptContext<Factory> CONTEXT = new ScriptContext<>("long_script_field", Factory.class);

static List<Whitelist> whitelist() {
return List.of(WhitelistLoader.loadFromResourceFiles(RuntimeFieldsPainlessExtension.class, "long_whitelist.txt"));
Expand All @@ -32,14 +32,47 @@ public interface Factory extends ScriptFactory {
}

public interface LeafFactory {
LongScriptFieldScript newInstance(LeafReaderContext ctx, LongConsumer sync) throws IOException;
LongScriptFieldScript newInstance(LeafReaderContext ctx) throws IOException;
}

private final LongConsumer sync;
private long[] values = new long[1];
private int count;

public LongScriptFieldScript(Map<String, Object> params, SearchLookup searchLookup, LeafReaderContext ctx, LongConsumer sync) {
public LongScriptFieldScript(Map<String, Object> params, SearchLookup searchLookup, LeafReaderContext ctx) {
super(params, searchLookup, ctx);
this.sync = sync;
}

/**
* Execute the script for the provided {@code docId}.
*/
public final void runForDoc(int docId) {
count = 0;
setDocument(docId);
execute();
}

/**
* Values from the last time {@link #runForDoc(int)} was called. This array
* is mutable and will change with the next call of {@link #runForDoc(int)}.
* It is also oversized and will contain garbage at all indices at and
* above {@link #count()}.
*/
public final long[] values() {
return values;
}

/**
* The number of results produced the last time {@link #runForDoc(int)} was called.
*/
public final int count() {
return count;
}

private void collectValue(long v) {
if (values.length < count + 1) {
values = ArrayUtil.grow(values, count + 1);
}
values[count++] = v;
}

public static class Value {
Expand All @@ -50,7 +83,7 @@ public Value(LongScriptFieldScript script) {
}

public void value(long v) {
script.sync.accept(v);
script.collectValue(v);
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ public interface LeafFactory {
StringScriptFieldScript newInstance(LeafReaderContext ctx) throws IOException;
}

protected final List<String> results = new ArrayList<>();
private final List<String> results = new ArrayList<>();
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

thanks! that was my bad after all :D


public StringScriptFieldScript(Map<String, Object> params, SearchLookup searchLookup, LeafReaderContext ctx) {
super(params, searchLookup, ctx);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.search.SortField;
import org.apache.lucene.util.SetOnce;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.ExceptionsHelper;
import org.elasticsearch.common.util.BigArrays;
import org.elasticsearch.index.AbstractIndexComponent;
import org.elasticsearch.index.IndexSettings;
Expand All @@ -34,8 +34,6 @@
import org.elasticsearch.xpack.runtimefields.StringScriptFieldScript;

import java.io.IOException;
import java.util.ArrayList;
import java.util.List;

public final class ScriptBinaryFieldData extends AbstractIndexComponent
implements
Expand Down Expand Up @@ -81,6 +79,7 @@ private ScriptBinaryFieldData(
this.scriptFactory = scriptFactory;
}

@Override
public void setSearchLookup(SearchLookup searchLookup) {
this.leafFactory.set(scriptFactory.newFactory(script.getParams(), searchLookup));
}
Expand All @@ -100,11 +99,7 @@ public ScriptBinaryLeafFieldData load(LeafReaderContext context) {
try {
return loadDirect(context);
} catch (Exception e) {
if (e instanceof ElasticsearchException) {
throw (ElasticsearchException) e;
} else {
throw new ElasticsearchException(e);
}
throw ExceptionsHelper.convertToElastic(e);
}
}

Expand Down Expand Up @@ -165,16 +160,4 @@ public void close() {

}
}

static class ScriptBinaryResult {
private final List<String> result = new ArrayList<>();

void accept(String value) {
this.result.add(value);
}

List<String> getResult() {
return result;
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/

package org.elasticsearch.xpack.runtimefields.fielddata;

import org.elasticsearch.index.fielddata.AbstractSortedNumericDocValues;
import org.elasticsearch.xpack.runtimefields.LongScriptFieldScript;

import java.io.IOException;
import java.util.Arrays;

public final class ScriptLongDocValues extends AbstractSortedNumericDocValues {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

you can use SortingNumericDocValues?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It'd force me to copy the longs from one array to the other rather than just sort. We already have to copy string because of BytesRef vs String stuff. But here I don't have to copy unless I want to.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I see, makes sense then

private final LongScriptFieldScript script;
private int cursor;

ScriptLongDocValues(LongScriptFieldScript script) {
this.script = script;
}

@Override
public boolean advanceExact(int docId) {
script.runForDoc(docId);
if (script.count() == 0) {
return false;
}
Arrays.sort(script.values(), 0, script.count());
cursor = 0;
return true;
}

@Override
public long nextValue() throws IOException {
return script.values()[cursor++];
}

@Override
public int docValueCount() {
return script.count();
}
}
Loading