Skip to content

Improve block loader for source only runtime boolean fields #135318

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Sep 24, 2025
Merged

Improve block loader for source only runtime boolean fields #135318

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
7c8aebb
Improve block loader for source only runtime boolean fields
Kubik42 Sep 23, 2025
0906b7d
Merge branch 'main' into kubik-boolean-script-block-loader-improvements
Kubik42 Sep 24, 2025
d510112
Merge branch 'main' into kubik-boolean-script-block-loader-improvements
Kubik42 Sep 24, 2025
1f7ae15
Simplify boolean casting
Kubik42 Sep 24, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
46 changes: 46 additions & 0 deletions server/src/main/java/org/elasticsearch/index/mapper/BooleanScriptFieldType.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,9 +27,12 @@
import org.elasticsearch.search.lookup.SearchLookup;
import org.elasticsearch.search.runtime.BooleanScriptFieldExistsQuery;
import org.elasticsearch.search.runtime.BooleanScriptFieldTermQuery;
import org.elasticsearch.xcontent.XContentParser;

import java.io.IOException;
import java.time.ZoneId;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.function.Function;

Expand Down Expand Up @@ -114,9 +117,52 @@ public DocValueFormat docValueFormat(String format, ZoneId timeZone) {

@Override
public BlockLoader blockLoader(BlockLoaderContext blContext) {
FallbackSyntheticSourceBlockLoader fallbackSyntheticSourceBlockLoader = fallbackSyntheticSourceBlockLoader(
blContext,
BlockLoader.BlockFactory::booleans,
this::fallbackSyntheticSourceBlockLoaderReader
);

if (fallbackSyntheticSourceBlockLoader != null) {
return fallbackSyntheticSourceBlockLoader;
}
return new BooleanScriptBlockDocValuesReader.BooleanScriptBlockLoader(leafFactory(blContext.lookup()));
}

private FallbackSyntheticSourceBlockLoader.Reader<?> fallbackSyntheticSourceBlockLoaderReader() {
return new FallbackSyntheticSourceBlockLoader.SingleValueReader<Boolean>(null) {
@Override
public void convertValue(Object value, List<Boolean> accumulator) {
try {
if (value instanceof Boolean b) {
accumulator.add(b);
} else {
accumulator.add(Booleans.parseBoolean(value.toString(), false));
}
} catch (Exception e) {
// value is malformed, skip it
}
}

@Override
public void writeToBlock(List<Boolean> values, BlockLoader.Builder blockBuilder) {
var booleanBuilder = (BlockLoader.BooleanBuilder) blockBuilder;
for (boolean value : values) {
booleanBuilder.appendBoolean(value);
}
}

@Override
protected void parseNonNullValue(XContentParser parser, List<Boolean> accumulator) throws IOException {
try {
accumulator.add(parser.booleanValue());
} catch (Exception e) {
// value is malformed, skip it
}
}
};
}

@Override
public BooleanScriptFieldData.Builder fielddataBuilder(FieldDataContext fieldDataContext) {
return new BooleanScriptFieldData.Builder(name(), leafFactory(fieldDataContext.lookupSupplier().get()), BooleanDocValuesField::new);
Expand Down
139 changes: 139 additions & 0 deletions server/src/test/java/org/elasticsearch/index/mapper/BooleanScriptFieldTypeTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,8 @@
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.common.bytes.BytesArray;
import org.elasticsearch.common.lucene.search.function.ScriptScoreQuery;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.core.Booleans;
import org.elasticsearch.index.IndexVersion;
import org.elasticsearch.index.fielddata.BooleanScriptFieldData;
import org.elasticsearch.index.fielddata.ScriptDocValues;
Expand All @@ -44,6 +46,7 @@
import org.elasticsearch.script.ScriptType;
import org.elasticsearch.script.field.BooleanDocValuesField;
import org.elasticsearch.search.MultiValueMode;
import org.elasticsearch.search.lookup.SearchLookup;
import org.elasticsearch.test.ESTestCase;
import org.elasticsearch.xcontent.XContentParser;
import org.elasticsearch.xcontent.XContentParser.Token;
Expand All @@ -52,6 +55,7 @@

import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Map;

Expand All @@ -60,9 +64,13 @@
import static org.hamcrest.Matchers.containsInAnyOrder;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.instanceOf;
import static org.hamcrest.Matchers.nullValue;

public class BooleanScriptFieldTypeTests extends AbstractNonTextScriptFieldTypeTestCase {

private static final Boolean MALFORMED_BOOLEAN = null;
private static final Boolean EMPTY_STR_BOOLEAN = false;

@Override
protected ScriptFactory parseFromSource() {
return BooleanFieldScript.PARSE_FROM_SOURCE;
Expand Down Expand Up @@ -453,6 +461,137 @@ public void testBlockLoader() throws IOException {
}
}

public void testBlockLoaderSourceOnlyRuntimeField() throws IOException {
try (
Directory directory = newDirectory();
RandomIndexWriter iw = new RandomIndexWriter(random(), directory, newIndexWriterConfig().setMergePolicy(NoMergePolicy.INSTANCE))
) {
// given
// try multiple variations of boolean as they're all encoded slightly differently
iw.addDocuments(
List.of(
List.of(new StoredField("_source", new BytesRef("{\"test\": [false]}"))),
List.of(new StoredField("_source", new BytesRef("{\"test\": [true]}"))),
List.of(new StoredField("_source", new BytesRef("{\"test\": [\"false\"]}"))),
List.of(new StoredField("_source", new BytesRef("{\"test\": [\"true\"]}"))),
List.of(new StoredField("_source", new BytesRef("{\"test\": [\"\"]}"))),
// ensure a malformed value doesn't crash
List.of(new StoredField("_source", new BytesRef("{\"test\": [\"potato\"]}")))
)
);
BooleanScriptFieldType fieldType = simpleSourceOnlyMappedFieldType();
List<Boolean> expected = Arrays.asList(false, true, false, true, EMPTY_STR_BOOLEAN, MALFORMED_BOOLEAN);

try (DirectoryReader reader = iw.getReader()) {
// when
BlockLoader loader = fieldType.blockLoader(blContext(Settings.EMPTY, true));

// then

// assert loader is of expected instance type
assertThat(loader, instanceOf(BooleanScriptBlockDocValuesReader.BooleanScriptBlockLoader.class));

// ignored source doesn't support column at a time loading:
var columnAtATimeLoader = loader.columnAtATimeReader(reader.leaves().getFirst());
assertThat(columnAtATimeLoader, instanceOf(BooleanScriptBlockDocValuesReader.class));

var rowStrideReader = loader.rowStrideReader(reader.leaves().getFirst());
assertThat(rowStrideReader, instanceOf(BooleanScriptBlockDocValuesReader.class));

// assert values
assertThat(blockLoaderReadValuesFromColumnAtATimeReader(reader, fieldType, 0), equalTo(expected));
assertThat(blockLoaderReadValuesFromRowStrideReader(reader, fieldType), equalTo(expected));
}
}
}

public void testBlockLoaderSourceOnlyRuntimeFieldWithSyntheticSource() throws IOException {
try (
Directory directory = newDirectory();
RandomIndexWriter iw = new RandomIndexWriter(random(), directory, newIndexWriterConfig().setMergePolicy(NoMergePolicy.INSTANCE))
) {
// given
// try multiple variations of boolean as they're all encoded slightly differently
iw.addDocuments(
List.of(
createDocumentWithIgnoredSource("false"),
createDocumentWithIgnoredSource("true"),
createDocumentWithIgnoredSource("[false]"),
createDocumentWithIgnoredSource("[true]"),
createDocumentWithIgnoredSource("[\"false\"]"),
createDocumentWithIgnoredSource("[\"true\"]"),
createDocumentWithIgnoredSource("[\"\"]"),
// ensure a malformed value doesn't crash
createDocumentWithIgnoredSource("[\"potato\"]")
)
);

Settings settings = Settings.builder().put("index.mapping.source.mode", "synthetic").build();
BooleanScriptFieldType fieldType = simpleSourceOnlyMappedFieldType();
List<Boolean> expected = Arrays.asList(false, true, false, true, false, true, EMPTY_STR_BOOLEAN, MALFORMED_BOOLEAN);

try (DirectoryReader reader = iw.getReader()) {
// when
BlockLoader loader = fieldType.blockLoader(blContext(settings, true));

// then

// assert loader is of expected instance type
assertThat(loader, instanceOf(FallbackSyntheticSourceBlockLoader.class));

// ignored source doesn't support column at a time loading:
var columnAtATimeLoader = loader.columnAtATimeReader(reader.leaves().getFirst());
assertThat(columnAtATimeLoader, nullValue());

var rowStrideReader = loader.rowStrideReader(reader.leaves().getFirst());
assertThat(
rowStrideReader.getClass().getName(),
equalTo("org.elasticsearch.index.mapper.FallbackSyntheticSourceBlockLoader$IgnoredSourceRowStrideReader")
);

// assert values
assertThat(blockLoaderReadValuesFromRowStrideReader(settings, reader, fieldType, true), equalTo(expected));
}
}
}

/**
* Returns a source only mapped field type. This is useful, since the available build() function doesn't override isParsedFromSource()
*/
private BooleanScriptFieldType simpleSourceOnlyMappedFieldType() {
Script script = new Script(ScriptType.INLINE, "test", "", emptyMap());
BooleanFieldScript.Factory factory = new BooleanFieldScript.Factory() {
@Override
public BooleanFieldScript.LeafFactory newFactory(
String fieldName,
Map<String, Object> params,
SearchLookup searchLookup,
OnScriptError onScriptError
) {
return ctx -> new BooleanFieldScript(fieldName, params, searchLookup, onScriptError, ctx) {
@Override
@SuppressWarnings("unchecked")
public void execute() {
Map<String, Object> source = (Map<String, Object>) this.getParams().get("_source");
for (Object foo : (List<?>) source.get("test")) {
try {
emit(Booleans.parseBoolean(foo.toString(), false));
} catch (Exception e) {
// skip
}
}
}
};
}

@Override
public boolean isParsedFromSource() {
return true;
}
};
return new BooleanScriptFieldType("test", factory, script, emptyMap(), OnScriptError.FAIL);
}

private void assertSameCount(IndexSearcher searcher, String source, Object queryDescription, Query scriptedQuery, Query ootbQuery)
throws IOException {
assertThat(
Expand Down