Skip to content

Commit b57e12a

Browse files
javannajavanna
authored
Add raw sort values to SearchSortValues transport serialization (#36617)
In order for CCS alternate execution mode (see #32125) to be able to do the final reduction step on the CCS coordinating node, we need to serialize additional info in the transport layer as part of each `SearchHit`. Sort values are already present but they are formatted according to the provided `DocValueFormat` provided. The CCS node needs to be able to reconstruct the lucene `FieldDoc` to include in the `TopFieldDocs` and `CollapseTopFieldDocs` which will feed the `mergeTopDocs` method used to reduce multiple search responses (one per cluster) into one. This commit adds such information to the `SearchSortValues` and exposes it through a new getter method added to `SearchHit` for retrieval. This info is only serialized at transport and never printed out at REST.
1 parent 7b9ca62 commit b57e12a

File tree

5 files changed

+125
-120
lines changed

5 files changed

+125
-120
lines changed

‎server/src/main/java/org/elasticsearch/search/SearchHit.java‎

Lines changed: 19 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -19,16 +19,6 @@
1919

2020
package org.elasticsearch.search;
2121

22-
import java.io.IOException;
23-
import java.util.ArrayList;
24-
import java.util.Arrays;
25-
import java.util.Collections;
26-
import java.util.HashMap;
27-
import java.util.Iterator;
28-
import java.util.List;
29-
import java.util.Map;
30-
import java.util.Objects;
31-
3222
import org.apache.lucene.search.Explanation;
3323
import org.elasticsearch.ElasticsearchParseException;
3424
import org.elasticsearch.action.OriginalIndices;
@@ -61,6 +51,16 @@
6151
import org.elasticsearch.search.lookup.SourceLookup;
6252
import org.elasticsearch.transport.RemoteClusterAware;
6353

54+
import java.io.IOException;
55+
import java.util.ArrayList;
56+
import java.util.Arrays;
57+
import java.util.Collections;
58+
import java.util.HashMap;
59+
import java.util.Iterator;
60+
import java.util.List;
61+
import java.util.Map;
62+
import java.util.Objects;
63+
6464
import static java.util.Collections.emptyMap;
6565
import static java.util.Collections.singletonMap;
6666
import static java.util.Collections.unmodifiableMap;
@@ -311,10 +311,17 @@ public void sortValues(SearchSortValues sortValues) {
311311
}
312312

313313
/**
314-
* An array of the sort values used.
314+
* An array of the (formatted) sort values used.
315315
*/
316316
public Object[] getSortValues() {
317-
return sortValues.sortValues();
317+
return sortValues.getFormattedSortValues();
318+
}
319+
320+
/**
321+
* An array of the (raw) sort values used.
322+
*/
323+
public Object[] getRawSortValues() {
324+
return sortValues.getRawSortValues();
318325
}
319326

320327
/**

‎server/src/main/java/org/elasticsearch/search/SearchSortValues.java‎

Lines changed: 52 additions & 82 deletions
Original file line numberDiff line numberDiff line change
@@ -20,9 +20,11 @@
2020
package org.elasticsearch.search;
2121

2222
import org.apache.lucene.util.BytesRef;
23+
import org.elasticsearch.Version;
2324
import org.elasticsearch.common.io.stream.StreamInput;
2425
import org.elasticsearch.common.io.stream.StreamOutput;
2526
import org.elasticsearch.common.io.stream.Writeable;
27+
import org.elasticsearch.common.lucene.Lucene;
2628
import org.elasticsearch.common.xcontent.ToXContentFragment;
2729
import org.elasticsearch.common.xcontent.XContentBuilder;
2830
import org.elasticsearch.common.xcontent.XContentParser;
@@ -35,101 +37,56 @@
3537

3638
public class SearchSortValues implements ToXContentFragment, Writeable {
3739

38-
static final SearchSortValues EMPTY = new SearchSortValues(new Object[0]);
39-
private final Object[] sortValues;
40+
private static final Object[] EMPTY_ARRAY = new Object[0];
41+
static final SearchSortValues EMPTY = new SearchSortValues(EMPTY_ARRAY);
42+
43+
private final Object[] formattedSortValues;
44+
private final Object[] rawSortValues;
4045

4146
SearchSortValues(Object[] sortValues) {
42-
this.sortValues = Objects.requireNonNull(sortValues, "sort values must not be empty");
47+
this.formattedSortValues = Objects.requireNonNull(sortValues, "sort values must not be empty");
48+
this.rawSortValues = EMPTY_ARRAY;
4349
}
4450

45-
public SearchSortValues(Object[] sortValues, DocValueFormat[] sortValueFormats) {
46-
Objects.requireNonNull(sortValues);
51+
public SearchSortValues(Object[] rawSortValues, DocValueFormat[] sortValueFormats) {
52+
Objects.requireNonNull(rawSortValues);
4753
Objects.requireNonNull(sortValueFormats);
48-
this.sortValues = Arrays.copyOf(sortValues, sortValues.length);
49-
for (int i = 0; i < sortValues.length; ++i) {
50-
if (this.sortValues[i] instanceof BytesRef) {
51-
this.sortValues[i] = sortValueFormats[i].format((BytesRef) sortValues[i]);
54+
if (rawSortValues.length != sortValueFormats.length) {
55+
throw new IllegalArgumentException("formattedSortValues and sortValueFormats must hold the same number of items");
56+
}
57+
this.rawSortValues = rawSortValues;
58+
this.formattedSortValues = Arrays.copyOf(rawSortValues, rawSortValues.length);
59+
for (int i = 0; i < rawSortValues.length; ++i) {
60+
//we currently format only BytesRef but we may want to change that in the future
61+
Object sortValue = rawSortValues[i];
62+
if (sortValue instanceof BytesRef) {
63+
this.formattedSortValues[i] = sortValueFormats[i].format((BytesRef) sortValue);
5264
}
5365
}
5466
}
5567

56-
public SearchSortValues(StreamInput in) throws IOException {
57-
int size = in.readVInt();
58-
if (size > 0) {
59-
sortValues = new Object[size];
60-
for (int i = 0; i < sortValues.length; i++) {
61-
byte type = in.readByte();
62-
if (type == 0) {
63-
sortValues[i] = null;
64-
} else if (type == 1) {
65-
sortValues[i] = in.readString();
66-
} else if (type == 2) {
67-
sortValues[i] = in.readInt();
68-
} else if (type == 3) {
69-
sortValues[i] = in.readLong();
70-
} else if (type == 4) {
71-
sortValues[i] = in.readFloat();
72-
} else if (type == 5) {
73-
sortValues[i] = in.readDouble();
74-
} else if (type == 6) {
75-
sortValues[i] = in.readByte();
76-
} else if (type == 7) {
77-
sortValues[i] = in.readShort();
78-
} else if (type == 8) {
79-
sortValues[i] = in.readBoolean();
80-
} else {
81-
throw new IOException("Can't match type [" + type + "]");
82-
}
83-
}
68+
SearchSortValues(StreamInput in) throws IOException {
69+
this.formattedSortValues = in.readArray(Lucene::readSortValue, Object[]::new);
70+
if (in.getVersion().onOrAfter(Version.V_7_0_0)) {
71+
this.rawSortValues = in.readArray(Lucene::readSortValue, Object[]::new);
8472
} else {
85-
sortValues = new Object[0];
73+
this.rawSortValues = EMPTY_ARRAY;
8674
}
8775
}
8876

8977
@Override
9078
public void writeTo(StreamOutput out) throws IOException {
91-
out.writeVInt(sortValues.length);
92-
for (Object sortValue : sortValues) {
93-
if (sortValue == null) {
94-
out.writeByte((byte) 0);
95-
} else {
96-
Class type = sortValue.getClass();
97-
if (type == String.class) {
98-
out.writeByte((byte) 1);
99-
out.writeString((String) sortValue);
100-
} else if (type == Integer.class) {
101-
out.writeByte((byte) 2);
102-
out.writeInt((Integer) sortValue);
103-
} else if (type == Long.class) {
104-
out.writeByte((byte) 3);
105-
out.writeLong((Long) sortValue);
106-
} else if (type == Float.class) {
107-
out.writeByte((byte) 4);
108-
out.writeFloat((Float) sortValue);
109-
} else if (type == Double.class) {
110-
out.writeByte((byte) 5);
111-
out.writeDouble((Double) sortValue);
112-
} else if (type == Byte.class) {
113-
out.writeByte((byte) 6);
114-
out.writeByte((Byte) sortValue);
115-
} else if (type == Short.class) {
116-
out.writeByte((byte) 7);
117-
out.writeShort((Short) sortValue);
118-
} else if (type == Boolean.class) {
119-
out.writeByte((byte) 8);
120-
out.writeBoolean((Boolean) sortValue);
121-
} else {
122-
throw new IOException("Can't handle sort field value of type [" + type + "]");
123-
}
124-
}
79+
out.writeArray(Lucene::writeSortValue, this.formattedSortValues);
80+
if (out.getVersion().onOrAfter(Version.V_7_0_0)) {
81+
out.writeArray(Lucene::writeSortValue, this.rawSortValues);
12582
}
12683
}
12784

12885
@Override
12986
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
130-
if (sortValues.length > 0) {
87+
if (formattedSortValues.length > 0) {
13188
builder.startArray(Fields.SORT);
132-
for (Object sortValue : sortValues) {
89+
for (Object sortValue : formattedSortValues) {
13390
builder.value(sortValue);
13491
}
13592
builder.endArray();
@@ -142,24 +99,37 @@ public static SearchSortValues fromXContent(XContentParser parser) throws IOExce
14299
return new SearchSortValues(parser.list().toArray());
143100
}
144101

145-
public Object[] sortValues() {
146-
return sortValues;
102+
/**
103+
* Returns the formatted version of the values that sorting was performed against
104+
*/
105+
public Object[] getFormattedSortValues() {
106+
return formattedSortValues;
107+
}
108+
109+
/**
110+
* Returns the raw version of the values that sorting was performed against
111+
*/
112+
public Object[] getRawSortValues() {
113+
return rawSortValues;
147114
}
148115

149116
@Override
150-
public boolean equals(Object obj) {
151-
if (this == obj) {
117+
public boolean equals(Object o) {
118+
if (this == o) {
152119
return true;
153120
}
154-
if (obj == null || getClass() != obj.getClass()) {
121+
if (o == null || getClass() != o.getClass()) {
155122
return false;
156123
}
157-
SearchSortValues other = (SearchSortValues) obj;
158-
return Arrays.equals(sortValues, other.sortValues);
124+
SearchSortValues that = (SearchSortValues) o;
125+
return Arrays.equals(formattedSortValues, that.formattedSortValues) &&
126+
Arrays.equals(rawSortValues, that.rawSortValues);
159127
}
160128

161129
@Override
162130
public int hashCode() {
163-
return Arrays.hashCode(sortValues);
131+
int result = Arrays.hashCode(formattedSortValues);
132+
result = 31 * result + Arrays.hashCode(rawSortValues);
133+
return result;
164134
}
165135
}

‎server/src/test/java/org/elasticsearch/common/lucene/LuceneTests.java‎

Lines changed: 11 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -531,24 +531,26 @@ public void testSortValueSerialization() throws IOException {
531531
}
532532

533533
public static Object randomSortValue() {
534-
switch(randomIntBetween(0, 8)) {
534+
switch(randomIntBetween(0, 9)) {
535535
case 0:
536-
return randomAlphaOfLengthBetween(3, 10);
536+
return null;
537537
case 1:
538-
return randomInt();
538+
return randomAlphaOfLengthBetween(3, 10);
539539
case 2:
540-
return randomLong();
540+
return randomInt();
541541
case 3:
542-
return randomFloat();
542+
return randomLong();
543543
case 4:
544-
return randomDouble();
544+
return randomFloat();
545545
case 5:
546-
return randomByte();
546+
return randomDouble();
547547
case 6:
548-
return randomShort();
548+
return randomByte();
549549
case 7:
550-
return randomBoolean();
550+
return randomShort();
551551
case 8:
552+
return randomBoolean();
553+
case 9:
552554
return new BytesRef(randomAlphaOfLengthBetween(3, 10));
553555
default:
554556
throw new UnsupportedOperationException();

‎server/src/test/java/org/elasticsearch/search/SearchSortValuesTests.java‎

Lines changed: 42 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
import org.apache.lucene.util.BytesRef;
2323
import org.elasticsearch.Version;
2424
import org.elasticsearch.common.Strings;
25+
import org.elasticsearch.common.io.stream.StreamInput;
2526
import org.elasticsearch.common.io.stream.Writeable;
2627
import org.elasticsearch.common.lucene.LuceneTests;
2728
import org.elasticsearch.common.xcontent.ToXContent;
@@ -31,23 +32,36 @@
3132
import org.elasticsearch.common.xcontent.json.JsonXContent;
3233
import org.elasticsearch.test.AbstractSerializingTestCase;
3334
import org.elasticsearch.test.RandomObjects;
35+
import org.elasticsearch.test.VersionUtils;
3436

3537
import java.io.IOException;
3638
import java.util.Arrays;
39+
import java.util.Base64;
3740

3841
public class SearchSortValuesTests extends AbstractSerializingTestCase<SearchSortValues> {
3942

4043
public static SearchSortValues createTestItem(XContentType xContentType, boolean transportSerialization) {
4144
int size = randomIntBetween(1, 20);
4245
Object[] values = new Object[size];
43-
DocValueFormat[] sortValueFormats = new DocValueFormat[size];
44-
for (int i = 0; i < size; i++) {
45-
Object sortValue = randomSortValue(xContentType, transportSerialization);
46-
values[i] = sortValue;
47-
//make sure that for BytesRef, we provide a specific doc value format that overrides format(BytesRef)
48-
sortValueFormats[i] = sortValue instanceof BytesRef ? DocValueFormat.RAW : randomDocValueFormat();
46+
if (transportSerialization) {
47+
DocValueFormat[] sortValueFormats = new DocValueFormat[size];
48+
for (int i = 0; i < size; i++) {
49+
Object sortValue = randomSortValue(xContentType, transportSerialization);
50+
values[i] = sortValue;
51+
//make sure that for BytesRef, we provide a specific doc value format that overrides format(BytesRef)
52+
sortValueFormats[i] = sortValue instanceof BytesRef ? DocValueFormat.RAW : randomDocValueFormat();
53+
}
54+
return new SearchSortValues(values, sortValueFormats);
55+
} else {
56+
//xcontent serialization doesn't write/parse the raw sort values, only the formatted ones
57+
for (int i = 0; i < size; i++) {
58+
Object sortValue = randomSortValue(xContentType, transportSerialization);
59+
//make sure that BytesRef are not provided as formatted values
60+
sortValue = sortValue instanceof BytesRef ? DocValueFormat.RAW.format((BytesRef)sortValue) : sortValue;
61+
values[i] = sortValue;
62+
}
63+
return new SearchSortValues(values);
4964
}
50-
return new SearchSortValues(values, sortValueFormats);
5165
}
5266

5367
private static Object randomSortValue(XContentType xContentType, boolean transportSerialization) {
@@ -79,7 +93,7 @@ protected SearchSortValues createXContextTestInstance(XContentType xContentType)
7993

8094
@Override
8195
protected SearchSortValues createTestInstance() {
82-
return createTestItem(randomFrom(XContentType.values()), true);
96+
return createTestItem(randomFrom(XContentType.values()), randomBoolean());
8397
}
8498

8599
@Override
@@ -113,20 +127,32 @@ public void testToXContent() throws IOException {
113127

114128
@Override
115129
protected SearchSortValues mutateInstance(SearchSortValues instance) {
116-
Object[] sortValues = instance.sortValues();
117-
if (sortValues.length == 0) {
118-
return createTestInstance();
119-
}
130+
Object[] sortValues = instance.getFormattedSortValues();
120131
if (randomBoolean()) {
121132
return new SearchSortValues(new Object[0]);
122133
}
123134
Object[] values = Arrays.copyOf(sortValues, sortValues.length + 1);
124-
values[sortValues.length] = randomSortValue(randomFrom(XContentType.values()), true);
135+
values[sortValues.length] = randomSortValue(randomFrom(XContentType.values()), randomBoolean());
125136
return new SearchSortValues(values);
126137
}
127138

128-
@Override
129-
protected SearchSortValues copyInstance(SearchSortValues instance, Version version) {
130-
return new SearchSortValues(Arrays.copyOf(instance.sortValues(), instance.sortValues().length));
139+
//TODO rename and update version after backport
140+
public void testSerializationPre70() throws IOException {
141+
Version version = VersionUtils.randomVersionBetween(random(), Version.V_6_0_0, VersionUtils.getPreviousVersion(Version.V_7_0_0));
142+
SearchSortValues original = createTestInstance();
143+
SearchSortValues deserialized = copyInstance(original, version);
144+
assertArrayEquals(original.getFormattedSortValues(), deserialized.getFormattedSortValues());
145+
assertEquals(0, deserialized.getRawSortValues().length);
146+
}
147+
148+
//TODO rename method and adapt versions after backport
149+
public void testReadFromPre70() throws IOException {
150+
try (StreamInput in = StreamInput.wrap(Base64.getDecoder().decode("AwIAAAABAQEyBUAIAAAAAAAAAAAAAAAA"))) {
151+
in.setVersion(VersionUtils.randomVersionBetween(random(), Version.V_6_0_0, VersionUtils.getPreviousVersion(Version.V_7_0_0)));
152+
SearchSortValues deserialized = new SearchSortValues(in);
153+
SearchSortValues expected = new SearchSortValues(new Object[]{1, "2", 3d});
154+
assertEquals(expected, deserialized);
155+
assertEquals(0, deserialized.getRawSortValues().length);
156+
}
131157
}
132158
}

‎test/framework/src/main/java/org/elasticsearch/test/AbstractWireSerializingTestCase.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,6 @@ public abstract class AbstractWireSerializingTestCase<T extends Writeable> exten
2727

2828
@Override
2929
protected T copyInstance(T instance, Version version) throws IOException {
30-
return copyWriteable(instance, getNamedWriteableRegistry(), instanceReader());
30+
return copyWriteable(instance, getNamedWriteableRegistry(), instanceReader(), version);
3131
}
3232
}

0 commit comments

Comments
 (0)