Skip to content

Commit 8f41a4b

Browse files
mromaioselasticsearchmachinekderusso
authored
refactor(semantic_text): fail early in pre-8.11 indices (#133080)
* fix(semantic_text): index underlying dense_vector field in older indices * Update docs/changelog/133080.yaml * [CI] Auto commit changes from spotless * update msg, change versions, add ut * indent * remove todo * [CI] Auto commit changes from spotless * update msg * [CI] Auto commit changes from spotless * add densevectormapper indexed ut * [CI] Auto commit changes from spotless * fix ut * undo accidental ut removal * [CI] Auto commit changes from spotless * Update docs/changelog/133080.yaml Co-authored-by: Kathleen DeRusso <kathleen.derusso@elastic.co> * remove sparse_vector exception change * [CI] Auto commit changes from spotless * reverting most of the stuff * removing unused import, rename test * [CI] Auto commit changes from spotless * syntax * revisit changelog * revisit changelog * Update docs/changelog/133080.yaml Co-authored-by: Kathleen DeRusso <kathleen.derusso@elastic.co> * Update x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java Co-authored-by: Kathleen DeRusso <kathleen.derusso@elastic.co> * changelog format * fix UT --------- Co-authored-by: elasticsearchmachine <infra-root+elasticsearchmachine@elastic.co> Co-authored-by: Kathleen DeRusso <kathleen.derusso@elastic.co>
1 parent 9a52696 commit 8f41a4b

File tree

4 files changed

+64
-1
lines changed

4 files changed

+64
-1
lines changed

‎docs/changelog/133080.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 133080
2+
summary: "Disallow creating `semantic_text` fields in indices created prior to 8.11.0"
3+
area: Relevance
4+
type: bug
5+
issues: []

‎server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,7 @@
6767
import static org.apache.lucene.tests.index.BaseKnnVectorsFormatTestCase.randomNormalizedVector;
6868
import static org.elasticsearch.index.codec.vectors.IVFVectorsFormat.DYNAMIC_VISIT_RATIO;
6969
import static org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.DEFAULT_OVERSAMPLE;
70+
import static org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.INDEXED_BY_DEFAULT_INDEX_VERSION;
7071
import static org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.IVF_FORMAT;
7172
import static org.hamcrest.Matchers.containsString;
7273
import static org.hamcrest.Matchers.equalTo;
@@ -106,7 +107,7 @@ private void indexMapping(XContentBuilder b, IndexVersion indexVersion) throws I
106107
if (elementType != ElementType.FLOAT) {
107108
b.field("element_type", elementType.toString());
108109
}
109-
if (indexVersion.onOrAfter(DenseVectorFieldMapper.INDEXED_BY_DEFAULT_INDEX_VERSION) || indexed) {
110+
if (indexVersion.onOrAfter(INDEXED_BY_DEFAULT_INDEX_VERSION) || indexed) {
110111
// Serialize if it's new index version, or it was not the default for previous indices
111112
b.field("index", indexed);
112113
}

‎x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -101,6 +101,7 @@
101101
import java.util.function.Function;
102102
import java.util.function.Supplier;
103103

104+
import static org.elasticsearch.index.IndexVersions.NEW_SPARSE_VECTOR;
104105
import static org.elasticsearch.index.IndexVersions.SEMANTIC_TEXT_DEFAULTS_TO_BBQ;
105106
import static org.elasticsearch.index.IndexVersions.SEMANTIC_TEXT_DEFAULTS_TO_BBQ_BACKPORT_8_X;
106107
import static org.elasticsearch.inference.TaskType.SPARSE_EMBEDDING;
@@ -127,6 +128,7 @@
127128
*/
128129
public class SemanticTextFieldMapper extends FieldMapper implements InferenceFieldMapper {
129130
private static final Logger logger = LogManager.getLogger(SemanticTextFieldMapper.class);
131+
public static final String UNSUPPORTED_INDEX_MESSAGE = "[semantic_text] is available on indices created with 8.11 or higher.";
130132
public static final NodeFeature SEMANTIC_TEXT_IN_OBJECT_FIELD_FIX = new NodeFeature("semantic_text.in_object_field_fix");
131133
public static final NodeFeature SEMANTIC_TEXT_SINGLE_FIELD_UPDATE_FIX = new NodeFeature("semantic_text.single_field_update_fix");
132134
public static final NodeFeature SEMANTIC_TEXT_DELETE_FIX = new NodeFeature("semantic_text.delete_fix");
@@ -165,6 +167,9 @@ public static final TypeParser parser(Supplier<ModelRegistry> modelRegistry) {
165167

166168
public static BiConsumer<String, MappingParserContext> validateParserContext(String type) {
167169
return (n, c) -> {
170+
if (c.getIndexSettings().getIndexVersionCreated().before(NEW_SPARSE_VECTOR)) {
171+
throw new UnsupportedOperationException(UNSUPPORTED_INDEX_MESSAGE);
172+
}
168173
if (InferenceMetadataFieldsMapper.isEnabled(c.getIndexSettings().getSettings()) == false) {
169174
notInMultiFields(type).accept(n, c);
170175
}

‎x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -111,6 +111,7 @@
111111
import static org.elasticsearch.xpack.inference.mapper.SemanticTextFieldMapper.DEFAULT_ELSER_2_INFERENCE_ID;
112112
import static org.elasticsearch.xpack.inference.mapper.SemanticTextFieldMapper.DEFAULT_RESCORE_OVERSAMPLE;
113113
import static org.elasticsearch.xpack.inference.mapper.SemanticTextFieldMapper.INDEX_OPTIONS_FIELD;
114+
import static org.elasticsearch.xpack.inference.mapper.SemanticTextFieldMapper.UNSUPPORTED_INDEX_MESSAGE;
114115
import static org.elasticsearch.xpack.inference.mapper.SemanticTextFieldTests.generateRandomChunkingSettings;
115116
import static org.elasticsearch.xpack.inference.mapper.SemanticTextFieldTests.generateRandomChunkingSettingsOtherThan;
116117
import static org.elasticsearch.xpack.inference.mapper.SemanticTextFieldTests.randomSemanticText;
@@ -414,6 +415,57 @@ public void testInvalidTaskTypes() {
414415
}
415416
}
416417

418+
@Override
419+
protected IndexVersion boostNotAllowedIndexVersion() {
420+
return IndexVersions.NEW_SPARSE_VECTOR;
421+
}
422+
423+
public void testOldIndexSemanticTextDenseVectorRaisesError() throws IOException {
424+
final String fieldName = "field";
425+
final XContentBuilder fieldMapping = fieldMapping(b -> {
426+
b.field("type", "semantic_text");
427+
b.field(INFERENCE_ID_FIELD, "test_inference_id");
428+
b.startObject("model_settings");
429+
b.field("task_type", "text_embedding");
430+
b.field("dimensions", 384);
431+
b.field("similarity", "cosine");
432+
b.field("element_type", "float");
433+
b.endObject();
434+
});
435+
assertOldIndexUnsupported(fieldMapping);
436+
}
437+
438+
public void testOldIndexSemanticTextMinimalMappingRaisesError() throws IOException {
439+
final XContentBuilder fieldMapping = fieldMapping(this::minimalMapping);
440+
assertOldIndexUnsupported(fieldMapping);
441+
}
442+
443+
public void testOldIndexSemanticTextSparseVersionRaisesError() throws IOException {
444+
final XContentBuilder fieldMapping = fieldMapping(b -> {
445+
b.field("type", "semantic_text");
446+
b.field("inference_id", "another_inference_id");
447+
b.startObject("model_settings");
448+
b.field("task_type", "sparse_embedding");
449+
b.endObject();
450+
});
451+
assertOldIndexUnsupported(fieldMapping);
452+
}
453+
454+
private void assertOldIndexUnsupported(XContentBuilder fieldMapping) {
455+
456+
MapperParsingException exception = assertThrows(
457+
MapperParsingException.class,
458+
() -> createMapperService(
459+
fieldMapping,
460+
true,
461+
IndexVersions.V_8_0_0,
462+
IndexVersionUtils.getPreviousVersion(IndexVersions.NEW_SPARSE_VECTOR)
463+
)
464+
);
465+
assertTrue(exception.getMessage().contains(UNSUPPORTED_INDEX_MESSAGE));
466+
assertTrue(exception.getRootCause() instanceof UnsupportedOperationException);
467+
}
468+
417469
public void testMultiFieldsSupport() throws IOException {
418470
if (useLegacyFormat) {
419471
Exception e = expectThrows(MapperParsingException.class, () -> createMapperService(fieldMapping(b -> {

0 commit comments

Comments
 (0)