diff --git a/REST_API_COMPATIBILITY.md b/REST_API_COMPATIBILITY.md index 4a6ad4e7e17f5..162935ec23c25 100644 --- a/REST_API_COMPATIBILITY.md +++ b/REST_API_COMPATIBILITY.md @@ -166,9 +166,9 @@ For example: ./gradlew :rest-api-spec:yamlRestCompatTest ``` -Since these are a variation of backward compatibility testing, the entire suite of compatibility tests will be skipped anytime the backward compatibility testing is disabled. Since the source code for these tests live in a branch of code, disabling a specific test should be done via the transformation task configuration in build.gradle (i.e. `yamlRestTestV7CompatTransform`). +Since these are a variation of backward compatibility testing, the entire suite of compatibility tests will be skipped anytime the backward compatibility testing is disabled. Since the source code for these tests live in a branch of code, disabling a specific test should be done via the transformation task configuration in build.gradle (i.e. `yamlRestCompatTestTransform`). -In some cases the prior version of the YAML REST tests are not sufficient to fully test changes. This can happen when the prior version has insufficient test coverage. In those cases, you can simply add more testing to the prior version or you can add custom REST tests that will run along side of the other compatibility tests. These custom tests can be found in the `yamlRestTestV7` (where Vnumber follows the same conventions) sourceset and the test directory is versioned for example: `yamlRestTestV7/resources/rest-api-spec/test/v7compat`). Custom REST tests for compatibility will not be modified prior to execution, so the correct headers need to be manually added. +In some cases the prior version of the YAML REST tests are not sufficient to fully test changes. This can happen when the prior version has insufficient test coverage. In those cases, you can simply add more testing to the prior version or you can add custom REST tests that will run along side of the other compatibility tests. These custom tests can be found in the `yamlRestCompatTest` sourceset. Custom REST tests for compatibility will not be modified prior to execution, so the correct headers need to be manually added. ### Developer's workflow @@ -182,7 +182,7 @@ Mixed clusters are not explicitly tested since the change should be applied at t By far the most common reason that compatibility tests can seemingly randomly fail is that your main branch is out of date with the upstream main. For this reason, it always suggested to ensure that your PR branch is up to date. -Test failure reproduction lines should behave identical to the non-compatible variant. However, to assure you are referencing the correct line number when reading the test, be sure to look at the line number from the transformed test on disk. Generally the fully transformed tests can be found at `build/restResources/v7/yamlTests/transformed/rest-api-spec/test/*` (where v7 will change with different versions). +Test failure reproduction lines should behave identical to the non-compatible variant. However, to assure you are referencing the correct line number when reading the test, be sure to look at the line number from the transformed test on disk. Generally the fully transformed tests can be found at `build/restResources/compat/yamlTests/transformed/rest-api-spec/test/*`. Muting compatibility tests should be done via a test transform. A per test skip or a file match can be used to skip the tests. diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/StandardVersusLogsIndexModeRandomDataChallengeRestIT.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/StandardVersusLogsIndexModeRandomDataChallengeRestIT.java index 404914cda6a7e..b269e8a3a6424 100644 --- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/StandardVersusLogsIndexModeRandomDataChallengeRestIT.java +++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/StandardVersusLogsIndexModeRandomDataChallengeRestIT.java @@ -43,7 +43,9 @@ public class StandardVersusLogsIndexModeRandomDataChallengeRestIT extends Standa public StandardVersusLogsIndexModeRandomDataChallengeRestIT() { super(); - this.subobjects = randomFrom(ObjectMapper.Subobjects.values()); + // TODO enable subobjects: auto + // It is disabled because it currently does not have auto flattening and that results in asserts being triggered when using copy_to. + this.subobjects = randomValueOtherThan(ObjectMapper.Subobjects.AUTO, () -> randomFrom(ObjectMapper.Subobjects.values())); this.keepArraySource = randomBoolean(); var specificationBuilder = DataGeneratorSpecification.builder().withFullyDynamicMapping(randomBoolean()); @@ -120,6 +122,14 @@ public CheckedConsumer fieldValueGenerator() { .build()); } + @Override + protected final Settings restClientSettings() { + return Settings.builder() + .put(super.restClientSettings()) + .put(org.elasticsearch.test.rest.ESRestTestCase.CLIENT_SOCKET_TIMEOUT, "9000s") + .build(); + } + @Override public void baselineMappings(XContentBuilder builder) throws IOException { dataGenerator.writeMapping(builder); diff --git a/muted-tests.yml b/muted-tests.yml index 88d532d39e81a..7dbb7f8f40333 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -214,6 +214,9 @@ tests: - class: org.elasticsearch.snapshots.ConcurrentSnapshotsIT method: testMasterFailoverOnFinalizationLoop issue: https://github.com/elastic/elasticsearch/issues/112811 +- class: org.elasticsearch.xpack.ml.integration.MlJobIT + method: testDeleteJobAfterMissingAliases + issue: https://github.com/elastic/elasticsearch/issues/112823 # Examples: # diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml index 9dd6cec6e657c..9c42471a2ccbe 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml @@ -974,7 +974,7 @@ subobjects auto: --- synthetic_source with copy_to: - requires: - cluster_features: ["mapper.source.synthetic_source_with_copy_to_and_doc_values_false"] + cluster_features: ["mapper.source.synthetic_source_copy_to_fix"] reason: requires copy_to support in synthetic source - do: @@ -1068,33 +1068,43 @@ synthetic_source with copy_to: fields: ["number_copy", "boolean_copy", "keyword_copy", "date_copy", "text_copy", "ip_copy", "ip_range_copy", "geo_point_copy", "binary_copy", "scaled_float_copy"] - match: { hits.hits.0._source.number: 100 } + - match: { hits.hits.0._source.number_copy: null } - match: { hits.hits.0.fields.number_copy.0: "100" } - match: { hits.hits.0._source.boolean: false } + - match: { hits.hits.0._source.boolean_copy: null } - match: { hits.hits.0.fields.boolean_copy.0: "false" } - match: { hits.hits.0._source.keyword: "hello_keyword" } + - match: { hits.hits.0._source.keyword_copy: null } - match: { hits.hits.0.fields.keyword_copy.0: "hello_keyword" } - match: { hits.hits.0._source.date: "2015-01-01T12:10:30Z" } + - match: { hits.hits.0._source.date_copy: null } - match: { hits.hits.0.fields.date_copy.0: "2015-01-01T12:10:30Z" } - match: { hits.hits.0._source.text: "hello_text" } + - match: { hits.hits.0._source.text_copy: null } - match: { hits.hits.0.fields.text_copy.0: "hello_text" } - match: { hits.hits.0._source.ip: "192.168.1.1" } + - match: { hits.hits.0._source.ip_copy: null } - match: { hits.hits.0.fields.ip_copy.0: "192.168.1.1" } - match: { hits.hits.0._source.ip_range: "10.0.0.0/24" } + - match: { hits.hits.0._source.ip_range_copy: null } - match: { hits.hits.0.fields.ip_range_copy.0: "10.0.0.0/24" } - match: { hits.hits.0._source.geo_point: "POINT (-71.34 41.12)" } + - match: { hits.hits.0._source.geo_point_copy: null } - match: { hits.hits.0.fields.geo_point_copy.0: "POINT (-71.34 41.12)" } - match: { hits.hits.0._source.binary: "aGVsbG8gY3VyaW91cyBwZXJzb24=" } + - match: { hits.hits.0._source.binary_copy: null } - match: { hits.hits.0.fields.binary_copy.0: "aGVsbG8gY3VyaW91cyBwZXJzb24=" } - match: { hits.hits.0._source.scaled_float: 1.5 } + - match: { hits.hits.0._source.scaled_float_copy: null } - match: { hits.hits.0.fields.scaled_float_copy.0: "1.5" } --- @@ -1214,3 +1224,331 @@ fallback synthetic_source for text field: hits.hits.0._source: text: [ "world", "hello", "world" ] +--- +synthetic_source with copy_to and ignored values: + - requires: + cluster_features: ["mapper.source.synthetic_source_copy_to_fix"] + reason: requires copy_to support in synthetic source + + - do: + indices.create: + index: test + body: + mappings: + _source: + mode: synthetic + properties: + name: + type: keyword + k: + type: keyword + ignore_above: 1 + copy_to: copy + long: + type: long + ignore_malformed: true + copy_to: copy + copy: + type: keyword + + - do: + index: + index: test + id: 1 + refresh: true + body: + name: "A" + k: "hello" + long: "world" + + - do: + index: + index: test + id: 2 + refresh: true + body: + name: "B" + k: ["5", "6"] + long: ["7", "8"] + + - do: + search: + index: test + sort: name + body: + docvalue_fields: [ "copy" ] + + - match: + hits.hits.0._source: + name: "A" + k: "hello" + long: "world" + - match: { hits.hits.0.fields.copy: ["hello", "world"] } + + - match: + hits.hits.1._source: + name: "B" + k: ["5", "6"] + long: ["7", "8"] + - match: { hits.hits.1.fields.copy: ["5", "6", "7", "8"] } + + +--- +synthetic_source with copy_to field having values in source: + - requires: + cluster_features: ["mapper.source.synthetic_source_copy_to_fix"] + reason: requires copy_to support in synthetic source + + - do: + indices.create: + index: test + body: + mappings: + _source: + mode: synthetic + properties: + name: + type: keyword + k: + type: keyword + copy_to: copy + copy: + type: keyword + + - do: + index: + index: test + id: 1 + refresh: true + body: + name: "A" + copy: "world" + k: "hello" + + - do: + index: + index: test + id: 2 + refresh: true + body: + name: "B" + k: ["5", "6"] + copy: ["7", "8"] + + - do: + search: + index: test + sort: name + body: + docvalue_fields: [ "copy" ] + + - match: + hits.hits.0._source: + name: "A" + k: "hello" + copy: "world" + - match: { hits.hits.0.fields.copy: ["hello", "world"] } + + - match: + hits.hits.1._source: + name: "B" + k: ["5", "6"] + copy: ["7", "8"] + - match: { hits.hits.1.fields.copy: ["5", "6", "7", "8"] } + +--- +synthetic_source with ignored source field using copy_to: + - requires: + cluster_features: ["mapper.source.synthetic_source_copy_to_fix"] + reason: requires copy_to support in synthetic source + + - do: + indices.create: + index: test + body: + mappings: + _source: + mode: synthetic + properties: + name: + type: keyword + k: + type: keyword + doc_values: false + copy_to: copy + copy: + type: keyword + + - do: + index: + index: test + id: 1 + refresh: true + body: + name: "A" + copy: "world" + k: "hello" + + - do: + index: + index: test + id: 2 + refresh: true + body: + name: "B" + k: ["5", "6"] + copy: ["7", "8"] + + - do: + search: + index: test + sort: name + body: + docvalue_fields: [ "copy" ] + + - match: + hits.hits.0._source: + name: "A" + k: "hello" + copy: "world" + - match: { hits.hits.0.fields.copy: ["hello", "world"] } + + - match: + hits.hits.1._source: + name: "B" + k: ["5", "6"] + copy: ["7", "8"] + - match: { hits.hits.1.fields.copy: ["5", "6", "7", "8"] } + +--- +synthetic_source with copy_to field from dynamic template having values in source: + - requires: + cluster_features: ["mapper.source.synthetic_source_copy_to_fix"] + reason: requires copy_to support in synthetic source + + - do: + indices.create: + index: test + body: + mappings: + _source: + mode: synthetic + dynamic_templates: + - copy_template: + match: "k" + mapping: + type: keyword + copy_to: copy + properties: + name: + type: keyword + copy: + type: keyword + + - do: + index: + index: test + id: 1 + refresh: true + body: + name: "A" + k: "hello" + + - do: + index: + index: test + id: 2 + refresh: true + body: + name: "B" + copy: "world" + k: "hello" + + - do: + index: + index: test + id: 3 + refresh: true + body: + name: "C" + k: ["5", "6"] + + - do: + index: + index: test + id: 4 + refresh: true + body: + name: "D" + k: ["5", "6"] + copy: ["7", "8"] + + - do: + search: + index: test + sort: name + body: + docvalue_fields: [ "copy" ] + + - match: + hits.hits.0._source: + name: "A" + k: "hello" + - match: { hits.hits.0.fields.copy: ["hello"] } + + - match: + hits.hits.1._source: + name: "B" + k: "hello" + copy: "world" + - match: { hits.hits.1.fields.copy: ["hello", "world"] } + + - match: + hits.hits.2._source: + name: "C" + k: ["5", "6"] + - match: { hits.hits.2.fields.copy: ["5", "6"] } + + - match: + hits.hits.3._source: + name: "D" + k: ["5", "6"] + copy: ["7", "8"] + - match: { hits.hits.3.fields.copy: ["5", "6", "7", "8"] } + +--- +synthetic_source with copy_to and invalid values for copy: + - requires: + cluster_features: ["mapper.source.synthetic_source_copy_to_fix"] + reason: requires copy_to support in synthetic source + test_runner_features: "contains" + + - do: + indices.create: + index: test + body: + mappings: + _source: + mode: synthetic + properties: + name: + type: keyword + p: + type: long_range + copy_to: copy + copy: + type: keyword + + - do: + catch: bad_request + index: + index: test + id: 1 + refresh: true + body: + name: "A" + p: + gte: 10 + + - match: { error.type: "document_parsing_exception" } + - contains: { error.reason: "Copy-to currently only works for value-type fields" } diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/reservedstate/ReservedRepositoryAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/reservedstate/ReservedRepositoryAction.java index d3dc7c916f066..2dae8c4d98685 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/reservedstate/ReservedRepositoryAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/reservedstate/ReservedRepositoryAction.java @@ -82,7 +82,7 @@ public TransformState transform(Object source, TransformState prevState) throws toDelete.removeAll(entities); for (var repositoryToDelete : toDelete) { - var task = new RepositoriesService.UnregisterRepositoryTask(DUMMY_TIMEOUT, repositoryToDelete); + var task = new RepositoriesService.UnregisterRepositoryTask(RESERVED_CLUSTER_STATE_HANDLER_IGNORED_TIMEOUT, repositoryToDelete); state = task.execute(state); } @@ -97,7 +97,11 @@ public List fromXContent(XContentParser parser) throws IOE Map source = parser.map(); for (var entry : source.entrySet()) { - PutRepositoryRequest putRepositoryRequest = new PutRepositoryRequest(DUMMY_TIMEOUT, DUMMY_TIMEOUT, entry.getKey()); + PutRepositoryRequest putRepositoryRequest = new PutRepositoryRequest( + RESERVED_CLUSTER_STATE_HANDLER_IGNORED_TIMEOUT, + RESERVED_CLUSTER_STATE_HANDLER_IGNORED_TIMEOUT, + entry.getKey() + ); @SuppressWarnings("unchecked") Map content = (Map) entry.getValue(); try (XContentParser repoParser = mapToXContentParser(XContentParserConfiguration.EMPTY, content)) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java index e2143aca0fd1d..e8d96979b51c0 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java @@ -441,7 +441,9 @@ static void parseObjectOrField(DocumentParserContext context, Mapper mapper) thr parseObjectOrNested(context.createFlattenContext(currentFieldName)); context.path().add(currentFieldName); } else { - if (context.canAddIgnoredField() && fieldMapper.syntheticSourceMode() == FieldMapper.SyntheticSourceMode.FALLBACK) { + if (context.canAddIgnoredField() + && (fieldMapper.syntheticSourceMode() == FieldMapper.SyntheticSourceMode.FALLBACK + || (context.isWithinCopyTo() == false && context.isCopyToDestinationField(mapper.fullPath())))) { Tuple contextWithSourceToStore = XContentDataHelper.cloneSubContext(context); context.addIgnoredField( @@ -685,6 +687,8 @@ private static void parseNonDynamicArray( ) throws IOException { // Check if we need to record the array source. This only applies to synthetic source. if (context.canAddIgnoredField()) { + String fullPath = context.path().pathAsText(arrayFieldName); + boolean objectRequiresStoringSource = mapper instanceof ObjectMapper objectMapper && (objectMapper.storeArraySource() || (context.sourceKeepModeFromIndexSettings() == Mapper.SourceKeepMode.ARRAYS @@ -692,27 +696,24 @@ private static void parseNonDynamicArray( || objectMapper.dynamic == ObjectMapper.Dynamic.RUNTIME); boolean fieldWithFallbackSyntheticSource = mapper instanceof FieldMapper fieldMapper && fieldMapper.syntheticSourceMode() == FieldMapper.SyntheticSourceMode.FALLBACK; - boolean fieldWithStoredArraySource = mapper instanceof FieldMapper fieldMapper + boolean fieldWithStoredArraySource = mapper instanceof FieldMapper && context.sourceKeepModeFromIndexSettings() == Mapper.SourceKeepMode.ARRAYS; boolean dynamicRuntimeContext = context.dynamic() == ObjectMapper.Dynamic.RUNTIME; - if (objectRequiresStoringSource || fieldWithFallbackSyntheticSource || dynamicRuntimeContext || fieldWithStoredArraySource) { + boolean copyToFieldHasValuesInDocument = context.isWithinCopyTo() == false && context.isCopyToDestinationField(fullPath); + if (objectRequiresStoringSource + || fieldWithFallbackSyntheticSource + || dynamicRuntimeContext + || fieldWithStoredArraySource + || copyToFieldHasValuesInDocument) { Tuple tuple = XContentDataHelper.cloneSubContext(context); context.addIgnoredField( - IgnoredSourceFieldMapper.NameValue.fromContext( - context, - context.path().pathAsText(arrayFieldName), - XContentDataHelper.encodeXContentBuilder(tuple.v2()) - ) + IgnoredSourceFieldMapper.NameValue.fromContext(context, fullPath, XContentDataHelper.encodeXContentBuilder(tuple.v2())) ); context = tuple.v1(); } else if (mapper instanceof ObjectMapper objectMapper && (objectMapper.isEnabled() == false || objectMapper.dynamic == ObjectMapper.Dynamic.FALSE)) { context.addIgnoredField( - IgnoredSourceFieldMapper.NameValue.fromContext( - context, - context.path().pathAsText(arrayFieldName), - XContentDataHelper.encodeToken(context.parser()) - ) + IgnoredSourceFieldMapper.NameValue.fromContext(context, fullPath, XContentDataHelper.encodeToken(context.parser())) ); return; } @@ -747,7 +748,7 @@ private static void postProcessDynamicArrayMapping(DocumentParserContext context final List mappers = context.getDynamicMappers(fullFieldName); if (mappers == null || context.isFieldAppliedFromTemplate(fullFieldName) - || context.isCopyToField(fullFieldName) + || context.isCopyToDestinationField(fullFieldName) || mappers.size() < MIN_DIMS_FOR_DYNAMIC_FLOAT_MAPPING || mappers.size() > MAX_DIMS_COUNT // Anything that is NOT a number or anything that IS a number but not mapped to `float` should NOT be mapped to dense_vector @@ -870,6 +871,7 @@ private static void parseCopyFields(DocumentParserContext context, List // ignore copy_to that targets inference fields, values are already extracted in the coordinating node to perform inference. continue; } + // In case of a hierarchy of nested documents, we need to figure out // which document the field should go to LuceneDocument targetDoc = null; @@ -902,14 +904,13 @@ private static Mapper getLeafMapper(final DocumentParserContext context, String if (fieldType != null) { // we haven't found a mapper with this name above, which means if a field type is found it is for sure a runtime field. assert fieldType.hasDocValues() == false && fieldType.isAggregatable() && fieldType.isSearchable(); - return NO_OP_FIELDMAPPER; + return noopFieldMapper(fieldPath); } return null; } - private static final FieldMapper NO_OP_FIELDMAPPER = new FieldMapper( - "no-op", - new MappedFieldType("no-op", false, false, false, TextSearchInfo.NONE, Collections.emptyMap()) { + private static FieldMapper noopFieldMapper(String path) { + return new FieldMapper("no-op", new MappedFieldType("no-op", false, false, false, TextSearchInfo.NONE, Collections.emptyMap()) { @Override public ValueFetcher valueFetcher(SearchExecutionContext context, String format) { throw new UnsupportedOperationException(); @@ -924,86 +925,84 @@ public String typeName() { public Query termQuery(Object value, SearchExecutionContext context) { throw new UnsupportedOperationException(); } - }, - FieldMapper.BuilderParams.empty() - ) { + }, FieldMapper.BuilderParams.empty()) { - @Override - protected void parseCreateField(DocumentParserContext context) { - // Run-time fields are mapped to this mapper, so it needs to handle storing values for use in synthetic source. - // #parseValue calls this method once the run-time field is created. - if (context.dynamic() == ObjectMapper.Dynamic.RUNTIME && context.canAddIgnoredField()) { - try { - context.addIgnoredField( - IgnoredSourceFieldMapper.NameValue.fromContext( - context, - context.path().pathAsText(context.parser().currentName()), - XContentDataHelper.encodeToken(context.parser()) - ) - ); - } catch (IOException e) { - throw new IllegalArgumentException("failed to parse run-time field under [" + context.path().pathAsText("") + " ]", e); + @Override + protected void parseCreateField(DocumentParserContext context) { + // Run-time fields are mapped to this mapper, so it needs to handle storing values for use in synthetic source. + // #parseValue calls this method once the run-time field is created. + if (context.dynamic() == ObjectMapper.Dynamic.RUNTIME && context.canAddIgnoredField()) { + try { + context.addIgnoredField( + IgnoredSourceFieldMapper.NameValue.fromContext(context, path, XContentDataHelper.encodeToken(context.parser())) + ); + } catch (IOException e) { + throw new IllegalArgumentException( + "failed to parse run-time field under [" + context.path().pathAsText("") + " ]", + e + ); + } } } - } - @Override - public String fullPath() { - throw new UnsupportedOperationException(); - } + @Override + public String fullPath() { + return path; + } - @Override - public String typeName() { - throw new UnsupportedOperationException(); - } + @Override + public String typeName() { + throw new UnsupportedOperationException(); + } - @Override - public MappedFieldType fieldType() { - throw new UnsupportedOperationException(); - } + @Override + public MappedFieldType fieldType() { + throw new UnsupportedOperationException(); + } - @Override - public MultiFields multiFields() { - throw new UnsupportedOperationException(); - } + @Override + public MultiFields multiFields() { + throw new UnsupportedOperationException(); + } - @Override - public Iterator iterator() { - throw new UnsupportedOperationException(); - } + @Override + public Iterator iterator() { + throw new UnsupportedOperationException(); + } - @Override - protected void doValidate(MappingLookup mappers) { - throw new UnsupportedOperationException(); - } + @Override + protected void doValidate(MappingLookup mappers) { + throw new UnsupportedOperationException(); + } - @Override - protected void checkIncomingMergeType(FieldMapper mergeWith) { - throw new UnsupportedOperationException(); - } + @Override + protected void checkIncomingMergeType(FieldMapper mergeWith) { + throw new UnsupportedOperationException(); + } - @Override - public Builder getMergeBuilder() { - throw new UnsupportedOperationException(); - } + @Override + public Builder getMergeBuilder() { + throw new UnsupportedOperationException(); + } - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) { - throw new UnsupportedOperationException(); - } + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) { + throw new UnsupportedOperationException(); + } - @Override - protected String contentType() { - throw new UnsupportedOperationException(); - } + @Override + protected String contentType() { + throw new UnsupportedOperationException(); + } - @Override - protected SyntheticSourceSupport syntheticSourceSupport() { - // Opt out of fallback synthetic source implementation - // since there is custom logic in #parseCreateField(). - return new SyntheticSourceSupport.Native(SourceLoader.SyntheticFieldLoader.NOTHING); - } - }; + @Override + protected SyntheticSourceSupport syntheticSourceSupport() { + // Opt out of fallback synthetic source implementation + // since there is custom logic in #parseCreateField(). + return new SyntheticSourceSupport.Native(SourceLoader.SyntheticFieldLoader.NOTHING); + } + }; + } private static class NoOpObjectMapper extends ObjectMapper { NoOpObjectMapper(String name, String fullPath) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java index 3a84162b86c27..9fd8dec4a3cb5 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java @@ -116,6 +116,12 @@ public int get() { private Field version; private final SeqNoFieldMapper.SequenceIDFields seqID; private final Set fieldsAppliedFromTemplates; + /** + * Fields that are copied from values of other fields via copy_to. + * This per-document state is needed since it is possible + * that copy_to field in introduced using a dynamic template + * in this document and therefore is not present in mapping yet. + */ private final Set copyToFields; // Indicates if the source for this context has been cloned and gets parsed multiple times. @@ -207,7 +213,7 @@ protected DocumentParserContext( parent, dynamic, new HashSet<>(), - new HashSet<>(), + new HashSet<>(mappingLookup.fieldTypesLookup().getCopyToDestinationFields()), new DynamicMapperSize(), false ); @@ -360,9 +366,23 @@ public boolean isFieldAppliedFromTemplate(String name) { public void markFieldAsCopyTo(String fieldName) { copyToFields.add(fieldName); + if (mappingLookup.isSourceSynthetic()) { + /* + Mark this field as containing copied data meaning it should not be present + in synthetic _source (to be consistent with stored _source). + Ignored source values take precedence over standard synthetic source implementation + so by adding this nothing entry we "disable" field in synthetic source. + Otherwise, it would be constructed f.e. from doc_values which leads to duplicate values + in copied field after reindexing. + + Note that this applies to fields that are copied from fields using ignored source themselves + and therefore we don't check for canAddIgnoredField(). + */ + ignoredFieldValues.add(IgnoredSourceFieldMapper.NameValue.fromContext(this, fieldName, XContentDataHelper.nothing())); + } } - public boolean isCopyToField(String name) { + public boolean isCopyToDestinationField(String name) { return copyToFields.contains(name); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/FieldTypeLookup.java b/server/src/main/java/org/elasticsearch/index/mapper/FieldTypeLookup.java index 65ee587d8cb50..b982612e4e3c0 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/FieldTypeLookup.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/FieldTypeLookup.java @@ -36,6 +36,11 @@ final class FieldTypeLookup { * For convenience, the set of copied fields includes the field itself. */ private final Map> fieldToCopiedFields; + /** + * Fields that are destinations of copy_to meaning fields that + * contain values copied from other fields. + */ + private final Set copyToDestinationFields; private final int maxParentPathDots; @@ -54,6 +59,7 @@ final class FieldTypeLookup { final Map fullSubfieldNameToParentPath = new HashMap<>(); final Map dynamicFieldTypes = new HashMap<>(); final Map> fieldToCopiedFields = new HashMap<>(); + final Set copiedFields = new HashSet<>(); for (FieldMapper fieldMapper : fieldMappers) { String fieldName = fieldMapper.fullPath(); MappedFieldType fieldType = fieldMapper.fieldType(); @@ -63,11 +69,13 @@ final class FieldTypeLookup { dynamicFieldTypes.put(fieldType.name(), (DynamicFieldType) fieldType); } for (String targetField : fieldMapper.copyTo().copyToFields()) { + copiedFields.add(targetField); + Set sourcePath = fieldToCopiedFields.get(targetField); if (sourcePath == null) { - Set copiedFields = new HashSet<>(); - copiedFields.add(targetField); - fieldToCopiedFields.put(targetField, copiedFields); + Set fieldCopiedFields = new HashSet<>(); + fieldCopiedFields.add(targetField); + fieldToCopiedFields.put(targetField, fieldCopiedFields); } fieldToCopiedFields.get(targetField).add(fieldName); } @@ -139,6 +147,7 @@ final class FieldTypeLookup { // make values into more compact immutable sets to save memory fieldToCopiedFields.entrySet().forEach(e -> e.setValue(Set.copyOf(e.getValue()))); this.fieldToCopiedFields = Map.copyOf(fieldToCopiedFields); + this.copyToDestinationFields = Set.copyOf(copiedFields); } public static int dotCount(String path) { @@ -260,4 +269,8 @@ public String parentField(String field) { public Map getFullNameToFieldType() { return fullNameToFieldType; } + + public Set getCopyToDestinationFields() { + return copyToDestinationFields; + } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java index 92412177d35e5..2f080fa97e0bb 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java @@ -36,7 +36,8 @@ public Set getFeatures() { KeywordFieldMapper.KEYWORD_NORMALIZER_SYNTHETIC_SOURCE, SourceFieldMapper.SYNTHETIC_SOURCE_STORED_FIELDS_ADVANCE_FIX, Mapper.SYNTHETIC_SOURCE_KEEP_FEATURE, - SourceFieldMapper.SYNTHETIC_SOURCE_WITH_COPY_TO_AND_DOC_VALUES_FALSE_SUPPORT + SourceFieldMapper.SYNTHETIC_SOURCE_WITH_COPY_TO_AND_DOC_VALUES_FALSE_SUPPORT, + SourceFieldMapper.SYNTHETIC_SOURCE_COPY_TO_FIX ); } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java index 1de4e11a1d3b1..b84218ba9dfb6 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java @@ -46,6 +46,7 @@ public class SourceFieldMapper extends MetadataFieldMapper { public static final NodeFeature SYNTHETIC_SOURCE_WITH_COPY_TO_AND_DOC_VALUES_FALSE_SUPPORT = new NodeFeature( "mapper.source.synthetic_source_with_copy_to_and_doc_values_false" ); + public static final NodeFeature SYNTHETIC_SOURCE_COPY_TO_FIX = new NodeFeature("mapper.source.synthetic_source_copy_to_fix"); public static final String NAME = "_source"; public static final String RECOVERY_SOURCE_NAME = "_recovery_source"; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/XContentDataHelper.java b/server/src/main/java/org/elasticsearch/index/mapper/XContentDataHelper.java index fefafbf13017b..cb7e356595eec 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/XContentDataHelper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/XContentDataHelper.java @@ -70,6 +70,19 @@ public static BytesRef encodeXContentBuilder(XContentBuilder builder) throws IOE return new BytesRef(TypeUtils.encode(builder)); } + /** + * Returns a special encoded value that signals that values of this field + * should not be present in synthetic source. + * + * An example is a field that has values copied to it using copy_to. + * While that field "looks" like a regular field it should not be present in + * synthetic _source same as it wouldn't be present in stored source. + * @return + */ + public static BytesRef nothing() { + return new BytesRef(new byte[] { VOID_ENCODING }); + } + /** * Decode the value in the passed {@link BytesRef} and add it as a value to the * passed build. The assumption is that the passed value has encoded using the function @@ -90,6 +103,7 @@ static void decodeAndWrite(XContentBuilder b, BytesRef r) throws IOException { case DOUBLE_ENCODING -> TypeUtils.DOUBLE.decodeAndWrite(b, r); case FLOAT_ENCODING -> TypeUtils.FLOAT.decodeAndWrite(b, r); case NULL_ENCODING -> TypeUtils.NULL.decodeAndWrite(b, r); + case VOID_ENCODING -> TypeUtils.VOID.decodeAndWrite(b, r); default -> throw new IllegalArgumentException("Can't decode " + r); } } @@ -103,19 +117,35 @@ static void decodeAndWrite(XContentBuilder b, BytesRef r) throws IOException { * @throws IOException */ static void writeMerged(XContentBuilder b, String fieldName, List encodedParts) throws IOException { - if (encodedParts.isEmpty()) { + var partsWithData = 0; + for (BytesRef encodedPart : encodedParts) { + if (isDataPresent(encodedPart)) { + partsWithData++; + } + } + + if (partsWithData == 0) { return; } - if (encodedParts.size() == 1) { + if (partsWithData == 1) { b.field(fieldName); - XContentDataHelper.decodeAndWrite(b, encodedParts.get(0)); + for (BytesRef encodedPart : encodedParts) { + if (isDataPresent(encodedPart)) { + XContentDataHelper.decodeAndWrite(b, encodedPart); + } + } + return; } b.startArray(fieldName); for (var encodedValue : encodedParts) { + if (isDataPresent(encodedValue) == false) { + continue; + } + Optional encodedXContentType = switch ((char) encodedValue.bytes[encodedValue.offset]) { case CBOR_OBJECT_ENCODING, JSON_OBJECT_ENCODING, YAML_OBJECT_ENCODING, SMILE_OBJECT_ENCODING -> Optional.of( getXContentType(encodedValue) @@ -158,6 +188,10 @@ static void writeMerged(XContentBuilder b, String fieldName, List enco b.endArray(); } + public static boolean isDataPresent(BytesRef encoded) { + return encoded.bytes[encoded.offset] != VOID_ENCODING; + } + /** * Returns the {@link XContentType} to use for creating an XContentBuilder to decode the passed value. */ @@ -256,6 +290,7 @@ private static Object processToken(XContentParser parser, CheckedFunction throw new IllegalArgumentException("Can't decode " + r); } } + }, + VOID(VOID_ENCODING) { + @Override + StoredField buildStoredField(String name, XContentParser parser) throws IOException { + return new StoredField(name, encode(parser)); + } + + @Override + byte[] encode(XContentParser parser) { + byte[] bytes = new byte[] { getEncoding() }; + assertValidEncoding(bytes); + return bytes; + } + + @Override + void decodeAndWrite(XContentBuilder b, BytesRef r) { + // NOOP + } }; TypeUtils(char encoding) { diff --git a/server/src/main/java/org/elasticsearch/reservedstate/ReservedClusterStateHandler.java b/server/src/main/java/org/elasticsearch/reservedstate/ReservedClusterStateHandler.java index 92cdf57102f42..0fe533dc9dc85 100644 --- a/server/src/main/java/org/elasticsearch/reservedstate/ReservedClusterStateHandler.java +++ b/server/src/main/java/org/elasticsearch/reservedstate/ReservedClusterStateHandler.java @@ -126,5 +126,5 @@ default void validate(MasterNodeRequest request) { /** * Reserved-state handlers create master-node requests but never actually send them to the master node so the timeouts are not relevant. */ - TimeValue DUMMY_TIMEOUT = TimeValue.THIRTY_SECONDS; + TimeValue RESERVED_CLUSTER_STATE_HANDLER_IGNORED_TIMEOUT = TimeValue.THIRTY_SECONDS; } diff --git a/server/src/main/java/org/elasticsearch/reservedstate/action/ReservedClusterSettingsAction.java b/server/src/main/java/org/elasticsearch/reservedstate/action/ReservedClusterSettingsAction.java index ebc0d5cdf50ec..dff9b5283c67e 100644 --- a/server/src/main/java/org/elasticsearch/reservedstate/action/ReservedClusterSettingsAction.java +++ b/server/src/main/java/org/elasticsearch/reservedstate/action/ReservedClusterSettingsAction.java @@ -14,7 +14,6 @@ import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsUpdater; -import org.elasticsearch.core.TimeValue; import org.elasticsearch.reservedstate.ReservedClusterStateHandler; import org.elasticsearch.reservedstate.TransformState; import org.elasticsearch.xcontent.XContentParser; @@ -60,9 +59,8 @@ private static ClusterUpdateSettingsRequest prepare(Object input, Set pr toDelete.forEach(k -> newSettings.put(k, (String) null)); final ClusterUpdateSettingsRequest clusterUpdateSettingsRequest = new ClusterUpdateSettingsRequest( - // timeouts are unused, any value will do - TimeValue.THIRTY_SECONDS, - TimeValue.THIRTY_SECONDS + RESERVED_CLUSTER_STATE_HANDLER_IGNORED_TIMEOUT, + RESERVED_CLUSTER_STATE_HANDLER_IGNORED_TIMEOUT ); clusterUpdateSettingsRequest.persistentSettings(newSettings); return clusterUpdateSettingsRequest; diff --git a/server/src/test/java/org/elasticsearch/index/mapper/XContentDataHelperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/XContentDataHelperTests.java index a4532bca67778..b0e88e17f6b5b 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/XContentDataHelperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/XContentDataHelperTests.java @@ -186,7 +186,7 @@ public void testWriteMergedWithSingleValue() throws IOException { } private void testWriteMergedWithSingleValue(Object value) throws IOException { - var map = executeWriteMergeOnRepeated(value); + var map = executeWriteMergedOnRepeated(value); assertEquals(Arrays.asList(value, value), map.get("foo")); } @@ -208,7 +208,7 @@ public void testWriteMergedWithMultipleValues() throws IOException { } private void testWriteMergedWithMultipleValues(List value) throws IOException { - var map = executeWriteMergeOnRepeated(value); + var map = executeWriteMergedOnRepeated(value); var expected = Stream.of(value, value).flatMap(Collection::stream).toList(); assertEquals(expected, map.get("foo")); } @@ -233,16 +233,74 @@ public void testWriteMergedWithMixedValues() throws IOException { } private void testWriteMergedWithMixedValues(Object value, List multipleValues) throws IOException { - var map = executeWriteMergeOnTwoEncodedValues(value, multipleValues); + var map = executeWriteMergedOnTwoEncodedValues(value, multipleValues); var expected = Stream.concat(Stream.of(value), multipleValues.stream()).toList(); assertEquals(expected, map.get("foo")); } - private Map executeWriteMergeOnRepeated(Object value) throws IOException { - return executeWriteMergeOnTwoEncodedValues(value, value); + public void testWriteMergedWithVoidValue() throws IOException { + var destination = XContentFactory.contentBuilder(XContentType.JSON); + destination.startObject(); + + XContentDataHelper.writeMerged(destination, "field", List.of(XContentDataHelper.nothing())); + + destination.endObject(); + + assertEquals("{}", Strings.toString(destination)); + } + + public void testWriteMergedWithMultipleVoidValues() throws IOException { + var destination = XContentFactory.contentBuilder(XContentType.JSON); + destination.startObject(); + + XContentDataHelper.writeMerged( + destination, + "field", + List.of(XContentDataHelper.nothing(), XContentDataHelper.nothing(), XContentDataHelper.nothing()) + ); + + destination.endObject(); + + assertEquals("{}", Strings.toString(destination)); + } + + public void testWriteMergedWithMixedVoidValues() throws IOException { + var destination = XContentFactory.contentBuilder(XContentType.JSON); + destination.startObject(); + + var value = XContentFactory.contentBuilder(XContentType.JSON).value(34); + XContentDataHelper.writeMerged( + destination, + "field", + List.of(XContentDataHelper.nothing(), XContentDataHelper.encodeXContentBuilder(value), XContentDataHelper.nothing()) + ); + + destination.endObject(); + + assertEquals("{\"field\":34}", Strings.toString(destination)); + } + + public void testWriteMergedWithArraysAndVoidValues() throws IOException { + var destination = XContentFactory.contentBuilder(XContentType.JSON); + destination.startObject(); + + var value = XContentFactory.contentBuilder(XContentType.JSON).value(List.of(3, 4)); + XContentDataHelper.writeMerged( + destination, + "field", + List.of(XContentDataHelper.nothing(), XContentDataHelper.encodeXContentBuilder(value), XContentDataHelper.nothing()) + ); + + destination.endObject(); + + assertEquals("{\"field\":[3,4]}", Strings.toString(destination)); + } + + private Map executeWriteMergedOnRepeated(Object value) throws IOException { + return executeWriteMergedOnTwoEncodedValues(value, value); } - private Map executeWriteMergeOnTwoEncodedValues(Object first, Object second) throws IOException { + private Map executeWriteMergedOnTwoEncodedValues(Object first, Object second) throws IOException { var xContentType = randomFrom(XContentType.values()); var firstEncoded = encodeSingleValue(first, xContentType); diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/FieldType.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/FieldType.java index 4ed36ea685238..d68f26909c57b 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/FieldType.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/FieldType.java @@ -9,6 +9,7 @@ package org.elasticsearch.logsdb.datageneration; import org.elasticsearch.logsdb.datageneration.datasource.DataSource; +import org.elasticsearch.logsdb.datageneration.datasource.DataSourceResponse; import org.elasticsearch.logsdb.datageneration.fields.leaf.ByteFieldDataGenerator; import org.elasticsearch.logsdb.datageneration.fields.leaf.DoubleFieldDataGenerator; import org.elasticsearch.logsdb.datageneration.fields.leaf.FloatFieldDataGenerator; @@ -35,18 +36,22 @@ public enum FieldType { HALF_FLOAT, SCALED_FLOAT; - public FieldDataGenerator generator(String fieldName, DataSource dataSource) { + public FieldDataGenerator generator( + String fieldName, + DataSource dataSource, + DataSourceResponse.LeafMappingParametersGenerator mappingParametersGenerator + ) { return switch (this) { - case KEYWORD -> new KeywordFieldDataGenerator(fieldName, dataSource); - case LONG -> new LongFieldDataGenerator(fieldName, dataSource); - case UNSIGNED_LONG -> new UnsignedLongFieldDataGenerator(fieldName, dataSource); - case INTEGER -> new IntegerFieldDataGenerator(fieldName, dataSource); - case SHORT -> new ShortFieldDataGenerator(fieldName, dataSource); - case BYTE -> new ByteFieldDataGenerator(fieldName, dataSource); - case DOUBLE -> new DoubleFieldDataGenerator(fieldName, dataSource); - case FLOAT -> new FloatFieldDataGenerator(fieldName, dataSource); - case HALF_FLOAT -> new HalfFloatFieldDataGenerator(fieldName, dataSource); - case SCALED_FLOAT -> new ScaledFloatFieldDataGenerator(fieldName, dataSource); + case KEYWORD -> new KeywordFieldDataGenerator(fieldName, dataSource, mappingParametersGenerator); + case LONG -> new LongFieldDataGenerator(fieldName, dataSource, mappingParametersGenerator); + case UNSIGNED_LONG -> new UnsignedLongFieldDataGenerator(fieldName, dataSource, mappingParametersGenerator); + case INTEGER -> new IntegerFieldDataGenerator(fieldName, dataSource, mappingParametersGenerator); + case SHORT -> new ShortFieldDataGenerator(fieldName, dataSource, mappingParametersGenerator); + case BYTE -> new ByteFieldDataGenerator(fieldName, dataSource, mappingParametersGenerator); + case DOUBLE -> new DoubleFieldDataGenerator(fieldName, dataSource, mappingParametersGenerator); + case FLOAT -> new FloatFieldDataGenerator(fieldName, dataSource, mappingParametersGenerator); + case HALF_FLOAT -> new HalfFloatFieldDataGenerator(fieldName, dataSource, mappingParametersGenerator); + case SCALED_FLOAT -> new ScaledFloatFieldDataGenerator(fieldName, dataSource, mappingParametersGenerator); }; } } diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceRequest.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceRequest.java index fadf51ee3ea10..87e9ce320fcfb 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceRequest.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceRequest.java @@ -12,6 +12,8 @@ import org.elasticsearch.logsdb.datageneration.FieldType; import org.elasticsearch.logsdb.datageneration.fields.DynamicMapping; +import java.util.Set; + public interface DataSourceRequest { TResponse accept(DataSourceHandler handler); @@ -101,7 +103,7 @@ public DataSourceResponse.ObjectArrayGenerator accept(DataSourceHandler handler) } } - record LeafMappingParametersGenerator(String fieldName, FieldType fieldType) + record LeafMappingParametersGenerator(String fieldName, FieldType fieldType, Set eligibleCopyToFields) implements DataSourceRequest { public DataSourceResponse.LeafMappingParametersGenerator accept(DataSourceHandler handler) { diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultMappingParametersHandler.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultMappingParametersHandler.java index 5babaec0cf43a..566c3d679cf6e 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultMappingParametersHandler.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultMappingParametersHandler.java @@ -13,64 +13,55 @@ import java.util.HashMap; import java.util.Map; import java.util.function.Supplier; +import java.util.stream.Collectors; public class DefaultMappingParametersHandler implements DataSourceHandler { @Override public DataSourceResponse.LeafMappingParametersGenerator handle(DataSourceRequest.LeafMappingParametersGenerator request) { + var map = new HashMap(); + map.put("store", ESTestCase.randomBoolean()); + map.put("index", ESTestCase.randomBoolean()); + map.put("doc_values", ESTestCase.randomBoolean()); + return new DataSourceResponse.LeafMappingParametersGenerator(switch (request.fieldType()) { - case KEYWORD -> keywordMapping(); - case LONG, INTEGER, SHORT, BYTE, DOUBLE, FLOAT, HALF_FLOAT -> numberMapping(); - case UNSIGNED_LONG -> unsignedLongMapping(); - case SCALED_FLOAT -> scaledFloatMapping(); + case KEYWORD -> keywordMapping(request, map); + case LONG, INTEGER, SHORT, BYTE, DOUBLE, FLOAT, HALF_FLOAT, UNSIGNED_LONG -> plain(map); + case SCALED_FLOAT -> scaledFloatMapping(map); }); } - private Supplier> keywordMapping() { - return () -> Map.of( - "store", - ESTestCase.randomBoolean(), - "index", - ESTestCase.randomBoolean(), - "doc_values", - ESTestCase.randomBoolean() - ); + private Supplier> plain(Map injected) { + return () -> injected; } - private Supplier> numberMapping() { - return () -> Map.of( - "store", - ESTestCase.randomBoolean(), - "index", - ESTestCase.randomBoolean(), - "doc_values", - ESTestCase.randomBoolean() - ); - } + private Supplier> keywordMapping( + DataSourceRequest.LeafMappingParametersGenerator request, + Map injected + ) { + return () -> { + // Inject copy_to sometimes but reflect that it is not widely used in reality. + // We only add copy_to to keywords because we get into trouble with numeric fields that are copied to dynamic fields. + // If first copied value is numeric, dynamic field is created with numeric field type and then copy of text values fail. + // Actual value being copied does not influence the core logic of copy_to anyway. + if (ESTestCase.randomDouble() <= 0.05) { + var options = request.eligibleCopyToFields() + .stream() + .filter(f -> f.equals(request.fieldName()) == false) + .collect(Collectors.toSet()); - private Supplier> unsignedLongMapping() { - return () -> Map.of( - "store", - ESTestCase.randomBoolean(), - "index", - ESTestCase.randomBoolean(), - "doc_values", - ESTestCase.randomBoolean() - ); + if (options.isEmpty() == false) { + injected.put("copy_to", ESTestCase.randomFrom(options)); + } + } + + return injected; + }; } - private Supplier> scaledFloatMapping() { + private Supplier> scaledFloatMapping(Map injected) { return () -> { - var scalingFactor = ESTestCase.randomFrom(10, 1000, 100000, 100.5); - return Map.of( - "scaling_factor", - scalingFactor, - "store", - ESTestCase.randomBoolean(), - "index", - ESTestCase.randomBoolean(), - "doc_values", - ESTestCase.randomBoolean() - ); + injected.put("scaling_factor", ESTestCase.randomFrom(10, 1000, 100000, 100.5)); + return injected; }; } diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/Context.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/Context.java index ef83ced13f0dc..06aa45eedf670 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/Context.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/Context.java @@ -12,8 +12,10 @@ import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest; import org.elasticsearch.logsdb.datageneration.datasource.DataSourceResponse; +import java.util.HashSet; import java.util.Map; import java.util.Optional; +import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; class Context { @@ -21,26 +23,33 @@ class Context { private final DataSourceResponse.ChildFieldGenerator childFieldGenerator; private final DataSourceResponse.ObjectArrayGenerator objectArrayGenerator; + + private final String path; private final int objectDepth; // We don't need atomicity, but we need to pass counter by reference to accumulate total value from sub-objects. private final AtomicInteger nestedFieldsCount; + private final Set eligibleCopyToDestinations; private final DynamicMapping parentDynamicMapping; Context(DataGeneratorSpecification specification, DynamicMapping parentDynamicMapping) { - this(specification, 0, new AtomicInteger(0), parentDynamicMapping); + this(specification, "", 0, new AtomicInteger(0), new HashSet<>(), parentDynamicMapping); } private Context( DataGeneratorSpecification specification, + String path, int objectDepth, AtomicInteger nestedFieldsCount, + Set eligibleCopyToDestinations, DynamicMapping parentDynamicMapping ) { this.specification = specification; this.childFieldGenerator = specification.dataSource().get(new DataSourceRequest.ChildFieldGenerator(specification)); this.objectArrayGenerator = specification.dataSource().get(new DataSourceRequest.ObjectArrayGenerator()); + this.path = path; this.objectDepth = objectDepth; this.nestedFieldsCount = nestedFieldsCount; + this.eligibleCopyToDestinations = eligibleCopyToDestinations; this.parentDynamicMapping = parentDynamicMapping; } @@ -56,13 +65,21 @@ public DataSourceResponse.FieldTypeGenerator fieldTypeGenerator(DynamicMapping d return specification.dataSource().get(new DataSourceRequest.FieldTypeGenerator(dynamicMapping)); } - public Context subObject(DynamicMapping dynamicMapping) { - return new Context(specification, objectDepth + 1, nestedFieldsCount, dynamicMapping); + public Context subObject(String name, DynamicMapping dynamicMapping) { + return new Context( + specification, + pathToField(name), + objectDepth + 1, + nestedFieldsCount, + eligibleCopyToDestinations, + dynamicMapping + ); } - public Context nestedObject(DynamicMapping dynamicMapping) { + public Context nestedObject(String name, DynamicMapping dynamicMapping) { nestedFieldsCount.incrementAndGet(); - return new Context(specification, objectDepth + 1, nestedFieldsCount, dynamicMapping); + // copy_to can't be used across nested documents so all currently eligible fields are not eligible inside nested document. + return new Context(specification, pathToField(name), objectDepth + 1, nestedFieldsCount, new HashSet<>(), dynamicMapping); } public boolean shouldAddDynamicObjectField(DynamicMapping dynamicMapping) { @@ -112,4 +129,16 @@ public DynamicMapping determineDynamicMapping(Map mappingParamet return dynamicParameter.equals("strict") ? DynamicMapping.FORBIDDEN : DynamicMapping.SUPPORTED; } + + public Set getEligibleCopyToDestinations() { + return eligibleCopyToDestinations; + } + + public void markFieldAsEligibleForCopyTo(String field) { + eligibleCopyToDestinations.add(pathToField(field)); + } + + private String pathToField(String field) { + return path.isEmpty() ? field : path + "." + field; + } } diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/GenericSubObjectFieldDataGenerator.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/GenericSubObjectFieldDataGenerator.java index e8f9724fee269..493b5f3e49f19 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/GenericSubObjectFieldDataGenerator.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/GenericSubObjectFieldDataGenerator.java @@ -10,6 +10,8 @@ import org.elasticsearch.core.CheckedConsumer; import org.elasticsearch.logsdb.datageneration.FieldDataGenerator; +import org.elasticsearch.logsdb.datageneration.FieldType; +import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest; import org.elasticsearch.xcontent.XContentBuilder; import java.io.IOException; @@ -38,14 +40,32 @@ List generateChildFields(DynamicMapping dynamicMapping) { var fieldName = generateFieldName(existingFieldNames); if (context.shouldAddDynamicObjectField(dynamicMapping)) { - result.add(new ChildField(fieldName, new ObjectFieldDataGenerator(context.subObject(DynamicMapping.FORCED)), true)); + result.add( + new ChildField(fieldName, new ObjectFieldDataGenerator(context.subObject(fieldName, DynamicMapping.FORCED)), true) + ); } else if (context.shouldAddObjectField()) { - result.add(new ChildField(fieldName, new ObjectFieldDataGenerator(context.subObject(dynamicMapping)), false)); + result.add(new ChildField(fieldName, new ObjectFieldDataGenerator(context.subObject(fieldName, dynamicMapping)), false)); } else if (context.shouldAddNestedField()) { - result.add(new ChildField(fieldName, new NestedFieldDataGenerator(context.nestedObject(dynamicMapping)), false)); + result.add(new ChildField(fieldName, new NestedFieldDataGenerator(context.nestedObject(fieldName, dynamicMapping)), false)); } else { var fieldTypeInfo = context.fieldTypeGenerator(dynamicMapping).generator().get(); - var generator = fieldTypeInfo.fieldType().generator(fieldName, context.specification().dataSource()); + + // For simplicity we only copy to keyword fields, synthetic source logic to handle copy_to is generic. + if (fieldTypeInfo.fieldType() == FieldType.KEYWORD) { + context.markFieldAsEligibleForCopyTo(fieldName); + } + + var mappingParametersGenerator = context.specification() + .dataSource() + .get( + new DataSourceRequest.LeafMappingParametersGenerator( + fieldName, + fieldTypeInfo.fieldType(), + context.getEligibleCopyToDestinations() + ) + ); + var generator = fieldTypeInfo.fieldType() + .generator(fieldName, context.specification().dataSource(), mappingParametersGenerator); result.add(new ChildField(fieldName, generator, fieldTypeInfo.dynamic())); } } diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/PredefinedField.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/PredefinedField.java index 81b73e4407bd7..c5d9dd5325e33 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/PredefinedField.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/PredefinedField.java @@ -11,6 +11,9 @@ import org.elasticsearch.logsdb.datageneration.FieldDataGenerator; import org.elasticsearch.logsdb.datageneration.FieldType; import org.elasticsearch.logsdb.datageneration.datasource.DataSource; +import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest; + +import java.util.Set; public interface PredefinedField { String name(); @@ -25,7 +28,11 @@ public String name() { @Override public FieldDataGenerator generator(DataSource dataSource) { - return fieldType().generator(fieldName, dataSource); + // copy_to currently not supported for predefined fields, use WithGenerator if needed + var mappingParametersGenerator = dataSource.get( + new DataSourceRequest.LeafMappingParametersGenerator(fieldName, fieldType, Set.of()) + ); + return fieldType().generator(fieldName, dataSource, mappingParametersGenerator); } } diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/ByteFieldDataGenerator.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/ByteFieldDataGenerator.java index d7461294456e9..d1586759996d2 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/ByteFieldDataGenerator.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/ByteFieldDataGenerator.java @@ -10,9 +10,9 @@ import org.elasticsearch.core.CheckedConsumer; import org.elasticsearch.logsdb.datageneration.FieldDataGenerator; -import org.elasticsearch.logsdb.datageneration.FieldType; import org.elasticsearch.logsdb.datageneration.datasource.DataSource; import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest; +import org.elasticsearch.logsdb.datageneration.datasource.DataSourceResponse; import org.elasticsearch.xcontent.XContentBuilder; import java.io.IOException; @@ -23,15 +23,17 @@ public class ByteFieldDataGenerator implements FieldDataGenerator { private final Supplier valueGenerator; private final Map mappingParameters; - public ByteFieldDataGenerator(String fieldName, DataSource dataSource) { + public ByteFieldDataGenerator( + String fieldName, + DataSource dataSource, + DataSourceResponse.LeafMappingParametersGenerator mappingParametersGenerator + ) { var bytes = dataSource.get(new DataSourceRequest.ByteGenerator()); var nulls = dataSource.get(new DataSourceRequest.NullWrapper()); var arrays = dataSource.get(new DataSourceRequest.ArrayWrapper()); this.valueGenerator = arrays.wrapper().compose(nulls.wrapper()).apply(() -> bytes.generator().get()); - this.mappingParameters = dataSource.get(new DataSourceRequest.LeafMappingParametersGenerator(fieldName, FieldType.BYTE)) - .mappingGenerator() - .get(); + this.mappingParameters = mappingParametersGenerator.mappingGenerator().get(); } @Override diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/DoubleFieldDataGenerator.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/DoubleFieldDataGenerator.java index 98d588754ef93..271fcb3e73b5c 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/DoubleFieldDataGenerator.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/DoubleFieldDataGenerator.java @@ -10,9 +10,9 @@ import org.elasticsearch.core.CheckedConsumer; import org.elasticsearch.logsdb.datageneration.FieldDataGenerator; -import org.elasticsearch.logsdb.datageneration.FieldType; import org.elasticsearch.logsdb.datageneration.datasource.DataSource; import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest; +import org.elasticsearch.logsdb.datageneration.datasource.DataSourceResponse; import org.elasticsearch.xcontent.XContentBuilder; import java.io.IOException; @@ -23,15 +23,17 @@ public class DoubleFieldDataGenerator implements FieldDataGenerator { private final Supplier valueGenerator; private final Map mappingParameters; - public DoubleFieldDataGenerator(String fieldName, DataSource dataSource) { + public DoubleFieldDataGenerator( + String fieldName, + DataSource dataSource, + DataSourceResponse.LeafMappingParametersGenerator mappingParametersGenerator + ) { var doubles = dataSource.get(new DataSourceRequest.DoubleGenerator()); var nulls = dataSource.get(new DataSourceRequest.NullWrapper()); var arrays = dataSource.get(new DataSourceRequest.ArrayWrapper()); this.valueGenerator = arrays.wrapper().compose(nulls.wrapper()).apply(() -> doubles.generator().get()); - this.mappingParameters = dataSource.get(new DataSourceRequest.LeafMappingParametersGenerator(fieldName, FieldType.DOUBLE)) - .mappingGenerator() - .get(); + this.mappingParameters = mappingParametersGenerator.mappingGenerator().get(); } @Override diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/FloatFieldDataGenerator.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/FloatFieldDataGenerator.java index 438617ee3c38d..9efdaa5b59302 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/FloatFieldDataGenerator.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/FloatFieldDataGenerator.java @@ -10,9 +10,9 @@ import org.elasticsearch.core.CheckedConsumer; import org.elasticsearch.logsdb.datageneration.FieldDataGenerator; -import org.elasticsearch.logsdb.datageneration.FieldType; import org.elasticsearch.logsdb.datageneration.datasource.DataSource; import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest; +import org.elasticsearch.logsdb.datageneration.datasource.DataSourceResponse; import org.elasticsearch.xcontent.XContentBuilder; import java.io.IOException; @@ -23,15 +23,17 @@ public class FloatFieldDataGenerator implements FieldDataGenerator { private final Supplier valueGenerator; private final Map mappingParameters; - public FloatFieldDataGenerator(String fieldName, DataSource dataSource) { + public FloatFieldDataGenerator( + String fieldName, + DataSource dataSource, + DataSourceResponse.LeafMappingParametersGenerator mappingParametersGenerator + ) { var floats = dataSource.get(new DataSourceRequest.FloatGenerator()); var nulls = dataSource.get(new DataSourceRequest.NullWrapper()); var arrays = dataSource.get(new DataSourceRequest.ArrayWrapper()); this.valueGenerator = arrays.wrapper().compose(nulls.wrapper()).apply(() -> floats.generator().get()); - this.mappingParameters = dataSource.get(new DataSourceRequest.LeafMappingParametersGenerator(fieldName, FieldType.FLOAT)) - .mappingGenerator() - .get(); + this.mappingParameters = mappingParametersGenerator.mappingGenerator().get(); } @Override diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/HalfFloatFieldDataGenerator.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/HalfFloatFieldDataGenerator.java index b21467a2afdcc..dfbcf0816a3b8 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/HalfFloatFieldDataGenerator.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/HalfFloatFieldDataGenerator.java @@ -10,9 +10,9 @@ import org.elasticsearch.core.CheckedConsumer; import org.elasticsearch.logsdb.datageneration.FieldDataGenerator; -import org.elasticsearch.logsdb.datageneration.FieldType; import org.elasticsearch.logsdb.datageneration.datasource.DataSource; import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest; +import org.elasticsearch.logsdb.datageneration.datasource.DataSourceResponse; import org.elasticsearch.xcontent.XContentBuilder; import java.io.IOException; @@ -23,15 +23,17 @@ public class HalfFloatFieldDataGenerator implements FieldDataGenerator { private final Supplier valueGenerator; private final Map mappingParameters; - public HalfFloatFieldDataGenerator(String fieldName, DataSource dataSource) { + public HalfFloatFieldDataGenerator( + String fieldName, + DataSource dataSource, + DataSourceResponse.LeafMappingParametersGenerator mappingParametersGenerator + ) { var halfFloats = dataSource.get(new DataSourceRequest.HalfFloatGenerator()); var nulls = dataSource.get(new DataSourceRequest.NullWrapper()); var arrays = dataSource.get(new DataSourceRequest.ArrayWrapper()); this.valueGenerator = arrays.wrapper().compose(nulls.wrapper()).apply(() -> halfFloats.generator().get()); - this.mappingParameters = dataSource.get(new DataSourceRequest.LeafMappingParametersGenerator(fieldName, FieldType.HALF_FLOAT)) - .mappingGenerator() - .get(); + this.mappingParameters = mappingParametersGenerator.mappingGenerator().get(); } @Override diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/IntegerFieldDataGenerator.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/IntegerFieldDataGenerator.java index 06f5f12fceac3..e5e77ef822546 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/IntegerFieldDataGenerator.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/IntegerFieldDataGenerator.java @@ -10,9 +10,9 @@ import org.elasticsearch.core.CheckedConsumer; import org.elasticsearch.logsdb.datageneration.FieldDataGenerator; -import org.elasticsearch.logsdb.datageneration.FieldType; import org.elasticsearch.logsdb.datageneration.datasource.DataSource; import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest; +import org.elasticsearch.logsdb.datageneration.datasource.DataSourceResponse; import org.elasticsearch.xcontent.XContentBuilder; import java.io.IOException; @@ -23,15 +23,17 @@ public class IntegerFieldDataGenerator implements FieldDataGenerator { private final Supplier valueGenerator; private final Map mappingParameters; - public IntegerFieldDataGenerator(String fieldName, DataSource dataSource) { + public IntegerFieldDataGenerator( + String fieldName, + DataSource dataSource, + DataSourceResponse.LeafMappingParametersGenerator mappingParametersGenerator + ) { var ints = dataSource.get(new DataSourceRequest.IntegerGenerator()); var nulls = dataSource.get(new DataSourceRequest.NullWrapper()); var arrays = dataSource.get(new DataSourceRequest.ArrayWrapper()); this.valueGenerator = arrays.wrapper().compose(nulls.wrapper()).apply(() -> ints.generator().get()); - this.mappingParameters = dataSource.get(new DataSourceRequest.LeafMappingParametersGenerator(fieldName, FieldType.INTEGER)) - .mappingGenerator() - .get(); + this.mappingParameters = mappingParametersGenerator.mappingGenerator().get(); } @Override diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/KeywordFieldDataGenerator.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/KeywordFieldDataGenerator.java index a33b4ce1b2365..73cd4458aa83d 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/KeywordFieldDataGenerator.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/KeywordFieldDataGenerator.java @@ -10,9 +10,9 @@ import org.elasticsearch.core.CheckedConsumer; import org.elasticsearch.logsdb.datageneration.FieldDataGenerator; -import org.elasticsearch.logsdb.datageneration.FieldType; import org.elasticsearch.logsdb.datageneration.datasource.DataSource; import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest; +import org.elasticsearch.logsdb.datageneration.datasource.DataSourceResponse; import org.elasticsearch.xcontent.XContentBuilder; import java.io.IOException; @@ -23,15 +23,17 @@ public class KeywordFieldDataGenerator implements FieldDataGenerator { private final Supplier valueGenerator; private final Map mappingParameters; - public KeywordFieldDataGenerator(String fieldName, DataSource dataSource) { + public KeywordFieldDataGenerator( + String fieldName, + DataSource dataSource, + DataSourceResponse.LeafMappingParametersGenerator mappingParametersGenerator + ) { var strings = dataSource.get(new DataSourceRequest.StringGenerator()); var nulls = dataSource.get(new DataSourceRequest.NullWrapper()); var arrays = dataSource.get(new DataSourceRequest.ArrayWrapper()); this.valueGenerator = arrays.wrapper().compose(nulls.wrapper()).apply(() -> strings.generator().get()); - this.mappingParameters = dataSource.get(new DataSourceRequest.LeafMappingParametersGenerator(fieldName, FieldType.KEYWORD)) - .mappingGenerator() - .get(); + this.mappingParameters = mappingParametersGenerator.mappingGenerator().get(); } @Override diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/LongFieldDataGenerator.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/LongFieldDataGenerator.java index 6f6fc5db253aa..4edb57c4597c4 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/LongFieldDataGenerator.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/LongFieldDataGenerator.java @@ -10,9 +10,9 @@ import org.elasticsearch.core.CheckedConsumer; import org.elasticsearch.logsdb.datageneration.FieldDataGenerator; -import org.elasticsearch.logsdb.datageneration.FieldType; import org.elasticsearch.logsdb.datageneration.datasource.DataSource; import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest; +import org.elasticsearch.logsdb.datageneration.datasource.DataSourceResponse; import org.elasticsearch.xcontent.XContentBuilder; import java.io.IOException; @@ -23,15 +23,17 @@ public class LongFieldDataGenerator implements FieldDataGenerator { private final Supplier valueGenerator; private final Map mappingParameters; - public LongFieldDataGenerator(String fieldName, DataSource dataSource) { + public LongFieldDataGenerator( + String fieldName, + DataSource dataSource, + DataSourceResponse.LeafMappingParametersGenerator mappingParametersGenerator + ) { var longs = dataSource.get(new DataSourceRequest.LongGenerator()); var nulls = dataSource.get(new DataSourceRequest.NullWrapper()); var arrays = dataSource.get(new DataSourceRequest.ArrayWrapper()); this.valueGenerator = arrays.wrapper().compose(nulls.wrapper()).apply(() -> longs.generator().get()); - this.mappingParameters = dataSource.get(new DataSourceRequest.LeafMappingParametersGenerator(fieldName, FieldType.LONG)) - .mappingGenerator() - .get(); + this.mappingParameters = mappingParametersGenerator.mappingGenerator().get(); } @Override diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/ScaledFloatFieldDataGenerator.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/ScaledFloatFieldDataGenerator.java index fb50933f74d70..a6cc27cf06689 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/ScaledFloatFieldDataGenerator.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/ScaledFloatFieldDataGenerator.java @@ -10,9 +10,9 @@ import org.elasticsearch.core.CheckedConsumer; import org.elasticsearch.logsdb.datageneration.FieldDataGenerator; -import org.elasticsearch.logsdb.datageneration.FieldType; import org.elasticsearch.logsdb.datageneration.datasource.DataSource; import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest; +import org.elasticsearch.logsdb.datageneration.datasource.DataSourceResponse; import org.elasticsearch.xcontent.XContentBuilder; import java.io.IOException; @@ -23,15 +23,17 @@ public class ScaledFloatFieldDataGenerator implements FieldDataGenerator { private final Supplier valueGenerator; private final Map mappingParameters; - public ScaledFloatFieldDataGenerator(String fieldName, DataSource dataSource) { + public ScaledFloatFieldDataGenerator( + String fieldName, + DataSource dataSource, + DataSourceResponse.LeafMappingParametersGenerator mappingParametersGenerator + ) { var doubles = dataSource.get(new DataSourceRequest.DoubleGenerator()); var nulls = dataSource.get(new DataSourceRequest.NullWrapper()); var arrays = dataSource.get(new DataSourceRequest.ArrayWrapper()); this.valueGenerator = arrays.wrapper().compose(nulls.wrapper()).apply(() -> doubles.generator().get()); - this.mappingParameters = dataSource.get(new DataSourceRequest.LeafMappingParametersGenerator(fieldName, FieldType.SCALED_FLOAT)) - .mappingGenerator() - .get(); + this.mappingParameters = mappingParametersGenerator.mappingGenerator().get(); } @Override diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/ShortFieldDataGenerator.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/ShortFieldDataGenerator.java index 43d390c0857b9..ab94586cc9802 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/ShortFieldDataGenerator.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/ShortFieldDataGenerator.java @@ -10,9 +10,9 @@ import org.elasticsearch.core.CheckedConsumer; import org.elasticsearch.logsdb.datageneration.FieldDataGenerator; -import org.elasticsearch.logsdb.datageneration.FieldType; import org.elasticsearch.logsdb.datageneration.datasource.DataSource; import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest; +import org.elasticsearch.logsdb.datageneration.datasource.DataSourceResponse; import org.elasticsearch.xcontent.XContentBuilder; import java.io.IOException; @@ -23,15 +23,17 @@ public class ShortFieldDataGenerator implements FieldDataGenerator { private final Supplier valueGenerator; private final Map mappingParameters; - public ShortFieldDataGenerator(String fieldName, DataSource dataSource) { + public ShortFieldDataGenerator( + String fieldName, + DataSource dataSource, + DataSourceResponse.LeafMappingParametersGenerator mappingParametersGenerator + ) { var shorts = dataSource.get(new DataSourceRequest.ShortGenerator()); var nulls = dataSource.get(new DataSourceRequest.NullWrapper()); var arrays = dataSource.get(new DataSourceRequest.ArrayWrapper()); this.valueGenerator = arrays.wrapper().compose(nulls.wrapper()).apply(() -> shorts.generator().get()); - this.mappingParameters = dataSource.get(new DataSourceRequest.LeafMappingParametersGenerator(fieldName, FieldType.SHORT)) - .mappingGenerator() - .get(); + this.mappingParameters = mappingParametersGenerator.mappingGenerator().get(); } @Override diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/UnsignedLongFieldDataGenerator.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/UnsignedLongFieldDataGenerator.java index 7e833d6486e52..a88902709d664 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/UnsignedLongFieldDataGenerator.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/UnsignedLongFieldDataGenerator.java @@ -10,9 +10,9 @@ import org.elasticsearch.core.CheckedConsumer; import org.elasticsearch.logsdb.datageneration.FieldDataGenerator; -import org.elasticsearch.logsdb.datageneration.FieldType; import org.elasticsearch.logsdb.datageneration.datasource.DataSource; import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest; +import org.elasticsearch.logsdb.datageneration.datasource.DataSourceResponse; import org.elasticsearch.xcontent.XContentBuilder; import java.io.IOException; @@ -23,15 +23,17 @@ public class UnsignedLongFieldDataGenerator implements FieldDataGenerator { private final Supplier valueGenerator; private final Map mappingParameters; - public UnsignedLongFieldDataGenerator(String fieldName, DataSource dataSource) { + public UnsignedLongFieldDataGenerator( + String fieldName, + DataSource dataSource, + DataSourceResponse.LeafMappingParametersGenerator mappingParametersGenerator + ) { var unsignedLongs = dataSource.get(new DataSourceRequest.UnsignedLongGenerator()); var nulls = dataSource.get(new DataSourceRequest.NullWrapper()); var arrays = dataSource.get(new DataSourceRequest.ArrayWrapper()); this.valueGenerator = arrays.wrapper().compose(nulls.wrapper()).apply(() -> unsignedLongs.generator().get()); - this.mappingParameters = dataSource.get(new DataSourceRequest.LeafMappingParametersGenerator(fieldName, FieldType.UNSIGNED_LONG)) - .mappingGenerator() - .get(); + this.mappingParameters = mappingParametersGenerator.mappingGenerator().get(); } @Override diff --git a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/action/ReservedAutoscalingPolicyAction.java b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/action/ReservedAutoscalingPolicyAction.java index 2a064afc591ce..ac09630f1bc9f 100644 --- a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/action/ReservedAutoscalingPolicyAction.java +++ b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/action/ReservedAutoscalingPolicyAction.java @@ -8,7 +8,6 @@ package org.elasticsearch.xpack.autoscaling.action; import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.core.TimeValue; import org.elasticsearch.reservedstate.ReservedClusterStateHandler; import org.elasticsearch.reservedstate.TransformState; import org.elasticsearch.xcontent.XContentParser; @@ -96,8 +95,8 @@ public List fromXContent(XContentParser pars PutAutoscalingPolicyAction.Request.parse( policyParser, (roles, deciders) -> new PutAutoscalingPolicyAction.Request( - TimeValue.MINUS_ONE, - TimeValue.MINUS_ONE, + RESERVED_CLUSTER_STATE_HANDLER_IGNORED_TIMEOUT, + RESERVED_CLUSTER_STATE_HANDLER_IGNORED_TIMEOUT, name, roles, deciders diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/ReservedLifecycleAction.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/ReservedLifecycleAction.java index ba41e29531b76..d68e83814915e 100644 --- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/ReservedLifecycleAction.java +++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/ReservedLifecycleAction.java @@ -10,7 +10,6 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.client.internal.Client; import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.core.TimeValue; import org.elasticsearch.license.XPackLicenseState; import org.elasticsearch.reservedstate.ReservedClusterStateHandler; import org.elasticsearch.reservedstate.TransformState; @@ -64,8 +63,11 @@ public Collection prepare(Object input) throws IOException List policies = (List) input; for (var policy : policies) { - // timeouts don't matter here - PutLifecycleRequest request = new PutLifecycleRequest(TimeValue.THIRTY_SECONDS, TimeValue.THIRTY_SECONDS, policy); + PutLifecycleRequest request = new PutLifecycleRequest( + RESERVED_CLUSTER_STATE_HANDLER_IGNORED_TIMEOUT, + RESERVED_CLUSTER_STATE_HANDLER_IGNORED_TIMEOUT, + policy + ); validate(request); result.add(request); } @@ -97,8 +99,11 @@ public TransformState transform(Object source, TransformState prevState) throws for (var policyToDelete : toDelete) { TransportDeleteLifecycleAction.DeleteLifecyclePolicyTask task = new TransportDeleteLifecycleAction.DeleteLifecyclePolicyTask( - // timeouts don't matter here - new DeleteLifecycleAction.Request(TimeValue.THIRTY_SECONDS, TimeValue.THIRTY_SECONDS, policyToDelete), + new DeleteLifecycleAction.Request( + RESERVED_CLUSTER_STATE_HANDLER_IGNORED_TIMEOUT, + RESERVED_CLUSTER_STATE_HANDLER_IGNORED_TIMEOUT, + policyToDelete + ), ActionListener.noop() ); state = task.execute(state); diff --git a/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/ReservedSnapshotAction.java b/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/ReservedSnapshotAction.java index 192b03aa385d5..a98e110ed88de 100644 --- a/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/ReservedSnapshotAction.java +++ b/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/ReservedSnapshotAction.java @@ -9,7 +9,6 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.core.TimeValue; import org.elasticsearch.features.FeatureService; import org.elasticsearch.reservedstate.ReservedClusterStateHandler; import org.elasticsearch.reservedstate.TransformState; @@ -59,10 +58,9 @@ private Collection prepare(List exceptions = new ArrayList<>(); for (var policy : policies) { - // timeouts don't matter here PutSnapshotLifecycleAction.Request request = new PutSnapshotLifecycleAction.Request( - TimeValue.THIRTY_SECONDS, - TimeValue.THIRTY_SECONDS, + RESERVED_CLUSTER_STATE_HANDLER_IGNORED_TIMEOUT, + RESERVED_CLUSTER_STATE_HANDLER_IGNORED_TIMEOUT, policy.getId(), policy ); @@ -106,9 +104,12 @@ public TransformState transform(Object source, TransformState prevState) throws toDelete.removeAll(entities); for (var policyToDelete : toDelete) { - // timeouts don't matter here var task = new TransportDeleteSnapshotLifecycleAction.DeleteSnapshotPolicyTask( - new DeleteSnapshotLifecycleAction.Request(TimeValue.THIRTY_SECONDS, TimeValue.THIRTY_SECONDS, policyToDelete), + new DeleteSnapshotLifecycleAction.Request( + RESERVED_CLUSTER_STATE_HANDLER_IGNORED_TIMEOUT, + RESERVED_CLUSTER_STATE_HANDLER_IGNORED_TIMEOUT, + policyToDelete + ), ActionListener.noop() ); state = task.execute(state);