Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Transform] Rename interim_results to align_checkpoints #76609

Merged
merged 3 commits into from
Aug 18, 2021
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -25,20 +25,20 @@ public class SettingsConfig implements ToXContentObject {
private static final ParseField MAX_PAGE_SEARCH_SIZE = new ParseField("max_page_search_size");
private static final ParseField DOCS_PER_SECOND = new ParseField("docs_per_second");
private static final ParseField DATES_AS_EPOCH_MILLIS = new ParseField("dates_as_epoch_millis");
private static final ParseField INTERIM_RESULTS = new ParseField("interim_results");
private static final ParseField ALIGN_CHECKPOINTS = new ParseField("align_checkpoints");
private static final int DEFAULT_MAX_PAGE_SEARCH_SIZE = -1;
private static final float DEFAULT_DOCS_PER_SECOND = -1F;

// use an integer as we need to code 4 states: true, false, null (unchanged), default (defined server side)
private static final int DEFAULT_DATES_AS_EPOCH_MILLIS = -1;

// use an integer as we need to code 4 states: true, false, null (unchanged), default (defined server side)
private static final int DEFAULT_INTERIM_RESULTS = -1;
private static final int DEFAULT_ALIGN_CHECKPOINTS = -1;

private final Integer maxPageSearchSize;
private final Float docsPerSecond;
private final Integer datesAsEpochMillis;
private final Integer interimResults;
private final Integer alignCheckpoints;

private static final ConstructingObjectParser<SettingsConfig, Void> PARSER = new ConstructingObjectParser<>(
"settings_config",
Expand All @@ -59,8 +59,8 @@ public class SettingsConfig implements ToXContentObject {
// this boolean requires 4 possible values: true, false, not_specified, default, therefore using a custom parser
PARSER.declareField(
optionalConstructorArg(),
p -> p.currentToken() == XContentParser.Token.VALUE_NULL ? DEFAULT_INTERIM_RESULTS : p.booleanValue() ? 1 : 0,
INTERIM_RESULTS,
p -> p.currentToken() == XContentParser.Token.VALUE_NULL ? DEFAULT_ALIGN_CHECKPOINTS : p.booleanValue() ? 1 : 0,
ALIGN_CHECKPOINTS,
ValueType.BOOLEAN_OR_NULL
);
}
Expand All @@ -69,11 +69,11 @@ public static SettingsConfig fromXContent(final XContentParser parser) {
return PARSER.apply(parser, null);
}

SettingsConfig(Integer maxPageSearchSize, Float docsPerSecond, Integer datesAsEpochMillis, Integer interimResults) {
SettingsConfig(Integer maxPageSearchSize, Float docsPerSecond, Integer datesAsEpochMillis, Integer alignCheckpoints) {
this.maxPageSearchSize = maxPageSearchSize;
this.docsPerSecond = docsPerSecond;
this.datesAsEpochMillis = datesAsEpochMillis;
this.interimResults = interimResults;
this.alignCheckpoints = alignCheckpoints;
}

@Override
Expand All @@ -100,11 +100,11 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
builder.field(DATES_AS_EPOCH_MILLIS.getPreferredName(), datesAsEpochMillis > 0 ? true : false);
}
}
if (interimResults != null) {
if (interimResults.equals(DEFAULT_INTERIM_RESULTS)) {
builder.field(INTERIM_RESULTS.getPreferredName(), (Boolean) null);
if (alignCheckpoints != null) {
if (alignCheckpoints.equals(DEFAULT_ALIGN_CHECKPOINTS)) {
builder.field(ALIGN_CHECKPOINTS.getPreferredName(), (Boolean) null);
} else {
builder.field(INTERIM_RESULTS.getPreferredName(), interimResults > 0 ? true : false);
builder.field(ALIGN_CHECKPOINTS.getPreferredName(), alignCheckpoints > 0 ? true : false);
}
}
builder.endObject();
Expand All @@ -123,8 +123,8 @@ public Boolean getDatesAsEpochMillis() {
return datesAsEpochMillis != null ? datesAsEpochMillis > 0 : null;
}

public Boolean getInterimResults() {
return interimResults != null ? interimResults > 0 : null;
public Boolean getAlignCheckpoints() {
return alignCheckpoints != null ? alignCheckpoints > 0 : null;
}

@Override
Expand All @@ -140,12 +140,12 @@ public boolean equals(Object other) {
return Objects.equals(maxPageSearchSize, that.maxPageSearchSize)
&& Objects.equals(docsPerSecond, that.docsPerSecond)
&& Objects.equals(datesAsEpochMillis, that.datesAsEpochMillis)
&& Objects.equals(interimResults, that.interimResults);
&& Objects.equals(alignCheckpoints, that.alignCheckpoints);
}

@Override
public int hashCode() {
return Objects.hash(maxPageSearchSize, docsPerSecond, datesAsEpochMillis, interimResults);
return Objects.hash(maxPageSearchSize, docsPerSecond, datesAsEpochMillis, alignCheckpoints);
}

public static Builder builder() {
Expand All @@ -156,7 +156,7 @@ public static class Builder {
private Integer maxPageSearchSize;
private Float docsPerSecond;
private Integer datesAsEpochMillis;
private Integer interimResults;
private Integer alignCheckpoints;

/**
* Sets the paging maximum paging maxPageSearchSize that transform can use when
Expand Down Expand Up @@ -203,20 +203,20 @@ public Builder setDatesAsEpochMillis(Boolean datesAsEpochMillis) {
}

/**
* Whether to write interim results in transform checkpoints.
* Whether to align transform checkpoint ranges with date histogram interval.
*
* An explicit `null` resets to default.
*
* @param interimResults true if interim results should be written.
* @return the {@link Builder} with interimResults set.
* @param alignCheckpoints true if checkpoint ranges should be aligned with date histogram interval.
* @return the {@link Builder} with alignCheckpoints set.
*/
public Builder setInterimResults(Boolean interimResults) {
this.interimResults = interimResults == null ? DEFAULT_INTERIM_RESULTS : interimResults ? 1 : 0;
public Builder setAlignCheckpoints(Boolean alignCheckpoints) {
this.alignCheckpoints = alignCheckpoints == null ? DEFAULT_ALIGN_CHECKPOINTS : alignCheckpoints ? 1 : 0;
return this;
}

public SettingsConfig build() {
return new SettingsConfig(maxPageSearchSize, docsPerSecond, datesAsEpochMillis, interimResults);
return new SettingsConfig(maxPageSearchSize, docsPerSecond, datesAsEpochMillis, alignCheckpoints);
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,7 @@ public void testExplicitNullOnWriteParser() throws IOException {
assertThat(settingsAsMap.getOrDefault("max_page_search_size", "not_set"), equalTo("not_set"));
assertNull(settingsAsMap.getOrDefault("docs_per_second", "not_set"));
assertThat(settingsAsMap.getOrDefault("dates_as_epoch_millis", "not_set"), equalTo("not_set"));
assertThat(settingsAsMap.getOrDefault("interim_results", "not_set"), equalTo("not_set"));
assertThat(settingsAsMap.getOrDefault("align_checkpoints", "not_set"), equalTo("not_set"));

config = fromString("{\"dates_as_epoch_millis\" : null}");
assertFalse(config.getDatesAsEpochMillis());
Expand All @@ -82,16 +82,16 @@ public void testExplicitNullOnWriteParser() throws IOException {
assertThat(settingsAsMap.getOrDefault("max_page_search_size", "not_set"), equalTo("not_set"));
assertThat(settingsAsMap.getOrDefault("docs_per_second", "not_set"), equalTo("not_set"));
assertNull(settingsAsMap.getOrDefault("dates_as_epoch_millis", "not_set"));
assertThat(settingsAsMap.getOrDefault("interim_results", "not_set"), equalTo("not_set"));
assertThat(settingsAsMap.getOrDefault("align_checkpoints", "not_set"), equalTo("not_set"));

config = fromString("{\"interim_results\" : null}");
assertFalse(config.getInterimResults());
config = fromString("{\"align_checkpoints\" : null}");
assertFalse(config.getAlignCheckpoints());

settingsAsMap = xContentToMap(config);
assertThat(settingsAsMap.getOrDefault("max_page_search_size", "not_set"), equalTo("not_set"));
assertThat(settingsAsMap.getOrDefault("docs_per_second", "not_set"), equalTo("not_set"));
assertThat(settingsAsMap.getOrDefault("dates_as_epoch_millis", "not_set"), equalTo("not_set"));
assertNull(settingsAsMap.getOrDefault("interim_results", "not_set"));
assertNull(settingsAsMap.getOrDefault("align_checkpoints", "not_set"));
}

public void testExplicitNullOnWriteBuilder() throws IOException {
Expand All @@ -103,12 +103,12 @@ public void testExplicitNullOnWriteBuilder() throws IOException {
assertNull(settingsAsMap.getOrDefault("max_page_search_size", "not_set"));
assertThat(settingsAsMap.getOrDefault("docs_per_second", "not_set"), equalTo("not_set"));
assertThat(settingsAsMap.getOrDefault("dates_as_epoch_millis", "not_set"), equalTo("not_set"));
assertThat(settingsAsMap.getOrDefault("interim_results", "not_set"), equalTo("not_set"));
assertThat(settingsAsMap.getOrDefault("align_checkpoints", "not_set"), equalTo("not_set"));

SettingsConfig emptyConfig = new SettingsConfig.Builder().build();
assertNull(emptyConfig.getMaxPageSearchSize());
assertNull(emptyConfig.getDatesAsEpochMillis());
assertNull(emptyConfig.getInterimResults());
assertNull(emptyConfig.getAlignCheckpoints());

settingsAsMap = xContentToMap(emptyConfig);
assertTrue(settingsAsMap.isEmpty());
Expand All @@ -120,7 +120,7 @@ public void testExplicitNullOnWriteBuilder() throws IOException {
assertThat(settingsAsMap.getOrDefault("max_page_search_size", "not_set"), equalTo("not_set"));
assertNull(settingsAsMap.getOrDefault("docs_per_second", "not_set"));
assertThat(settingsAsMap.getOrDefault("dates_as_epoch_millis", "not_set"), equalTo("not_set"));
assertThat(settingsAsMap.getOrDefault("interim_results", "not_set"), equalTo("not_set"));
assertThat(settingsAsMap.getOrDefault("align_checkpoints", "not_set"), equalTo("not_set"));

config = new SettingsConfig.Builder().setDatesAsEpochMillis(null).build();
// returns false, however it's `null` as in "use default", checked next
Expand All @@ -130,17 +130,17 @@ public void testExplicitNullOnWriteBuilder() throws IOException {
assertThat(settingsAsMap.getOrDefault("max_page_search_size", "not_set"), equalTo("not_set"));
assertThat(settingsAsMap.getOrDefault("docs_per_second", "not_set"), equalTo("not_set"));
assertNull(settingsAsMap.getOrDefault("dates_as_epoch_millis", "not_set"));
assertThat(settingsAsMap.getOrDefault("interim_results", "not_set"), equalTo("not_set"));
assertThat(settingsAsMap.getOrDefault("align_checkpoints", "not_set"), equalTo("not_set"));

config = new SettingsConfig.Builder().setInterimResults(null).build();
config = new SettingsConfig.Builder().setAlignCheckpoints(null).build();
// returns false, however it's `null` as in "use default", checked next
assertFalse(config.getInterimResults());
assertFalse(config.getAlignCheckpoints());

settingsAsMap = xContentToMap(config);
assertThat(settingsAsMap.getOrDefault("max_page_search_size", "not_set"), equalTo("not_set"));
assertThat(settingsAsMap.getOrDefault("docs_per_second", "not_set"), equalTo("not_set"));
assertThat(settingsAsMap.getOrDefault("dates_as_epoch_millis", "not_set"), equalTo("not_set"));
assertNull(settingsAsMap.getOrDefault("interim_results", "not_set"));
assertNull(settingsAsMap.getOrDefault("align_checkpoints", "not_set"));
}

private Map<String, Object> xContentToMap(ToXContent xcontent) throws IOException {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ public static void assertHlrcEquals(
assertEquals(serverTestInstance.getMaxPageSearchSize(), clientInstance.getMaxPageSearchSize());
assertEquals(serverTestInstance.getDocsPerSecond(), clientInstance.getDocsPerSecond());
assertEquals(serverTestInstance.getDatesAsEpochMillis(), clientInstance.getDatesAsEpochMillis());
assertEquals(serverTestInstance.getInterimResults(), clientInstance.getInterimResults());
assertEquals(serverTestInstance.getAlignCheckpoints(), clientInstance.getAlignCheckpoints());
}

@Override
Expand Down
8 changes: 8 additions & 0 deletions docs/reference/rest-api/common-parms.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -1005,6 +1005,14 @@ throttles the {transform} by adding a wait time between search requests. The
default value is `null`, which disables throttling.
end::transform-settings-docs-per-second[]

tag::transform-settings-align-checkpoints[]
Specifies whether the transform checkpoint ranges should be optimized for performance.
Such optimization can align checkpoint ranges with date histogram interval when date histogram
is specified as a group source in the transform config. As an effect, less document updates in the
destination index will be performed thus improving overall performance.
The default value is `true`, which means the checkpoint ranges will be optimized if possible.
end::transform-settings-align-checkpoints[]

tag::transform-settings-max-page-search-size[]
Defines the initial page size to use for the composite aggregation for each
checkpoint. If circuit breaker exceptions occur, the page size is dynamically
Expand Down
3 changes: 3 additions & 0 deletions docs/reference/transform/apis/preview-transform.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -204,6 +204,9 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-settings]
`docs_per_second`:::
(Optional, float)
include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-settings-docs-per-second]
`align_checkpoints`:::
(Optional, boolean)
include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-settings-align-checkpoints]
`max_page_search_size`:::
(Optional, integer)
include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-settings-max-page-search-size]
Expand Down
3 changes: 3 additions & 0 deletions docs/reference/transform/apis/put-transform.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -186,6 +186,9 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-settings-dat
`docs_per_second`:::
(Optional, float)
include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-settings-docs-per-second]
`align_checkpoints`:::
(Optional, boolean)
include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-settings-align-checkpoints]
`max_page_search_size`:::
(Optional, integer)
include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-settings-max-page-search-size]
Expand Down
3 changes: 3 additions & 0 deletions docs/reference/transform/apis/update-transform.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -138,6 +138,9 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-settings-dat
`docs_per_second`:::
(Optional, float)
include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-settings-docs-per-second]
`align_checkpoints`:::
(Optional, boolean)
include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-settings-align-checkpoints]
`max_page_search_size`:::
(Optional, integer)
include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-settings-max-page-search-size]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ public final class TransformField {
public static final ParseField MAX_PAGE_SEARCH_SIZE = new ParseField("max_page_search_size");
public static final ParseField DOCS_PER_SECOND = new ParseField("docs_per_second");
public static final ParseField DATES_AS_EPOCH_MILLIS = new ParseField("dates_as_epoch_millis");
public static final ParseField INTERIM_RESULTS = new ParseField("interim_results");
public static final ParseField ALIGN_CHECKPOINTS = new ParseField("align_checkpoints");
public static final ParseField FIELD = new ParseField("field");
public static final ParseField SYNC = new ParseField("sync");
public static final ParseField TIME = new ParseField("time");
Expand Down
Loading