Skip to content

Commit

Permalink
Merge remote-tracking branch 'upstream/main' into 13302
Browse files Browse the repository at this point in the history
Signed-off-by: Ashish Singh <ssashish@amazon.com>
  • Loading branch information
ashking94 committed Apr 26, 2024
2 parents bd3ab77 + a69cd08 commit e92892c
Show file tree
Hide file tree
Showing 30 changed files with 2,059 additions and 143 deletions.
3 changes: 3 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
- [Streaming Indexing] Ensure support of the new transport by security plugin ([#13174](https://github.com/opensearch-project/OpenSearch/pull/13174))
- Add cluster setting to dynamically configure the buckets for filter rewrite optimization. ([#13179](https://github.com/opensearch-project/OpenSearch/pull/13179))
- [Remote Store] Add capability of doing refresh as determined by the translog ([#12992](https://github.com/opensearch-project/OpenSearch/pull/12992))
- Batch mode for async fetching shard information in GatewayAllocator for unassigned shards ([#8746](https://github.com/opensearch-project/OpenSearch/pull/8746))
- [Remote Store] Add settings for remote path type and hash algorithm ([#13225](https://github.com/opensearch-project/OpenSearch/pull/13225))
- [Remote Store] Upload remote paths during remote enabled index creation ([#13386](https://github.com/opensearch-project/OpenSearch/pull/13386))

Expand All @@ -48,6 +49,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
- Bump `com.github.spullara.mustache.java:compiler` from 0.9.10 to 0.9.11 ([#13329](https://github.com/opensearch-project/OpenSearch/pull/13329))
- Bump `jakarta.enterprise:jakarta.enterprise.cdi-api` from 4.0.1 to 4.1.0 ([#13328](https://github.com/opensearch-project/OpenSearch/pull/13328))
- Bump `com.google.api.grpc:proto-google-iam-v1` from 0.12.0 to 1.33.0 ([#13332](https://github.com/opensearch-project/OpenSearch/pull/13332))
- OpenJDK Update (April 2024 Patch releases), update to Eclipse Temurin 21.0.3+9 ([#13389](https://github.com/opensearch-project/OpenSearch/pull/13389))
- Bump `com.squareup.okio:okio` from 3.8.0 to 3.9.0 ([#12997](https://github.com/opensearch-project/OpenSearch/pull/12997))

### Changed
- [BWC and API enforcement] Enforcing the presence of API annotations at build time ([#12872](https://github.com/opensearch-project/OpenSearch/pull/12872))
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -77,9 +77,9 @@
import java.util.stream.Stream;

public class DistroTestPlugin implements Plugin<Project> {
private static final String SYSTEM_JDK_VERSION = "21.0.2+13";
private static final String SYSTEM_JDK_VERSION = "21.0.3+9";
private static final String SYSTEM_JDK_VENDOR = "adoptium";
private static final String GRADLE_JDK_VERSION = "21.0.2+13";
private static final String GRADLE_JDK_VERSION = "21.0.3+9";
private static final String GRADLE_JDK_VENDOR = "adoptium";

// all distributions used by distro tests. this is temporary until tests are per distribution
Expand Down
2 changes: 1 addition & 1 deletion buildSrc/version.properties
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ opensearch = 3.0.0
lucene = 9.11.0-snapshot-fb97840

bundled_jdk_vendor = adoptium
bundled_jdk = 21.0.2+13
bundled_jdk = 21.0.3+9

# optional dependencies
spatial4j = 0.7
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -235,33 +235,19 @@ public void testDefaultRefreshIntervalWithUpdateClusterAndIndexSettings() throws
}

public void testRefreshIntervalDisabled() throws ExecutionException, InterruptedException {
TimeValue clusterMinimumRefreshInterval = client().settings()
.getAsTime(IndicesService.CLUSTER_MINIMUM_INDEX_REFRESH_INTERVAL_SETTING.getKey(), TimeValue.MINUS_ONE);
boolean createIndexSuccess = clusterMinimumRefreshInterval.equals(TimeValue.MINUS_ONE);
String clusterManagerName = internalCluster().getClusterManagerName();
List<String> dataNodes = new ArrayList<>(internalCluster().getDataNodeNames());
Settings settings = Settings.builder()
.put(indexSettings())
.put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), IndexSettings.MINIMUM_REFRESH_INTERVAL)
.build();
if (createIndexSuccess) {
createIndex(INDEX_NAME, settings);
ensureYellowAndNoInitializingShards(INDEX_NAME);
ensureGreen(INDEX_NAME);
GetIndexResponse getIndexResponse = client(clusterManagerName).admin().indices().getIndex(new GetIndexRequest()).get();
IndicesService indicesService = internalCluster().getInstance(IndicesService.class, randomFrom(dataNodes));
String uuid = getIndexResponse.getSettings().get(INDEX_NAME).get(IndexMetadata.SETTING_INDEX_UUID);
IndexService indexService = indicesService.indexService(new Index(INDEX_NAME, uuid));
assertEquals(IndexSettings.MINIMUM_REFRESH_INTERVAL, indexService.getRefreshTaskInterval());
} else {
IllegalArgumentException exception = assertThrows(IllegalArgumentException.class, () -> createIndex(INDEX_NAME, settings));
assertEquals(
"invalid index.refresh_interval [-1]: cannot be smaller than cluster.minimum.index.refresh_interval ["
+ getMinRefreshIntervalForRefreshDisabled()
+ "]",
exception.getMessage()
);
}
createIndex(INDEX_NAME, settings);
ensureGreen(INDEX_NAME);
GetIndexResponse getIndexResponse = client(clusterManagerName).admin().indices().getIndex(new GetIndexRequest()).get();
IndicesService indicesService = internalCluster().getInstance(IndicesService.class, randomFrom(dataNodes));
String uuid = getIndexResponse.getSettings().get(INDEX_NAME).get(IndexMetadata.SETTING_INDEX_UUID);
IndexService indexService = indicesService.indexService(new Index(INDEX_NAME, uuid));
assertEquals(IndexSettings.MINIMUM_REFRESH_INTERVAL, indexService.getRefreshTaskInterval());
}

protected TimeValue getMinRefreshIntervalForRefreshDisabled() {
Expand Down Expand Up @@ -366,6 +352,147 @@ public void testClusterMinimumChangeOnIndexWithCustomRefreshInterval() throws Ex
assertEquals(customRefreshInterval, indexService.getRefreshTaskInterval());
}

public void testClusterMinimumRefreshIntervalOfMinusOneFails() {
// This test checks that we can not set cluster minimum refresh interval as -1 (or -1ms).
String clusterManagerName = internalCluster().getClusterManagerName();
String refreshInterval = randomFrom("-1", "-1ms");
IllegalArgumentException ex = assertThrows(
IllegalArgumentException.class,
() -> client(clusterManagerName).admin()
.cluster()
.prepareUpdateSettings()
.setTransientSettings(Settings.builder().put(CLUSTER_MINIMUM_INDEX_REFRESH_INTERVAL_SETTING.getKey(), refreshInterval))
.get()
);
assertEquals(
"failed to parse value [" + refreshInterval + "] for setting [cluster.minimum.index.refresh_interval], must be >= [0ms]",
ex.getMessage()
);
}

public void testClusterMinimumRefreshIntervalOfZero() {
// This test checks that we can set the cluster minimum refresh interval as 0.
String clusterManagerName = internalCluster().getClusterManagerName();
client(clusterManagerName).admin()
.cluster()
.prepareUpdateSettings()
.setTransientSettings(Settings.builder().put(CLUSTER_MINIMUM_INDEX_REFRESH_INTERVAL_SETTING.getKey(), "0"))
.get();
}

public void testDefaultRefreshIntervalOfMinusOneIrrespectiveOfMinimum() {
// This test checks that we are able to set the cluster default refresh interval to one regardless of what the
// minimum is set to. -1 corresponds to no period background refreshes.
String clusterManagerName = internalCluster().getClusterManagerName();
client(clusterManagerName).admin()
.cluster()
.prepareUpdateSettings()
.setTransientSettings(
Settings.builder()
.put(CLUSTER_MINIMUM_INDEX_REFRESH_INTERVAL_SETTING.getKey(), randomFrom("0", "1ms", "1s", "10s"))
.put(CLUSTER_DEFAULT_INDEX_REFRESH_INTERVAL_SETTING.getKey(), randomFrom("-1", "-1ms"))
)
.get();
}

public void testCreateIndexWithMinusOneRefreshInterval() throws ExecutionException, InterruptedException {
// This test checks that we are able to create index with -1 refresh interval using index settings and default interval both.
String clusterManagerName = internalCluster().getClusterManagerName();
client(clusterManagerName).admin()
.cluster()
.prepareUpdateSettings()
.setTransientSettings(
Settings.builder()
.put(CLUSTER_MINIMUM_INDEX_REFRESH_INTERVAL_SETTING.getKey(), "10s")
.put(CLUSTER_DEFAULT_INDEX_REFRESH_INTERVAL_SETTING.getKey(), "10s")
)
.get();

Settings indexSettings = Settings.builder()
.put(indexSettings())
.put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), randomFrom("-1", "-1ms"))
.build();
createIndex(INDEX_NAME, indexSettings);
ensureGreen(INDEX_NAME);

IndexService indexService = getIndexServiceFromRandomDataNode(INDEX_NAME);
assertEquals(-1, indexService.getRefreshTaskInterval().millis());

client(clusterManagerName).admin()
.cluster()
.prepareUpdateSettings()
.setTransientSettings(Settings.builder().put(CLUSTER_DEFAULT_INDEX_REFRESH_INTERVAL_SETTING.getKey(), randomFrom("-1", "-1ms")))
.get();
createIndex(OTHER_INDEX_NAME);
ensureGreen(OTHER_INDEX_NAME);
indexService = getIndexServiceFromRandomDataNode(OTHER_INDEX_NAME);
assertEquals(-1, indexService.getRefreshTaskInterval().millis());
}

public void testUpdateIndexWithMinusOneRefreshInterval() throws ExecutionException, InterruptedException {
// This test checks that we are able to update index with -1 refresh interval using index settings and default interval both.
String clusterManagerName = internalCluster().getClusterManagerName();
client(clusterManagerName).admin()
.cluster()
.prepareUpdateSettings()
.setTransientSettings(
Settings.builder()
.put(CLUSTER_MINIMUM_INDEX_REFRESH_INTERVAL_SETTING.getKey(), "10s")
.put(CLUSTER_DEFAULT_INDEX_REFRESH_INTERVAL_SETTING.getKey(), "10s")
)
.get();

createIndex(INDEX_NAME);
ensureGreen(INDEX_NAME);
IndexService indexService = getIndexServiceFromRandomDataNode(INDEX_NAME);
assertEquals(10, indexService.getRefreshTaskInterval().seconds());

client(clusterManagerName).admin()
.indices()
.updateSettings(
new UpdateSettingsRequest(INDEX_NAME).settings(
Settings.builder().put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), randomFrom("-1", "-1ms"))
)
)
.actionGet();
assertEquals(-1, indexService.getRefreshTaskInterval().millis());

client(clusterManagerName).admin()
.indices()
.updateSettings(
new UpdateSettingsRequest(INDEX_NAME).settings(
Settings.builder().put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), "100s")
)
)
.actionGet();
assertEquals(100, indexService.getRefreshTaskInterval().seconds());

client(clusterManagerName).admin()
.cluster()
.prepareUpdateSettings()
.setTransientSettings(Settings.builder().put(CLUSTER_DEFAULT_INDEX_REFRESH_INTERVAL_SETTING.getKey(), randomFrom("-1", "-1ms")))
.get();

client(clusterManagerName).admin()
.indices()
.updateSettings(
new UpdateSettingsRequest(INDEX_NAME).settings(
Settings.builder().putNull(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey())
)
)
.actionGet();
assertEquals(-1, indexService.getRefreshTaskInterval().millis());
}

private IndexService getIndexServiceFromRandomDataNode(String indexName) throws ExecutionException, InterruptedException {
String clusterManagerName = internalCluster().getClusterManagerName();
List<String> dataNodes = new ArrayList<>(internalCluster().getDataNodeNames());
GetIndexResponse getIndexResponse = client(clusterManagerName).admin().indices().getIndex(new GetIndexRequest()).get();
IndicesService indicesService = internalCluster().getInstance(IndicesService.class, randomFrom(dataNodes));
String uuid = getIndexResponse.getSettings().get(indexName).get(IndexMetadata.SETTING_INDEX_UUID);
return indicesService.indexService(new Index(indexName, uuid));
}

protected TimeValue getDefaultRefreshInterval() {
return IndexSettings.DEFAULT_REFRESH_INTERVAL;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -198,11 +198,15 @@ public void testIsolateClusterManagerAndVerifyClusterStateConsensus() throws Exc
+ nodeState
);
}

}
ClusterStateStats clusterStateStats = internalCluster().clusterService().getClusterManagerService().getClusterStateStats();
assertTrue(clusterStateStats.getUpdateFailed() > 0);

});

ClusterStateStats clusterStateStats = internalCluster().clusterService(isolatedNode)
.getClusterManagerService()
.getClusterStateStats();
assertTrue(clusterStateStats.getUpdateFailed() > 0);

}

/**
Expand Down
Loading

0 comments on commit e92892c

Please sign in to comment.