Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Enforce soft deletes #1903

Merged
merged 2 commits into from
Jan 18, 2022
+144 −724
Merged

Enforce soft deletes #1903

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
7f42949
Enforce soft deletes
nknize Jan 14, 2022
5f7d3f9
remove straggler soft delete disabled tests
nknize Jan 14, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
83 changes: 79 additions & 4 deletions qa/full-cluster-restart/src/test/java/org/opensearch/upgrades/FullClusterRestartIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -696,7 +696,6 @@ public void testEmptyShard() throws IOException {
// before timing out
.put(INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), "100ms")
.put(SETTING_ALLOCATION_MAX_RETRY.getKey(), "0"); // fail faster
settings.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean());
if (randomBoolean()) {
settings.put(IndexSettings.INDEX_TRANSLOG_RETENTION_SIZE_SETTING.getKey(), "-1");
}
Expand Down Expand Up @@ -828,6 +827,11 @@ public void testSnapshotRestore() throws IOException {
if (isRunningAgainstOldCluster()) {
// Create the index
count = between(200, 300);
Settings.Builder settings = Settings.builder();
if (minimumNodeVersion().before(Version.V_2_0_0) && randomBoolean()) {
settings.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean());
}
createIndex(index, settings.build());
indexRandomDocuments(count, true, true, i -> jsonBuilder().startObject().field("field", "value").endObject());
} else {
count = countOfIndexedRandomDocuments();
Expand Down Expand Up @@ -1337,7 +1341,9 @@ public void testOperationBasedRecovery() throws Exception {
final Settings.Builder settings = Settings.builder()
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1);
settings.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean());
if (minimumNodeVersion().before(Version.V_2_0_0)) {
settings.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean());
}
createIndex(index, settings.build());
ensureGreen(index);
int committedDocs = randomIntBetween(100, 200);
Expand Down Expand Up @@ -1392,7 +1398,9 @@ public void testRecoveryWithTranslogRetentionDisabled() throws Exception {
final Settings.Builder settings = Settings.builder()
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1);
settings.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean());
if (minimumNodeVersion().before(Version.V_2_0_0)) {
settings.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean());
}
if (randomBoolean()) {
settings.put(IndexSettings.INDEX_TRANSLOG_RETENTION_SIZE_SETTING.getKey(), "-1");
}
Expand Down Expand Up @@ -1424,6 +1432,72 @@ public void testRecoveryWithTranslogRetentionDisabled() throws Exception {
assertTotalHits(numDocs, entityAsMap(client().performRequest(new Request("GET", "/" + index + "/_search"))));
}

public void testResize() throws Exception {
int numDocs;
if (isRunningAgainstOldCluster()) {
final Settings.Builder settings = Settings.builder()
.put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 3)
.put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 1);
if (minimumNodeVersion().before(Version.V_2_0_0) && randomBoolean()) {
settings.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), false);
}
final String mappings = randomBoolean() ? "\"_source\": { \"enabled\": false}" : null;
createIndex(index, settings.build(), mappings);
numDocs = randomIntBetween(10, 1000);
for (int i = 0; i < numDocs; i++) {
indexDocument(Integer.toString(i));
if (rarely()) {
flush(index, randomBoolean());
}
}
saveInfoDocument("num_doc_" + index, Integer.toString(numDocs));
ensureGreen(index);
} else {
ensureGreen(index);
numDocs = Integer.parseInt(loadInfoDocument("num_doc_" + index));
int moreDocs = randomIntBetween(0, 100);
for (int i = 0; i < moreDocs; i++) {
indexDocument(Integer.toString(numDocs + i));
if (rarely()) {
flush(index, randomBoolean());
}
}
Request updateSettingsRequest = new Request("PUT", "/" + index + "/_settings");
updateSettingsRequest.setJsonEntity("{\"settings\": {\"index.blocks.write\": true}}");
client().performRequest(updateSettingsRequest);
{
final String target = index + "_shrunken";
Request shrinkRequest = new Request("PUT", "/" + index + "/_shrink/" + target);
Settings.Builder settings = Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1);
if (randomBoolean()) {
settings.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true);
}
shrinkRequest.setJsonEntity("{\"settings\":" + Strings.toString(settings.build()) + "}");
client().performRequest(shrinkRequest);
ensureGreenLongWait(target);
assertNumHits(target, numDocs + moreDocs, 1);
}
{
final String target = index + "_split";
Settings.Builder settings = Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 6);
if (randomBoolean()) {
settings.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true);
}
Request splitRequest = new Request("PUT", "/" + index + "/_split/" + target);
splitRequest.setJsonEntity("{\"settings\":" + Strings.toString(settings.build()) + "}");
client().performRequest(splitRequest);
ensureGreenLongWait(target);
assertNumHits(target, numDocs + moreDocs, 6);
}
{
final String target = index + "_cloned";
client().performRequest(new Request("PUT", "/" + index + "/_clone/" + target));
ensureGreenLongWait(target);
assertNumHits(target, numDocs + moreDocs, 3);
}
}
}

@SuppressWarnings("unchecked")
public void testSystemIndexMetadataIsUpgraded() throws Exception {
final String systemIndexWarning = "this request accesses system indices: [.tasks], but in a future major version, direct " +
Expand Down Expand Up @@ -1529,12 +1603,13 @@ public void testSystemIndexMetadataIsUpgraded() throws Exception {
}

public void testEnableSoftDeletesOnRestore() throws Exception {
assumeTrue("soft deletes must be enabled on 2.0+", getOldClusterVersion().before(Version.V_2_0_0));
final String snapshot = "snapshot-" + index;
if (isRunningAgainstOldCluster()) {
final Settings.Builder settings = Settings.builder()
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1);
settings.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean());
settings.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), false);
createIndex(index, settings.build());
ensureGreen(index);
int numDocs = randomIntBetween(0, 100);
Expand Down
25 changes: 16 additions & 9 deletions qa/rolling-upgrade/src/test/java/org/opensearch/upgrades/RecoveryIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -339,7 +339,7 @@ public void testRecovery() throws Exception {
// before timing out
.put(INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), "100ms")
.put(SETTING_ALLOCATION_MAX_RETRY.getKey(), "0"); // fail faster
if (randomBoolean()) {
if (minimumNodeVersion().before(Version.V_2_0_0) && randomBoolean()) {
settings.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean());
}
createIndex(index, settings.build());
Expand Down Expand Up @@ -370,8 +370,10 @@ public void testRetentionLeasesEstablishedWhenPromotingPrimary() throws Exceptio
.put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), between(1, 5))
.put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), between(1, 2)) // triggers nontrivial promotion
.put(INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), "100ms")
.put(SETTING_ALLOCATION_MAX_RETRY.getKey(), "0") // fail faster
.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean());
.put(SETTING_ALLOCATION_MAX_RETRY.getKey(), "0"); // fail faster
if (minimumNodeVersion().before(Version.V_2_0_0) && randomBoolean()) {
settings.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean());
}
createIndex(index, settings.build());
int numDocs = randomInt(10);
indexDocs(index, 0, numDocs);
Expand All @@ -391,8 +393,10 @@ public void testRetentionLeasesEstablishedWhenRelocatingPrimary() throws Excepti
.put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), between(1, 5))
.put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), between(0, 1))
.put(INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), "100ms")
.put(SETTING_ALLOCATION_MAX_RETRY.getKey(), "0") // fail faster
.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean());
.put(SETTING_ALLOCATION_MAX_RETRY.getKey(), "0"); // fail faster
if (minimumNodeVersion().before(Version.V_2_0_0) && randomBoolean()) {
settings.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean());
}
createIndex(index, settings.build());
int numDocs = randomInt(10);
indexDocs(index, 0, numDocs);
Expand Down Expand Up @@ -713,10 +717,13 @@ private void assertNoopRecoveries(String indexName, Predicate<String> targetNode
public void testOperationBasedRecovery() throws Exception {
final String index = "test_operation_based_recovery";
if (CLUSTER_TYPE == ClusterType.OLD) {
createIndex(index, Settings.builder()
final Settings.Builder settings = Settings.builder()
.put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1)
.put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 2)
.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean()).build());
.put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 2);
if (minimumNodeVersion().before(Version.V_2_0_0) && randomBoolean()) {
settings.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean());
}
createIndex(index, settings.build());
ensureGreen(index);
indexDocs(index, 0, randomIntBetween(100, 200));
flush(index, randomBoolean());
Expand Down Expand Up @@ -791,7 +798,7 @@ public void testSoftDeletesDisabledWarning() throws Exception {
if (CLUSTER_TYPE == ClusterType.OLD) {
boolean softDeletesEnabled = true;
Settings.Builder settings = Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1);
if (randomBoolean()) {
if (minimumNodeVersion().before(Version.V_2_0_0) && randomBoolean()) {
softDeletesEnabled = randomBoolean();
settings.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), softDeletesEnabled);
}
Expand Down
16 changes: 0 additions & 16 deletions rest-api-spec/src/main/resources/rest-api-spec/test/indices.create/10_basic.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -155,19 +155,3 @@
- match: { error.type: "illegal_argument_exception" }
- match: { error.reason: "The mapping definition cannot be nested under a type [_doc] unless include_type_name is set to true." }

---
"Create index without soft deletes":
- skip:
version: " - 7.5.99"
reason: "indices without soft deletes are deprecated in 7.6"
features: "allowed_warnings"

- do:
allowed_warnings:
- Creating indices with soft-deletes disabled is deprecated and will be removed in future OpenSearch versions.
Please do not specify value for setting [index.soft_deletes.enabled] of index [test_index].
indices.create:
index: test_index
body:
settings:
soft_deletes.enabled: false
145 changes: 2 additions & 143 deletions rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/20_translog.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,83 +1,5 @@
---
"Translog retention without soft_deletes":
- skip:
version: " - 7.5.99"
reason: "indices without soft deletes are deprecated in 7.6"
features: "allowed_warnings"

- do:
indices.create:
index: test
body:
settings:
soft_deletes.enabled: false
allowed_warnings:
- Creating indices with soft-deletes disabled is deprecated and will be removed in future OpenSearch versions.
Please do not specify value for setting [index.soft_deletes.enabled] of index [test].
- do:
cluster.health:
wait_for_no_initializing_shards: true
wait_for_events: languid
- do:
indices.stats:
metric: [ translog ]
- set: { indices.test.primaries.translog.size_in_bytes: creation_size }

- do:
index:
index: test
id: 1
body: { "foo": "bar" }

- do:
indices.stats:
metric: [ translog ]
- gt: { indices.test.primaries.translog.size_in_bytes: $creation_size }
- match: { indices.test.primaries.translog.operations: 1 }
# we can't check this yet as creation size will contain two empty translog generations. A single
# non empty generation with one op may be smaller or larger than that.
# - gt: { indices.test.primaries.translog.uncommitted_size_in_bytes: $creation_size }
- match: { indices.test.primaries.translog.uncommitted_operations: 1 }

- do:
indices.flush:
index: test

- do:
indices.stats:
metric: [ translog ]
- gt: { indices.test.primaries.translog.size_in_bytes: $creation_size }
- match: { indices.test.primaries.translog.operations: 1 }
## creation translog size has some overhead due to an initial empty generation that will be trimmed later
- lt: { indices.test.primaries.translog.uncommitted_size_in_bytes: $creation_size }
- match: { indices.test.primaries.translog.uncommitted_operations: 0 }

- do:
indices.put_settings:
index: test
body:
index.translog.retention.size: -1
index.translog.retention.age: -1

- do:
indices.flush:
index: test
force: true # force flush as we don't have pending ops

- do:
indices.stats:
metric: [ translog ]
## creation translog size has some overhead due to an initial empty generation that will be trimmed later
- lte: { indices.test.primaries.translog.size_in_bytes: $creation_size }
- match: { indices.test.primaries.translog.operations: 0 }
- lte: { indices.test.primaries.translog.uncommitted_size_in_bytes: $creation_size }
- match: { indices.test.primaries.translog.uncommitted_operations: 0 }

---
"Translog retention with soft_deletes":
- skip:
version: " - 7.3.99"
reason: "start ignoring translog retention policy with soft-deletes enabled in 7.4"
"Translog retention":
- do:
indices.create:
index: test
Expand Down Expand Up @@ -179,70 +101,7 @@
- gte: { indices.test.primaries.translog.earliest_last_modified_age: 0 }

---
"Translog stats on closed indices without soft-deletes":
- skip:
version: " - 7.5.99"
reason: "indices without soft deletes are deprecated in 7.6"
features: "allowed_warnings"

- do:
indices.create:
index: test
body:
settings:
soft_deletes.enabled: false
routing.rebalance.enable: "none" # prevents shard relocations while we are closing an index
allowed_warnings:
- Creating indices with soft-deletes disabled is deprecated and will be removed in future OpenSearch versions.
Please do not specify value for setting [index.soft_deletes.enabled] of index [test].

- do:
cluster.health:
wait_for_no_initializing_shards: true
wait_for_events: languid
- do:
index:
index: test
id: 1
body: { "foo": "bar" }

- do:
index:
index: test
id: 2
body: { "foo": "bar" }

- do:
index:
index: test
id: 3
body: { "foo": "bar" }

- do:
indices.stats:
metric: [ translog ]
- match: { indices.test.primaries.translog.operations: 3 }
- match: { indices.test.primaries.translog.uncommitted_operations: 3 }

- do:
indices.close:
index: test
wait_for_active_shards: 1
- is_true: acknowledged

- do:
indices.stats:
metric: [ translog ]
expand_wildcards: all
forbid_closed_indices: false
- match: { indices.test.primaries.translog.operations: 3 }
- match: { indices.test.primaries.translog.uncommitted_operations: 0 }

---
"Translog stats on closed indices with soft-deletes":
- skip:
version: " - 7.3.99"
reason: "start ignoring translog retention policy with soft-deletes enabled in 7.4"
"Translog stats on closed indices":
- do:
indices.create:
index: test
Expand Down
4 changes: 0 additions & 4 deletions server/src/internalClusterTest/java/org/opensearch/gateway/ReplicaShardAllocatorIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,6 @@ public void testPreferCopyCanPerformNoopRecovery() throws Exception {
.prepareCreate(indexName)
.setSettings(
Settings.builder()
.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean())
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1)
.put(IndexSettings.FILE_BASED_RECOVERY_THRESHOLD_SETTING.getKey(), 1.0f)
Expand Down Expand Up @@ -280,7 +279,6 @@ public void testFullClusterRestartPerformNoopRecovery() throws Exception {
.prepareCreate(indexName)
.setSettings(
Settings.builder()
.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean())
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
.put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), randomIntBetween(10, 100) + "kb")
.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, numOfReplicas)
Expand Down Expand Up @@ -342,7 +340,6 @@ public void testPreferCopyWithHighestMatchingOperations() throws Exception {
.prepareCreate(indexName)
.setSettings(
Settings.builder()
.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean())
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
.put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), randomIntBetween(10, 100) + "kb")
.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1)
Expand Down Expand Up @@ -473,7 +470,6 @@ public void testPeerRecoveryForClosedIndices() throws Exception {
Settings.builder()
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean())
.put(IndexService.GLOBAL_CHECKPOINT_SYNC_INTERVAL_SETTING.getKey(), "100ms")
.put(IndexService.RETENTION_LEASE_SYNC_INTERVAL_SETTING.getKey(), "100ms")
.build()
Expand Down
Loading