Skip to content

Commit

Permalink
Enforce soft deletes (#1903)
Browse files Browse the repository at this point in the history 
This commit enforces soft deletes as required in 2.0.0.

Signed-off-by: Nicholas Walter Knize <[email protected]>
  • Loading branch information
@nknize
nknize authored Jan 18, 2022
1 parent 7691f07 commit 7058256
Show file tree
Hide file tree
Showing 24 changed files with 144 additions and 724 deletions.
83 changes: 79 additions & 4 deletions qa/full-cluster-restart/src/test/java/org/opensearch/upgrades/FullClusterRestartIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -696,7 +696,6 @@ public void testEmptyShard() throws IOException {
// before timing out
.put(INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), "100ms")
.put(SETTING_ALLOCATION_MAX_RETRY.getKey(), "0"); // fail faster
settings.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean());
if (randomBoolean()) {
settings.put(IndexSettings.INDEX_TRANSLOG_RETENTION_SIZE_SETTING.getKey(), "-1");
}
Expand Down Expand Up @@ -828,6 +827,11 @@ public void testSnapshotRestore() throws IOException {
if (isRunningAgainstOldCluster()) {
// Create the index
count = between(200, 300);
Settings.Builder settings = Settings.builder();
if (minimumNodeVersion().before(Version.V_2_0_0) && randomBoolean()) {
settings.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean());
}
createIndex(index, settings.build());
indexRandomDocuments(count, true, true, i -> jsonBuilder().startObject().field("field", "value").endObject());
} else {
count = countOfIndexedRandomDocuments();
Expand Down Expand Up @@ -1337,7 +1341,9 @@ public void testOperationBasedRecovery() throws Exception {
final Settings.Builder settings = Settings.builder()
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1);
settings.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean());
if (minimumNodeVersion().before(Version.V_2_0_0)) {
settings.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean());
}
createIndex(index, settings.build());
ensureGreen(index);
int committedDocs = randomIntBetween(100, 200);
Expand Down Expand Up @@ -1392,7 +1398,9 @@ public void testRecoveryWithTranslogRetentionDisabled() throws Exception {
final Settings.Builder settings = Settings.builder()
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1);
settings.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean());
if (minimumNodeVersion().before(Version.V_2_0_0)) {
settings.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean());
}
if (randomBoolean()) {
settings.put(IndexSettings.INDEX_TRANSLOG_RETENTION_SIZE_SETTING.getKey(), "-1");
}
Expand Down Expand Up @@ -1424,6 +1432,72 @@ public void testRecoveryWithTranslogRetentionDisabled() throws Exception {
assertTotalHits(numDocs, entityAsMap(client().performRequest(new Request("GET", "/" + index + "/_search"))));
}

public void testResize() throws Exception {
int numDocs;
if (isRunningAgainstOldCluster()) {
final Settings.Builder settings = Settings.builder()
.put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 3)
.put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 1);
if (minimumNodeVersion().before(Version.V_2_0_0) && randomBoolean()) {
settings.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), false);
}
final String mappings = randomBoolean() ? "\"_source\": { \"enabled\": false}" : null;
createIndex(index, settings.build(), mappings);
numDocs = randomIntBetween(10, 1000);
for (int i = 0; i < numDocs; i++) {
indexDocument(Integer.toString(i));
if (rarely()) {
flush(index, randomBoolean());
}
}
saveInfoDocument("num_doc_" + index, Integer.toString(numDocs));
ensureGreen(index);
} else {
ensureGreen(index);
numDocs = Integer.parseInt(loadInfoDocument("num_doc_" + index));
int moreDocs = randomIntBetween(0, 100);
for (int i = 0; i < moreDocs; i++) {
indexDocument(Integer.toString(numDocs + i));
if (rarely()) {
flush(index, randomBoolean());
}
}
Request updateSettingsRequest = new Request("PUT", "/" + index + "/_settings");
updateSettingsRequest.setJsonEntity("{\"settings\": {\"index.blocks.write\": true}}");
client().performRequest(updateSettingsRequest);
{
final String target = index + "_shrunken";
Request shrinkRequest = new Request("PUT", "/" + index + "/_shrink/" + target);
Settings.Builder settings = Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1);
if (randomBoolean()) {
settings.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true);
}
shrinkRequest.setJsonEntity("{\"settings\":" + Strings.toString(settings.build()) + "}");
client().performRequest(shrinkRequest);
ensureGreenLongWait(target);
assertNumHits(target, numDocs + moreDocs, 1);
}
{
final String target = index + "_split";
Settings.Builder settings = Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 6);
if (randomBoolean()) {
settings.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true);
}
Request splitRequest = new Request("PUT", "/" + index + "/_split/" + target);
splitRequest.setJsonEntity("{\"settings\":" + Strings.toString(settings.build()) + "}");
client().performRequest(splitRequest);
ensureGreenLongWait(target);
assertNumHits(target, numDocs + moreDocs, 6);
}
{
final String target = index + "_cloned";
client().performRequest(new Request("PUT", "/" + index + "/_clone/" + target));
ensureGreenLongWait(target);
assertNumHits(target, numDocs + moreDocs, 3);
}
}
}

@SuppressWarnings("unchecked")
public void testSystemIndexMetadataIsUpgraded() throws Exception {
final String systemIndexWarning = "this request accesses system indices: [.tasks], but in a future major version, direct " +
Expand Down Expand Up @@ -1529,12 +1603,13 @@ public void testSystemIndexMetadataIsUpgraded() throws Exception {
}

public void testEnableSoftDeletesOnRestore() throws Exception {
assumeTrue("soft deletes must be enabled on 2.0+", getOldClusterVersion().before(Version.V_2_0_0));
final String snapshot = "snapshot-" + index;
if (isRunningAgainstOldCluster()) {
final Settings.Builder settings = Settings.builder()
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1);
settings.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean());
settings.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), false);
createIndex(index, settings.build());
ensureGreen(index);
int numDocs = randomIntBetween(0, 100);
Expand Down
25 changes: 16 additions & 9 deletions qa/rolling-upgrade/src/test/java/org/opensearch/upgrades/RecoveryIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -339,7 +339,7 @@ public void testRecovery() throws Exception {
// before timing out
.put(INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), "100ms")
.put(SETTING_ALLOCATION_MAX_RETRY.getKey(), "0"); // fail faster
if (randomBoolean()) {
if (minimumNodeVersion().before(Version.V_2_0_0) && randomBoolean()) {
settings.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean());
}
createIndex(index, settings.build());
Expand Down Expand Up @@ -370,8 +370,10 @@ public void testRetentionLeasesEstablishedWhenPromotingPrimary() throws Exceptio
.put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), between(1, 5))
.put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), between(1, 2)) // triggers nontrivial promotion
.put(INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), "100ms")
.put(SETTING_ALLOCATION_MAX_RETRY.getKey(), "0") // fail faster
.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean());
.put(SETTING_ALLOCATION_MAX_RETRY.getKey(), "0"); // fail faster
if (minimumNodeVersion().before(Version.V_2_0_0) && randomBoolean()) {
settings.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean());
}
createIndex(index, settings.build());
int numDocs = randomInt(10);
indexDocs(index, 0, numDocs);
Expand All @@ -391,8 +393,10 @@ public void testRetentionLeasesEstablishedWhenRelocatingPrimary() throws Excepti
.put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), between(1, 5))
.put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), between(0, 1))
.put(INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), "100ms")
.put(SETTING_ALLOCATION_MAX_RETRY.getKey(), "0") // fail faster
.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean());
.put(SETTING_ALLOCATION_MAX_RETRY.getKey(), "0"); // fail faster
if (minimumNodeVersion().before(Version.V_2_0_0) && randomBoolean()) {
settings.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean());
}
createIndex(index, settings.build());
int numDocs = randomInt(10);
indexDocs(index, 0, numDocs);
Expand Down Expand Up @@ -713,10 +717,13 @@ private void assertNoopRecoveries(String indexName, Predicate<String> targetNode
public void testOperationBasedRecovery() throws Exception {
final String index = "test_operation_based_recovery";
if (CLUSTER_TYPE == ClusterType.OLD) {
createIndex(index, Settings.builder()
final Settings.Builder settings = Settings.builder()
.put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1)
.put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 2)
.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean()).build());
.put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 2);
if (minimumNodeVersion().before(Version.V_2_0_0) && randomBoolean()) {
settings.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean());
}
createIndex(index, settings.build());
ensureGreen(index);
indexDocs(index, 0, randomIntBetween(100, 200));
flush(index, randomBoolean());
Expand Down Expand Up @@ -791,7 +798,7 @@ public void testSoftDeletesDisabledWarning() throws Exception {
if (CLUSTER_TYPE == ClusterType.OLD) {
boolean softDeletesEnabled = true;
Settings.Builder settings = Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1);
if (randomBoolean()) {
if (minimumNodeVersion().before(Version.V_2_0_0) && randomBoolean()) {
softDeletesEnabled = randomBoolean();
settings.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), softDeletesEnabled);
}
Expand Down
16 changes: 0 additions & 16 deletions rest-api-spec/src/main/resources/rest-api-spec/test/indices.create/10_basic.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -155,19 +155,3 @@
- match: { error.type: "illegal_argument_exception" }
- match: { error.reason: "The mapping definition cannot be nested under a type [_doc] unless include_type_name is set to true." }

---
"Create index without soft deletes":
- skip:
version: " - 7.5.99"
reason: "indices without soft deletes are deprecated in 7.6"
features: "allowed_warnings"

- do:
allowed_warnings:
- Creating indices with soft-deletes disabled is deprecated and will be removed in future OpenSearch versions.
Please do not specify value for setting [index.soft_deletes.enabled] of index [test_index].
indices.create:
index: test_index
body:
settings:
soft_deletes.enabled: false
145 changes: 2 additions & 143 deletions rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/20_translog.yml
View file
Original file line number Diff line number Diff line change
@@ -1,83 +1,5 @@
---
"Translog retention without soft_deletes":
- skip:
version: " - 7.5.99"
reason: "indices without soft deletes are deprecated in 7.6"
features: "allowed_warnings"

- do:
indices.create:
index: test
body:
settings:
soft_deletes.enabled: false
allowed_warnings:
- Creating indices with soft-deletes disabled is deprecated and will be removed in future OpenSearch versions.
Please do not specify value for setting [index.soft_deletes.enabled] of index [test].
- do:
cluster.health:
wait_for_no_initializing_shards: true
wait_for_events: languid
- do:
indices.stats:
metric: [ translog ]
- set: { indices.test.primaries.translog.size_in_bytes: creation_size }

- do:
index:
index: test
id: 1
body: { "foo": "bar" }

- do:
indices.stats:
metric: [ translog ]
- gt: { indices.test.primaries.translog.size_in_bytes: $creation_size }
- match: { indices.test.primaries.translog.operations: 1 }
# we can't check this yet as creation size will contain two empty translog generations. A single
# non empty generation with one op may be smaller or larger than that.
# - gt: { indices.test.primaries.translog.uncommitted_size_in_bytes: $creation_size }
- match: { indices.test.primaries.translog.uncommitted_operations: 1 }

- do:
indices.flush:
index: test

- do:
indices.stats:
metric: [ translog ]
- gt: { indices.test.primaries.translog.size_in_bytes: $creation_size }
- match: { indices.test.primaries.translog.operations: 1 }
## creation translog size has some overhead due to an initial empty generation that will be trimmed later
- lt: { indices.test.primaries.translog.uncommitted_size_in_bytes: $creation_size }
- match: { indices.test.primaries.translog.uncommitted_operations: 0 }

- do:
indices.put_settings:
index: test
body:
index.translog.retention.size: -1
index.translog.retention.age: -1

- do:
indices.flush:
index: test
force: true # force flush as we don't have pending ops

- do:
indices.stats:
metric: [ translog ]
## creation translog size has some overhead due to an initial empty generation that will be trimmed later
- lte: { indices.test.primaries.translog.size_in_bytes: $creation_size }
- match: { indices.test.primaries.translog.operations: 0 }
- lte: { indices.test.primaries.translog.uncommitted_size_in_bytes: $creation_size }
- match: { indices.test.primaries.translog.uncommitted_operations: 0 }

---
"Translog retention with soft_deletes":
- skip:
version: " - 7.3.99"
reason: "start ignoring translog retention policy with soft-deletes enabled in 7.4"
"Translog retention":
- do:
indices.create:
index: test
Expand Down Expand Up @@ -179,70 +101,7 @@
- gte: { indices.test.primaries.translog.earliest_last_modified_age: 0 }

---
"Translog stats on closed indices without soft-deletes":
- skip:
version: " - 7.5.99"
reason: "indices without soft deletes are deprecated in 7.6"
features: "allowed_warnings"

- do:
indices.create:
index: test
body:
settings:
soft_deletes.enabled: false
routing.rebalance.enable: "none" # prevents shard relocations while we are closing an index
allowed_warnings:
- Creating indices with soft-deletes disabled is deprecated and will be removed in future OpenSearch versions.
Please do not specify value for setting [index.soft_deletes.enabled] of index [test].

- do:
cluster.health:
wait_for_no_initializing_shards: true
wait_for_events: languid
- do:
index:
index: test
id: 1
body: { "foo": "bar" }

- do:
index:
index: test
id: 2
body: { "foo": "bar" }

- do:
index:
index: test
id: 3
body: { "foo": "bar" }

- do:
indices.stats:
metric: [ translog ]
- match: { indices.test.primaries.translog.operations: 3 }
- match: { indices.test.primaries.translog.uncommitted_operations: 3 }

- do:
indices.close:
index: test
wait_for_active_shards: 1
- is_true: acknowledged

- do:
indices.stats:
metric: [ translog ]
expand_wildcards: all
forbid_closed_indices: false
- match: { indices.test.primaries.translog.operations: 3 }
- match: { indices.test.primaries.translog.uncommitted_operations: 0 }

---
"Translog stats on closed indices with soft-deletes":
- skip:
version: " - 7.3.99"
reason: "start ignoring translog retention policy with soft-deletes enabled in 7.4"
"Translog stats on closed indices":
- do:
indices.create:
index: test
Expand Down
4 changes: 0 additions & 4 deletions server/src/internalClusterTest/java/org/opensearch/gateway/ReplicaShardAllocatorIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,6 @@ public void testPreferCopyCanPerformNoopRecovery() throws Exception {
.prepareCreate(indexName)
.setSettings(
Settings.builder()
.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean())
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1)
.put(IndexSettings.FILE_BASED_RECOVERY_THRESHOLD_SETTING.getKey(), 1.0f)
Expand Down Expand Up @@ -280,7 +279,6 @@ public void testFullClusterRestartPerformNoopRecovery() throws Exception {
.prepareCreate(indexName)
.setSettings(
Settings.builder()
.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean())
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
.put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), randomIntBetween(10, 100) + "kb")
.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, numOfReplicas)
Expand Down Expand Up @@ -342,7 +340,6 @@ public void testPreferCopyWithHighestMatchingOperations() throws Exception {
.prepareCreate(indexName)
.setSettings(
Settings.builder()
.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean())
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
.put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), randomIntBetween(10, 100) + "kb")
.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1)
Expand Down Expand Up @@ -473,7 +470,6 @@ public void testPeerRecoveryForClosedIndices() throws Exception {
Settings.builder()
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean())
.put(IndexService.GLOBAL_CHECKPOINT_SYNC_INTERVAL_SETTING.getKey(), "100ms")
.put(IndexService.RETENTION_LEASE_SYNC_INTERVAL_SETTING.getKey(), "100ms")
.build()
Expand Down
Loading

0 comments on commit 7058256

Please sign in to comment.