Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Custom toXContent implementations #13833

Closed
wants to merge 3 commits into from
Closed

Custom toXContent implementations #13833

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
d844a7f
Custom toXContent implementations
shiv0408 May 26, 2024
5a0cf23
Fix snapshot de/ser
shiv0408 Jun 4, 2024
5a2cc20
Add UT
shiv0408 Jun 5, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 36 additions & 0 deletions server/src/main/java/org/opensearch/cluster/RepositoryCleanupInProgress.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,12 +33,15 @@

import org.opensearch.LegacyESVersion;
import org.opensearch.Version;
import org.opensearch.cluster.metadata.Metadata;
import org.opensearch.core.common.Strings;
import org.opensearch.core.common.io.stream.StreamInput;
import org.opensearch.core.common.io.stream.StreamOutput;
import org.opensearch.core.common.io.stream.Writeable;
import org.opensearch.core.xcontent.MediaTypeRegistry;
import org.opensearch.core.xcontent.XContentBuilder;
import org.opensearch.core.xcontent.XContentParser;
import org.opensearch.core.xcontent.XContentParserUtils;
import org.opensearch.repositories.RepositoryOperation;

import java.io.IOException;
Expand Down Expand Up @@ -101,13 +104,46 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
builder.startObject();
{
builder.field("repository", entry.repository);
if (params.param(Metadata.CONTEXT_MODE_PARAM, Metadata.CONTEXT_MODE_API).equals(Metadata.CONTEXT_MODE_GATEWAY)) {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What about the BWC for this?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

By default when we are using toXContent API mode is passed, we only pass Gateway when we are saving cluster state as persisted state in remote. So, by default I am putting the params as API and only adding extra fields if the mode is Gateway mode which is only used in our use case.

builder.field("repository_state_id", entry.repositoryStateId);
} // else we don't serialize it
}
builder.endObject();
}
builder.endArray();
return builder;
}

public static RepositoryCleanupInProgress fromXContent(XContentParser parser) throws IOException {
if (parser.currentToken() == null) {
parser.nextToken();
}
XContentParserUtils.ensureFieldName(parser, parser.currentToken(), TYPE);
parser.nextToken();
XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_ARRAY, parser.currentToken(), parser);
List<Entry> entries = new ArrayList<>();
while (parser.nextToken() != XContentParser.Token.END_ARRAY) {
XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser);
String repository = null;
long repositoryStateId = -1L;
while (parser.nextToken() != XContentParser.Token.END_OBJECT) {
XContentParserUtils.ensureExpectedToken(XContentParser.Token.FIELD_NAME, parser.currentToken(), parser);
String currentFieldName = parser.currentName();
parser.nextToken();
if ("repository".equals(currentFieldName)) {
repository = parser.text();
} else if ("repository_state_id".equals(currentFieldName)) {
// only XContent parsed with {@link Metadata.CONTEXT_MODE_GATEWAY} will have the repository state id and can be deserialized
repositoryStateId = parser.longValue();
} else {
throw new IllegalArgumentException("unknown field [" + currentFieldName + "]");
}
}
entries.add(new Entry(repository, repositoryStateId));
}
return new RepositoryCleanupInProgress(entries);
}

@Override
public String toString() {
return Strings.toString(MediaTypeRegistry.JSON, this);
Expand Down
211 changes: 180 additions & 31 deletions server/src/main/java/org/opensearch/cluster/RestoreInProgress.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,18 +32,25 @@

package org.opensearch.cluster;

import org.elasticsearch.snapshots.SnapshotId;
import org.opensearch.Version;
import org.opensearch.cluster.ClusterState.Custom;
import org.opensearch.cluster.metadata.Metadata;
import org.opensearch.common.annotation.PublicApi;
import org.opensearch.core.common.io.stream.StreamInput;
import org.opensearch.core.common.io.stream.StreamOutput;
import org.opensearch.core.common.io.stream.Writeable;
import org.opensearch.core.index.shard.ShardId;
import org.opensearch.core.xcontent.ToXContent;
import org.opensearch.core.xcontent.ToXContentFragment;
import org.opensearch.core.xcontent.XContentBuilder;
import org.opensearch.core.xcontent.XContentParser;
import org.opensearch.core.xcontent.XContentParserUtils;
import org.opensearch.snapshots.Snapshot;
import org.opensearch.snapshots.SnapshotId;

import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.Iterator;
Expand All @@ -52,6 +59,8 @@
import java.util.Objects;
import java.util.UUID;

import static org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken;

/**
* Meta data about restore processes that are currently executing
*
Expand Down Expand Up @@ -144,7 +153,7 @@ public RestoreInProgress build() {
* @opensearch.api
*/
@PublicApi(since = "1.0.0")
public static class Entry {
public static class Entry implements ToXContentFragment {
private final String uuid;
private final State state;
private final Snapshot snapshot;
Expand Down Expand Up @@ -236,6 +245,135 @@ public boolean equals(Object o) {
public int hashCode() {
return Objects.hash(uuid, snapshot, state, indices, shards);
}

@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
boolean isGatewayXContent = params.param(Metadata.CONTEXT_MODE_PARAM, Metadata.CONTEXT_MODE_API)
.equals(Metadata.CONTEXT_MODE_GATEWAY);
builder.startObject();
builder.field("snapshot", snapshot().getSnapshotId().getName());
builder.field("repository", snapshot().getRepository());
builder.field("state", state());
if (isGatewayXContent) {
builder.field("snapshot_uuid", snapshot().getSnapshotId().getUUID());
builder.field("uuid", uuid());
}
builder.startArray("indices");
{
for (String index : indices()) {
builder.value(index);
}
}
builder.endArray();
builder.startArray("shards");
{
for (final Map.Entry<ShardId, ShardRestoreStatus> shardEntry : shards.entrySet()) {
ShardId shardId = shardEntry.getKey();
ShardRestoreStatus status = shardEntry.getValue();
builder.startObject();
{
builder.field("index", shardId.getIndex());
builder.field("shard", shardId.getId());
builder.field("state", status.state());
if (isGatewayXContent) {
builder.field("index_uuid", shardId.getIndex().getUUID());
if (status.nodeId() != null) builder.field("node_id", status.nodeId());
if (status.reason() != null) builder.field("reason", status.reason());
}
}
builder.endObject();
}
}

builder.endArray();
builder.endObject();
return builder;
}

public static Entry fromXContent(XContentParser parser) throws IOException {
if (parser.currentToken() == null) {
parser.nextToken();
}
ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser);
String snapshotName = null;
String snapshotRepository = null;
String snapshotUUID = null;
int state = -1;
String uuid = null;
List<String> indices = new ArrayList<>();
Map<ShardId, ShardRestoreStatus> shards = new HashMap<>();
while (parser.nextToken() != XContentParser.Token.END_OBJECT) {
ensureExpectedToken(XContentParser.Token.FIELD_NAME, parser.currentToken(), parser);
String currentFieldName = parser.currentName();
parser.nextToken();
switch (currentFieldName) {
case "snapshot":
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: could be made constant.

snapshotName = parser.text();
break;
case "repository":
snapshotRepository = parser.text();
break;
case "state":
state = parser.intValue();
break;
case "snapshot_uuid":
snapshotUUID = parser.text();
break;
case "uuid":
uuid = parser.text();
break;
case "indices":
ensureExpectedToken(XContentParser.Token.START_ARRAY, parser.currentToken(), parser);
while (parser.nextToken() != XContentParser.Token.END_ARRAY) {
indices.add(parser.text());
}
break;
case "shards":
ensureExpectedToken(XContentParser.Token.START_ARRAY, parser.currentToken(), parser);
while (parser.nextToken() != XContentParser.Token.END_ARRAY) {
String indexName = null;
String indexUUID = null;
int shardId = -1;
int restoreState = -1;
String nodeId = null;
String reason = null;
ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser);
while (parser.nextToken() != XContentParser.Token.END_OBJECT) {
ensureExpectedToken(XContentParser.Token.FIELD_NAME, parser.currentToken(), parser);
String currentShardFieldName = parser.currentName();
parser.nextToken();
switch (currentShardFieldName) {
case "index":
indexName = parser.text();
break;
case "shard":
shardId = parser.intValue();
break;
case "state":
restoreState = parser.intValue();
break;
case "index_uuid":
indexUUID = parser.text();
break;
case "node_id":
nodeId = parser.text();
break;
case "reason":
reason = parser.text();
break;
default:
throw new IllegalArgumentException("unknown field [" + currentFieldName + "]");
}
}
shards.put(new ShardId(indexName, indexUUID, shardId), new ShardRestoreStatus(nodeId, State.fromValue((byte) restoreState), reason));
}
break;
default:
throw new IllegalArgumentException("unknown field [" + currentFieldName + "]");
}
}
return new Entry(uuid, new Snapshot(snapshotRepository, new SnapshotId(snapshotName, snapshotUUID)), State.fromValue((byte) state), indices, shards);
}
}

/**
Expand Down Expand Up @@ -495,46 +633,57 @@ public void writeTo(StreamOutput out) throws IOException {
public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException {
builder.startArray("snapshots");
for (final Entry entry : entries.values()) {
toXContent(entry, builder);
toXContent(entry, builder, ToXContent.EMPTY_PARAMS);
}
builder.endArray();
return builder;
}

public static RestoreInProgress fromXContent(XContentParser parser) throws IOException {
if (parser.currentToken() == null) {
parser.nextToken();
}
final Map<String, Entry> entries = new HashMap<>();
XContentParserUtils.ensureFieldName(parser, parser.currentToken(), "snapshots");
while (parser.nextToken() != XContentParser.Token.END_ARRAY) {
final Entry entry = Entry.fromXContent(parser);
entries.put(entry.uuid, entry);
}
return new RestoreInProgress(entries);
}

/**
* Serializes single restore operation
*
* @param entry restore operation metadata
* @param builder XContent builder
* @param params
*/
public void toXContent(Entry entry, XContentBuilder builder) throws IOException {
builder.startObject();
builder.field("snapshot", entry.snapshot().getSnapshotId().getName());
builder.field("repository", entry.snapshot().getRepository());
builder.field("state", entry.state());
builder.startArray("indices");
{
for (String index : entry.indices()) {
builder.value(index);
}
}
builder.endArray();
builder.startArray("shards");
{
for (final Map.Entry<ShardId, ShardRestoreStatus> shardEntry : entry.shards.entrySet()) {
ShardId shardId = shardEntry.getKey();
ShardRestoreStatus status = shardEntry.getValue();
builder.startObject();
{
builder.field("index", shardId.getIndex());
builder.field("shard", shardId.getId());
builder.field("state", status.state());
}
builder.endObject();
}
}

builder.endArray();
builder.endObject();
public void toXContent(Entry entry, XContentBuilder builder, ToXContent.Params params) throws IOException {
entry.toXContent(builder, params);
}
}

shiv0408 marked this conversation as resolved.
Show resolved Hide resolved























Loading
Loading