Split the remote global metadata file to metadata attribute files

server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java

@@ -752,6 +752,10 @@ public String getIndexUUID() {
         return index.getUUID();
     }
 
+    public String getIndexName() {
+        return index.getName();
+    }
+
     /**
      * Test whether the current index UUID is the same as the given one. Returns true if either are _na_
      */

server/src/main/java/org/opensearch/cluster/metadata/Metadata.java

@@ -175,6 +175,27 @@ public enum XContentContext {
     public interface Custom extends NamedDiffable<Custom>, ToXContentFragment, ClusterState.FeatureAware {
 
         EnumSet<XContentContext> context();
+
+        static Custom fromXContent(XContentParser parser, String name) throws IOException {
+            try {
+                return parser.namedObject(Custom.class, name, null);
+            } catch (NamedObjectNotFoundException e) {
+                logger.warn("Unknown custom object with type {}", name);
+                parser.skipChildren();
+                throw e;
+            }
+        }
+
+        static Custom fromXContent(XContentParser parser) throws IOException {
+            String currentFieldName = parser.currentName();
+            try {
+                return parser.namedObject(Custom.class, currentFieldName, null);
+            } catch (NamedObjectNotFoundException e) {
+                logger.warn("Unknown custom object with type {}", currentFieldName);
+                parser.skipChildren();
+                throw e;
+            }
+        }
     }
 
     public static final Setting<Integer> DEFAULT_REPLICA_COUNT_SETTING = Setting.intSetting(
@@ -260,7 +281,7 @@ public interface Custom extends NamedDiffable<Custom>, ToXContentFragment, Clust
     private final Settings settings;
     private final DiffableStringMap hashesOfConsistentSettings;
     private final Map<String, IndexMetadata> indices;
-    private final Map<String, IndexTemplateMetadata> templates;
+    private final TemplatesMetadata templates;
     private final Map<String, Custom> customs;
 
     private final transient int totalNumberOfShards; // Transient ? not serializable anyway?
@@ -304,7 +325,7 @@ public interface Custom extends NamedDiffable<Custom>, ToXContentFragment, Clust
         this.hashesOfConsistentSettings = hashesOfConsistentSettings;
         this.indices = Collections.unmodifiableMap(indices);
         this.customs = Collections.unmodifiableMap(customs);
-        this.templates = Collections.unmodifiableMap(templates);
+        this.templates = new TemplatesMetadata(templates);
         int totalNumberOfShards = 0;
         int totalOpenIndexShards = 0;
         for (IndexMetadata cursor : indices.values()) {
@@ -806,13 +827,17 @@ public Map<String, IndexMetadata> getIndices() {
     }
 
     public Map<String, IndexTemplateMetadata> templates() {
-        return this.templates;
+        return this.templates.getTemplates();
     }
 
     public Map<String, IndexTemplateMetadata> getTemplates() {
         return templates();
     }
 
+    public TemplatesMetadata templatesMetadata() {
+        return this.templates;
+    }
+
     public Map<String, ComponentTemplate> componentTemplates() {
         return Optional.ofNullable((ComponentTemplateMetadata) this.custom(ComponentTemplateMetadata.TYPE))
             .map(ComponentTemplateMetadata::componentTemplates)
@@ -923,7 +948,7 @@ public Iterator<IndexMetadata> iterator() {
     }
 
     public static boolean isGlobalStateEquals(Metadata metadata1, Metadata metadata2) {
-        if (!metadata1.coordinationMetadata.equals(metadata2.coordinationMetadata)) {
+        if (!isCoordinationMetadataEqual(metadata1, metadata2)) {
             return false;
         }
         if (!metadata1.hashesOfConsistentSettings.equals(metadata2.hashesOfConsistentSettings)) {
@@ -942,13 +967,29 @@ public static boolean isGlobalStateEquals(Metadata metadata1, Metadata metadata2
      * Compares Metadata entities persisted in Remote Store.
      */
     public static boolean isGlobalResourcesMetadataEquals(Metadata metadata1, Metadata metadata2) {
-        if (!metadata1.persistentSettings.equals(metadata2.persistentSettings)) {
+        if (!isSettingsMetadataEqual(metadata1, metadata2)) {
             return false;
         }
-        if (!metadata1.templates.equals(metadata2.templates())) {
+        if (!isTemplatesMetadataEqual(metadata1, metadata2)) {
             return false;
         }
         // Check if any persistent metadata needs to be saved
+        return isCustomMetadataEqual(metadata1, metadata2);
+    }
+
+    public static boolean isCoordinationMetadataEqual(Metadata metadata1, Metadata metadata2) {
+        return metadata1.coordinationMetadata.equals(metadata2.coordinationMetadata);
+    }
+
+    public static boolean isSettingsMetadataEqual(Metadata metadata1, Metadata metadata2) {
+        return metadata1.persistentSettings.equals(metadata2.persistentSettings);
+    }
+
+    public static boolean isTemplatesMetadataEqual(Metadata metadata1, Metadata metadata2) {
+        return metadata1.templates.equals(metadata2.templates);
+    }
+
+    public static boolean isCustomMetadataEqual(Metadata metadata1, Metadata metadata2) {
         int customCount1 = 0;
         for (Map.Entry<String, Custom> cursor : metadata1.customs.entrySet()) {
             if (cursor.getValue().context().contains(XContentContext.GATEWAY)) {
@@ -962,8 +1003,7 @@ public static boolean isGlobalResourcesMetadataEquals(Metadata metadata1, Metada
                 customCount2++;
             }
         }
-        if (customCount1 != customCount2) return false;
-        return true;
+        return customCount1 == customCount2;
     }
 
     @Override
@@ -1012,7 +1052,11 @@ private static class MetadataDiff implements Diff<Metadata> {
             persistentSettings = after.persistentSettings;
             hashesOfConsistentSettings = after.hashesOfConsistentSettings.diff(before.hashesOfConsistentSettings);
             indices = DiffableUtils.diff(before.indices, after.indices, DiffableUtils.getStringKeySerializer());
-            templates = DiffableUtils.diff(before.templates, after.templates, DiffableUtils.getStringKeySerializer());
+            templates = DiffableUtils.diff(
+                before.templates.getTemplates(),
+                after.templates.getTemplates(),
+                DiffableUtils.getStringKeySerializer()
+            );
             customs = DiffableUtils.diff(before.customs, after.customs, DiffableUtils.getStringKeySerializer(), CUSTOM_VALUE_SERIALIZER);
         }
 
@@ -1059,7 +1103,7 @@ public Metadata apply(Metadata part) {
             builder.persistentSettings(persistentSettings);
             builder.hashesOfConsistentSettings(hashesOfConsistentSettings.apply(part.hashesOfConsistentSettings));
             builder.indices(indices.apply(part.indices));
-            builder.templates(templates.apply(part.templates));
+            builder.templates(templates.apply(part.templates.getTemplates()));
             builder.customs(customs.apply(part.customs));
             return builder.build();
         }
@@ -1103,10 +1147,7 @@ public void writeTo(StreamOutput out) throws IOException {
         for (IndexMetadata indexMetadata : this) {
             indexMetadata.writeTo(out);
         }
-        out.writeVInt(templates.size());
-        for (final IndexTemplateMetadata cursor : templates.values()) {
-            cursor.writeTo(out);
-        }
+        templates.writeTo(out);
         // filter out custom states not supported by the other node
         int numberOfCustoms = 0;
         for (final Custom cursor : customs.values()) {
@@ -1170,7 +1211,7 @@ public Builder(Metadata metadata) {
             this.hashesOfConsistentSettings = metadata.hashesOfConsistentSettings;
             this.version = metadata.version;
             this.indices = new HashMap<>(metadata.indices);
-            this.templates = new HashMap<>(metadata.templates);
+            this.templates = new HashMap<>(metadata.templates.getTemplates());
             this.customs = new HashMap<>(metadata.customs);
             this.previousMetadata = metadata;
         }
@@ -1249,6 +1290,11 @@ public Builder templates(Map<String, IndexTemplateMetadata> templates) {
             return this;
         }
 
+        public Builder templates(TemplatesMetadata templatesMetadata) {
+            this.templates.putAll(templatesMetadata.getTemplates());
+            return this;
+        }
+
         public Builder put(String name, ComponentTemplate componentTemplate) {
             Objects.requireNonNull(componentTemplate, "it is invalid to add a null component template: " + name);
             Map<String, ComponentTemplate> existingTemplates = Optional.ofNullable(
@@ -1739,9 +1785,7 @@ public static void toXContent(Metadata metadata, XContentBuilder builder, ToXCon
             }
 
             builder.startObject("templates");
-            for (final IndexTemplateMetadata cursor : metadata.templates().values()) {
-                IndexTemplateMetadata.Builder.toXContentWithTypes(cursor, builder, params);
-            }
+            metadata.templatesMetadata().toXContent(builder, params);
             builder.endObject();
 
             if (context == XContentContext.API) {
@@ -1804,16 +1848,13 @@ public static Metadata fromXContent(XContentParser parser) throws IOException {
                     } else if ("hashes_of_consistent_settings".equals(currentFieldName)) {
                         builder.hashesOfConsistentSettings(parser.mapStrings());
                     } else if ("templates".equals(currentFieldName)) {
-                        while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
-                            builder.put(IndexTemplateMetadata.Builder.fromXContent(parser, parser.currentName()));
-                        }
+                        builder.templates(TemplatesMetadata.fromXContent(parser));
                     } else {
                         try {
-                            Custom custom = parser.namedObject(Custom.class, currentFieldName, null);
+                            Custom custom = Custom.fromXContent(parser, currentFieldName);
                             builder.putCustom(custom.getWriteableName(), custom);
                         } catch (NamedObjectNotFoundException ex) {
                             logger.warn("Skipping unknown custom object with type {}", currentFieldName);
-                            parser.skipChildren();
                         }
                     }
                 } else if (token.isValue()) {

server/src/main/java/org/opensearch/cluster/metadata/RepositoriesMetadata.java

@@ -202,6 +202,10 @@ public static RepositoriesMetadata fromXContent(XContentParser parser) throws IO
         XContentParser.Token token;
         List<RepositoryMetadata> repository = new ArrayList<>();
         while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+            if (token == XContentParser.Token.START_OBJECT) {
+                // move to next token if parsing the whole object
+                token = parser.nextToken();
+            }
             if (token == XContentParser.Token.FIELD_NAME) {
                 String name = parser.currentName();
                 if (parser.nextToken() != XContentParser.Token.START_OBJECT) {

server/src/main/java/org/opensearch/cluster/metadata/TemplatesMetadata.java

@@ -0,0 +1,144 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.cluster.metadata;
+
+import org.opensearch.cluster.AbstractDiffable;
+import org.opensearch.core.common.io.stream.StreamOutput;
+import org.opensearch.core.xcontent.ToXContentFragment;
+import org.opensearch.core.xcontent.XContentBuilder;
+import org.opensearch.core.xcontent.XContentParser;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+
+public class TemplatesMetadata extends AbstractDiffable<TemplatesMetadata> implements ToXContentFragment {
+    public static TemplatesMetadata EMPTY_METADATA = builder().build();
+    private final Map<String, IndexTemplateMetadata> templates;
+
+    public TemplatesMetadata() {
+        this(Collections.emptyMap());
+    }
+
+    public TemplatesMetadata(Map<String, IndexTemplateMetadata> templates) {
+        this.templates = Collections.unmodifiableMap(templates);
+    }
+
+    public static Builder builder() {
+        return new Builder();
+    }
+
+    public Map<String, IndexTemplateMetadata> getTemplates() {
+        return this.templates;
+    }
+
+    public static TemplatesMetadata fromXContent(XContentParser parser) throws IOException {
+        return Builder.fromXContent(parser);
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        Builder.toXContent(this, builder, params);
+        return builder;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeVInt(templates.size());
+        for (final IndexTemplateMetadata cursor : templates.values()) {
+            cursor.writeTo(out);
+        }
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+
+        TemplatesMetadata that = (TemplatesMetadata) o;
+
+        return Objects.equals(templates, that.templates);
+    }
+
+    @Override
+    public int hashCode() {
+        return templates != null ? templates.hashCode() : 0;
+    }
+
+    public static class Builder {
+        private final Map<String, IndexTemplateMetadata> templates;
+
+        public Builder() {
+            this.templates = new HashMap<String, IndexTemplateMetadata>();
+        }
+
+        public Builder(Map<String, IndexTemplateMetadata> templates) {
+            this.templates = templates;
+        }
+
+        public Builder put(IndexTemplateMetadata.Builder template) {
+            return put(template.build());
+        }
+
+        public Builder put(IndexTemplateMetadata template) {
+            templates.put(template.name(), template);
+            return this;
+        }
+
+        public Builder removeTemplate(String templateName) {
+            templates.remove(templateName);
+            return this;
+        }
+
+        public Builder templates(Map<String, IndexTemplateMetadata> templates) {
+            this.templates.putAll(templates);
+            return this;
+        }
+
+        public TemplatesMetadata build() {
+            return new TemplatesMetadata(templates);
+        }
+
+        public static void toXContent(TemplatesMetadata templates, XContentBuilder builder, Params params) throws IOException {
+            // builder.startObject("templates-metadata");
+            for (IndexTemplateMetadata cursor : templates.getTemplates().values()) {
+                IndexTemplateMetadata.Builder.toXContentWithTypes(cursor, builder, params);
+            }
+            // builder.endObject();
+        }
+
+        public static TemplatesMetadata fromXContent(XContentParser parser) throws IOException {
+            Builder builder = new Builder();
+
+            // we might get here after the templates-metadata element, or on a fresh parser
+            XContentParser.Token token = parser.currentToken();
+            String currentFieldName = parser.currentName();
+            if (currentFieldName == null) {
+                token = parser.nextToken();
+                if (token == XContentParser.Token.START_OBJECT) {
+                    // move to the field name
+                    token = parser.nextToken();
+                    if (token == XContentParser.Token.FIELD_NAME) {
+                        // move to the next object
+                        token = parser.nextToken();
+                    }
+                }
+                currentFieldName = parser.currentName();
+            }
+            if (currentFieldName != null && token == XContentParser.Token.START_OBJECT) {
+                while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+                    builder.put(IndexTemplateMetadata.Builder.fromXContent(parser, parser.currentName()));
+                }
+            }
+            return builder.build();
+        }
+    }
+}