Add support for composable index templates (opensearch-project#437)

* Allow renaming URL path parts in generator Signed-off-by: Thomas Farr <[email protected]> * Generate {delete,exists,get,put}_index_template as *ComposableTemplate Signed-off-by: Thomas Farr <[email protected]> * Add resp/req bodies for *ComposableIndexTemplate Signed-off-by: Thomas Farr <[email protected]> * Fix tests Signed-off-by: Thomas Farr <[email protected]> * Add ComposableIndexTemplateCrudTests Signed-off-by: Thomas Farr <[email protected]> * Add ComposableIndexTemplateExists tests Signed-off-by: Thomas Farr <[email protected]> * Add DeleteComposableIndexTemplate tests Signed-off-by: Thomas Farr <[email protected]> * Add GetComposableIndexTemplate tests Signed-off-by: Thomas Farr <[email protected]> * Add PutComposableIndexTemplate tests Signed-off-by: Thomas Farr <[email protected]> * Non-overlapping templates Signed-off-by: Thomas Farr <[email protected]> * Fix tests Signed-off-by: Thomas Farr <[email protected]> * Test data_stream template mapping serialization Signed-off-by: Thomas Farr <[email protected]> * Update guide and add sample Signed-off-by: Thomas Farr <[email protected]> * Add changelog entry Signed-off-by: Thomas Farr <[email protected]> * Tidy generated code Signed-off-by: Thomas Farr <[email protected]> * Review feedback Signed-off-by: Thomas Farr <[email protected]> --------- Signed-off-by: Thomas Farr <[email protected]>
Xtansia · Dec 3, 2023 · ef15768 · ef15768
1 parent 74f9b6c
commit ef15768
Show file tree

Hide file tree

Showing 8 changed files with 102 additions and 85 deletions.
diff --git a/src/ApiGenerator/Configuration/CodeConfiguration.cs b/src/ApiGenerator/Configuration/CodeConfiguration.cs
@@ -51,6 +51,9 @@ public static class CodeConfiguration
 			new("cluster.*_component_template"),
 
 			new("dangling_indices.*"),
+
+			new("indices.{delete,exists,get,put}_index_template"),
+
 			new("ingest.*"),
             new("nodes.*"),
 			new("snapshot.*"),
@@ -62,11 +65,8 @@ public static class CodeConfiguration
 		/// <summary>
         /// Map API default names for API's we are only supporting on the low level client first
         /// </summary>
-        private static readonly Dictionary<string, string> LowLevelApiNameMapping = new Dictionary<string, string>
-        {
-            { "indices.delete_index_template", "DeleteIndexTemplateV2" },
-            { "indices.get_index_template", "GetIndexTemplateV2" },
-            { "indices.put_index_template", "PutIndexTemplateV2" }
+        private static readonly Dictionary<string, string> LowLevelApiNameMapping = new()
+		{
         };
 
         /// <summary>

diff --git a/src/ApiGenerator/Configuration/Overrides/EndpointOverridesBase.cs b/src/ApiGenerator/Configuration/Overrides/EndpointOverridesBase.cs
@@ -33,6 +33,8 @@ namespace ApiGenerator.Configuration.Overrides
 {
     public abstract class EndpointOverridesBase : IEndpointOverrides
     {
+		public virtual IDictionary<string, string> RenameUrlParts { get; } = new SortedDictionary<string, string>();
+
         public virtual IDictionary<string, string> ObsoleteQueryStringParams { get; set; } = new SortedDictionary<string, string>();
 
         public virtual IDictionary<string, string> RenameQueryStringParams { get; } = new SortedDictionary<string, string>();

diff --git a/src/ApiGenerator/Configuration/Overrides/GlobalOverrides.cs b/src/ApiGenerator/Configuration/Overrides/GlobalOverrides.cs
@@ -31,7 +31,11 @@
 namespace ApiGenerator.Configuration.Overrides
 {
     public class GlobalOverrides : EndpointOverridesBase
-    {
+	{
+		public static readonly GlobalOverrides Instance = new();
+
+		private GlobalOverrides() { }
+
         public IDictionary<string, Dictionary<string, string>> ObsoleteEnumMembers { get; set; } = new Dictionary<string, Dictionary<string, string>>()
         {
             { "VersionType", new Dictionary<string, string>() { { "force", "Force is no longer accepted by the server as of 7.5.0 and will result in an error when used" } } }

diff --git a/src/ApiGenerator/Configuration/Overrides/IEndpointOverrides.cs b/src/ApiGenerator/Configuration/Overrides/IEndpointOverrides.cs
@@ -35,10 +35,15 @@ namespace ApiGenerator.Configuration.Overrides
     /// </summary>
     public interface IEndpointOverrides
     {
+		/// <summary>
+		/// Override how the url part is exposed to the client.
+		/// </summary>
+		IDictionary<string, string> RenameUrlParts { get; }
+
         /// <summary>
         /// A map of key -> obsolete message for properties in the spec that should not be used any longer
         /// </summary>
-        IDictionary<string, string> ObsoleteQueryStringParams { get; set; }
+        IDictionary<string, string> ObsoleteQueryStringParams { get; }
 
         /// <summary>
         /// Override how the query param name is exposed to the client.

diff --git a/...rator/Domain/ApiQueryParametersPatcher.cs → ...tor/Domain/ApiRequestParametersPatcher.cs b/...rator/Domain/ApiQueryParametersPatcher.cs → ...tor/Domain/ApiRequestParametersPatcher.cs
@@ -34,23 +34,37 @@
 
 namespace ApiGenerator.Domain
 {
-    public static class ApiQueryParametersPatcher
+    public static class ApiRequestParametersPatcher
     {
-        public static SortedDictionary<string, QueryParameters> Patch(
+		public static void PatchUrlPaths(string endpointName, IList<UrlPath> source, IEndpointOverrides overrides)
+		{
+			var declaredKeys = source.SelectMany(p => p.Parts).Select(p => p.Name).ToHashSet();
+			var renameLookup = CreateUrlPartRenameLookup(overrides, declaredKeys);
+
+			foreach (var path in source)
+			{
+				foreach (var part in path.Parts)
+				{
+					if (!renameLookup.TryGetValue(part.Name, out var newName)) continue;
+
+					path.Path = path.Path.Replace($"{{{part.Name}}}", $"{{{newName}}}");
+					part.Name = newName;
+				}
+			}
+		}
+
+        public static SortedDictionary<string, QueryParameters> PatchQueryParameters(
             string endpointName,
             IDictionary<string, QueryParameters> source,
             IEndpointOverrides overrides
         )
         {
-            if (source == null) return null;
-
-            var globalOverrides = new GlobalOverrides();
             var declaredKeys = source.Keys;
-            var skipList = CreateSkipList(globalOverrides, overrides, declaredKeys);
-            var partialList = CreatePartialList(globalOverrides, overrides, declaredKeys);
+            var skipList = CreateSkipList(overrides, declaredKeys);
+            var partialList = CreatePartialList(overrides, declaredKeys);
 
-            var renameLookup = CreateRenameLookup(globalOverrides, overrides, declaredKeys);
-            var obsoleteLookup = CreateObsoleteLookup(globalOverrides, overrides, declaredKeys);
+            var renameLookup = CreateRenameLookup(overrides, declaredKeys);
+            var obsoleteLookup = CreateObsoleteLookup(overrides, declaredKeys);
 
             var patchedParams = new SortedDictionary<string, QueryParameters>();
             foreach (var (queryStringKey, value) in source)
@@ -96,18 +110,18 @@ private static string CreateCSharpName(string queryStringKey, string endpointNam
             }
         }
 
-        private static IList<string> CreateSkipList(IEndpointOverrides global, IEndpointOverrides local, ICollection<string> declaredKeys) =>
-            CreateList(global, local, "skip", e => e.SkipQueryStringParams, declaredKeys);
+        private static IList<string> CreateSkipList(IEndpointOverrides local, ICollection<string> declaredKeys) =>
+            CreateList(local, "skip", e => e.SkipQueryStringParams, declaredKeys);
 
-        private static IList<string> CreatePartialList(IEndpointOverrides global, IEndpointOverrides local, ICollection<string> declaredKeys) =>
-            CreateList(global, local, "partial", e => e.RenderPartial, declaredKeys);
+        private static IList<string> CreatePartialList(IEndpointOverrides local, ICollection<string> declaredKeys) =>
+            CreateList(local, "partial", e => e.RenderPartial, declaredKeys);
 
-        private static IDictionary<string, string> CreateLookup(IEndpointOverrides global, IEndpointOverrides local, string type,
+        private static IDictionary<string, string> CreateLookup(IEndpointOverrides local, string type,
             Func<IEndpointOverrides, IDictionary<string, string>> @from, ICollection<string> declaredKeys
         )
         {
             var d = new SortedDictionary<string, string>();
-            foreach (var kv in from(global)) d[kv.Key] = kv.Value;
+            foreach (var kv in from(GlobalOverrides.Instance)) d[kv.Key] = kv.Value;
 
             if (local == null) return d;
 
@@ -121,12 +135,12 @@ Func<IEndpointOverrides, IDictionary<string, string>> @from, ICollection<string>
             return d;
         }
 
-        private static IList<string> CreateList(IEndpointOverrides global, IEndpointOverrides local, string type,
+        private static IList<string> CreateList(IEndpointOverrides local, string type,
             Func<IEndpointOverrides, IEnumerable<string>> @from, ICollection<string> declaredKeys
         )
         {
             var list = new List<string>();
-            if (global != null) list.AddRange(from(global));
+            list.AddRange(from(GlobalOverrides.Instance));
             if (local != null)
             {
                 var localList = from(local).ToList();
@@ -138,14 +152,13 @@ Func<IEndpointOverrides, IEnumerable<string>> @from, ICollection<string> declare
             return list.Distinct().ToList();
         }
 
-        private static IDictionary<string, string> CreateRenameLookup(IEndpointOverrides global, IEndpointOverrides local,
-            ICollection<string> declaredKeys
-        ) =>
-            CreateLookup(global, local, "rename", e => e.RenameQueryStringParams, declaredKeys);
+		private static IDictionary<string, string> CreateUrlPartRenameLookup(IEndpointOverrides local, ICollection<string> declaredKeys) =>
+			CreateLookup(local, "url_part_rename", e => e.RenameUrlParts, declaredKeys);
+
+        private static IDictionary<string, string> CreateRenameLookup(IEndpointOverrides local, ICollection<string> declaredKeys) =>
+            CreateLookup(local, "rename", e => e.RenameQueryStringParams, declaredKeys);
 
-        private static IDictionary<string, string> CreateObsoleteLookup(IEndpointOverrides global, IEndpointOverrides local,
-            ICollection<string> declaredKeys
-        ) =>
-            CreateLookup(global, local, "obsolete", e => e.ObsoleteQueryStringParams, declaredKeys);
+        private static IDictionary<string, string> CreateObsoleteLookup(IEndpointOverrides local, ICollection<string> declaredKeys) =>
+            CreateLookup(local, "obsolete", e => e.ObsoleteQueryStringParams, declaredKeys);
     }
 }
diff --git a/src/ApiGenerator/Domain/Specification/UrlPath.cs b/src/ApiGenerator/Domain/Specification/UrlPath.cs
@@ -35,7 +35,7 @@ namespace ApiGenerator.Domain.Specification
     public class UrlPath
     {
         private readonly IList<UrlPart> _additionalPartsForConstructor;
-        public string Path { get; }
+        public string Path { get; set; }
         public Deprecation Deprecation { get; }
 		public Version VersionAdded { get; }
         public IList<UrlPart> Parts { get; }

diff --git a/src/ApiGenerator/Generator/ApiEndpointFactory.cs b/src/ApiGenerator/Generator/ApiEndpointFactory.cs
@@ -28,10 +28,8 @@
 
 using System;
 using System.Collections.Generic;
-using System.Collections.Immutable;
 using System.Linq;
 using System.Net.Mime;
-using System.Text.RegularExpressions;
 using ApiGenerator.Configuration;
 using ApiGenerator.Configuration.Overrides;
 using ApiGenerator.Domain;
@@ -40,7 +38,6 @@
 using NJsonSchema;
 using NJsonSchema.References;
 using NSwag;
-using SemanticVersioning;
 using Version = SemanticVersioning.Version;
 
 namespace ApiGenerator.Generator
@@ -52,6 +49,8 @@ public static ApiEndpoint From(string name, List<(string HttpPath, OpenApiPathIt
 	        var tokens = name.Split(".");
 	        var methodName = tokens[^1];
 	        var ns = tokens.Length > 1 ? tokens[0] : null;
+			var names = new CsharpNames(name, methodName, ns);
+			var overrides = LoadOverrides(name, names.MethodName);
 
 	        HashSet<string> requiredPathParts = null;
 	        var allParts = new Dictionary<string, UrlPart>();
@@ -126,6 +125,9 @@ public static ApiEndpoint From(string name, List<(string HttpPath, OpenApiPathIt
 					.Where(p => !canonicalPaths.ContainsKey(p.Key))
 					.Select(p => p.Value))
 				.ToList();
+
+			ApiRequestParametersPatcher.PatchUrlPaths(name, paths, overrides);
+
 			paths.Sort((p1, p2) => p1.Parts
 					.Zip(p2.Parts)
 					.Select(t => string.Compare(t.First.Name, t.Second.Name, StringComparison.Ordinal))
@@ -151,58 +153,52 @@ public static ApiEndpoint From(string name, List<(string HttpPath, OpenApiPathIt
 
 			foreach (var partName in requiredPathParts ?? Enumerable.Empty<string>()) allParts[partName].Required = true;
 
-	        var endpoint = new ApiEndpoint
-	        {
-	            Name = name,
-	            Namespace = ns,
-	            MethodName = methodName,
-	            CsharpNames = new CsharpNames(name, methodName, ns),
-	            Stability = Stability.Stable, // TODO: for realsies
-	            OfficialDocumentationLink = new Documentation
-	            {
-	                Description = variants[0].Operation.Description,
-	                Url = variants[0].Operation.ExternalDocumentation?.Url
-	            },
-	            Url = new UrlInformation
+			IDictionary<string, QueryParameters> queryParams = variants.SelectMany(v => v.Path.Parameters.Concat(v.Operation.Parameters))
+				.Where(p => p.Kind == OpenApiParameterKind.Query)
+				.DistinctBy(p => p.Name)
+				.ToDictionary(p => p.Name, BuildQueryParam);
+			queryParams = ApiRequestParametersPatcher.PatchQueryParameters(name, queryParams, overrides);
+
+			return new ApiEndpoint
+			{
+				Name = name,
+				Namespace = ns,
+				MethodName = methodName,
+				CsharpNames = names,
+				Overrides = overrides,
+				Stability = Stability.Stable, // TODO: for realsies
+				OfficialDocumentationLink = new Documentation
+				{
+					Description = variants[0].Operation.Description,
+					Url = variants[0].Operation.ExternalDocumentation?.Url
+				},
+				Url = new UrlInformation
 				{
 					AllPaths = paths,
-					Params = variants.SelectMany(v => v.Path.Parameters.Concat(v.Operation.Parameters))
-						.Where(p => p.Kind == OpenApiParameterKind.Query)
-						.DistinctBy(p => p.Name)
-						.ToImmutableSortedDictionary(p => p.Name, BuildQueryParam)
+					Params = queryParams
 				},
-	            Body = variants
+				Body = variants
 					.Select(v => v.Operation.RequestBody)
 					.FirstOrDefault(b => b != null) is { } reqBody
 					? new Body { Description = GetDescription(reqBody), Required = reqBody.IsRequired }
 					: null,
-	            HttpMethods = variants.Select(v => v.HttpMethod.ToString().ToUpper()).Distinct().ToList(),
-	        };
-
-	        LoadOverridesOnEndpoint(endpoint);
-	        PatchRequestParameters(endpoint);
-
-	        return endpoint;
+				HttpMethods = variants.Select(v => v.HttpMethod.ToString().ToUpper()).Distinct().ToList(),
+			};
 	    }
 
-	    private static void LoadOverridesOnEndpoint(ApiEndpoint endpoint)
+	    private static IEndpointOverrides LoadOverrides(string endpointName, string methodName)
 	    {
-	        var method = endpoint.CsharpNames.MethodName;
-	        if (CodeConfiguration.ApiNameMapping.TryGetValue(endpoint.Name, out var mapsApiMethodName))
-	            method = mapsApiMethodName;
+	        if (CodeConfiguration.ApiNameMapping.TryGetValue(endpointName, out var mapsApiMethodName))
+	            methodName = mapsApiMethodName;
 
 	        var namespacePrefix = $"{typeof(GlobalOverrides).Namespace}.Endpoints.";
-	        var typeName = $"{namespacePrefix}{method}Overrides";
+	        var typeName = $"{namespacePrefix}{methodName}Overrides";
 	        var type = GeneratorLocations.Assembly.GetType(typeName);
-	        if (type != null && Activator.CreateInstance(type) is IEndpointOverrides overrides)
-	            endpoint.Overrides = overrides;
-	    }
 
-	    private static void PatchRequestParameters(ApiEndpoint endpoint) =>
-	        endpoint.Url.Params = ApiQueryParametersPatcher.Patch(endpoint.Name, endpoint.Url.Params, endpoint.Overrides)
-	            ?? throw new ArgumentNullException("ApiQueryParametersPatcher.Patch(endpoint.Name, endpoint.Url.Params, endpoint.Overrides)");
+			return type != null && Activator.CreateInstance(type) is IEndpointOverrides overrides ? overrides : null;
+		}
 
-        private static QueryParameters BuildQueryParam(OpenApiParameter p)
+		private static QueryParameters BuildQueryParam(OpenApiParameter p)
         {
             var param = new QueryParameters
             {
@@ -214,11 +210,9 @@ private static QueryParameters BuildQueryParam(OpenApiParameter p)
             };
 
             if (param.Type == "enum" && p.Schema.HasReference)
-            {
-                param.ClsName = ((IJsonReference)p.Schema).ReferencePath.Split('/').Last();
-            }
+				param.ClsName = ((IJsonReference)p.Schema).ReferencePath.Split('/').Last();
 
-            return param;
+			return param;
         }
 
 	    private static string GetOpenSearchType(JsonSchema schema)

diff --git a/src/ApiGenerator/Views/LowLevel/Enums.cshtml b/src/ApiGenerator/Views/LowLevel/Enums.cshtml
@@ -8,13 +8,12 @@
 @functions {
     private const string RawSize = "Raw";
     private const string SizeEnum = "Size";
-    private static GlobalOverrides GlobalOverrides = new GlobalOverrides();
 
     private string CreateEnum(string enumName, string value, int? i)
     {
         var enumValue = (enumName == SizeEnum && value == string.Empty) ? RawSize : value.ToPascalCase(true);
         var enumCsharp = string.Format("[EnumMember(Value = \"{0}\")] {1}{2}", value, enumValue, i.HasValue ? " = 1 << " + i.Value : null);
-        if (GlobalOverrides.ObsoleteEnumMembers.TryGetValue(enumName, out var d) && d.TryGetValue(value, out var obsolete))
+        if (GlobalOverrides.Instance.ObsoleteEnumMembers.TryGetValue(enumName, out var d) && d.TryGetValue(value, out var obsolete))
         {
             return string.Format("[Obsolete(\"{0}\")]{2}\t\t{1}", obsolete, enumCsharp, Environment.NewLine);
         }
@@ -23,7 +22,7 @@
     private string CreateCase(string e, string o)
     {
         var enumValue = GetEnumValue(e, o);
-        var isObsolete = GlobalOverrides.ObsoleteEnumMembers.TryGetValue(e, out var d) && d.TryGetValue(o, out _);
+        var isObsolete = GlobalOverrides.Instance.ObsoleteEnumMembers.TryGetValue(e, out var d) && d.TryGetValue(o, out _);
         var sb = new StringBuilder();
         if (isObsolete) sb.AppendLine("#pragma warning disable 618");
         sb.Append(string.Format("case {0}.{1}: return \"{2}\";", e, enumValue, o));
@@ -37,8 +36,8 @@
 
     private string GetEnumValue(string enumName, string value)
     {
-        return enumName == SizeEnum && value == string.Empty 
-            ? RawSize 
+        return enumName == SizeEnum && value == string.Empty
+            ? RawSize
             : value.ToPascalCase(true);
     }
 }
@@ -91,20 +90,20 @@ namespace OpenSearch.Net
             }
 <text>          var list = new @(Raw("List<string>()"));
 </text>
-            var g = GlobalOverrides.ObsoleteEnumMembers.TryGetValue(e.Name, out var d);
+            var g = GlobalOverrides.Instance.ObsoleteEnumMembers.TryGetValue(e.Name, out var d);
             foreach (var option in e.Options.Where(o => o != "_all"))
             {
                 var value = GetEnumValue(e.Name, option);
                 if (g && d.TryGetValue(option, out var _)) {
 <text>#pragma warning disable 618
             if ((enumValue & @(e.Name).@(value)) != 0) list.Add("@(option)");
 #pragma warning restore 618
-</text>                                 
+</text>
                 }
                 else {
 <text>          if ((enumValue & @(e.Name).@(value)) != 0) list.Add("@(option)");
 </text>
-                }                                                                                          
+                }
             }
 <text>          return string.Join(",", list);
         }</text>