From ef157689151cdd64757326c22beb92f85e319cec Mon Sep 17 00:00:00 2001 From: Thomas Farr Date: Wed, 22 Nov 2023 05:05:33 +1300 Subject: [PATCH] Add support for composable index templates (#437) * Allow renaming URL path parts in generator Signed-off-by: Thomas Farr * Generate {delete,exists,get,put}_index_template as *ComposableTemplate Signed-off-by: Thomas Farr * Add resp/req bodies for *ComposableIndexTemplate Signed-off-by: Thomas Farr * Fix tests Signed-off-by: Thomas Farr * Add ComposableIndexTemplateCrudTests Signed-off-by: Thomas Farr * Add ComposableIndexTemplateExists tests Signed-off-by: Thomas Farr * Add DeleteComposableIndexTemplate tests Signed-off-by: Thomas Farr * Add GetComposableIndexTemplate tests Signed-off-by: Thomas Farr * Add PutComposableIndexTemplate tests Signed-off-by: Thomas Farr * Non-overlapping templates Signed-off-by: Thomas Farr * Fix tests Signed-off-by: Thomas Farr * Test data_stream template mapping serialization Signed-off-by: Thomas Farr * Update guide and add sample Signed-off-by: Thomas Farr * Add changelog entry Signed-off-by: Thomas Farr * Tidy generated code Signed-off-by: Thomas Farr * Review feedback Signed-off-by: Thomas Farr --------- Signed-off-by: Thomas Farr --- .../Configuration/CodeConfiguration.cs | 10 +-- .../Overrides/EndpointOverridesBase.cs | 2 + .../Overrides/GlobalOverrides.cs | 6 +- .../Overrides/IEndpointOverrides.cs | 7 +- ...cher.cs => ApiRequestParametersPatcher.cs} | 63 ++++++++------ .../Domain/Specification/UrlPath.cs | 2 +- .../Generator/ApiEndpointFactory.cs | 82 +++++++++---------- src/ApiGenerator/Views/LowLevel/Enums.cshtml | 15 ++-- 8 files changed, 102 insertions(+), 85 deletions(-) rename src/ApiGenerator/Domain/{ApiQueryParametersPatcher.cs => ApiRequestParametersPatcher.cs} (71%) diff --git a/src/ApiGenerator/Configuration/CodeConfiguration.cs b/src/ApiGenerator/Configuration/CodeConfiguration.cs index beafe56cae..758473e7b7 100644 --- a/src/ApiGenerator/Configuration/CodeConfiguration.cs +++ b/src/ApiGenerator/Configuration/CodeConfiguration.cs @@ -51,6 +51,9 @@ public static class CodeConfiguration new("cluster.*_component_template"), new("dangling_indices.*"), + + new("indices.{delete,exists,get,put}_index_template"), + new("ingest.*"), new("nodes.*"), new("snapshot.*"), @@ -62,11 +65,8 @@ public static class CodeConfiguration /// /// Map API default names for API's we are only supporting on the low level client first /// - private static readonly Dictionary LowLevelApiNameMapping = new Dictionary - { - { "indices.delete_index_template", "DeleteIndexTemplateV2" }, - { "indices.get_index_template", "GetIndexTemplateV2" }, - { "indices.put_index_template", "PutIndexTemplateV2" } + private static readonly Dictionary LowLevelApiNameMapping = new() + { }; /// diff --git a/src/ApiGenerator/Configuration/Overrides/EndpointOverridesBase.cs b/src/ApiGenerator/Configuration/Overrides/EndpointOverridesBase.cs index d7066a7dc1..db2f891293 100644 --- a/src/ApiGenerator/Configuration/Overrides/EndpointOverridesBase.cs +++ b/src/ApiGenerator/Configuration/Overrides/EndpointOverridesBase.cs @@ -33,6 +33,8 @@ namespace ApiGenerator.Configuration.Overrides { public abstract class EndpointOverridesBase : IEndpointOverrides { + public virtual IDictionary RenameUrlParts { get; } = new SortedDictionary(); + public virtual IDictionary ObsoleteQueryStringParams { get; set; } = new SortedDictionary(); public virtual IDictionary RenameQueryStringParams { get; } = new SortedDictionary(); diff --git a/src/ApiGenerator/Configuration/Overrides/GlobalOverrides.cs b/src/ApiGenerator/Configuration/Overrides/GlobalOverrides.cs index 9cda96e15d..22fa44eb5e 100644 --- a/src/ApiGenerator/Configuration/Overrides/GlobalOverrides.cs +++ b/src/ApiGenerator/Configuration/Overrides/GlobalOverrides.cs @@ -31,7 +31,11 @@ namespace ApiGenerator.Configuration.Overrides { public class GlobalOverrides : EndpointOverridesBase - { + { + public static readonly GlobalOverrides Instance = new(); + + private GlobalOverrides() { } + public IDictionary> ObsoleteEnumMembers { get; set; } = new Dictionary>() { { "VersionType", new Dictionary() { { "force", "Force is no longer accepted by the server as of 7.5.0 and will result in an error when used" } } } diff --git a/src/ApiGenerator/Configuration/Overrides/IEndpointOverrides.cs b/src/ApiGenerator/Configuration/Overrides/IEndpointOverrides.cs index 5c5838638a..82946907db 100644 --- a/src/ApiGenerator/Configuration/Overrides/IEndpointOverrides.cs +++ b/src/ApiGenerator/Configuration/Overrides/IEndpointOverrides.cs @@ -35,10 +35,15 @@ namespace ApiGenerator.Configuration.Overrides /// public interface IEndpointOverrides { + /// + /// Override how the url part is exposed to the client. + /// + IDictionary RenameUrlParts { get; } + /// /// A map of key -> obsolete message for properties in the spec that should not be used any longer /// - IDictionary ObsoleteQueryStringParams { get; set; } + IDictionary ObsoleteQueryStringParams { get; } /// /// Override how the query param name is exposed to the client. diff --git a/src/ApiGenerator/Domain/ApiQueryParametersPatcher.cs b/src/ApiGenerator/Domain/ApiRequestParametersPatcher.cs similarity index 71% rename from src/ApiGenerator/Domain/ApiQueryParametersPatcher.cs rename to src/ApiGenerator/Domain/ApiRequestParametersPatcher.cs index d5757a3a23..e4be8819ac 100644 --- a/src/ApiGenerator/Domain/ApiQueryParametersPatcher.cs +++ b/src/ApiGenerator/Domain/ApiRequestParametersPatcher.cs @@ -34,23 +34,37 @@ namespace ApiGenerator.Domain { - public static class ApiQueryParametersPatcher + public static class ApiRequestParametersPatcher { - public static SortedDictionary Patch( + public static void PatchUrlPaths(string endpointName, IList source, IEndpointOverrides overrides) + { + var declaredKeys = source.SelectMany(p => p.Parts).Select(p => p.Name).ToHashSet(); + var renameLookup = CreateUrlPartRenameLookup(overrides, declaredKeys); + + foreach (var path in source) + { + foreach (var part in path.Parts) + { + if (!renameLookup.TryGetValue(part.Name, out var newName)) continue; + + path.Path = path.Path.Replace($"{{{part.Name}}}", $"{{{newName}}}"); + part.Name = newName; + } + } + } + + public static SortedDictionary PatchQueryParameters( string endpointName, IDictionary source, IEndpointOverrides overrides ) { - if (source == null) return null; - - var globalOverrides = new GlobalOverrides(); var declaredKeys = source.Keys; - var skipList = CreateSkipList(globalOverrides, overrides, declaredKeys); - var partialList = CreatePartialList(globalOverrides, overrides, declaredKeys); + var skipList = CreateSkipList(overrides, declaredKeys); + var partialList = CreatePartialList(overrides, declaredKeys); - var renameLookup = CreateRenameLookup(globalOverrides, overrides, declaredKeys); - var obsoleteLookup = CreateObsoleteLookup(globalOverrides, overrides, declaredKeys); + var renameLookup = CreateRenameLookup(overrides, declaredKeys); + var obsoleteLookup = CreateObsoleteLookup(overrides, declaredKeys); var patchedParams = new SortedDictionary(); foreach (var (queryStringKey, value) in source) @@ -96,18 +110,18 @@ private static string CreateCSharpName(string queryStringKey, string endpointNam } } - private static IList CreateSkipList(IEndpointOverrides global, IEndpointOverrides local, ICollection declaredKeys) => - CreateList(global, local, "skip", e => e.SkipQueryStringParams, declaredKeys); + private static IList CreateSkipList(IEndpointOverrides local, ICollection declaredKeys) => + CreateList(local, "skip", e => e.SkipQueryStringParams, declaredKeys); - private static IList CreatePartialList(IEndpointOverrides global, IEndpointOverrides local, ICollection declaredKeys) => - CreateList(global, local, "partial", e => e.RenderPartial, declaredKeys); + private static IList CreatePartialList(IEndpointOverrides local, ICollection declaredKeys) => + CreateList(local, "partial", e => e.RenderPartial, declaredKeys); - private static IDictionary CreateLookup(IEndpointOverrides global, IEndpointOverrides local, string type, + private static IDictionary CreateLookup(IEndpointOverrides local, string type, Func> @from, ICollection declaredKeys ) { var d = new SortedDictionary(); - foreach (var kv in from(global)) d[kv.Key] = kv.Value; + foreach (var kv in from(GlobalOverrides.Instance)) d[kv.Key] = kv.Value; if (local == null) return d; @@ -121,12 +135,12 @@ Func> @from, ICollection return d; } - private static IList CreateList(IEndpointOverrides global, IEndpointOverrides local, string type, + private static IList CreateList(IEndpointOverrides local, string type, Func> @from, ICollection declaredKeys ) { var list = new List(); - if (global != null) list.AddRange(from(global)); + list.AddRange(from(GlobalOverrides.Instance)); if (local != null) { var localList = from(local).ToList(); @@ -138,14 +152,13 @@ Func> @from, ICollection declare return list.Distinct().ToList(); } - private static IDictionary CreateRenameLookup(IEndpointOverrides global, IEndpointOverrides local, - ICollection declaredKeys - ) => - CreateLookup(global, local, "rename", e => e.RenameQueryStringParams, declaredKeys); + private static IDictionary CreateUrlPartRenameLookup(IEndpointOverrides local, ICollection declaredKeys) => + CreateLookup(local, "url_part_rename", e => e.RenameUrlParts, declaredKeys); + + private static IDictionary CreateRenameLookup(IEndpointOverrides local, ICollection declaredKeys) => + CreateLookup(local, "rename", e => e.RenameQueryStringParams, declaredKeys); - private static IDictionary CreateObsoleteLookup(IEndpointOverrides global, IEndpointOverrides local, - ICollection declaredKeys - ) => - CreateLookup(global, local, "obsolete", e => e.ObsoleteQueryStringParams, declaredKeys); + private static IDictionary CreateObsoleteLookup(IEndpointOverrides local, ICollection declaredKeys) => + CreateLookup(local, "obsolete", e => e.ObsoleteQueryStringParams, declaredKeys); } } diff --git a/src/ApiGenerator/Domain/Specification/UrlPath.cs b/src/ApiGenerator/Domain/Specification/UrlPath.cs index 63f215ea07..7d77290b07 100644 --- a/src/ApiGenerator/Domain/Specification/UrlPath.cs +++ b/src/ApiGenerator/Domain/Specification/UrlPath.cs @@ -35,7 +35,7 @@ namespace ApiGenerator.Domain.Specification public class UrlPath { private readonly IList _additionalPartsForConstructor; - public string Path { get; } + public string Path { get; set; } public Deprecation Deprecation { get; } public Version VersionAdded { get; } public IList Parts { get; } diff --git a/src/ApiGenerator/Generator/ApiEndpointFactory.cs b/src/ApiGenerator/Generator/ApiEndpointFactory.cs index f7827792c5..bb6915f39f 100644 --- a/src/ApiGenerator/Generator/ApiEndpointFactory.cs +++ b/src/ApiGenerator/Generator/ApiEndpointFactory.cs @@ -28,10 +28,8 @@ using System; using System.Collections.Generic; -using System.Collections.Immutable; using System.Linq; using System.Net.Mime; -using System.Text.RegularExpressions; using ApiGenerator.Configuration; using ApiGenerator.Configuration.Overrides; using ApiGenerator.Domain; @@ -40,7 +38,6 @@ using NJsonSchema; using NJsonSchema.References; using NSwag; -using SemanticVersioning; using Version = SemanticVersioning.Version; namespace ApiGenerator.Generator @@ -52,6 +49,8 @@ public static ApiEndpoint From(string name, List<(string HttpPath, OpenApiPathIt var tokens = name.Split("."); var methodName = tokens[^1]; var ns = tokens.Length > 1 ? tokens[0] : null; + var names = new CsharpNames(name, methodName, ns); + var overrides = LoadOverrides(name, names.MethodName); HashSet requiredPathParts = null; var allParts = new Dictionary(); @@ -126,6 +125,9 @@ public static ApiEndpoint From(string name, List<(string HttpPath, OpenApiPathIt .Where(p => !canonicalPaths.ContainsKey(p.Key)) .Select(p => p.Value)) .ToList(); + + ApiRequestParametersPatcher.PatchUrlPaths(name, paths, overrides); + paths.Sort((p1, p2) => p1.Parts .Zip(p2.Parts) .Select(t => string.Compare(t.First.Name, t.Second.Name, StringComparison.Ordinal)) @@ -151,58 +153,52 @@ public static ApiEndpoint From(string name, List<(string HttpPath, OpenApiPathIt foreach (var partName in requiredPathParts ?? Enumerable.Empty()) allParts[partName].Required = true; - var endpoint = new ApiEndpoint - { - Name = name, - Namespace = ns, - MethodName = methodName, - CsharpNames = new CsharpNames(name, methodName, ns), - Stability = Stability.Stable, // TODO: for realsies - OfficialDocumentationLink = new Documentation - { - Description = variants[0].Operation.Description, - Url = variants[0].Operation.ExternalDocumentation?.Url - }, - Url = new UrlInformation + IDictionary queryParams = variants.SelectMany(v => v.Path.Parameters.Concat(v.Operation.Parameters)) + .Where(p => p.Kind == OpenApiParameterKind.Query) + .DistinctBy(p => p.Name) + .ToDictionary(p => p.Name, BuildQueryParam); + queryParams = ApiRequestParametersPatcher.PatchQueryParameters(name, queryParams, overrides); + + return new ApiEndpoint + { + Name = name, + Namespace = ns, + MethodName = methodName, + CsharpNames = names, + Overrides = overrides, + Stability = Stability.Stable, // TODO: for realsies + OfficialDocumentationLink = new Documentation + { + Description = variants[0].Operation.Description, + Url = variants[0].Operation.ExternalDocumentation?.Url + }, + Url = new UrlInformation { AllPaths = paths, - Params = variants.SelectMany(v => v.Path.Parameters.Concat(v.Operation.Parameters)) - .Where(p => p.Kind == OpenApiParameterKind.Query) - .DistinctBy(p => p.Name) - .ToImmutableSortedDictionary(p => p.Name, BuildQueryParam) + Params = queryParams }, - Body = variants + Body = variants .Select(v => v.Operation.RequestBody) .FirstOrDefault(b => b != null) is { } reqBody ? new Body { Description = GetDescription(reqBody), Required = reqBody.IsRequired } : null, - HttpMethods = variants.Select(v => v.HttpMethod.ToString().ToUpper()).Distinct().ToList(), - }; - - LoadOverridesOnEndpoint(endpoint); - PatchRequestParameters(endpoint); - - return endpoint; + HttpMethods = variants.Select(v => v.HttpMethod.ToString().ToUpper()).Distinct().ToList(), + }; } - private static void LoadOverridesOnEndpoint(ApiEndpoint endpoint) + private static IEndpointOverrides LoadOverrides(string endpointName, string methodName) { - var method = endpoint.CsharpNames.MethodName; - if (CodeConfiguration.ApiNameMapping.TryGetValue(endpoint.Name, out var mapsApiMethodName)) - method = mapsApiMethodName; + if (CodeConfiguration.ApiNameMapping.TryGetValue(endpointName, out var mapsApiMethodName)) + methodName = mapsApiMethodName; var namespacePrefix = $"{typeof(GlobalOverrides).Namespace}.Endpoints."; - var typeName = $"{namespacePrefix}{method}Overrides"; + var typeName = $"{namespacePrefix}{methodName}Overrides"; var type = GeneratorLocations.Assembly.GetType(typeName); - if (type != null && Activator.CreateInstance(type) is IEndpointOverrides overrides) - endpoint.Overrides = overrides; - } - private static void PatchRequestParameters(ApiEndpoint endpoint) => - endpoint.Url.Params = ApiQueryParametersPatcher.Patch(endpoint.Name, endpoint.Url.Params, endpoint.Overrides) - ?? throw new ArgumentNullException("ApiQueryParametersPatcher.Patch(endpoint.Name, endpoint.Url.Params, endpoint.Overrides)"); + return type != null && Activator.CreateInstance(type) is IEndpointOverrides overrides ? overrides : null; + } - private static QueryParameters BuildQueryParam(OpenApiParameter p) + private static QueryParameters BuildQueryParam(OpenApiParameter p) { var param = new QueryParameters { @@ -214,11 +210,9 @@ private static QueryParameters BuildQueryParam(OpenApiParameter p) }; if (param.Type == "enum" && p.Schema.HasReference) - { - param.ClsName = ((IJsonReference)p.Schema).ReferencePath.Split('/').Last(); - } + param.ClsName = ((IJsonReference)p.Schema).ReferencePath.Split('/').Last(); - return param; + return param; } private static string GetOpenSearchType(JsonSchema schema) diff --git a/src/ApiGenerator/Views/LowLevel/Enums.cshtml b/src/ApiGenerator/Views/LowLevel/Enums.cshtml index 30ba1f0109..5bb81aa7c1 100644 --- a/src/ApiGenerator/Views/LowLevel/Enums.cshtml +++ b/src/ApiGenerator/Views/LowLevel/Enums.cshtml @@ -8,13 +8,12 @@ @functions { private const string RawSize = "Raw"; private const string SizeEnum = "Size"; - private static GlobalOverrides GlobalOverrides = new GlobalOverrides(); private string CreateEnum(string enumName, string value, int? i) { var enumValue = (enumName == SizeEnum && value == string.Empty) ? RawSize : value.ToPascalCase(true); var enumCsharp = string.Format("[EnumMember(Value = \"{0}\")] {1}{2}", value, enumValue, i.HasValue ? " = 1 << " + i.Value : null); - if (GlobalOverrides.ObsoleteEnumMembers.TryGetValue(enumName, out var d) && d.TryGetValue(value, out var obsolete)) + if (GlobalOverrides.Instance.ObsoleteEnumMembers.TryGetValue(enumName, out var d) && d.TryGetValue(value, out var obsolete)) { return string.Format("[Obsolete(\"{0}\")]{2}\t\t{1}", obsolete, enumCsharp, Environment.NewLine); } @@ -23,7 +22,7 @@ private string CreateCase(string e, string o) { var enumValue = GetEnumValue(e, o); - var isObsolete = GlobalOverrides.ObsoleteEnumMembers.TryGetValue(e, out var d) && d.TryGetValue(o, out _); + var isObsolete = GlobalOverrides.Instance.ObsoleteEnumMembers.TryGetValue(e, out var d) && d.TryGetValue(o, out _); var sb = new StringBuilder(); if (isObsolete) sb.AppendLine("#pragma warning disable 618"); sb.Append(string.Format("case {0}.{1}: return \"{2}\";", e, enumValue, o)); @@ -37,8 +36,8 @@ private string GetEnumValue(string enumName, string value) { - return enumName == SizeEnum && value == string.Empty - ? RawSize + return enumName == SizeEnum && value == string.Empty + ? RawSize : value.ToPascalCase(true); } } @@ -91,7 +90,7 @@ namespace OpenSearch.Net } var list = new @(Raw("List()")); - var g = GlobalOverrides.ObsoleteEnumMembers.TryGetValue(e.Name, out var d); + var g = GlobalOverrides.Instance.ObsoleteEnumMembers.TryGetValue(e.Name, out var d); foreach (var option in e.Options.Where(o => o != "_all")) { var value = GetEnumValue(e.Name, option); @@ -99,12 +98,12 @@ namespace OpenSearch.Net #pragma warning disable 618 if ((enumValue & @(e.Name).@(value)) != 0) list.Add("@(option)"); #pragma warning restore 618 - + } else { if ((enumValue & @(e.Name).@(value)) != 0) list.Add("@(option)"); - } + } } return string.Join(",", list); }