From dd208324f890aece8706cfdd80f5b70473ed4f6e Mon Sep 17 00:00:00 2001 From: tombuildsstuff Date: Wed, 25 Oct 2023 10:37:02 +0200 Subject: [PATCH 1/6] tools/importer-rest-api-specs: updating the `metadata` file to output the DataSource and SourceInformation This makes more sense as the directory level, rather than nested under each API Version and allows us to merge directories of data together when parsing them --- .../models/api_version_definition.go | 3 --- .../dataapigeneratorjson/models/metadata.go | 20 +++++++++++++++++-- .../dataapigeneratorjson/revision_id.go | 4 +++- .../templater_api_version_definition.go | 1 - 4 files changed, 21 insertions(+), 7 deletions(-) diff --git a/tools/importer-rest-api-specs/components/dataapigeneratorjson/models/api_version_definition.go b/tools/importer-rest-api-specs/components/dataapigeneratorjson/models/api_version_definition.go index 4fcb6639cdd..ff16dc0a883 100644 --- a/tools/importer-rest-api-specs/components/dataapigeneratorjson/models/api_version_definition.go +++ b/tools/importer-rest-api-specs/components/dataapigeneratorjson/models/api_version_definition.go @@ -12,9 +12,6 @@ type ApiVersionDefinition struct { // Generate specifies whether this API Version should be generated or not. Generate bool `json:"generate"` - // Source specifies where the source data for this API version came from. - Source ApiDefinitionsSource `json:"source"` // TODO: move this into the MetaData file? - // Resources specifies a list of Api Resource names that exist within this API version. Resources []string `json:"resources"` } diff --git a/tools/importer-rest-api-specs/components/dataapigeneratorjson/models/metadata.go b/tools/importer-rest-api-specs/components/dataapigeneratorjson/models/metadata.go index 051ccaeab4a..d7d892b50f1 100644 --- a/tools/importer-rest-api-specs/components/dataapigeneratorjson/models/metadata.go +++ b/tools/importer-rest-api-specs/components/dataapigeneratorjson/models/metadata.go @@ -2,8 +2,14 @@ package models // MetaData contains Meta Data about this type MetaData struct { - // TODO: the license information could make sense to go in here, since all items within a - // directory should be under the same license + // DataSource specifies the type of Data that this Source is related to + // for example `AzureResourceManager`. This allows multiple directories + // to be used to populate the same Data Source (for example, auto-generated + // and handwritten) and then coalesced together. + DataSource DataSource `json:"dataSource"` + + // SourceInformation specifies where the data within this directory was sourced. + SourceInformation ApiDefinitionsSource `json:"sourceInformation"` // GitRevision specified the Git Revision (SHA) of the Repository // where this data has been sourced from. @@ -12,6 +18,16 @@ type MetaData struct { GitRevision *string `json:"gitRevision"` } +type DataSource string + +const ( + // AzureResourceManagerDataSource specifies that this Data is related to Azure Resource Manager. + AzureResourceManagerDataSource DataSource = "AzureResourceManager" + + // MicrosoftGraphDataSource specifies that this Data is for Microsoft Graph. + MicrosoftGraphDataSource DataSource = "MicrosoftGraph" +) + type ApiDefinitionsSource string const ( diff --git a/tools/importer-rest-api-specs/components/dataapigeneratorjson/revision_id.go b/tools/importer-rest-api-specs/components/dataapigeneratorjson/revision_id.go index 673e24acc64..01a25cb5c84 100644 --- a/tools/importer-rest-api-specs/components/dataapigeneratorjson/revision_id.go +++ b/tools/importer-rest-api-specs/components/dataapigeneratorjson/revision_id.go @@ -11,7 +11,9 @@ import ( func OutputMetaData(workingDirectory, swaggerGitSha string) error { metaData := dataApiModels.MetaData{ - GitRevision: pointer.To(swaggerGitSha), + DataSource: dataApiModels.AzureResourceManagerDataSource, + SourceInformation: dataApiModels.AzureRestApiSpecsRepositoryApiDefinitionsSource, + GitRevision: pointer.To(swaggerGitSha), } data, err := json.MarshalIndent(&metaData, "", "\t") diff --git a/tools/importer-rest-api-specs/components/dataapigeneratorjson/templater_api_version_definition.go b/tools/importer-rest-api-specs/components/dataapigeneratorjson/templater_api_version_definition.go index d744682b332..f2fbe4a0b89 100644 --- a/tools/importer-rest-api-specs/components/dataapigeneratorjson/templater_api_version_definition.go +++ b/tools/importer-rest-api-specs/components/dataapigeneratorjson/templater_api_version_definition.go @@ -11,7 +11,6 @@ func buildApiVersionDefinition(apiVersion string, isPreview bool, resources map[ versionDefinition := dataApiModels.ApiVersionDefinition{ ApiVersion: apiVersion, IsPreview: isPreview, - Source: dataApiModels.AzureRestApiSpecsRepositoryApiDefinitionsSource, Generate: true, } From 7072fe0ab4ab62f8a66aadaa219ceb9b07ab0ee7 Mon Sep 17 00:00:00 2001 From: tombuildsstuff Date: Wed, 25 Oct 2023 10:43:49 +0200 Subject: [PATCH 2/6] tools/importer-rest-api-specs: fixing the resource id templater - subscriptions aren't Static values so don't need this check --- .../components/dataapigeneratorjson/templater_resource_id.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/tools/importer-rest-api-specs/components/dataapigeneratorjson/templater_resource_id.go b/tools/importer-rest-api-specs/components/dataapigeneratorjson/templater_resource_id.go index 32adcc5fd09..c498a13be95 100644 --- a/tools/importer-rest-api-specs/components/dataapigeneratorjson/templater_resource_id.go +++ b/tools/importer-rest-api-specs/components/dataapigeneratorjson/templater_resource_id.go @@ -83,9 +83,6 @@ func mapResourceIdSegment(input resourcemanager.ResourceIdSegment) (*dataApiMode } if input.Type == resourcemanager.SubscriptionIdSegment { - if input.FixedValue == nil { - return nil, fmt.Errorf("static segment type with missing fixed value: %+v", input) - } return &dataApiModels.ResourceIdSegment{ Type: dataApiModels.SubscriptionIdResourceIdSegmentType, Name: input.Name, From ef85f38d3aa7763980385d7610c5df0aae9299b5 Mon Sep 17 00:00:00 2001 From: tombuildsstuff Date: Wed, 25 Oct 2023 10:44:20 +0200 Subject: [PATCH 3/6] tools/importer-rest-api-specs: conditionally mapping the request/response object These are optional so mapping them will fail --- .../templater_operations.go | 27 ++++++++++--------- 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/tools/importer-rest-api-specs/components/dataapigeneratorjson/templater_operations.go b/tools/importer-rest-api-specs/components/dataapigeneratorjson/templater_operations.go index 087b303b018..d2c25b9d2f6 100644 --- a/tools/importer-rest-api-specs/components/dataapigeneratorjson/templater_operations.go +++ b/tools/importer-rest-api-specs/components/dataapigeneratorjson/templater_operations.go @@ -42,16 +42,6 @@ func codeForOperation(operationName string, operation importerModels.OperationDe uriSuffix = *operation.UriSuffix } - responseObject, err := mapObjectDefinition(operation.ResponseObject, knownConstants, knownModels) - if err != nil { - return nil, fmt.Errorf("mapping the object definition: %+v", err) - } - - requestObject, err := mapObjectDefinition(operation.RequestObject, knownConstants, knownModels) - if err != nil { - return nil, fmt.Errorf("mapping the object definition: %+v", err) - } - op := dataApiModels.Operation{ Name: operationName, ContentType: contentType, @@ -60,11 +50,24 @@ func codeForOperation(operationName string, operation importerModels.OperationDe LongRunning: longRunning, HTTPMethod: strings.ToUpper(operation.Method), ResourceIdName: pointer.To(resourceId), - RequestObject: responseObject, - ResponseObject: requestObject, UriSuffix: pointer.To(uriSuffix), } + if operation.RequestObject != nil { + requestObject, err := mapObjectDefinition(operation.RequestObject, knownConstants, knownModels) + if err != nil { + return nil, fmt.Errorf("mapping the request object definition: %+v", err) + } + op.RequestObject = requestObject + } + if operation.ResponseObject != nil { + responseObject, err := mapObjectDefinition(operation.ResponseObject, knownConstants, knownModels) + if err != nil { + return nil, fmt.Errorf("mapping the response object definition: %+v", err) + } + op.ResponseObject = responseObject + } + if len(operation.Options) > 0 { options := make([]dataApiModels.Option, 0) sortedOptionsKeys := make([]string, 0) From f3d7c0e31b572c84ac623e50344a4d4cde1b16c8 Mon Sep 17 00:00:00 2001 From: tombuildsstuff Date: Wed, 25 Oct 2023 10:45:48 +0200 Subject: [PATCH 4/6] tools/importer-rest-api-specs: updating `operations` to use `input` and `output` rather than `operation` and `op` Also ensuring the UriSuffix, ResourceIDName and ContentType aren't nil --- .../templater_operations.go | 63 +++++++------------ 1 file changed, 21 insertions(+), 42 deletions(-) diff --git a/tools/importer-rest-api-specs/components/dataapigeneratorjson/templater_operations.go b/tools/importer-rest-api-specs/components/dataapigeneratorjson/templater_operations.go index d2c25b9d2f6..991ddf91b96 100644 --- a/tools/importer-rest-api-specs/components/dataapigeneratorjson/templater_operations.go +++ b/tools/importer-rest-api-specs/components/dataapigeneratorjson/templater_operations.go @@ -12,72 +12,51 @@ import ( "github.com/hashicorp/pandora/tools/sdk/resourcemanager" ) -func codeForOperation(operationName string, operation importerModels.OperationDetails, knownConstants map[string]resourcemanager.ConstantDetails, knownModels map[string]importerModels.ModelDetails) ([]byte, error) { - contentType := "" - if !strings.Contains(strings.ToLower(operation.ContentType), "application/json") { - contentType = operation.ContentType - } - +func codeForOperation(operationName string, input importerModels.OperationDetails, knownConstants map[string]resourcemanager.ConstantDetails, knownModels map[string]importerModels.ModelDetails) ([]byte, error) { statusCodes := make([]int, 0) - if usesNonDefaultStatusCodes(operation) { - for _, sc := range operation.ExpectedStatusCodes { + if usesNonDefaultStatusCodes(input) { + for _, sc := range input.ExpectedStatusCodes { statusCodes = append(statusCodes, sc) } sort.Ints(statusCodes) } - longRunning := false - if operation.LongRunning { - // TODO: we can use the `LongRunning` operation base types too - longRunning = true - } - - resourceId := "" - if operation.ResourceIdName != nil { - resourceId = *operation.ResourceIdName - } - - uriSuffix := "" - if operation.UriSuffix != nil { - uriSuffix = *operation.UriSuffix - } - - op := dataApiModels.Operation{ + output := dataApiModels.Operation{ Name: operationName, - ContentType: contentType, + ContentType: input.ContentType, ExpectedStatusCodes: statusCodes, - FieldContainingPaginationDetails: operation.FieldContainingPaginationDetails, - LongRunning: longRunning, - HTTPMethod: strings.ToUpper(operation.Method), - ResourceIdName: pointer.To(resourceId), - UriSuffix: pointer.To(uriSuffix), + FieldContainingPaginationDetails: input.FieldContainingPaginationDetails, + LongRunning: input.LongRunning, + HTTPMethod: strings.ToUpper(input.Method), + ResourceIdName: input.ResourceIdName, + UriSuffix: input.UriSuffix, } - if operation.RequestObject != nil { - requestObject, err := mapObjectDefinition(operation.RequestObject, knownConstants, knownModels) + if input.RequestObject != nil { + requestObject, err := mapObjectDefinition(input.RequestObject, knownConstants, knownModels) if err != nil { return nil, fmt.Errorf("mapping the request object definition: %+v", err) } - op.RequestObject = requestObject + output.RequestObject = requestObject } - if operation.ResponseObject != nil { - responseObject, err := mapObjectDefinition(operation.ResponseObject, knownConstants, knownModels) + if input.ResponseObject != nil { + responseObject, err := mapObjectDefinition(input.ResponseObject, knownConstants, knownModels) if err != nil { return nil, fmt.Errorf("mapping the response object definition: %+v", err) } - op.ResponseObject = responseObject + output.ResponseObject = responseObject } - if len(operation.Options) > 0 { + if len(input.Options) > 0 { options := make([]dataApiModels.Option, 0) sortedOptionsKeys := make([]string, 0) - for k := range operation.Options { + for k := range input.Options { sortedOptionsKeys = append(sortedOptionsKeys, k) } sort.Strings(sortedOptionsKeys) for _, optionName := range sortedOptionsKeys { - optionDetails := operation.Options[optionName] + optionDetails := input.Options[optionName] if optionDetails.ObjectDefinition == nil { return nil, fmt.Errorf("missing object definition") @@ -103,10 +82,10 @@ func codeForOperation(operationName string, operation importerModels.OperationDe options = append(options, option) } - op.Options = pointer.To(options) + output.Options = pointer.To(options) } - data, err := json.MarshalIndent(op, "", " ") + data, err := json.MarshalIndent(output, "", " ") if err != nil { return nil, err } From c82730616857e9098183e7157fd900a04cd25fb5 Mon Sep 17 00:00:00 2001 From: tombuildsstuff Date: Wed, 25 Oct 2023 10:49:34 +0200 Subject: [PATCH 5/6] tools/importer-rest-api-specs: always outputting the expectedStatusCodes The Go SDK Generator expects these to be returned, these weren't output previously due to the base types --- .../templater_operations.go | 40 +------------------ 1 file changed, 1 insertion(+), 39 deletions(-) diff --git a/tools/importer-rest-api-specs/components/dataapigeneratorjson/templater_operations.go b/tools/importer-rest-api-specs/components/dataapigeneratorjson/templater_operations.go index 991ddf91b96..d633ea23797 100644 --- a/tools/importer-rest-api-specs/components/dataapigeneratorjson/templater_operations.go +++ b/tools/importer-rest-api-specs/components/dataapigeneratorjson/templater_operations.go @@ -13,18 +13,10 @@ import ( ) func codeForOperation(operationName string, input importerModels.OperationDetails, knownConstants map[string]resourcemanager.ConstantDetails, knownModels map[string]importerModels.ModelDetails) ([]byte, error) { - statusCodes := make([]int, 0) - if usesNonDefaultStatusCodes(input) { - for _, sc := range input.ExpectedStatusCodes { - statusCodes = append(statusCodes, sc) - } - sort.Ints(statusCodes) - } - output := dataApiModels.Operation{ Name: operationName, ContentType: input.ContentType, - ExpectedStatusCodes: statusCodes, + ExpectedStatusCodes: input.ExpectedStatusCodes, FieldContainingPaginationDetails: input.FieldContainingPaginationDetails, LongRunning: input.LongRunning, HTTPMethod: strings.ToUpper(input.Method), @@ -93,36 +85,6 @@ func codeForOperation(operationName string, input importerModels.OperationDetail return data, nil } -func usesNonDefaultStatusCodes(operation importerModels.OperationDetails) bool { - defaultStatusCodes := map[string][]int{ - "get": {200}, - "post": {200, 201}, - "put": {200, 201}, - "delete": {200, 201}, - "patch": {200, 201}, - } - expected, ok := defaultStatusCodes[strings.ToLower(operation.Method)] - if !ok { - // potentially an unsupported use-case but fine for now - return true - } - - if len(expected) != len(operation.ExpectedStatusCodes) { - return true - } - - sort.Ints(expected) - sort.Ints(operation.ExpectedStatusCodes) - for i, ev := range expected { - av := operation.ExpectedStatusCodes[i] - if ev != av { - return true - } - } - - return false -} - var internalObjectDefinitionsToOptionObjectDefinitionTypes = map[importerModels.ObjectDefinitionType]dataApiModels.OptionObjectDefinitionType{ importerModels.ObjectDefinitionBoolean: dataApiModels.BooleanOptionObjectDefinitionType, importerModels.ObjectDefinitionCsv: dataApiModels.CsvOptionObjectDefinitionType, From 5b450d7d0d3a7e577c06edfc9e1fb6e255ce595e Mon Sep 17 00:00:00 2001 From: tombuildsstuff Date: Wed, 25 Oct 2023 11:04:08 +0200 Subject: [PATCH 6/6] tools/importer-rest-api-specs: API Definitions v2 - outputting the model name --- .../components/dataapigeneratorjson/templater_models.go | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/importer-rest-api-specs/components/dataapigeneratorjson/templater_models.go b/tools/importer-rest-api-specs/components/dataapigeneratorjson/templater_models.go index 6070de173a1..27d1312633e 100644 --- a/tools/importer-rest-api-specs/components/dataapigeneratorjson/templater_models.go +++ b/tools/importer-rest-api-specs/components/dataapigeneratorjson/templater_models.go @@ -25,6 +25,7 @@ func codeForModel(modelName string, model importerModels.ModelDetails, parentMod } dataApiModel := dataApiModels.Model{ + Name: modelName, Fields: *fields, }