Skip to content

Commit

Permalink
Data API V2: extending NewServicesRepository to build a complete li…
Browse files Browse the repository at this point in the history
…st of services (hashicorp#3392)

* auto discover api definition directories with metadata.json

* handle and return errors in NewServiceRepository

* return error when processing service definitions

* use log.Fatal, refactor mappings into a separate file and add discovery functions for finding service type directories and services

* load services from cache in GetAll and GetByName, call discovery functions to populate all available services and their file paths

* fix nit
  • Loading branch information
stephybun authored Nov 28, 2023
1 parent c2d93e2 commit 2142f47
Show file tree
Hide file tree
Showing 6 changed files with 406 additions and 197 deletions.
20 changes: 17 additions & 3 deletions tools/data-api/internal/endpoints/routing.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
package endpoints

import (
"log"

"github.com/go-chi/chi/v5"
"github.com/hashicorp/pandora/tools/data-api/internal/endpoints/infrastructure"
"github.com/hashicorp/pandora/tools/data-api/internal/endpoints/v1"
Expand All @@ -16,7 +18,11 @@ func Router(directory string, serviceNames *[]string) func(chi.Router) {
UriPrefix: "/v1/microsoft-graph/beta",
UsesCommonTypes: true,
}
serviceRepo := repositories.NewServicesRepository(directory, opts.ServiceType, serviceNames)
serviceRepo, err := repositories.NewServicesRepository(directory, opts.ServiceType, serviceNames)
if err != nil {
// TODO logging
log.Fatalf("Error: %+v", err)
}
v1.Router(r, opts, serviceRepo)
})
router.Route("/v1/microsoft-graph/stable-v1", func(r chi.Router) {
Expand All @@ -25,7 +31,11 @@ func Router(directory string, serviceNames *[]string) func(chi.Router) {
UriPrefix: "/v1/microsoft-graph/stable-v1",
UsesCommonTypes: true,
}
serviceRepo := repositories.NewServicesRepository(directory, opts.ServiceType, serviceNames)
serviceRepo, err := repositories.NewServicesRepository(directory, opts.ServiceType, serviceNames)
if err != nil {
// TODO logging
log.Fatalf("Error: %+v", err)
}
v1.Router(r, opts, serviceRepo)
})
router.Route("/v1/resource-manager", func(r chi.Router) {
Expand All @@ -34,7 +44,11 @@ func Router(directory string, serviceNames *[]string) func(chi.Router) {
UriPrefix: "/v1/resource-manager",
UsesCommonTypes: false,
}
serviceRepo := repositories.NewServicesRepository(directory, opts.ServiceType, serviceNames)
serviceRepo, err := repositories.NewServicesRepository(directory, opts.ServiceType, serviceNames)
if err != nil {
// TODO logging
log.Fatalf("Error: %+v", err)
}
v1.Router(r, opts, serviceRepo)
})
router.Get("/", HomePage(router))
Expand Down
118 changes: 118 additions & 0 deletions tools/data-api/internal/repositories/discovery.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,118 @@
package repositories

import (
"encoding/json"
"errors"
"fmt"
"os"
"path"

"github.com/hashicorp/pandora/tools/sdk/dataapimodels"
)

func (s *ServicesRepositoryImpl) discoverServiceTypeDirectories() (*[]string, error) {
// discoverServiceTypeDirectories finds all directories under the root directory that contain api definitions for a given
// service type by checking for a metadata.json and comparing the Data Source value defined within it to the Data Source
// value we're expecting for the Services Repository
dirs, err := listSubDirectories(s.rootDirectory)
if err != nil {
return nil, fmt.Errorf("listing directories under %q: %+v", s.rootDirectory, err)
}

serviceTypeDirectories := make([]string, 0)

for _, d := range *dirs {
serviceTypeDir := path.Join(s.rootDirectory, d)

// check whether directory contains a metadata.json
var metadata dataapimodels.MetaData
contents, err := loadJson(path.Join(serviceTypeDir, "metadata.json"))
if err != nil {
var pathError *os.PathError
if errors.As(err, &pathError) {
// this folder has no metadata.json, so we skip it
continue
}
return nil, fmt.Errorf("loading metadata.json: %+v", err)
}

if err := json.Unmarshal(*contents, &metadata); err != nil {
return nil, fmt.Errorf("unmarshaling metadata.json: %+v", err)
}

if metadata.DataSource != s.expectedDataSource {
// this folder contains definitions not belonging to this service type, so we skip it
continue
}
serviceTypeDirectories = append(serviceTypeDirectories, serviceTypeDir)
}

return &serviceTypeDirectories, nil
}

func (s *ServicesRepositoryImpl) discoverSubsetOfServices() error {
// discoverSubsetOfServices populates the serviceNamesToDirectory attribute of the ServicesRepositoryImpl.
// This function is called if we're spinning up the data API for a subset of services and avoids iterating over
// all available services.
dirs, err := s.discoverServiceTypeDirectories()
if err != nil {
return fmt.Errorf("discovering service type directories for service type %q: %+v", s.serviceType, err)
}

services := make(map[string]string, 0)
for _, d := range *dirs {
for _, service := range *s.serviceNames {
serviceDir := path.Join(d, service)
if _, err := os.Stat(serviceDir); os.IsNotExist(err) {
// we continue here since the service we're looking for could exist in another source directory e.g. under handwritten definitions
continue
}
if _, ok := services[service]; ok {
return fmt.Errorf("duplicate definitions for service %q", service)
}
services[service] = serviceDir
}
}

// this checks if all services have been found if we're running the data API for a subset
for _, service := range *s.serviceNames {
if _, ok := services[service]; !ok {
return fmt.Errorf("service %q was not found", service)
}
}

s.serviceNamesToDirectory = &services

return nil
}

func (s *ServicesRepositoryImpl) discoverAllServices() error {
// discoverAllServices populates the serviceNamesToDirectory attribute of the ServicesRepositoryImpl.
// It iterates through all available services to build a complete list of available services for a given
// service type and checks if there are duplicate definitions for a service.
dirs, err := s.discoverServiceTypeDirectories()
if err != nil {
return fmt.Errorf("discovering service type directories for service type %q: %+v", s.serviceType, err)
}

allServices := make(map[string]string, 0)
for _, d := range *dirs {
files, err := os.ReadDir(d)
if err != nil {
return fmt.Errorf("getting all services: %+v", err)
}

for _, f := range files {
if f.IsDir() {
if _, ok := allServices[f.Name()]; ok {
return fmt.Errorf("duplicate definitions for service %q", f.Name())
}
allServices[f.Name()] = path.Join(d, f.Name())
}
}
}

s.serviceNamesToDirectory = &allServices

return nil
}
155 changes: 155 additions & 0 deletions tools/data-api/internal/repositories/mappings.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,155 @@
package repositories

import (
"fmt"

"github.com/hashicorp/go-azure-helpers/lang/pointer"
"github.com/hashicorp/pandora/tools/sdk/dataapimodels"
)

func mapObjectDefinition(input *dataapimodels.ObjectDefinition) (*ObjectDefinition, error) {
if input == nil {
return nil, nil
}

objectDefinitionType, err := mapObjectDefinitionType(input.Type)
if err != nil {
return nil, err
}

output := ObjectDefinition{
ReferenceName: input.ReferenceName,
Type: pointer.From(objectDefinitionType),
}

if input.NestedItem != nil {
nestedItem, err := mapObjectDefinition(input.NestedItem)
if err != nil {
return nil, fmt.Errorf("mapping Nested Item for Object Definition: %+v", err)
}
output.NestedItem = nestedItem
}

return &output, nil
}

func mapOptionObjectDefinition(input *dataapimodels.OptionObjectDefinition, constants map[string]ConstantDetails, apiModels map[string]ModelDetails) (*OptionObjectDefinition, error) {
optionObjectType, err := mapOptionObjectDefinitionType(input.Type)
if err != nil {
return nil, err
}

output := OptionObjectDefinition{
ReferenceName: input.ReferenceName,
Type: pointer.From(optionObjectType),
}

if input.NestedItem != nil {
nestedItem, err := mapOptionObjectDefinition(input.NestedItem, constants, apiModels)
if err != nil {
return nil, fmt.Errorf("mapping Nested Item for Option Object Definition: %+v", err)
}
output.NestedItem = nestedItem
}

if err := validateOptionObjectDefinition(output, constants, apiModels); err != nil {
return nil, fmt.Errorf("validating mapped Option Object Definition: %+v", err)
}

return &output, nil
}

func mapDateFormatType(input dataapimodels.DateFormat) (*DateFormat, error) {
mappings := map[dataapimodels.DateFormat]DateFormat{
dataapimodels.RFC3339DateFormat: RFC3339DateFormat,
}
if v, ok := mappings[input]; ok {
return &v, nil
}

return nil, fmt.Errorf("unmapped Date Format Type %q", string(input))
}

func mapObjectDefinitionType(input dataapimodels.ObjectDefinitionType) (*ObjectDefinitionType, error) {
mappings := map[dataapimodels.ObjectDefinitionType]ObjectDefinitionType{
dataapimodels.BooleanObjectDefinitionType: BooleanObjectDefinitionType,
dataapimodels.DateTimeObjectDefinitionType: DateTimeObjectDefinitionType,
dataapimodels.IntegerObjectDefinitionType: IntegerObjectDefinitionType,
dataapimodels.FloatObjectDefinitionType: FloatObjectDefinitionType,
dataapimodels.RawFileObjectDefinitionType: RawFileObjectDefinitionType,
dataapimodels.RawObjectObjectDefinitionType: RawObjectObjectDefinitionType,
dataapimodels.ReferenceObjectDefinitionType: ReferenceObjectDefinitionType,
dataapimodels.StringObjectDefinitionType: StringObjectDefinitionType,
dataapimodels.CsvObjectDefinitionType: CsvObjectDefinitionType,
dataapimodels.DictionaryObjectDefinitionType: DictionaryObjectDefinitionType,
dataapimodels.ListObjectDefinitionType: ListObjectDefinitionType,

dataapimodels.EdgeZoneObjectDefinitionType: EdgeZoneObjectDefinitionType,
dataapimodels.LocationObjectDefinitionType: LocationObjectDefinitionType,
dataapimodels.TagsObjectDefinitionType: TagsObjectDefinitionType,
dataapimodels.SystemAssignedIdentityObjectDefinitionType: SystemAssignedIdentityObjectDefinitionType,
dataapimodels.SystemAndUserAssignedIdentityListObjectDefinitionType: SystemAndUserAssignedIdentityListObjectDefinitionType,
dataapimodels.SystemAndUserAssignedIdentityMapObjectDefinitionType: SystemAndUserAssignedIdentityMapObjectDefinitionType,
dataapimodels.LegacySystemAndUserAssignedIdentityListObjectDefinitionType: LegacySystemAndUserAssignedIdentityListObjectDefinitionType,
dataapimodels.LegacySystemAndUserAssignedIdentityMapObjectDefinitionType: LegacySystemAndUserAssignedIdentityMapObjectDefinitionType,
dataapimodels.SystemOrUserAssignedIdentityListObjectDefinitionType: SystemOrUserAssignedIdentityListObjectDefinitionType,
dataapimodels.SystemOrUserAssignedIdentityMapObjectDefinitionType: SystemOrUserAssignedIdentityMapObjectDefinitionType,
dataapimodels.UserAssignedIdentityListObjectDefinitionType: UserAssignedIdentityListObjectDefinitionType,
dataapimodels.UserAssignedIdentityMapObjectDefinitionType: UserAssignedIdentityMapObjectDefinitionType,
dataapimodels.SystemDataObjectDefinitionType: SystemDataObjectDefinitionType,
dataapimodels.ZoneObjectDefinitionType: ZoneObjectDefinitionType,
dataapimodels.ZonesObjectDefinitionType: ZonesObjectDefinitionType,
}
if v, ok := mappings[input]; ok {
return &v, nil
}

return nil, fmt.Errorf("unmapped Object Definition Type %q", string(input))
}

func mapOptionObjectDefinitionType(input dataapimodels.OptionObjectDefinitionType) (*OptionObjectDefinitionType, error) {
mappings := map[dataapimodels.OptionObjectDefinitionType]OptionObjectDefinitionType{
dataapimodels.BooleanOptionObjectDefinitionType: BooleanOptionObjectDefinition,
dataapimodels.IntegerOptionObjectDefinitionType: IntegerOptionObjectDefinition,
dataapimodels.FloatOptionObjectDefinitionType: FloatOptionObjectDefinitionType,
dataapimodels.StringOptionObjectDefinitionType: StringOptionObjectDefinitionType,
dataapimodels.CsvOptionObjectDefinitionType: CsvOptionObjectDefinitionType,
dataapimodels.ListOptionObjectDefinitionType: ListOptionObjectDefinitionType,
dataapimodels.ReferenceOptionObjectDefinitionType: ReferenceOptionObjectDefinitionType,
}
if v, ok := mappings[input]; ok {
return &v, nil
}

return nil, fmt.Errorf("unmapped Options Object Definition Type %q", string(input))
}

func mapConstantFieldType(input dataapimodels.ConstantType) (*ConstantType, error) {
mappings := map[dataapimodels.ConstantType]ConstantType{
dataapimodels.FloatConstant: FloatConstant,
dataapimodels.IntegerConstant: IntegerConstant,
dataapimodels.StringConstant: StringConstant,
}
if v, ok := mappings[input]; ok {
return &v, nil
}

return nil, fmt.Errorf("unmapped Constant Type %q", string(input))
}

func mapResourceIdSegmentType(input dataapimodels.ResourceIdSegmentType) (*ResourceIdSegmentType, error) {
mappings := map[dataapimodels.ResourceIdSegmentType]ResourceIdSegmentType{
dataapimodels.ConstantResourceIdSegmentType: ConstantResourceIdSegmentType,
dataapimodels.ResourceGroupResourceIdSegmentType: ResourceGroupResourceIdSegmentType,
dataapimodels.ResourceProviderResourceIdSegmentType: ResourceProviderResourceIdSegmentType,
dataapimodels.ScopeResourceIdSegmentType: ScopeResourceIdSegmentType,
dataapimodels.StaticResourceIdSegmentType: StaticResourceIdSegmentType,
dataapimodels.SubscriptionIdResourceIdSegmentType: SubscriptionIdResourceIdSegmentType,
dataapimodels.UserSpecifiedResourceIdSegmentType: UserSpecifiedResourceIdSegmentType,
}
if v, ok := mappings[input]; ok {
return &v, nil
}

return nil, fmt.Errorf("unmapped Resource Id Segment Type %q", string(input))
}
2 changes: 1 addition & 1 deletion tools/data-api/internal/repositories/models.go
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,7 @@ type ServiceDetails struct {
Name string
ApiVersions map[string]*ServiceApiVersionDetails
Generate bool
ResourceProvider string
ResourceProvider *string
TerraformPackageName *string
TerraformDetails TerraformDetails
}
Expand Down
Loading

0 comments on commit 2142f47

Please sign in to comment.