Generate a Microsoft Graph SDK

.github/workflows/automation-msgraph-metadata-importer.yaml

@@ -5,7 +5,6 @@ on:
     branches:
       - fake_name_to_disable_trigger
     paths:
-      - '.github/workflows/**'
       - 'config/microsoft-graph.hcl'
       - 'submodules/msgraph-metadata'
       - 'tools/importer-msgraph/**'

.github/workflows/automation-regenerate-terraform.yaml

@@ -5,7 +5,8 @@ on:
     branches:
       - main
     paths:
-      - 'api-definitions/**'
+      - 'api-definitions/handwritten-resource-manager/**'
+      - 'api-definitions/resource-manager/**'
       - 'tools/generator-terraform/**'
   workflow_dispatch: # for manual invocations
 

.github/workflows/automation-rest-api-specs-importer.yaml

@@ -4,8 +4,8 @@ on:
     branches:
       - main
     paths:
-      - '.github/workflows/**'
-      - 'config/**'
+      - 'config/resource-manager.hcl'
+      - 'config/resources/**'
       - 'submodules/rest-api-specs'
       - 'tools/importer-rest-api-specs/**'
   workflow_dispatch: # for manual invocations

.github/workflows/automation-version-bumper.yaml

@@ -4,8 +4,8 @@ on:
     branches:
       - main
     paths:
-      - '.github/workflows/**'
-      - 'config/**'
+      - '.github/workflows/automation-version-bumper.yaml'
+      - 'config/resource-manager.hcl'
       - 'submodules/msgraph-metadata'
       - 'submodules/rest-api-specs'
       - 'tools/version-bumper/**'

.github/workflows/unit-test-data-api-differ.yaml

@@ -4,7 +4,7 @@ on:
   pull_request:
     types: ['opened', 'synchronize']
     paths:
-      - '.github/workflows/**'
+      - '.github/workflows/unit-test-data-api-differ.yaml'
       - 'tools/data-api-differ/**'
 
 jobs:

.github/workflows/unit-test-data-api-v2.yaml

@@ -4,7 +4,7 @@ on:
   pull_request:
     types: ['opened', 'synchronize']
     paths:
-      - '.github/workflows/**'
+      - '.github/workflows/unit-test-data-api-v2.yaml'
       - 'api-definitions/**'
       - 'tools/data-api/**'
 

.github/workflows/unit-test-end-to-end-microsoft-graph.yaml

@@ -0,0 +1,37 @@
+---
+name: Verify All Microsoft Graph Services can be Imported and Generated
+on:
+  pull_request:
+    types: ['opened', 'synchronize']
+    paths:
+      - '.github/workflows/unit-test-end-to-end-microsoft-graph.yaml'
+      - 'config/microsoft-graph.hcl'
+      - 'tools/generator-go-sdk/**'
+      - 'tools/importer-msgraph-metadata/**'
+
+jobs:
+  test:
+    runs-on: custom-linux-xl
+    strategy:
+      fail-fast: true
+    steps:
+      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+        with:
+          submodules: recursive
+
+      - uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2
+        with:
+          go-version-file: ./.go-version
+
+      - name: "Build and Run importer-msgraph-metadata"
+        id: import-data
+        run: |
+          cd ./tools/importer-msgraph-metadata
+          make tools
+          make build
+          make import
+
+      - name: "Run the Go SDK Generator"
+        run: |
+          # go go gadget generator
+          ./scripts/automation-generate-go-sdk.sh microsoft-graph

.github/workflows/unit-test-end-to-end-resource-manager.yaml

@@ -4,8 +4,11 @@ on:
   pull_request:
     types: ['opened', 'synchronize']
     paths:
-      - '.github/workflows/**'
-      - 'config/**'
+      - '.github/workflows/unit-test-end-to-end-resource-manager.yaml'
+      - 'config/resource-manager.hcl'
+      - 'config/resources/**'
+      - 'tools/generator-go-sdk/**'
+      - 'tools/importer-rest-api-specs/**'
 
 jobs:
   test:
@@ -32,4 +35,4 @@ jobs:
       - name: "Run the Go SDK Generator"
         run: |
           # go go gadget generator
-          ./scripts/automation-generate-go-sdk.sh
+          ./scripts/automation-generate-go-sdk.sh resource-manager

.github/workflows/unit-test-generator-go-sdk.yaml

@@ -4,7 +4,7 @@ on:
   pull_request:
     types: ['opened', 'synchronize']
     paths:
-      - '.github/workflows/**'
+      - '.github/workflows/unit-test-generator-go-sdk.yaml'
       - 'tools/generator-go-sdk/**'
 
 jobs:

.github/workflows/unit-test-generator-terraform.yaml

@@ -4,7 +4,7 @@ on:
   pull_request:
     types: ['opened', 'synchronize']
     paths:
-      - '.github/workflows/**'
+      - '.github/workflows/unit-test-generator-terraform.yaml'
       - 'tools/generator-terraform/**'
 
 jobs:

.github/workflows/unit-test-rest-api-specs-importer.yaml

@@ -4,8 +4,9 @@ on:
   pull_request:
     types: ['opened', 'synchronize']
     paths:
-      - '.github/workflows/**'
-      - 'config/**'
+      - '.github/workflows/unit-test-rest-api-specs-importer.yaml'
+      - 'config/resource-manager.hcl'
+      - 'config/resources/**'
       - 'submodules/rest-api-specs'
       - 'tools/importer-rest-api-specs/**'
 

config/microsoft-graph.hcl

@@ -21,6 +21,16 @@ service "applicationTemplates" {
   available = ["stable", "beta"]
 }
 
+service "auditLogs" {
+  name      = "AuditLogs"
+  available = ["stable", "beta"]
+}
+
+service "deviceManagement" {
+  name      = "DeviceManagement"
+  available = ["stable", "beta"]
+}
+
 service "directory" {
   name      = "Directory"
   available = ["stable", "beta"]
@@ -31,6 +41,16 @@ service "directoryObjects" {
   available = ["stable", "beta"]
 }
 
+service "directoryRoles" {
+  name      = "DirectoryRoles"
+  available = ["stable", "beta"]
+}
+
+service "directoryRoleTemplates" {
+  name      = "DirectoryRoleTemplates"
+  available = ["stable", "beta"]
+}
+
 service "domains" {
   name      = "Domains"
   available = ["stable", "beta"]
@@ -46,6 +66,11 @@ service "identity" {
   available = ["stable"]
 }
 
+service "identityGovernance" {
+  name      = "IdentityGovernance"
+  available = ["stable", "beta"]
+}
+
 service "invitations" {
   name      = "Invitations"
   available = ["stable", "beta"]
@@ -66,11 +91,21 @@ service "policies" {
   available = ["stable", "beta"]
 }
 
+service "reports" {
+  name      = "Reports"
+  available = ["stable", "beta"]
+}
+
 service "roleManagement" {
   name      = "RoleManagement"
   available = ["stable", "beta"]
 }
 
+service "schemaExtensions" {
+  name      = "SchemaExtensions"
+  available = ["stable", "beta"]
+}
+
 service "servicePrincipals" {
   name      = "ServicePrincipals"
   available = ["stable", "beta"]

scripts/automation-generate-go-sdk.sh

@@ -7,6 +7,19 @@ set -e
 
 DIR="$(cd "$(dirname "$0")" && pwd)/.."
 
+sdkToGenerate="${1}"
+if [[ "${sdkToGenerate}" == "" ]]; then
+  echo "must specify SDK to generate!" >&2
+  echo "" >&2
+  echo "supported values are:" >&2
+  echo "  microsoft-graph" >&2
+  echo "  resource-manager" >&2
+  echo "" >&2
+  echo "example usage:" >&2
+  echo "${0} resource-manager" >&2
+  exit 1
+fi
+
 function buildAndInstallDependencies {
   echo "Outputting Go Version.."
   go version
@@ -31,15 +44,9 @@ function runWrapper {
   local apiDefinitionsDirectory=$1
   local outputDirectory=$2
 
-  echo "Running Wrapper for Resource Manager.."
-  cd "${DIR}/tools/wrapper-automation"
-  ./wrapper-automation resource-manager go-sdk \
-    --api-definitions-dir="../../$apiDefinitionsDirectory"\
-    --output-dir="../../$outputDirectory"
-
-  echo "Running Wrapper for Microsoft Graph.."
+  echo "Running Wrapper for ${sdkToGenerate}.."
   cd "${DIR}/tools/wrapper-automation"
-  ./wrapper-automation microsoft-graph go-sdk \
+  ./wrapper-automation "${sdkToGenerate}" go-sdk \
     --api-definitions-dir="../../$apiDefinitionsDirectory"\
     --output-dir="../../$outputDirectory"
 
@@ -61,11 +68,12 @@ function runWrapper {
 function runGoSDKUnitTests {
   local outputDirectory=$1
 
-  cd "${DIR}"
+  echo "Running unit tests within the SDK codebase.."
+  cd "${outputDirectory}/${sdkToGenerate}"
+  go mod tidy
+  go test -v ./...
 
-  echo "Running 'make test' within the SDK codebase.."
-  cd "${outputDirectory}"
-  make test
+  cd "${DIR}"
 }
 
 function prepareGoSdk {

tools/data-api-differ/internal/differ/differ_helpers.go

@@ -12,7 +12,7 @@ import (
 
 // stringifySDKObjectDefinition returns a human readable, string version of this SDKObjectDefinition.
 func (d differ) stringifySDKObjectDefinition(input models.SDKObjectDefinition) (*string, error) {
-	return helpers.GolangTypeForSDKObjectDefinition(input, nil)
+	return helpers.GolangTypeForSDKObjectDefinition(input, nil, nil)
 }
 
 // stringifySDKOperationOptionObjectDefinition returns a human readable, string version of this Object Definition.

tools/data-api-repository/repository/helpers_parse.go

@@ -57,8 +57,9 @@ func parseCommonTypesWithin(workingDirectory string, logger hclog.Logger) (*map[
 		apiVersion = strings.TrimPrefix(apiVersion, "/")
 
 		commonTypes := sdkModels.CommonTypes{
-			Constants: map[string]sdkModels.SDKConstant{},
-			Models:    map[string]sdkModels.SDKModel{},
+			Constants:   map[string]sdkModels.SDKConstant{},
+			Models:      map[string]sdkModels.SDKModel{},
+			ResourceIDs: make(map[string]sdkModels.ResourceID),
 		}
 
 		logger.Trace(fmt.Sprintf("Discovering the Common Type Constants within %q..", subDirectory))
@@ -75,7 +76,14 @@ func parseCommonTypesWithin(workingDirectory string, logger hclog.Logger) (*map[
 		}
 		commonTypes.Models = *models
 
-		if len(commonTypes.Constants) > 0 || len(commonTypes.Models) > 0 {
+		logger.Trace(fmt.Sprintf("Discovering the Common Type Resource IDs within %q..", subDirectory))
+		resourceIds, err := parseResourceIDsWithin(subDirectory, *constants, logger)
+		if err != nil {
+			return nil, fmt.Errorf("parsing the Common Type Models within %q: %+v", subDirectory, err)
+		}
+		commonTypes.ResourceIDs = *resourceIds
+
+		if len(commonTypes.Constants) > 0 || len(commonTypes.Models) > 0 || len(commonTypes.ResourceIDs) > 0 {
 			output[apiVersion] = commonTypes
 		}
 	}
@@ -183,15 +191,17 @@ func parseAPIResourceWithin(workingDirectory, resourceName string, commonTypesFo
 	}
 
 	knownData := helpers.KnownData{
-		Constants:           *constants,
-		Models:              *models,
-		ResourceIds:         *resourceIds,
-		CommonTypeConstants: make(map[string]sdkModels.SDKConstant),
-		CommonTypeModels:    make(map[string]sdkModels.SDKModel),
+		Constants:              *constants,
+		Models:                 *models,
+		ResourceIds:            *resourceIds,
+		CommonTypeConstants:    make(map[string]sdkModels.SDKConstant),
+		CommonTypeModels:       make(map[string]sdkModels.SDKModel),
+		CommonTypesResourceIds: make(map[string]sdkModels.ResourceID),
 	}
 	if commonTypesForThisAPIVersion != nil {
 		knownData.CommonTypeConstants = commonTypesForThisAPIVersion.Constants
 		knownData.CommonTypeModels = commonTypesForThisAPIVersion.Models
+		knownData.CommonTypesResourceIds = commonTypesForThisAPIVersion.ResourceIDs
 	}
 
 	logger.Trace(fmt.Sprintf("Parsing the Operations within %q..", workingDirectory))

tools/data-api-repository/repository/internal/helpers/known_data.go

@@ -10,8 +10,9 @@ type KnownData struct {
 	Models      map[string]sdkModels.SDKModel
 	ResourceIds map[string]sdkModels.ResourceID
 
-	CommonTypeConstants map[string]sdkModels.SDKConstant
-	CommonTypeModels    map[string]sdkModels.SDKModel
+	CommonTypeConstants    map[string]sdkModels.SDKConstant
+	CommonTypeModels       map[string]sdkModels.SDKModel
+	CommonTypesResourceIds map[string]sdkModels.ResourceID
 }
 
 func (d KnownData) ConstantExists(constantName string) bool {
@@ -38,5 +39,6 @@ func (d KnownData) ModelExists(modelName string) bool {
 
 func (d KnownData) ResourceIDExists(resourceIdName string) bool {
 	_, resourceIdExists := d.ResourceIds[resourceIdName]
-	return resourceIdExists
+	_, commonTypesResourceIdExists := d.CommonTypesResourceIds[resourceIdName]
+	return resourceIdExists || commonTypesResourceIdExists
 }

tools/data-api-repository/repository/internal/models/models.go

@@ -13,6 +13,9 @@ type Model struct {
 	// Fields is an array of fields contained in the Model
 	Fields []ModelField `json:"fields"`
 
+	// IsParent specifies whether this model is a known parent model, typically for discriminated child models.
+	IsParent bool
+
 	// NOTE: If DiscriminatedParentModelName and DiscriminatedTypeValue are both populated then this Model
 	// represents a discriminated type
 	// DiscriminatedParentModelName contains the name of the Parent Model that this Model would implement

tools/data-api-repository/repository/internal/models/object_definition.go

@@ -8,10 +8,15 @@ type ObjectDefinition struct {
 	// ObjectDefinitionType defines what kind of ObjectDefinition this is, such as a Reference, String or List
 	Type ObjectDefinitionType `json:"type"`
 
-	// NOTE: we could split this into ConstantName and ModelName in time, but not today.
+	// Nullable specifies that this type should be unset by sending `null` as the JSON value.
+	Nullable bool `json:"nullable"`
+
 	// ReferenceName is the name of the Constant or Model that this is a reference to
 	ReferenceName *string `json:"referenceName"`
 
+	// ReferenceNameIsCommonType indicates whether ReferenceName refers to a common type (true) or a type that is local to the service (false)
+	ReferenceNameIsCommonType *bool `json:"referenceNameIsCommonType"`
+
 	// NestedItem is a nested ObjectDefinition when Type is a Dictionary or List
 	// NOTE: that it's possible to have deeply-nested ObjectDefinitions, e.g. a List of a List of a Dictionary[String (key, fixed as a string) : Integer (value)
 	NestedItem *ObjectDefinition `json:"nestedItem,omitempty"`