Centralize SDK generation logic

Presently, the majority of Java code generation is driven by the
`pulumi-java-gen` binary, since Java usage began before we had time to implement
the `Generate*` family of language host gRPC methods. Recently, these gRPC
methods were implemented, to support (among other things) conformance testing.
Unfortunately, while both routes end in `pkg/codegen/java`'s `Generate*`
functions, each had accumulated its own special "setup logic" ahead of the call
into `pkg/codegen`. This commit attempts to sort this out, pushing all that
logic into `pkg/codegen` so that both routes behave identically. As a result of
this, we should be able to deprecate `pulumi-java-gen` more safely when the time
comes, remove direct build-time dependencies on `pulumi-java` from
`pulumi/pulumi` and fix some issues that have arisen as a result of the historic
differences, such as #1404 (which looks like it may have already been fixed, but
this should cement it) and #1508.

Closes #1404
Fixes #1508

pkg/cmd/pulumi-java-gen/generate.go

@@ -53,10 +53,13 @@ func generateJava(cfg generateJavaOptions) error {
 		return fmt.Errorf("failed to read schema from %s: %w", cfg.Schema, err)
 	}
 
-	pkgSpec, err := dedupTypes(rawPkgSpec)
+	pkgSpec, diags, err := javagen.DeduplicateTypes(rawPkgSpec)
 	if err != nil {
 		return fmt.Errorf("failed to dedup types in schema from %s: %w", cfg.Schema, err)
 	}
+	for _, diag := range diags {
+		fmt.Println(diag.Error())
+	}
 
 	pkg, err := pschema.ImportSpec(*pkgSpec, nil)
 	if err != nil {

pkg/cmd/pulumi-language-java/main.go

@@ -19,6 +19,7 @@ import (
 	"time"
 
 	pbempty "github.com/golang/protobuf/ptypes/empty"
+	"github.com/hashicorp/hcl/v2"
 	"github.com/pkg/errors"
 	hclsyntax "github.com/pulumi/pulumi/pkg/v3/codegen/hcl2/syntax"
 	"github.com/pulumi/pulumi/pkg/v3/codegen/pcl"
@@ -703,37 +704,44 @@ func (host *javaLanguageHost) GeneratePackage(
 		return nil, err
 	}
 
-	pkg, diags, err := schema.BindSpec(spec, loader)
+	diags := hcl.Diagnostics{}
+
+	// Historically, Java has "deduplicated" PackageSpecs to reduce sets of multiple types whose names differ only in
+	// case down to just one type that is then shared (assuming that, apart from name, the types are otherwise
+	// identical). We thus perform that deduplication here before we bind the schema and resolve any references.
+	dedupedSpec, dedupeDiags, err := codegen.DeduplicateTypes(&spec)
 	if err != nil {
 		return nil, err
 	}
-	rpcDiagnostics := plugin.HclDiagnosticsToRPCDiagnostics(diags)
-	if diags.HasErrors() {
+	diags = diags.Extend(dedupeDiags)
+	if dedupeDiags.HasErrors() {
 		return &pulumirpc.GeneratePackageResponse{
-			Diagnostics: rpcDiagnostics,
+			Diagnostics: plugin.HclDiagnosticsToRPCDiagnostics(diags),
 		}, nil
 	}
 
-	if pkg.Description == "" {
-		pkg.Description = " "
+	pkg, bindDiags, err := schema.BindSpec(*dedupedSpec, loader)
+	if err != nil {
+		return nil, err
 	}
-	if pkg.Repository == "" {
-		pkg.Repository = "https://example.com"
+	diags = diags.Extend(bindDiags)
+	if bindDiags.HasErrors() {
+		return &pulumirpc.GeneratePackageResponse{
+			Diagnostics: plugin.HclDiagnosticsToRPCDiagnostics(diags),
+		}, nil
 	}
 
-	// Presently, we only support generating Java SDKs which use Gradle as a build system. Specify that here, as well as
-	// the set of dependencies that all generated SDKs rely on.
-	pkgInfo := codegen.PackageInfo{
-		BuildFiles: "gradle",
-		Dependencies: map[string]string{
-			"com.google.code.gson:gson":       "2.8.9",
-			"com.google.code.findbugs:jsr305": "3.0.2",
-		},
-	}
+	// While the consumer can specify a PackageInfo object (e.g. from the schema, or a set of manual specifications), we
+	// need to ensure that local dependencies are reflected in the lists of dependencies and repositories. We'll do this
+	// here before calling into the codegen package.
+	pkgOverrides := codegen.PackageInfo{}
 
+	dependencies := map[string]string{}
 	repositories := map[string]bool{}
-
 	for name, dep := range req.LocalDependencies {
+		// A local dependency has the form groupId:artifactId:version[:repositoryPath]. We'll parse this and add an
+		// entry to the dependency map for groupId:artifactId -> version, and add the repositoryPath to the list of
+		// repositories if it's present.
 		parts := strings.Split(dep, ":")
 		if len(parts) < 3 {
 			return nil, fmt.Errorf(
@@ -743,15 +751,22 @@ func (host *javaLanguageHost) GeneratePackage(
 		}
 
 		k := parts[0] + ":" + parts[1]
-		pkgInfo.Dependencies[k] = parts[2]
+		dependencies[k] = parts[2]
 
 		if len(parts) == 4 {
 			repositories[parts[3]] = true
 		}
 	}
 
-	pkgInfo.Repositories = maps.Keys(repositories)
-	pkg.Language["java"] = pkgInfo
+	pkgOverrides.Dependencies = dependencies
+	pkgOverrides.Repositories = maps.Keys(repositories)
+
+	var pkgInfo codegen.PackageInfo
+	if javaInfo, ok := pkg.Language["java"].(codegen.PackageInfo); ok {
+		pkgInfo = javaInfo
+	}
+
+	pkg.Language["java"] = pkgInfo.With(pkgOverrides)
 
 	files, err := codegen.GeneratePackage("pulumi-language-java", pkg, req.ExtraFiles, req.Local)
 	if err != nil {
@@ -772,7 +787,7 @@ func (host *javaLanguageHost) GeneratePackage(
 	}
 
 	return &pulumirpc.GeneratePackageResponse{
-		Diagnostics: rpcDiagnostics,
+		Diagnostics: plugin.HclDiagnosticsToRPCDiagnostics(diags),
 	}, nil
 }
 

pkg/cmd/pulumi-java-gen/dedup.go → pkg/codegen/java/dedup.go

@@ -1,6 +1,6 @@
-// Copyright 2022, Pulumi Corporation.  All rights reserved.
+// Copyright 2024, Pulumi Corporation.  All rights reserved.
 
-package main
+package java
 
 import (
 	"bytes"
@@ -9,15 +9,16 @@ import (
 	"reflect"
 	"strings"
 
-	pschema "github.com/pulumi/pulumi/pkg/v3/codegen/schema"
+	"github.com/hashicorp/hcl/v2"
+	"github.com/pulumi/pulumi/pkg/v3/codegen/schema"
 )
 
-// Detects cases when identical types have similar names modulo case
-// such as `azure-native:network:IpAllocationMethod` vs
-// `azure-native:network:IPAllocationMethod`, deterministically picks
-// one of these names, and rewrites the schema as if there was only
-// one such type.
-func dedupTypes(spec *pschema.PackageSpec) (*pschema.PackageSpec, error) {
+// DeduplicateTypes detects multiple types in a PackageSpec whose names are the same modulo case, such as
+// `azure-native:network:IpAllocationMethod` and `azure-native:network:IPAllocationMethod`, deterministically picks one
+// of these names, and rewrites the schema as if there was only one such type.
+func DeduplicateTypes(spec *schema.PackageSpec) (*schema.PackageSpec, hcl.Diagnostics, error) {
+	diags := hcl.Diagnostics{}
+
 	normalizedTokens := map[string]string{}
 	for typeToken := range spec.Types {
 		key := strings.ToUpper(typeToken)
@@ -41,12 +42,12 @@ func dedupTypes(spec *pschema.PackageSpec) (*pschema.PackageSpec, error) {
 
 	var buf bytes.Buffer
 	if err := json.NewEncoder(&buf).Encode(spec); err != nil {
-		return nil, err
+		return nil, nil, err
 	}
 
 	var rawSchema interface{}
 	if err := json.NewDecoder(bytes.NewReader(buf.Bytes())).Decode(&rawSchema); err != nil {
-		return nil, err
+		return nil, nil, err
 	}
 
 	types := map[string]interface{}{}
@@ -64,13 +65,19 @@ func dedupTypes(spec *pschema.PackageSpec) (*pschema.PackageSpec, error) {
 			transformJSONTree(stripDescription, types[newToken]),
 		)
 		if eq {
-			fmt.Printf("WARN renaming %s to %s in the schema\n",
-				oldToken, newToken)
+			diags = append(diags, &hcl.Diagnostic{
+				Severity: hcl.DiagWarning,
+				Summary:  fmt.Sprintf("Renaming '%s' to '%s' in the schema", oldToken, newToken),
+			})
 			delete(types, oldToken)
 		} else {
-			fmt.Printf("WARN not renaming %s to %s in the schema "+
-				"because they differ structurally\n",
-				oldToken, newToken)
+			diags = append(diags, &hcl.Diagnostic{
+				Severity: hcl.DiagWarning,
+				Summary: fmt.Sprintf(
+					"Not renaming '%s' to '%s' in the schema because they differ structurally",
+					oldToken, newToken,
+				),
+			})
 		}
 	}
 
@@ -89,7 +96,10 @@ func dedupTypes(spec *pschema.PackageSpec) (*pschema.PackageSpec, error) {
 			return node
 		}
 		if r, isRenamed := renamedRefs[s]; isRenamed {
-			fmt.Printf("Rewritten %s to %s\n", s, r)
+			diags = append(diags, &hcl.Diagnostic{
+				Severity: hcl.DiagWarning,
+				Summary:  fmt.Sprintf("Rewrote reference '%s' to '%s'", s, r),
+			})
 			return r
 		}
 		return node
@@ -100,15 +110,15 @@ func dedupTypes(spec *pschema.PackageSpec) (*pschema.PackageSpec, error) {
 	buf.Reset()
 
 	if err := json.NewEncoder(&buf).Encode(&rawSchema); err != nil {
-		return nil, err
+		return nil, nil, err
 	}
 
-	var fixedSpec pschema.PackageSpec
+	var fixedSpec schema.PackageSpec
 	if err := json.NewDecoder(bytes.NewReader(buf.Bytes())).Decode(&fixedSpec); err != nil {
-		return nil, err
+		return nil, nil, err
 	}
 
-	return &fixedSpec, nil
+	return &fixedSpec, diags, nil
 }
 
 func transformJSONTree(t func(interface{}) interface{}, tree interface{}) interface{} {

pkg/codegen/java/dedup_test.go

@@ -0,0 +1,138 @@
+// Copyright 2024, Pulumi Corporation.  All rights reserved.
+
+package java
+
+import (
+	"testing"
+
+	"github.com/hashicorp/hcl/v2"
+	"github.com/pulumi/pulumi/pkg/v3/codegen/schema"
+	"github.com/stretchr/testify/require"
+)
+
+func TestDeduplicateTypes(t *testing.T) {
+	cases := []struct {
+		name          string
+		input         *schema.PackageSpec
+		expectedSpec  *schema.PackageSpec
+		expectedDiags hcl.Diagnostics
+	}{
+		{
+			name:          "no duplicates",
+			input:         &schema.PackageSpec{},
+			expectedSpec:  &schema.PackageSpec{},
+			expectedDiags: hcl.Diagnostics{},
+		},
+		{
+			name: "duplicates, lowercase",
+			input: &schema.PackageSpec{
+				Types: map[string]schema.ComplexTypeSpec{
+					"azure-native:network:ipallocationmethod": {},
+					"azure-native:network:IpAllocationMethod": {},
+				},
+			},
+			expectedSpec: &schema.PackageSpec{
+				Types: map[string]schema.ComplexTypeSpec{
+					"azure-native:network:IpAllocationMethod": {},
+				},
+			},
+			expectedDiags: hcl.Diagnostics{
+				&hcl.Diagnostic{
+					Severity: hcl.DiagWarning,
+					Summary: "Renaming 'azure-native:network:ipallocationmethod' to " +
+						"'azure-native:network:IpAllocationMethod' in the schema",
+				},
+			},
+		},
+		{
+			name: "duplicates, uppercase",
+			input: &schema.PackageSpec{
+				Types: map[string]schema.ComplexTypeSpec{
+					"azure-native:network:IPAllocationMethod": {},
+					"azure-native:network:IpAllocationMethod": {},
+				},
+			},
+			expectedSpec: &schema.PackageSpec{
+				Types: map[string]schema.ComplexTypeSpec{
+					"azure-native:network:IPAllocationMethod": {},
+				},
+			},
+			expectedDiags: hcl.Diagnostics{
+				&hcl.Diagnostic{
+					Severity: hcl.DiagWarning,
+					Summary: "Renaming 'azure-native:network:IpAllocationMethod' to " +
+						"'azure-native:network:IPAllocationMethod' in the schema",
+				},
+			},
+		},
+		{
+			name: "multiple duplicates",
+			input: &schema.PackageSpec{
+				Types: map[string]schema.ComplexTypeSpec{
+					"azure-native:network:ipallocationmethod": {},
+					"azure-native:network:IPAllocationMethod": {},
+					"azure-native:network:IpAllocationMethod": {},
+				},
+			},
+			expectedSpec: &schema.PackageSpec{
+				Types: map[string]schema.ComplexTypeSpec{
+					"azure-native:network:IPAllocationMethod": {},
+				},
+			},
+			expectedDiags: hcl.Diagnostics{
+				&hcl.Diagnostic{
+					Severity: hcl.DiagWarning,
+					Summary: "Renaming 'azure-native:network:ipallocationmethod' to " +
+						"'azure-native:network:IPAllocationMethod' in the schema",
+				},
+				&hcl.Diagnostic{
+					Severity: hcl.DiagWarning,
+					Summary: "Renaming 'azure-native:network:IpAllocationMethod' to " +
+						"'azure-native:network:IPAllocationMethod' in the schema",
+				},
+			},
+		},
+		{
+			name: "multiple duplicates and non-duplicates",
+			input: &schema.PackageSpec{
+				Types: map[string]schema.ComplexTypeSpec{
+					"azure-native:network:other":               {},
+					"azure-native:network:ipsallocationmethod": {},
+					"azure-native:network:ip_allocationmethod": {},
+					"azure-native:network:ipallocationmethod":  {},
+					"azure-native:network:IPAllocationMethod":  {},
+					"azure-native:network:IpAllocationMethod":  {},
+				},
+			},
+			expectedSpec: &schema.PackageSpec{
+				Types: map[string]schema.ComplexTypeSpec{
+					"azure-native:network:other":               {},
+					"azure-native:network:ipsallocationmethod": {},
+					"azure-native:network:ip_allocationmethod": {},
+					"azure-native:network:IPAllocationMethod":  {},
+				},
+			},
+			expectedDiags: hcl.Diagnostics{
+				&hcl.Diagnostic{
+					Severity: hcl.DiagWarning,
+					Summary: "Renaming 'azure-native:network:ipallocationmethod' to " +
+						"'azure-native:network:IPAllocationMethod' in the schema",
+				},
+				&hcl.Diagnostic{
+					Severity: hcl.DiagWarning,
+					Summary: "Renaming 'azure-native:network:IpAllocationMethod' to " +
+						"'azure-native:network:IPAllocationMethod' in the schema",
+				},
+			},
+		},
+	}
+
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			actualSpec, actualDiags, err := DeduplicateTypes(c.input)
+			require.NoError(t, err)
+			require.ElementsMatch(t, c.expectedDiags, actualDiags)
+			require.Equal(t, c.expectedSpec, actualSpec)
+		})
+	}
+}