Download upstream docs for dynamically bridged provider (#2664)

This pull request adds logic that enables us to generate resource docs
for a dynamically bridged provider using the upstream dependency.

Usage for remote providers:
`pulumi package get-schema terraform-provider <registry address>
<version> fullDocs=<true|false>`.

Adds a parameterized arg field to `pulumi package get-schema
terraform-provider <registry address> <version>` called `fullDocs`,
which when set to `true` will instruct the bridge to `git clone --depth
1 -b <version> <terraform provider github repo> <local dir for dynamic
docs>`. This allows us to keep the exact same docs logic we have
established. The shallow clone targets the exact doc version we need.
(*)

For this, we infer the github repo from the OpenTofu org name. It is
based on the assumption that `registry.opentofu.org/org/foo` is based on
a provider that lives at `github.com/org/terraform-provider-foo`.
OpenTofu says their protocol follows that of the HashiCorp Terraform
Registry which [requires an org/user and a provider name of the format
`terraform-provider-foo`](https://developer.hashicorp.com/terraform/registry/providers/publishing)
and we have historical evidence that GitHub is the most commonly used
source host for terraform providers. See also
opentofu/registry#1337.

Usage for local providers:
`pulumi package get-schema terraform-provider <local-path>
upstreamRepoPath=<localPath>`. Here, the local docs path gets read
directly into `upstreamRepoPath`.

The one thing that is perhaps missing here is to expand the remote args
to allow to take a source repo as an additional argument. However, our
internal use case is primarily one of automation, so this is not an
immediate need.

(*)Cloning docs for the AWS terraform provider at v5.70.0 has the
following time performance:
```
git clone --depth 1 -b v5.70.0  awsDir  
2.06s user 
1.62s system 
34% cpu 
10.713 total
```

Fixes #2607

Makefile

@@ -6,7 +6,7 @@ export PULUMI_DISABLE_AUTOMATIC_PLUGIN_ACQUISITION := true
 PROJECT_DIR := $(patsubst %/,%,$(dir $(abspath $(lastword $(MAKEFILE_LIST)))))
 
 install_plugins::
-	pulumi plugin install converter terraform 1.0.19
+	pulumi plugin install converter terraform 1.0.20
 	pulumi plugin install resource random 4.16.3
 	pulumi plugin install resource aws 6.22.2
 	pulumi plugin install resource archive 0.0.4

dynamic/go.mod

@@ -189,7 +189,7 @@ require (
 	github.com/pgavlin/fx v0.1.6 // indirect
 	github.com/pgavlin/goldmark v1.1.33-0.20200616210433-b5eb04559386 // indirect
 	github.com/pjbgf/sha1cd v0.3.0 // indirect
-	github.com/pkg/errors v0.9.1 // indirect
+	github.com/pkg/errors v0.9.1
 	github.com/pkg/term v1.1.0 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/posener/complete v1.2.3 // indirect
@@ -212,7 +212,7 @@ require (
 	github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect
 	github.com/shopspring/decimal v1.3.1 // indirect
 	github.com/skeema/knownhosts v1.2.2 // indirect
-	github.com/spf13/afero v1.9.5 // indirect
+	github.com/spf13/afero v1.9.5
 	github.com/spf13/cast v1.5.0 // indirect
 	github.com/spf13/cobra v1.8.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect

dynamic/info.go

@@ -33,18 +33,15 @@ import (
 
 func providerInfo(ctx context.Context, p run.Provider, value parameterize.Value) (tfbridge.ProviderInfo, error) {
 	provider := proto.New(ctx, p)
-	prov := tfbridge.ProviderInfo{
-		P:           provider,
-		Name:        p.Name(),
-		Version:     p.Version(),
-		Description: "A Pulumi provider dynamically bridged from " + p.Name() + ".",
-		Publisher:   "Pulumi",
 
+	prov := tfbridge.ProviderInfo{
+		P:              provider,
+		Name:           p.Name(),
+		Version:        p.Version(),
+		Description:    "A Pulumi provider dynamically bridged from " + p.Name() + ".",
+		Publisher:      "Pulumi",
 		ResourcePrefix: inferResourcePrefix(provider),
 
-		// To avoid bogging down schema generation speed, we skip all examples.
-		SkipExamples: func(tfbridge.SkipExamplesArgs) bool { return true },
-
 		MetadataInfo: &tfbridge.MetadataInfo{
 			Path: "", Data: tfbridge.ProviderMetadata(nil),
 		},
@@ -84,6 +81,24 @@ func providerInfo(ctx context.Context, p run.Provider, value parameterize.Value)
 			}
 		},
 	}
+	// Add presumed best-effort GitHub org to the provider info.
+	// We do not set the GitHubOrg field for a local dynamic provider.
+	if value.Remote != nil {
+		// https://github.com/opentofu/registry/issues/1337:
+		// Due to discrepancies in the registry protocol/implementation,
+		// we infer the Terraform provider's source code repository via the following assumptions:
+		// - The provider's source code is hosted at github.com
+		// - The provider's github org, for providers, is the namespace field of the registry name
+		// Example:
+		//
+		// opentofu.org/provider/hashicorp/random -> "hashicorp" is deduced to be the github org.
+		// Note that this will only work for the provider (not the module) protocol.
+		urlFields := strings.Split(value.Remote.URL, "/")
+		ghOrg := urlFields[len(urlFields)-2]
+		name := urlFields[len(urlFields)-1]
+		prov.GitHubOrg = ghOrg
+		prov.Repository = "https://github.com/" + ghOrg + "/terraform-provider-" + name
+	}
 
 	if err := fixup.Default(&prov); err != nil {
 		return prov, err

dynamic/main.go

@@ -19,14 +19,17 @@ import (
 	"encoding/json"
 	"fmt"
 	"os"
+	"os/exec"
 
 	"github.com/blang/semver"
 	"github.com/opentofu/opentofu/shim/run"
+	"github.com/pkg/errors"
 	"github.com/pulumi/pulumi/pkg/v3/codegen/schema"
 	"github.com/pulumi/pulumi/sdk/v3/go/common/diag"
 	"github.com/pulumi/pulumi/sdk/v3/go/common/diag/colors"
 	"github.com/pulumi/pulumi/sdk/v3/go/common/resource/plugin"
 	"github.com/pulumi/pulumi/sdk/v3/go/common/util/contract"
+	"github.com/spf13/afero"
 
 	"github.com/pulumi/pulumi-terraform-bridge/dynamic/parameterize"
 	"github.com/pulumi/pulumi-terraform-bridge/dynamic/version"
@@ -63,15 +66,26 @@ func initialSetup() (info.Provider, pfbridge.ProviderMetadata, func() error) {
 	}
 
 	var metadata pfbridge.ProviderMetadata
+	var fullDocs bool
 	metadata = pfbridge.ProviderMetadata{
 		XGetSchema: func(ctx context.Context, req plugin.GetSchemaRequest) ([]byte, error) {
-			packageSchema, err := tfgen.GenerateSchemaWithOptions(tfgen.GenerateSchemaOptions{
+			// Create a custom generator for schema. Examples will only be generated if `fullDocs` is set.
+			g, err := tfgen.NewGenerator(tfgen.GeneratorOptions{
+				Package:      info.Name,
+				Version:      info.Version,
+				Language:     tfgen.Schema,
 				ProviderInfo: info,
-				DiagnosticsSink: diag.DefaultSink(os.Stdout, os.Stderr, diag.FormatOptions{
+				Root:         afero.NewMemMapFs(),
+				Sink: diag.DefaultSink(os.Stdout, os.Stderr, diag.FormatOptions{
 					Color: colors.Always,
 				}),
-				XInMemoryDocs: true,
+				XInMemoryDocs: !fullDocs,
+				SkipExamples:  !fullDocs,
 			})
+			if err != nil {
+				return nil, errors.Wrapf(err, "failed to create generator")
+			}
+			packageSchema, err := g.Generate()
 			if err != nil {
 				return nil, err
 			}
@@ -148,6 +162,33 @@ func initialSetup() (info.Provider, pfbridge.ProviderMetadata, func() error) {
 				return plugin.ParameterizeResponse{}, err
 			}
 
+			switch args.Remote {
+			case nil:
+				// We're using local args.
+				if args.Local.UpstreamRepoPath != "" {
+					info.UpstreamRepoPath = args.Local.UpstreamRepoPath
+					fullDocs = true
+				}
+			default:
+				fullDocs = args.Remote.Docs
+				if fullDocs {
+					// Write the upstream files at this version to a temporary directory
+					tmpDir, err := os.MkdirTemp("", "upstreamRepoDir")
+					if err != nil {
+						return plugin.ParameterizeResponse{}, err
+					}
+					versionTag := "v" + info.Version
+					cmd := exec.Command(
+						"git", "clone", "--depth", "1", "-b", versionTag, info.Repository, tmpDir,
+					)
+					err = cmd.Run()
+					if err != nil {
+						return plugin.ParameterizeResponse{}, err
+					}
+					info.UpstreamRepoPath = tmpDir
+				}
+			}
+
 			return plugin.ParameterizeResponse{
 				Name:    p.Name(),
 				Version: v,

dynamic/parameterize/args.go

@@ -31,27 +31,68 @@ type RemoteArgs struct {
 	Name string
 	// Version is the (possibly empty) version constraint on the provider.
 	Version string
+	// Docs indicates if full schema documentation should be generated.
+	Docs bool
 }
 
 // LocalArgs represents a local TF provider referenced by path.
 type LocalArgs struct {
 	// Path is the path to the provider binary. It can be relative or absolute.
 	Path string
+	// UpstreamRepoPath (if provided) is the local path to the dynamically bridged Terraform provider's repo.
+	//
+	// If set, full documentation will be generated for the provider.
+	// If not set, only documentation from the TF provider's schema will be used.
+	UpstreamRepoPath string
 }
 
 func ParseArgs(args []string) (Args, error) {
 	// Check for a leading '.' or '/' to indicate a path
 	if len(args) >= 1 &&
 		(strings.HasPrefix(args[0], "./") || strings.HasPrefix(args[0], "/")) {
 		if len(args) > 1 {
-			return Args{}, fmt.Errorf("path based providers are only parameterized by 1 argument: <path>")
+			docsArg := args[1]
+			upstreamRepoPath, found := strings.CutPrefix(docsArg, "upstreamRepoPath=")
+			if !found {
+				return Args{}, fmt.Errorf(
+					"path based providers are only parameterized by 2 arguments: <path> " +
+						"[upstreamRepoPath=<path/to/files>]",
+				)
+			}
+			if upstreamRepoPath == "" {
+				return Args{}, fmt.Errorf(
+					"upstreamRepoPath must be set to a non-empty value: " +
+						"upstreamRepoPath=path/to/files",
+				)
+			}
+			return Args{Local: &LocalArgs{Path: args[0], UpstreamRepoPath: upstreamRepoPath}}, nil
 		}
 		return Args{Local: &LocalArgs{Path: args[0]}}, nil
 	}
 
 	// This is a registry based provider
 	var remote RemoteArgs
 	switch len(args) {
+	// The third argument, if any, is the full docs option for when we need to generate docs
+	case 3:
+		docsArg := args[2]
+		errMsg := "expected third parameterized argument to be 'fullDocs=<true|false>' or be empty"
+
+		fullDocs, found := strings.CutPrefix(docsArg, "fullDocs=")
+		if !found {
+			return Args{}, fmt.Errorf("%s", errMsg)
+		}
+
+		switch fullDocs {
+		case "true":
+			remote.Docs = true
+		case "false":
+			// Do nothing
+		default:
+			return Args{}, fmt.Errorf("%s", errMsg)
+		}
+
+		fallthrough
 	// The second argument, if any is the version
 	case 2:
 		remote.Version = args[1]
@@ -61,6 +102,6 @@ func ParseArgs(args []string) (Args, error) {
 		remote.Name = args[0]
 		return Args{Remote: &remote}, nil
 	default:
-		return Args{}, fmt.Errorf("expected to be parameterized by 1-2 arguments: <name> [version]")
+		return Args{}, fmt.Errorf("expected to be parameterized by 1-3 arguments: <name> [version] [fullDocs=<true|false>]")
 	}
 }

dynamic/parameterize/args_test.go

@@ -35,6 +35,30 @@ func TestParseArgs(t *testing.T) {
 			args:   []string{"./my-provider"},
 			expect: Args{Local: &LocalArgs{Path: "./my-provider"}},
 		},
+		{
+			name: "local too many args",
+			args: []string{"./my-provider", "nonsense"},
+			errMsg: autogold.Expect(
+				"path based providers are only parameterized by 2 arguments: <path> [upstreamRepoPath=<path/to/files>]",
+			),
+		},
+		{
+			name: "local with docs location",
+			args: []string{"./my-provider", "upstreamRepoPath=./my-provider"},
+			expect: Args{
+				Local: &LocalArgs{
+					Path:             "./my-provider",
+					UpstreamRepoPath: "./my-provider",
+				},
+			},
+		},
+		{
+			name: "local empty upstreamRepoPath",
+			args: []string{"./my-provider", "upstreamRepoPath="},
+			errMsg: autogold.Expect(
+				"upstreamRepoPath must be set to a non-empty value: upstreamRepoPath=path/to/files",
+			),
+		},
 		{
 			name:   "remote",
 			args:   []string{"my-registry.io/typ"},
@@ -51,12 +75,53 @@ func TestParseArgs(t *testing.T) {
 		{
 			name:   "no args",
 			args:   []string{},
-			errMsg: autogold.Expect("expected to be parameterized by 1-2 arguments: <name> [version]"),
+			errMsg: autogold.Expect("expected to be parameterized by 1-3 arguments: <name> [version] [fullDocs=<true|false>]"),
 		},
 		{
 			name:   "too many args",
-			args:   []string{"arg1", "arg2", "arg3"},
-			errMsg: autogold.Expect("expected to be parameterized by 1-2 arguments: <name> [version]"),
+			args:   []string{"arg1", "arg2", "arg3", "arg4"},
+			errMsg: autogold.Expect("expected to be parameterized by 1-3 arguments: <name> [version] [fullDocs=<true|false>]"),
+		},
+		{
+			name: "invalid third arg",
+			args: []string{"arg1", "arg2", "arg3"},
+			errMsg: autogold.Expect(
+				"expected third parameterized argument to be 'fullDocs=<true|false>' or be empty",
+			),
+		},
+		{
+			name: "empty third arg",
+			args: []string{"arg1", "arg2"},
+			expect: Args{Remote: &RemoteArgs{
+				Name:    "arg1",
+				Version: "arg2",
+				Docs:    false,
+			}},
+		},
+		{
+			name: "valid third arg true",
+			args: []string{"my-registry.io/typ", "1.2.3", "fullDocs=true"},
+			expect: Args{Remote: &RemoteArgs{
+				Name:    "my-registry.io/typ",
+				Version: "1.2.3",
+				Docs:    true,
+			}},
+		},
+		{
+			name: "valid third arg false",
+			args: []string{"my-registry.io/typ", "1.2.3", "fullDocs=false"},
+			expect: Args{Remote: &RemoteArgs{
+				Name:    "my-registry.io/typ",
+				Version: "1.2.3",
+				Docs:    false,
+			}},
+		},
+		{
+			name: "third arg invalid input",
+			args: []string{"my-registry.io/typ", "1.2.3", "fullDocs=invalid-input"},
+			errMsg: autogold.Expect(
+				"expected third parameterized argument to be 'fullDocs=<true|false>' or be empty",
+			),
 		},
 	}
 

dynamic/provider_test.go

@@ -457,6 +457,49 @@ func TestSchemaGeneration(t *testing.T) { //nolint:paralleltest
 	testSchema("databricks/databricks", "1.50.0")
 }
 
+func TestSchemaGenerationFullDocs(t *testing.T) { //nolint:paralleltest
+	skipWindows(t)
+	type testCase struct {
+		name     string
+		version  string
+		fullDocs string
+	}
+
+	tc := testCase{
+		name:     "hashicorp/random",
+		version:  "3.6.3",
+		fullDocs: "fullDocs=true",
+	}
+
+	t.Run(strings.Join([]string{tc.name, tc.version}, "-"), func(t *testing.T) {
+		helper.Integration(t)
+		ctx := context.Background()
+
+		server := grpcTestServer(ctx, t)
+
+		result, err := server.Parameterize(ctx, &pulumirpc.ParameterizeRequest{
+			Parameters: &pulumirpc.ParameterizeRequest_Args{
+				Args: &pulumirpc.ParameterizeRequest_ParametersArgs{
+					Args: []string{tc.name, tc.version, tc.fullDocs},
+				},
+			},
+		})
+		require.NoError(t, err)
+
+		assert.Equal(t, tc.version, result.Version)
+
+		schema, err := server.GetSchema(ctx, &pulumirpc.GetSchemaRequest{
+			SubpackageName:    result.Name,
+			SubpackageVersion: result.Version,
+		})
+
+		require.NoError(t, err)
+		var fmtSchema bytes.Buffer
+		require.NoError(t, json.Indent(&fmtSchema, []byte(schema.Schema), "", "    "))
+		autogold.ExpectFile(t, autogold.Raw(fmtSchema.String()))
+	})
+}
+
 func TestRandomCreate(t *testing.T) {
 	t.Parallel()
 	ctx := context.Background()