Optimize RAM usage for the provider

patches/0031-Optimize-startup-performance.patch

@@ -61,18 +61,10 @@ index 0000000000..35202ebd58
 +	return r.SchemaMap()
 +}
 diff --git a/shim/shim.go b/shim/shim.go
-index e24e53fe17..00297dbe77 100644
+index e24e53fe17..3378f955bb 100644
 --- a/shim/shim.go
 +++ b/shim/shim.go
-@@ -3,6 +3,7 @@ package shim
- import (
- 	"context"
- 	"fmt"
-+	"sync"
-
- 	pfprovider "github.com/hashicorp/terraform-plugin-framework/provider"
- 	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema"
-@@ -17,7 +18,7 @@ type UpstreamProvider struct {
+@@ -17,7 +17,7 @@ type UpstreamProvider struct {
  }
 
  func NewUpstreamProvider(ctx context.Context) (UpstreamProvider, error) {
@@ -81,7 +73,7 @@ index e24e53fe17..00297dbe77 100644
  	if err != nil {
  		return UpstreamProvider{}, err
  	}
-@@ -44,44 +45,47 @@ func NewTagConfig(ctx context.Context, i interface{}) TagConfig {
+@@ -44,44 +44,42 @@ func NewTagConfig(ctx context.Context, i interface{}) TagConfig {
  // rationale for this is that Pulumi copies tags to tags_all before it hits the TF layer, so these
  // attributes must match in schema.
  func markTagsAllNotComputedForResources(sdkV2Provider *schema.Provider) {
@@ -115,13 +107,8 @@ index e24e53fe17..00297dbe77 100644
 +	u := *r
 +	if r.SchemaFunc != nil {
 +		old := r.SchemaFunc
-+		var once sync.Once
-+		var cache map[string]*schema.Schema
 +		u.SchemaFunc = func() map[string]*schema.Schema {
-+			once.Do(func() {
-+				cache = markTagsAllNotComputedForSchema(rn, old())
-+			})
-+			return cache
++			return markTagsAllNotComputedForSchema(rn, old())
  		}
 +	} else {
 +		u.Schema = markTagsAllNotComputedForSchema(rn, r.Schema)

patches/0033-DisableTagSchemaCheck-for-PF-provider.patch

@@ -68,10 +68,10 @@ index 0000000000..f790acb4e2
 +	return &resp
 +}
 diff --git a/shim/shim.go b/shim/shim.go
-index 00297dbe77..9ef51a5245 100644
+index 3378f955bb..b94f722d26 100644
 --- a/shim/shim.go
 +++ b/shim/shim.go
-@@ -18,6 +18,7 @@ type UpstreamProvider struct {
+@@ -17,6 +17,7 @@ type UpstreamProvider struct {
  }
 
  func NewUpstreamProvider(ctx context.Context) (UpstreamProvider, error) {

provider/resources.go

@@ -715,11 +715,16 @@ func Provider() *tfbridge.ProviderInfo {
 	return ProviderFromMeta(tfbridge.NewProviderMetadata(runtimeMetadata))
 }
 
+func newUpstreamProvider(ctx context.Context) awsShim.UpstreamProvider {
+	upstreamProvider, err := awsShim.NewUpstreamProvider(ctx)
+	contract.AssertNoErrorf(err, "NewUpstreamProvider failed to initialize")
+	return upstreamProvider
+}
+
 // Provider returns additional overlaid schema and metadata associated with the aws package.
 func ProviderFromMeta(metaInfo *tfbridge.MetadataInfo) *tfbridge.ProviderInfo {
 	ctx := context.Background()
-	upstreamProvider, err := awsShim.NewUpstreamProvider(ctx)
-	contract.AssertNoErrorf(err, "NewUpstreamProvider failed to initialize")
+	upstreamProvider := newUpstreamProvider(ctx)
 
 	p := pftfbridge.MuxShimWithDisjointgPF(ctx, shimv2.NewProvider(upstreamProvider.SDKV2Provider, shimv2.WithDiffStrategy(shimv2.PlanState)), upstreamProvider.PluginFrameworkProvider)
 
@@ -6144,20 +6149,8 @@ $ pulumi import aws:networkfirewall/resourcePolicy:ResourcePolicy example arn:aw
 			return awsResource(mod, name).String(), nil
 		}))
 
-	prov.P.ResourcesMap().Range(func(key string, value shim.Resource) bool {
-		// Skip resources that don't have tags.
-		tagsF, ok := value.Schema().GetOk("tags")
-		if !ok {
-			return true
-		}
-		// Skip resources that don't have tags_all.
-		_, ok = value.Schema().GetOk("tags_all")
-		if !ok {
-			return true
-		}
-
-		// tags must be non-computed.
-		if tagsF.Computed() {
+	prov.P.ResourcesMap().Range(func(key string, res shim.Resource) bool {
+		if !hasNonComputedTagsAndTagsAllOptimized(key, res) {
 			return true
 		}
 
@@ -6212,11 +6205,139 @@ $ pulumi import aws:networkfirewall/resourcePolicy:ResourcePolicy example arn:aw
 	// Fixes a spurious diff on repeat pulumi up for the aws_wafv2_web_acl resource (pulumi/pulumi#1423).
 	shimv2.SetInstanceStateStrategy(prov.P.ResourcesMap().Get("aws_wafv2_web_acl"), shimv2.CtyInstanceState)
 
-	prov.SetAutonaming(255, "-")
+	setAutonaming(&prov)
 
 	prov.MustApplyAutoAliases()
 
 	prov.XSkipDetailedDiffForChanges = true
 
 	return &prov
 }
+
+const nameProperty = "name"
+
+// prov.SetAutonaming is too inefficient for AWS as it forces SchemaFunc calls for large resources
+// that use up RAM. Instead of doing prov.SetAutonaming(255, "-") we call a surgically crafted
+// setAutonaming that avoids these calls.
+func setAutonaming(p *tfbridge.ProviderInfo) {
+	maxLength := 255
+	separator := "-"
+	for resname, res := range p.Resources {
+		// Only apply auto-name to input properties (Optional || Required) named `name`
+		if !hasOptionalOrRequiredNamePropertyOptimized(p.P, resname) {
+			continue
+		}
+		if _, hasfield := res.Fields[nameProperty]; !hasfield {
+			if res.Fields == nil {
+				res.Fields = make(map[string]*tfbridge.SchemaInfo)
+			}
+			res.Fields[nameProperty] = tfbridge.AutoName(nameProperty, maxLength, separator)
+		}
+	}
+}
+
+func hasOptionalOrRequiredNameProperty(p shim.Provider, tfResourceName string) bool {
+	if schema := p.ResourcesMap().Get(tfResourceName); schema != nil {
+		// Only apply auto-name to input properties (Optional || Required) named `name`
+		if sch := schema.Schema().Get(nameProperty); sch != nil && (sch.Optional() || sch.Required()) {
+			return true
+		}
+	}
+	return false
+}
+
+func hasOptionalOrRequiredNamePropertyOptimized(p shim.Provider, tfResourceName string) bool {
+	switch tfResourceName {
+	case
+		"aws_quicksight_account_subscription",
+		"aws_quicksight_group",
+		"aws_quicksight_group_membership",
+		"aws_quicksight_user",
+		"aws_wafv2_web_acl_association",
+		"aws_wafv2_web_acl_logging_configuration":
+		return false
+	case "aws_medialive_channel",
+		"aws_opsworks_custom_layer",
+		"aws_opsworks_ecs_cluster_layer",
+		"aws_opsworks_ganglia_layer",
+		"aws_opsworks_haproxy_layer",
+		"aws_opsworks_java_app_layer",
+		"aws_opsworks_memcached_layer",
+		"aws_opsworks_mysql_layer",
+		"aws_opsworks_nodejs_app_layer",
+		"aws_opsworks_php_app_layer",
+		"aws_opsworks_rails_app_layer",
+		"aws_quicksight_analysis",
+		"aws_quicksight_dashboard",
+		"aws_quicksight_data_set",
+		"aws_quicksight_data_source",
+		"aws_quicksight_template",
+		"aws_quicksight_theme",
+		"aws_wafv2_ip_set",
+		"aws_wafv2_regex_pattern_set",
+		"aws_wafv2_rule_group",
+		"aws_wafv2_web_acl",
+		"aws_kinesis_firehose_delivery_stream",
+		"aws_opsworks_static_web_layer":
+		return true
+	}
+	return hasOptionalOrRequiredNameProperty(p, tfResourceName)
+}
+
+// Like hasNonComputedTagsAndTagsAll but optimized with an ad-hoc cache to avoid calling
+// SchemaFunc() and allocating memory at startup.
+func hasNonComputedTagsAndTagsAllOptimized(tfResourceName string, res shim.Resource) bool {
+	switch tfResourceName {
+	case "aws_kinesis_firehose_delivery_stream",
+		"aws_opsworks_custom_layer",
+		"aws_opsworks_ecs_cluster_layer",
+		"aws_opsworks_ganglia_layer",
+		"aws_opsworks_haproxy_layer",
+		"aws_opsworks_java_app_layer",
+		"aws_opsworks_memcached_layer",
+		"aws_opsworks_mysql_layer",
+		"aws_opsworks_nodejs_app_layer",
+		"aws_opsworks_php_app_layer",
+		"aws_opsworks_rails_app_layer",
+		"aws_opsworks_static_web_layer",
+		"aws_quicksight_analysis",
+		"aws_quicksight_dashboard",
+		"aws_quicksight_data_set",
+		"aws_quicksight_data_source",
+		"aws_quicksight_template",
+		"aws_quicksight_theme",
+		"aws_wafv2_ip_set",
+		"aws_wafv2_regex_pattern_set",
+		"aws_wafv2_rule_group",
+		"aws_wafv2_web_acl",
+		"aws_medialive_channel":
+		return true
+	case "aws_quicksight_user",
+		"aws_wafv2_web_acl_logging_configuration",
+		"aws_quicksight_group_membership",
+		"aws_wafv2_web_acl_association",
+		"aws_quicksight_group",
+		"aws_quicksight_account_subscription":
+		return false
+	}
+
+	return hasNonComputedTagsAndTagsAll(tfResourceName, res)
+}
+
+func hasNonComputedTagsAndTagsAll(tfResourceName string, res shim.Resource) bool {
+	// Skip resources that don't have tags.
+	tagsF, ok := res.Schema().GetOk("tags")
+	if !ok {
+		return false
+	}
+	// Skip resources that don't have tags_all.
+	_, ok = res.Schema().GetOk("tags_all")
+	if !ok {
+		return false
+	}
+	// tags must be non-computed.
+	if tagsF.Computed() {
+		return false
+	}
+	return true
+}

provider/resources_test.go

@@ -1,8 +1,10 @@
 package provider
 
 import (
+	"context"
 	"testing"
 
+	"github.com/pulumi/pulumi-terraform-bridge/v3/pkg/tfshim"
 	"github.com/pulumi/pulumi/sdk/v3/go/common/resource"
 	"github.com/stretchr/testify/assert"
 )
@@ -25,3 +27,42 @@ func TestParseDuration(t *testing.T) {
 		assert.NotNil(t, d)
 	}
 }
+
+func TestHasNonComputedTagsAndTagsAllOptimized(t *testing.T) {
+	p := Provider()
+	p.P.ResourcesMap().Range(func(key string, value shim.Resource) bool {
+		actual := hasNonComputedTagsAndTagsAllOptimized(key, value)
+		expected := hasNonComputedTagsAndTagsAll(key, value)
+		assert.Equal(t, expected, actual, "%q", key)
+		return true
+	})
+	ctx := context.Background()
+	upstreamProvider := newUpstreamProvider(ctx)
+	for rn, r := range upstreamProvider.SDKV2Provider.ResourcesMap {
+		if r.SchemaFunc != nil {
+			res, ok := p.P.ResourcesMap().GetOk(rn)
+			if ok {
+				v := hasNonComputedTagsAndTagsAll(rn, res)
+				t.Logf("Should cache %v: %s", v, rn)
+			}
+		}
+	}
+}
+
+func TestHasOptionalOrRequiredNamePropertyOptimized(t *testing.T) {
+	p := Provider()
+	p.P.ResourcesMap().Range(func(key string, value shim.Resource) bool {
+		actual := hasOptionalOrRequiredNameProperty(p.P, key)
+		expected := hasOptionalOrRequiredNamePropertyOptimized(p.P, key)
+		assert.Equal(t, expected, actual, "%q", key)
+		return true
+	})
+	ctx := context.Background()
+	upstreamProvider := newUpstreamProvider(ctx)
+	for rn, r := range upstreamProvider.SDKV2Provider.ResourcesMap {
+		if r.SchemaFunc != nil {
+			v := hasOptionalOrRequiredNameProperty(p.P, rn)
+			t.Logf("Should cache %v: %s", v, rn)
+		}
+	}
+}