diff --git a/patches/0031-Optimize-startup-performance.patch b/patches/0031-Optimize-startup-performance.patch index 36df0472228..8471bbfe9c4 100644 --- a/patches/0031-Optimize-startup-performance.patch +++ b/patches/0031-Optimize-startup-performance.patch @@ -61,18 +61,10 @@ index 0000000000..35202ebd58 + return r.SchemaMap() +} diff --git a/shim/shim.go b/shim/shim.go -index e24e53fe17..00297dbe77 100644 +index e24e53fe17..3378f955bb 100644 --- a/shim/shim.go +++ b/shim/shim.go -@@ -3,6 +3,7 @@ package shim - import ( - "context" - "fmt" -+ "sync" - - pfprovider "github.com/hashicorp/terraform-plugin-framework/provider" - "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" -@@ -17,7 +18,7 @@ type UpstreamProvider struct { +@@ -17,7 +17,7 @@ type UpstreamProvider struct { } func NewUpstreamProvider(ctx context.Context) (UpstreamProvider, error) { @@ -81,7 +73,7 @@ index e24e53fe17..00297dbe77 100644 if err != nil { return UpstreamProvider{}, err } -@@ -44,44 +45,47 @@ func NewTagConfig(ctx context.Context, i interface{}) TagConfig { +@@ -44,44 +44,42 @@ func NewTagConfig(ctx context.Context, i interface{}) TagConfig { // rationale for this is that Pulumi copies tags to tags_all before it hits the TF layer, so these // attributes must match in schema. func markTagsAllNotComputedForResources(sdkV2Provider *schema.Provider) { @@ -115,13 +107,8 @@ index e24e53fe17..00297dbe77 100644 + u := *r + if r.SchemaFunc != nil { + old := r.SchemaFunc -+ var once sync.Once -+ var cache map[string]*schema.Schema + u.SchemaFunc = func() map[string]*schema.Schema { -+ once.Do(func() { -+ cache = markTagsAllNotComputedForSchema(rn, old()) -+ }) -+ return cache ++ return markTagsAllNotComputedForSchema(rn, old()) } + } else { + u.Schema = markTagsAllNotComputedForSchema(rn, r.Schema) diff --git a/patches/0033-DisableTagSchemaCheck-for-PF-provider.patch b/patches/0033-DisableTagSchemaCheck-for-PF-provider.patch index 6f62b87d1e8..9a149f27e02 100644 --- a/patches/0033-DisableTagSchemaCheck-for-PF-provider.patch +++ b/patches/0033-DisableTagSchemaCheck-for-PF-provider.patch @@ -68,10 +68,10 @@ index 0000000000..f790acb4e2 + return &resp +} diff --git a/shim/shim.go b/shim/shim.go -index 00297dbe77..9ef51a5245 100644 +index 3378f955bb..b94f722d26 100644 --- a/shim/shim.go +++ b/shim/shim.go -@@ -18,6 +18,7 @@ type UpstreamProvider struct { +@@ -17,6 +17,7 @@ type UpstreamProvider struct { } func NewUpstreamProvider(ctx context.Context) (UpstreamProvider, error) { diff --git a/provider/resources.go b/provider/resources.go index cdd0fb02f24..14d18b98c02 100644 --- a/provider/resources.go +++ b/provider/resources.go @@ -715,11 +715,16 @@ func Provider() *tfbridge.ProviderInfo { return ProviderFromMeta(tfbridge.NewProviderMetadata(runtimeMetadata)) } +func newUpstreamProvider(ctx context.Context) awsShim.UpstreamProvider { + upstreamProvider, err := awsShim.NewUpstreamProvider(ctx) + contract.AssertNoErrorf(err, "NewUpstreamProvider failed to initialize") + return upstreamProvider +} + // Provider returns additional overlaid schema and metadata associated with the aws package. func ProviderFromMeta(metaInfo *tfbridge.MetadataInfo) *tfbridge.ProviderInfo { ctx := context.Background() - upstreamProvider, err := awsShim.NewUpstreamProvider(ctx) - contract.AssertNoErrorf(err, "NewUpstreamProvider failed to initialize") + upstreamProvider := newUpstreamProvider(ctx) p := pftfbridge.MuxShimWithDisjointgPF(ctx, shimv2.NewProvider(upstreamProvider.SDKV2Provider, shimv2.WithDiffStrategy(shimv2.PlanState)), upstreamProvider.PluginFrameworkProvider) @@ -6144,20 +6149,8 @@ $ pulumi import aws:networkfirewall/resourcePolicy:ResourcePolicy example arn:aw return awsResource(mod, name).String(), nil })) - prov.P.ResourcesMap().Range(func(key string, value shim.Resource) bool { - // Skip resources that don't have tags. - tagsF, ok := value.Schema().GetOk("tags") - if !ok { - return true - } - // Skip resources that don't have tags_all. - _, ok = value.Schema().GetOk("tags_all") - if !ok { - return true - } - - // tags must be non-computed. - if tagsF.Computed() { + prov.P.ResourcesMap().Range(func(key string, res shim.Resource) bool { + if !hasNonComputedTagsAndTagsAllOptimized(key, res) { return true } @@ -6212,7 +6205,7 @@ $ pulumi import aws:networkfirewall/resourcePolicy:ResourcePolicy example arn:aw // Fixes a spurious diff on repeat pulumi up for the aws_wafv2_web_acl resource (pulumi/pulumi#1423). shimv2.SetInstanceStateStrategy(prov.P.ResourcesMap().Get("aws_wafv2_web_acl"), shimv2.CtyInstanceState) - prov.SetAutonaming(255, "-") + setAutonaming(&prov) prov.MustApplyAutoAliases() @@ -6220,3 +6213,131 @@ $ pulumi import aws:networkfirewall/resourcePolicy:ResourcePolicy example arn:aw return &prov } + +const nameProperty = "name" + +// prov.SetAutonaming is too inefficient for AWS as it forces SchemaFunc calls for large resources +// that use up RAM. Instead of doing prov.SetAutonaming(255, "-") we call a surgically crafted +// setAutonaming that avoids these calls. +func setAutonaming(p *tfbridge.ProviderInfo) { + maxLength := 255 + separator := "-" + for resname, res := range p.Resources { + // Only apply auto-name to input properties (Optional || Required) named `name` + if !hasOptionalOrRequiredNamePropertyOptimized(p.P, resname) { + continue + } + if _, hasfield := res.Fields[nameProperty]; !hasfield { + if res.Fields == nil { + res.Fields = make(map[string]*tfbridge.SchemaInfo) + } + res.Fields[nameProperty] = tfbridge.AutoName(nameProperty, maxLength, separator) + } + } +} + +func hasOptionalOrRequiredNameProperty(p shim.Provider, tfResourceName string) bool { + if schema := p.ResourcesMap().Get(tfResourceName); schema != nil { + // Only apply auto-name to input properties (Optional || Required) named `name` + if sch := schema.Schema().Get(nameProperty); sch != nil && (sch.Optional() || sch.Required()) { + return true + } + } + return false +} + +func hasOptionalOrRequiredNamePropertyOptimized(p shim.Provider, tfResourceName string) bool { + switch tfResourceName { + case + "aws_quicksight_account_subscription", + "aws_quicksight_group", + "aws_quicksight_group_membership", + "aws_quicksight_user", + "aws_wafv2_web_acl_association", + "aws_wafv2_web_acl_logging_configuration": + return false + case "aws_medialive_channel", + "aws_opsworks_custom_layer", + "aws_opsworks_ecs_cluster_layer", + "aws_opsworks_ganglia_layer", + "aws_opsworks_haproxy_layer", + "aws_opsworks_java_app_layer", + "aws_opsworks_memcached_layer", + "aws_opsworks_mysql_layer", + "aws_opsworks_nodejs_app_layer", + "aws_opsworks_php_app_layer", + "aws_opsworks_rails_app_layer", + "aws_quicksight_analysis", + "aws_quicksight_dashboard", + "aws_quicksight_data_set", + "aws_quicksight_data_source", + "aws_quicksight_template", + "aws_quicksight_theme", + "aws_wafv2_ip_set", + "aws_wafv2_regex_pattern_set", + "aws_wafv2_rule_group", + "aws_wafv2_web_acl", + "aws_kinesis_firehose_delivery_stream", + "aws_opsworks_static_web_layer": + return true + } + return hasOptionalOrRequiredNameProperty(p, tfResourceName) +} + +// Like hasNonComputedTagsAndTagsAll but optimized with an ad-hoc cache to avoid calling +// SchemaFunc() and allocating memory at startup. +func hasNonComputedTagsAndTagsAllOptimized(tfResourceName string, res shim.Resource) bool { + switch tfResourceName { + case "aws_kinesis_firehose_delivery_stream", + "aws_opsworks_custom_layer", + "aws_opsworks_ecs_cluster_layer", + "aws_opsworks_ganglia_layer", + "aws_opsworks_haproxy_layer", + "aws_opsworks_java_app_layer", + "aws_opsworks_memcached_layer", + "aws_opsworks_mysql_layer", + "aws_opsworks_nodejs_app_layer", + "aws_opsworks_php_app_layer", + "aws_opsworks_rails_app_layer", + "aws_opsworks_static_web_layer", + "aws_quicksight_analysis", + "aws_quicksight_dashboard", + "aws_quicksight_data_set", + "aws_quicksight_data_source", + "aws_quicksight_template", + "aws_quicksight_theme", + "aws_wafv2_ip_set", + "aws_wafv2_regex_pattern_set", + "aws_wafv2_rule_group", + "aws_wafv2_web_acl", + "aws_medialive_channel": + return true + case "aws_quicksight_user", + "aws_wafv2_web_acl_logging_configuration", + "aws_quicksight_group_membership", + "aws_wafv2_web_acl_association", + "aws_quicksight_group", + "aws_quicksight_account_subscription": + return false + } + + return hasNonComputedTagsAndTagsAll(tfResourceName, res) +} + +func hasNonComputedTagsAndTagsAll(tfResourceName string, res shim.Resource) bool { + // Skip resources that don't have tags. + tagsF, ok := res.Schema().GetOk("tags") + if !ok { + return false + } + // Skip resources that don't have tags_all. + _, ok = res.Schema().GetOk("tags_all") + if !ok { + return false + } + // tags must be non-computed. + if tagsF.Computed() { + return false + } + return true +} diff --git a/provider/resources_test.go b/provider/resources_test.go index c66d89465da..3cfea066214 100644 --- a/provider/resources_test.go +++ b/provider/resources_test.go @@ -1,8 +1,10 @@ package provider import ( + "context" "testing" + "github.com/pulumi/pulumi-terraform-bridge/v3/pkg/tfshim" "github.com/pulumi/pulumi/sdk/v3/go/common/resource" "github.com/stretchr/testify/assert" ) @@ -25,3 +27,42 @@ func TestParseDuration(t *testing.T) { assert.NotNil(t, d) } } + +func TestHasNonComputedTagsAndTagsAllOptimized(t *testing.T) { + p := Provider() + p.P.ResourcesMap().Range(func(key string, value shim.Resource) bool { + actual := hasNonComputedTagsAndTagsAllOptimized(key, value) + expected := hasNonComputedTagsAndTagsAll(key, value) + assert.Equal(t, expected, actual, "%q", key) + return true + }) + ctx := context.Background() + upstreamProvider := newUpstreamProvider(ctx) + for rn, r := range upstreamProvider.SDKV2Provider.ResourcesMap { + if r.SchemaFunc != nil { + res, ok := p.P.ResourcesMap().GetOk(rn) + if ok { + v := hasNonComputedTagsAndTagsAll(rn, res) + t.Logf("Should cache %v: %s", v, rn) + } + } + } +} + +func TestHasOptionalOrRequiredNamePropertyOptimized(t *testing.T) { + p := Provider() + p.P.ResourcesMap().Range(func(key string, value shim.Resource) bool { + actual := hasOptionalOrRequiredNameProperty(p.P, key) + expected := hasOptionalOrRequiredNamePropertyOptimized(p.P, key) + assert.Equal(t, expected, actual, "%q", key) + return true + }) + ctx := context.Background() + upstreamProvider := newUpstreamProvider(ctx) + for rn, r := range upstreamProvider.SDKV2Provider.ResourcesMap { + if r.SchemaFunc != nil { + v := hasOptionalOrRequiredNameProperty(p.P, rn) + t.Logf("Should cache %v: %s", v, rn) + } + } +}