diff --git a/examples/examples_dotnet_test.go b/examples/examples_dotnet_test.go index d12016e5321..46acaf3d880 100644 --- a/examples/examples_dotnet_test.go +++ b/examples/examples_dotnet_test.go @@ -1,4 +1,5 @@ // Copyright 2016-2017, Pulumi Corporation. All rights reserved. +//go:build dotnet || all // +build dotnet all package examples @@ -36,15 +37,14 @@ func TestAccFifoSqsQueueCs(t *testing.T) { func getCSBaseOptions(t *testing.T) integration.ProgramTestOptions { envRegion := getEnvRegion(t) - base := getBaseOptions() - csharpBase := base.With(integration.ProgramTestOptions{ + csharpBase := integration.ProgramTestOptions{ Config: map[string]string{ "aws:region": envRegion, }, Dependencies: []string{ "Pulumi.Aws", }, - }) + } return csharpBase } diff --git a/examples/examples_nodejs_test.go b/examples/examples_nodejs_test.go index 3cf1d6803f9..35e9d4451cb 100644 --- a/examples/examples_nodejs_test.go +++ b/examples/examples_nodejs_test.go @@ -37,8 +37,7 @@ func TestAccDedicatedHosts(t *testing.T) { // This is a specific test to ensure that we are testing for a missing region and erroring func TestAccCredentialsConfigTest(t *testing.T) { t.Skip("STACK72: Temp skip until we investigate the cause of https://github.com/pulumi/pulumi-aws/issues/1995") - base := getBaseOptions() - baseJS := base.With(integration.ProgramTestOptions{ + baseJS := integration.ProgramTestOptions{ Config: map[string]string{ "aws:region": "INVALID_REGION", }, @@ -47,7 +46,7 @@ func TestAccCredentialsConfigTest(t *testing.T) { }, Dir: filepath.Join(getCwd(t), "credentialsConfigTest"), ExpectFailure: true, - }) + } integration.ProgramTest(t, &baseJS) } @@ -67,7 +66,7 @@ func TestAccExpress(t *testing.T) { Dir: filepath.Join(getCwd(t), "express"), RunUpdateTest: true, }) - + skipRefresh(&test) integration.ProgramTest(t, &test) } @@ -107,6 +106,7 @@ func TestAccBucket(t *testing.T) { }, }) + skipRefresh(&test) integration.ProgramTest(t, &test) } @@ -116,15 +116,19 @@ func TestAccCloudWatch(t *testing.T) { Dir: filepath.Join(getCwd(t), "cloudwatch"), RunUpdateTest: true, }) - + skipRefresh(&test) integration.ProgramTest(t, &test) } func TestAccCloudWatchOidcManual(t *testing.T) { test := getJSBaseOptions(t). With(integration.ProgramTestOptions{ - Dir: filepath.Join(getCwd(t), "cloudwatchOidcManual"), - RunUpdateTest: true, + Dir: filepath.Join(getCwd(t), "cloudwatchOidcManual"), + + // TODO[pulumi/pulumi-aws#3193] multiple issues with refreshing and updating cleanly. + SkipRefresh: true, + AllowEmptyPreviewChanges: true, + AllowEmptyUpdateChanges: true, }) integration.ProgramTest(t, &test) @@ -136,7 +140,7 @@ func TestAccLogGroup(t *testing.T) { Dir: filepath.Join(getCwd(t), "logGroup"), RunUpdateTest: false, }) - + skipRefresh(&test) integration.ProgramTest(t, &test) } @@ -146,7 +150,7 @@ func TestAccQueue(t *testing.T) { Dir: filepath.Join(getCwd(t), "queue"), RunUpdateTest: true, }) - + skipRefresh(&test) integration.ProgramTest(t, &test) } @@ -155,7 +159,7 @@ func TestAccEventBus(t *testing.T) { With(integration.ProgramTestOptions{ Dir: filepath.Join(getCwd(t), "eventbus"), }) - + skipRefresh(&test) integration.ProgramTest(t, &test) } @@ -165,7 +169,7 @@ func TestAccStream(t *testing.T) { Dir: filepath.Join(getCwd(t), "stream"), RunUpdateTest: true, }) - + skipRefresh(&test) integration.ProgramTest(t, &test) } @@ -175,7 +179,7 @@ func TestAccTable(t *testing.T) { Dir: filepath.Join(getCwd(t), "table"), RunUpdateTest: true, }) - + skipRefresh(&test) integration.ProgramTest(t, &test) } @@ -185,7 +189,7 @@ func TestAccTopic(t *testing.T) { Dir: filepath.Join(getCwd(t), "topic"), RunUpdateTest: true, }) - + skipRefresh(&test) integration.ProgramTest(t, &test) } @@ -200,7 +204,7 @@ func TestAccSecretCapture(t *testing.T) { assert.NotContains(t, "s3cr3t", string(byts)) }, }) - + skipRefresh(&test) integration.ProgramTest(t, &test) } @@ -236,6 +240,7 @@ func TestAccCallbackFunction(t *testing.T) { } }, }) + skipRefresh(&test) integration.ProgramTest(t, &test) } @@ -266,7 +271,7 @@ func TestAccRoute53(t *testing.T) { Dir: filepath.Join(getCwd(t), "route53"), RunUpdateTest: true, }) - + skipRefresh(&test) integration.ProgramTest(t, &test) } @@ -276,7 +281,7 @@ func TestAccLambdaLayer(t *testing.T) { Dir: filepath.Join(getCwd(t), "lambda-layer-old"), RunUpdateTest: true, }) - + skipRefresh(&test) integration.ProgramTest(t, &test) } @@ -286,7 +291,7 @@ func TestAccLambdaContainerImages(t *testing.T) { RunUpdateTest: false, // new feature! Dir: filepath.Join(getCwd(t), "lambda-container-image"), }) - + skipRefresh(&test) integration.ProgramTest(t, &test) } @@ -296,7 +301,7 @@ func TestAccLambdaLayerNewEnums(t *testing.T) { Dir: filepath.Join(getCwd(t), "lambda-layer-new"), RunUpdateTest: false, }) - + skipRefresh(&test) integration.ProgramTest(t, &test) } @@ -327,7 +332,7 @@ func TestAccDeleteBeforeCreate(t *testing.T) { }, }, }) - + skipRefresh(&test) integration.ProgramTest(t, &test) } @@ -457,6 +462,12 @@ func TestAccWafV2(t *testing.T) { With(integration.ProgramTestOptions{ Dir: filepath.Join(getCwd(t), "wafv2"), }) + skipRefresh(&test) + + // TODO[pulumi/pulumi-aws#3190] there is a bug with non-empty diff after pulumi up. + test.AllowEmptyPreviewChanges = true + test.AllowEmptyUpdateChanges = true + integration.ProgramTest(t, &test) } @@ -491,8 +502,7 @@ func TestRegress2818(t *testing.T) { func getJSBaseOptions(t *testing.T) integration.ProgramTestOptions { envRegion := getEnvRegion(t) - base := getBaseOptions() - baseJS := base.With(integration.ProgramTestOptions{ + baseJS := integration.ProgramTestOptions{ Config: map[string]string{ "aws:region": "INVALID_REGION", "aws:envRegion": envRegion, @@ -500,7 +510,7 @@ func getJSBaseOptions(t *testing.T) integration.ProgramTestOptions { Dependencies: []string{ "@pulumi/aws", }, - }) + } return baseJS } diff --git a/examples/examples_py_test.go b/examples/examples_py_test.go index 293a6a0ad72..a2974a349b9 100644 --- a/examples/examples_py_test.go +++ b/examples/examples_py_test.go @@ -41,7 +41,7 @@ func TestAccCodeBuildProjectPy(t *testing.T) { Dir: filepath.Join(getCwd(t), "codebuild-project-py"), RunUpdateTest: false, }) - + skipRefresh(&test) integration.ProgramTest(t, &test) } @@ -69,15 +69,14 @@ func TestSecretManagerPy(t *testing.T) { func getPythonBaseOptions(t *testing.T) integration.ProgramTestOptions { envRegion := getEnvRegion(t) - base := getBaseOptions() - pythonBase := base.With(integration.ProgramTestOptions{ + pythonBase := integration.ProgramTestOptions{ Config: map[string]string{ "aws:region": envRegion, }, Dependencies: []string{ filepath.Join("..", "sdk", "python", "bin"), }, - }) + } return pythonBase } diff --git a/examples/examples_test.go b/examples/examples_test.go index 98a12e04955..f6632119226 100644 --- a/examples/examples_test.go +++ b/examples/examples_test.go @@ -50,14 +50,6 @@ func getCwd(t *testing.T) string { return cwd } -func getBaseOptions() integration.ProgramTestOptions { - return integration.ProgramTestOptions{ - ExpectRefreshChanges: true, - SkipRefresh: true, - Quick: true, - } -} - func validateAPITest(isValid func(body string)) func(t *testing.T, stack integration.RuntimeValidationStackInfo) { return func(t *testing.T, stack integration.RuntimeValidationStackInfo) { var resp *http.Response @@ -593,3 +585,13 @@ func TestWrongStateMaxItemOneDiffProduced(t *testing.T) { ` replay(t, repro) } + +// A lot of tests do not currently refresh cleanly. The work to root cause each tests has not been +// done yet but the common causes are listed here: +// +// TODO[pulumi/pulumi-aws#2246] specifically affects overlays such as bucket.onObjectCreated; may be worked around +// TODO[pulumi/pulumi#6235] +// TODO[pulumi/pulumi-terraform-bridge#1595] +func skipRefresh(opts *integration.ProgramTestOptions) { + opts.SkipRefresh = true +} diff --git a/examples/main_test.go b/examples/main_test.go index 2c47e67d6a5..c73831a798b 100644 --- a/examples/main_test.go +++ b/examples/main_test.go @@ -18,9 +18,9 @@ func TestMain(m *testing.M) { } command := exec.Command("make", "provider_no_deps") command.Dir = filepath.Join(cwd, "..") - err = command.Run() - if err != nil { - log.Println("Unable to build provider!") + if combinedOutput, err := command.CombinedOutput(); err != nil { + log.Printf("Unable to build provider. Command `make provider_no_deps` failed with: %v\nOutput:\n%s", + err, combinedOutput) os.Exit(1) }