Make integration tests stricter (#3183)

Prior to this change, all example-based acceptance tests inherited a
relaxed configuration.

```go
	return integration.ProgramTestOptions{
		ExpectRefreshChanges: true,
		SkipRefresh:          true,
		Quick:                true,
	}
```

This was masking issues with unexpected diffs during `refresh` and any
possible regressions as there provider evolves. After the change, only
the test that need it are configured with these relaxed options and the
code is cross-correlated to root cause issues where appropriate.

examples/examples_dotnet_test.go

@@ -1,4 +1,5 @@
 // Copyright 2016-2017, Pulumi Corporation.  All rights reserved.
+//go:build dotnet || all
 // +build dotnet all
 
 package examples
@@ -36,15 +37,14 @@ func TestAccFifoSqsQueueCs(t *testing.T) {
 
 func getCSBaseOptions(t *testing.T) integration.ProgramTestOptions {
 	envRegion := getEnvRegion(t)
-	base := getBaseOptions()
-	csharpBase := base.With(integration.ProgramTestOptions{
+	csharpBase := integration.ProgramTestOptions{
 		Config: map[string]string{
 			"aws:region": envRegion,
 		},
 		Dependencies: []string{
 			"Pulumi.Aws",
 		},
-	})
+	}
 
 	return csharpBase
 }

examples/examples_nodejs_test.go

@@ -37,8 +37,7 @@ func TestAccDedicatedHosts(t *testing.T) {
 // This is a specific test to ensure that we are testing for a missing region and erroring
 func TestAccCredentialsConfigTest(t *testing.T) {
 	t.Skip("STACK72: Temp skip until we investigate the cause of https://github.com/pulumi/pulumi-aws/issues/1995")
-	base := getBaseOptions()
-	baseJS := base.With(integration.ProgramTestOptions{
+	baseJS := integration.ProgramTestOptions{
 		Config: map[string]string{
 			"aws:region": "INVALID_REGION",
 		},
@@ -47,7 +46,7 @@ func TestAccCredentialsConfigTest(t *testing.T) {
 		},
 		Dir:           filepath.Join(getCwd(t), "credentialsConfigTest"),
 		ExpectFailure: true,
-	})
+	}
 
 	integration.ProgramTest(t, &baseJS)
 }
@@ -67,7 +66,7 @@ func TestAccExpress(t *testing.T) {
 			Dir:           filepath.Join(getCwd(t), "express"),
 			RunUpdateTest: true,
 		})
-
+	skipRefresh(&test)
 	integration.ProgramTest(t, &test)
 }
 
@@ -107,6 +106,7 @@ func TestAccBucket(t *testing.T) {
 			},
 		})
 
+	skipRefresh(&test)
 	integration.ProgramTest(t, &test)
 }
 
@@ -116,15 +116,19 @@ func TestAccCloudWatch(t *testing.T) {
 			Dir:           filepath.Join(getCwd(t), "cloudwatch"),
 			RunUpdateTest: true,
 		})
-
+	skipRefresh(&test)
 	integration.ProgramTest(t, &test)
 }
 
 func TestAccCloudWatchOidcManual(t *testing.T) {
 	test := getJSBaseOptions(t).
 		With(integration.ProgramTestOptions{
-			Dir:           filepath.Join(getCwd(t), "cloudwatchOidcManual"),
-			RunUpdateTest: true,
+			Dir: filepath.Join(getCwd(t), "cloudwatchOidcManual"),
+
+			// TODO[pulumi/pulumi-aws#3193] multiple issues with refreshing and updating cleanly.
+			SkipRefresh:              true,
+			AllowEmptyPreviewChanges: true,
+			AllowEmptyUpdateChanges:  true,
 		})
 
 	integration.ProgramTest(t, &test)
@@ -136,7 +140,7 @@ func TestAccLogGroup(t *testing.T) {
 			Dir:           filepath.Join(getCwd(t), "logGroup"),
 			RunUpdateTest: false,
 		})
-
+	skipRefresh(&test)
 	integration.ProgramTest(t, &test)
 }
 
@@ -146,7 +150,7 @@ func TestAccQueue(t *testing.T) {
 			Dir:           filepath.Join(getCwd(t), "queue"),
 			RunUpdateTest: true,
 		})
-
+	skipRefresh(&test)
 	integration.ProgramTest(t, &test)
 }
 
@@ -155,7 +159,7 @@ func TestAccEventBus(t *testing.T) {
 		With(integration.ProgramTestOptions{
 			Dir: filepath.Join(getCwd(t), "eventbus"),
 		})
-
+	skipRefresh(&test)
 	integration.ProgramTest(t, &test)
 }
 
@@ -165,7 +169,7 @@ func TestAccStream(t *testing.T) {
 			Dir:           filepath.Join(getCwd(t), "stream"),
 			RunUpdateTest: true,
 		})
-
+	skipRefresh(&test)
 	integration.ProgramTest(t, &test)
 }
 
@@ -175,7 +179,7 @@ func TestAccTable(t *testing.T) {
 			Dir:           filepath.Join(getCwd(t), "table"),
 			RunUpdateTest: true,
 		})
-
+	skipRefresh(&test)
 	integration.ProgramTest(t, &test)
 }
 
@@ -185,7 +189,7 @@ func TestAccTopic(t *testing.T) {
 			Dir:           filepath.Join(getCwd(t), "topic"),
 			RunUpdateTest: true,
 		})
-
+	skipRefresh(&test)
 	integration.ProgramTest(t, &test)
 }
 
@@ -200,7 +204,7 @@ func TestAccSecretCapture(t *testing.T) {
 				assert.NotContains(t, "s3cr3t", string(byts))
 			},
 		})
-
+	skipRefresh(&test)
 	integration.ProgramTest(t, &test)
 }
 
@@ -236,6 +240,7 @@ func TestAccCallbackFunction(t *testing.T) {
 				}
 			},
 		})
+	skipRefresh(&test)
 	integration.ProgramTest(t, &test)
 }
 
@@ -266,7 +271,7 @@ func TestAccRoute53(t *testing.T) {
 			Dir:           filepath.Join(getCwd(t), "route53"),
 			RunUpdateTest: true,
 		})
-
+	skipRefresh(&test)
 	integration.ProgramTest(t, &test)
 }
 
@@ -276,7 +281,7 @@ func TestAccLambdaLayer(t *testing.T) {
 			Dir:           filepath.Join(getCwd(t), "lambda-layer-old"),
 			RunUpdateTest: true,
 		})
-
+	skipRefresh(&test)
 	integration.ProgramTest(t, &test)
 }
 
@@ -286,7 +291,7 @@ func TestAccLambdaContainerImages(t *testing.T) {
 			RunUpdateTest: false, // new feature!
 			Dir:           filepath.Join(getCwd(t), "lambda-container-image"),
 		})
-
+	skipRefresh(&test)
 	integration.ProgramTest(t, &test)
 }
 
@@ -296,7 +301,7 @@ func TestAccLambdaLayerNewEnums(t *testing.T) {
 			Dir:           filepath.Join(getCwd(t), "lambda-layer-new"),
 			RunUpdateTest: false,
 		})
-
+	skipRefresh(&test)
 	integration.ProgramTest(t, &test)
 }
 
@@ -327,7 +332,7 @@ func TestAccDeleteBeforeCreate(t *testing.T) {
 				},
 			},
 		})
-
+	skipRefresh(&test)
 	integration.ProgramTest(t, &test)
 }
 
@@ -457,6 +462,12 @@ func TestAccWafV2(t *testing.T) {
 		With(integration.ProgramTestOptions{
 			Dir: filepath.Join(getCwd(t), "wafv2"),
 		})
+	skipRefresh(&test)
+
+	// TODO[pulumi/pulumi-aws#3190] there is a bug with non-empty diff after pulumi up.
+	test.AllowEmptyPreviewChanges = true
+	test.AllowEmptyUpdateChanges = true
+
 	integration.ProgramTest(t, &test)
 }
 
@@ -491,16 +502,15 @@ func TestRegress2818(t *testing.T) {
 
 func getJSBaseOptions(t *testing.T) integration.ProgramTestOptions {
 	envRegion := getEnvRegion(t)
-	base := getBaseOptions()
-	baseJS := base.With(integration.ProgramTestOptions{
+	baseJS := integration.ProgramTestOptions{
 		Config: map[string]string{
 			"aws:region":    "INVALID_REGION",
 			"aws:envRegion": envRegion,
 		},
 		Dependencies: []string{
 			"@pulumi/aws",
 		},
-	})
+	}
 
 	return baseJS
 }

examples/examples_py_test.go

@@ -41,7 +41,7 @@ func TestAccCodeBuildProjectPy(t *testing.T) {
 			Dir:           filepath.Join(getCwd(t), "codebuild-project-py"),
 			RunUpdateTest: false,
 		})
-
+	skipRefresh(&test)
 	integration.ProgramTest(t, &test)
 }
 
@@ -69,15 +69,14 @@ func TestSecretManagerPy(t *testing.T) {
 
 func getPythonBaseOptions(t *testing.T) integration.ProgramTestOptions {
 	envRegion := getEnvRegion(t)
-	base := getBaseOptions()
-	pythonBase := base.With(integration.ProgramTestOptions{
+	pythonBase := integration.ProgramTestOptions{
 		Config: map[string]string{
 			"aws:region": envRegion,
 		},
 		Dependencies: []string{
 			filepath.Join("..", "sdk", "python", "bin"),
 		},
-	})
+	}
 
 	return pythonBase
 }

examples/examples_test.go

@@ -50,14 +50,6 @@ func getCwd(t *testing.T) string {
 	return cwd
 }
 
-func getBaseOptions() integration.ProgramTestOptions {
-	return integration.ProgramTestOptions{
-		ExpectRefreshChanges: true,
-		SkipRefresh:          true,
-		Quick:                true,
-	}
-}
-
 func validateAPITest(isValid func(body string)) func(t *testing.T, stack integration.RuntimeValidationStackInfo) {
 	return func(t *testing.T, stack integration.RuntimeValidationStackInfo) {
 		var resp *http.Response
@@ -593,3 +585,13 @@ func TestWrongStateMaxItemOneDiffProduced(t *testing.T) {
 	`
 	replay(t, repro)
 }
+
+// A lot of tests do not currently refresh cleanly. The work to root cause each tests has not been
+// done yet but the common causes are listed here:
+//
+// TODO[pulumi/pulumi-aws#2246] specifically affects overlays such as bucket.onObjectCreated; may be worked around
+// TODO[pulumi/pulumi#6235]
+// TODO[pulumi/pulumi-terraform-bridge#1595]
+func skipRefresh(opts *integration.ProgramTestOptions) {
+	opts.SkipRefresh = true
+}

examples/main_test.go

@@ -18,9 +18,9 @@ func TestMain(m *testing.M) {
 		}
 		command := exec.Command("make", "provider_no_deps")
 		command.Dir = filepath.Join(cwd, "..")
-		err = command.Run()
-		if err != nil {
-			log.Println("Unable to build provider!")
+		if combinedOutput, err := command.CombinedOutput(); err != nil {
+			log.Printf("Unable to build provider. Command `make provider_no_deps` failed with: %v\nOutput:\n%s",
+				err, combinedOutput)
 			os.Exit(1)
 		}