Test error matching

replay/replay.go

@@ -183,13 +183,10 @@ func replay[Req protoreflect.ProtoMessage, Resp protoreflect.ProtoMessage](
 	assert.NoError(t, err)
 
 	resp, err := serve(ctx, req)
-	if err != nil && entry.Errors != nil {
-		assert.Equal(t, *entry.Errors, err.Error())
+	if done := assertErrorMatchesSpec(t, entry.Errors, err); done {
 		return
 	}
 
-	require.NoError(t, err)
-
 	if normalizeResponse != nil {
 		normalizeResponse(resp)
 	}
@@ -201,6 +198,36 @@ func replay[Req protoreflect.ProtoMessage, Resp protoreflect.ProtoMessage](
 	AssertJSONMatchesPattern(t, expected, actual)
 }
 
+func assertErrorMatchesSpec(t *testing.T, expectedErrors []string, err error) bool {
+	switch {
+	case len(expectedErrors) == 0:
+		require.NoError(t, err)
+		return false
+	case len(expectedErrors) == 1:
+		require.Error(t, err)
+		e := expectedErrors[0]
+		if e != "*" {
+			require.Equal(t, e, err.Error())
+		}
+		return true
+	default:
+		// The cases where there are actual multiple errors returned from gRPC intereceptor
+		// seem a little unclear, the code is in interceptors.go:
+		//
+		// https://github.com/pulumi/pulumi/blob/master/pkg/util/rpcdebug/interceptors.go#L34
+		//
+		// It seems we have at most one logical error coming from the provider, but we also
+		// may record errors coming from interceptor-related issues in the error list.
+		//
+		// If this reasoning is correct, when reusing GRPC logs as expectations, a
+		// reasonable default assert is to check that the actual error message is contained
+		// in the list of expectations.
+		require.Error(t, err)
+		require.Contains(t, expectedErrors, err.Error())
+		return true
+	}
+}
+
 // ReplayFile executes ReplaySequence on all pulumirpc.ResourceProvider events found in the file produced with
 // PULUMI_DEBUG_GPRC. For example:
 //
@@ -242,11 +269,12 @@ func ReplayFile(t *testing.T, server pulumirpc.ResourceProviderServer, traceFile
 	assert.Greater(t, count, 0)
 }
 
+// See also: https://github.com/pulumi/pulumi/blob/master/pkg/util/rpcdebug/logformat.go#L28
 type jsonLogEntry struct {
 	Method   string          `json:"method"`
 	Request  json.RawMessage `json:"request,omitempty"`
 	Response json.RawMessage `json:"response,omitempty"`
-	Errors   *string         `json:"errors,omitempty"`
+	Errors   []string        `json:"errors,omitempty"`
 }
 
 func normInvokeResponse(resp *pulumirpc.InvokeResponse) {

replay/replay_test.go

@@ -2,6 +2,7 @@ package replay
 
 import (
 	"context"
+	"fmt"
 	"testing"
 
 	"github.com/stretchr/testify/require"
@@ -121,3 +122,42 @@ func TestReplayNormalizesCheckFailureOrder(t *testing.T) {
 	  }
 	}`)
 }
+
+func TestMatchingErrors(t *testing.T) {
+	p, err := providers.NewProviderMock(context.Background(), providers.ProviderMocks{
+		Check: func(ctx context.Context, in *pulumirpc.CheckRequest) (*pulumirpc.CheckResponse, error) {
+			return &pulumirpc.CheckResponse{}, fmt.Errorf("An error has occurred")
+		},
+	})
+	require.NoError(t, err)
+
+	Replay(t, p, `
+	{
+	  "method": "/pulumirpc.ResourceProvider/Check",
+	  "request": {
+	    "urn": "u",
+	    "news": {}
+	  },
+          "errors": ["An error has occurred"]
+	}`)
+
+	Replay(t, p, `
+	{
+	  "method": "/pulumirpc.ResourceProvider/Check",
+	  "request": {
+	    "urn": "u",
+	    "news": {}
+	  },
+          "errors": ["An error has occurred", "Unrelated error"]
+	}`)
+
+	Replay(t, p, `
+	{
+	  "method": "/pulumirpc.ResourceProvider/Check",
+	  "request": {
+	    "urn": "u",
+	    "news": {}
+	  },
+          "errors": ["*"]
+	}`)
+}