From 0b72b50f41c8a14fe4ae2b41671e352ed99dff46 Mon Sep 17 00:00:00 2001
From: Joseph Schorr <josephschorr@users.noreply.github.com>
Date: Tue, 3 Jan 2023 12:14:52 -0500
Subject: [PATCH] Parallelize the datastore consistency tests

---
 .github/workflows/build-test.yaml             | 18 ++++++++++
 .../consistency_datastore_test.go             | 35 ++++++++-----------
 2 files changed, 33 insertions(+), 20 deletions(-)

diff --git a/.github/workflows/build-test.yaml b/.github/workflows/build-test.yaml
index b44fa1de4f..68af1479b4 100644
--- a/.github/workflows/build-test.yaml
+++ b/.github/workflows/build-test.yaml
@@ -114,6 +114,24 @@ jobs:
           tags: "ci,docker"
           timeout: "10m"
           working_directory: "internal/datastore/${{ matrix.datastore }}"
+
+  datastoreconsistency:
+    name: "Datastore Consistency Tests"
+    runs-on: "ubuntu-latest"
+    strategy:
+      fail-fast: false
+      matrix:
+        datastore: ["cockroachdb", "mysql", "postgres", "spanner"]
+    steps:
+      - uses: "actions/checkout@v3"
+      - uses: "actions/setup-go@v3"
+        with:
+          go-version: "${{ env.GO_VERSION }}"
+          cache: "true"
+      - name: "Run Datastore Consistency Tests"
+        working-directory: "internal/services/integrationtesting"
+        run: "go test --failfast -count=1 -timeout '10m' --tags='ci,docker,datastoreconsistency' ./... -run TestConsistencyPerDatastore/${{ matrix.datastore }}"
+
   e2e:
     name: "E2E"
     runs-on: "ubuntu-latest"
diff --git a/internal/services/integrationtesting/consistency_datastore_test.go b/internal/services/integrationtesting/consistency_datastore_test.go
index 4362c1bb55..5d2a045e82 100644
--- a/internal/services/integrationtesting/consistency_datastore_test.go
+++ b/internal/services/integrationtesting/consistency_datastore_test.go
@@ -1,15 +1,12 @@
-//go:build ci && docker
-// +build ci,docker
+//go:build ci && docker && datastoreconsistency
+// +build ci,docker,datastoreconsistency
 
 package integrationtesting_test
 
 import (
 	"context"
-	"os"
 	"path"
-	"path/filepath"
 	"runtime"
-	"strings"
 	"testing"
 	"time"
 
@@ -27,26 +24,22 @@ import (
 )
 
 func TestConsistencyPerDatastore(t *testing.T) {
+	// TODO(jschorr): Re-enable for *all* files once we make this faster.
 	_, filename, _, _ := runtime.Caller(0)
-	consistencyTestFiles := []string{}
-	err := filepath.Walk(path.Join(path.Dir(filename), "testconfigs"), func(path string, info os.FileInfo, err error) error {
-		if info == nil || info.IsDir() {
-			return nil
-		}
-
-		if strings.HasSuffix(info.Name(), ".yaml") {
-			consistencyTestFiles = append(consistencyTestFiles, path)
-		}
-
-		return nil
-	})
-
-	rrequire := require.New(t)
-	rrequire.NoError(err)
+	consistencyTestFiles := []string{
+		path.Join(path.Join(path.Dir(filename), "testconfigs"), "document.yaml"),
+		path.Join(path.Join(path.Dir(filename), "testconfigs"), "basicrbac.yaml"),
+		path.Join(path.Join(path.Dir(filename), "testconfigs"), "public.yaml"),
+		path.Join(path.Join(path.Dir(filename), "testconfigs"), "nil.yaml"),
+	}
 
 	for _, engineID := range datastore.Engines {
+		engineID := engineID
+
 		t.Run(engineID, func(t *testing.T) {
 			for _, filePath := range consistencyTestFiles {
+				filePath := filePath
+
 				t.Run(path.Base(filePath), func(t *testing.T) {
 					lrequire := require.New(t)
 
@@ -73,6 +66,8 @@ func TestConsistencyPerDatastore(t *testing.T) {
 
 					// Run the assertions within each file.
 					for _, tester := range testers {
+						tester := tester
+
 						t.Run(tester.Name(), func(t *testing.T) {
 							runAssertions(t, tester, dispatcher, fullyResolved, revision)
 						})