walk kustomize files

this allows us to more reliably trigger diffs for all apps

cmd/controller_cmd.go

@@ -7,13 +7,13 @@ import (
 	"syscall"
 	"time"
 
-	"github.com/zapier/kubechecks/pkg/events"
-
 	_ "github.com/argoproj/argo-cd/v2/pkg/apis/application/v1alpha1"
 	"github.com/rs/zerolog/log"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 	"github.com/zapier/kubechecks/pkg"
+	"github.com/zapier/kubechecks/pkg/config"
+	"github.com/zapier/kubechecks/pkg/events"
 	"github.com/zapier/kubechecks/pkg/server"
 )
 
@@ -25,7 +25,7 @@ var controllerCmd = &cobra.Command{
 	Run: func(cmd *cobra.Command, args []string) {
 		fmt.Println("Starting KubeChecks:", pkg.GitTag, pkg.GitCommit)
 
-		server := server.NewServer(&pkg.ServerConfig{
+		server := server.NewServer(&config.ServerConfig{
 			UrlPrefix:     viper.GetString("webhook-url-prefix"),
 			WebhookSecret: viper.GetString("webhook-secret"),
 		})

pkg/affected_apps/argocd_matcher.go

@@ -2,24 +2,38 @@ package affected_apps
 
 import (
 	"context"
+	"os"
 
 	"github.com/rs/zerolog/log"
-	"github.com/zapier/kubechecks/pkg"
-	"github.com/zapier/kubechecks/pkg/app_directory"
+	"github.com/zapier/kubechecks/pkg/config"
 	"github.com/zapier/kubechecks/pkg/repo"
 )
 
 type ArgocdMatcher struct {
-	appsDirectory *app_directory.AppDirectory
+	appsDirectory *config.AppDirectory
 }
 
-func NewArgocdMatcher(vcsToArgoMap pkg.VcsToArgoMap, repo *repo.Repo) *ArgocdMatcher {
+func NewArgocdMatcher(vcsToArgoMap config.VcsToArgoMap, repo *repo.Repo, repoPath string) (*ArgocdMatcher, error) {
 	log.Debug().Msgf("looking for %s repos", repo.CloneURL)
 	repoApps := vcsToArgoMap.GetAppsInRepo(repo.CloneURL)
 	log.Debug().Msgf("found %d apps", repoApps.Count())
-	return &ArgocdMatcher{
-		appsDirectory: repoApps,
+
+	log.Debug().Msgf("creating fs for %s", repoPath)
+	fs := os.DirFS(repoPath)
+	log.Debug().Msg("following kustomize apps")
+	kustomizeAppFiles, err := vcsToArgoMap.WalkKustomizeApps(repo, fs)
+	if err != nil {
+		log.Warn().Err(err).Msgf("failed to follow kustomize files")
 	}
+	log.Debug().Msgf("found %d apps", kustomizeAppFiles.Count())
+
+	appDirectory := new(config.AppDirectory).
+		Union(repoApps).
+		Union(kustomizeAppFiles)
+
+	return &ArgocdMatcher{
+		appsDirectory: appDirectory,
+	}, nil
 }
 
 func (a *ArgocdMatcher) AffectedApps(ctx context.Context, changeList []string) (AffectedItems, error) {

pkg/affected_apps/best_effort.go

@@ -7,7 +7,7 @@ import (
 	"strings"
 
 	"github.com/rs/zerolog/log"
-	"github.com/zapier/kubechecks/pkg/app_directory"
+	"github.com/zapier/kubechecks/pkg/config"
 )
 
 // TODO: move this out to config and or in an .kubechecks.yaml as well
@@ -87,9 +87,9 @@ func (b *BestEffort) AffectedApps(ctx context.Context, changeList []string) (Aff
 		}
 	}
 
-	var appsSlice []app_directory.ApplicationStub
+	var appsSlice []config.ApplicationStub
 	for name, path := range appsMap {
-		appsSlice = append(appsSlice, app_directory.ApplicationStub{Name: name, Path: path})
+		appsSlice = append(appsSlice, config.ApplicationStub{Name: name, Path: path})
 	}
 
 	return AffectedItems{Applications: appsSlice}, nil

pkg/affected_apps/best_effort_test.go

@@ -6,7 +6,7 @@ import (
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
-	"github.com/zapier/kubechecks/pkg/app_directory"
+	"github.com/zapier/kubechecks/pkg/config"
 )
 
 func TestBestEffortMatcher(t *testing.T) {
@@ -28,7 +28,7 @@ func TestBestEffortMatcher(t *testing.T) {
 				repoName: "",
 			},
 			want: AffectedItems{
-				Applications: []app_directory.ApplicationStub{
+				Applications: []config.ApplicationStub{
 					{Name: "foo-eks-01-echo-server", Path: "apps/echo-server/foo-eks-01/"},
 				},
 			},
@@ -42,7 +42,7 @@ func TestBestEffortMatcher(t *testing.T) {
 				repoName: "",
 			},
 			want: AffectedItems{
-				Applications: []app_directory.ApplicationStub{
+				Applications: []config.ApplicationStub{
 					{Name: "foo-eks-01-echo-server", Path: "apps/echo-server/foo-eks-01/"},
 					{Name: "foo-eks-02-echo-server", Path: "apps/echo-server/foo-eks-02/"},
 				},
@@ -58,7 +58,7 @@ func TestBestEffortMatcher(t *testing.T) {
 				repoName: "",
 			},
 			want: AffectedItems{
-				Applications: []app_directory.ApplicationStub{
+				Applications: []config.ApplicationStub{
 					{Name: "foo-eks-01-echo-server", Path: "apps/echo-server/foo-eks-01/"},
 					{Name: "foo-eks-02-echo-server", Path: "apps/echo-server/foo-eks-02/"},
 				},
@@ -75,7 +75,7 @@ func TestBestEffortMatcher(t *testing.T) {
 				repoName: "",
 			},
 			want: AffectedItems{
-				Applications: []app_directory.ApplicationStub{
+				Applications: []config.ApplicationStub{
 					{Name: "foo-eks-01-echo-server", Path: "apps/echo-server/foo-eks-01/"},
 					{Name: "foo-eks-02-echo-server", Path: "apps/echo-server/foo-eks-02/"},
 				},
@@ -90,7 +90,7 @@ func TestBestEffortMatcher(t *testing.T) {
 				repoName: "",
 			},
 			want: AffectedItems{
-				Applications: []app_directory.ApplicationStub{
+				Applications: []config.ApplicationStub{
 					{Name: "foo-eks-01-httpbin", Path: "apps/httpbin/overlays/foo-eks-01/"},
 				},
 			},
@@ -104,7 +104,7 @@ func TestBestEffortMatcher(t *testing.T) {
 				repoName: "",
 			},
 			want: AffectedItems{
-				Applications: []app_directory.ApplicationStub{
+				Applications: []config.ApplicationStub{
 					{Name: "foo-eks-01-httpbin", Path: "apps/httpbin/overlays/foo-eks-01/"},
 				},
 			},
@@ -118,7 +118,7 @@ func TestBestEffortMatcher(t *testing.T) {
 				repoName: "",
 			},
 			want: AffectedItems{
-				Applications: []app_directory.ApplicationStub{
+				Applications: []config.ApplicationStub{
 					{Name: "foo-eks-01-httpbin", Path: "apps/httpbin/overlays/foo-eks-01/"},
 				},
 			},
@@ -132,7 +132,7 @@ func TestBestEffortMatcher(t *testing.T) {
 				repoName: "",
 			},
 			want: AffectedItems{
-				Applications: []app_directory.ApplicationStub{
+				Applications: []config.ApplicationStub{
 					{Name: "foo-eks-01-httpbin", Path: "apps/httpbin/overlays/foo-eks-01/"},
 				},
 			},
@@ -146,7 +146,7 @@ func TestBestEffortMatcher(t *testing.T) {
 				repoName: "",
 			},
 			want: AffectedItems{
-				Applications: []app_directory.ApplicationStub{
+				Applications: []config.ApplicationStub{
 					{Name: "foo-eks-01-httpbin", Path: "apps/httpbin/overlays/foo-eks-01/"},
 				},
 			},
@@ -180,7 +180,7 @@ func appSetKey(item ApplicationSet) string {
 	return item.Name
 }
 
-func appStubKey(stub app_directory.ApplicationStub) string {
+func appStubKey(stub config.ApplicationStub) string {
 	return stub.Name
 }
 

pkg/affected_apps/config_matcher.go

@@ -7,9 +7,8 @@ import (
 	"strings"
 
 	"github.com/rs/zerolog/log"
-	"github.com/zapier/kubechecks/pkg/app_directory"
-
 	"github.com/zapier/kubechecks/pkg/argo_client"
+	"github.com/zapier/kubechecks/pkg/config"
 	"github.com/zapier/kubechecks/pkg/repo_config"
 )
 
@@ -40,9 +39,9 @@ func (b *ConfigMatcher) AffectedApps(ctx context.Context, changeList []string) (
 		appSetList = append(appSetList, ApplicationSet{appset.Name})
 	}
 
-	var appsSlice []app_directory.ApplicationStub
+	var appsSlice []config.ApplicationStub
 	for name, appPath := range appsMap {
-		appsSlice = append(appsSlice, app_directory.ApplicationStub{Name: name, Path: appPath})
+		appsSlice = append(appsSlice, config.ApplicationStub{Name: name, Path: appPath})
 	}
 
 	return AffectedItems{Applications: appsSlice, ApplicationSets: appSetList}, nil

pkg/affected_apps/matcher.go

@@ -4,11 +4,11 @@ import (
 	"context"
 	"path"
 
-	"github.com/zapier/kubechecks/pkg/app_directory"
+	"github.com/zapier/kubechecks/pkg/config"
 )
 
 type AffectedItems struct {
-	Applications    []app_directory.ApplicationStub
+	Applications    []config.ApplicationStub
 	ApplicationSets []ApplicationSet
 }
 

pkg/app_directory/app_directory.go → pkg/config/app_directory.go

@@ -1,4 +1,4 @@
-package app_directory
+package config
 
 import (
 	"path/filepath"
@@ -10,18 +10,20 @@ import (
 
 type ApplicationStub struct {
 	Name, Path string
+
+	IsHelm, IsKustomize bool
 }
 
 type AppDirectory struct {
-	appPaths map[string][]string // directory -> array of app names
+	appDirs  map[string][]string // directory -> array of app names
 	appFiles map[string][]string // file path -> array of app names
 
 	appsMap map[string]ApplicationStub // app name -> app stub
 }
 
 func NewAppDirectory() *AppDirectory {
 	return &AppDirectory{
-		appPaths: make(map[string][]string),
+		appDirs:  make(map[string][]string),
 		appFiles: make(map[string][]string),
 		appsMap:  make(map[string]ApplicationStub),
 	}
@@ -31,7 +33,15 @@ func (d *AppDirectory) Count() int {
 	return len(d.appsMap)
 }
 
-func (d *AppDirectory) AddApp(app v1alpha1.Application) {
+func (d *AppDirectory) Union(other *AppDirectory) *AppDirectory {
+	var join AppDirectory
+	join.appsMap = mergeMaps(d.appsMap, other.appsMap, takeFirst[ApplicationStub])
+	join.appDirs = mergeMaps(d.appDirs, other.appDirs, mergeLists[string])
+	join.appFiles = mergeMaps(d.appFiles, other.appFiles, mergeLists[string])
+	return &join
+}
+
+func (d *AppDirectory) ProcessApp(app v1alpha1.Application) {
 	appName := app.Name
 
 	src := app.Spec.Source
@@ -41,19 +51,18 @@ func (d *AppDirectory) AddApp(app v1alpha1.Application) {
 
 	// common data
 	srcPath := src.Path
-	d.appsMap[appName] = ApplicationStub{Name: appName, Path: srcPath}
-	d.appPaths[srcPath] = append(d.appPaths[srcPath], appName)
+	d.AddAppStub(appName, srcPath, src.IsHelm(), !src.Kustomize.IsZero())
 
 	// handle extra helm paths
 	if helm := src.Helm; helm != nil {
 		for _, param := range helm.FileParameters {
 			path := filepath.Join(srcPath, param.Path)
-			d.appFiles[path] = append(d.appFiles[path], appName)
+			d.AddFile(appName, path)
 		}
 
 		for _, valueFilePath := range helm.ValueFiles {
 			path := filepath.Join(srcPath, valueFilePath)
-			d.appFiles[path] = append(d.appFiles[path], appName)
+			d.AddFile(appName, path)
 		}
 	}
 }
@@ -66,7 +75,7 @@ func (d *AppDirectory) FindAppsBasedOnChangeList(changeList []string) []Applicat
 	for _, changePath := range changeList {
 		log.Debug().Msgf("change: %s", changePath)
 
-		for dir, appNames := range d.appPaths {
+		for dir, appNames := range d.appDirs {
 			log.Debug().Msgf("- app path: %s", dir)
 			if strings.HasPrefix(changePath, dir) {
 				log.Debug().Msg("dir match!")
@@ -98,3 +107,56 @@ func (d *AppDirectory) FindAppsBasedOnChangeList(changeList []string) []Applicat
 	log.Debug().Msgf("matched %d files into %d apps", len(appsMap), len(appsSet))
 	return appsSlice
 }
+
+func (d *AppDirectory) GetApps(filter func(stub ApplicationStub) bool) []ApplicationStub {
+	var result []ApplicationStub
+	for _, value := range d.appsMap {
+		if filter != nil && !filter(value) {
+			continue
+		}
+		result = append(result, value)
+	}
+	return result
+}
+
+func (d *AppDirectory) AddAppStub(appName, srcPath string, isHelm, isKustomize bool) {
+	d.appsMap[appName] = ApplicationStub{
+		Name:        appName,
+		Path:        srcPath,
+		IsHelm:      isHelm,
+		IsKustomize: isKustomize,
+	}
+	d.AddDir(appName, srcPath)
+}
+
+func (d *AppDirectory) AddDir(appName, path string) {
+	d.appDirs[path] = append(d.appDirs[path], appName)
+}
+
+func (d *AppDirectory) AddFile(appName, path string) {
+	d.appFiles[path] = append(d.appFiles[path], appName)
+}
+
+func mergeMaps[T any](first map[string]T, second map[string]T, combine func(T, T) T) map[string]T {
+	result := make(map[string]T)
+	for key, value := range first {
+		result[key] = value
+	}
+	for key, value := range second {
+		exist, ok := result[key]
+		if ok {
+			result[key] = combine(exist, value)
+		} else {
+			result[key] = value
+		}
+	}
+	return result
+}
+
+func mergeLists[T any](a []T, b []T) []T {
+	return append(a, b...)
+}
+
+func takeFirst[T any](a, _ T) T {
+	return a
+}

pkg/app_directory/app_directory_test.go → pkg/config/app_directory_test.go

@@ -1,4 +1,4 @@
-package app_directory
+package config
 
 import (
 	"testing"
@@ -29,7 +29,7 @@ func TestPathsAreJoinedProperly(t *testing.T) {
 		},
 	}
 
-	rad.AddApp(app1)
+	rad.ProcessApp(app1)
 
 	assert.Equal(t, map[string]ApplicationStub{
 		"test-app": {
@@ -39,7 +39,7 @@ func TestPathsAreJoinedProperly(t *testing.T) {
 	}, rad.appsMap)
 	assert.Equal(t, map[string][]string{
 		"/test1/test2": {"test-app"},
-	}, rad.appPaths)
+	}, rad.appDirs)
 	assert.Equal(t, map[string][]string{
 		"/test1/test2/one.json": {"test-app"},
 		"/test1/test2/two.json": {"test-app"},