support multiple locations for schemas and policies

also check and enforce linting

.editorconfig

@@ -0,0 +1,10 @@
+[*.go]
+ij_any_blank_lines_after_imports = 1
+ij_go_add_parentheses_for_single_import = true
+ij_go_GROUP_CURRENT_PROJECT_IMPORTS = true
+ij_go_group_stdlib_imports = true
+ij_go_import_sorting = gofmt
+ij_go_move_all_imports_in_one_declaration = true
+ij_go_move_all_stdlib_imports_in_one_group = true
+ij_go_remove_redundant_import_aliases = true
+ij_go_use_back_quotes_for_imports = false

.github/workflows/on_pull-request_docs.yaml

@@ -27,16 +27,4 @@ jobs:
               --GOLANG_VERSION="${{ env.GOLANG_TOOL_VERSION }}"
 
       - name: verify that the checked in file has not changed
-        run: |
-          #!/usr/bin/env bash
-          
-          exitCode=0
-
-          # Log the actual diff for debugging purposes
-          git diff --name-only | cat
-          if ! git diff --exit-code --quiet; then
-            echo "Please run 'earthly +rebuild-docs' and commit the results to this PR"
-            exitCode=1
-          fi
-          
-          exit $exitCode
+        run: ./hacks/exit-on-changed-files.sh "Please run 'earthly +rebuild-docs' and commit the results to this PR"

Earthfile

@@ -5,6 +5,7 @@ test:
     BUILD +ci-helm
 
 ci-golang:
+    BUILD +fmt-golang
     BUILD +lint-golang
     BUILD +validate-golang
     BUILD +test-golang
@@ -105,10 +106,7 @@ docker:
 
     WORKDIR /app
 
-    COPY ./policy ./policy
-    VOLUME /app/policy
-
-    COPY ./schemas ./schemas
+    VOLUME /app/policies
     VOLUME /app/schemas
 
     COPY (+build-binary/kubechecks --GOARCH=amd64 --VARIANT=$TARGETVARIANT) .
@@ -148,6 +146,15 @@ docker-debug:
 
     SAVE IMAGE --push $CI_REGISTRY_IMAGE
 
+fmt-golang:
+    FROM +go-deps
+
+    WORKDIR /src
+    COPY . /src
+
+    RUN go fmt \
+        && ./hacks/exit-on-changed-files.sh
+
 lint-golang:
     ARG STATICCHECK_VERSION="2023.1.3"
 

cmd/flags.go

@@ -40,10 +40,18 @@ func newStringOpts() DocOpt[string] {
 	return DocOpt[string]{}
 }
 
+func newStringSliceOpts() DocOpt[[]string] {
+	return DocOpt[[]string]{}
+}
+
 func stringFlag(flags *pflag.FlagSet, name, usage string, opts ...DocOpt[string]) {
 	addFlag(name, usage, opts, flags.String, flags.StringP)
 }
 
+func stringSliceFlag(flags *pflag.FlagSet, name, usage string, opts ...DocOpt[[]string]) {
+	addFlag(name, usage, opts, flags.StringArray, flags.StringArrayP)
+}
+
 func addFlag[D any](
 		name, usage string,
 		opts []DocOpt[D],

cmd/root.go

@@ -10,6 +10,7 @@ import (
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
+
 	"github.com/zapier/kubechecks/telemetry"
 )
 
@@ -79,14 +80,17 @@ func init() {
 			withChoices("hide", "delete").
 			withDefault("hide"),
 	)
-	stringFlag(flags, "schemas-location", "Sets the schema location. Can be local path or git repository.",
-		newStringOpts().
-			withDefault("./schemas"))
+	stringSliceFlag(flags, "schemas-location", "Sets schema locations to be used for every check request. Can be common paths inside the repos being checked or git urls in either git or http(s) format.",
+		newStringSliceOpts().
+			withDefault([]string{"./schemas"}))
+
+	stringSliceFlag(flags, "policies-location", "Sets rego policy locations to be used for every check request. Can be common path inside the repos being checked or git urls in either git or http(s) format.",
+		newStringSliceOpts().
+			withDefault([]string{"./policies"}))
 
 	panicIfError(viper.BindPFlags(flags))
 
 	setupLogOutput()
-
 }
 
 func initTelemetry(ctx context.Context) (*telemetry.OperatorTelemetry, error) {

docs/usage.md

@@ -50,7 +50,8 @@ The full list of supported environment variables is described below:
 |`KUBECHECKS_OTEL_COLLECTOR_PORT`|The OpenTelemetry collector port.||
 |`KUBECHECKS_OTEL_ENABLED`|Enable OpenTelemetry.|`false`|
 |`KUBECHECKS_PERSIST_LOG_LEVEL`|Persists the set log level down to other module loggers.|`false`|
-|`KUBECHECKS_SCHEMAS_LOCATION`|Sets the schema location. Can be local path or git repository.|`./schemas`|
+|`KUBECHECKS_POLICIES_LOCATION`|Sets rego policy locations to be used for every check request. Can be common path inside the repos being checked or git urls in either git or http(s) format.|`[./policies]`|
+|`KUBECHECKS_SCHEMAS_LOCATION`|Sets schema locations to be used for every check request. Can be common paths inside the repos being checked or git urls in either git or http(s) format.|`[./schemas]`|
 |`KUBECHECKS_SHOW_DEBUG_INFO`|Set to true to print debug info to the footer of MR comments.|`false`|
 |`KUBECHECKS_TIDY_OUTDATED_COMMENTS_MODE`|Sets the mode to use when tidying outdated comments. One of hide, delete.|`hide`|
 |`KUBECHECKS_VCS_BASE_URL`|VCS base url, useful if self hosting gitlab, enterprise github, etc.||

hacks/exit-on-changed-files.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+set -e
+
+exitCode=0
+message=$1
+
+# Log the actual diff for debugging purposes
+git diff --name-only | cat
+if ! git diff --exit-code --quiet; then
+  echo "$message"
+  exitCode=1
+fi
+
+exit $exitCode

pkg/affected_apps/best_effort.go

@@ -115,9 +115,9 @@ func isKustomizeApp(file string) bool {
 
 func isKustomizeBaseComponentsChange(file string) bool {
 	return strings.Contains(file, "base/") ||
-			strings.Contains(file, "bases/") ||
-			strings.Contains(file, "components/") ||
-			strings.Contains(file, "resources/")
+		strings.Contains(file, "bases/") ||
+		strings.Contains(file, "components/") ||
+		strings.Contains(file, "resources/")
 }
 
 func overlaysDir(file string) string {

pkg/conftest/conftest.go

@@ -11,12 +11,14 @@ import (
 	"github.com/argoproj/argo-cd/v2/pkg/apis/application/v1alpha1"
 	"github.com/olekukonko/tablewriter"
 	"github.com/open-policy-agent/conftest/output"
+	"github.com/open-policy-agent/conftest/runner"
 	"github.com/rs/zerolog/log"
-	"github.com/zapier/kubechecks/pkg"
-	"github.com/zapier/kubechecks/telemetry"
+	"github.com/spf13/viper"
 	"go.opentelemetry.io/otel"
 
-	"github.com/open-policy-agent/conftest/runner"
+	"github.com/zapier/kubechecks/pkg"
+	"github.com/zapier/kubechecks/pkg/local"
+	"github.com/zapier/kubechecks/telemetry"
 )
 
 var gitLabCommentFormat = `
@@ -32,6 +34,8 @@ var gitLabCommentFormat = `
 
 const passedMessage = "\nPassed all policy checks."
 
+var reposCache = local.NewReposDirectory()
+
 // Conftest runs the conftest validation against an application in a given repository
 // path. It generates a summary string with the results, which can later be posted
 // as a GitLab comment. The validation checks resources against Zapier policies and
@@ -45,12 +49,26 @@ func Conftest(ctx context.Context, app *v1alpha1.Application, repoPath string) (
 
 	log.Debug().Str("dir", confTestDir).Str("app", app.Name).Msg("running conftest in dir for application")
 
-	var r = runner.TestRunner{}
+	policiesLocations := viper.GetStringSlice("policies-location")
+	var locations []string
+	for _, policiesLocation := range policiesLocations {
+		log.Debug().Str("schemas-location", policiesLocation).Msg("viper")
+		schemaPath := reposCache.EnsurePath(ctx, repoPath, policiesLocation)
+		if schemaPath != "" {
+			locations = append(locations, schemaPath)
+		}
+	}
+
+	if len(locations) == 0 {
+		return "no policies locations configured", nil
+	}
+
+	var r runner.TestRunner
 
 	r.NoColor = true
 	r.AllNamespaces = true
 	// PATH To Rego Polices
-	r.Policy = []string{"./policy"}
+	r.Policy = locations
 	r.SuppressExceptions = false
 	r.Trace = false
 

pkg/events/check.go

@@ -344,7 +344,7 @@ func (ce *CheckEvent) processApp(ctx context.Context, app, dir string) error {
 	grp, grpCtx := errgroup.WithContext(ctx)
 	wrap := ce.createWrapper(span, grpCtx, app)
 
-	grp.Go(wrap("validating app against schema", ce.validateSchemas(grpCtx, app, k8sVersion, formattedManifests)))
+	grp.Go(wrap("validating app against schema", ce.validateSchemas(grpCtx, app, k8sVersion, ce.TempWorkingDir, formattedManifests)))
 	grp.Go(wrap("generating diff for app", ce.generateDiff(grpCtx, app, manifests)))
 
 	if viper.GetBool("enable-conftest") {
@@ -431,9 +431,9 @@ func (ce *CheckEvent) generateDiff(ctx context.Context, app string, manifests []
 	}
 }
 
-func (ce *CheckEvent) validateSchemas(ctx context.Context, app string, k8sVersion string, formattedManifests []string) func() (string, error) {
+func (ce *CheckEvent) validateSchemas(ctx context.Context, app, k8sVersion, tempRepoPath string, formattedManifests []string) func() (string, error) {
 	return func() (string, error) {
-		s, err := validate.ArgoCdAppValidate(ctx, app, k8sVersion, formattedManifests)
+		s, err := validate.ArgoCdAppValidate(ctx, app, k8sVersion, tempRepoPath, formattedManifests)
 		if err != nil {
 			return "", err
 		}

pkg/gitlab_client/pipeline.go

@@ -1,7 +1,6 @@
 package gitlab_client
 
 import (
-
 	"github.com/rs/zerolog/log"
 	"github.com/xanzy/go-gitlab"
 )

pkg/local/gitRepos.go

@@ -0,0 +1,94 @@
+package local
+
+import (
+	"context"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"sync"
+
+	"github.com/rs/zerolog/log"
+
+	"github.com/zapier/kubechecks/pkg/repo"
+)
+
+type ReposDirectory struct {
+	paths map[string]string
+
+	mutex sync.Mutex
+}
+
+func NewReposDirectory() *ReposDirectory {
+	rd := &ReposDirectory{
+		paths: make(map[string]string),
+	}
+
+	return rd
+}
+
+func (rd *ReposDirectory) EnsurePath(ctx context.Context, tempRepoPath, location string) string {
+	if location == "" {
+		return ""
+	}
+
+	if strings.HasPrefix(location, "https://") || strings.HasPrefix(location, "http://") || strings.HasPrefix(location, "git@") {
+		log.Debug().Str("location", location).Msg("registering remote repository")
+		localPath := rd.Register(ctx, location)
+		return localPath
+	}
+
+	schemaPath := filepath.Join(tempRepoPath, location)
+	if stat, err := os.Stat(schemaPath); err == nil && stat.IsDir() {
+		log.Debug().Str("location", location).Msg("registering in-repo path")
+		return schemaPath
+	} else {
+		log.Warn().Str("location", location).Err(err).Msg("failed to find in-repo path")
+	}
+
+	return ""
+}
+
+func (rd *ReposDirectory) Register(ctx context.Context, cloneUrl string) string {
+	var (
+		ok      bool
+		repoDir string
+	)
+
+	rd.mutex.Lock()
+	defer rd.mutex.Unlock()
+
+	repoDir, ok = rd.paths[cloneUrl]
+	if ok {
+		rd.fetchLatest()
+		return repoDir
+	}
+
+	return rd.clone(ctx, cloneUrl)
+}
+
+func (rd *ReposDirectory) fetchLatest() {
+	cmd := exec.Command("git", "pull")
+	err := cmd.Run()
+	if err != nil {
+		log.Err(err).Msg("failed to pull latest")
+	}
+}
+
+func (rd *ReposDirectory) clone(ctx context.Context, cloneUrl string) string {
+	repoDir, err := os.MkdirTemp("/tmp", "schemas")
+	if err != nil {
+		log.Err(err).Msg("failed to make temp dir")
+		return ""
+	}
+
+	r := repo.Repo{CloneURL: cloneUrl}
+	err = r.CloneRepoLocal(ctx, repoDir)
+	if err != nil {
+		log.Err(err).Str("clone-url", cloneUrl).Msg("failed to clone repository")
+		return ""
+	}
+
+	rd.paths[cloneUrl] = repoDir
+	return repoDir
+}

pkg/repo/repo.go

@@ -15,13 +15,14 @@ import (
 	"github.com/pkg/errors"
 	"github.com/rs/zerolog/log"
 	"github.com/spf13/viper"
-	"github.com/zapier/kubechecks/telemetry"
 	"go.opentelemetry.io/otel"
 	"go.opentelemetry.io/otel/attribute"
 	"go.opentelemetry.io/otel/trace"
+
+	"github.com/zapier/kubechecks/telemetry"
 )
 
-// Represents a local Repostiory on disk, based off of a PR/MR
+// Repo represents a local Repostiory on disk, based off of a PR/MR
 type Repo struct {
 	BaseRef       string   // base ref is the branch that the PR is being merged into
 	HeadRef       string   // head ref is the branch that the PR is coming from