From 2c1a493e5068cf00b973ce10bef6bf454f2cb5a2 Mon Sep 17 00:00:00 2001 From: Stig Otnes Kolstad Date: Mon, 25 Nov 2024 14:41:49 +0100 Subject: [PATCH 1/8] fix: remove debug log for unchanged watchFile --- cmd/terraform-registry/main.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/cmd/terraform-registry/main.go b/cmd/terraform-registry/main.go index 7ca8646..1731a05 100644 --- a/cmd/terraform-registry/main.go +++ b/cmd/terraform-registry/main.go @@ -242,9 +242,6 @@ func watchFile(ctx context.Context, filename string, interval time.Duration, cal return } if sum := h.Sum(b); bytes.Equal(sum, lastSum) { - logger.Debug("watchFile: file contents unchanged. do nothing.", - zap.String("filename", filename), - ) return } else { logger.Debug("watchFile: file contents updated. triggering callback.", From 6032403a5a9c2d24f01c83e7073f0633f5f12127 Mon Sep 17 00:00:00 2001 From: Stig Otnes Kolstad Date: Mon, 25 Nov 2024 15:07:49 +0100 Subject: [PATCH 2/8] refactor: use shorthand zap.Error for err logging --- cmd/terraform-registry/main.go | 16 ++++++++-------- pkg/registry/registry.go | 30 +++++++++++++++--------------- 2 files changed, 23 insertions(+), 23 deletions(-) diff --git a/cmd/terraform-registry/main.go b/cmd/terraform-registry/main.go index 1731a05..543ecdc 100644 --- a/cmd/terraform-registry/main.go +++ b/cmd/terraform-registry/main.go @@ -141,7 +141,7 @@ func main() { } if err := setEnvironmentFromJSONFile(prefix, filename); err != nil { logger.Fatal("failed to load environment from file(s)", - zap.Errors("err", []error{err}), + zap.Error(err), ) } } @@ -174,7 +174,7 @@ func main() { tokens, err := parseAuthTokens(b) if err != nil { logger.Error("failed to load auth tokens", - zap.Errors("err", []error{err}), + zap.Error(err), ) } @@ -237,7 +237,7 @@ func watchFile(ctx context.Context, filename string, interval time.Duration, cal if err != nil { logger.Error("watchFile: failed to read file", zap.String("filename", filename), - zap.Errors("err", []error{err}), + zap.Error(err), ) return } @@ -295,7 +295,7 @@ func gitHubRegistry(reg *registry.Registry) { err := store.ReloadProviderCache(context.Background()) if err != nil { logger.Error("failed to load GitHub provider store cache", - zap.Errors("err", []error{err}), + zap.Error(err), ) } } @@ -304,7 +304,7 @@ func gitHubRegistry(reg *registry.Registry) { logger.Debug("loading GitHub module store cache") if err := store.ReloadCache(context.Background()); err != nil { logger.Error("failed to load GitHub module store cache", - zap.Errors("err", []error{err}), + zap.Error(err), ) } @@ -318,7 +318,7 @@ func gitHubRegistry(reg *registry.Registry) { logger.Debug("reloading GitHub module store cache") if err := store.ReloadCache(context.Background()); err != nil { logger.Error("failed to reload GitHub module store cache", - zap.Errors("err", []error{err}), + zap.Error(err), ) } if reg.IsProviderEnabled { @@ -326,7 +326,7 @@ func gitHubRegistry(reg *registry.Registry) { err := store.ReloadProviderCache(context.Background()) if err != nil { logger.Error("failed to load GitHub provider store cache", - zap.Errors("err", []error{err}), + zap.Error(err), ) } } @@ -359,7 +359,7 @@ func s3Registry(reg *registry.Registry) { store := s3.NewS3Store(s3Sess, S3Region, S3Bucket, logger.Named("s3 store")) if err != nil { logger.Fatal("failed to create S3 store", - zap.Errors("err", []error{err}), + zap.Error(err), ) } reg.SetModuleStore(store) diff --git a/pkg/registry/registry.go b/pkg/registry/registry.go index 48586a7..ce7da90 100644 --- a/pkg/registry/registry.go +++ b/pkg/registry/registry.go @@ -243,7 +243,7 @@ func (reg *Registry) Index() http.HandlerFunc { return } if _, err := w.Write(WelcomeMessage); err != nil { - reg.logger.Error("Index", zap.Errors("err", []error{err})) + reg.logger.Error("Index", zap.Error(err)) } } } @@ -264,7 +264,7 @@ func (reg *Registry) Health() http.HandlerFunc { w.Header().Set("Content-Type", "application/json") enc := json.NewEncoder(w) if err := enc.Encode(resp); err != nil { - reg.logger.Error("Health", zap.Errors("err", []error{err})) + reg.logger.Error("Health", zap.Error(err)) } } } @@ -284,7 +284,7 @@ func (reg *Registry) ServiceDiscovery() http.HandlerFunc { resp, err := json.Marshal(spec) if err != nil { - reg.logger.Panic("ServiceDiscovery", zap.Errors("err", []error{err})) + reg.logger.Panic("ServiceDiscovery", zap.Error(err)) } return func(w http.ResponseWriter, r *http.Request) { @@ -294,7 +294,7 @@ func (reg *Registry) ServiceDiscovery() http.HandlerFunc { } w.Header().Set("Content-Type", "application/json") if _, err := w.Write(resp); err != nil { - reg.logger.Error("ServiceDiscovery", zap.Errors("err", []error{err})) + reg.logger.Error("ServiceDiscovery", zap.Error(err)) } } } @@ -324,7 +324,7 @@ func (reg *Registry) ModuleVersions() http.HandlerFunc { versions, err := reg.moduleStore.ListModuleVersions(r.Context(), namespace, name, provider) if err != nil { http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound) - reg.logger.Error("ListModuleVersions", zap.Errors("err", []error{err})) + reg.logger.Debug("ListModuleVersions", zap.Error(err)) return } @@ -339,12 +339,12 @@ func (reg *Registry) ModuleVersions() http.HandlerFunc { b, err := json.Marshal(respObj) if err != nil { - reg.logger.Error("ModuleVersions", zap.Errors("err", []error{err})) + reg.logger.Error("ModuleVersions", zap.Error(err)) } w.Header().Set("Content-Type", "application/json") if _, err := w.Write(b); err != nil { - reg.logger.Error("ModuleVersions", zap.Errors("err", []error{err})) + reg.logger.Error("ModuleVersions", zap.Error(err)) } } } @@ -363,7 +363,7 @@ func (reg *Registry) ModuleDownload() http.HandlerFunc { ver, err := reg.moduleStore.GetModuleVersion(r.Context(), namespace, name, provider, version) if err != nil { http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound) - reg.logger.Error("GetModuleVersion", zap.Errors("err", []error{err})) + reg.logger.Error("GetModuleVersion", zap.Error(err)) return } @@ -384,14 +384,14 @@ func (reg *Registry) ProviderVersions() http.HandlerFunc { ver, err := reg.providerStore.ListProviderVersions(r.Context(), namespace, name) if err != nil { http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound) - reg.logger.Error("ListProviderVersions", zap.Errors("err", []error{err})) + reg.logger.Error("ListProviderVersions", zap.Error(err)) return } err = json.NewEncoder(w).Encode(ver) if err != nil { http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound) - reg.logger.Error("ListProviderVersions", zap.Errors("err", []error{err})) + reg.logger.Error("ListProviderVersions", zap.Error(err)) return } @@ -414,7 +414,7 @@ func (reg *Registry) ProviderDownload() http.HandlerFunc { provider, err := reg.providerStore.GetProviderVersion(r.Context(), namespace, name, version, os, arch) if err != nil { http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound) - reg.logger.Error("GetProviderVersion", zap.Errors("err", []error{err})) + reg.logger.Error("GetProviderVersion", zap.Error(err)) return } @@ -434,7 +434,7 @@ func (reg *Registry) ProviderDownload() http.HandlerFunc { tokenString, err := token.SignedString(reg.AssetDownloadAuthSecret) if err != nil { http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) - reg.logger.Error("GetProviderVersion: unable to create token", zap.Errors("err", []error{err})) + reg.logger.Error("GetProviderVersion: unable to create token", zap.Error(err)) return } @@ -446,7 +446,7 @@ func (reg *Registry) ProviderDownload() http.HandlerFunc { err = json.NewEncoder(w).Encode(provider) if err != nil { http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound) - reg.logger.Error("GetProviderVersion", zap.Errors("err", []error{err})) + reg.logger.Error("GetProviderVersion", zap.Error(err)) return } w.WriteHeader(http.StatusOK) @@ -468,7 +468,7 @@ func (reg *Registry) ProviderAssetDownload() http.HandlerFunc { asset, err := reg.providerStore.GetProviderAsset(r.Context(), owner, repo, tag, assetName) if err != nil { http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound) - reg.logger.Error("ProviderAssetDownload", zap.Errors("err", []error{err})) + reg.logger.Error("ProviderAssetDownload", zap.Error(err)) return } defer asset.Close() @@ -476,7 +476,7 @@ func (reg *Registry) ProviderAssetDownload() http.HandlerFunc { written, err := io.Copy(w, asset) if err != nil { http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) - reg.logger.Error("ProviderAssetDownload", zap.Errors("err", []error{err})) + reg.logger.Error("ProviderAssetDownload", zap.Error(err)) return } From 00b80b3e91482dd29ae8970bda608f9b1473bad1 Mon Sep 17 00:00:00 2001 From: Stig Otnes Kolstad Date: Mon, 25 Nov 2024 15:10:08 +0100 Subject: [PATCH 3/8] chore: lint imports --- pkg/registry/registry.go | 2 +- pkg/store/github/github.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/registry/registry.go b/pkg/registry/registry.go index ce7da90..c881930 100644 --- a/pkg/registry/registry.go +++ b/pkg/registry/registry.go @@ -8,7 +8,6 @@ import ( "encoding/json" "errors" "fmt" - "github.com/golang-jwt/jwt/v5" "io" "net/http" "slices" @@ -18,6 +17,7 @@ import ( "github.com/go-chi/chi/v5" "github.com/go-chi/chi/v5/middleware" + "github.com/golang-jwt/jwt/v5" "github.com/nrkno/terraform-registry/pkg/core" "go.uber.org/zap" ) diff --git a/pkg/store/github/github.go b/pkg/store/github/github.go index 6a1db87..c3b59c9 100644 --- a/pkg/store/github/github.go +++ b/pkg/store/github/github.go @@ -10,7 +10,6 @@ import ( "context" "encoding/json" "fmt" - "github.com/ProtonMail/go-crypto/openpgp" "io" "net/http" "regexp" @@ -18,6 +17,7 @@ import ( "sync" "time" + "github.com/ProtonMail/go-crypto/openpgp" "github.com/google/go-github/v43/github" goversion "github.com/hashicorp/go-version" "github.com/nrkno/terraform-registry/pkg/core" From af9af4523fa81f58ad24c9aa8fae31100c9256c1 Mon Sep 17 00:00:00 2001 From: Stig Otnes Kolstad Date: Mon, 25 Nov 2024 15:26:54 +0100 Subject: [PATCH 4/8] refactor: init vars using shorthand notation No need to be overly performant in cold code paths. --- pkg/store/github/github.go | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/pkg/store/github/github.go b/pkg/store/github/github.go index c3b59c9..b2ba72f 100644 --- a/pkg/store/github/github.go +++ b/pkg/store/github/github.go @@ -213,11 +213,6 @@ func (s *GitHubStore) findAsset(ctx context.Context, owner string, repo string, // Should be called at least once after initialisation and probably on regular // intervals afterward to keep providerCache up-to-date. func (s *GitHubStore) ReloadProviderCache(ctx context.Context) error { - var ( - owner string - name string - ) - repos, err := s.searchProviderRepositories(ctx) if err != nil { return err @@ -231,7 +226,8 @@ func (s *GitHubStore) ReloadProviderCache(ctx context.Context) error { providerCache := make(map[string]*core.Provider) for _, repo := range repos { - if owner, name, err = getOwnerRepoName(repo); err != nil { + owner, name, err := getOwnerRepoName(repo) + if err != nil { return err } @@ -389,11 +385,6 @@ func (s *GitHubStore) getGPGPublicKey(ctx context.Context, release *github.Repos // Should be called at least once after initialisation and probably on regular // intervals afterward to keep moduleCache up-to-date. func (s *GitHubStore) ReloadCache(ctx context.Context) error { - var ( - owner string - name string - ) - repos, err := s.searchModuleRepositories(ctx) if err != nil { return err @@ -402,7 +393,8 @@ func (s *GitHubStore) ReloadCache(ctx context.Context) error { fresh := make(map[string][]*core.ModuleVersion) for _, repo := range repos { - if owner, name, err = getOwnerRepoName(repo); err != nil { + owner, name, err := getOwnerRepoName(repo) + if err != nil { return err } From 64b90ac6e05f28d488cfd30bc23e992101e7c2f8 Mon Sep 17 00:00:00 2001 From: Stig Otnes Kolstad Date: Mon, 25 Nov 2024 15:30:50 +0100 Subject: [PATCH 5/8] fix(store/github): improve details in empty provider result warning --- pkg/store/github/github.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pkg/store/github/github.go b/pkg/store/github/github.go index b2ba72f..b15b53d 100644 --- a/pkg/store/github/github.go +++ b/pkg/store/github/github.go @@ -219,7 +219,9 @@ func (s *GitHubStore) ReloadProviderCache(ctx context.Context) error { } if len(repos) == 0 { - s.logger.Warn("could not find any repos matching filter") + s.logger.Warn("could not find any provider repos matching filter", + zap.String("topic", s.providerTopicFilter), + zap.String("owner", s.providerOwnerFilter)) } providerVersionsCache := make(map[string]*core.ProviderVersions) From 6e9e87ec50c569f05e2b5307a2c6d869bdba8f32 Mon Sep 17 00:00:00 2001 From: Stig Otnes Kolstad Date: Mon, 25 Nov 2024 15:31:18 +0100 Subject: [PATCH 6/8] fix(store/github): log warning when no module repos were found --- pkg/store/github/github.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pkg/store/github/github.go b/pkg/store/github/github.go index b15b53d..954584b 100644 --- a/pkg/store/github/github.go +++ b/pkg/store/github/github.go @@ -392,6 +392,12 @@ func (s *GitHubStore) ReloadCache(ctx context.Context) error { return err } + if len(repos) == 0 { + s.logger.Warn("could not find any module repos matching filter", + zap.String("topic", s.topicFilter), + zap.String("owner", s.ownerFilter)) + } + fresh := make(map[string][]*core.ModuleVersion) for _, repo := range repos { From 4b5fd4d906308151062b1600e7b06c8508f8b182 Mon Sep 17 00:00:00 2001 From: Stig Otnes Kolstad Date: Mon, 25 Nov 2024 15:50:38 +0100 Subject: [PATCH 7/8] fix(store/github): load module cache before providers Because module loading is much faster. --- cmd/terraform-registry/main.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/cmd/terraform-registry/main.go b/cmd/terraform-registry/main.go index 543ecdc..a631f0b 100644 --- a/cmd/terraform-registry/main.go +++ b/cmd/terraform-registry/main.go @@ -289,6 +289,14 @@ func gitHubRegistry(reg *registry.Registry) { reg.SetModuleStore(store) reg.SetProviderStore(store) + // Fill module store cache initially + logger.Debug("loading GitHub module store cache") + if err := store.ReloadCache(context.Background()); err != nil { + logger.Error("failed to load GitHub module store cache", + zap.Error(err), + ) + } + // Fill provider store cache initially if reg.IsProviderEnabled { logger.Debug("loading GitHub provider store cache") @@ -300,14 +308,6 @@ func gitHubRegistry(reg *registry.Registry) { } } - // Fill module store cache initially - logger.Debug("loading GitHub module store cache") - if err := store.ReloadCache(context.Background()); err != nil { - logger.Error("failed to load GitHub module store cache", - zap.Error(err), - ) - } - // Reload store caches on regular intervals go func() { t := time.NewTicker(5 * time.Minute) From f9d8c958d6b044a4293e47481f49269233db6fce Mon Sep 17 00:00:00 2001 From: Stig Otnes Kolstad Date: Mon, 25 Nov 2024 16:27:15 +0100 Subject: [PATCH 8/8] ci: use aws mirror for trivy-db --- .github/workflows/docker-build.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/docker-build.yaml b/.github/workflows/docker-build.yaml index 47b2c11..49e5f6d 100644 --- a/.github/workflows/docker-build.yaml +++ b/.github/workflows/docker-build.yaml @@ -37,3 +37,6 @@ jobs: ignore-unfixed: true vuln-type: 'os,library' severity: 'CRITICAL,HIGH' + env: + TRIVY_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-db + TRIVY_JAVA_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-java-db