Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Loading and logging improvements #415

Merged
merged 8 commits into from
Nov 26, 2024
Merged

Loading and logging improvements #415

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
2c1a493
fix: remove debug log for unchanged watchFile
stigok Nov 25, 2024
6032403
refactor: use shorthand zap.Error for err logging
stigok Nov 25, 2024
00b80b3
chore: lint imports
stigok Nov 25, 2024
af9af45
refactor: init vars using shorthand notation
stigok Nov 25, 2024
64b90ac
fix(store/github): improve details in empty provider result warning
stigok Nov 25, 2024
6e9e87e
fix(store/github): log warning when no module repos were found
stigok Nov 25, 2024
4b5fd4d
fix(store/github): load module cache before providers
stigok Nov 25, 2024
f9d8c95
ci: use aws mirror for trivy-db
stigok Nov 25, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions .github/workflows/docker-build.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,3 +37,6 @@ jobs:
ignore-unfixed: true
vuln-type: 'os,library'
severity: 'CRITICAL,HIGH'
env:
TRIVY_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-db
TRIVY_JAVA_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-java-db
33 changes: 15 additions & 18 deletions cmd/terraform-registry/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -141,7 +141,7 @@ func main() {
}
if err := setEnvironmentFromJSONFile(prefix, filename); err != nil {
logger.Fatal("failed to load environment from file(s)",
zap.Errors("err", []error{err}),
zap.Error(err),
)
}
}
Expand Down Expand Up @@ -174,7 +174,7 @@ func main() {
tokens, err := parseAuthTokens(b)
if err != nil {
logger.Error("failed to load auth tokens",
zap.Errors("err", []error{err}),
zap.Error(err),
)
}

Expand Down Expand Up @@ -237,14 +237,11 @@ func watchFile(ctx context.Context, filename string, interval time.Duration, cal
if err != nil {
logger.Error("watchFile: failed to read file",
zap.String("filename", filename),
zap.Errors("err", []error{err}),
zap.Error(err),
)
return
}
if sum := h.Sum(b); bytes.Equal(sum, lastSum) {
logger.Debug("watchFile: file contents unchanged. do nothing.",
zap.String("filename", filename),
)
return
} else {
logger.Debug("watchFile: file contents updated. triggering callback.",
Expand Down Expand Up @@ -292,25 +289,25 @@ func gitHubRegistry(reg *registry.Registry) {
reg.SetModuleStore(store)
reg.SetProviderStore(store)

// Fill module store cache initially
logger.Debug("loading GitHub module store cache")
if err := store.ReloadCache(context.Background()); err != nil {
logger.Error("failed to load GitHub module store cache",
zap.Error(err),
)
}

// Fill provider store cache initially
if reg.IsProviderEnabled {
logger.Debug("loading GitHub provider store cache")
err := store.ReloadProviderCache(context.Background())
if err != nil {
logger.Error("failed to load GitHub provider store cache",
zap.Errors("err", []error{err}),
zap.Error(err),
)
}
}

// Fill module store cache initially
logger.Debug("loading GitHub module store cache")
if err := store.ReloadCache(context.Background()); err != nil {
logger.Error("failed to load GitHub module store cache",
zap.Errors("err", []error{err}),
)
}

// Reload store caches on regular intervals
go func() {
t := time.NewTicker(5 * time.Minute)
Expand All @@ -321,15 +318,15 @@ func gitHubRegistry(reg *registry.Registry) {
logger.Debug("reloading GitHub module store cache")
if err := store.ReloadCache(context.Background()); err != nil {
logger.Error("failed to reload GitHub module store cache",
zap.Errors("err", []error{err}),
zap.Error(err),
)
}
if reg.IsProviderEnabled {
logger.Debug("loading GitHub provider store cache")
err := store.ReloadProviderCache(context.Background())
if err != nil {
logger.Error("failed to load GitHub provider store cache",
zap.Errors("err", []error{err}),
zap.Error(err),
)
}
}
Expand Down Expand Up @@ -362,7 +359,7 @@ func s3Registry(reg *registry.Registry) {
store := s3.NewS3Store(s3Sess, S3Region, S3Bucket, logger.Named("s3 store"))
if err != nil {
logger.Fatal("failed to create S3 store",
zap.Errors("err", []error{err}),
zap.Error(err),
)
}
reg.SetModuleStore(store)
Expand Down
32 changes: 16 additions & 16 deletions pkg/registry/registry.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@ import (
"encoding/json"
"errors"
"fmt"
"github.com/golang-jwt/jwt/v5"
"io"
"net/http"
"slices"
Expand All @@ -18,6 +17,7 @@ import (

"github.com/go-chi/chi/v5"
"github.com/go-chi/chi/v5/middleware"
"github.com/golang-jwt/jwt/v5"
"github.com/nrkno/terraform-registry/pkg/core"
"go.uber.org/zap"
)
Expand Down Expand Up @@ -243,7 +243,7 @@ func (reg *Registry) Index() http.HandlerFunc {
return
}
if _, err := w.Write(WelcomeMessage); err != nil {
reg.logger.Error("Index", zap.Errors("err", []error{err}))
reg.logger.Error("Index", zap.Error(err))
}
}
}
Expand All @@ -264,7 +264,7 @@ func (reg *Registry) Health() http.HandlerFunc {
w.Header().Set("Content-Type", "application/json")
enc := json.NewEncoder(w)
if err := enc.Encode(resp); err != nil {
reg.logger.Error("Health", zap.Errors("err", []error{err}))
reg.logger.Error("Health", zap.Error(err))
}
}
}
Expand All @@ -284,7 +284,7 @@ func (reg *Registry) ServiceDiscovery() http.HandlerFunc {

resp, err := json.Marshal(spec)
if err != nil {
reg.logger.Panic("ServiceDiscovery", zap.Errors("err", []error{err}))
reg.logger.Panic("ServiceDiscovery", zap.Error(err))
}

return func(w http.ResponseWriter, r *http.Request) {
Expand All @@ -294,7 +294,7 @@ func (reg *Registry) ServiceDiscovery() http.HandlerFunc {
}
w.Header().Set("Content-Type", "application/json")
if _, err := w.Write(resp); err != nil {
reg.logger.Error("ServiceDiscovery", zap.Errors("err", []error{err}))
reg.logger.Error("ServiceDiscovery", zap.Error(err))
}
}
}
Expand Down Expand Up @@ -324,7 +324,7 @@ func (reg *Registry) ModuleVersions() http.HandlerFunc {
versions, err := reg.moduleStore.ListModuleVersions(r.Context(), namespace, name, provider)
if err != nil {
http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound)
reg.logger.Error("ListModuleVersions", zap.Errors("err", []error{err}))
reg.logger.Debug("ListModuleVersions", zap.Error(err))
return
}

Expand All @@ -339,12 +339,12 @@ func (reg *Registry) ModuleVersions() http.HandlerFunc {

b, err := json.Marshal(respObj)
if err != nil {
reg.logger.Error("ModuleVersions", zap.Errors("err", []error{err}))
reg.logger.Error("ModuleVersions", zap.Error(err))
}

w.Header().Set("Content-Type", "application/json")
if _, err := w.Write(b); err != nil {
reg.logger.Error("ModuleVersions", zap.Errors("err", []error{err}))
reg.logger.Error("ModuleVersions", zap.Error(err))
}
}
}
Expand All @@ -363,7 +363,7 @@ func (reg *Registry) ModuleDownload() http.HandlerFunc {
ver, err := reg.moduleStore.GetModuleVersion(r.Context(), namespace, name, provider, version)
if err != nil {
http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound)
reg.logger.Error("GetModuleVersion", zap.Errors("err", []error{err}))
reg.logger.Error("GetModuleVersion", zap.Error(err))
return
}

Expand All @@ -384,14 +384,14 @@ func (reg *Registry) ProviderVersions() http.HandlerFunc {
ver, err := reg.providerStore.ListProviderVersions(r.Context(), namespace, name)
if err != nil {
http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound)
reg.logger.Error("ListProviderVersions", zap.Errors("err", []error{err}))
reg.logger.Error("ListProviderVersions", zap.Error(err))
return
}

err = json.NewEncoder(w).Encode(ver)
if err != nil {
http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound)
reg.logger.Error("ListProviderVersions", zap.Errors("err", []error{err}))
reg.logger.Error("ListProviderVersions", zap.Error(err))
return
}

Expand All @@ -414,7 +414,7 @@ func (reg *Registry) ProviderDownload() http.HandlerFunc {
provider, err := reg.providerStore.GetProviderVersion(r.Context(), namespace, name, version, os, arch)
if err != nil {
http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound)
reg.logger.Error("GetProviderVersion", zap.Errors("err", []error{err}))
reg.logger.Error("GetProviderVersion", zap.Error(err))
return
}

Expand All @@ -434,7 +434,7 @@ func (reg *Registry) ProviderDownload() http.HandlerFunc {
tokenString, err := token.SignedString(reg.AssetDownloadAuthSecret)
if err != nil {
http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
reg.logger.Error("GetProviderVersion: unable to create token", zap.Errors("err", []error{err}))
reg.logger.Error("GetProviderVersion: unable to create token", zap.Error(err))
return
}

Expand All @@ -446,7 +446,7 @@ func (reg *Registry) ProviderDownload() http.HandlerFunc {
err = json.NewEncoder(w).Encode(provider)
if err != nil {
http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound)
reg.logger.Error("GetProviderVersion", zap.Errors("err", []error{err}))
reg.logger.Error("GetProviderVersion", zap.Error(err))
return
}
w.WriteHeader(http.StatusOK)
Expand All @@ -468,15 +468,15 @@ func (reg *Registry) ProviderAssetDownload() http.HandlerFunc {
asset, err := reg.providerStore.GetProviderAsset(r.Context(), owner, repo, tag, assetName)
if err != nil {
http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound)
reg.logger.Error("ProviderAssetDownload", zap.Errors("err", []error{err}))
reg.logger.Error("ProviderAssetDownload", zap.Error(err))
return
}
defer asset.Close()

written, err := io.Copy(w, asset)
if err != nil {
http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
reg.logger.Error("ProviderAssetDownload", zap.Errors("err", []error{err}))
reg.logger.Error("ProviderAssetDownload", zap.Error(err))
return
}

Expand Down
28 changes: 14 additions & 14 deletions pkg/store/github/github.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,14 +10,14 @@ import (
"context"
"encoding/json"
"fmt"
"github.com/ProtonMail/go-crypto/openpgp"
"io"
"net/http"
"regexp"
"strings"
"sync"
"time"

"github.com/ProtonMail/go-crypto/openpgp"
"github.com/google/go-github/v43/github"
goversion "github.com/hashicorp/go-version"
"github.com/nrkno/terraform-registry/pkg/core"
Expand Down Expand Up @@ -213,25 +213,23 @@ func (s *GitHubStore) findAsset(ctx context.Context, owner string, repo string,
// Should be called at least once after initialisation and probably on regular
// intervals afterward to keep providerCache up-to-date.
func (s *GitHubStore) ReloadProviderCache(ctx context.Context) error {
var (
owner string
name string
)

repos, err := s.searchProviderRepositories(ctx)
if err != nil {
return err
}

if len(repos) == 0 {
s.logger.Warn("could not find any repos matching filter")
s.logger.Warn("could not find any provider repos matching filter",
zap.String("topic", s.providerTopicFilter),
zap.String("owner", s.providerOwnerFilter))
}

providerVersionsCache := make(map[string]*core.ProviderVersions)
providerCache := make(map[string]*core.Provider)

for _, repo := range repos {
if owner, name, err = getOwnerRepoName(repo); err != nil {
owner, name, err := getOwnerRepoName(repo)
if err != nil {
return err
}

Expand Down Expand Up @@ -389,20 +387,22 @@ func (s *GitHubStore) getGPGPublicKey(ctx context.Context, release *github.Repos
// Should be called at least once after initialisation and probably on regular
// intervals afterward to keep moduleCache up-to-date.
func (s *GitHubStore) ReloadCache(ctx context.Context) error {
var (
owner string
name string
)

repos, err := s.searchModuleRepositories(ctx)
if err != nil {
return err
}

if len(repos) == 0 {
s.logger.Warn("could not find any module repos matching filter",
zap.String("topic", s.topicFilter),
zap.String("owner", s.ownerFilter))
}

fresh := make(map[string][]*core.ModuleVersion)

for _, repo := range repos {
if owner, name, err = getOwnerRepoName(repo); err != nil {
owner, name, err := getOwnerRepoName(repo)
if err != nil {
return err
}

Expand Down