From 36ceca692ef762fd1816c17ef6de81377995593e Mon Sep 17 00:00:00 2001 From: Adam Buran Date: Wed, 11 Dec 2024 19:40:15 +0000 Subject: [PATCH 01/24] httpcache stuff Signed-off-by: Adam Buran Signed-off-by: aburan28 --- applicationset/generators/scm_provider.go | 6 +++++- .../services/internal/github_app/client.go | 17 ++++++++++++++++- applicationset/services/scm_provider/github.go | 16 +++++++++++++++- .../services/scm_provider/github_app.go | 4 ++-- go.mod | 2 +- .../v1alpha1/applicationset_types.go | 2 ++ 6 files changed, 41 insertions(+), 6 deletions(-) diff --git a/applicationset/generators/scm_provider.go b/applicationset/generators/scm_provider.go index 417b682e50511..73f1f358735b7 100644 --- a/applicationset/generators/scm_provider.go +++ b/applicationset/generators/scm_provider.go @@ -7,6 +7,8 @@ import ( "strings" "time" + "github.com/gregjones/httpcache" + "sigs.k8s.io/controller-runtime/pkg/client" log "github.com/sirupsen/logrus" @@ -35,6 +37,7 @@ type SCMConfig struct { allowedSCMProviders []string enableSCMProviders bool GitHubApps github_app_auth.Credentials + GitHubClientCache httpcache.Cache tokenRefStrictMode bool } @@ -282,6 +285,7 @@ func (g *SCMProviderGenerator) githubProvider(ctx context.Context, github *argop github.Organization, github.API, github.AllBranches, + github.CachingEnabled, ) } @@ -289,5 +293,5 @@ func (g *SCMProviderGenerator) githubProvider(ctx context.Context, github *argop if err != nil { return nil, fmt.Errorf("error fetching Github token: %w", err) } - return scm_provider.NewGithubProvider(ctx, github.Organization, token, github.API, github.AllBranches) + return scm_provider.NewGithubProvider(ctx, github.Organization, token, github.API, github.AllBranches, github.CachingEnabled) } diff --git a/applicationset/services/internal/github_app/client.go b/applicationset/services/internal/github_app/client.go index 742b2bc001383..adb9296bde002 100644 --- a/applicationset/services/internal/github_app/client.go +++ b/applicationset/services/internal/github_app/client.go @@ -6,12 +6,13 @@ import ( "github.com/bradleyfalzon/ghinstallation/v2" "github.com/google/go-github/v63/github" + "github.com/gregjones/httpcache" "github.com/argoproj/argo-cd/v2/applicationset/services/github_app_auth" ) // Client builds a github client for the given app authentication. -func Client(g github_app_auth.Authentication, url string) (*github.Client, error) { +func Client(g github_app_auth.Authentication, url string, cacheEnabled bool) (*github.Client, error) { rt, err := ghinstallation.New(http.DefaultTransport, g.Id, g.InstallationId, []byte(g.PrivateKey)) if err != nil { return nil, fmt.Errorf("failed to create github app install: %w", err) @@ -20,9 +21,23 @@ func Client(g github_app_auth.Authentication, url string) (*github.Client, error url = g.EnterpriseBaseURL } var client *github.Client + + // determine if the http client should use a cache if url == "" { httpClient := http.Client{Transport: rt} + client = github.NewClient(&httpClient) + } else if cacheEnabled { + cache := httpcache.NewMemoryCache() + cachingHttpClient := http.Client{ + Transport: &httpcache.Transport{ + Cache: cache, + }, + } + client, err = github.NewClient(&cachingHttpClient).WithEnterpriseURLs(url, url) + if err != nil { + return nil, fmt.Errorf("failed to create http cache client: %w", err) + } } else { rt.BaseURL = url httpClient := http.Client{Transport: rt} diff --git a/applicationset/services/scm_provider/github.go b/applicationset/services/scm_provider/github.go index 961c08066672d..3d06d5fc941d7 100644 --- a/applicationset/services/scm_provider/github.go +++ b/applicationset/services/scm_provider/github.go @@ -7,6 +7,7 @@ import ( "os" "github.com/google/go-github/v63/github" + "github.com/gregjones/httpcache" "golang.org/x/oauth2" ) @@ -18,7 +19,7 @@ type GithubProvider struct { var _ SCMProviderService = &GithubProvider{} -func NewGithubProvider(ctx context.Context, organization string, token string, url string, allBranches bool) (*GithubProvider, error) { +func NewGithubProvider(ctx context.Context, organization string, token string, url string, allBranches bool, cacheEnabled bool) (*GithubProvider, error) { var ts oauth2.TokenSource // Undocumented environment variable to set a default token, to be used in testing to dodge anonymous rate limits. if token == "" { @@ -29,7 +30,20 @@ func NewGithubProvider(ctx context.Context, organization string, token string, u &oauth2.Token{AccessToken: token}, ) } + + if cacheEnabled { + cache := httpcache.NewMemoryCache() + cachingHttpClient := http.Client{ + Transport: &httpcache.Transport{ + Cache: cache, + }, + } + oauth2.NewClient(ctx, ) + } + + httpClient := oauth2.NewClient(ctx, ts) + var client *github.Client if url == "" { client = github.NewClient(httpClient) diff --git a/applicationset/services/scm_provider/github_app.go b/applicationset/services/scm_provider/github_app.go index 5429ed48ee8ab..16db0ad010347 100644 --- a/applicationset/services/scm_provider/github_app.go +++ b/applicationset/services/scm_provider/github_app.go @@ -5,8 +5,8 @@ import ( "github.com/argoproj/argo-cd/v2/applicationset/services/internal/github_app" ) -func NewGithubAppProviderFor(g github_app_auth.Authentication, organization string, url string, allBranches bool) (*GithubProvider, error) { - client, err := github_app.Client(g, url) +func NewGithubAppProviderFor(g github_app_auth.Authentication, organization string, url string, allBranches bool, cacheEnabled bool) (*GithubProvider, error) { + client, err := github_app.Client(g, url, cacheEnabled) if err != nil { return nil, err } diff --git a/go.mod b/go.mod index df8ecce49e953..df80b876488c3 100644 --- a/go.mod +++ b/go.mod @@ -215,7 +215,7 @@ require ( github.com/google/gofuzz v1.2.0 // indirect github.com/gosimple/unidecode v1.0.1 // indirect github.com/gregdel/pushover v1.2.1 // indirect - github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 // indirect + github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 github.com/grpc-ecosystem/grpc-gateway/v2 v2.22.0 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect github.com/hashicorp/go-version v1.6.0 // indirect diff --git a/pkg/apis/application/v1alpha1/applicationset_types.go b/pkg/apis/application/v1alpha1/applicationset_types.go index 16f9eeecd5d85..493db43791455 100644 --- a/pkg/apis/application/v1alpha1/applicationset_types.go +++ b/pkg/apis/application/v1alpha1/applicationset_types.go @@ -487,6 +487,8 @@ type SCMProviderGeneratorGithub struct { AppSecretName string `json:"appSecretName,omitempty" protobuf:"bytes,4,opt,name=appSecretName"` // Scan all branches instead of just the default branch. AllBranches bool `json:"allBranches,omitempty" protobuf:"varint,5,opt,name=allBranches"` + // should caching be enabled for github requests + CachingEnabled bool `json:"cachingEnabled,omitempty" protobuf:"varint,6,opt,name=cachingEnabled` } // SCMProviderGeneratorGitlab defines connection info specific to Gitlab. From 24cb96f53f0bd8f3ade8d0c1550dd42ccbfb84ea Mon Sep 17 00:00:00 2001 From: "aburan28@gmail.com" Date: Sun, 22 Dec 2024 10:05:16 -0300 Subject: [PATCH 02/24] minor updates Signed-off-by: aburan28 Signed-off-by: aburan28 --- applicationset/generators/scm_provider.go | 10 ++++++++-- applicationset/services/pull_request/github_app.go | 4 ++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/applicationset/generators/scm_provider.go b/applicationset/generators/scm_provider.go index 73f1f358735b7..5f33b64475ec2 100644 --- a/applicationset/generators/scm_provider.go +++ b/applicationset/generators/scm_provider.go @@ -41,14 +41,20 @@ type SCMConfig struct { tokenRefStrictMode bool } -func NewSCMConfig(scmRootCAPath string, allowedSCMProviders []string, enableSCMProviders bool, gitHubApps github_app_auth.Credentials, tokenRefStrictMode bool) SCMConfig { - return SCMConfig{ +func NewSCMConfig(scmRootCAPath string, allowedSCMProviders []string, enableSCMProviders bool, gitHubApps github_app_auth.Credentials, tokenRefStrictMode bool, cacheEnabled bool) SCMConfig { + scmConfig := SCMConfig{ scmRootCAPath: scmRootCAPath, allowedSCMProviders: allowedSCMProviders, enableSCMProviders: enableSCMProviders, GitHubApps: gitHubApps, tokenRefStrictMode: tokenRefStrictMode, } + + if cacheEnabled { + scmConfig.GitHubClientCache = httpcache.NewMemoryCache() + } + + return scmConfig } func NewSCMProviderGenerator(client client.Client, scmConfig SCMConfig) Generator { diff --git a/applicationset/services/pull_request/github_app.go b/applicationset/services/pull_request/github_app.go index 8879a777ad277..6705b2b829fd0 100644 --- a/applicationset/services/pull_request/github_app.go +++ b/applicationset/services/pull_request/github_app.go @@ -5,8 +5,8 @@ import ( "github.com/argoproj/argo-cd/v2/applicationset/services/internal/github_app" ) -func NewGithubAppService(g github_app_auth.Authentication, url, owner, repo string, labels []string) (PullRequestService, error) { - client, err := github_app.Client(g, url) +func NewGithubAppService(g github_app_auth.Authentication, url, owner, repo string, labels []string, cacheEnabled bool) (PullRequestService, error) { + client, err := github_app.Client(g, url, cacheEnabled) if err != nil { return nil, err } From 2cb2d67279b6542533a4fe94d7a0376d4c05e077 Mon Sep 17 00:00:00 2001 From: "aburan28@gmail.com" Date: Sun, 22 Dec 2024 10:40:40 -0300 Subject: [PATCH 03/24] updates Signed-off-by: aburan28@gmail.com Signed-off-by: aburan28 --- applicationset/generators/pull_request.go | 2 +- .../services/scm_provider/github.go | 27 +++++++++++++------ .../commands/applicationset_controller.go | 7 ++++- .../v1alpha1/applicationset_types.go | 4 ++- 4 files changed, 29 insertions(+), 11 deletions(-) diff --git a/applicationset/generators/pull_request.go b/applicationset/generators/pull_request.go index f0c2bfaacfcf5..25f2067d08109 100644 --- a/applicationset/generators/pull_request.go +++ b/applicationset/generators/pull_request.go @@ -215,7 +215,7 @@ func (g *PullRequestGenerator) github(ctx context.Context, cfg *argoprojiov1alph if err != nil { return nil, fmt.Errorf("error getting GitHub App secret: %w", err) } - return pullrequest.NewGithubAppService(*auth, cfg.API, cfg.Owner, cfg.Repo, cfg.Labels) + return pullrequest.NewGithubAppService(*auth, cfg.API, cfg.Owner, cfg.Repo, cfg.Labels, g.cacheEnabled) } // always default to token, even if not set (public access) diff --git a/applicationset/services/scm_provider/github.go b/applicationset/services/scm_provider/github.go index 3d06d5fc941d7..be930e4e8b75c 100644 --- a/applicationset/services/scm_provider/github.go +++ b/applicationset/services/scm_provider/github.go @@ -11,6 +11,14 @@ import ( "golang.org/x/oauth2" ) +type contextKey struct{} + +var cacheContextKey = contextKey{} + +func ContextWithGithubCache(ctx context.Context, cache httpcache.Cache) context.Context { + return context.WithValue(ctx, cacheContextKey, cache) +} + type GithubProvider struct { client *github.Client organization string @@ -29,18 +37,21 @@ func NewGithubProvider(ctx context.Context, organization string, token string, u ts = oauth2.StaticTokenSource( &oauth2.Token{AccessToken: token}, ) + if cache, ok := ctx.Value(cacheContextKey).(httpcache.Cache); ok { + ctx = context.WithValue(ctx, oauth2.HTTPClient, &http.Client{ + Transport: &httpcache.Transport{ + Cache: cache, + }, + }) + } + + oauth2.NewClient(ctx, ts) + } if cacheEnabled { - cache := httpcache.NewMemoryCache() - cachingHttpClient := http.Client{ - Transport: &httpcache.Transport{ - Cache: cache, - }, - } - oauth2.NewClient(ctx, ) - } + } httpClient := oauth2.NewClient(ctx, ts) diff --git a/cmd/argocd-applicationset-controller/commands/applicationset_controller.go b/cmd/argocd-applicationset-controller/commands/applicationset_controller.go index 345454b7e7a2c..c68a88341687e 100644 --- a/cmd/argocd-applicationset-controller/commands/applicationset_controller.go +++ b/cmd/argocd-applicationset-controller/commands/applicationset_controller.go @@ -53,6 +53,8 @@ func NewCommand() *cobra.Command { probeBindAddr string webhookAddr string enableLeaderElection bool + enableGithubCache bool + githubCacheSize int64 applicationSetNamespaces []string argocdRepoServer string policy string @@ -164,7 +166,7 @@ func NewCommand() *cobra.Command { argoSettingsMgr := argosettings.NewSettingsManager(ctx, k8sClient, namespace) argoCDDB := db.NewDB(namespace, argoSettingsMgr, k8sClient) - scmConfig := generators.NewSCMConfig(scmRootCAPath, allowedScmProviders, enableScmProviders, github_app.NewAuthCredentials(argoCDDB.(db.RepoCredsDB)), tokenRefStrictMode) + scmConfig := generators.NewSCMConfig(scmRootCAPath, allowedScmProviders, enableScmProviders, github_app.NewAuthCredentials(argoCDDB.(db.RepoCredsDB)), tokenRefStrictMode, enableGithubCache) tlsConfig := apiclient.TLSConfiguration{ DisableTLS: repoServerPlaintext, @@ -262,6 +264,9 @@ func NewCommand() *cobra.Command { command.Flags().StringSliceVar(&globalPreservedLabels, "preserved-labels", env.StringsFromEnv("ARGOCD_APPLICATIONSET_CONTROLLER_GLOBAL_PRESERVED_LABELS", []string{}, ","), "Sets global preserved field values for labels") command.Flags().IntVar(&webhookParallelism, "webhook-parallelism-limit", env.ParseNumFromEnv("ARGOCD_APPLICATIONSET_CONTROLLER_WEBHOOK_PARALLELISM_LIMIT", 50, 1, 1000), "Number of webhook requests processed concurrently") command.Flags().StringSliceVar(&metricsAplicationsetLabels, "metrics-applicationset-labels", []string{}, "List of Application labels that will be added to the argocd_applicationset_labels metric") + command.Flags().BoolVar(&enableGithubCache, "enabled-github-cache", env.ParseBoolFromEnv("ARGOCD_APPLICATIONSET_CONTROLLER_ENABLED_GITHUB_CACHE", false), "Enable caching for GitHub client") + command.Flags().Int64Var(&githubCacheSize, "github-cache-size", env.ParseInt64FromEnv("ARGOCD_APPLICATIONSET_CONTROLLER_GITHUB_CACHE_SIZE", 0, 0, 0), "Max size of the cache for GitHub client in kb") + return &command } diff --git a/pkg/apis/application/v1alpha1/applicationset_types.go b/pkg/apis/application/v1alpha1/applicationset_types.go index 493db43791455..801be1cccab8f 100644 --- a/pkg/apis/application/v1alpha1/applicationset_types.go +++ b/pkg/apis/application/v1alpha1/applicationset_types.go @@ -488,7 +488,7 @@ type SCMProviderGeneratorGithub struct { // Scan all branches instead of just the default branch. AllBranches bool `json:"allBranches,omitempty" protobuf:"varint,5,opt,name=allBranches"` // should caching be enabled for github requests - CachingEnabled bool `json:"cachingEnabled,omitempty" protobuf:"varint,6,opt,name=cachingEnabled` + CachingEnabled bool `json:"cachingEnabled,omitempty" protobuf:"varint,6,opt,name=cachingEnabled"` } // SCMProviderGeneratorGitlab defines connection info specific to Gitlab. @@ -680,6 +680,8 @@ type PullRequestGeneratorGithub struct { AppSecretName string `json:"appSecretName,omitempty" protobuf:"bytes,5,opt,name=appSecretName"` // Labels is used to filter the PRs that you want to target Labels []string `json:"labels,omitempty" protobuf:"bytes,6,rep,name=labels"` + // should caching be enabled for github requests + CachingEnabled bool `json:"cachingEnabled,omitempty" protobuf:"varint,6,opt,name=cachingEnabled"` } // PullRequestGeneratorGitLab defines connection info specific to GitLab. From 7e8048da84fc88236fefcb851ef000e95a8d07e0 Mon Sep 17 00:00:00 2001 From: "aburan28@gmail.com" Date: Sun, 22 Dec 2024 11:17:18 -0300 Subject: [PATCH 04/24] try a hack Signed-off-by: aburan28@gmail.com Signed-off-by: aburan28 --- applicationset/generators/pull_request.go | 2 +- go.mod | 1 + go.sum | 2 ++ server/applicationset/applicationset.go | 4 ++-- 4 files changed, 6 insertions(+), 3 deletions(-) diff --git a/applicationset/generators/pull_request.go b/applicationset/generators/pull_request.go index 25f2067d08109..456e83f5a6a1f 100644 --- a/applicationset/generators/pull_request.go +++ b/applicationset/generators/pull_request.go @@ -215,7 +215,7 @@ func (g *PullRequestGenerator) github(ctx context.Context, cfg *argoprojiov1alph if err != nil { return nil, fmt.Errorf("error getting GitHub App secret: %w", err) } - return pullrequest.NewGithubAppService(*auth, cfg.API, cfg.Owner, cfg.Repo, cfg.Labels, g.cacheEnabled) + return pullrequest.NewGithubAppService(*auth, cfg.API, cfg.Owner, cfg.Repo, cfg.Labels, cfg.CachingEnabled) } // always default to token, even if not set (public access) diff --git a/go.mod b/go.mod index df80b876488c3..0593bc5a9c8c6 100644 --- a/go.mod +++ b/go.mod @@ -43,6 +43,7 @@ require ( github.com/golang/protobuf v1.5.4 github.com/google/btree v1.1.3 github.com/google/go-cmp v0.6.0 + github.com/google/go-github/v35 v35.3.0 github.com/google/go-github/v63 v63.0.0 github.com/google/go-jsonnet v0.20.0 github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 diff --git a/go.sum b/go.sum index a0b074c903cb9..d3bee45689dac 100644 --- a/go.sum +++ b/go.sum @@ -431,6 +431,8 @@ github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeN github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-github/v35 v35.3.0 h1:fU+WBzuukn0VssbayTT+Zo3/ESKX9JYWjbZTLOTEyho= +github.com/google/go-github/v35 v35.3.0/go.mod h1:yWB7uCcVWaUbUP74Aq3whuMySRMatyRmq5U9FTNlbio= github.com/google/go-github/v41 v41.0.0 h1:HseJrM2JFf2vfiZJ8anY2hqBjdfY1Vlj/K27ueww4gg= github.com/google/go-github/v41 v41.0.0/go.mod h1:XgmCA5H323A9rtgExdTcnDkcqp6S30AVACCBDOonIxg= github.com/google/go-github/v63 v63.0.0 h1:13xwK/wk9alSokujB9lJkuzdmQuVn2QCPeck76wR3nE= diff --git a/server/applicationset/applicationset.go b/server/applicationset/applicationset.go index b5288c71c1509..febd734d0aa68 100644 --- a/server/applicationset/applicationset.go +++ b/server/applicationset/applicationset.go @@ -264,8 +264,8 @@ func (s *Server) Create(ctx context.Context, q *applicationset.ApplicationSetCre func (s *Server) generateApplicationSetApps(ctx context.Context, logEntry *log.Entry, appset v1alpha1.ApplicationSet, namespace string) ([]v1alpha1.Application, error) { argoCDDB := s.db - - scmConfig := generators.NewSCMConfig(s.ScmRootCAPath, s.AllowedScmProviders, s.EnableScmProviders, github_app.NewAuthCredentials(argoCDDB.(db.RepoCredsDB)), true) + // TODO: this is a hack to get the github httpcache to load. + scmConfig := generators.NewSCMConfig(s.ScmRootCAPath, s.AllowedScmProviders, s.EnableScmProviders, github_app.NewAuthCredentials(argoCDDB.(db.RepoCredsDB)), true, true) getRepository := func(ctx context.Context, url, project string) (*v1alpha1.Repository, error) { return s.db.GetRepository(ctx, url, project) From 884ff175c1a264ed735b9978a5a4999205ba7ff0 Mon Sep 17 00:00:00 2001 From: "aburan28@gmail.com" Date: Sun, 22 Dec 2024 13:04:21 -0300 Subject: [PATCH 05/24] use new method for auth token Signed-off-by: aburan28@gmail.com Signed-off-by: aburan28 --- .../services/scm_provider/github.go | 32 ++++++++----------- 1 file changed, 14 insertions(+), 18 deletions(-) diff --git a/applicationset/services/scm_provider/github.go b/applicationset/services/scm_provider/github.go index be930e4e8b75c..84a4b58b6454c 100644 --- a/applicationset/services/scm_provider/github.go +++ b/applicationset/services/scm_provider/github.go @@ -8,7 +8,6 @@ import ( "github.com/google/go-github/v63/github" "github.com/gregjones/httpcache" - "golang.org/x/oauth2" ) type contextKey struct{} @@ -28,34 +27,31 @@ type GithubProvider struct { var _ SCMProviderService = &GithubProvider{} func NewGithubProvider(ctx context.Context, organization string, token string, url string, allBranches bool, cacheEnabled bool) (*GithubProvider, error) { - var ts oauth2.TokenSource // Undocumented environment variable to set a default token, to be used in testing to dodge anonymous rate limits. + var client *github.Client + if token == "" { token = os.Getenv("GITHUB_TOKEN") } - if token != "" { - ts = oauth2.StaticTokenSource( - &oauth2.Token{AccessToken: token}, - ) - if cache, ok := ctx.Value(cacheContextKey).(httpcache.Cache); ok { - ctx = context.WithValue(ctx, oauth2.HTTPClient, &http.Client{ - Transport: &httpcache.Transport{ - Cache: cache, - }, - }) - } - oauth2.NewClient(ctx, ts) + httpClient := &http.Client{} + if cacheEnabled { + cache := httpcache.NewMemoryCache() + httpClient = &http.Client{ + Transport: &httpcache.Transport{ + Cache: cache, + }, + } + } else { + httpClient = &http.Client{} } - if cacheEnabled { + if token != "" { + client.WithAuthToken(token) } - httpClient := oauth2.NewClient(ctx, ts) - - var client *github.Client if url == "" { client = github.NewClient(httpClient) } else { From a2ce309ddd985c7515b9af886f7b0531671e45a1 Mon Sep 17 00:00:00 2001 From: "aburan28@gmail.com" Date: Tue, 24 Dec 2024 14:35:32 -0300 Subject: [PATCH 06/24] hopefully good to pass some checks Signed-off-by: aburan28 --- applicationset/generators/pull_request.go | 4 +- applicationset/generators/scm_provider.go | 13 ++--- .../services/internal/github_app/client.go | 52 +++++++++---------- .../services/pull_request/github.go | 27 ++++++---- .../services/pull_request/github_app.go | 5 +- .../services/pull_request/github_test.go | 23 ++++++++ .../services/scm_provider/github.go | 9 ++-- .../services/scm_provider/github_app.go | 5 +- .../commands/applicationset_controller.go | 7 ++- go.mod | 5 +- go.sum | 8 ++- server/applicationset/applicationset.go | 2 +- 12 files changed, 97 insertions(+), 63 deletions(-) diff --git a/applicationset/generators/pull_request.go b/applicationset/generators/pull_request.go index 456e83f5a6a1f..1c9826804dfc5 100644 --- a/applicationset/generators/pull_request.go +++ b/applicationset/generators/pull_request.go @@ -215,7 +215,7 @@ func (g *PullRequestGenerator) github(ctx context.Context, cfg *argoprojiov1alph if err != nil { return nil, fmt.Errorf("error getting GitHub App secret: %w", err) } - return pullrequest.NewGithubAppService(*auth, cfg.API, cfg.Owner, cfg.Repo, cfg.Labels, cfg.CachingEnabled) + return pullrequest.NewGithubAppService(*auth, cfg.API, cfg.Owner, cfg.Repo, cfg.Labels, g.SCMConfig.GitHubClientCache) } // always default to token, even if not set (public access) @@ -223,5 +223,5 @@ func (g *PullRequestGenerator) github(ctx context.Context, cfg *argoprojiov1alph if err != nil { return nil, fmt.Errorf("error fetching Secret token: %w", err) } - return pullrequest.NewGithubService(ctx, token, cfg.API, cfg.Owner, cfg.Repo, cfg.Labels) + return pullrequest.NewGithubService(ctx, token, cfg.API, cfg.Owner, cfg.Repo, cfg.Labels, g.SCMConfig.GitHubClientCache) } diff --git a/applicationset/generators/scm_provider.go b/applicationset/generators/scm_provider.go index 5f33b64475ec2..0136f809d89b8 100644 --- a/applicationset/generators/scm_provider.go +++ b/applicationset/generators/scm_provider.go @@ -7,7 +7,8 @@ import ( "strings" "time" - "github.com/gregjones/httpcache" + "github.com/aburan28/httpcache" + "github.com/aburan28/httpcache/lrucache" "sigs.k8s.io/controller-runtime/pkg/client" @@ -41,7 +42,7 @@ type SCMConfig struct { tokenRefStrictMode bool } -func NewSCMConfig(scmRootCAPath string, allowedSCMProviders []string, enableSCMProviders bool, gitHubApps github_app_auth.Credentials, tokenRefStrictMode bool, cacheEnabled bool) SCMConfig { +func NewSCMConfig(scmRootCAPath string, allowedSCMProviders []string, enableSCMProviders bool, gitHubApps github_app_auth.Credentials, tokenRefStrictMode bool, enableGithubCache bool, githubCacheSize int) SCMConfig { scmConfig := SCMConfig{ scmRootCAPath: scmRootCAPath, allowedSCMProviders: allowedSCMProviders, @@ -50,8 +51,8 @@ func NewSCMConfig(scmRootCAPath string, allowedSCMProviders []string, enableSCMP tokenRefStrictMode: tokenRefStrictMode, } - if cacheEnabled { - scmConfig.GitHubClientCache = httpcache.NewMemoryCache() + if enableGithubCache { + scmConfig.GitHubClientCache = lrucache.NewLRUCache(githubCacheSize) } return scmConfig @@ -291,7 +292,7 @@ func (g *SCMProviderGenerator) githubProvider(ctx context.Context, github *argop github.Organization, github.API, github.AllBranches, - github.CachingEnabled, + g.GitHubClientCache, ) } @@ -299,5 +300,5 @@ func (g *SCMProviderGenerator) githubProvider(ctx context.Context, github *argop if err != nil { return nil, fmt.Errorf("error fetching Github token: %w", err) } - return scm_provider.NewGithubProvider(ctx, github.Organization, token, github.API, github.AllBranches, github.CachingEnabled) + return scm_provider.NewGithubProvider(ctx, github.Organization, token, github.API, github.AllBranches, g.GitHubClientCache) } diff --git a/applicationset/services/internal/github_app/client.go b/applicationset/services/internal/github_app/client.go index adb9296bde002..668f0a1506628 100644 --- a/applicationset/services/internal/github_app/client.go +++ b/applicationset/services/internal/github_app/client.go @@ -4,46 +4,46 @@ import ( "fmt" "net/http" + "github.com/aburan28/httpcache" "github.com/bradleyfalzon/ghinstallation/v2" "github.com/google/go-github/v63/github" - "github.com/gregjones/httpcache" "github.com/argoproj/argo-cd/v2/applicationset/services/github_app_auth" ) // Client builds a github client for the given app authentication. -func Client(g github_app_auth.Authentication, url string, cacheEnabled bool) (*github.Client, error) { - rt, err := ghinstallation.New(http.DefaultTransport, g.Id, g.InstallationId, []byte(g.PrivateKey)) - if err != nil { - return nil, fmt.Errorf("failed to create github app install: %w", err) - } - if url == "" { - url = g.EnterpriseBaseURL - } +func Client(g github_app_auth.Authentication, url string, cache httpcache.Cache) (*github.Client, error) { + var httpClient *http.Client + var err error var client *github.Client - // determine if the http client should use a cache - if url == "" { - httpClient := http.Client{Transport: rt} - - client = github.NewClient(&httpClient) - } else if cacheEnabled { - cache := httpcache.NewMemoryCache() - cachingHttpClient := http.Client{ - Transport: &httpcache.Transport{ - Cache: cache, - }, + if cache != nil { + tr := httpcache.NewTransport(cache) + at, err := ghinstallation.NewAppsTransport(tr, g.Id, []byte(g.PrivateKey)) + if err != nil { + return nil, fmt.Errorf("failed to create github app transport: %w", err) + } + + httpClient = &http.Client{ + Transport: at, } - client, err = github.NewClient(&cachingHttpClient).WithEnterpriseURLs(url, url) + } else { + rt, err := ghinstallation.New(http.DefaultTransport, g.Id, g.InstallationId, []byte(g.PrivateKey)) if err != nil { - return nil, fmt.Errorf("failed to create http cache client: %w", err) + return nil, fmt.Errorf("failed to create github app install: %w", err) + } + httpClient = &http.Client{ + Transport: rt, } + + } + + if url == "" { + client = github.NewClient(httpClient) } else { - rt.BaseURL = url - httpClient := http.Client{Transport: rt} - client, err = github.NewClient(&httpClient).WithEnterpriseURLs(url, url) + client, err = github.NewClient(httpClient).WithEnterpriseURLs(url, url) if err != nil { - return nil, fmt.Errorf("failed to create github enterprise client: %w", err) + return nil, fmt.Errorf("failed to create github client: %w", err) } } return client, nil diff --git a/applicationset/services/pull_request/github.go b/applicationset/services/pull_request/github.go index b63f2a9de6a8e..07c41b247c7a8 100644 --- a/applicationset/services/pull_request/github.go +++ b/applicationset/services/pull_request/github.go @@ -3,10 +3,11 @@ package pull_request import ( "context" "fmt" + "net/http" "os" + "github.com/aburan28/httpcache" "github.com/google/go-github/v63/github" - "golang.org/x/oauth2" ) type GithubService struct { @@ -18,21 +19,27 @@ type GithubService struct { var _ PullRequestService = (*GithubService)(nil) -func NewGithubService(ctx context.Context, token, url, owner, repo string, labels []string) (PullRequestService, error) { - var ts oauth2.TokenSource +func NewGithubService(ctx context.Context, token, url, owner, repo string, labels []string, cache httpcache.Cache) (PullRequestService, error) { + // var ts oauth2.TokenSource // Undocumented environment variable to set a default token, to be used in testing to dodge anonymous rate limits. if token == "" { token = os.Getenv("GITHUB_TOKEN") } - if token != "" { - ts = oauth2.StaticTokenSource( - &oauth2.Token{AccessToken: token}, - ) - } - httpClient := oauth2.NewClient(ctx, ts) + var httpClient *http.Client var client *github.Client + + if cache != nil { + httpClient = &http.Client{ + Transport: &httpcache.Transport{ + Cache: cache, + }, + } + } else { + httpClient = &http.Client{} + } + if url == "" { - client = github.NewClient(httpClient) + client = github.NewClient(httpClient).WithAuthToken(token) } else { var err error client, err = github.NewClient(httpClient).WithEnterpriseURLs(url, url) diff --git a/applicationset/services/pull_request/github_app.go b/applicationset/services/pull_request/github_app.go index 6705b2b829fd0..1b2f5f65332a6 100644 --- a/applicationset/services/pull_request/github_app.go +++ b/applicationset/services/pull_request/github_app.go @@ -1,12 +1,13 @@ package pull_request import ( + "github.com/aburan28/httpcache" "github.com/argoproj/argo-cd/v2/applicationset/services/github_app_auth" "github.com/argoproj/argo-cd/v2/applicationset/services/internal/github_app" ) -func NewGithubAppService(g github_app_auth.Authentication, url, owner, repo string, labels []string, cacheEnabled bool) (PullRequestService, error) { - client, err := github_app.Client(g, url, cacheEnabled) +func NewGithubAppService(g github_app_auth.Authentication, url, owner, repo string, labels []string, cache httpcache.Cache) (PullRequestService, error) { + client, err := github_app.Client(g, url, cache) if err != nil { return nil, err } diff --git a/applicationset/services/pull_request/github_test.go b/applicationset/services/pull_request/github_test.go index d68bcd8f124de..e223c52cbf40f 100644 --- a/applicationset/services/pull_request/github_test.go +++ b/applicationset/services/pull_request/github_test.go @@ -1,6 +1,7 @@ package pull_request import ( + "context" "testing" "github.com/google/go-github/v63/github" @@ -11,6 +12,28 @@ func toPtr(s string) *string { return &s } +func TestGithubToken(t *testing.T) { + ctx := context.Background() + testCases := []struct { + name string + token string + }{ + { + name: "No token", + }, + { + name: "With token", + token: "test", + }, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + _, err := NewGithubService(ctx, tc.token, "", "test", "test", []string{}) + require.NoError(t, err) + }) + } +} + func TestContainLabels(t *testing.T) { cases := []struct { Name string diff --git a/applicationset/services/scm_provider/github.go b/applicationset/services/scm_provider/github.go index 84a4b58b6454c..4108b8a112b09 100644 --- a/applicationset/services/scm_provider/github.go +++ b/applicationset/services/scm_provider/github.go @@ -6,8 +6,8 @@ import ( "net/http" "os" + "github.com/aburan28/httpcache" "github.com/google/go-github/v63/github" - "github.com/gregjones/httpcache" ) type contextKey struct{} @@ -26,7 +26,7 @@ type GithubProvider struct { var _ SCMProviderService = &GithubProvider{} -func NewGithubProvider(ctx context.Context, organization string, token string, url string, allBranches bool, cacheEnabled bool) (*GithubProvider, error) { +func NewGithubProvider(ctx context.Context, organization string, token string, url string, allBranches bool, cache httpcache.Cache) (*GithubProvider, error) { // Undocumented environment variable to set a default token, to be used in testing to dodge anonymous rate limits. var client *github.Client @@ -35,9 +35,7 @@ func NewGithubProvider(ctx context.Context, organization string, token string, u } httpClient := &http.Client{} - if cacheEnabled { - cache := httpcache.NewMemoryCache() - + if cache != nil { httpClient = &http.Client{ Transport: &httpcache.Transport{ Cache: cache, @@ -49,7 +47,6 @@ func NewGithubProvider(ctx context.Context, organization string, token string, u if token != "" { client.WithAuthToken(token) - } if url == "" { diff --git a/applicationset/services/scm_provider/github_app.go b/applicationset/services/scm_provider/github_app.go index 16db0ad010347..286ff22399313 100644 --- a/applicationset/services/scm_provider/github_app.go +++ b/applicationset/services/scm_provider/github_app.go @@ -1,12 +1,13 @@ package scm_provider import ( + "github.com/aburan28/httpcache" "github.com/argoproj/argo-cd/v2/applicationset/services/github_app_auth" "github.com/argoproj/argo-cd/v2/applicationset/services/internal/github_app" ) -func NewGithubAppProviderFor(g github_app_auth.Authentication, organization string, url string, allBranches bool, cacheEnabled bool) (*GithubProvider, error) { - client, err := github_app.Client(g, url, cacheEnabled) +func NewGithubAppProviderFor(g github_app_auth.Authentication, organization string, url string, allBranches bool, cache httpcache.Cache) (*GithubProvider, error) { + client, err := github_app.Client(g, url, cache) if err != nil { return nil, err } diff --git a/cmd/argocd-applicationset-controller/commands/applicationset_controller.go b/cmd/argocd-applicationset-controller/commands/applicationset_controller.go index c68a88341687e..a47aa7c611074 100644 --- a/cmd/argocd-applicationset-controller/commands/applicationset_controller.go +++ b/cmd/argocd-applicationset-controller/commands/applicationset_controller.go @@ -54,7 +54,7 @@ func NewCommand() *cobra.Command { webhookAddr string enableLeaderElection bool enableGithubCache bool - githubCacheSize int64 + githubCacheSize int applicationSetNamespaces []string argocdRepoServer string policy string @@ -166,7 +166,7 @@ func NewCommand() *cobra.Command { argoSettingsMgr := argosettings.NewSettingsManager(ctx, k8sClient, namespace) argoCDDB := db.NewDB(namespace, argoSettingsMgr, k8sClient) - scmConfig := generators.NewSCMConfig(scmRootCAPath, allowedScmProviders, enableScmProviders, github_app.NewAuthCredentials(argoCDDB.(db.RepoCredsDB)), tokenRefStrictMode, enableGithubCache) + scmConfig := generators.NewSCMConfig(scmRootCAPath, allowedScmProviders, enableScmProviders, github_app.NewAuthCredentials(argoCDDB.(db.RepoCredsDB)), tokenRefStrictMode, enableGithubCache, githubCacheSize) tlsConfig := apiclient.TLSConfiguration{ DisableTLS: repoServerPlaintext, @@ -265,8 +265,7 @@ func NewCommand() *cobra.Command { command.Flags().IntVar(&webhookParallelism, "webhook-parallelism-limit", env.ParseNumFromEnv("ARGOCD_APPLICATIONSET_CONTROLLER_WEBHOOK_PARALLELISM_LIMIT", 50, 1, 1000), "Number of webhook requests processed concurrently") command.Flags().StringSliceVar(&metricsAplicationsetLabels, "metrics-applicationset-labels", []string{}, "List of Application labels that will be added to the argocd_applicationset_labels metric") command.Flags().BoolVar(&enableGithubCache, "enabled-github-cache", env.ParseBoolFromEnv("ARGOCD_APPLICATIONSET_CONTROLLER_ENABLED_GITHUB_CACHE", false), "Enable caching for GitHub client") - command.Flags().Int64Var(&githubCacheSize, "github-cache-size", env.ParseInt64FromEnv("ARGOCD_APPLICATIONSET_CONTROLLER_GITHUB_CACHE_SIZE", 0, 0, 0), "Max size of the cache for GitHub client in kb") - + command.Flags().IntVar(&githubCacheSize, "github-cache-size", env.ParseNumFromEnv("ARGOCD_APPLICATIONSET_CONTROLLER_GITHUB_CACHE_SIZE", 5000, 1000, 100000), "Max items to hold in the cache for github") return &command } diff --git a/go.mod b/go.mod index 0593bc5a9c8c6..ab325c55294dc 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/argoproj/argo-cd/v2 -go 1.22.0 +go 1.23.3 require ( code.gitea.io/sdk/gitea v0.19.0 @@ -8,6 +8,7 @@ require ( github.com/Masterminds/semver/v3 v3.3.1 github.com/Masterminds/sprig/v3 v3.3.0 github.com/TomOnTime/utfutil v0.0.0-20180511104225-09c41003ee1d + github.com/aburan28/httpcache v0.0.1 github.com/alicebob/miniredis/v2 v2.33.0 github.com/antonmedv/expr v1.15.1 github.com/argoproj/gitops-engine v0.7.1-0.20241107145828-847cfc9f8b20 @@ -43,7 +44,6 @@ require ( github.com/golang/protobuf v1.5.4 github.com/google/btree v1.1.3 github.com/google/go-cmp v0.6.0 - github.com/google/go-github/v35 v35.3.0 github.com/google/go-github/v63 v63.0.0 github.com/google/go-jsonnet v0.20.0 github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 @@ -144,6 +144,7 @@ require ( github.com/google/s2a-go v0.1.7 // indirect github.com/googleapis/enterprise-certificate-proxy v0.2.5 // indirect github.com/googleapis/gax-go/v2 v2.12.0 // indirect + github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect github.com/kylelemons/godebug v1.1.0 // indirect github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect diff --git a/go.sum b/go.sum index d3bee45689dac..f5d3f36b18ff0 100644 --- a/go.sum +++ b/go.sum @@ -66,6 +66,8 @@ github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMx github.com/TomOnTime/utfutil v0.0.0-20180511104225-09c41003ee1d h1:WtAMR0fPCOfK7TPGZ8ZpLLY18HRvL7XJ3xcs0wnREgo= github.com/TomOnTime/utfutil v0.0.0-20180511104225-09c41003ee1d/go.mod h1:WML6KOYjeU8N6YyusMjj2qRvaPNUEvrQvaxuFcMRFJY= github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g= +github.com/aburan28/httpcache v0.0.1 h1:Ow0+bnxO0KYeF4CpU8E9RaASrfcfW0dw36uy/Dou7uE= +github.com/aburan28/httpcache v0.0.1/go.mod h1:9l7da/RFyyILtLd4HoDzCpk2Dl2F5AEvuHx5APrFd/M= github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= @@ -431,8 +433,6 @@ github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeN github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/go-github/v35 v35.3.0 h1:fU+WBzuukn0VssbayTT+Zo3/ESKX9JYWjbZTLOTEyho= -github.com/google/go-github/v35 v35.3.0/go.mod h1:yWB7uCcVWaUbUP74Aq3whuMySRMatyRmq5U9FTNlbio= github.com/google/go-github/v41 v41.0.0 h1:HseJrM2JFf2vfiZJ8anY2hqBjdfY1Vlj/K27ueww4gg= github.com/google/go-github/v41 v41.0.0/go.mod h1:XgmCA5H323A9rtgExdTcnDkcqp6S30AVACCBDOonIxg= github.com/google/go-github/v63 v63.0.0 h1:13xwK/wk9alSokujB9lJkuzdmQuVn2QCPeck76wR3nE= @@ -527,6 +527,8 @@ github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09 github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= +github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= @@ -930,6 +932,8 @@ github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/tj/assert v0.0.3 h1:Df/BlaZ20mq6kuai7f5z2TvPFiwC3xaWJSDQNiIS3Rk= +github.com/tj/assert v0.0.3/go.mod h1:Ne6X72Q+TB1AteidzQncjw9PabbMp4PBMZ1k+vd1Pvk= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo= github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= diff --git a/server/applicationset/applicationset.go b/server/applicationset/applicationset.go index febd734d0aa68..765e36615c285 100644 --- a/server/applicationset/applicationset.go +++ b/server/applicationset/applicationset.go @@ -265,7 +265,7 @@ func (s *Server) Create(ctx context.Context, q *applicationset.ApplicationSetCre func (s *Server) generateApplicationSetApps(ctx context.Context, logEntry *log.Entry, appset v1alpha1.ApplicationSet, namespace string) ([]v1alpha1.Application, error) { argoCDDB := s.db // TODO: this is a hack to get the github httpcache to load. - scmConfig := generators.NewSCMConfig(s.ScmRootCAPath, s.AllowedScmProviders, s.EnableScmProviders, github_app.NewAuthCredentials(argoCDDB.(db.RepoCredsDB)), true, true) + scmConfig := generators.NewSCMConfig(s.ScmRootCAPath, s.AllowedScmProviders, s.EnableScmProviders, github_app.NewAuthCredentials(argoCDDB.(db.RepoCredsDB)), true, true, 10000) getRepository := func(ctx context.Context, url, project string) (*v1alpha1.Repository, error) { return s.db.GetRepository(ctx, url, project) From 1cae1b9ee25142d3371b8c8dc05b8382ea84fe27 Mon Sep 17 00:00:00 2001 From: aburan28 Date: Tue, 24 Dec 2024 14:50:20 -0300 Subject: [PATCH 07/24] remove caching enabled api field Signed-off-by: aburan28 --- pkg/apis/application/v1alpha1/applicationset_types.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/pkg/apis/application/v1alpha1/applicationset_types.go b/pkg/apis/application/v1alpha1/applicationset_types.go index 801be1cccab8f..16f9eeecd5d85 100644 --- a/pkg/apis/application/v1alpha1/applicationset_types.go +++ b/pkg/apis/application/v1alpha1/applicationset_types.go @@ -487,8 +487,6 @@ type SCMProviderGeneratorGithub struct { AppSecretName string `json:"appSecretName,omitempty" protobuf:"bytes,4,opt,name=appSecretName"` // Scan all branches instead of just the default branch. AllBranches bool `json:"allBranches,omitempty" protobuf:"varint,5,opt,name=allBranches"` - // should caching be enabled for github requests - CachingEnabled bool `json:"cachingEnabled,omitempty" protobuf:"varint,6,opt,name=cachingEnabled"` } // SCMProviderGeneratorGitlab defines connection info specific to Gitlab. @@ -680,8 +678,6 @@ type PullRequestGeneratorGithub struct { AppSecretName string `json:"appSecretName,omitempty" protobuf:"bytes,5,opt,name=appSecretName"` // Labels is used to filter the PRs that you want to target Labels []string `json:"labels,omitempty" protobuf:"bytes,6,rep,name=labels"` - // should caching be enabled for github requests - CachingEnabled bool `json:"cachingEnabled,omitempty" protobuf:"varint,6,opt,name=cachingEnabled"` } // PullRequestGeneratorGitLab defines connection info specific to GitLab. From 37b9a00aa9cc830affe104393f8a0a772d43c9eb Mon Sep 17 00:00:00 2001 From: Adam Buran Date: Tue, 24 Dec 2024 14:54:30 -0300 Subject: [PATCH 08/24] Update go.mod Signed-off-by: Adam Buran --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index ad7c25b990b5e..2f674bc48481a 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/argoproj/argo-cd/v2 -go 1.23.3 +go 1.22.0 require ( code.gitea.io/sdk/gitea v0.19.0 From bfd020666aa7bbac87ca802c6405d315ed9d71b2 Mon Sep 17 00:00:00 2001 From: aburan28 Date: Tue, 24 Dec 2024 14:59:44 -0300 Subject: [PATCH 09/24] remove unused dep oauth2 Signed-off-by: aburan28 --- applicationset/services/pull_request/github.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/applicationset/services/pull_request/github.go b/applicationset/services/pull_request/github.go index 1b3fc0a85764e..5425ad43b6b78 100644 --- a/applicationset/services/pull_request/github.go +++ b/applicationset/services/pull_request/github.go @@ -6,8 +6,8 @@ import ( "net/http" "os" + "github.com/aburan28/httpcache" "github.com/google/go-github/v66/github" - "golang.org/x/oauth2" ) type GithubService struct { From d43e8586f6a6b3d23375dbf3e676b65dace069cd Mon Sep 17 00:00:00 2001 From: aburan28 Date: Tue, 24 Dec 2024 15:06:59 -0300 Subject: [PATCH 10/24] reorder go.mod Signed-off-by: aburan28 --- go.mod | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 2f674bc48481a..bd73a3e89992f 100644 --- a/go.mod +++ b/go.mod @@ -25,7 +25,7 @@ require ( github.com/coreos/go-oidc/v3 v3.11.0 github.com/cyphar/filepath-securejoin v0.3.6 github.com/dustin/go-humanize v1.0.1 - github.com/evanphx/json-patch v5.9.0+incompatible + github.com/evanphx/json-patch v5.9.0+incompatibgolanle github.com/expr-lang/expr v1.16.9 github.com/felixge/httpsnoop v1.0.4 github.com/fsnotify/fsnotify v1.8.0 @@ -141,9 +141,9 @@ require ( github.com/golang-jwt/jwt/v5 v5.2.1 // indirect github.com/google/gnostic-models v0.6.8 // indirect github.com/google/s2a-go v0.1.7 // indirect - github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect github.com/googleapis/gax-go/v2 v2.12.3 // indirect + github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect github.com/kylelemons/godebug v1.1.0 // indirect github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect From 28d2f6e8a828cac35e575941b2bbe3112c4258c2 Mon Sep 17 00:00:00 2001 From: aburan28 Date: Tue, 24 Dec 2024 15:07:25 -0300 Subject: [PATCH 11/24] reorder go.mod Signed-off-by: aburan28 --- go.mod | 1 - 1 file changed, 1 deletion(-) diff --git a/go.mod b/go.mod index bd73a3e89992f..7168685bc255c 100644 --- a/go.mod +++ b/go.mod @@ -218,7 +218,6 @@ require ( github.com/google/gofuzz v1.2.0 // indirect github.com/gosimple/unidecode v1.0.1 // indirect github.com/gregdel/pushover v1.2.1 // indirect - github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 github.com/grpc-ecosystem/grpc-gateway/v2 v2.23.0 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect github.com/hashicorp/go-version v1.6.0 // indirect From c6e58afd46612df571feb455156644d8277525db Mon Sep 17 00:00:00 2001 From: aburan28 Date: Tue, 7 Jan 2025 13:22:15 -0300 Subject: [PATCH 12/24] add enableGithubCache and maxCacheSize to server cli Signed-off-by: aburan28 --- cmd/argocd-server/commands/argocd_server.go | 2 ++ go.mod | 3 ++- server/applicationset/applicationset.go | 6 ++++-- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/cmd/argocd-server/commands/argocd_server.go b/cmd/argocd-server/commands/argocd_server.go index 403cfdae16633..280d1597f5639 100644 --- a/cmd/argocd-server/commands/argocd_server.go +++ b/cmd/argocd-server/commands/argocd_server.go @@ -330,6 +330,8 @@ func NewCommand() *cobra.Command { command.Flags().BoolVar(&enableScmProviders, "appset-enable-scm-providers", env.ParseBoolFromEnv("ARGOCD_APPLICATIONSET_CONTROLLER_ENABLE_SCM_PROVIDERS", true), "Enable retrieving information from SCM providers, used by the SCM and PR generators (Default: true)") command.Flags().StringSliceVar(&allowedScmProviders, "appset-allowed-scm-providers", env.StringsFromEnv("ARGOCD_APPLICATIONSET_CONTROLLER_ALLOWED_SCM_PROVIDERS", []string{}, ","), "The list of allowed custom SCM provider API URLs. This restriction does not apply to SCM or PR generators which do not accept a custom API URL. (Default: Empty = all)") command.Flags().BoolVar(&enableNewGitFileGlobbing, "appset-enable-new-git-file-globbing", env.ParseBoolFromEnv("ARGOCD_APPLICATIONSET_CONTROLLER_ENABLE_NEW_GIT_FILE_GLOBBING", false), "Enable new globbing in Git files generator.") + command.Flags().BoolVar(&enableGithubCache, "enabled-github-cache", env.ParseBoolFromEnv("ARGOCD_APPLICATIONSET_CONTROLLER_ENABLED_GITHUB_CACHE", false), "Enable caching for GitHub client") + command.Flags().IntVar(&githubCacheSize, "github-cache-size", env.ParseNumFromEnv("ARGOCD_APPLICATIONSET_CONTROLLER_GITHUB_CACHE_SIZE", 5000, 1000, 100000), "Max items to hold in the cache for github") tlsConfigCustomizerSrc = tls.AddTLSFlagsToCmd(command) cacheSrc = servercache.AddCacheFlagsToCmd(command, cacheutil.Options{ diff --git a/go.mod b/go.mod index 7168685bc255c..61362256d30f9 100644 --- a/go.mod +++ b/go.mod @@ -25,7 +25,7 @@ require ( github.com/coreos/go-oidc/v3 v3.11.0 github.com/cyphar/filepath-securejoin v0.3.6 github.com/dustin/go-humanize v1.0.1 - github.com/evanphx/json-patch v5.9.0+incompatibgolanle + github.com/evanphx/json-patch v5.9.0+incompatible github.com/expr-lang/expr v1.16.9 github.com/felixge/httpsnoop v1.0.4 github.com/fsnotify/fsnotify v1.8.0 @@ -143,6 +143,7 @@ require ( github.com/google/s2a-go v0.1.7 // indirect github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect github.com/googleapis/gax-go/v2 v2.12.3 // indirect + github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 // indirect github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect github.com/kylelemons/godebug v1.1.0 // indirect github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect diff --git a/server/applicationset/applicationset.go b/server/applicationset/applicationset.go index 765e36615c285..32a1d7e4c3fb4 100644 --- a/server/applicationset/applicationset.go +++ b/server/applicationset/applicationset.go @@ -65,6 +65,8 @@ type Server struct { ScmRootCAPath string AllowedScmProviders []string EnableScmProviders bool + EnableGithubCache bool + GithubCacheSize int } // NewServer returns a new instance of the ApplicationSet service @@ -264,8 +266,8 @@ func (s *Server) Create(ctx context.Context, q *applicationset.ApplicationSetCre func (s *Server) generateApplicationSetApps(ctx context.Context, logEntry *log.Entry, appset v1alpha1.ApplicationSet, namespace string) ([]v1alpha1.Application, error) { argoCDDB := s.db - // TODO: this is a hack to get the github httpcache to load. - scmConfig := generators.NewSCMConfig(s.ScmRootCAPath, s.AllowedScmProviders, s.EnableScmProviders, github_app.NewAuthCredentials(argoCDDB.(db.RepoCredsDB)), true, true, 10000) + + scmConfig := generators.NewSCMConfig(s.ScmRootCAPath, s.AllowedScmProviders, s.EnableScmProviders, github_app.NewAuthCredentials(argoCDDB.(db.RepoCredsDB)), true, s.EnableGithubCache, s.GithubCacheSize) getRepository := func(ctx context.Context, url, project string) (*v1alpha1.Repository, error) { return s.db.GetRepository(ctx, url, project) From a4d12819f714e16369f4c5897f7b489cdd909265 Mon Sep 17 00:00:00 2001 From: aburan28 Date: Tue, 7 Jan 2025 15:23:14 -0300 Subject: [PATCH 13/24] update to add missing parameter Signed-off-by: aburan28 --- cmd/argocd-server/commands/argocd_server.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cmd/argocd-server/commands/argocd_server.go b/cmd/argocd-server/commands/argocd_server.go index 2aeaa0496c6ee..caf8736ec55a8 100644 --- a/cmd/argocd-server/commands/argocd_server.go +++ b/cmd/argocd-server/commands/argocd_server.go @@ -94,6 +94,8 @@ func NewCommand() *cobra.Command { scmRootCAPath string allowedScmProviders []string enableScmProviders bool + enableGithubCache bool + githubCacheSize int // argocd k8s event logging flag enableK8sEvent []string From a3ebc5e0c9d7ccdcd57bf6f4c469ea832b24613c Mon Sep 17 00:00:00 2001 From: aburan28 Date: Mon, 13 Jan 2025 14:44:37 -0300 Subject: [PATCH 14/24] add missing parameters Signed-off-by: aburan28 --- .../commands/applicationset_controller.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/argocd-applicationset-controller/commands/applicationset_controller.go b/cmd/argocd-applicationset-controller/commands/applicationset_controller.go index 4eb740d9557bf..0eb6da665ccce 100644 --- a/cmd/argocd-applicationset-controller/commands/applicationset_controller.go +++ b/cmd/argocd-applicationset-controller/commands/applicationset_controller.go @@ -271,7 +271,7 @@ func NewCommand() *cobra.Command { command.Flags().StringSliceVar(&globalPreservedLabels, "preserved-labels", env.StringsFromEnv("ARGOCD_APPLICATIONSET_CONTROLLER_GLOBAL_PRESERVED_LABELS", []string{}, ","), "Sets global preserved field values for labels") command.Flags().IntVar(&webhookParallelism, "webhook-parallelism-limit", env.ParseNumFromEnv("ARGOCD_APPLICATIONSET_CONTROLLER_WEBHOOK_PARALLELISM_LIMIT", 50, 1, 1000), "Number of webhook requests processed concurrently") command.Flags().StringSliceVar(&metricsAplicationsetLabels, "metrics-applicationset-labels", []string{}, "List of Application labels that will be added to the argocd_applicationset_labels metric") - command.Flags().BoolVar(&enableGithubCache, "enabled-github-cache", env.ParseBoolFromEnv("ARGOCD_APPLICATIONSET_CONTROLLER_ENABLED_GITHUB_CACHE", false), "Enable caching for GitHub client") + command.Flags().BoolVar(&enableGithubCache, "enable-github-cache", env.ParseBoolFromEnv("ARGOCD_APPLICATIONSET_CONTROLLER_ENABLE_GITHUB_CACHE", false), "Enable caching for GitHub client") command.Flags().IntVar(&githubCacheSize, "github-cache-size", env.ParseNumFromEnv("ARGOCD_APPLICATIONSET_CONTROLLER_GITHUB_CACHE_SIZE", 5000, 1000, 100000), "Max items to hold in the cache for github") return &command } From 814cbdcca578ccf7cf29617de25b23c04432853e Mon Sep 17 00:00:00 2001 From: aburan28 Date: Mon, 13 Jan 2025 14:48:14 -0300 Subject: [PATCH 15/24] remove unneeded params Signed-off-by: aburan28 --- applicationset/services/scm_provider/github.go | 8 -------- 1 file changed, 8 deletions(-) diff --git a/applicationset/services/scm_provider/github.go b/applicationset/services/scm_provider/github.go index 0b43386ada670..3bdd2237c2c4a 100644 --- a/applicationset/services/scm_provider/github.go +++ b/applicationset/services/scm_provider/github.go @@ -10,14 +10,6 @@ import ( "github.com/google/go-github/v66/github" ) -type contextKey struct{} - -var cacheContextKey = contextKey{} - -func ContextWithGithubCache(ctx context.Context, cache httpcache.Cache) context.Context { - return context.WithValue(ctx, cacheContextKey, cache) -} - type GithubProvider struct { client *github.Client organization string From a2d6c40db41c62de35233ff236fc685d1c40b2d9 Mon Sep 17 00:00:00 2001 From: aburan28 Date: Mon, 13 Jan 2025 14:52:02 -0300 Subject: [PATCH 16/24] fix up go.mod Signed-off-by: aburan28 --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 700161670fcce..878ce368b27fc 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/argoproj/argo-cd/v3 -go 1.22.0 +go 1.23.3 require ( code.gitea.io/sdk/gitea v0.20.0 From 3f24cc0b574e89033276b6ba5201188052f8e8ca Mon Sep 17 00:00:00 2001 From: aburan28 Date: Mon, 13 Jan 2025 15:17:22 -0300 Subject: [PATCH 17/24] clean up tests Signed-off-by: aburan28 --- applicationset/controllers/requeue_after_test.go | 2 +- applicationset/services/scm_provider/github_test.go | 10 +++++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/applicationset/controllers/requeue_after_test.go b/applicationset/controllers/requeue_after_test.go index 6ecb792b18eee..7cc836ff24fa6 100644 --- a/applicationset/controllers/requeue_after_test.go +++ b/applicationset/controllers/requeue_after_test.go @@ -57,7 +57,7 @@ func TestRequeueAfter(t *testing.T) { }, } fakeDynClient := dynfake.NewSimpleDynamicClientWithCustomListKinds(runtime.NewScheme(), gvrToListKind, duckType) - scmConfig := generators.NewSCMConfig("", []string{""}, true, nil, true) + scmConfig := generators.NewSCMConfig("", []string{""}, true, nil, true, false, 0) terminalGenerators := map[string]generators.Generator{ "List": generators.NewListGenerator(), "Clusters": generators.NewClusterGenerator(ctx, k8sClient, appClientset, "argocd"), diff --git a/applicationset/services/scm_provider/github_test.go b/applicationset/services/scm_provider/github_test.go index f1fb0e412abb6..006fffe61add3 100644 --- a/applicationset/services/scm_provider/github_test.go +++ b/applicationset/services/scm_provider/github_test.go @@ -7,6 +7,7 @@ import ( "net/http/httptest" "testing" + "github.com/aburan28/httpcache" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -242,8 +243,9 @@ func TestGithubListRepos(t *testing.T) { })) defer ts.Close() for _, c := range cases { + t.Run(c.name, func(t *testing.T) { - provider, _ := NewGithubProvider(context.Background(), "argoproj", "", ts.URL, c.allBranches) + provider, _ := NewGithubProvider(context.Background(), "argoproj", "", ts.URL, c.allBranches, cache) rawRepos, err := ListRepos(context.Background(), provider, c.filters, c.proto) if c.hasError { require.Error(t, err) @@ -273,7 +275,9 @@ func TestGithubHasPath(t *testing.T) { githubMockHandler(t)(w, r) })) defer ts.Close() - host, _ := NewGithubProvider(context.Background(), "argoproj", "", ts.URL, false) + cache := httpcache.Cache{} + + host, _ := NewGithubProvider(context.Background(), "argoproj", "", ts.URL, false, cache) repo := &Repository{ Organization: "argoproj", Repository: "argo-cd", @@ -293,7 +297,7 @@ func TestGithubGetBranches(t *testing.T) { githubMockHandler(t)(w, r) })) defer ts.Close() - host, _ := NewGithubProvider(context.Background(), "argoproj", "", ts.URL, false) + host, _ := NewGithubProvider("argoproj", "", ts.URL, false) repo := &Repository{ Organization: "argoproj", Repository: "argo-cd", From e7bda20db03a1d2d14c4feff26e9b0dbd378a645 Mon Sep 17 00:00:00 2001 From: aburan28 Date: Mon, 13 Jan 2025 15:17:58 -0300 Subject: [PATCH 18/24] clean up tests Signed-off-by: aburan28 --- applicationset/generators/pull_request_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/applicationset/generators/pull_request_test.go b/applicationset/generators/pull_request_test.go index 5bde48ea1695d..b61a3106cb43d 100644 --- a/applicationset/generators/pull_request_test.go +++ b/applicationset/generators/pull_request_test.go @@ -283,7 +283,7 @@ func TestAllowedSCMProviderPullRequest(t *testing.T) { "gitea.myorg.com", "bitbucket.myorg.com", "azuredevops.myorg.com", - }, true, nil, true)) + }, true, nil, true, false, 0)) applicationSetInfo := argoprojiov1alpha1.ApplicationSet{ ObjectMeta: metav1.ObjectMeta{ @@ -306,7 +306,7 @@ func TestAllowedSCMProviderPullRequest(t *testing.T) { } func TestSCMProviderDisabled_PRGenerator(t *testing.T) { - generator := NewPullRequestGenerator(nil, NewSCMConfig("", []string{}, false, nil, true)) + generator := NewPullRequestGenerator(nil, NewSCMConfig("", []string{}, false, nil, true, false, 0)) applicationSetInfo := argoprojiov1alpha1.ApplicationSet{ ObjectMeta: metav1.ObjectMeta{ From acebe5df8513867d45d6858dfc95851568e82ed4 Mon Sep 17 00:00:00 2001 From: aburan28 Date: Tue, 14 Jan 2025 08:31:34 -0300 Subject: [PATCH 19/24] add prometheus metrics Signed-off-by: aburan28 --- applicationset/metrics/metrics.go | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/applicationset/metrics/metrics.go b/applicationset/metrics/metrics.go index df75012c0a462..30e28ea5e08f7 100644 --- a/applicationset/metrics/metrics.go +++ b/applicationset/metrics/metrics.go @@ -31,7 +31,9 @@ var ( ) type ApplicationsetMetrics struct { - reconcileHistogram *prometheus.HistogramVec + reconcileHistogram *prometheus.HistogramVec + githubCacheHistogram *prometheus.HistogramVec + githubCacheCounter *prometheus.CounterVec } type appsetCollector struct { @@ -50,15 +52,34 @@ func NewApplicationsetMetrics(appsetLister applisters.ApplicationSetLister, apps }, descAppsetDefaultLabels, ) + githubCacheHistogram := prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Name: "argocd_github_cache_latency", + Help: "", + }, + descAppsetDefaultLabels, + ) + + githubCacheCounter := prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "argocd_github_cache_items", + Help: "Number of items in the github cache", + }, + descAppsetDefaultLabels, + ) appsetCollector := newAppsetCollector(appsetLister, appsetLabels, appsetFilter) // Register collectors and metrics metrics.Registry.MustRegister(reconcileHistogram) metrics.Registry.MustRegister(appsetCollector) + metrics.Registry.MustRegister(githubCacheHistogram) + metrics.Registry.MustRegister(githubCacheCounter) return ApplicationsetMetrics{ - reconcileHistogram: reconcileHistogram, + reconcileHistogram: reconcileHistogram, + githubCacheHistogram: githubCacheHistogram, + githubCacheCounter: githubCacheCounter, } } From 48e8eff6baab655434ffb792369b9e9cc7ad0205 Mon Sep 17 00:00:00 2001 From: aburan28 Date: Tue, 14 Jan 2025 08:41:55 -0300 Subject: [PATCH 20/24] adding the codegen items Signed-off-by: aburan28 --- applicationset/metrics/metrics.go | 8 ++++++++ docs/operator-manual/server-commands/argocd-server.md | 2 ++ 2 files changed, 10 insertions(+) diff --git a/applicationset/metrics/metrics.go b/applicationset/metrics/metrics.go index 30e28ea5e08f7..50360da0cc1e6 100644 --- a/applicationset/metrics/metrics.go +++ b/applicationset/metrics/metrics.go @@ -87,6 +87,14 @@ func (m *ApplicationsetMetrics) ObserveReconcile(appset *argoappv1.ApplicationSe m.reconcileHistogram.WithLabelValues(appset.Namespace, appset.Name).Observe(duration.Seconds()) } +func (m *ApplicationsetMetrics) ObserveGithubCacheLatency(appset *argoappv1.ApplicationSet, duration time.Duration) { + m.githubCacheHistogram.WithLabelValues(appset.Namespace, appset.Name).Observe(duration.Seconds()) +} + +func (m *ApplicationsetMetrics) IncGithubCacheItems(appset *argoappv1.ApplicationSet, count int) { + m.githubCacheCounter.WithLabelValues(appset.Namespace, appset.Name).Add(float64(count)) +} + func newAppsetCollector(lister applisters.ApplicationSetLister, labels []string, filter func(appset *argoappv1.ApplicationSet) bool) *appsetCollector { descAppsetDefaultLabels = []string{"namespace", "name"} diff --git a/docs/operator-manual/server-commands/argocd-server.md b/docs/operator-manual/server-commands/argocd-server.md index fe284a5940733..1fc7547055fca 100644 --- a/docs/operator-manual/server-commands/argocd-server.md +++ b/docs/operator-manual/server-commands/argocd-server.md @@ -53,6 +53,8 @@ argocd-server [flags] --enable-gzip Enable GZIP compression (default true) --enable-k8s-event none Enable ArgoCD to use k8s event. For disabling all events, set the value as none. (e.g --enable-k8s-event=none), For enabling specific events, set the value as `event reason`. (e.g --enable-k8s-event=StatusRefreshed,ResourceCreated) (default [all]) --enable-proxy-extension Enable Proxy Extension feature + --enabled-github-cache Enable caching for GitHub client + --github-cache-size int Max items to hold in the cache for github (default 5000) --gloglevel int Set the glog logging level -h, --help help for argocd-server --hydrator-enabled Feature flag to enable Hydrator. Default ("false") From e3330f393c62b230c27de59addda5ec28bc06a83 Mon Sep 17 00:00:00 2001 From: aburan28 Date: Tue, 14 Jan 2025 09:20:27 -0300 Subject: [PATCH 21/24] adding the codegen items Signed-off-by: aburan28 --- applicationset/services/pull_request/github_test.go | 3 ++- applicationset/services/scm_provider/github_test.go | 5 +++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/applicationset/services/pull_request/github_test.go b/applicationset/services/pull_request/github_test.go index 203f2a42e1c74..e7f5af055d542 100644 --- a/applicationset/services/pull_request/github_test.go +++ b/applicationset/services/pull_request/github_test.go @@ -14,6 +14,7 @@ func toPtr(s string) *string { func TestGithubToken(t *testing.T) { ctx := context.Background() + testCases := []struct { name string token string @@ -28,7 +29,7 @@ func TestGithubToken(t *testing.T) { } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - _, err := NewGithubService(ctx, tc.token, "", "test", "test", []string{}) + _, err := NewGithubService(ctx, tc.token, "", "test", "test", []string{}, nil) require.NoError(t, err) }) } diff --git a/applicationset/services/scm_provider/github_test.go b/applicationset/services/scm_provider/github_test.go index 006fffe61add3..4c5a92ec098d5 100644 --- a/applicationset/services/scm_provider/github_test.go +++ b/applicationset/services/scm_provider/github_test.go @@ -245,7 +245,7 @@ func TestGithubListRepos(t *testing.T) { for _, c := range cases { t.Run(c.name, func(t *testing.T) { - provider, _ := NewGithubProvider(context.Background(), "argoproj", "", ts.URL, c.allBranches, cache) + provider, _ := NewGithubProvider(context.Background(), "argoproj", "", ts.URL, c.allBranches, nil) rawRepos, err := ListRepos(context.Background(), provider, c.filters, c.proto) if c.hasError { require.Error(t, err) @@ -297,7 +297,8 @@ func TestGithubGetBranches(t *testing.T) { githubMockHandler(t)(w, r) })) defer ts.Close() - host, _ := NewGithubProvider("argoproj", "", ts.URL, false) + cache := httpcache.NewMemoryCache() + host, _ := NewGithubProvider("argoproj", "", ts.URL, false, cache) repo := &Repository{ Organization: "argoproj", Repository: "argo-cd", From 27e29828923e4f48bea9939fb0c3a80adc0dcbc6 Mon Sep 17 00:00:00 2001 From: aburan28 Date: Tue, 14 Jan 2025 10:33:59 -0300 Subject: [PATCH 22/24] update to documentation for the github cache params Signed-off-by: aburan28 --- docs/operator-manual/argocd-cmd-params-cm.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/operator-manual/argocd-cmd-params-cm.yaml b/docs/operator-manual/argocd-cmd-params-cm.yaml index 59382a30e9db7..0e06c3d74bf3d 100644 --- a/docs/operator-manual/argocd-cmd-params-cm.yaml +++ b/docs/operator-manual/argocd-cmd-params-cm.yaml @@ -272,7 +272,10 @@ data: applicationsetcontroller.requeue.after: "3m" # Enable strict mode for tokenRef in ApplicationSet resources. When enabled, the referenced secret must have a label `argocd.argoproj.io/secret-type` with value `scm-creds`. applicationsetcontroller.enable.tokenref.strict.mode: "false" - + # Enable caching of Github API requests for scm and pull request generators + applicationsetcontroller.enable.github.cache: "false" + # Max number of items to store in the in-memory cache for Github API requests. Note that if enabled, to adjust the resource requests/limits accordingly. + applicationsetcontroller.github.cache.size: "1000" ## Argo CD Notifications Controller Properties # Set the logging level. One of: debug|info|warn|error (default "info") notificationscontroller.log.level: "info" From 69159123166910184b245185efaa491bf6aff408 Mon Sep 17 00:00:00 2001 From: aburan28 Date: Tue, 14 Jan 2025 11:28:32 -0300 Subject: [PATCH 23/24] add a counter for github cache Signed-off-by: aburan28 --- applicationset/metrics/metrics.go | 21 ++++++++++++++------- applicationset/metrics/metrics_test.go | 19 +++++++++++++++++++ 2 files changed, 33 insertions(+), 7 deletions(-) diff --git a/applicationset/metrics/metrics.go b/applicationset/metrics/metrics.go index 50360da0cc1e6..503775e88d00e 100644 --- a/applicationset/metrics/metrics.go +++ b/applicationset/metrics/metrics.go @@ -28,6 +28,13 @@ var ( descAppsetDefaultLabels, nil, ) + + descAppsetGithubCacheCounter = prometheus.NewDesc( + "argocd_appset_github_cached_items_count", + "Number of github api requests cached", + []string{}, + nil, + ) ) type ApplicationsetMetrics struct { @@ -54,7 +61,7 @@ func NewApplicationsetMetrics(appsetLister applisters.ApplicationSetLister, apps ) githubCacheHistogram := prometheus.NewHistogramVec( prometheus.HistogramOpts{ - Name: "argocd_github_cache_latency", + Name: "argocd_appset_github_cache_latency", Help: "", }, descAppsetDefaultLabels, @@ -62,7 +69,7 @@ func NewApplicationsetMetrics(appsetLister applisters.ApplicationSetLister, apps githubCacheCounter := prometheus.NewCounterVec( prometheus.CounterOpts{ - Name: "argocd_github_cache_items", + Name: "argocd_appset_github_cached_items_count", Help: "Number of items in the github cache", }, descAppsetDefaultLabels, @@ -77,9 +84,8 @@ func NewApplicationsetMetrics(appsetLister applisters.ApplicationSetLister, apps metrics.Registry.MustRegister(githubCacheCounter) return ApplicationsetMetrics{ - reconcileHistogram: reconcileHistogram, - githubCacheHistogram: githubCacheHistogram, - githubCacheCounter: githubCacheCounter, + reconcileHistogram: reconcileHistogram, + githubCacheCounter: githubCacheCounter, } } @@ -88,11 +94,12 @@ func (m *ApplicationsetMetrics) ObserveReconcile(appset *argoappv1.ApplicationSe } func (m *ApplicationsetMetrics) ObserveGithubCacheLatency(appset *argoappv1.ApplicationSet, duration time.Duration) { + // TODO: come back to this m.githubCacheHistogram.WithLabelValues(appset.Namespace, appset.Name).Observe(duration.Seconds()) } -func (m *ApplicationsetMetrics) IncGithubCacheItems(appset *argoappv1.ApplicationSet, count int) { - m.githubCacheCounter.WithLabelValues(appset.Namespace, appset.Name).Add(float64(count)) +func (m *ApplicationsetMetrics) IncGithubCacheItems() { + m.githubCacheCounter.WithLabelValues().Inc() } func newAppsetCollector(lister applisters.ApplicationSetLister, labels []string, filter func(appset *argoappv1.ApplicationSet) bool) *appsetCollector { diff --git a/applicationset/metrics/metrics_test.go b/applicationset/metrics/metrics_test.go index 1db5b3efbef20..2f61f901ddf64 100644 --- a/applicationset/metrics/metrics_test.go +++ b/applicationset/metrics/metrics_test.go @@ -209,6 +209,25 @@ argocd_appset_owned_applications{name="test2",namespace="argocd"} 0 assert.NotContains(t, rr.Body.String(), `name="should-be-filtered-out"`) } +func IncGithubCacheItems(t *testing.T) { + appsetList := newFakeAppsets(fakeAppsetList) + client := initializeClient(appsetList) + metrics.Registry = prometheus.NewRegistry() + + appsetMetrics := NewApplicationsetMetrics(utils.NewAppsetLister(client), collectedLabels, filter) + + req, err := http.NewRequest(http.MethodGet, "/metrics", nil) + require.NoError(t, err) + rr := httptest.NewRecorder() + handler := promhttp.HandlerFor(metrics.Registry, promhttp.HandlerOpts{}) + appsetMetrics.IncGithubCacheItems() + handler.ServeHTTP(rr, req) + assert.Contains(t, rr.Body.String(), ` +argocd_appset_github_cached_items_count{} 1 +`) + +} + func TestObserveReconcile(t *testing.T) { appsetList := newFakeAppsets(fakeAppsetList) client := initializeClient(appsetList) From b69e3bd95a4af29509650485f3797df9d2f31c22 Mon Sep 17 00:00:00 2001 From: aburan28 Date: Tue, 14 Jan 2025 18:14:39 -0300 Subject: [PATCH 24/24] testing updates Signed-off-by: aburan28 --- applicationset/metrics/metrics.go | 2 +- applicationset/services/scm_provider/github_test.go | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/applicationset/metrics/metrics.go b/applicationset/metrics/metrics.go index 503775e88d00e..81ebbc81fade6 100644 --- a/applicationset/metrics/metrics.go +++ b/applicationset/metrics/metrics.go @@ -95,7 +95,7 @@ func (m *ApplicationsetMetrics) ObserveReconcile(appset *argoappv1.ApplicationSe func (m *ApplicationsetMetrics) ObserveGithubCacheLatency(appset *argoappv1.ApplicationSet, duration time.Duration) { // TODO: come back to this - m.githubCacheHistogram.WithLabelValues(appset.Namespace, appset.Name).Observe(duration.Seconds()) + m.githubCacheHistogram.WithLabelValues().Observe(duration.Seconds()) } func (m *ApplicationsetMetrics) IncGithubCacheItems() { diff --git a/applicationset/services/scm_provider/github_test.go b/applicationset/services/scm_provider/github_test.go index 4c5a92ec098d5..132ade662b108 100644 --- a/applicationset/services/scm_provider/github_test.go +++ b/applicationset/services/scm_provider/github_test.go @@ -297,7 +297,8 @@ func TestGithubGetBranches(t *testing.T) { githubMockHandler(t)(w, r) })) defer ts.Close() - cache := httpcache.NewMemoryCache() + cache := httpcache.Cache{} + host, _ := NewGithubProvider("argoproj", "", ts.URL, false, cache) repo := &Repository{ Organization: "argoproj",