From 801aae40c379aff83f3158e24bd15eb3ff6896f5 Mon Sep 17 00:00:00 2001 From: Matthew Bennett Date: Tue, 30 Jul 2024 16:08:35 -0700 Subject: [PATCH 1/9] allow ssh/altssh subdomains in repo URLs to match webhook payload Signed-off-by: Matthew Bennett --- .../webhook/testdata/gitlab-event.json | 24 +++++++++---------- applicationset/webhook/webhook.go | 13 ++++++++-- applicationset/webhook/webhook_test.go | 18 ++++++++------ util/webhook/webhook.go | 2 +- util/webhook/webhook_test.go | 5 +++- 5 files changed, 39 insertions(+), 23 deletions(-) diff --git a/applicationset/webhook/testdata/gitlab-event.json b/applicationset/webhook/testdata/gitlab-event.json index 83ac0b4fcb059..f4c129586ad68 100644 --- a/applicationset/webhook/testdata/gitlab-event.json +++ b/applicationset/webhook/testdata/gitlab-event.json @@ -16,26 +16,26 @@ "id": 1, "name": "project", "description": "", - "web_url": "https://gitlab/group/name", + "web_url": "https://gitlab.com/group/name", "avatar_url": null, - "git_ssh_url": "ssh://git@gitlab:2222/group/name.git", - "git_http_url": "https://gitlab/group/name.git", + "git_ssh_url": "ssh://git@gitlab.com:2222/group/name.git", + "git_http_url": "https://gitlab.com/group/name.git", "namespace": "group", "visibility_level": 1, "path_with_namespace": "group/name", "default_branch": "master", "ci_config_path": null, - "homepage": "https://gitlab/group/name", - "url": "ssh://git@gitlab:2222/group/name.git", - "ssh_url": "ssh://git@gitlab:2222/group/name.git", - "http_url": "https://gitlab/group/name.git" + "homepage": "https://gitlab.com/group/name", + "url": "ssh://git@gitlab.com:2222/group/name.git", + "ssh_url": "ssh://git@gitlab.com:2222/group/name.git", + "http_url": "https://gitlab.com/group/name.git" }, "commits": [ { "id": "bb0748feaa336d841c251017e4e374c22d0c8a98", "message": "Test commit message\n", "timestamp": "2020-01-06T03:47:55Z", - "url": "https://gitlab/group/name/commit/bb0748feaa336d841c251017e4e374c22d0c8a98", + "url": "https://gitlab.com/group/name/commit/bb0748feaa336d841c251017e4e374c22d0c8a98", "author": { "name": "User", "email": "user@example.com" @@ -55,11 +55,11 @@ }, "repository": { "name": "name", - "url": "ssh://git@gitlab:2222/group/name.git", + "url": "ssh://git@gitlab.com:2222/group/name.git", "description": "", - "homepage": "https://gitlab/group/name", - "git_http_url": "https://gitlab/group/name.git", - "git_ssh_url": "ssh://git@gitlab:2222/group/name.git", + "homepage": "https://gitlab.com/group/name", + "git_http_url": "https://gitlab.com/group/name.git", + "git_ssh_url": "ssh://git@gitlab.com:2222/group/name.git", "visibility_level": 10 } } \ No newline at end of file diff --git a/applicationset/webhook/webhook.go b/applicationset/webhook/webhook.go index 4fb4d6668bc2f..295a989b03f53 100644 --- a/applicationset/webhook/webhook.go +++ b/applicationset/webhook/webhook.go @@ -27,6 +27,10 @@ import ( log "github.com/sirupsen/logrus" ) +// https://www.rfc-editor.org/rfc/rfc3986#section-3.2.1 +// https://github.com/shadow-maint/shadow/blob/master/libmisc/chkname.c#L36 +const usernameRegex = `[a-zA-Z0-9_\.][a-zA-Z0-9_\.-]{0,30}[a-zA-Z0-9_\.\$-]?` + const payloadQueueSize = 50000 var errBasicAuthVerificationFailed = errors.New("basic auth verification failed") @@ -246,7 +250,10 @@ func getGitGeneratorInfo(payload interface{}) *gitGeneratorInfo { log.Errorf("Failed to parse repoURL '%s'", webURL) return nil } - regexpStr := `(?i)(http://|https://|\w+@|ssh://(\w+@)?)` + urlObj.Hostname() + "(:[0-9]+|)[:/]" + urlObj.Path[1:] + "(\\.git)?" + regexEscapedHostname := regexp.QuoteMeta(urlObj.Hostname()) + regexEscapedPath := regexp.QuoteMeta(urlObj.EscapedPath()[1:]) + regexpStr := fmt.Sprintf(`(?i)^(http://|https://|%s@|ssh://(%s@)?((alt)?ssh\.)?)%s(:[0-9]+|)[:/]%s(\.git)?$`, + usernameRegex, usernameRegex, regexEscapedHostname, regexEscapedPath) repoRegexp, err := regexp.Compile(regexpStr) if err != nil { log.Errorf("Failed to compile regexp for repoURL '%s'", webURL) @@ -274,7 +281,9 @@ func getPRGeneratorInfo(payload interface{}) *prGeneratorInfo { log.Errorf("Failed to parse repoURL '%s'", apiURL) return nil } - regexpStr := `(?i)(http://|https://|\w+@|ssh://(\w+@)?)` + urlObj.Hostname() + "(:[0-9]+|)[:/]" + regexEscapedHostname := regexp.QuoteMeta(urlObj.Hostname()) + regexpStr := fmt.Sprintf(`(?i)^(http://|https://|%s@|ssh://(%s@)?)%s(:[0-9]+|)[:/]$`, + usernameRegex, usernameRegex, regexEscapedHostname) apiRegexp, err := regexp.Compile(regexpStr) if err != nil { log.Errorf("Failed to compile regexp for repoURL '%s'", apiURL) diff --git a/applicationset/webhook/webhook_test.go b/applicationset/webhook/webhook_test.go index 683928635bd51..7eaf5b08478c8 100644 --- a/applicationset/webhook/webhook_test.go +++ b/applicationset/webhook/webhook_test.go @@ -63,7 +63,7 @@ func TestWebhookHandler(t *testing.T) { headerKey: "X-GitHub-Event", headerValue: "push", payloadFile: "github-commit-event.json", - effectedAppSets: []string{"git-github", "matrix-git-github", "merge-git-github", "matrix-scm-git-github", "matrix-nested-git-github", "merge-nested-git-github", "plugin", "matrix-pull-request-github-plugin"}, + effectedAppSets: []string{"git-github", "git-github-ssh", "git-github-alt-ssh", "matrix-git-github", "merge-git-github", "matrix-scm-git-github", "matrix-nested-git-github", "merge-nested-git-github", "plugin", "matrix-pull-request-github-plugin"}, expectedStatusCode: http.StatusOK, expectedRefresh: true, }, @@ -72,7 +72,7 @@ func TestWebhookHandler(t *testing.T) { headerKey: "X-GitHub-Event", headerValue: "push", payloadFile: "github-commit-branch-event.json", - effectedAppSets: []string{"git-github", "plugin", "matrix-pull-request-github-plugin"}, + effectedAppSets: []string{"git-github", "git-github-ssh", "git-github-alt-ssh", "plugin", "matrix-pull-request-github-plugin"}, expectedStatusCode: http.StatusOK, expectedRefresh: true, }, @@ -81,7 +81,7 @@ func TestWebhookHandler(t *testing.T) { headerKey: "X-GitHub-Event", headerValue: "ping", payloadFile: "github-ping-event.json", - effectedAppSets: []string{"git-github", "plugin"}, + effectedAppSets: []string{"git-github", "git-github-ssh", "git-github-alt-ssh", "plugin"}, expectedStatusCode: http.StatusOK, expectedRefresh: false, }, @@ -90,7 +90,7 @@ func TestWebhookHandler(t *testing.T) { headerKey: "X-Gitlab-Event", headerValue: "Push Hook", payloadFile: "gitlab-event.json", - effectedAppSets: []string{"git-gitlab", "plugin", "matrix-pull-request-github-plugin"}, + effectedAppSets: []string{"git-gitlab", "git-gitlab-ssh", "git-gitlab-alt-ssh", "plugin", "matrix-pull-request-github-plugin"}, expectedStatusCode: http.StatusOK, expectedRefresh: true, }, @@ -99,7 +99,7 @@ func TestWebhookHandler(t *testing.T) { headerKey: "X-Random-Event", headerValue: "Push Hook", payloadFile: "gitlab-event.json", - effectedAppSets: []string{"git-gitlab", "plugin"}, + effectedAppSets: []string{"git-gitlab", "git-gitlab-ssh", "git-gitlab-alt-ssh", "plugin"}, expectedStatusCode: http.StatusBadRequest, expectedRefresh: false, }, @@ -108,7 +108,7 @@ func TestWebhookHandler(t *testing.T) { headerKey: "X-Random-Event", headerValue: "Push Hook", payloadFile: "invalid-event.json", - effectedAppSets: []string{"git-gitlab", "plugin"}, + effectedAppSets: []string{"git-gitlab", "git-gitlab-ssh", "git-gitlab-alt-ssh", "plugin"}, expectedStatusCode: http.StatusBadRequest, expectedRefresh: false, }, @@ -190,7 +190,11 @@ func TestWebhookHandler(t *testing.T) { t.Run(test.desc, func(t *testing.T) { fc := fake.NewClientBuilder().WithScheme(scheme).WithObjects( fakeAppWithGitGenerator("git-github", namespace, "https://github.com/org/repo"), - fakeAppWithGitGenerator("git-gitlab", namespace, "https://gitlab/group/name"), + fakeAppWithGitGenerator("git-github-ssh", namespace, "ssh://git@github.com/org/repo"), + fakeAppWithGitGenerator("git-github-alt-ssh", namespace, "ssh://git@ssh.github.com:443/org/repo"), + fakeAppWithGitGenerator("git-gitlab", namespace, "https://gitlab.com/group/name"), + fakeAppWithGitGenerator("git-gitlab-ssh", namespace, "ssh://git@gitlab.com/group/name"), + fakeAppWithGitGenerator("git-gitlab-alt-ssh", namespace, "ssh://git@altssh.gitlab.com:443/group/name"), fakeAppWithGitGenerator("git-azure-devops", namespace, "https://dev.azure.com/fabrikam-fiber-inc/DefaultCollection/_git/Fabrikam-Fiber-Git"), fakeAppWithGithubPullRequestGenerator("pull-request-github", namespace, "CodErTOcat", "Hello-World"), fakeAppWithGitlabPullRequestGenerator("pull-request-gitlab", namespace, "100500"), diff --git a/util/webhook/webhook.go b/util/webhook/webhook.go index 31d39621325d4..11750cb84cf73 100644 --- a/util/webhook/webhook.go +++ b/util/webhook/webhook.go @@ -347,7 +347,7 @@ func getWebUrlRegex(webURL string) (*regexp.Regexp, error) { regexEscapedHostname := regexp.QuoteMeta(urlObj.Hostname()) regexEscapedPath := regexp.QuoteMeta(urlObj.EscapedPath()[1:]) - regexpStr := fmt.Sprintf(`(?i)^(http://|https://|%s@|ssh://(%s@)?)%s(:[0-9]+|)[:/]%s(\.git)?$`, + regexpStr := fmt.Sprintf(`(?i)^(http://|https://|%s@|ssh://(%s@)?((alt)?ssh\.)?)%s(:[0-9]+|)[:/]%s(\.git)?$`, usernameRegex, usernameRegex, regexEscapedHostname, regexEscapedPath) repoRegexp, err := regexp.Compile(regexpStr) if err != nil { diff --git a/util/webhook/webhook_test.go b/util/webhook/webhook_test.go index 82cc353a8364c..f981ed8926a7a 100644 --- a/util/webhook/webhook_test.go +++ b/util/webhook/webhook_test.go @@ -616,10 +616,13 @@ func Test_getWebUrlRegex(t *testing.T) { {false, "https://example.com/org/repo", "https://example.com/org/repo-2", "partial match should not match"}, {true, "https://example.com/org/repo", "https://example.com/org/repo.git", "no .git should match with .git"}, {true, "https://example.com/org/repo", "git@example.com:org/repo", "git without protocol should match"}, - {true, "https://example.com/org/repo", "user@example.com:org/repo", "git with non-git username shout match"}, + {true, "https://example.com/org/repo", "user@example.com:org/repo", "git with non-git username should match"}, {true, "https://example.com/org/repo", "ssh://git@example.com/org/repo", "git with protocol should match"}, {true, "https://example.com/org/repo", "ssh://git@example.com:22/org/repo", "git with port number should match"}, {true, "https://example.com:443/org/repo", "ssh://git@example.com:22/org/repo", "https and ssh w/ different port numbers should match"}, + {true, "https://example.com:443/org/repo", "ssh://git@ssh.example.com:22/org/repo", "https and ssh w/ ssh subdomain should match"}, + {true, "https://example.com:443/org/repo", "ssh://git@altssh.example.com:22/org/repo", "https and ssh w/ altssh subdomain should match"}, + {false, "https://example.com:443/org/repo", "ssh://git@unknown.example.com:22/org/repo", "https and ssh w/ unknown subdomain should not match"}, {true, "https://example.com/org/repo", "ssh://user-name@example.com/org/repo", "valid usernames with hyphens in repo should match"}, {false, "https://example.com/org/repo", "ssh://-user-name@example.com/org/repo", "invalid usernames with hyphens in repo should not match"}, {true, "https://example.com:443/org/repo", "GIT@EXAMPLE.COM:22:ORG/REPO", "matches aren't case-sensitive"}, From c79b60d0aee7319dcd8935ac020cce7225ad2938 Mon Sep 17 00:00:00 2001 From: Matthew Bennett Date: Thu, 1 Aug 2024 10:38:31 -0700 Subject: [PATCH 2/9] test nit Signed-off-by: Matthew Bennett --- util/webhook/webhook_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/util/webhook/webhook_test.go b/util/webhook/webhook_test.go index f981ed8926a7a..0ce976fff03b0 100644 --- a/util/webhook/webhook_test.go +++ b/util/webhook/webhook_test.go @@ -620,9 +620,9 @@ func Test_getWebUrlRegex(t *testing.T) { {true, "https://example.com/org/repo", "ssh://git@example.com/org/repo", "git with protocol should match"}, {true, "https://example.com/org/repo", "ssh://git@example.com:22/org/repo", "git with port number should match"}, {true, "https://example.com:443/org/repo", "ssh://git@example.com:22/org/repo", "https and ssh w/ different port numbers should match"}, - {true, "https://example.com:443/org/repo", "ssh://git@ssh.example.com:22/org/repo", "https and ssh w/ ssh subdomain should match"}, - {true, "https://example.com:443/org/repo", "ssh://git@altssh.example.com:22/org/repo", "https and ssh w/ altssh subdomain should match"}, - {false, "https://example.com:443/org/repo", "ssh://git@unknown.example.com:22/org/repo", "https and ssh w/ unknown subdomain should not match"}, + {true, "https://example.com:443/org/repo", "ssh://git@ssh.example.com:443/org/repo", "https and ssh w/ ssh subdomain should match"}, + {true, "https://example.com:443/org/repo", "ssh://git@altssh.example.com:443/org/repo", "https and ssh w/ altssh subdomain should match"}, + {false, "https://example.com:443/org/repo", "ssh://git@unknown.example.com:443/org/repo", "https and ssh w/ unknown subdomain should not match"}, {true, "https://example.com/org/repo", "ssh://user-name@example.com/org/repo", "valid usernames with hyphens in repo should match"}, {false, "https://example.com/org/repo", "ssh://-user-name@example.com/org/repo", "invalid usernames with hyphens in repo should not match"}, {true, "https://example.com:443/org/repo", "GIT@EXAMPLE.COM:22:ORG/REPO", "matches aren't case-sensitive"}, From eb25202de02126f8cee48c0260d46c0e8b05a790 Mon Sep 17 00:00:00 2001 From: Matthew Bennett Date: Mon, 12 Aug 2024 14:15:26 -0700 Subject: [PATCH 3/9] wip Signed-off-by: Matthew Bennett --- .../testdata/azuredevops-pull-request.json | 0 .../testdata/azuredevops-push.json | 0 .../testdata/github-commit-branch-event.json | 0 .../testdata/github-commit-event.json | 0 .../testdata/github-ping-event.json | 0 .../github-pull-request-assigned-event.json | 0 .../github-pull-request-labeled-event.json | 0 .../github-pull-request-opened-event.json | 0 .../testdata/gitlab-event.json | 0 .../gitlab-merge-request-approval-event.json | 0 .../gitlab-merge-request-open-event.json | 0 .../testdata/invalid-event.json | 0 .../webhookhandler.go} | 215 ++---- .../webhookhandler_test.go} | 11 +- .../commands/applicationset_controller.go | 28 +- server/server.go | 18 +- .../testdata/azuredevops-git-push-event.json | 0 .../testdata/bitbucket-server-event.json | 0 .../testdata/github-commit-event.json | 0 .../testdata/github-ping-event.json | 0 .../testdata/github-tag-event.json | 0 .../testdata/gitlab-event.json | 0 .../webhookhandler}/testdata/gogs-event.json | 0 server/webhookhandler/webhookhandler.go | 355 ++++++++++ server/webhookhandler/webhookhandler_test.go | 668 ++++++++++++++++++ util/webhook/webhook.go | 484 ++++--------- util/webhook/webhook_test.go | 617 +--------------- 27 files changed, 1268 insertions(+), 1128 deletions(-) rename applicationset/{webhook => webhookhandler}/testdata/azuredevops-pull-request.json (100%) rename applicationset/{webhook => webhookhandler}/testdata/azuredevops-push.json (100%) rename applicationset/{webhook => webhookhandler}/testdata/github-commit-branch-event.json (100%) rename applicationset/{webhook => webhookhandler}/testdata/github-commit-event.json (100%) rename applicationset/{webhook => webhookhandler}/testdata/github-ping-event.json (100%) rename applicationset/{webhook => webhookhandler}/testdata/github-pull-request-assigned-event.json (100%) rename applicationset/{webhook => webhookhandler}/testdata/github-pull-request-labeled-event.json (100%) rename applicationset/{webhook => webhookhandler}/testdata/github-pull-request-opened-event.json (100%) rename applicationset/{webhook => webhookhandler}/testdata/gitlab-event.json (100%) rename applicationset/{webhook => webhookhandler}/testdata/gitlab-merge-request-approval-event.json (100%) rename applicationset/{webhook => webhookhandler}/testdata/gitlab-merge-request-open-event.json (100%) rename applicationset/{webhook => webhookhandler}/testdata/invalid-event.json (100%) rename applicationset/{webhook/webhook.go => webhookhandler/webhookhandler.go} (71%) rename applicationset/{webhook/webhook_test.go => webhookhandler/webhookhandler_test.go} (99%) rename {util/webhook => server/webhookhandler}/testdata/azuredevops-git-push-event.json (100%) rename {util/webhook => server/webhookhandler}/testdata/bitbucket-server-event.json (100%) rename {util/webhook => server/webhookhandler}/testdata/github-commit-event.json (100%) rename {util/webhook => server/webhookhandler}/testdata/github-ping-event.json (100%) rename {util/webhook => server/webhookhandler}/testdata/github-tag-event.json (100%) rename {util/webhook => server/webhookhandler}/testdata/gitlab-event.json (100%) rename {util/webhook => server/webhookhandler}/testdata/gogs-event.json (100%) create mode 100644 server/webhookhandler/webhookhandler.go create mode 100644 server/webhookhandler/webhookhandler_test.go diff --git a/applicationset/webhook/testdata/azuredevops-pull-request.json b/applicationset/webhookhandler/testdata/azuredevops-pull-request.json similarity index 100% rename from applicationset/webhook/testdata/azuredevops-pull-request.json rename to applicationset/webhookhandler/testdata/azuredevops-pull-request.json diff --git a/applicationset/webhook/testdata/azuredevops-push.json b/applicationset/webhookhandler/testdata/azuredevops-push.json similarity index 100% rename from applicationset/webhook/testdata/azuredevops-push.json rename to applicationset/webhookhandler/testdata/azuredevops-push.json diff --git a/applicationset/webhook/testdata/github-commit-branch-event.json b/applicationset/webhookhandler/testdata/github-commit-branch-event.json similarity index 100% rename from applicationset/webhook/testdata/github-commit-branch-event.json rename to applicationset/webhookhandler/testdata/github-commit-branch-event.json diff --git a/applicationset/webhook/testdata/github-commit-event.json b/applicationset/webhookhandler/testdata/github-commit-event.json similarity index 100% rename from applicationset/webhook/testdata/github-commit-event.json rename to applicationset/webhookhandler/testdata/github-commit-event.json diff --git a/applicationset/webhook/testdata/github-ping-event.json b/applicationset/webhookhandler/testdata/github-ping-event.json similarity index 100% rename from applicationset/webhook/testdata/github-ping-event.json rename to applicationset/webhookhandler/testdata/github-ping-event.json diff --git a/applicationset/webhook/testdata/github-pull-request-assigned-event.json b/applicationset/webhookhandler/testdata/github-pull-request-assigned-event.json similarity index 100% rename from applicationset/webhook/testdata/github-pull-request-assigned-event.json rename to applicationset/webhookhandler/testdata/github-pull-request-assigned-event.json diff --git a/applicationset/webhook/testdata/github-pull-request-labeled-event.json b/applicationset/webhookhandler/testdata/github-pull-request-labeled-event.json similarity index 100% rename from applicationset/webhook/testdata/github-pull-request-labeled-event.json rename to applicationset/webhookhandler/testdata/github-pull-request-labeled-event.json diff --git a/applicationset/webhook/testdata/github-pull-request-opened-event.json b/applicationset/webhookhandler/testdata/github-pull-request-opened-event.json similarity index 100% rename from applicationset/webhook/testdata/github-pull-request-opened-event.json rename to applicationset/webhookhandler/testdata/github-pull-request-opened-event.json diff --git a/applicationset/webhook/testdata/gitlab-event.json b/applicationset/webhookhandler/testdata/gitlab-event.json similarity index 100% rename from applicationset/webhook/testdata/gitlab-event.json rename to applicationset/webhookhandler/testdata/gitlab-event.json diff --git a/applicationset/webhook/testdata/gitlab-merge-request-approval-event.json b/applicationset/webhookhandler/testdata/gitlab-merge-request-approval-event.json similarity index 100% rename from applicationset/webhook/testdata/gitlab-merge-request-approval-event.json rename to applicationset/webhookhandler/testdata/gitlab-merge-request-approval-event.json diff --git a/applicationset/webhook/testdata/gitlab-merge-request-open-event.json b/applicationset/webhookhandler/testdata/gitlab-merge-request-open-event.json similarity index 100% rename from applicationset/webhook/testdata/gitlab-merge-request-open-event.json rename to applicationset/webhookhandler/testdata/gitlab-merge-request-open-event.json diff --git a/applicationset/webhook/testdata/invalid-event.json b/applicationset/webhookhandler/testdata/invalid-event.json similarity index 100% rename from applicationset/webhook/testdata/invalid-event.json rename to applicationset/webhookhandler/testdata/invalid-event.json diff --git a/applicationset/webhook/webhook.go b/applicationset/webhookhandler/webhookhandler.go similarity index 71% rename from applicationset/webhook/webhook.go rename to applicationset/webhookhandler/webhookhandler.go index 7d1d367308439..c3948505ed96e 100644 --- a/applicationset/webhook/webhook.go +++ b/applicationset/webhookhandler/webhookhandler.go @@ -1,48 +1,37 @@ -package webhook +package webhookhandler import ( "context" "fmt" - "html" - "net/http" "net/url" "regexp" "strconv" "strings" - "sync" - "k8s.io/apimachinery/pkg/types" - "k8s.io/client-go/util/retry" "sigs.k8s.io/controller-runtime/pkg/client" - - "github.com/argoproj/argo-cd/v2/applicationset/generators" - "github.com/argoproj/argo-cd/v2/common" - "github.com/argoproj/argo-cd/v2/pkg/apis/application/v1alpha1" - argosettings "github.com/argoproj/argo-cd/v2/util/settings" + log "github.com/sirupsen/logrus" + "k8s.io/client-go/util/retry" "github.com/go-playground/webhooks/v6/azuredevops" "github.com/go-playground/webhooks/v6/github" "github.com/go-playground/webhooks/v6/gitlab" - log "github.com/sirupsen/logrus" -) + "k8s.io/apimachinery/pkg/types" -// https://www.rfc-editor.org/rfc/rfc3986#section-3.2.1 -// https://github.com/shadow-maint/shadow/blob/master/libmisc/chkname.c#L36 -const usernameRegex = `[a-zA-Z0-9_\.][a-zA-Z0-9_\.-]{0,30}[a-zA-Z0-9_\.\$-]?` + "github.com/argoproj/argo-cd/v2/common" -const payloadQueueSize = 50000 + "github.com/argoproj/argo-cd/v2/applicationset/generators" + "github.com/argoproj/argo-cd/v2/pkg/apis/application/v1alpha1" + "github.com/argoproj/argo-cd/v2/util/webhook" + "github.com/argoproj/argo-cd/v2/util/settings" +) -type WebhookHandler struct { - sync.WaitGroup // for testing - namespace string - github *github.Webhook - gitlab *gitlab.Webhook - azuredevops *azuredevops.Webhook +type ApplicationSetWebhookPayloadHandler struct { client client.Client generators map[string]generators.Generator - queue chan interface{} } + + type gitGeneratorInfo struct { Revision string TouchedHead bool @@ -71,129 +60,53 @@ type prGeneratorGitlabInfo struct { APIHostname string } -func NewWebhookHandler(namespace string, webhookParallelism int, argocdSettingsMgr *argosettings.SettingsManager, client client.Client, generators map[string]generators.Generator) (*WebhookHandler, error) { - // register the webhook secrets stored under "argocd-secret" for verifying incoming payloads - argocdSettings, err := argocdSettingsMgr.GetSettings() - if err != nil { - return nil, fmt.Errorf("Failed to get argocd settings: %w", err) - } - githubHandler, err := github.New(github.Options.Secret(argocdSettings.WebhookGitHubSecret)) - if err != nil { - return nil, fmt.Errorf("Unable to init GitHub webhook: %w", err) - } - gitlabHandler, err := gitlab.New(gitlab.Options.Secret(argocdSettings.WebhookGitLabSecret)) - if err != nil { - return nil, fmt.Errorf("Unable to init GitLab webhook: %w", err) - } - azuredevopsHandler, err := azuredevops.New(azuredevops.Options.BasicAuth(argocdSettings.WebhookAzureDevOpsUsername, argocdSettings.WebhookAzureDevOpsPassword)) - if err != nil { - return nil, fmt.Errorf("Unable to init Azure DevOps webhook: %w", err) - } - - webhookHandler := &WebhookHandler{ - namespace: namespace, - github: githubHandler, - gitlab: gitlabHandler, - azuredevops: azuredevopsHandler, - client: client, - generators: generators, - queue: make(chan interface{}, payloadQueueSize), - } - - webhookHandler.startWorkerPool(webhookParallelism) - - return webhookHandler, nil -} - -func (h *WebhookHandler) startWorkerPool(webhookParallelism int) { - for i := 0; i < webhookParallelism; i++ { - h.Add(1) - go func() { - defer h.Done() - for { - payload, ok := <-h.queue - if !ok { - return - } - h.HandleEvent(payload) - } - }() - } -} - -func (h *WebhookHandler) HandleEvent(payload interface{}) { +func (handler *ApplicationSetWebhookPayloadHandler) HandlePayload(payload interface{}, handlerWebhook *webhook.Webhook) { gitGenInfo := getGitGeneratorInfo(payload) prGenInfo := getPRGeneratorInfo(payload) + if gitGenInfo == nil && prGenInfo == nil { return } appSetList := &v1alpha1.ApplicationSetList{} - err := h.client.List(context.Background(), appSetList, &client.ListOptions{}) + err := handler.client.List(context.Background(), appSetList, &client.ListOptions{}) + if err != nil { log.Errorf("Failed to list applicationsets: %v", err) + return } for _, appSet := range appSetList.Items { shouldRefresh := false + for _, gen := range appSet.Spec.Generators { // check if the ApplicationSet uses any generator that is relevant to the payload shouldRefresh = shouldRefreshGitGenerator(gen.Git, gitGenInfo) || shouldRefreshPRGenerator(gen.PullRequest, prGenInfo) || shouldRefreshPluginGenerator(gen.Plugin) || - h.shouldRefreshMatrixGenerator(gen.Matrix, &appSet, gitGenInfo, prGenInfo) || - h.shouldRefreshMergeGenerator(gen.Merge, &appSet, gitGenInfo, prGenInfo) + handler.shouldRefreshMatrixGenerator(gen.Matrix, &appSet, gitGenInfo, prGenInfo) || + handler.shouldRefreshMergeGenerator(gen.Merge, &appSet, gitGenInfo, prGenInfo) + if shouldRefresh { break } } + if shouldRefresh { - err := refreshApplicationSet(h.client, &appSet) + err := refreshApplicationSet(handler.client, &appSet) + if err != nil { log.Errorf("Failed to refresh ApplicationSet '%s' for controller reprocessing", appSet.Name) + continue } + log.Infof("refresh ApplicationSet %v/%v from webhook", appSet.Namespace, appSet.Name) } } } -func (h *WebhookHandler) Handler(w http.ResponseWriter, r *http.Request) { - var payload interface{} - var err error - - switch { - case r.Header.Get("X-GitHub-Event") != "": - payload, err = h.github.Parse(r, github.PushEvent, github.PullRequestEvent, github.PingEvent) - case r.Header.Get("X-Gitlab-Event") != "": - payload, err = h.gitlab.Parse(r, gitlab.PushEvents, gitlab.TagEvents, gitlab.MergeRequestEvents) - case r.Header.Get("X-Vss-Activityid") != "": - payload, err = h.azuredevops.Parse(r, azuredevops.GitPushEventType, azuredevops.GitPullRequestCreatedEventType, azuredevops.GitPullRequestUpdatedEventType, azuredevops.GitPullRequestMergedEventType) - default: - log.Debug("Ignoring unknown webhook event") - http.Error(w, "Unknown webhook event", http.StatusBadRequest) - return - } - - if err != nil { - log.Infof("Webhook processing failed: %s", err) - status := http.StatusBadRequest - if r.Method != http.MethodPost { - status = http.StatusMethodNotAllowed - } - http.Error(w, fmt.Sprintf("Webhook processing failed: %s", html.EscapeString(err.Error())), status) - return - } - - select { - case h.queue <- payload: - default: - log.Info("Queue is full, discarding webhook payload") - http.Error(w, "Queue is full, discarding webhook payload", http.StatusServiceUnavailable) - } -} - func parseRevision(ref string) string { refParts := strings.SplitN(ref, "/", 3) return refParts[len(refParts)-1] @@ -225,18 +138,11 @@ func getGitGeneratorInfo(payload interface{}) *gitGeneratorInfo { } log.Infof("Received push event repo: %s, revision: %s, touchedHead: %v", webURL, revision, touchedHead) - urlObj, err := url.Parse(webURL) - if err != nil { - log.Errorf("Failed to parse repoURL '%s'", webURL) - return nil - } - regexEscapedHostname := regexp.QuoteMeta(urlObj.Hostname()) - regexEscapedPath := regexp.QuoteMeta(urlObj.EscapedPath()[1:]) - regexpStr := fmt.Sprintf(`(?i)^(http://|https://|%s@|ssh://(%s@)?((alt)?ssh\.)?)%s(:[0-9]+|)[:/]%s(\.git)?$`, - usernameRegex, usernameRegex, regexEscapedHostname, regexEscapedPath) - repoRegexp, err := regexp.Compile(regexpStr) + repoRegexp, err := webhook.GetWebUrlRegex(webURL) + if err != nil { - log.Errorf("Failed to compile regexp for repoURL '%s'", webURL) + log.Errorf("Failed to get repoRegexp: %s", err) + return nil } @@ -256,19 +162,14 @@ func getPRGeneratorInfo(payload interface{}) *prGeneratorInfo { } apiURL := payload.Repository.URL - urlObj, err := url.Parse(apiURL) - if err != nil { - log.Errorf("Failed to parse repoURL '%s'", apiURL) - return nil - } - regexEscapedHostname := regexp.QuoteMeta(urlObj.Hostname()) - regexpStr := fmt.Sprintf(`(?i)^(http://|https://|%s@|ssh://(%s@)?)%s(:[0-9]+|)[:/]$`, - usernameRegex, usernameRegex, regexEscapedHostname) - apiRegexp, err := regexp.Compile(regexpStr) + apiRegexp, err := webhook.GetApiUrlRegex(apiURL) + if err != nil { log.Errorf("Failed to compile regexp for repoURL '%s'", apiURL) + return nil } + info.Github = &prGeneratorGithubInfo{ Repo: payload.Repository.Name, Owner: payload.Repository.Owner.Login, @@ -463,7 +364,7 @@ func shouldRefreshPRGenerator(gen *v1alpha1.PullRequestGenerator, info *prGenera return false } -func (h *WebhookHandler) shouldRefreshMatrixGenerator(gen *v1alpha1.MatrixGenerator, appSet *v1alpha1.ApplicationSet, gitGenInfo *gitGeneratorInfo, prGenInfo *prGeneratorInfo) bool { +func (handler *ApplicationSetWebhookPayloadHandler) shouldRefreshMatrixGenerator(gen *v1alpha1.MatrixGenerator, appSet *v1alpha1.ApplicationSet, gitGenInfo *gitGeneratorInfo, prGenInfo *prGeneratorInfo) bool { if gen == nil { return false } @@ -492,7 +393,7 @@ func (h *WebhookHandler) shouldRefreshMatrixGenerator(gen *v1alpha1.MatrixGenera } if nestedMatrix != nil { matrixGenerator0 = nestedMatrix.ToMatrixGenerator() - if h.shouldRefreshMatrixGenerator(matrixGenerator0, appSet, gitGenInfo, prGenInfo) { + if handler.shouldRefreshMatrixGenerator(matrixGenerator0, appSet, gitGenInfo, prGenInfo) { return true } } @@ -509,7 +410,7 @@ func (h *WebhookHandler) shouldRefreshMatrixGenerator(gen *v1alpha1.MatrixGenera } if nestedMerge != nil { mergeGenerator0 = nestedMerge.ToMergeGenerator() - if h.shouldRefreshMergeGenerator(mergeGenerator0, appSet, gitGenInfo, prGenInfo) { + if handler.shouldRefreshMergeGenerator(mergeGenerator0, appSet, gitGenInfo, prGenInfo) { return true } } @@ -529,10 +430,10 @@ func (h *WebhookHandler) shouldRefreshMatrixGenerator(gen *v1alpha1.MatrixGenera } // Generate params for first child generator - relGenerators := generators.GetRelevantGenerators(requestedGenerator0, h.generators) + relGenerators := generators.GetRelevantGenerators(requestedGenerator0, handler.generators) params := []map[string]interface{}{} for _, g := range relGenerators { - p, err := g.GenerateParams(requestedGenerator0, appSet, h.client) + p, err := g.GenerateParams(requestedGenerator0, appSet, handler.client) if err != nil { log.Error(err) return false @@ -597,8 +498,8 @@ func (h *WebhookHandler) shouldRefreshMatrixGenerator(gen *v1alpha1.MatrixGenera if shouldRefreshGitGenerator(interpolatedGenerator.Git, gitGenInfo) || shouldRefreshPRGenerator(interpolatedGenerator.PullRequest, prGenInfo) || shouldRefreshPluginGenerator(interpolatedGenerator.Plugin) || - h.shouldRefreshMatrixGenerator(interpolatedGenerator.Matrix, appSet, gitGenInfo, prGenInfo) || - h.shouldRefreshMergeGenerator(requestedGenerator1.Merge, appSet, gitGenInfo, prGenInfo) { + handler.shouldRefreshMatrixGenerator(interpolatedGenerator.Matrix, appSet, gitGenInfo, prGenInfo) || + handler.shouldRefreshMergeGenerator(requestedGenerator1.Merge, appSet, gitGenInfo, prGenInfo) { return true } } @@ -608,11 +509,11 @@ func (h *WebhookHandler) shouldRefreshMatrixGenerator(gen *v1alpha1.MatrixGenera return shouldRefreshGitGenerator(requestedGenerator1.Git, gitGenInfo) || shouldRefreshPRGenerator(requestedGenerator1.PullRequest, prGenInfo) || shouldRefreshPluginGenerator(requestedGenerator1.Plugin) || - h.shouldRefreshMatrixGenerator(requestedGenerator1.Matrix, appSet, gitGenInfo, prGenInfo) || - h.shouldRefreshMergeGenerator(requestedGenerator1.Merge, appSet, gitGenInfo, prGenInfo) + handler.shouldRefreshMatrixGenerator(requestedGenerator1.Matrix, appSet, gitGenInfo, prGenInfo) || + handler.shouldRefreshMergeGenerator(requestedGenerator1.Merge, appSet, gitGenInfo, prGenInfo) } -func (h *WebhookHandler) shouldRefreshMergeGenerator(gen *v1alpha1.MergeGenerator, appSet *v1alpha1.ApplicationSet, gitGenInfo *gitGeneratorInfo, prGenInfo *prGeneratorInfo) bool { +func (handler *ApplicationSetWebhookPayloadHandler) shouldRefreshMergeGenerator(gen *v1alpha1.MergeGenerator, appSet *v1alpha1.ApplicationSet, gitGenInfo *gitGeneratorInfo, prGenInfo *prGeneratorInfo) bool { if gen == nil { return false } @@ -633,7 +534,7 @@ func (h *WebhookHandler) shouldRefreshMergeGenerator(gen *v1alpha1.MergeGenerato return false } if nestedMatrix != nil { - if h.shouldRefreshMatrixGenerator(nestedMatrix.ToMatrixGenerator(), appSet, gitGenInfo, prGenInfo) { + if handler.shouldRefreshMatrixGenerator(nestedMatrix.ToMatrixGenerator(), appSet, gitGenInfo, prGenInfo) { return true } } @@ -648,7 +549,7 @@ func (h *WebhookHandler) shouldRefreshMergeGenerator(gen *v1alpha1.MergeGenerato return false } if nestedMerge != nil { - if h.shouldRefreshMergeGenerator(nestedMerge.ToMergeGenerator(), appSet, gitGenInfo, prGenInfo) { + if handler.shouldRefreshMergeGenerator(nestedMerge.ToMergeGenerator(), appSet, gitGenInfo, prGenInfo) { return true } } @@ -672,3 +573,25 @@ func refreshApplicationSet(c client.Client, appSet *v1alpha1.ApplicationSet) err return c.Patch(context.Background(), appSet, client.Merge) }) } + +func NewWebhook( + parallelism int, + maxPayloadSize int64, + argoCdSettings *settings.ArgoCDSettings, + argoCdSettingsMgr *settings.SettingsManager, + client client.Client, + generators map[string]generators.Generator, +) *webhook.Webhook { + payloadHandler := &ApplicationSetWebhookPayloadHandler{ + client: client, + generators: generators, + } + + return webhook.NewWebhook( + parallelism, + maxPayloadSize, + argoCdSettings, + argoCdSettingsMgr, + payloadHandler, + ) +} diff --git a/applicationset/webhook/webhook_test.go b/applicationset/webhookhandler/webhookhandler_test.go similarity index 99% rename from applicationset/webhook/webhook_test.go rename to applicationset/webhookhandler/webhookhandler_test.go index 7eaf5b08478c8..daef3b4684258 100644 --- a/applicationset/webhook/webhook_test.go +++ b/applicationset/webhookhandler/webhookhandler_test.go @@ -1,4 +1,4 @@ -package webhook +package webhookhandler import ( "bytes" @@ -211,8 +211,8 @@ func TestWebhookHandler(t *testing.T) { fakeAppWithMergeAndNestedGitGenerator("merge-nested-git-github", namespace, "https://github.com/org/repo"), ).Build() set := argosettings.NewSettingsManager(context.TODO(), fakeClient, namespace) - h, err := NewWebhookHandler(namespace, webhookParallelism, set, fc, mockGenerators()) - require.NoError(t, err) + h := NewWebhook(webhookParallelism, int64(1) * 1024 * 1024 * 1024, &argosettings.ArgoCDSettings{}, set, fc, mockGenerators()) + //require.NoError(t, err) req := httptest.NewRequest(http.MethodPost, "/api/webhook", nil) req.Header.Set(test.headerKey, test.headerValue) @@ -221,9 +221,8 @@ func TestWebhookHandler(t *testing.T) { req.Body = io.NopCloser(bytes.NewReader(eventJSON)) w := httptest.NewRecorder() - h.Handler(w, req) - close(h.queue) - h.Wait() + h.HandleRequest(w, req) + h.CloseAndWait() assert.Equal(t, test.expectedStatusCode, w.Code) list := &v1alpha1.ApplicationSetList{} diff --git a/cmd/argocd-applicationset-controller/commands/applicationset_controller.go b/cmd/argocd-applicationset-controller/commands/applicationset_controller.go index 17bd5444f4a2b..e5fb7c9e9e6d8 100644 --- a/cmd/argocd-applicationset-controller/commands/applicationset_controller.go +++ b/cmd/argocd-applicationset-controller/commands/applicationset_controller.go @@ -18,11 +18,12 @@ import ( "github.com/argoproj/argo-cd/v2/applicationset/controllers" "github.com/argoproj/argo-cd/v2/applicationset/generators" "github.com/argoproj/argo-cd/v2/applicationset/utils" - "github.com/argoproj/argo-cd/v2/applicationset/webhook" cmdutil "github.com/argoproj/argo-cd/v2/cmd/util" "github.com/argoproj/argo-cd/v2/common" "github.com/argoproj/argo-cd/v2/util/env" "github.com/argoproj/argo-cd/v2/util/github_app" + "github.com/argoproj/argo-cd/v2/util/webhook" + webhookHandler "github.com/argoproj/argo-cd/v2/applicationset/webhookhandler" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" @@ -184,16 +185,24 @@ func NewCommand() *cobra.Command { errors.CheckError(err) topLevelGenerators := generators.GetGenerators(ctx, mgr.GetClient(), k8sClient, namespace, argoCDService, dynamicClient, scmConfig) + argoSettings, err := argoSettingsMgr.GetSettings() - // start a webhook server that listens to incoming webhook payloads - webhookHandler, err := webhook.NewWebhookHandler(namespace, webhookParallelism, argoSettingsMgr, mgr.GetClient(), topLevelGenerators) if err != nil { - log.Error(err, "failed to create webhook handler") - } - if webhookHandler != nil { - startWebhookServer(webhookHandler, webhookAddr) + log.Error(err, "Failed to get argocd settings") + os.Exit(1) } + appSetWebhook := webhookHandler.NewWebhook( + webhookParallelism, + argoSettingsMgr.GetMaxWebhookPayloadSize(), + argoSettings, + argoSettingsMgr, + mgr.GetClient(), + topLevelGenerators, + ) + + startWebhookServer(appSetWebhook, webhookAddr) + if err = (&controllers.ApplicationSetReconciler{ Generators: topLevelGenerators, Client: mgr.GetClient(), @@ -256,9 +265,10 @@ func NewCommand() *cobra.Command { return &command } -func startWebhookServer(webhookHandler *webhook.WebhookHandler, webhookAddr string) { +type handleRequest func() +func startWebhookServer(webhook *webhook.Webhook, webhookAddr string) { mux := http.NewServeMux() - mux.HandleFunc("/api/webhook", webhookHandler.Handler) + mux.HandleFunc("/api/webhook", webhook.HandleRequest) go func() { log.Info("Starting webhook server") err := http.ListenAndServe(webhookAddr, mux) diff --git a/server/server.go b/server/server.go index 2222b7b3df87f..b018b73c70088 100644 --- a/server/server.go +++ b/server/server.go @@ -122,7 +122,7 @@ import ( settings_util "github.com/argoproj/argo-cd/v2/util/settings" "github.com/argoproj/argo-cd/v2/util/swagger" tlsutil "github.com/argoproj/argo-cd/v2/util/tls" - "github.com/argoproj/argo-cd/v2/util/webhook" + webhookHandler "github.com/argoproj/argo-cd/v2/server/webhookhandler" ) const ( @@ -1078,9 +1078,21 @@ func (a *ArgoCDServer) newHTTPServer(ctx context.Context, port int, grpcWebHandl // Webhook handler for git events (Note: cache timeouts are hardcoded because API server does not write to cache and not really using them) argoDB := db.NewDB(a.Namespace, a.settingsMgr, a.KubeClientset) - acdWebhookHandler := webhook.NewHandler(a.Namespace, a.ArgoCDServerOpts.ApplicationNamespaces, a.ArgoCDServerOpts.WebhookParallelism, a.AppClientset, a.settings, a.settingsMgr, a.RepoServerCache, a.Cache, argoDB, a.settingsMgr.GetMaxWebhookPayloadSize()) - mux.HandleFunc("/api/webhook", acdWebhookHandler.Handler) + appWebhook := webhookHandler.NewWebhook( + a.ArgoCDServerOpts.WebhookParallelism, + a.settingsMgr.GetMaxWebhookPayloadSize(), + a.settings, + a.settingsMgr, + argoDB, + a.Namespace, + a.ArgoCDServerOpts.ApplicationNamespaces, + a.AppClientset, + a.RepoServerCache, + a.Cache, + ) + + mux.HandleFunc("/api/webhook", appWebhook.HandleRequest) // Serve cli binaries directly from API server registerDownloadHandlers(mux, "/download") diff --git a/util/webhook/testdata/azuredevops-git-push-event.json b/server/webhookhandler/testdata/azuredevops-git-push-event.json similarity index 100% rename from util/webhook/testdata/azuredevops-git-push-event.json rename to server/webhookhandler/testdata/azuredevops-git-push-event.json diff --git a/util/webhook/testdata/bitbucket-server-event.json b/server/webhookhandler/testdata/bitbucket-server-event.json similarity index 100% rename from util/webhook/testdata/bitbucket-server-event.json rename to server/webhookhandler/testdata/bitbucket-server-event.json diff --git a/util/webhook/testdata/github-commit-event.json b/server/webhookhandler/testdata/github-commit-event.json similarity index 100% rename from util/webhook/testdata/github-commit-event.json rename to server/webhookhandler/testdata/github-commit-event.json diff --git a/util/webhook/testdata/github-ping-event.json b/server/webhookhandler/testdata/github-ping-event.json similarity index 100% rename from util/webhook/testdata/github-ping-event.json rename to server/webhookhandler/testdata/github-ping-event.json diff --git a/util/webhook/testdata/github-tag-event.json b/server/webhookhandler/testdata/github-tag-event.json similarity index 100% rename from util/webhook/testdata/github-tag-event.json rename to server/webhookhandler/testdata/github-tag-event.json diff --git a/util/webhook/testdata/gitlab-event.json b/server/webhookhandler/testdata/gitlab-event.json similarity index 100% rename from util/webhook/testdata/gitlab-event.json rename to server/webhookhandler/testdata/gitlab-event.json diff --git a/util/webhook/testdata/gogs-event.json b/server/webhookhandler/testdata/gogs-event.json similarity index 100% rename from util/webhook/testdata/gogs-event.json rename to server/webhookhandler/testdata/gogs-event.json diff --git a/server/webhookhandler/webhookhandler.go b/server/webhookhandler/webhookhandler.go new file mode 100644 index 0000000000000..8bf3f90777bca --- /dev/null +++ b/server/webhookhandler/webhookhandler.go @@ -0,0 +1,355 @@ +package webhookhandler + +import ( + "context" + "fmt" + "regexp" + "strings" + + azureDevOps "github.com/go-playground/webhooks/v6/azuredevops" + "github.com/go-playground/webhooks/v6/bitbucket" + bitbucketServer "github.com/go-playground/webhooks/v6/bitbucket-server" + gitHub "github.com/go-playground/webhooks/v6/github" + gitLab "github.com/go-playground/webhooks/v6/gitlab" + gogsClient "github.com/gogits/go-gogs-client" + log "github.com/sirupsen/logrus" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/argoproj/argo-cd/v2/pkg/apis/application/v1alpha1" + appClientset "github.com/argoproj/argo-cd/v2/pkg/client/clientset/versioned" + "github.com/argoproj/argo-cd/v2/reposerver/cache" + serverCache "github.com/argoproj/argo-cd/v2/server/cache" + "github.com/argoproj/argo-cd/v2/util/app/path" + "github.com/argoproj/argo-cd/v2/util/argo" + "github.com/argoproj/argo-cd/v2/util/db" + "github.com/argoproj/argo-cd/v2/util/glob" + "github.com/argoproj/argo-cd/v2/util/settings" + "github.com/argoproj/argo-cd/v2/util/webhook" +) + +type ApplicationWebhookPayloadHandler struct { + db db.ArgoDB + ns string + appNs []string + appClientset appClientset.Interface + repoCache *cache.Cache + serverCache *serverCache.Cache + argoCdSettingsMgr *settings.SettingsManager +} + +type changeInfo struct { + shaBefore string + shaAfter string +} + +func parseRevision(ref string) string { + refParts := strings.SplitN(ref, "/", 3) + + return refParts[len(refParts)-1] +} + +// affectedRevisionInfo examines a payload from a webhook event, and extracts the repo web URL, the +// revision, and whether or not this affected origin/HEAD (the default branch of the repository). +func affectedRevisionInfo(payloadIf interface{}) (webURLs []string, revision string, change changeInfo, touchedHead bool, changedFiles []string) { + switch payload := payloadIf.(type) { + case azureDevOps.GitPushEvent: + // See: https://learn.microsoft.com/en-us/azure/devops/service-hooks/events?view=azure-devops#git.push + webURLs = append(webURLs, payload.Resource.Repository.RemoteURL) + revision = parseRevision(payload.Resource.RefUpdates[0].Name) + change.shaAfter = parseRevision(payload.Resource.RefUpdates[0].NewObjectID) + change.shaBefore = parseRevision(payload.Resource.RefUpdates[0].OldObjectID) + touchedHead = payload.Resource.RefUpdates[0].Name == payload.Resource.Repository.DefaultBranch + // Unfortunately, Azure DevOps doesn't provide a list of changed files. + case gitHub.PushPayload: + // See: https://developer.github.com/v3/activity/events/types/#pushevent + webURLs = append(webURLs, payload.Repository.HTMLURL) + revision = parseRevision(payload.Ref) + change.shaAfter = parseRevision(payload.After) + change.shaBefore = parseRevision(payload.Before) + touchedHead = bool(payload.Repository.DefaultBranch == revision) + + for _, commit := range payload.Commits { + changedFiles = append(changedFiles, commit.Added...) + changedFiles = append(changedFiles, commit.Modified...) + changedFiles = append(changedFiles, commit.Removed...) + } + case gitLab.PushEventPayload: + // See: https://docs.gitlab.com/ee/user/project/integrations/webhooks.html + webURLs = append(webURLs, payload.Project.WebURL) + revision = parseRevision(payload.Ref) + change.shaAfter = parseRevision(payload.After) + change.shaBefore = parseRevision(payload.Before) + touchedHead = bool(payload.Project.DefaultBranch == revision) + + for _, commit := range payload.Commits { + changedFiles = append(changedFiles, commit.Added...) + changedFiles = append(changedFiles, commit.Modified...) + changedFiles = append(changedFiles, commit.Removed...) + } + case gitLab.TagEventPayload: + // See: https://docs.gitlab.com/ee/user/project/integrations/webhooks.html + // NOTE: This is untested. + webURLs = append(webURLs, payload.Project.WebURL) + revision = parseRevision(payload.Ref) + change.shaAfter = parseRevision(payload.After) + change.shaBefore = parseRevision(payload.Before) + touchedHead = bool(payload.Project.DefaultBranch == revision) + + for _, commit := range payload.Commits { + changedFiles = append(changedFiles, commit.Added...) + changedFiles = append(changedFiles, commit.Modified...) + changedFiles = append(changedFiles, commit.Removed...) + } + case bitbucket.RepoPushPayload: + // See: https://confluence.atlassian.com/bitbucket/event-payloads-740262817.html#EventPayloads-Push + // NOTE: This is untested. + webURLs = append(webURLs, payload.Repository.Links.HTML.Href) + + // TODO: Bitbucket includes multiple changes as part of a single event. We only pick the first + // but need to consider how to handle multiple. + for _, change := range payload.Push.Changes { + revision = change.New.Name + + break + } + + // Not actually sure how to check if the incoming change affected HEAD just by examining the + // payload alone. To be safe, we just return true and let the controller check for itself. + touchedHead = true + case bitbucketServer.RepositoryReferenceChangedPayload: + // Webhook module does not parse the inner links. + if payload.Repository.Links != nil { + for _, l := range payload.Repository.Links["clone"].([]interface{}) { + link := l.(map[string]interface{}) + + if (link["name"] == "http" || link["name"] == "ssh") { + webURLs = append(webURLs, link["href"].(string)) + } + } + } + + // TODO: Bitbucket includes multiple changes as part of a single event. We only pick the first + // but need to consider how to handle multiple. + for _, change := range payload.Changes { + revision = parseRevision(change.Reference.ID) + + break + } + + // Not actually sure how to check if the incoming change affected HEAD just by examining the + // payload alone. To be safe, we just return true and let the controller check for itself. + touchedHead = true + + // Bitbucket does not include a list of changed files anywhere in its payload so we cannot + // update changedFiles for this type of payload. + case gogsClient.PushPayload: + webURLs = append(webURLs, payload.Repo.HTMLURL) + revision = parseRevision(payload.Ref) + change.shaAfter = parseRevision(payload.After) + change.shaBefore = parseRevision(payload.Before) + touchedHead = bool(payload.Repo.DefaultBranch == revision) + + for _, commit := range payload.Commits { + changedFiles = append(changedFiles, commit.Added...) + changedFiles = append(changedFiles, commit.Modified...) + changedFiles = append(changedFiles, commit.Removed...) + } + } + + return webURLs, revision, change, touchedHead, changedFiles +} + +func sourceRevisionHasChanged(source v1alpha1.ApplicationSource, revision string, touchedHead bool) bool { + targetRev := parseRevision(source.TargetRevision) + + if targetRev == "HEAD" || targetRev == "" { // revision is head + return touchedHead + } + + targetRevisionHasPrefixList := []string{"refs/heads/", "refs/tags/"} + + for _, prefix := range targetRevisionHasPrefixList { + if strings.HasPrefix(source.TargetRevision, prefix) { + return revision == targetRev + } + } + + return source.TargetRevision == revision +} + +func sourceUsesURL(source v1alpha1.ApplicationSource, webURL string, repoRegexp *regexp.Regexp) bool { + if !repoRegexp.MatchString(source.RepoURL) { + log.Debugf("%s does not match %s", source.RepoURL, repoRegexp.String()) + + return false + } + + log.Debugf("%s uses repoURL %s", source.RepoURL, webURL) + + return true +} + +func (handler *ApplicationWebhookPayloadHandler) HandlePayload(payload interface{}, handlerWebhook *webhook.Webhook) { + webURLs, revision, change, touchedHead, changedFiles := affectedRevisionInfo(payload) + + // NOTE: The webURL does not include the .git extension. + + if len(webURLs) == 0 { + log.Info("Ignoring webhook event") + + return + } + + for _, webURL := range webURLs { + log.Infof("Received push event repo: %s, revision: %s, touchedHead: %v", webURL, revision, touchedHead) + } + + nsFilter := handler.ns + + if len(handler.appNs) > 0 { + // Retrieve app from all namespaces + nsFilter = "" + } + + appIf := handler.appClientset.ArgoprojV1alpha1().Applications(nsFilter) + apps, err := appIf.List(context.Background(), metav1.ListOptions{}) + + if err != nil { + log.Warnf("Failed to list applications: %v", err) + + return + } + + trackingMethod, err := handler.argoCdSettingsMgr.GetTrackingMethod() + + if err != nil { + log.Warnf("Failed to get trackingMethod: %v", err) + + return + } + + appInstanceLabelKey, err := handler.argoCdSettingsMgr.GetAppInstanceLabelKey() + + if err != nil { + log.Warnf("Failed to get appInstanceLabelKey: %v", err) + + return + } + + // Skip any application that is neither in the control plane's namespace + // nor in the list of enabled namespaces. + var filteredApps []v1alpha1.Application + + for _, app := range apps.Items { + if app.Namespace == handler.ns || glob.MatchStringInList(handler.appNs, app.Namespace, false) { + filteredApps = append(filteredApps, app) + } + } + + for _, webURL := range webURLs { + repoRegexp, err := webhook.GetWebUrlRegex(webURL) + + if err != nil { + log.Warnf("Failed to get repoRegexp: %s", err) + + continue + } + + for _, app := range filteredApps { + for _, source := range app.Spec.GetSources() { + if sourceRevisionHasChanged(source, revision, touchedHead) && sourceUsesURL(source, webURL, repoRegexp) { + refreshPaths := path.GetAppRefreshPaths(&app) + + if path.AppFilesHaveChanged(refreshPaths, changedFiles) { + namespacedAppInterface := handler.appClientset.ArgoprojV1alpha1().Applications(app.ObjectMeta.Namespace) + _, err = argo.RefreshApp(namespacedAppInterface, app.ObjectMeta.Name, v1alpha1.RefreshTypeNormal) + + if err != nil { + log.Warnf("Failed to refresh app '%s' for controller reprocessing: %v", app.ObjectMeta.Name, err) + + continue + } + + // No need to refresh multiple times if multiple sources match. + break + } else if change.shaBefore != "" && change.shaAfter != "" { + if err := handler.storePreviouslyCachedManifests(&app, change, trackingMethod, appInstanceLabelKey); err != nil { + log.Warnf("Failed to store cached manifests of previous revision for app '%s': %v", app.Name, err) + } + } + } + } + } + } +} + +func (handler *ApplicationWebhookPayloadHandler) storePreviouslyCachedManifests(app *v1alpha1.Application, change changeInfo, trackingMethod string, appInstanceLabelKey string) error { + err := argo.ValidateDestination(context.Background(), &app.Spec.Destination, handler.db) + + if err != nil { + return fmt.Errorf("error validating destination: %w", err) + } + + var clusterInfo v1alpha1.ClusterInfo + + err = handler.serverCache.GetClusterInfo(app.Spec.Destination.Server, &clusterInfo) + + if err != nil { + return fmt.Errorf("error getting cluster info: %w", err) + } + + var sources v1alpha1.ApplicationSources + + if app.Spec.HasMultipleSources() { + sources = app.Spec.GetSources() + } else { + sources = append(sources, app.Spec.GetSource()) + } + + refSources, err := argo.GetRefSources(context.Background(), sources, app.Spec.Project, handler.db.GetRepository, []string{}, false) + + if err != nil { + return fmt.Errorf("error getting ref sources: %w", err) + } + + source := app.Spec.GetSource() + + cache.LogDebugManifestCacheKeyFields("moving manifests cache", "webhook app revision changed", change.shaBefore, &source, refSources, &clusterInfo, app.Spec.Destination.Namespace, trackingMethod, appInstanceLabelKey, app.Name, nil) + + if err := handler.repoCache.SetNewRevisionManifests(change.shaAfter, change.shaBefore, &source, refSources, &clusterInfo, app.Spec.Destination.Namespace, trackingMethod, appInstanceLabelKey, app.Name, nil); err != nil { + return fmt.Errorf("error setting new revision manifests: %w", err) + } + + return nil +} + +func NewWebhook( + parallelism int, + maxPayloadSize int64, + argoCdSettings *settings.ArgoCDSettings, + argoCdSettingsMgr *settings.SettingsManager, + db db.ArgoDB, + ns string, + appNs []string, + appClientset appClientset.Interface, + repoCache *cache.Cache, + serverCache *serverCache.Cache, +) *webhook.Webhook { + payloadHandler := &ApplicationWebhookPayloadHandler{ + db: db, + ns: ns, + appNs: appNs, + appClientset: appClientset, + repoCache: repoCache, + serverCache: serverCache, + argoCdSettingsMgr: argoCdSettingsMgr, + } + + return webhook.NewWebhook( + parallelism, + maxPayloadSize, + argoCdSettings, + argoCdSettingsMgr, + payloadHandler, + ) +} diff --git a/server/webhookhandler/webhookhandler_test.go b/server/webhookhandler/webhookhandler_test.go new file mode 100644 index 0000000000000..816a32df00d4a --- /dev/null +++ b/server/webhookhandler/webhookhandler_test.go @@ -0,0 +1,668 @@ +package webhookhandler + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/http/httptest" + "os" + "testing" + "time" + + "k8s.io/apimachinery/pkg/types" + corev1 "k8s.io/api/core/v1" + + "github.com/go-playground/webhooks/v6/bitbucket" + bitbucketserver "github.com/go-playground/webhooks/v6/bitbucket-server" + "github.com/go-playground/webhooks/v6/github" + "github.com/go-playground/webhooks/v6/gitlab" + gogsclient "github.com/gogits/go-gogs-client" + "k8s.io/apimachinery/pkg/runtime" + kubefake "k8s.io/client-go/kubernetes/fake" + kubetesting "k8s.io/client-go/testing" + + "github.com/argoproj/argo-cd/v2/util/cache/appstate" + + "github.com/argoproj/argo-cd/v2/util/db/mocks" + + servercache "github.com/argoproj/argo-cd/v2/server/cache" + + "github.com/sirupsen/logrus/hooks/test" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/argoproj/argo-cd/v2/common" + "github.com/argoproj/argo-cd/v2/pkg/apis/application/v1alpha1" + appclientset "github.com/argoproj/argo-cd/v2/pkg/client/clientset/versioned/fake" + "github.com/argoproj/argo-cd/v2/reposerver/cache" + cacheutil "github.com/argoproj/argo-cd/v2/util/cache" + "github.com/argoproj/argo-cd/v2/util/settings" + webhookUtil "github.com/argoproj/argo-cd/v2/util/webhook" +) + +type fakeSettingsSrc struct{} + +func (f fakeSettingsSrc) GetAppInstanceLabelKey() (string, error) { + return "mycompany.com/appname", nil +} + +func (f fakeSettingsSrc) GetTrackingMethod() (string, error) { + return "", nil +} + +type reactorDef struct { + verb string + resource string + reaction kubetesting.ReactionFunc +} + +func NewMockHandler(reactor *reactorDef, applicationNamespaces []string, objects ...runtime.Object) *webhookUtil.Webhook { + defaultMaxPayloadSize := int64(1) * 1024 * 1024 * 1024 + return NewMockHandlerWithPayloadLimit(reactor, applicationNamespaces, defaultMaxPayloadSize, objects...) +} + +func NewMockHandlerWithPayloadLimit(reactor *reactorDef, applicationNamespaces []string, maxPayloadSize int64, objects ...runtime.Object) *webhookUtil.Webhook { + appClientset := appclientset.NewSimpleClientset(objects...) + if reactor != nil { + defaultReactor := appClientset.ReactionChain[0] + appClientset.ReactionChain = nil + appClientset.AddReactor("list", "*", func(action kubetesting.Action) (handled bool, ret runtime.Object, err error) { + return defaultReactor.React(action) + }) + appClientset.AddReactor(reactor.verb, reactor.resource, reactor.reaction) + } + namespace := "argocd" + fakeClient := newFakeClient(namespace) + settingsMgr := settings.NewSettingsManager(context.TODO(), fakeClient, namespace) + cacheClient := cacheutil.NewCache(cacheutil.NewInMemoryCache(1 * time.Hour)) + + return NewWebhook( + 10, + maxPayloadSize, + &settings.ArgoCDSettings{}, + settingsMgr, + &mocks.ArgoDB{}, + namespace, + applicationNamespaces, + appClientset, + cache.NewCache( + cacheClient, + 1*time.Minute, + 1*time.Minute, + 10*time.Second, + ), + servercache.NewCache(appstate.NewCache(cacheClient, time.Minute), time.Minute, time.Minute, time.Minute), + ) +} + +func TestGitHubCommitEvent(t *testing.T) { + hook := test.NewGlobal() + h := NewMockHandler(nil, []string{}) + req := httptest.NewRequest(http.MethodPost, "/api/webhook", nil) + req.Header.Set("X-GitHub-Event", "push") + eventJSON, err := os.ReadFile("testdata/github-commit-event.json") + require.NoError(t, err) + req.Body = io.NopCloser(bytes.NewReader(eventJSON)) + w := httptest.NewRecorder() + h.HandleRequest(w, req) + h.CloseAndWait() + assert.Equal(t, http.StatusOK, w.Code) + expectedLogResult := "Received push event repo: https://github.com/jessesuen/test-repo, revision: master, touchedHead: true" + actualLogResult := getLogEntry(hook, -3) + assert.Equal(t, expectedLogResult, actualLogResult) + hook.Reset() +} + +func TestAzureDevOpsCommitEvent(t *testing.T) { + hook := test.NewGlobal() + h := NewMockHandler(nil, []string{}) + req := httptest.NewRequest(http.MethodPost, "/api/webhook", nil) + req.Header.Set("X-Vss-Activityid", "abc") + eventJSON, err := os.ReadFile("testdata/azuredevops-git-push-event.json") + require.NoError(t, err) + req.Body = io.NopCloser(bytes.NewReader(eventJSON)) + w := httptest.NewRecorder() + h.HandleRequest(w, req) + h.CloseAndWait() + assert.Equal(t, http.StatusOK, w.Code) + expectedLogResult := "Received push event repo: https://dev.azure.com/alexander0053/alex-test/_git/alex-test, revision: master, touchedHead: true" + actualLogResult := getLogEntry(hook, -3) + assert.Equal(t, expectedLogResult, actualLogResult) + hook.Reset() +} + +// TestGitHubCommitEvent_MultiSource_Refresh makes sure that a webhook will refresh a multi-source app when at least +// one source matches. +func TestGitHubCommitEvent_MultiSource_Refresh(t *testing.T) { + hook := test.NewGlobal() + var patched bool + reaction := func(action kubetesting.Action) (handled bool, ret runtime.Object, err error) { + patchAction := action.(kubetesting.PatchAction) + assert.Equal(t, "app-to-refresh", patchAction.GetName()) + patched = true + return true, nil, nil + } + h := NewMockHandler(&reactorDef{"patch", "applications", reaction}, []string{}, &v1alpha1.Application{ + ObjectMeta: metav1.ObjectMeta{ + Name: "app-to-refresh", + Namespace: "argocd", + }, + Spec: v1alpha1.ApplicationSpec{ + Sources: v1alpha1.ApplicationSources{ + { + RepoURL: "https://github.com/some/unrelated-repo", + Path: ".", + }, + { + RepoURL: "https://github.com/jessesuen/test-repo", + Path: ".", + }, + }, + }, + }, &v1alpha1.Application{ + ObjectMeta: metav1.ObjectMeta{ + Name: "app-to-ignore", + }, + Spec: v1alpha1.ApplicationSpec{ + Sources: v1alpha1.ApplicationSources{ + { + RepoURL: "https://github.com/some/unrelated-repo", + Path: ".", + }, + }, + }, + }, + ) + req := httptest.NewRequest(http.MethodPost, "/api/webhook", nil) + req.Header.Set("X-GitHub-Event", "push") + eventJSON, err := os.ReadFile("testdata/github-commit-event.json") + require.NoError(t, err) + req.Body = io.NopCloser(bytes.NewReader(eventJSON)) + w := httptest.NewRecorder() + h.HandleRequest(w, req) + h.CloseAndWait() + assert.Equal(t, http.StatusOK, w.Code) + expectedLogResult := "Requested app 'app-to-refresh' refresh" + actualLogResult := getLogEntry(hook, -1) + assert.Equal(t, expectedLogResult, actualLogResult) + assert.True(t, patched) + hook.Reset() +} + +// TestGitHubCommitEvent_AppsInOtherNamespaces makes sure that webhooks properly find apps in the configured set of +// allowed namespaces when Apps are allowed in any namespace +func TestGitHubCommitEvent_AppsInOtherNamespaces(t *testing.T) { + hook := test.NewGlobal() + + patchedApps := make([]types.NamespacedName, 0, 3) + reaction := func(action kubetesting.Action) (handled bool, ret runtime.Object, err error) { + patchAction := action.(kubetesting.PatchAction) + patchedApps = append(patchedApps, types.NamespacedName{Name: patchAction.GetName(), Namespace: patchAction.GetNamespace()}) + return true, nil, nil + } + + h := NewMockHandler(&reactorDef{"patch", "applications", reaction}, []string{"end-to-end-tests", "app-team-*"}, + &v1alpha1.Application{ + ObjectMeta: metav1.ObjectMeta{ + Name: "app-to-refresh-in-default-namespace", + Namespace: "argocd", + }, + Spec: v1alpha1.ApplicationSpec{ + Sources: v1alpha1.ApplicationSources{ + { + RepoURL: "https://github.com/jessesuen/test-repo", + Path: ".", + }, + }, + }, + }, &v1alpha1.Application{ + ObjectMeta: metav1.ObjectMeta{ + Name: "app-to-ignore", + Namespace: "kube-system", + }, + Spec: v1alpha1.ApplicationSpec{ + Sources: v1alpha1.ApplicationSources{ + { + RepoURL: "https://github.com/jessesuen/test-repo", + Path: ".", + }, + }, + }, + }, &v1alpha1.Application{ + ObjectMeta: metav1.ObjectMeta{ + Name: "app-to-refresh-in-exact-match-namespace", + Namespace: "end-to-end-tests", + }, + Spec: v1alpha1.ApplicationSpec{ + Sources: v1alpha1.ApplicationSources{ + { + RepoURL: "https://github.com/jessesuen/test-repo", + Path: ".", + }, + }, + }, + }, &v1alpha1.Application{ + ObjectMeta: metav1.ObjectMeta{ + Name: "app-to-refresh-in-globbed-namespace", + Namespace: "app-team-two", + }, + Spec: v1alpha1.ApplicationSpec{ + Sources: v1alpha1.ApplicationSources{ + { + RepoURL: "https://github.com/jessesuen/test-repo", + Path: ".", + }, + }, + }, + }, + ) + req := httptest.NewRequest("POST", "/api/webhook", nil) + req.Header.Set("X-GitHub-Event", "push") + eventJSON, err := os.ReadFile("testdata/github-commit-event.json") + require.NoError(t, err) + req.Body = io.NopCloser(bytes.NewReader(eventJSON)) + w := httptest.NewRecorder() + h.HandleRequest(w, req) + h.CloseAndWait() + assert.Equal(t, http.StatusOK, w.Code) + + logMessages := make([]string, 0, len(hook.Entries)) + + for _, entry := range hook.Entries { + logMessages = append(logMessages, entry.Message) + } + + assert.Contains(t, logMessages, "Requested app 'app-to-refresh-in-default-namespace' refresh") + assert.Contains(t, logMessages, "Requested app 'app-to-refresh-in-exact-match-namespace' refresh") + assert.Contains(t, logMessages, "Requested app 'app-to-refresh-in-globbed-namespace' refresh") + assert.NotContains(t, logMessages, "Requested app 'app-to-ignore' refresh") + + assert.Contains(t, patchedApps, types.NamespacedName{Name: "app-to-refresh-in-default-namespace", Namespace: "argocd"}) + assert.Contains(t, patchedApps, types.NamespacedName{Name: "app-to-refresh-in-exact-match-namespace", Namespace: "end-to-end-tests"}) + assert.Contains(t, patchedApps, types.NamespacedName{Name: "app-to-refresh-in-globbed-namespace", Namespace: "app-team-two"}) + assert.NotContains(t, patchedApps, types.NamespacedName{Name: "app-to-ignore", Namespace: "kube-system"}) + assert.Len(t, patchedApps, 3) + + hook.Reset() +} + +func TestGitHubTagEvent(t *testing.T) { + hook := test.NewGlobal() + h := NewMockHandler(nil, []string{}) + req := httptest.NewRequest(http.MethodPost, "/api/webhook", nil) + req.Header.Set("X-GitHub-Event", "push") + eventJSON, err := os.ReadFile("testdata/github-tag-event.json") + require.NoError(t, err) + req.Body = io.NopCloser(bytes.NewReader(eventJSON)) + w := httptest.NewRecorder() + h.HandleRequest(w, req) + h.CloseAndWait() + assert.Equal(t, http.StatusOK, w.Code) + expectedLogResult := "Received push event repo: https://github.com/jessesuen/test-repo, revision: v1.0, touchedHead: false" + actualLogResult := getLogEntry(hook, -3) + assert.Equal(t, expectedLogResult, actualLogResult) + hook.Reset() +} + +func TestGitHubPingEvent(t *testing.T) { + hook := test.NewGlobal() + h := NewMockHandler(nil, []string{}) + req := httptest.NewRequest(http.MethodPost, "/api/webhook", nil) + req.Header.Set("X-GitHub-Event", "ping") + eventJSON, err := os.ReadFile("testdata/github-ping-event.json") + require.NoError(t, err) + req.Body = io.NopCloser(bytes.NewReader(eventJSON)) + w := httptest.NewRecorder() + h.HandleRequest(w, req) + h.CloseAndWait() + assert.Equal(t, http.StatusOK, w.Code) + expectedLogResult := "Ignoring webhook event" + actualLogResult := getLogEntry(hook, -1) + assert.Equal(t, expectedLogResult, actualLogResult) + hook.Reset() +} + +func TestBitbucketServerRepositoryReferenceChangedEvent(t *testing.T) { + hook := test.NewGlobal() + h := NewMockHandler(nil, []string{}) + req := httptest.NewRequest(http.MethodPost, "/api/webhook", nil) + req.Header.Set("X-Event-Key", "repo:refs_changed") + eventJSON, err := os.ReadFile("testdata/bitbucket-server-event.json") + require.NoError(t, err) + req.Body = io.NopCloser(bytes.NewReader(eventJSON)) + w := httptest.NewRecorder() + h.HandleRequest(w, req) + h.CloseAndWait() + assert.Equal(t, http.StatusOK, w.Code) + expectedLogResultSsh := "Received push event repo: ssh://git@bitbucketserver:7999/myproject/test-repo.git, revision: master, touchedHead: true" + actualLogResultSsh := getLogEntry(hook, -4) + assert.Equal(t, expectedLogResultSsh, actualLogResultSsh) + expectedLogResultHttps := "Received push event repo: https://bitbucketserver/scm/myproject/test-repo.git, revision: master, touchedHead: true" + actualLogResultHttps := getLogEntry(hook, -3) + assert.Equal(t, expectedLogResultHttps, actualLogResultHttps) + hook.Reset() +} + +func TestBitbucketServerRepositoryDiagnosticPingEvent(t *testing.T) { + hook := test.NewGlobal() + h := NewMockHandler(nil, []string{}) + eventJSON := "{\"test\": true}" + req := httptest.NewRequest(http.MethodPost, "/api/webhook", bytes.NewBufferString(eventJSON)) + req.Header.Set("X-Event-Key", "diagnostics:ping") + w := httptest.NewRecorder() + h.HandleRequest(w, req) + h.CloseAndWait() + assert.Equal(t, http.StatusOK, w.Code) + expectedLogResult := "Ignoring webhook event" + actualLogResult := getLogEntry(hook, -1) + assert.Equal(t, expectedLogResult, actualLogResult) + hook.Reset() +} + +func TestGogsPushEvent(t *testing.T) { + hook := test.NewGlobal() + h := NewMockHandler(nil, []string{}) + req := httptest.NewRequest(http.MethodPost, "/api/webhook", nil) + req.Header.Set("X-Gogs-Event", "push") + eventJSON, err := os.ReadFile("testdata/gogs-event.json") + require.NoError(t, err) + req.Body = io.NopCloser(bytes.NewReader(eventJSON)) + w := httptest.NewRecorder() + h.HandleRequest(w, req) + h.CloseAndWait() + assert.Equal(t, http.StatusOK, w.Code) + expectedLogResult := "Received push event repo: http://gogs-server/john/repo-test, revision: master, touchedHead: true" + actualLogResult := getLogEntry(hook, -3) + assert.Equal(t, expectedLogResult, actualLogResult) + hook.Reset() +} + +func TestGitLabPushEvent(t *testing.T) { + hook := test.NewGlobal() + h := NewMockHandler(nil, []string{}) + req := httptest.NewRequest(http.MethodPost, "/api/webhook", nil) + req.Header.Set("X-Gitlab-Event", "Push Hook") + eventJSON, err := os.ReadFile("testdata/gitlab-event.json") + require.NoError(t, err) + req.Body = io.NopCloser(bytes.NewReader(eventJSON)) + w := httptest.NewRecorder() + h.HandleRequest(w, req) + h.CloseAndWait() + assert.Equal(t, http.StatusOK, w.Code) + expectedLogResult := "Received push event repo: https://gitlab/group/name, revision: master, touchedHead: true" + actualLogResult := getLogEntry(hook, -3) + assert.Equal(t, expectedLogResult, actualLogResult) + hook.Reset() +} + +func TestGitLabSystemEvent(t *testing.T) { + hook := test.NewGlobal() + h := NewMockHandler(nil, []string{}) + req := httptest.NewRequest(http.MethodPost, "/api/webhook", nil) + req.Header.Set("X-Gitlab-Event", "System Hook") + eventJSON, err := os.ReadFile("testdata/gitlab-event.json") + require.NoError(t, err) + req.Body = io.NopCloser(bytes.NewReader(eventJSON)) + w := httptest.NewRecorder() + h.HandleRequest(w, req) + h.CloseAndWait() + assert.Equal(t, http.StatusOK, w.Code) + expectedLogResult := "Received push event repo: https://gitlab/group/name, revision: master, touchedHead: true" + actualLogResult := getLogEntry(hook, -3) + assert.Equal(t, expectedLogResult, actualLogResult) + hook.Reset() +} + +func TestInvalidMethod(t *testing.T) { + hook := test.NewGlobal() + h := NewMockHandler(nil, []string{}) + req := httptest.NewRequest(http.MethodGet, "/api/webhook", nil) + req.Header.Set("X-GitHub-Event", "push") + w := httptest.NewRecorder() + h.HandleRequest(w, req) + h.CloseAndWait() + assert.Equal(t, http.StatusMethodNotAllowed, w.Code) + expectedLogResult := "Webhook processing failed: invalid HTTP Method" + actualLogResult := getLogEntry(hook, -1) + assert.Equal(t, expectedLogResult, actualLogResult) + assert.Equal(t, expectedLogResult+"\n", w.Body.String()) + hook.Reset() +} + +func TestInvalidEvent(t *testing.T) { + hook := test.NewGlobal() + h := NewMockHandler(nil, []string{}) + req := httptest.NewRequest(http.MethodPost, "/api/webhook", nil) + req.Header.Set("X-GitHub-Event", "push") + w := httptest.NewRecorder() + h.HandleRequest(w, req) + h.CloseAndWait() + assert.Equal(t, http.StatusBadRequest, w.Code) + expectedLogResult := "Webhook processing failed: The payload is either too large or corrupted. Please check the payload size (must be under 1024 MB) and ensure it is valid JSON" + actualLogResult := getLogEntry(hook, -1) + assert.Equal(t, expectedLogResult, actualLogResult) + assert.Equal(t, expectedLogResult+"\n", w.Body.String()) + hook.Reset() +} + +func TestUnknownEvent(t *testing.T) { + hook := test.NewGlobal() + h := NewMockHandler(nil, []string{}) + req := httptest.NewRequest(http.MethodPost, "/api/webhook", nil) + req.Header.Set("X-Unknown-Event", "push") + w := httptest.NewRecorder() + h.HandleRequest(w, req) + h.CloseAndWait() + assert.Equal(t, http.StatusBadRequest, w.Code) + assert.Equal(t, "Unknown webhook event\n", w.Body.String()) + hook.Reset() +} + +func TestAppRevisionHasChanged(t *testing.T) { + getSource := func(targetRevision string) v1alpha1.ApplicationSource { + return v1alpha1.ApplicationSource{TargetRevision: targetRevision} + } + + testCases := []struct { + name string + source v1alpha1.ApplicationSource + revision string + touchedHead bool + expectHasChanged bool + }{ + {"no target revision, master, touched head", getSource(""), "master", true, true}, + {"no target revision, master, did not touch head", getSource(""), "master", false, false}, + {"dev target revision, master, touched head", getSource("dev"), "master", true, false}, + {"dev target revision, dev, did not touch head", getSource("dev"), "dev", false, true}, + {"refs/heads/dev target revision, master, touched head", getSource("refs/heads/dev"), "master", true, false}, + {"refs/heads/dev target revision, dev, did not touch head", getSource("refs/heads/dev"), "dev", false, true}, + {"env/test target revision, env/test, did not touch head", getSource("env/test"), "env/test", false, true}, + {"refs/heads/env/test target revision, env/test, did not touch head", getSource("refs/heads/env/test"), "env/test", false, true}, + } + + for _, tc := range testCases { + tcc := tc + t.Run(tcc.name, func(t *testing.T) { + t.Parallel() + changed := sourceRevisionHasChanged(tcc.source, tcc.revision, tcc.touchedHead) + assert.Equal(t, tcc.expectHasChanged, changed) + }) + } +} + +func Test_affectedRevisionInfo_appRevisionHasChanged(t *testing.T) { + sourceWithRevision := func(targetRevision string) v1alpha1.ApplicationSource { + return v1alpha1.ApplicationSource{TargetRevision: targetRevision} + } + + githubPushPayload := func(branchName string) github.PushPayload { + // This payload's "ref" member always has the full git ref, according to the field description. + // https://docs.github.com/en/developers/webhooks-and-events/webhooks/webhook-events-and-payloads#push + return github.PushPayload{Ref: "refs/heads/" + branchName} + } + + gitlabPushPayload := func(branchName string) gitlab.PushEventPayload { + // This payload's "ref" member seems to always have the full git ref (based on the example payload). + // https://docs.gitlab.com/ee/user/project/integrations/webhook_events.html#push-events + return gitlab.PushEventPayload{Ref: "refs/heads/" + branchName} + } + + gitlabTagPayload := func(tagName string) gitlab.TagEventPayload { + // This payload's "ref" member seems to always have the full git ref (based on the example payload). + // https://docs.gitlab.com/ee/user/project/integrations/webhook_events.html#tag-events + return gitlab.TagEventPayload{Ref: "refs/tags/" + tagName} + } + + bitbucketPushPayload := func(branchName string) bitbucket.RepoPushPayload { + // The payload's "push.changes[0].new.name" member seems to only have the branch name (based on the example payload). + // https://support.atlassian.com/bitbucket-cloud/docs/event-payloads/#EventPayloads-Push + var pl bitbucket.RepoPushPayload + _ = json.Unmarshal([]byte(fmt.Sprintf(`{"push":{"changes":[{"new":{"name":"%s"}}]}}`, branchName)), &pl) + return pl + } + + bitbucketRefChangedPayload := func(branchName string) bitbucketserver.RepositoryReferenceChangedPayload { + // This payload's "changes[0].ref.id" member seems to always have the full git ref (based on the example payload). + // https://confluence.atlassian.com/bitbucketserver/event-payload-938025882.html#Eventpayload-Push + return bitbucketserver.RepositoryReferenceChangedPayload{ + Changes: []bitbucketserver.RepositoryChange{ + {Reference: bitbucketserver.RepositoryReference{ID: "refs/heads/" + branchName}}, + }, + Repository: bitbucketserver.Repository{Links: map[string]interface{}{"clone": []interface{}{}}}, + } + } + + gogsPushPayload := func(branchName string) gogsclient.PushPayload { + // This payload's "ref" member seems to always have the full git ref (based on the example payload). + // https://gogs.io/docs/features/webhook#event-information + return gogsclient.PushPayload{Ref: "refs/heads/" + branchName, Repo: &gogsclient.Repository{}} + } + + tests := []struct { + hasChanged bool + targetRevision string + hookPayload interface{} + name string + }{ + // Edge cases for bitbucket. + // Bitbucket push events just have tag or branch names instead of fully-qualified refs. If someone were to create + // a branch starting with refs/heads/ or refs/tags/, they couldn't use the branch name in targetRevision. + {false, "refs/heads/x", bitbucketPushPayload("refs/heads/x"), "bitbucket branch name containing 'refs/heads/'"}, + {false, "refs/tags/x", bitbucketPushPayload("refs/tags/x"), "bitbucket branch name containing 'refs/tags/'"}, + {false, "x", bitbucketPushPayload("refs/heads/x"), "bitbucket branch name containing 'refs/heads/', targetRevision with just the part after refs/heads/"}, + {false, "x", bitbucketPushPayload("refs/tags/x"), "bitbucket branch name containing 'refs/tags/', targetRevision with just the part after refs/tags/"}, + // However, a targetRevision prefixed with refs/heads/ or refs/tags/ would match a payload with just the suffix. + {true, "refs/heads/x", bitbucketPushPayload("x"), "bitbucket branch name containing 'refs/heads/', targetRevision with just the part after refs/heads/"}, + {true, "refs/tags/x", bitbucketPushPayload("x"), "bitbucket branch name containing 'refs/tags/', targetRevision with just the part after refs/tags/"}, + // They could also hack around the issue by prepending another refs/heads/ + {true, "refs/heads/refs/heads/x", bitbucketPushPayload("refs/heads/x"), "bitbucket branch name containing 'refs/heads/'"}, + {true, "refs/heads/refs/tags/x", bitbucketPushPayload("refs/tags/x"), "bitbucket branch name containing 'refs/tags/'"}, + + // Standard cases. These tests show that + // 1) Slashes in branch names do not cause missed refreshes. + // 2) Fully-qualifying branches/tags by adding the refs/(heads|tags)/ prefix does not cause missed refreshes. + // 3) Branches and tags are not differentiated. A branch event with branch name 'x' will match all the following: + // a. targetRevision: x + // b. targetRevision: refs/heads/x + // c. targetRevision: refs/tags/x + // A tag event with tag name 'x' will match all of those as well. + + {true, "has/slashes", githubPushPayload("has/slashes"), "github push branch name with slashes, targetRevision not prefixed"}, + {true, "has/slashes", gitlabPushPayload("has/slashes"), "gitlab push branch name with slashes, targetRevision not prefixed"}, + {true, "has/slashes", bitbucketPushPayload("has/slashes"), "bitbucket push branch name with slashes, targetRevision not prefixed"}, + {true, "has/slashes", bitbucketRefChangedPayload("has/slashes"), "bitbucket ref changed branch name with slashes, targetRevision not prefixed"}, + {true, "has/slashes", gogsPushPayload("has/slashes"), "gogs push branch name with slashes, targetRevision not prefixed"}, + + {true, "refs/heads/has/slashes", githubPushPayload("has/slashes"), "github push branch name with slashes, targetRevision branch prefixed"}, + {true, "refs/heads/has/slashes", gitlabPushPayload("has/slashes"), "gitlab push branch name with slashes, targetRevision branch prefixed"}, + {true, "refs/heads/has/slashes", bitbucketPushPayload("has/slashes"), "bitbucket push branch name with slashes, targetRevision branch prefixed"}, + {true, "refs/heads/has/slashes", bitbucketRefChangedPayload("has/slashes"), "bitbucket ref changed branch name with slashes, targetRevision branch prefixed"}, + {true, "refs/heads/has/slashes", gogsPushPayload("has/slashes"), "gogs push branch name with slashes, targetRevision branch prefixed"}, + + // Not testing for refs/tags/has/slashes, because apparently tags can't have slashes: https://stackoverflow.com/a/32850142/684776 + + {true, "no-slashes", githubPushPayload("no-slashes"), "github push branch or tag name without slashes, targetRevision not prefixed"}, + {true, "no-slashes", gitlabTagPayload("no-slashes"), "gitlab tag branch or tag name without slashes, targetRevision not prefixed"}, + {true, "no-slashes", gitlabPushPayload("no-slashes"), "gitlab push branch or tag name without slashes, targetRevision not prefixed"}, + {true, "no-slashes", bitbucketPushPayload("no-slashes"), "bitbucket push branch or tag name without slashes, targetRevision not prefixed"}, + {true, "no-slashes", bitbucketRefChangedPayload("no-slashes"), "bitbucket ref changed branch or tag name without slashes, targetRevision not prefixed"}, + {true, "no-slashes", gogsPushPayload("no-slashes"), "gogs push branch or tag name without slashes, targetRevision not prefixed"}, + + {true, "refs/heads/no-slashes", githubPushPayload("no-slashes"), "github push branch or tag name without slashes, targetRevision branch prefixed"}, + {true, "refs/heads/no-slashes", gitlabTagPayload("no-slashes"), "gitlab tag branch or tag name without slashes, targetRevision branch prefixed"}, + {true, "refs/heads/no-slashes", gitlabPushPayload("no-slashes"), "gitlab push branch or tag name without slashes, targetRevision branch prefixed"}, + {true, "refs/heads/no-slashes", bitbucketPushPayload("no-slashes"), "bitbucket push branch or tag name without slashes, targetRevision branch prefixed"}, + {true, "refs/heads/no-slashes", bitbucketRefChangedPayload("no-slashes"), "bitbucket ref changed branch or tag name without slashes, targetRevision branch prefixed"}, + {true, "refs/heads/no-slashes", gogsPushPayload("no-slashes"), "gogs push branch or tag name without slashes, targetRevision branch prefixed"}, + + {true, "refs/tags/no-slashes", githubPushPayload("no-slashes"), "github push branch or tag name without slashes, targetRevision tag prefixed"}, + {true, "refs/tags/no-slashes", gitlabTagPayload("no-slashes"), "gitlab tag branch or tag name without slashes, targetRevision tag prefixed"}, + {true, "refs/tags/no-slashes", gitlabPushPayload("no-slashes"), "gitlab push branch or tag name without slashes, targetRevision tag prefixed"}, + {true, "refs/tags/no-slashes", bitbucketPushPayload("no-slashes"), "bitbucket push branch or tag name without slashes, targetRevision tag prefixed"}, + {true, "refs/tags/no-slashes", bitbucketRefChangedPayload("no-slashes"), "bitbucket ref changed branch or tag name without slashes, targetRevision tag prefixed"}, + {true, "refs/tags/no-slashes", gogsPushPayload("no-slashes"), "gogs push branch or tag name without slashes, targetRevision tag prefixed"}, + } + for _, testCase := range tests { + testCopy := testCase + t.Run(testCopy.name, func(t *testing.T) { + t.Parallel() + _, revisionFromHook, _, _, _ := affectedRevisionInfo(testCopy.hookPayload) + if got := sourceRevisionHasChanged(sourceWithRevision(testCopy.targetRevision), revisionFromHook, false); got != testCopy.hasChanged { + t.Errorf("sourceRevisionHasChanged() = %v, want %v", got, testCopy.hasChanged) + } + }) + } +} + +func TestGitHubCommitEventMaxPayloadSize(t *testing.T) { + hook := test.NewGlobal() + maxPayloadSize := int64(100) + h := NewMockHandlerWithPayloadLimit(nil, []string{}, maxPayloadSize) + req := httptest.NewRequest(http.MethodPost, "/api/webhook", nil) + req.Header.Set("X-GitHub-Event", "push") + eventJSON, err := os.ReadFile("testdata/github-commit-event.json") + require.NoError(t, err) + req.Body = io.NopCloser(bytes.NewReader(eventJSON)) + w := httptest.NewRecorder() + h.HandleRequest(w, req) + h.CloseAndWait() + assert.Equal(t, http.StatusBadRequest, w.Code) + expectedLogResult := "Webhook processing failed: The payload is either too large or corrupted. Please check the payload size (must be under 0 MB) and ensure it is valid JSON" + actualLogResult := getLogEntry(hook, -1) + assert.Equal(t, expectedLogResult, actualLogResult) + hook.Reset() +} + +func getLogEntry(hook *test.Hook, offset int) string { + allEntries := hook.AllEntries() + index := offset + + if offset < 0 { + index = len(allEntries) + offset + } + + return allEntries[index].Message +} + +func newFakeClient(ns string) *kubefake.Clientset { + s := runtime.NewScheme() + s.AddKnownTypes(v1alpha1.SchemeGroupVersion, &v1alpha1.ApplicationSet{}) + return kubefake.NewSimpleClientset(&corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "argocd-cm", Namespace: ns, Labels: map[string]string{ + "app.kubernetes.io/part-of": "argocd", + }}}, &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: common.ArgoCDSecretName, + Namespace: ns, + Labels: map[string]string{ + "app.kubernetes.io/part-of": "argocd", + }, + }, + Data: map[string][]byte{ + "server.secretkey": nil, + }, + }) +} diff --git a/util/webhook/webhook.go b/util/webhook/webhook.go index 9739055707555..bbcc7470e5928 100644 --- a/util/webhook/webhook.go +++ b/util/webhook/webhook.go @@ -1,41 +1,43 @@ package webhook import ( - "context" "errors" "fmt" "html" "net/http" "net/url" "regexp" - "strings" "sync" - "github.com/go-playground/webhooks/v6/azuredevops" + azureDevOps "github.com/go-playground/webhooks/v6/azuredevops" "github.com/go-playground/webhooks/v6/bitbucket" - bitbucketserver "github.com/go-playground/webhooks/v6/bitbucket-server" - "github.com/go-playground/webhooks/v6/github" - "github.com/go-playground/webhooks/v6/gitlab" + bitbucketServer "github.com/go-playground/webhooks/v6/bitbucket-server" + gitHub "github.com/go-playground/webhooks/v6/github" + gitLab "github.com/go-playground/webhooks/v6/gitlab" "github.com/go-playground/webhooks/v6/gogs" - gogsclient "github.com/gogits/go-gogs-client" log "github.com/sirupsen/logrus" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/argoproj/argo-cd/v2/common" - "github.com/argoproj/argo-cd/v2/pkg/apis/application/v1alpha1" - appclientset "github.com/argoproj/argo-cd/v2/pkg/client/clientset/versioned" - "github.com/argoproj/argo-cd/v2/reposerver/cache" - servercache "github.com/argoproj/argo-cd/v2/server/cache" - "github.com/argoproj/argo-cd/v2/util/app/path" - "github.com/argoproj/argo-cd/v2/util/argo" - "github.com/argoproj/argo-cd/v2/util/db" - "github.com/argoproj/argo-cd/v2/util/glob" "github.com/argoproj/argo-cd/v2/util/settings" ) -type settingsSource interface { - GetAppInstanceLabelKey() (string, error) - GetTrackingMethod() (string, error) +type WebhookPayloadHandler interface { + HandlePayload(payload interface{}, webhook *Webhook) +} + +type Webhook struct { + sync.WaitGroup + parallelism int + maxPayloadSize int64 + argoCdSettingsMgr *settings.SettingsManager + payloadHandler WebhookPayloadHandler + payloadQueue chan interface{} + gitHub *gitHub.Webhook + gitLab *gitLab.Webhook + bitbucket *bitbucket.Webhook + bitbucketServer *bitbucketServer.Webhook + azureDevOps *azureDevOps.Webhook + gogs *gogs.Webhook } // https://www.rfc-editor.org/rfc/rfc3986#section-3.2.1 @@ -44,429 +46,213 @@ const usernameRegex = `[a-zA-Z0-9_\.][a-zA-Z0-9_\.-]{0,30}[a-zA-Z0-9_\.\$-]?` const payloadQueueSize = 50000 -var _ settingsSource = &settings.SettingsManager{} - -type ArgoCDWebhookHandler struct { - sync.WaitGroup // for testing - repoCache *cache.Cache - serverCache *servercache.Cache - db db.ArgoDB - ns string - appNs []string - appClientset appclientset.Interface - github *github.Webhook - gitlab *gitlab.Webhook - bitbucket *bitbucket.Webhook - bitbucketserver *bitbucketserver.Webhook - azuredevops *azuredevops.Webhook - gogs *gogs.Webhook - settingsSrc settingsSource - queue chan interface{} - maxWebhookPayloadSizeB int64 -} +func NewWebhook( + parallelism int, + maxPayloadSize int64, + argoCdSettings *settings.ArgoCDSettings, + argoCdSettingsMgr *settings.SettingsManager, + payloadHandler WebhookPayloadHandler, +) *Webhook { + gitHubWebhook, err := gitHub.New(gitHub.Options.Secret(argoCdSettings.WebhookGitHubSecret)) -func NewHandler(namespace string, applicationNamespaces []string, webhookParallelism int, appClientset appclientset.Interface, set *settings.ArgoCDSettings, settingsSrc settingsSource, repoCache *cache.Cache, serverCache *servercache.Cache, argoDB db.ArgoDB, maxWebhookPayloadSizeB int64) *ArgoCDWebhookHandler { - githubWebhook, err := github.New(github.Options.Secret(set.WebhookGitHubSecret)) if err != nil { log.Warnf("Unable to init the GitHub webhook") } - gitlabWebhook, err := gitlab.New(gitlab.Options.Secret(set.WebhookGitLabSecret)) + + gitLabWebhook, err := gitLab.New(gitLab.Options.Secret(argoCdSettings.WebhookGitLabSecret)) + if err != nil { log.Warnf("Unable to init the GitLab webhook") } - bitbucketWebhook, err := bitbucket.New(bitbucket.Options.UUID(set.WebhookBitbucketUUID)) + + bitbucketWebhook, err := bitbucket.New(bitbucket.Options.UUID(argoCdSettings.WebhookBitbucketUUID)) + if err != nil { log.Warnf("Unable to init the Bitbucket webhook") } - bitbucketserverWebhook, err := bitbucketserver.New(bitbucketserver.Options.Secret(set.WebhookBitbucketServerSecret)) + + bitbucketServerWebhook, err := bitbucketServer.New(bitbucketServer.Options.Secret(argoCdSettings.WebhookBitbucketServerSecret)) + if err != nil { log.Warnf("Unable to init the Bitbucket Server webhook") } - gogsWebhook, err := gogs.New(gogs.Options.Secret(set.WebhookGogsSecret)) + + gogsWebhook, err := gogs.New(gogs.Options.Secret(argoCdSettings.WebhookGogsSecret)) + if err != nil { log.Warnf("Unable to init the Gogs webhook") } - azuredevopsWebhook, err := azuredevops.New(azuredevops.Options.BasicAuth(set.WebhookAzureDevOpsUsername, set.WebhookAzureDevOpsPassword)) + + azureDevOpsWebhook, err := azureDevOps.New(azureDevOps.Options.BasicAuth(argoCdSettings.WebhookAzureDevOpsUsername, argoCdSettings.WebhookAzureDevOpsPassword)) + if err != nil { log.Warnf("Unable to init the Azure DevOps webhook") } - acdWebhook := ArgoCDWebhookHandler{ - ns: namespace, - appNs: applicationNamespaces, - appClientset: appClientset, - github: githubWebhook, - gitlab: gitlabWebhook, - bitbucket: bitbucketWebhook, - bitbucketserver: bitbucketserverWebhook, - azuredevops: azuredevopsWebhook, - gogs: gogsWebhook, - settingsSrc: settingsSrc, - repoCache: repoCache, - serverCache: serverCache, - db: argoDB, - queue: make(chan interface{}, payloadQueueSize), - maxWebhookPayloadSizeB: maxWebhookPayloadSizeB, + webhook := Webhook{ + parallelism: parallelism, + maxPayloadSize: maxPayloadSize, + argoCdSettingsMgr: argoCdSettingsMgr, + payloadHandler: payloadHandler, + payloadQueue: make(chan interface{}, payloadQueueSize), + gitHub: gitHubWebhook, + gitLab: gitLabWebhook, + bitbucket: bitbucketWebhook, + bitbucketServer: bitbucketServerWebhook, + azureDevOps: azureDevOpsWebhook, + gogs: gogsWebhook, } - acdWebhook.startWorkerPool(webhookParallelism) + webhook.startWorkerPool() - return &acdWebhook + return &webhook } -func (a *ArgoCDWebhookHandler) startWorkerPool(webhookParallelism int) { - for i := 0; i < webhookParallelism; i++ { - a.Add(1) +func (webhook *Webhook) startWorkerPool() { + for i := 0; i < webhook.parallelism; i++ { + webhook.Add(1) + go func() { - defer a.Done() + defer webhook.Done() + for { - payload, ok := <-a.queue + payload, ok := <-webhook.payloadQueue + if !ok { return } - a.HandleEvent(payload) - } - }() - } -} -func parseRevision(ref string) string { - refParts := strings.SplitN(ref, "/", 3) - return refParts[len(refParts)-1] -} - -// affectedRevisionInfo examines a payload from a webhook event, and extracts the repo web URL, -// the revision, and whether or not this affected origin/HEAD (the default branch of the repository) -func affectedRevisionInfo(payloadIf interface{}) (webURLs []string, revision string, change changeInfo, touchedHead bool, changedFiles []string) { - switch payload := payloadIf.(type) { - case azuredevops.GitPushEvent: - // See: https://learn.microsoft.com/en-us/azure/devops/service-hooks/events?view=azure-devops#git.push - webURLs = append(webURLs, payload.Resource.Repository.RemoteURL) - revision = parseRevision(payload.Resource.RefUpdates[0].Name) - change.shaAfter = parseRevision(payload.Resource.RefUpdates[0].NewObjectID) - change.shaBefore = parseRevision(payload.Resource.RefUpdates[0].OldObjectID) - touchedHead = payload.Resource.RefUpdates[0].Name == payload.Resource.Repository.DefaultBranch - // unfortunately, Azure DevOps doesn't provide a list of changed files - case github.PushPayload: - // See: https://developer.github.com/v3/activity/events/types/#pushevent - webURLs = append(webURLs, payload.Repository.HTMLURL) - revision = parseRevision(payload.Ref) - change.shaAfter = parseRevision(payload.After) - change.shaBefore = parseRevision(payload.Before) - touchedHead = bool(payload.Repository.DefaultBranch == revision) - for _, commit := range payload.Commits { - changedFiles = append(changedFiles, commit.Added...) - changedFiles = append(changedFiles, commit.Modified...) - changedFiles = append(changedFiles, commit.Removed...) - } - case gitlab.PushEventPayload: - // See: https://docs.gitlab.com/ee/user/project/integrations/webhooks.html - webURLs = append(webURLs, payload.Project.WebURL) - revision = parseRevision(payload.Ref) - change.shaAfter = parseRevision(payload.After) - change.shaBefore = parseRevision(payload.Before) - touchedHead = bool(payload.Project.DefaultBranch == revision) - for _, commit := range payload.Commits { - changedFiles = append(changedFiles, commit.Added...) - changedFiles = append(changedFiles, commit.Modified...) - changedFiles = append(changedFiles, commit.Removed...) - } - case gitlab.TagEventPayload: - // See: https://docs.gitlab.com/ee/user/project/integrations/webhooks.html - // NOTE: this is untested - webURLs = append(webURLs, payload.Project.WebURL) - revision = parseRevision(payload.Ref) - change.shaAfter = parseRevision(payload.After) - change.shaBefore = parseRevision(payload.Before) - touchedHead = bool(payload.Project.DefaultBranch == revision) - for _, commit := range payload.Commits { - changedFiles = append(changedFiles, commit.Added...) - changedFiles = append(changedFiles, commit.Modified...) - changedFiles = append(changedFiles, commit.Removed...) - } - case bitbucket.RepoPushPayload: - // See: https://confluence.atlassian.com/bitbucket/event-payloads-740262817.html#EventPayloads-Push - // NOTE: this is untested - webURLs = append(webURLs, payload.Repository.Links.HTML.Href) - // TODO: bitbucket includes multiple changes as part of a single event. - // We only pick the first but need to consider how to handle multiple - for _, change := range payload.Push.Changes { - revision = change.New.Name - break - } - // Not actually sure how to check if the incoming change affected HEAD just by examining the - // payload alone. To be safe, we just return true and let the controller check for himself. - touchedHead = true - - // Bitbucket does not include a list of changed files anywhere in it's payload - // so we cannot update changedFiles for this type of payload - case bitbucketserver.RepositoryReferenceChangedPayload: - - // Webhook module does not parse the inner links - if payload.Repository.Links != nil { - for _, l := range payload.Repository.Links["clone"].([]interface{}) { - link := l.(map[string]interface{}) - if link["name"] == "http" { - webURLs = append(webURLs, link["href"].(string)) - } - if link["name"] == "ssh" { - webURLs = append(webURLs, link["href"].(string)) - } + webhook.payloadHandler.HandlePayload(payload, webhook) } - } - - // TODO: bitbucket includes multiple changes as part of a single event. - // We only pick the first but need to consider how to handle multiple - for _, change := range payload.Changes { - revision = parseRevision(change.Reference.ID) - break - } - // Not actually sure how to check if the incoming change affected HEAD just by examining the - // payload alone. To be safe, we just return true and let the controller check for himself. - touchedHead = true - - // Bitbucket does not include a list of changed files anywhere in it's payload - // so we cannot update changedFiles for this type of payload - - case gogsclient.PushPayload: - webURLs = append(webURLs, payload.Repo.HTMLURL) - revision = parseRevision(payload.Ref) - change.shaAfter = parseRevision(payload.After) - change.shaBefore = parseRevision(payload.Before) - touchedHead = bool(payload.Repo.DefaultBranch == revision) - for _, commit := range payload.Commits { - changedFiles = append(changedFiles, commit.Added...) - changedFiles = append(changedFiles, commit.Modified...) - changedFiles = append(changedFiles, commit.Removed...) - } + }() } - return webURLs, revision, change, touchedHead, changedFiles } -type changeInfo struct { - shaBefore string - shaAfter string -} - -// HandleEvent handles webhook events for repo push events -func (a *ArgoCDWebhookHandler) HandleEvent(payload interface{}) { - webURLs, revision, change, touchedHead, changedFiles := affectedRevisionInfo(payload) - // NOTE: the webURL does not include the .git extension - if len(webURLs) == 0 { - log.Info("Ignoring webhook event") - return - } - for _, webURL := range webURLs { - log.Infof("Received push event repo: %s, revision: %s, touchedHead: %v", webURL, revision, touchedHead) - } - - nsFilter := a.ns - if len(a.appNs) > 0 { - // Retrieve app from all namespaces - nsFilter = "" - } - - appIf := a.appClientset.ArgoprojV1alpha1().Applications(nsFilter) - apps, err := appIf.List(context.Background(), metav1.ListOptions{}) - if err != nil { - log.Warnf("Failed to list applications: %v", err) - return - } - - trackingMethod, err := a.settingsSrc.GetTrackingMethod() - if err != nil { - log.Warnf("Failed to get trackingMethod: %v", err) - return - } - appInstanceLabelKey, err := a.settingsSrc.GetAppInstanceLabelKey() - if err != nil { - log.Warnf("Failed to get appInstanceLabelKey: %v", err) - return - } - - // Skip any application that is neither in the control plane's namespace - // nor in the list of enabled namespaces. - var filteredApps []v1alpha1.Application - for _, app := range apps.Items { - if app.Namespace == a.ns || glob.MatchStringInList(a.appNs, app.Namespace, false) { - filteredApps = append(filteredApps, app) - } - } +func getUrlRegex(originalUrl string, includePath bool) (*regexp.Regexp, error) { + urlObj, err := url.Parse(originalUrl) - for _, webURL := range webURLs { - repoRegexp, err := getWebUrlRegex(webURL) - if err != nil { - log.Warnf("Failed to get repoRegexp: %s", err) - continue - } - for _, app := range filteredApps { - for _, source := range app.Spec.GetSources() { - if sourceRevisionHasChanged(source, revision, touchedHead) && sourceUsesURL(source, webURL, repoRegexp) { - refreshPaths := path.GetAppRefreshPaths(&app) - if path.AppFilesHaveChanged(refreshPaths, changedFiles) { - namespacedAppInterface := a.appClientset.ArgoprojV1alpha1().Applications(app.ObjectMeta.Namespace) - _, err = argo.RefreshApp(namespacedAppInterface, app.ObjectMeta.Name, v1alpha1.RefreshTypeNormal) - if err != nil { - log.Warnf("Failed to refresh app '%s' for controller reprocessing: %v", app.ObjectMeta.Name, err) - continue - } - // No need to refresh multiple times if multiple sources match. - break - } else if change.shaBefore != "" && change.shaAfter != "" { - if err := a.storePreviouslyCachedManifests(&app, change, trackingMethod, appInstanceLabelKey); err != nil { - log.Warnf("Failed to store cached manifests of previous revision for app '%s': %v", app.Name, err) - } - } - } - } - } - } -} - -// getWebUrlRegex compiles a regex that will match any targetRevision referring to the same repo as the given webURL. -// webURL is expected to be a URL from an SCM webhook payload pointing to the web page for the repo. -func getWebUrlRegex(webURL string) (*regexp.Regexp, error) { - urlObj, err := url.Parse(webURL) if err != nil { - return nil, fmt.Errorf("failed to parse repoURL '%s'", webURL) + return nil, fmt.Errorf("failed to parse repoURL '%s'", originalUrl) } regexEscapedHostname := regexp.QuoteMeta(urlObj.Hostname()) - regexEscapedPath := regexp.QuoteMeta(urlObj.EscapedPath()[1:]) - regexpStr := fmt.Sprintf(`(?i)^(http://|https://|%s@|ssh://(%s@)?((alt)?ssh\.)?)%s(:[0-9]+|)[:/]%s(\.git)?$`, - usernameRegex, usernameRegex, regexEscapedHostname, regexEscapedPath) - repoRegexp, err := regexp.Compile(regexpStr) - if err != nil { - return nil, fmt.Errorf("failed to compile regexp for repoURL '%s'", webURL) - } + regexEscapedPath := "" - return repoRegexp, nil -} - -func (a *ArgoCDWebhookHandler) storePreviouslyCachedManifests(app *v1alpha1.Application, change changeInfo, trackingMethod string, appInstanceLabelKey string) error { - err := argo.ValidateDestination(context.Background(), &app.Spec.Destination, a.db) - if err != nil { - return fmt.Errorf("error validating destination: %w", err) + if includePath { + regexEscapedPath = regexp.QuoteMeta(urlObj.EscapedPath()[1:]) + `(\.git)?` } - var clusterInfo v1alpha1.ClusterInfo - err = a.serverCache.GetClusterInfo(app.Spec.Destination.Server, &clusterInfo) - if err != nil { - return fmt.Errorf("error getting cluster info: %w", err) - } + regexpStr := fmt.Sprintf( + `(?i)^(http://|https://|%s@|ssh://(%s@)?((alt)?ssh\.)?)%s(:[0-9]+|)[:/]%s$`, + usernameRegex, usernameRegex, regexEscapedHostname, regexEscapedPath) - var sources v1alpha1.ApplicationSources - if app.Spec.HasMultipleSources() { - sources = app.Spec.GetSources() - } else { - sources = append(sources, app.Spec.GetSource()) - } + repoRegexp, err := regexp.Compile(regexpStr) - refSources, err := argo.GetRefSources(context.Background(), sources, app.Spec.Project, a.db.GetRepository, []string{}, false) if err != nil { - return fmt.Errorf("error getting ref sources: %w", err) - } - source := app.Spec.GetSource() - cache.LogDebugManifestCacheKeyFields("moving manifests cache", "webhook app revision changed", change.shaBefore, &source, refSources, &clusterInfo, app.Spec.Destination.Namespace, trackingMethod, appInstanceLabelKey, app.Name, nil) - - if err := a.repoCache.SetNewRevisionManifests(change.shaAfter, change.shaBefore, &source, refSources, &clusterInfo, app.Spec.Destination.Namespace, trackingMethod, appInstanceLabelKey, app.Name, nil); err != nil { - return fmt.Errorf("error setting new revision manifests: %w", err) + return nil, fmt.Errorf("failed to compile regexp for repoURL '%s'", originalUrl) } - return nil + return repoRegexp, nil } -func sourceRevisionHasChanged(source v1alpha1.ApplicationSource, revision string, touchedHead bool) bool { - targetRev := parseRevision(source.TargetRevision) - if targetRev == "HEAD" || targetRev == "" { // revision is head - return touchedHead - } - targetRevisionHasPrefixList := []string{"refs/heads/", "refs/tags/"} - for _, prefix := range targetRevisionHasPrefixList { - if strings.HasPrefix(source.TargetRevision, prefix) { - return revision == targetRev - } - } - - return source.TargetRevision == revision +func GetWebUrlRegex(webURL string) (*regexp.Regexp, error) { + return getUrlRegex(webURL, true) } -func sourceUsesURL(source v1alpha1.ApplicationSource, webURL string, repoRegexp *regexp.Regexp) bool { - if !repoRegexp.MatchString(source.RepoURL) { - log.Debugf("%s does not match %s", source.RepoURL, repoRegexp.String()) - return false - } - - log.Debugf("%s uses repoURL %s", source.RepoURL, webURL) - return true +func GetApiUrlRegex(apiURL string) (*regexp.Regexp, error) { + return getUrlRegex(apiURL, false) } -func (a *ArgoCDWebhookHandler) Handler(w http.ResponseWriter, r *http.Request) { +func (webhook *Webhook) HandleRequest(writer http.ResponseWriter, request *http.Request) { var payload interface{} var err error - r.Body = http.MaxBytesReader(w, r.Body, a.maxWebhookPayloadSizeB) + request.Body = http.MaxBytesReader(writer, request.Body, webhook.maxPayloadSize) switch { - case r.Header.Get("X-Vss-Activityid") != "": - payload, err = a.azuredevops.Parse(r, azuredevops.GitPushEventType) - if errors.Is(err, azuredevops.ErrBasicAuthVerificationFailed) { + case request.Header.Get("X-Vss-Activityid") != "": + payload, err = webhook.azureDevOps.Parse(request, azureDevOps.GitPushEventType, azureDevOps.GitPullRequestCreatedEventType, azureDevOps.GitPullRequestUpdatedEventType, azureDevOps.GitPullRequestMergedEventType) + + if errors.Is(err, azureDevOps.ErrBasicAuthVerificationFailed) { log.WithField(common.SecurityField, common.SecurityHigh).Infof("Azure DevOps webhook basic auth verification failed") } - // Gogs needs to be checked before GitHub since it carries both Gogs and (incompatible) GitHub headers - case r.Header.Get("X-Gogs-Event") != "": - payload, err = a.gogs.Parse(r, gogs.PushEvent) + + // Gogs needs to be checked before GitHub since it carries both Gogs and (incompatible) GitHub headers. + case request.Header.Get("X-Gogs-Event") != "": + payload, err = webhook.gogs.Parse(request, gogs.PushEvent) + if errors.Is(err, gogs.ErrHMACVerificationFailed) { log.WithField(common.SecurityField, common.SecurityHigh).Infof("Gogs webhook HMAC verification failed") } - case r.Header.Get("X-GitHub-Event") != "": - payload, err = a.github.Parse(r, github.PushEvent, github.PingEvent) - if errors.Is(err, github.ErrHMACVerificationFailed) { + + case request.Header.Get("X-GitHub-Event") != "": + payload, err = webhook.gitHub.Parse(request, gitHub.PushEvent, gitHub.PullRequestEvent, gitHub.PingEvent) + + if errors.Is(err, gitHub.ErrHMACVerificationFailed) { log.WithField(common.SecurityField, common.SecurityHigh).Infof("GitHub webhook HMAC verification failed") } - case r.Header.Get("X-Gitlab-Event") != "": - payload, err = a.gitlab.Parse(r, gitlab.PushEvents, gitlab.TagEvents, gitlab.SystemHookEvents) - if errors.Is(err, gitlab.ErrGitLabTokenVerificationFailed) { + + case request.Header.Get("X-Gitlab-Event") != "": + payload, err = webhook.gitLab.Parse(request, gitLab.PushEvents, gitLab.TagEvents, gitLab.MergeRequestEvents, gitLab.SystemHookEvents) + + if errors.Is(err, gitLab.ErrGitLabTokenVerificationFailed) { log.WithField(common.SecurityField, common.SecurityHigh).Infof("GitLab webhook token verification failed") } - case r.Header.Get("X-Hook-UUID") != "": - payload, err = a.bitbucket.Parse(r, bitbucket.RepoPushEvent) + + case request.Header.Get("X-Hook-UUID") != "": + payload, err = webhook.bitbucket.Parse(request, bitbucket.RepoPushEvent) + if errors.Is(err, bitbucket.ErrUUIDVerificationFailed) { log.WithField(common.SecurityField, common.SecurityHigh).Infof("BitBucket webhook UUID verification failed") } - case r.Header.Get("X-Event-Key") != "": - payload, err = a.bitbucketserver.Parse(r, bitbucketserver.RepositoryReferenceChangedEvent, bitbucketserver.DiagnosticsPingEvent) - if errors.Is(err, bitbucketserver.ErrHMACVerificationFailed) { + + case request.Header.Get("X-Event-Key") != "": + payload, err = webhook.bitbucketServer.Parse(request, bitbucketServer.RepositoryReferenceChangedEvent, bitbucketServer.DiagnosticsPingEvent) + + if errors.Is(err, bitbucketServer.ErrHMACVerificationFailed) { log.WithField(common.SecurityField, common.SecurityHigh).Infof("BitBucket webhook HMAC verification failed") } + default: log.Debug("Ignoring unknown webhook event") - http.Error(w, "Unknown webhook event", http.StatusBadRequest) + http.Error(writer, "Unknown webhook event", http.StatusBadRequest) + return } if err != nil { - // If the error is due to a large payload, return a more user-friendly error message + // If the error is due to a large payload, return a more user-friendly error message. if err.Error() == "error parsing payload" { - msg := fmt.Sprintf("Webhook processing failed: The payload is either too large or corrupted. Please check the payload size (must be under %v MB) and ensure it is valid JSON", a.maxWebhookPayloadSizeB/1024/1024) + msg := fmt.Sprintf("Webhook processing failed: The payload is either too large or corrupted. Please check the payload size (must be under %v MB) and ensure it is valid JSON", webhook.maxPayloadSize / 1024 / 1024) + log.WithField(common.SecurityField, common.SecurityHigh).Warn(msg) - http.Error(w, msg, http.StatusBadRequest) + http.Error(writer, msg, http.StatusBadRequest) + return } log.Infof("Webhook processing failed: %s", err) + status := http.StatusBadRequest - if r.Method != http.MethodPost { + + if request.Method != http.MethodPost { status = http.StatusMethodNotAllowed } - http.Error(w, fmt.Sprintf("Webhook processing failed: %s", html.EscapeString(err.Error())), status) + + http.Error(writer, fmt.Sprintf("Webhook processing failed: %s", html.EscapeString(err.Error())), status) + return } select { - case a.queue <- payload: + case webhook.payloadQueue <- payload: default: log.Info("Queue is full, discarding webhook payload") - http.Error(w, "Queue is full, discarding webhook payload", http.StatusServiceUnavailable) + http.Error(writer, "Queue is full, discarding webhook payload", http.StatusServiceUnavailable) } } + +func (webhook *Webhook) CloseAndWait() { + close(webhook.payloadQueue) + webhook.Wait() +} diff --git a/util/webhook/webhook_test.go b/util/webhook/webhook_test.go index 0ce976fff03b0..5544add157846 100644 --- a/util/webhook/webhook_test.go +++ b/util/webhook/webhook_test.go @@ -1,606 +1,12 @@ package webhook import ( - "bytes" - "encoding/json" - "fmt" - "io" - "net/http" - "net/http/httptest" - "os" "testing" - "time" - "k8s.io/apimachinery/pkg/types" - - "github.com/go-playground/webhooks/v6/bitbucket" - bitbucketserver "github.com/go-playground/webhooks/v6/bitbucket-server" - "github.com/go-playground/webhooks/v6/github" - "github.com/go-playground/webhooks/v6/gitlab" - gogsclient "github.com/gogits/go-gogs-client" - "k8s.io/apimachinery/pkg/runtime" - kubetesting "k8s.io/client-go/testing" - - "github.com/argoproj/argo-cd/v2/util/cache/appstate" - - "github.com/argoproj/argo-cd/v2/util/db/mocks" - - servercache "github.com/argoproj/argo-cd/v2/server/cache" - - "github.com/sirupsen/logrus/hooks/test" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - "github.com/argoproj/argo-cd/v2/pkg/apis/application/v1alpha1" - appclientset "github.com/argoproj/argo-cd/v2/pkg/client/clientset/versioned/fake" - "github.com/argoproj/argo-cd/v2/reposerver/cache" - cacheutil "github.com/argoproj/argo-cd/v2/util/cache" - "github.com/argoproj/argo-cd/v2/util/settings" ) -type fakeSettingsSrc struct{} - -func (f fakeSettingsSrc) GetAppInstanceLabelKey() (string, error) { - return "mycompany.com/appname", nil -} - -func (f fakeSettingsSrc) GetTrackingMethod() (string, error) { - return "", nil -} - -type reactorDef struct { - verb string - resource string - reaction kubetesting.ReactionFunc -} - -func NewMockHandler(reactor *reactorDef, applicationNamespaces []string, objects ...runtime.Object) *ArgoCDWebhookHandler { - defaultMaxPayloadSize := int64(1) * 1024 * 1024 * 1024 - return NewMockHandlerWithPayloadLimit(reactor, applicationNamespaces, defaultMaxPayloadSize, objects...) -} - -func NewMockHandlerWithPayloadLimit(reactor *reactorDef, applicationNamespaces []string, maxPayloadSize int64, objects ...runtime.Object) *ArgoCDWebhookHandler { - appClientset := appclientset.NewSimpleClientset(objects...) - if reactor != nil { - defaultReactor := appClientset.ReactionChain[0] - appClientset.ReactionChain = nil - appClientset.AddReactor("list", "*", func(action kubetesting.Action) (handled bool, ret runtime.Object, err error) { - return defaultReactor.React(action) - }) - appClientset.AddReactor(reactor.verb, reactor.resource, reactor.reaction) - } - cacheClient := cacheutil.NewCache(cacheutil.NewInMemoryCache(1 * time.Hour)) - - return NewHandler("argocd", applicationNamespaces, 10, appClientset, &settings.ArgoCDSettings{}, &fakeSettingsSrc{}, cache.NewCache( - cacheClient, - 1*time.Minute, - 1*time.Minute, - 10*time.Second, - ), servercache.NewCache(appstate.NewCache(cacheClient, time.Minute), time.Minute, time.Minute, time.Minute), &mocks.ArgoDB{}, maxPayloadSize) -} - -func TestGitHubCommitEvent(t *testing.T) { - hook := test.NewGlobal() - h := NewMockHandler(nil, []string{}) - req := httptest.NewRequest(http.MethodPost, "/api/webhook", nil) - req.Header.Set("X-GitHub-Event", "push") - eventJSON, err := os.ReadFile("testdata/github-commit-event.json") - require.NoError(t, err) - req.Body = io.NopCloser(bytes.NewReader(eventJSON)) - w := httptest.NewRecorder() - h.Handler(w, req) - close(h.queue) - h.Wait() - assert.Equal(t, http.StatusOK, w.Code) - expectedLogResult := "Received push event repo: https://github.com/jessesuen/test-repo, revision: master, touchedHead: true" - assert.Equal(t, expectedLogResult, hook.LastEntry().Message) - hook.Reset() -} - -func TestAzureDevOpsCommitEvent(t *testing.T) { - hook := test.NewGlobal() - h := NewMockHandler(nil, []string{}) - req := httptest.NewRequest(http.MethodPost, "/api/webhook", nil) - req.Header.Set("X-Vss-Activityid", "abc") - eventJSON, err := os.ReadFile("testdata/azuredevops-git-push-event.json") - require.NoError(t, err) - req.Body = io.NopCloser(bytes.NewReader(eventJSON)) - w := httptest.NewRecorder() - h.Handler(w, req) - close(h.queue) - h.Wait() - assert.Equal(t, http.StatusOK, w.Code) - expectedLogResult := "Received push event repo: https://dev.azure.com/alexander0053/alex-test/_git/alex-test, revision: master, touchedHead: true" - assert.Equal(t, expectedLogResult, hook.LastEntry().Message) - hook.Reset() -} - -// TestGitHubCommitEvent_MultiSource_Refresh makes sure that a webhook will refresh a multi-source app when at least -// one source matches. -func TestGitHubCommitEvent_MultiSource_Refresh(t *testing.T) { - hook := test.NewGlobal() - var patched bool - reaction := func(action kubetesting.Action) (handled bool, ret runtime.Object, err error) { - patchAction := action.(kubetesting.PatchAction) - assert.Equal(t, "app-to-refresh", patchAction.GetName()) - patched = true - return true, nil, nil - } - h := NewMockHandler(&reactorDef{"patch", "applications", reaction}, []string{}, &v1alpha1.Application{ - ObjectMeta: metav1.ObjectMeta{ - Name: "app-to-refresh", - Namespace: "argocd", - }, - Spec: v1alpha1.ApplicationSpec{ - Sources: v1alpha1.ApplicationSources{ - { - RepoURL: "https://github.com/some/unrelated-repo", - Path: ".", - }, - { - RepoURL: "https://github.com/jessesuen/test-repo", - Path: ".", - }, - }, - }, - }, &v1alpha1.Application{ - ObjectMeta: metav1.ObjectMeta{ - Name: "app-to-ignore", - }, - Spec: v1alpha1.ApplicationSpec{ - Sources: v1alpha1.ApplicationSources{ - { - RepoURL: "https://github.com/some/unrelated-repo", - Path: ".", - }, - }, - }, - }, - ) - req := httptest.NewRequest(http.MethodPost, "/api/webhook", nil) - req.Header.Set("X-GitHub-Event", "push") - eventJSON, err := os.ReadFile("testdata/github-commit-event.json") - require.NoError(t, err) - req.Body = io.NopCloser(bytes.NewReader(eventJSON)) - w := httptest.NewRecorder() - h.Handler(w, req) - close(h.queue) - h.Wait() - assert.Equal(t, http.StatusOK, w.Code) - expectedLogResult := "Requested app 'app-to-refresh' refresh" - assert.Equal(t, expectedLogResult, hook.LastEntry().Message) - assert.True(t, patched) - hook.Reset() -} - -// TestGitHubCommitEvent_AppsInOtherNamespaces makes sure that webhooks properly find apps in the configured set of -// allowed namespaces when Apps are allowed in any namespace -func TestGitHubCommitEvent_AppsInOtherNamespaces(t *testing.T) { - hook := test.NewGlobal() - - patchedApps := make([]types.NamespacedName, 0, 3) - reaction := func(action kubetesting.Action) (handled bool, ret runtime.Object, err error) { - patchAction := action.(kubetesting.PatchAction) - patchedApps = append(patchedApps, types.NamespacedName{Name: patchAction.GetName(), Namespace: patchAction.GetNamespace()}) - return true, nil, nil - } - - h := NewMockHandler(&reactorDef{"patch", "applications", reaction}, []string{"end-to-end-tests", "app-team-*"}, - &v1alpha1.Application{ - ObjectMeta: metav1.ObjectMeta{ - Name: "app-to-refresh-in-default-namespace", - Namespace: "argocd", - }, - Spec: v1alpha1.ApplicationSpec{ - Sources: v1alpha1.ApplicationSources{ - { - RepoURL: "https://github.com/jessesuen/test-repo", - Path: ".", - }, - }, - }, - }, &v1alpha1.Application{ - ObjectMeta: metav1.ObjectMeta{ - Name: "app-to-ignore", - Namespace: "kube-system", - }, - Spec: v1alpha1.ApplicationSpec{ - Sources: v1alpha1.ApplicationSources{ - { - RepoURL: "https://github.com/jessesuen/test-repo", - Path: ".", - }, - }, - }, - }, &v1alpha1.Application{ - ObjectMeta: metav1.ObjectMeta{ - Name: "app-to-refresh-in-exact-match-namespace", - Namespace: "end-to-end-tests", - }, - Spec: v1alpha1.ApplicationSpec{ - Sources: v1alpha1.ApplicationSources{ - { - RepoURL: "https://github.com/jessesuen/test-repo", - Path: ".", - }, - }, - }, - }, &v1alpha1.Application{ - ObjectMeta: metav1.ObjectMeta{ - Name: "app-to-refresh-in-globbed-namespace", - Namespace: "app-team-two", - }, - Spec: v1alpha1.ApplicationSpec{ - Sources: v1alpha1.ApplicationSources{ - { - RepoURL: "https://github.com/jessesuen/test-repo", - Path: ".", - }, - }, - }, - }, - ) - req := httptest.NewRequest("POST", "/api/webhook", nil) - req.Header.Set("X-GitHub-Event", "push") - eventJSON, err := os.ReadFile("testdata/github-commit-event.json") - require.NoError(t, err) - req.Body = io.NopCloser(bytes.NewReader(eventJSON)) - w := httptest.NewRecorder() - h.Handler(w, req) - close(h.queue) - h.Wait() - assert.Equal(t, http.StatusOK, w.Code) - - logMessages := make([]string, 0, len(hook.Entries)) - - for _, entry := range hook.Entries { - logMessages = append(logMessages, entry.Message) - } - - assert.Contains(t, logMessages, "Requested app 'app-to-refresh-in-default-namespace' refresh") - assert.Contains(t, logMessages, "Requested app 'app-to-refresh-in-exact-match-namespace' refresh") - assert.Contains(t, logMessages, "Requested app 'app-to-refresh-in-globbed-namespace' refresh") - assert.NotContains(t, logMessages, "Requested app 'app-to-ignore' refresh") - - assert.Contains(t, patchedApps, types.NamespacedName{Name: "app-to-refresh-in-default-namespace", Namespace: "argocd"}) - assert.Contains(t, patchedApps, types.NamespacedName{Name: "app-to-refresh-in-exact-match-namespace", Namespace: "end-to-end-tests"}) - assert.Contains(t, patchedApps, types.NamespacedName{Name: "app-to-refresh-in-globbed-namespace", Namespace: "app-team-two"}) - assert.NotContains(t, patchedApps, types.NamespacedName{Name: "app-to-ignore", Namespace: "kube-system"}) - assert.Len(t, patchedApps, 3) - - hook.Reset() -} - -func TestGitHubTagEvent(t *testing.T) { - hook := test.NewGlobal() - h := NewMockHandler(nil, []string{}) - req := httptest.NewRequest(http.MethodPost, "/api/webhook", nil) - req.Header.Set("X-GitHub-Event", "push") - eventJSON, err := os.ReadFile("testdata/github-tag-event.json") - require.NoError(t, err) - req.Body = io.NopCloser(bytes.NewReader(eventJSON)) - w := httptest.NewRecorder() - h.Handler(w, req) - close(h.queue) - h.Wait() - assert.Equal(t, http.StatusOK, w.Code) - expectedLogResult := "Received push event repo: https://github.com/jessesuen/test-repo, revision: v1.0, touchedHead: false" - assert.Equal(t, expectedLogResult, hook.LastEntry().Message) - hook.Reset() -} - -func TestGitHubPingEvent(t *testing.T) { - hook := test.NewGlobal() - h := NewMockHandler(nil, []string{}) - req := httptest.NewRequest(http.MethodPost, "/api/webhook", nil) - req.Header.Set("X-GitHub-Event", "ping") - eventJSON, err := os.ReadFile("testdata/github-ping-event.json") - require.NoError(t, err) - req.Body = io.NopCloser(bytes.NewReader(eventJSON)) - w := httptest.NewRecorder() - h.Handler(w, req) - close(h.queue) - h.Wait() - assert.Equal(t, http.StatusOK, w.Code) - expectedLogResult := "Ignoring webhook event" - assert.Equal(t, expectedLogResult, hook.LastEntry().Message) - hook.Reset() -} - -func TestBitbucketServerRepositoryReferenceChangedEvent(t *testing.T) { - hook := test.NewGlobal() - h := NewMockHandler(nil, []string{}) - req := httptest.NewRequest(http.MethodPost, "/api/webhook", nil) - req.Header.Set("X-Event-Key", "repo:refs_changed") - eventJSON, err := os.ReadFile("testdata/bitbucket-server-event.json") - require.NoError(t, err) - req.Body = io.NopCloser(bytes.NewReader(eventJSON)) - w := httptest.NewRecorder() - h.Handler(w, req) - close(h.queue) - h.Wait() - assert.Equal(t, http.StatusOK, w.Code) - expectedLogResultSsh := "Received push event repo: ssh://git@bitbucketserver:7999/myproject/test-repo.git, revision: master, touchedHead: true" - assert.Equal(t, expectedLogResultSsh, hook.AllEntries()[len(hook.AllEntries())-2].Message) - expectedLogResultHttps := "Received push event repo: https://bitbucketserver/scm/myproject/test-repo.git, revision: master, touchedHead: true" - assert.Equal(t, expectedLogResultHttps, hook.LastEntry().Message) - hook.Reset() -} - -func TestBitbucketServerRepositoryDiagnosticPingEvent(t *testing.T) { - hook := test.NewGlobal() - h := NewMockHandler(nil, []string{}) - eventJSON := "{\"test\": true}" - req := httptest.NewRequest(http.MethodPost, "/api/webhook", bytes.NewBufferString(eventJSON)) - req.Header.Set("X-Event-Key", "diagnostics:ping") - w := httptest.NewRecorder() - h.Handler(w, req) - close(h.queue) - h.Wait() - assert.Equal(t, http.StatusOK, w.Code) - expectedLogResult := "Ignoring webhook event" - assert.Equal(t, expectedLogResult, hook.LastEntry().Message) - hook.Reset() -} - -func TestGogsPushEvent(t *testing.T) { - hook := test.NewGlobal() - h := NewMockHandler(nil, []string{}) - req := httptest.NewRequest(http.MethodPost, "/api/webhook", nil) - req.Header.Set("X-Gogs-Event", "push") - eventJSON, err := os.ReadFile("testdata/gogs-event.json") - require.NoError(t, err) - req.Body = io.NopCloser(bytes.NewReader(eventJSON)) - w := httptest.NewRecorder() - h.Handler(w, req) - close(h.queue) - h.Wait() - assert.Equal(t, http.StatusOK, w.Code) - expectedLogResult := "Received push event repo: http://gogs-server/john/repo-test, revision: master, touchedHead: true" - assert.Equal(t, expectedLogResult, hook.LastEntry().Message) - hook.Reset() -} - -func TestGitLabPushEvent(t *testing.T) { - hook := test.NewGlobal() - h := NewMockHandler(nil, []string{}) - req := httptest.NewRequest(http.MethodPost, "/api/webhook", nil) - req.Header.Set("X-Gitlab-Event", "Push Hook") - eventJSON, err := os.ReadFile("testdata/gitlab-event.json") - require.NoError(t, err) - req.Body = io.NopCloser(bytes.NewReader(eventJSON)) - w := httptest.NewRecorder() - h.Handler(w, req) - close(h.queue) - h.Wait() - assert.Equal(t, http.StatusOK, w.Code) - expectedLogResult := "Received push event repo: https://gitlab/group/name, revision: master, touchedHead: true" - assert.Equal(t, expectedLogResult, hook.LastEntry().Message) - hook.Reset() -} - -func TestGitLabSystemEvent(t *testing.T) { - hook := test.NewGlobal() - h := NewMockHandler(nil, []string{}) - req := httptest.NewRequest(http.MethodPost, "/api/webhook", nil) - req.Header.Set("X-Gitlab-Event", "System Hook") - eventJSON, err := os.ReadFile("testdata/gitlab-event.json") - require.NoError(t, err) - req.Body = io.NopCloser(bytes.NewReader(eventJSON)) - w := httptest.NewRecorder() - h.Handler(w, req) - close(h.queue) - h.Wait() - assert.Equal(t, http.StatusOK, w.Code) - expectedLogResult := "Received push event repo: https://gitlab/group/name, revision: master, touchedHead: true" - assert.Equal(t, expectedLogResult, hook.LastEntry().Message) - hook.Reset() -} - -func TestInvalidMethod(t *testing.T) { - hook := test.NewGlobal() - h := NewMockHandler(nil, []string{}) - req := httptest.NewRequest(http.MethodGet, "/api/webhook", nil) - req.Header.Set("X-GitHub-Event", "push") - w := httptest.NewRecorder() - h.Handler(w, req) - close(h.queue) - h.Wait() - assert.Equal(t, http.StatusMethodNotAllowed, w.Code) - expectedLogResult := "Webhook processing failed: invalid HTTP Method" - assert.Equal(t, expectedLogResult, hook.LastEntry().Message) - assert.Equal(t, expectedLogResult+"\n", w.Body.String()) - hook.Reset() -} - -func TestInvalidEvent(t *testing.T) { - hook := test.NewGlobal() - h := NewMockHandler(nil, []string{}) - req := httptest.NewRequest(http.MethodPost, "/api/webhook", nil) - req.Header.Set("X-GitHub-Event", "push") - w := httptest.NewRecorder() - h.Handler(w, req) - close(h.queue) - h.Wait() - assert.Equal(t, http.StatusBadRequest, w.Code) - expectedLogResult := "Webhook processing failed: The payload is either too large or corrupted. Please check the payload size (must be under 1024 MB) and ensure it is valid JSON" - assert.Equal(t, expectedLogResult, hook.LastEntry().Message) - assert.Equal(t, expectedLogResult+"\n", w.Body.String()) - hook.Reset() -} - -func TestUnknownEvent(t *testing.T) { - hook := test.NewGlobal() - h := NewMockHandler(nil, []string{}) - req := httptest.NewRequest(http.MethodPost, "/api/webhook", nil) - req.Header.Set("X-Unknown-Event", "push") - w := httptest.NewRecorder() - h.Handler(w, req) - close(h.queue) - h.Wait() - assert.Equal(t, http.StatusBadRequest, w.Code) - assert.Equal(t, "Unknown webhook event\n", w.Body.String()) - hook.Reset() -} - -func TestAppRevisionHasChanged(t *testing.T) { - getSource := func(targetRevision string) v1alpha1.ApplicationSource { - return v1alpha1.ApplicationSource{TargetRevision: targetRevision} - } - - testCases := []struct { - name string - source v1alpha1.ApplicationSource - revision string - touchedHead bool - expectHasChanged bool - }{ - {"no target revision, master, touched head", getSource(""), "master", true, true}, - {"no target revision, master, did not touch head", getSource(""), "master", false, false}, - {"dev target revision, master, touched head", getSource("dev"), "master", true, false}, - {"dev target revision, dev, did not touch head", getSource("dev"), "dev", false, true}, - {"refs/heads/dev target revision, master, touched head", getSource("refs/heads/dev"), "master", true, false}, - {"refs/heads/dev target revision, dev, did not touch head", getSource("refs/heads/dev"), "dev", false, true}, - {"env/test target revision, env/test, did not touch head", getSource("env/test"), "env/test", false, true}, - {"refs/heads/env/test target revision, env/test, did not touch head", getSource("refs/heads/env/test"), "env/test", false, true}, - } - - for _, tc := range testCases { - tcc := tc - t.Run(tcc.name, func(t *testing.T) { - t.Parallel() - changed := sourceRevisionHasChanged(tcc.source, tcc.revision, tcc.touchedHead) - assert.Equal(t, tcc.expectHasChanged, changed) - }) - } -} - -func Test_affectedRevisionInfo_appRevisionHasChanged(t *testing.T) { - sourceWithRevision := func(targetRevision string) v1alpha1.ApplicationSource { - return v1alpha1.ApplicationSource{TargetRevision: targetRevision} - } - - githubPushPayload := func(branchName string) github.PushPayload { - // This payload's "ref" member always has the full git ref, according to the field description. - // https://docs.github.com/en/developers/webhooks-and-events/webhooks/webhook-events-and-payloads#push - return github.PushPayload{Ref: "refs/heads/" + branchName} - } - - gitlabPushPayload := func(branchName string) gitlab.PushEventPayload { - // This payload's "ref" member seems to always have the full git ref (based on the example payload). - // https://docs.gitlab.com/ee/user/project/integrations/webhook_events.html#push-events - return gitlab.PushEventPayload{Ref: "refs/heads/" + branchName} - } - - gitlabTagPayload := func(tagName string) gitlab.TagEventPayload { - // This payload's "ref" member seems to always have the full git ref (based on the example payload). - // https://docs.gitlab.com/ee/user/project/integrations/webhook_events.html#tag-events - return gitlab.TagEventPayload{Ref: "refs/tags/" + tagName} - } - - bitbucketPushPayload := func(branchName string) bitbucket.RepoPushPayload { - // The payload's "push.changes[0].new.name" member seems to only have the branch name (based on the example payload). - // https://support.atlassian.com/bitbucket-cloud/docs/event-payloads/#EventPayloads-Push - var pl bitbucket.RepoPushPayload - _ = json.Unmarshal([]byte(fmt.Sprintf(`{"push":{"changes":[{"new":{"name":"%s"}}]}}`, branchName)), &pl) - return pl - } - - bitbucketRefChangedPayload := func(branchName string) bitbucketserver.RepositoryReferenceChangedPayload { - // This payload's "changes[0].ref.id" member seems to always have the full git ref (based on the example payload). - // https://confluence.atlassian.com/bitbucketserver/event-payload-938025882.html#Eventpayload-Push - return bitbucketserver.RepositoryReferenceChangedPayload{ - Changes: []bitbucketserver.RepositoryChange{ - {Reference: bitbucketserver.RepositoryReference{ID: "refs/heads/" + branchName}}, - }, - Repository: bitbucketserver.Repository{Links: map[string]interface{}{"clone": []interface{}{}}}, - } - } - - gogsPushPayload := func(branchName string) gogsclient.PushPayload { - // This payload's "ref" member seems to always have the full git ref (based on the example payload). - // https://gogs.io/docs/features/webhook#event-information - return gogsclient.PushPayload{Ref: "refs/heads/" + branchName, Repo: &gogsclient.Repository{}} - } - - tests := []struct { - hasChanged bool - targetRevision string - hookPayload interface{} - name string - }{ - // Edge cases for bitbucket. - // Bitbucket push events just have tag or branch names instead of fully-qualified refs. If someone were to create - // a branch starting with refs/heads/ or refs/tags/, they couldn't use the branch name in targetRevision. - {false, "refs/heads/x", bitbucketPushPayload("refs/heads/x"), "bitbucket branch name containing 'refs/heads/'"}, - {false, "refs/tags/x", bitbucketPushPayload("refs/tags/x"), "bitbucket branch name containing 'refs/tags/'"}, - {false, "x", bitbucketPushPayload("refs/heads/x"), "bitbucket branch name containing 'refs/heads/', targetRevision with just the part after refs/heads/"}, - {false, "x", bitbucketPushPayload("refs/tags/x"), "bitbucket branch name containing 'refs/tags/', targetRevision with just the part after refs/tags/"}, - // However, a targetRevision prefixed with refs/heads/ or refs/tags/ would match a payload with just the suffix. - {true, "refs/heads/x", bitbucketPushPayload("x"), "bitbucket branch name containing 'refs/heads/', targetRevision with just the part after refs/heads/"}, - {true, "refs/tags/x", bitbucketPushPayload("x"), "bitbucket branch name containing 'refs/tags/', targetRevision with just the part after refs/tags/"}, - // They could also hack around the issue by prepending another refs/heads/ - {true, "refs/heads/refs/heads/x", bitbucketPushPayload("refs/heads/x"), "bitbucket branch name containing 'refs/heads/'"}, - {true, "refs/heads/refs/tags/x", bitbucketPushPayload("refs/tags/x"), "bitbucket branch name containing 'refs/tags/'"}, - - // Standard cases. These tests show that - // 1) Slashes in branch names do not cause missed refreshes. - // 2) Fully-qualifying branches/tags by adding the refs/(heads|tags)/ prefix does not cause missed refreshes. - // 3) Branches and tags are not differentiated. A branch event with branch name 'x' will match all the following: - // a. targetRevision: x - // b. targetRevision: refs/heads/x - // c. targetRevision: refs/tags/x - // A tag event with tag name 'x' will match all of those as well. - - {true, "has/slashes", githubPushPayload("has/slashes"), "github push branch name with slashes, targetRevision not prefixed"}, - {true, "has/slashes", gitlabPushPayload("has/slashes"), "gitlab push branch name with slashes, targetRevision not prefixed"}, - {true, "has/slashes", bitbucketPushPayload("has/slashes"), "bitbucket push branch name with slashes, targetRevision not prefixed"}, - {true, "has/slashes", bitbucketRefChangedPayload("has/slashes"), "bitbucket ref changed branch name with slashes, targetRevision not prefixed"}, - {true, "has/slashes", gogsPushPayload("has/slashes"), "gogs push branch name with slashes, targetRevision not prefixed"}, - - {true, "refs/heads/has/slashes", githubPushPayload("has/slashes"), "github push branch name with slashes, targetRevision branch prefixed"}, - {true, "refs/heads/has/slashes", gitlabPushPayload("has/slashes"), "gitlab push branch name with slashes, targetRevision branch prefixed"}, - {true, "refs/heads/has/slashes", bitbucketPushPayload("has/slashes"), "bitbucket push branch name with slashes, targetRevision branch prefixed"}, - {true, "refs/heads/has/slashes", bitbucketRefChangedPayload("has/slashes"), "bitbucket ref changed branch name with slashes, targetRevision branch prefixed"}, - {true, "refs/heads/has/slashes", gogsPushPayload("has/slashes"), "gogs push branch name with slashes, targetRevision branch prefixed"}, - - // Not testing for refs/tags/has/slashes, because apparently tags can't have slashes: https://stackoverflow.com/a/32850142/684776 - - {true, "no-slashes", githubPushPayload("no-slashes"), "github push branch or tag name without slashes, targetRevision not prefixed"}, - {true, "no-slashes", gitlabTagPayload("no-slashes"), "gitlab tag branch or tag name without slashes, targetRevision not prefixed"}, - {true, "no-slashes", gitlabPushPayload("no-slashes"), "gitlab push branch or tag name without slashes, targetRevision not prefixed"}, - {true, "no-slashes", bitbucketPushPayload("no-slashes"), "bitbucket push branch or tag name without slashes, targetRevision not prefixed"}, - {true, "no-slashes", bitbucketRefChangedPayload("no-slashes"), "bitbucket ref changed branch or tag name without slashes, targetRevision not prefixed"}, - {true, "no-slashes", gogsPushPayload("no-slashes"), "gogs push branch or tag name without slashes, targetRevision not prefixed"}, - - {true, "refs/heads/no-slashes", githubPushPayload("no-slashes"), "github push branch or tag name without slashes, targetRevision branch prefixed"}, - {true, "refs/heads/no-slashes", gitlabTagPayload("no-slashes"), "gitlab tag branch or tag name without slashes, targetRevision branch prefixed"}, - {true, "refs/heads/no-slashes", gitlabPushPayload("no-slashes"), "gitlab push branch or tag name without slashes, targetRevision branch prefixed"}, - {true, "refs/heads/no-slashes", bitbucketPushPayload("no-slashes"), "bitbucket push branch or tag name without slashes, targetRevision branch prefixed"}, - {true, "refs/heads/no-slashes", bitbucketRefChangedPayload("no-slashes"), "bitbucket ref changed branch or tag name without slashes, targetRevision branch prefixed"}, - {true, "refs/heads/no-slashes", gogsPushPayload("no-slashes"), "gogs push branch or tag name without slashes, targetRevision branch prefixed"}, - - {true, "refs/tags/no-slashes", githubPushPayload("no-slashes"), "github push branch or tag name without slashes, targetRevision tag prefixed"}, - {true, "refs/tags/no-slashes", gitlabTagPayload("no-slashes"), "gitlab tag branch or tag name without slashes, targetRevision tag prefixed"}, - {true, "refs/tags/no-slashes", gitlabPushPayload("no-slashes"), "gitlab push branch or tag name without slashes, targetRevision tag prefixed"}, - {true, "refs/tags/no-slashes", bitbucketPushPayload("no-slashes"), "bitbucket push branch or tag name without slashes, targetRevision tag prefixed"}, - {true, "refs/tags/no-slashes", bitbucketRefChangedPayload("no-slashes"), "bitbucket ref changed branch or tag name without slashes, targetRevision tag prefixed"}, - {true, "refs/tags/no-slashes", gogsPushPayload("no-slashes"), "gogs push branch or tag name without slashes, targetRevision tag prefixed"}, - } - for _, testCase := range tests { - testCopy := testCase - t.Run(testCopy.name, func(t *testing.T) { - t.Parallel() - _, revisionFromHook, _, _, _ := affectedRevisionInfo(testCopy.hookPayload) - if got := sourceRevisionHasChanged(sourceWithRevision(testCopy.targetRevision), revisionFromHook, false); got != testCopy.hasChanged { - t.Errorf("sourceRevisionHasChanged() = %v, want %v", got, testCopy.hasChanged) - } - }) - } -} - -func Test_getWebUrlRegex(t *testing.T) { +func Test_GetWebUrlRegex(t *testing.T) { tests := []struct { shouldMatch bool webURL string @@ -632,7 +38,7 @@ func Test_getWebUrlRegex(t *testing.T) { testCopy := testCase t.Run(testCopy.name, func(t *testing.T) { t.Parallel() - regexp, err := getWebUrlRegex(testCopy.webURL) + regexp, err := GetWebUrlRegex(testCopy.webURL) require.NoError(t, err) if matches := regexp.MatchString(testCopy.repo); matches != testCopy.shouldMatch { t.Errorf("sourceRevisionHasChanged() = %v, want %v", matches, testCopy.shouldMatch) @@ -640,22 +46,3 @@ func Test_getWebUrlRegex(t *testing.T) { }) } } - -func TestGitHubCommitEventMaxPayloadSize(t *testing.T) { - hook := test.NewGlobal() - maxPayloadSize := int64(100) - h := NewMockHandlerWithPayloadLimit(nil, []string{}, maxPayloadSize) - req := httptest.NewRequest(http.MethodPost, "/api/webhook", nil) - req.Header.Set("X-GitHub-Event", "push") - eventJSON, err := os.ReadFile("testdata/github-commit-event.json") - require.NoError(t, err) - req.Body = io.NopCloser(bytes.NewReader(eventJSON)) - w := httptest.NewRecorder() - h.Handler(w, req) - close(h.queue) - h.Wait() - assert.Equal(t, http.StatusBadRequest, w.Code) - expectedLogResult := "Webhook processing failed: The payload is either too large or corrupted. Please check the payload size (must be under 0 MB) and ensure it is valid JSON" - assert.Equal(t, expectedLogResult, hook.LastEntry().Message) - hook.Reset() -} From a58f88aa47b2e603938ac772dd748af832c2d25a Mon Sep 17 00:00:00 2001 From: Matthew Bennett Date: Mon, 12 Aug 2024 18:49:03 -0700 Subject: [PATCH 4/9] unify return types and allow for nil webhook Signed-off-by: Matthew Bennett --- .../webhookhandler/webhookhandler.go | 10 ++++- .../webhookhandler/webhookhandler_test.go | 4 +- .../commands/applicationset_controller.go | 11 +++-- server/server.go | 10 ++++- server/webhookhandler/webhookhandler.go | 10 ++++- server/webhookhandler/webhookhandler_test.go | 43 +++++++++++-------- util/webhook/webhook.go | 32 +++++++------- 7 files changed, 74 insertions(+), 46 deletions(-) diff --git a/applicationset/webhookhandler/webhookhandler.go b/applicationset/webhookhandler/webhookhandler.go index c3948505ed96e..b871cd5635582 100644 --- a/applicationset/webhookhandler/webhookhandler.go +++ b/applicationset/webhookhandler/webhookhandler.go @@ -581,17 +581,23 @@ func NewWebhook( argoCdSettingsMgr *settings.SettingsManager, client client.Client, generators map[string]generators.Generator, -) *webhook.Webhook { +) (*webhook.Webhook, error) { payloadHandler := &ApplicationSetWebhookPayloadHandler{ client: client, generators: generators, } - return webhook.NewWebhook( + webhook, err := webhook.NewWebhook( parallelism, maxPayloadSize, argoCdSettings, argoCdSettingsMgr, payloadHandler, ) + + if err != nil { + return nil, err + } + + return webhook, nil } diff --git a/applicationset/webhookhandler/webhookhandler_test.go b/applicationset/webhookhandler/webhookhandler_test.go index daef3b4684258..a8784d8db2059 100644 --- a/applicationset/webhookhandler/webhookhandler_test.go +++ b/applicationset/webhookhandler/webhookhandler_test.go @@ -211,8 +211,8 @@ func TestWebhookHandler(t *testing.T) { fakeAppWithMergeAndNestedGitGenerator("merge-nested-git-github", namespace, "https://github.com/org/repo"), ).Build() set := argosettings.NewSettingsManager(context.TODO(), fakeClient, namespace) - h := NewWebhook(webhookParallelism, int64(1) * 1024 * 1024 * 1024, &argosettings.ArgoCDSettings{}, set, fc, mockGenerators()) - //require.NoError(t, err) + h, err := NewWebhook(webhookParallelism, int64(1) * 1024 * 1024 * 1024, &argosettings.ArgoCDSettings{}, set, fc, mockGenerators()) + require.NoError(t, err) req := httptest.NewRequest(http.MethodPost, "/api/webhook", nil) req.Header.Set(test.headerKey, test.headerValue) diff --git a/cmd/argocd-applicationset-controller/commands/applicationset_controller.go b/cmd/argocd-applicationset-controller/commands/applicationset_controller.go index e5fb7c9e9e6d8..d06bf3b7c1611 100644 --- a/cmd/argocd-applicationset-controller/commands/applicationset_controller.go +++ b/cmd/argocd-applicationset-controller/commands/applicationset_controller.go @@ -192,7 +192,7 @@ func NewCommand() *cobra.Command { os.Exit(1) } - appSetWebhook := webhookHandler.NewWebhook( + appSetWebhook, err := webhookHandler.NewWebhook( webhookParallelism, argoSettingsMgr.GetMaxWebhookPayloadSize(), argoSettings, @@ -201,7 +201,13 @@ func NewCommand() *cobra.Command { topLevelGenerators, ) - startWebhookServer(appSetWebhook, webhookAddr) + if err != nil { + log.Error(err, "failed to create webhook handler") + } + + if appSetWebhook != nil { + startWebhookServer(appSetWebhook, webhookAddr) + } if err = (&controllers.ApplicationSetReconciler{ Generators: topLevelGenerators, @@ -265,7 +271,6 @@ func NewCommand() *cobra.Command { return &command } -type handleRequest func() func startWebhookServer(webhook *webhook.Webhook, webhookAddr string) { mux := http.NewServeMux() mux.HandleFunc("/api/webhook", webhook.HandleRequest) diff --git a/server/server.go b/server/server.go index b018b73c70088..5b631d31c7cd4 100644 --- a/server/server.go +++ b/server/server.go @@ -1079,7 +1079,7 @@ func (a *ArgoCDServer) newHTTPServer(ctx context.Context, port int, grpcWebHandl // Webhook handler for git events (Note: cache timeouts are hardcoded because API server does not write to cache and not really using them) argoDB := db.NewDB(a.Namespace, a.settingsMgr, a.KubeClientset) - appWebhook := webhookHandler.NewWebhook( + appWebhook, err := webhookHandler.NewWebhook( a.ArgoCDServerOpts.WebhookParallelism, a.settingsMgr.GetMaxWebhookPayloadSize(), a.settings, @@ -1092,7 +1092,13 @@ func (a *ArgoCDServer) newHTTPServer(ctx context.Context, port int, grpcWebHandl a.Cache, ) - mux.HandleFunc("/api/webhook", appWebhook.HandleRequest) + if err != nil { + log.Error(err, "failed to create webhook handler") + } + + if appWebhook != nil { + mux.HandleFunc("/api/webhook", appWebhook.HandleRequest) + } // Serve cli binaries directly from API server registerDownloadHandlers(mux, "/download") diff --git a/server/webhookhandler/webhookhandler.go b/server/webhookhandler/webhookhandler.go index 8bf3f90777bca..8ae0df5613ccd 100644 --- a/server/webhookhandler/webhookhandler.go +++ b/server/webhookhandler/webhookhandler.go @@ -334,7 +334,7 @@ func NewWebhook( appClientset appClientset.Interface, repoCache *cache.Cache, serverCache *serverCache.Cache, -) *webhook.Webhook { +) (*webhook.Webhook, error) { payloadHandler := &ApplicationWebhookPayloadHandler{ db: db, ns: ns, @@ -345,11 +345,17 @@ func NewWebhook( argoCdSettingsMgr: argoCdSettingsMgr, } - return webhook.NewWebhook( + webhook, err := webhook.NewWebhook( parallelism, maxPayloadSize, argoCdSettings, argoCdSettingsMgr, payloadHandler, ) + + if err != nil { + return nil, err + } + + return webhook, nil } diff --git a/server/webhookhandler/webhookhandler_test.go b/server/webhookhandler/webhookhandler_test.go index 816a32df00d4a..7ab32bee503c5 100644 --- a/server/webhookhandler/webhookhandler_test.go +++ b/server/webhookhandler/webhookhandler_test.go @@ -60,12 +60,13 @@ type reactorDef struct { reaction kubetesting.ReactionFunc } -func NewMockHandler(reactor *reactorDef, applicationNamespaces []string, objects ...runtime.Object) *webhookUtil.Webhook { +func NewMockHandler(t *testing.T, reactor *reactorDef, applicationNamespaces []string, objects ...runtime.Object) *webhookUtil.Webhook { defaultMaxPayloadSize := int64(1) * 1024 * 1024 * 1024 - return NewMockHandlerWithPayloadLimit(reactor, applicationNamespaces, defaultMaxPayloadSize, objects...) + + return NewMockHandlerWithPayloadLimit(t, reactor, applicationNamespaces, defaultMaxPayloadSize, objects...) } -func NewMockHandlerWithPayloadLimit(reactor *reactorDef, applicationNamespaces []string, maxPayloadSize int64, objects ...runtime.Object) *webhookUtil.Webhook { +func NewMockHandlerWithPayloadLimit(t *testing.T, reactor *reactorDef, applicationNamespaces []string, maxPayloadSize int64, objects ...runtime.Object) *webhookUtil.Webhook { appClientset := appclientset.NewSimpleClientset(objects...) if reactor != nil { defaultReactor := appClientset.ReactionChain[0] @@ -80,7 +81,7 @@ func NewMockHandlerWithPayloadLimit(reactor *reactorDef, applicationNamespaces [ settingsMgr := settings.NewSettingsManager(context.TODO(), fakeClient, namespace) cacheClient := cacheutil.NewCache(cacheutil.NewInMemoryCache(1 * time.Hour)) - return NewWebhook( + webhook, err := NewWebhook( 10, maxPayloadSize, &settings.ArgoCDSettings{}, @@ -97,11 +98,15 @@ func NewMockHandlerWithPayloadLimit(reactor *reactorDef, applicationNamespaces [ ), servercache.NewCache(appstate.NewCache(cacheClient, time.Minute), time.Minute, time.Minute, time.Minute), ) + + require.NoError(t, err) + + return webhook } func TestGitHubCommitEvent(t *testing.T) { hook := test.NewGlobal() - h := NewMockHandler(nil, []string{}) + h := NewMockHandler(t, nil, []string{}) req := httptest.NewRequest(http.MethodPost, "/api/webhook", nil) req.Header.Set("X-GitHub-Event", "push") eventJSON, err := os.ReadFile("testdata/github-commit-event.json") @@ -119,7 +124,7 @@ func TestGitHubCommitEvent(t *testing.T) { func TestAzureDevOpsCommitEvent(t *testing.T) { hook := test.NewGlobal() - h := NewMockHandler(nil, []string{}) + h := NewMockHandler(t, nil, []string{}) req := httptest.NewRequest(http.MethodPost, "/api/webhook", nil) req.Header.Set("X-Vss-Activityid", "abc") eventJSON, err := os.ReadFile("testdata/azuredevops-git-push-event.json") @@ -146,7 +151,7 @@ func TestGitHubCommitEvent_MultiSource_Refresh(t *testing.T) { patched = true return true, nil, nil } - h := NewMockHandler(&reactorDef{"patch", "applications", reaction}, []string{}, &v1alpha1.Application{ + h := NewMockHandler(t, &reactorDef{"patch", "applications", reaction}, []string{}, &v1alpha1.Application{ ObjectMeta: metav1.ObjectMeta{ Name: "app-to-refresh", Namespace: "argocd", @@ -205,7 +210,7 @@ func TestGitHubCommitEvent_AppsInOtherNamespaces(t *testing.T) { return true, nil, nil } - h := NewMockHandler(&reactorDef{"patch", "applications", reaction}, []string{"end-to-end-tests", "app-team-*"}, + h := NewMockHandler(t, &reactorDef{"patch", "applications", reaction}, []string{"end-to-end-tests", "app-team-*"}, &v1alpha1.Application{ ObjectMeta: metav1.ObjectMeta{ Name: "app-to-refresh-in-default-namespace", @@ -292,7 +297,7 @@ func TestGitHubCommitEvent_AppsInOtherNamespaces(t *testing.T) { func TestGitHubTagEvent(t *testing.T) { hook := test.NewGlobal() - h := NewMockHandler(nil, []string{}) + h := NewMockHandler(t, nil, []string{}) req := httptest.NewRequest(http.MethodPost, "/api/webhook", nil) req.Header.Set("X-GitHub-Event", "push") eventJSON, err := os.ReadFile("testdata/github-tag-event.json") @@ -310,7 +315,7 @@ func TestGitHubTagEvent(t *testing.T) { func TestGitHubPingEvent(t *testing.T) { hook := test.NewGlobal() - h := NewMockHandler(nil, []string{}) + h := NewMockHandler(t, nil, []string{}) req := httptest.NewRequest(http.MethodPost, "/api/webhook", nil) req.Header.Set("X-GitHub-Event", "ping") eventJSON, err := os.ReadFile("testdata/github-ping-event.json") @@ -328,7 +333,7 @@ func TestGitHubPingEvent(t *testing.T) { func TestBitbucketServerRepositoryReferenceChangedEvent(t *testing.T) { hook := test.NewGlobal() - h := NewMockHandler(nil, []string{}) + h := NewMockHandler(t, nil, []string{}) req := httptest.NewRequest(http.MethodPost, "/api/webhook", nil) req.Header.Set("X-Event-Key", "repo:refs_changed") eventJSON, err := os.ReadFile("testdata/bitbucket-server-event.json") @@ -349,7 +354,7 @@ func TestBitbucketServerRepositoryReferenceChangedEvent(t *testing.T) { func TestBitbucketServerRepositoryDiagnosticPingEvent(t *testing.T) { hook := test.NewGlobal() - h := NewMockHandler(nil, []string{}) + h := NewMockHandler(t, nil, []string{}) eventJSON := "{\"test\": true}" req := httptest.NewRequest(http.MethodPost, "/api/webhook", bytes.NewBufferString(eventJSON)) req.Header.Set("X-Event-Key", "diagnostics:ping") @@ -365,7 +370,7 @@ func TestBitbucketServerRepositoryDiagnosticPingEvent(t *testing.T) { func TestGogsPushEvent(t *testing.T) { hook := test.NewGlobal() - h := NewMockHandler(nil, []string{}) + h := NewMockHandler(t, nil, []string{}) req := httptest.NewRequest(http.MethodPost, "/api/webhook", nil) req.Header.Set("X-Gogs-Event", "push") eventJSON, err := os.ReadFile("testdata/gogs-event.json") @@ -383,7 +388,7 @@ func TestGogsPushEvent(t *testing.T) { func TestGitLabPushEvent(t *testing.T) { hook := test.NewGlobal() - h := NewMockHandler(nil, []string{}) + h := NewMockHandler(t, nil, []string{}) req := httptest.NewRequest(http.MethodPost, "/api/webhook", nil) req.Header.Set("X-Gitlab-Event", "Push Hook") eventJSON, err := os.ReadFile("testdata/gitlab-event.json") @@ -401,7 +406,7 @@ func TestGitLabPushEvent(t *testing.T) { func TestGitLabSystemEvent(t *testing.T) { hook := test.NewGlobal() - h := NewMockHandler(nil, []string{}) + h := NewMockHandler(t, nil, []string{}) req := httptest.NewRequest(http.MethodPost, "/api/webhook", nil) req.Header.Set("X-Gitlab-Event", "System Hook") eventJSON, err := os.ReadFile("testdata/gitlab-event.json") @@ -419,7 +424,7 @@ func TestGitLabSystemEvent(t *testing.T) { func TestInvalidMethod(t *testing.T) { hook := test.NewGlobal() - h := NewMockHandler(nil, []string{}) + h := NewMockHandler(t, nil, []string{}) req := httptest.NewRequest(http.MethodGet, "/api/webhook", nil) req.Header.Set("X-GitHub-Event", "push") w := httptest.NewRecorder() @@ -435,7 +440,7 @@ func TestInvalidMethod(t *testing.T) { func TestInvalidEvent(t *testing.T) { hook := test.NewGlobal() - h := NewMockHandler(nil, []string{}) + h := NewMockHandler(t, nil, []string{}) req := httptest.NewRequest(http.MethodPost, "/api/webhook", nil) req.Header.Set("X-GitHub-Event", "push") w := httptest.NewRecorder() @@ -451,7 +456,7 @@ func TestInvalidEvent(t *testing.T) { func TestUnknownEvent(t *testing.T) { hook := test.NewGlobal() - h := NewMockHandler(nil, []string{}) + h := NewMockHandler(t, nil, []string{}) req := httptest.NewRequest(http.MethodPost, "/api/webhook", nil) req.Header.Set("X-Unknown-Event", "push") w := httptest.NewRecorder() @@ -621,7 +626,7 @@ func Test_affectedRevisionInfo_appRevisionHasChanged(t *testing.T) { func TestGitHubCommitEventMaxPayloadSize(t *testing.T) { hook := test.NewGlobal() maxPayloadSize := int64(100) - h := NewMockHandlerWithPayloadLimit(nil, []string{}, maxPayloadSize) + h := NewMockHandlerWithPayloadLimit(t, nil, []string{}, maxPayloadSize) req := httptest.NewRequest(http.MethodPost, "/api/webhook", nil) req.Header.Set("X-GitHub-Event", "push") eventJSON, err := os.ReadFile("testdata/github-commit-event.json") diff --git a/util/webhook/webhook.go b/util/webhook/webhook.go index bbcc7470e5928..5b0789284ea1c 100644 --- a/util/webhook/webhook.go +++ b/util/webhook/webhook.go @@ -52,60 +52,60 @@ func NewWebhook( argoCdSettings *settings.ArgoCDSettings, argoCdSettingsMgr *settings.SettingsManager, payloadHandler WebhookPayloadHandler, -) *Webhook { +) (*Webhook, error) { gitHubWebhook, err := gitHub.New(gitHub.Options.Secret(argoCdSettings.WebhookGitHubSecret)) if err != nil { - log.Warnf("Unable to init the GitHub webhook") + return nil, fmt.Errorf("Unable to init the GitHub webhook: %w", err) } gitLabWebhook, err := gitLab.New(gitLab.Options.Secret(argoCdSettings.WebhookGitLabSecret)) if err != nil { - log.Warnf("Unable to init the GitLab webhook") + return nil, fmt.Errorf("Unable to init the GitLab webhook: %w", err) } bitbucketWebhook, err := bitbucket.New(bitbucket.Options.UUID(argoCdSettings.WebhookBitbucketUUID)) if err != nil { - log.Warnf("Unable to init the Bitbucket webhook") + return nil, fmt.Errorf("Unable to init the Bitbucket webhook: %w", err) } bitbucketServerWebhook, err := bitbucketServer.New(bitbucketServer.Options.Secret(argoCdSettings.WebhookBitbucketServerSecret)) if err != nil { - log.Warnf("Unable to init the Bitbucket Server webhook") + return nil, fmt.Errorf("Unable to init the Bitbucket Server webhook: %w", err) } gogsWebhook, err := gogs.New(gogs.Options.Secret(argoCdSettings.WebhookGogsSecret)) if err != nil { - log.Warnf("Unable to init the Gogs webhook") + return nil, fmt.Errorf("Unable to init the Gogs webhook: %w", err) } azureDevOpsWebhook, err := azureDevOps.New(azureDevOps.Options.BasicAuth(argoCdSettings.WebhookAzureDevOpsUsername, argoCdSettings.WebhookAzureDevOpsPassword)) if err != nil { - log.Warnf("Unable to init the Azure DevOps webhook") + return nil, fmt.Errorf("Unable to init the Azure DevOps webhook: %w", err) } webhook := Webhook{ parallelism: parallelism, maxPayloadSize: maxPayloadSize, argoCdSettingsMgr: argoCdSettingsMgr, - payloadHandler: payloadHandler, - payloadQueue: make(chan interface{}, payloadQueueSize), - gitHub: gitHubWebhook, - gitLab: gitLabWebhook, - bitbucket: bitbucketWebhook, - bitbucketServer: bitbucketServerWebhook, - azureDevOps: azureDevOpsWebhook, - gogs: gogsWebhook, + payloadHandler: payloadHandler, + payloadQueue: make(chan interface{}, payloadQueueSize), + gitHub: gitHubWebhook, + gitLab: gitLabWebhook, + bitbucket: bitbucketWebhook, + bitbucketServer: bitbucketServerWebhook, + azureDevOps: azureDevOpsWebhook, + gogs: gogsWebhook, } webhook.startWorkerPool() - return &webhook + return &webhook, nil } func (webhook *Webhook) startWorkerPool() { From dba501e56ae2b8781986d75e83daeb36509da4b5 Mon Sep 17 00:00:00 2001 From: Matthew Bennett Date: Mon, 12 Aug 2024 22:05:12 -0700 Subject: [PATCH 5/9] lint fixes Signed-off-by: Matthew Bennett --- .../webhookhandler/webhookhandler.go | 31 ++++------ .../webhookhandler/webhookhandler_test.go | 2 +- .../commands/applicationset_controller.go | 4 +- server/server.go | 3 +- server/webhookhandler/webhookhandler.go | 57 ++++++++----------- server/webhookhandler/webhookhandler_test.go | 12 +--- util/webhook/webhook.go | 32 ++++------- 7 files changed, 51 insertions(+), 90 deletions(-) diff --git a/applicationset/webhookhandler/webhookhandler.go b/applicationset/webhookhandler/webhookhandler.go index b871cd5635582..ce49661d050f5 100644 --- a/applicationset/webhookhandler/webhookhandler.go +++ b/applicationset/webhookhandler/webhookhandler.go @@ -8,30 +8,26 @@ import ( "strconv" "strings" - "sigs.k8s.io/controller-runtime/pkg/client" log "github.com/sirupsen/logrus" "k8s.io/client-go/util/retry" + "sigs.k8s.io/controller-runtime/pkg/client" "github.com/go-playground/webhooks/v6/azuredevops" "github.com/go-playground/webhooks/v6/github" "github.com/go-playground/webhooks/v6/gitlab" "k8s.io/apimachinery/pkg/types" - "github.com/argoproj/argo-cd/v2/common" - "github.com/argoproj/argo-cd/v2/applicationset/generators" "github.com/argoproj/argo-cd/v2/pkg/apis/application/v1alpha1" - "github.com/argoproj/argo-cd/v2/util/webhook" "github.com/argoproj/argo-cd/v2/util/settings" + "github.com/argoproj/argo-cd/v2/util/webhook" ) type ApplicationSetWebhookPayloadHandler struct { - client client.Client - generators map[string]generators.Generator + client client.Client + generators map[string]generators.Generator } - - type gitGeneratorInfo struct { Revision string TouchedHead bool @@ -70,7 +66,6 @@ func (handler *ApplicationSetWebhookPayloadHandler) HandlePayload(payload interf appSetList := &v1alpha1.ApplicationSetList{} err := handler.client.List(context.Background(), appSetList, &client.ListOptions{}) - if err != nil { log.Errorf("Failed to list applicationsets: %v", err) @@ -95,7 +90,6 @@ func (handler *ApplicationSetWebhookPayloadHandler) HandlePayload(payload interf if shouldRefresh { err := refreshApplicationSet(handler.client, &appSet) - if err != nil { log.Errorf("Failed to refresh ApplicationSet '%s' for controller reprocessing", appSet.Name) @@ -139,7 +133,6 @@ func getGitGeneratorInfo(payload interface{}) *gitGeneratorInfo { log.Infof("Received push event repo: %s, revision: %s, touchedHead: %v", webURL, revision, touchedHead) repoRegexp, err := webhook.GetWebUrlRegex(webURL) - if err != nil { log.Errorf("Failed to get repoRegexp: %s", err) @@ -163,7 +156,6 @@ func getPRGeneratorInfo(payload interface{}) *prGeneratorInfo { apiURL := payload.Repository.URL apiRegexp, err := webhook.GetApiUrlRegex(apiURL) - if err != nil { log.Errorf("Failed to compile regexp for repoURL '%s'", apiURL) @@ -575,15 +567,15 @@ func refreshApplicationSet(c client.Client, appSet *v1alpha1.ApplicationSet) err } func NewWebhook( - parallelism int, - maxPayloadSize int64, - argoCdSettings *settings.ArgoCDSettings, - argoCdSettingsMgr *settings.SettingsManager, - client client.Client, - generators map[string]generators.Generator, + parallelism int, + maxPayloadSize int64, + argoCdSettings *settings.ArgoCDSettings, + argoCdSettingsMgr *settings.SettingsManager, + client client.Client, + generators map[string]generators.Generator, ) (*webhook.Webhook, error) { payloadHandler := &ApplicationSetWebhookPayloadHandler{ - client: client, + client: client, generators: generators, } @@ -594,7 +586,6 @@ func NewWebhook( argoCdSettingsMgr, payloadHandler, ) - if err != nil { return nil, err } diff --git a/applicationset/webhookhandler/webhookhandler_test.go b/applicationset/webhookhandler/webhookhandler_test.go index a8784d8db2059..10bdfbc511fcf 100644 --- a/applicationset/webhookhandler/webhookhandler_test.go +++ b/applicationset/webhookhandler/webhookhandler_test.go @@ -211,7 +211,7 @@ func TestWebhookHandler(t *testing.T) { fakeAppWithMergeAndNestedGitGenerator("merge-nested-git-github", namespace, "https://github.com/org/repo"), ).Build() set := argosettings.NewSettingsManager(context.TODO(), fakeClient, namespace) - h, err := NewWebhook(webhookParallelism, int64(1) * 1024 * 1024 * 1024, &argosettings.ArgoCDSettings{}, set, fc, mockGenerators()) + h, err := NewWebhook(webhookParallelism, int64(1)*1024*1024*1024, &argosettings.ArgoCDSettings{}, set, fc, mockGenerators()) require.NoError(t, err) req := httptest.NewRequest(http.MethodPost, "/api/webhook", nil) diff --git a/cmd/argocd-applicationset-controller/commands/applicationset_controller.go b/cmd/argocd-applicationset-controller/commands/applicationset_controller.go index d06bf3b7c1611..13833244908b4 100644 --- a/cmd/argocd-applicationset-controller/commands/applicationset_controller.go +++ b/cmd/argocd-applicationset-controller/commands/applicationset_controller.go @@ -18,12 +18,12 @@ import ( "github.com/argoproj/argo-cd/v2/applicationset/controllers" "github.com/argoproj/argo-cd/v2/applicationset/generators" "github.com/argoproj/argo-cd/v2/applicationset/utils" + webhookHandler "github.com/argoproj/argo-cd/v2/applicationset/webhookhandler" cmdutil "github.com/argoproj/argo-cd/v2/cmd/util" "github.com/argoproj/argo-cd/v2/common" "github.com/argoproj/argo-cd/v2/util/env" "github.com/argoproj/argo-cd/v2/util/github_app" "github.com/argoproj/argo-cd/v2/util/webhook" - webhookHandler "github.com/argoproj/argo-cd/v2/applicationset/webhookhandler" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" @@ -186,7 +186,6 @@ func NewCommand() *cobra.Command { topLevelGenerators := generators.GetGenerators(ctx, mgr.GetClient(), k8sClient, namespace, argoCDService, dynamicClient, scmConfig) argoSettings, err := argoSettingsMgr.GetSettings() - if err != nil { log.Error(err, "Failed to get argocd settings") os.Exit(1) @@ -200,7 +199,6 @@ func NewCommand() *cobra.Command { mgr.GetClient(), topLevelGenerators, ) - if err != nil { log.Error(err, "failed to create webhook handler") } diff --git a/server/server.go b/server/server.go index 5b631d31c7cd4..5d2f6697da8c7 100644 --- a/server/server.go +++ b/server/server.go @@ -99,6 +99,7 @@ import ( "github.com/argoproj/argo-cd/v2/server/session" "github.com/argoproj/argo-cd/v2/server/settings" "github.com/argoproj/argo-cd/v2/server/version" + webhookHandler "github.com/argoproj/argo-cd/v2/server/webhookhandler" "github.com/argoproj/argo-cd/v2/ui" "github.com/argoproj/argo-cd/v2/util/assets" cacheutil "github.com/argoproj/argo-cd/v2/util/cache" @@ -122,7 +123,6 @@ import ( settings_util "github.com/argoproj/argo-cd/v2/util/settings" "github.com/argoproj/argo-cd/v2/util/swagger" tlsutil "github.com/argoproj/argo-cd/v2/util/tls" - webhookHandler "github.com/argoproj/argo-cd/v2/server/webhookhandler" ) const ( @@ -1091,7 +1091,6 @@ func (a *ArgoCDServer) newHTTPServer(ctx context.Context, port int, grpcWebHandl a.RepoServerCache, a.Cache, ) - if err != nil { log.Error(err, "failed to create webhook handler") } diff --git a/server/webhookhandler/webhookhandler.go b/server/webhookhandler/webhookhandler.go index 8ae0df5613ccd..3fffa8ae36083 100644 --- a/server/webhookhandler/webhookhandler.go +++ b/server/webhookhandler/webhookhandler.go @@ -28,13 +28,13 @@ import ( ) type ApplicationWebhookPayloadHandler struct { - db db.ArgoDB - ns string - appNs []string - appClientset appClientset.Interface - repoCache *cache.Cache - serverCache *serverCache.Cache - argoCdSettingsMgr *settings.SettingsManager + db db.ArgoDB + ns string + appNs []string + appClientset appClientset.Interface + repoCache *cache.Cache + serverCache *serverCache.Cache + argoCdSettingsMgr *settings.SettingsManager } type changeInfo struct { @@ -122,7 +122,7 @@ func affectedRevisionInfo(payloadIf interface{}) (webURLs []string, revision str for _, l := range payload.Repository.Links["clone"].([]interface{}) { link := l.(map[string]interface{}) - if (link["name"] == "http" || link["name"] == "ssh") { + if link["name"] == "http" || link["name"] == "ssh" { webURLs = append(webURLs, link["href"].(string)) } } @@ -213,7 +213,6 @@ func (handler *ApplicationWebhookPayloadHandler) HandlePayload(payload interface appIf := handler.appClientset.ArgoprojV1alpha1().Applications(nsFilter) apps, err := appIf.List(context.Background(), metav1.ListOptions{}) - if err != nil { log.Warnf("Failed to list applications: %v", err) @@ -221,7 +220,6 @@ func (handler *ApplicationWebhookPayloadHandler) HandlePayload(payload interface } trackingMethod, err := handler.argoCdSettingsMgr.GetTrackingMethod() - if err != nil { log.Warnf("Failed to get trackingMethod: %v", err) @@ -229,7 +227,6 @@ func (handler *ApplicationWebhookPayloadHandler) HandlePayload(payload interface } appInstanceLabelKey, err := handler.argoCdSettingsMgr.GetAppInstanceLabelKey() - if err != nil { log.Warnf("Failed to get appInstanceLabelKey: %v", err) @@ -248,7 +245,6 @@ func (handler *ApplicationWebhookPayloadHandler) HandlePayload(payload interface for _, webURL := range webURLs { repoRegexp, err := webhook.GetWebUrlRegex(webURL) - if err != nil { log.Warnf("Failed to get repoRegexp: %s", err) @@ -263,7 +259,6 @@ func (handler *ApplicationWebhookPayloadHandler) HandlePayload(payload interface if path.AppFilesHaveChanged(refreshPaths, changedFiles) { namespacedAppInterface := handler.appClientset.ArgoprojV1alpha1().Applications(app.ObjectMeta.Namespace) _, err = argo.RefreshApp(namespacedAppInterface, app.ObjectMeta.Name, v1alpha1.RefreshTypeNormal) - if err != nil { log.Warnf("Failed to refresh app '%s' for controller reprocessing: %v", app.ObjectMeta.Name, err) @@ -285,7 +280,6 @@ func (handler *ApplicationWebhookPayloadHandler) HandlePayload(payload interface func (handler *ApplicationWebhookPayloadHandler) storePreviouslyCachedManifests(app *v1alpha1.Application, change changeInfo, trackingMethod string, appInstanceLabelKey string) error { err := argo.ValidateDestination(context.Background(), &app.Spec.Destination, handler.db) - if err != nil { return fmt.Errorf("error validating destination: %w", err) } @@ -293,7 +287,6 @@ func (handler *ApplicationWebhookPayloadHandler) storePreviouslyCachedManifests( var clusterInfo v1alpha1.ClusterInfo err = handler.serverCache.GetClusterInfo(app.Spec.Destination.Server, &clusterInfo) - if err != nil { return fmt.Errorf("error getting cluster info: %w", err) } @@ -307,7 +300,6 @@ func (handler *ApplicationWebhookPayloadHandler) storePreviouslyCachedManifests( } refSources, err := argo.GetRefSources(context.Background(), sources, app.Spec.Project, handler.db.GetRepository, []string{}, false) - if err != nil { return fmt.Errorf("error getting ref sources: %w", err) } @@ -324,24 +316,24 @@ func (handler *ApplicationWebhookPayloadHandler) storePreviouslyCachedManifests( } func NewWebhook( - parallelism int, - maxPayloadSize int64, - argoCdSettings *settings.ArgoCDSettings, - argoCdSettingsMgr *settings.SettingsManager, - db db.ArgoDB, - ns string, - appNs []string, - appClientset appClientset.Interface, - repoCache *cache.Cache, - serverCache *serverCache.Cache, + parallelism int, + maxPayloadSize int64, + argoCdSettings *settings.ArgoCDSettings, + argoCdSettingsMgr *settings.SettingsManager, + db db.ArgoDB, + ns string, + appNs []string, + appClientset appClientset.Interface, + repoCache *cache.Cache, + serverCache *serverCache.Cache, ) (*webhook.Webhook, error) { payloadHandler := &ApplicationWebhookPayloadHandler{ - db: db, - ns: ns, - appNs: appNs, - appClientset: appClientset, - repoCache: repoCache, - serverCache: serverCache, + db: db, + ns: ns, + appNs: appNs, + appClientset: appClientset, + repoCache: repoCache, + serverCache: serverCache, argoCdSettingsMgr: argoCdSettingsMgr, } @@ -352,7 +344,6 @@ func NewWebhook( argoCdSettingsMgr, payloadHandler, ) - if err != nil { return nil, err } diff --git a/server/webhookhandler/webhookhandler_test.go b/server/webhookhandler/webhookhandler_test.go index 7ab32bee503c5..767dbbc93bec7 100644 --- a/server/webhookhandler/webhookhandler_test.go +++ b/server/webhookhandler/webhookhandler_test.go @@ -12,7 +12,6 @@ import ( "testing" "time" - "k8s.io/apimachinery/pkg/types" corev1 "k8s.io/api/core/v1" "github.com/go-playground/webhooks/v6/bitbucket" @@ -21,6 +20,7 @@ import ( "github.com/go-playground/webhooks/v6/gitlab" gogsclient "github.com/gogits/go-gogs-client" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" kubefake "k8s.io/client-go/kubernetes/fake" kubetesting "k8s.io/client-go/testing" @@ -44,16 +44,6 @@ import ( webhookUtil "github.com/argoproj/argo-cd/v2/util/webhook" ) -type fakeSettingsSrc struct{} - -func (f fakeSettingsSrc) GetAppInstanceLabelKey() (string, error) { - return "mycompany.com/appname", nil -} - -func (f fakeSettingsSrc) GetTrackingMethod() (string, error) { - return "", nil -} - type reactorDef struct { verb string resource string diff --git a/util/webhook/webhook.go b/util/webhook/webhook.go index 5b0789284ea1c..2f339a415dd3e 100644 --- a/util/webhook/webhook.go +++ b/util/webhook/webhook.go @@ -27,17 +27,17 @@ type WebhookPayloadHandler interface { type Webhook struct { sync.WaitGroup - parallelism int - maxPayloadSize int64 - argoCdSettingsMgr *settings.SettingsManager - payloadHandler WebhookPayloadHandler - payloadQueue chan interface{} - gitHub *gitHub.Webhook - gitLab *gitLab.Webhook - bitbucket *bitbucket.Webhook - bitbucketServer *bitbucketServer.Webhook - azureDevOps *azureDevOps.Webhook - gogs *gogs.Webhook + parallelism int + maxPayloadSize int64 + argoCdSettingsMgr *settings.SettingsManager + payloadHandler WebhookPayloadHandler + payloadQueue chan interface{} + gitHub *gitHub.Webhook + gitLab *gitLab.Webhook + bitbucket *bitbucket.Webhook + bitbucketServer *bitbucketServer.Webhook + azureDevOps *azureDevOps.Webhook + gogs *gogs.Webhook } // https://www.rfc-editor.org/rfc/rfc3986#section-3.2.1 @@ -54,37 +54,31 @@ func NewWebhook( payloadHandler WebhookPayloadHandler, ) (*Webhook, error) { gitHubWebhook, err := gitHub.New(gitHub.Options.Secret(argoCdSettings.WebhookGitHubSecret)) - if err != nil { return nil, fmt.Errorf("Unable to init the GitHub webhook: %w", err) } gitLabWebhook, err := gitLab.New(gitLab.Options.Secret(argoCdSettings.WebhookGitLabSecret)) - if err != nil { return nil, fmt.Errorf("Unable to init the GitLab webhook: %w", err) } bitbucketWebhook, err := bitbucket.New(bitbucket.Options.UUID(argoCdSettings.WebhookBitbucketUUID)) - if err != nil { return nil, fmt.Errorf("Unable to init the Bitbucket webhook: %w", err) } bitbucketServerWebhook, err := bitbucketServer.New(bitbucketServer.Options.Secret(argoCdSettings.WebhookBitbucketServerSecret)) - if err != nil { return nil, fmt.Errorf("Unable to init the Bitbucket Server webhook: %w", err) } gogsWebhook, err := gogs.New(gogs.Options.Secret(argoCdSettings.WebhookGogsSecret)) - if err != nil { return nil, fmt.Errorf("Unable to init the Gogs webhook: %w", err) } azureDevOpsWebhook, err := azureDevOps.New(azureDevOps.Options.BasicAuth(argoCdSettings.WebhookAzureDevOpsUsername, argoCdSettings.WebhookAzureDevOpsPassword)) - if err != nil { return nil, fmt.Errorf("Unable to init the Azure DevOps webhook: %w", err) } @@ -130,7 +124,6 @@ func (webhook *Webhook) startWorkerPool() { func getUrlRegex(originalUrl string, includePath bool) (*regexp.Regexp, error) { urlObj, err := url.Parse(originalUrl) - if err != nil { return nil, fmt.Errorf("failed to parse repoURL '%s'", originalUrl) } @@ -147,7 +140,6 @@ func getUrlRegex(originalUrl string, includePath bool) (*regexp.Regexp, error) { usernameRegex, usernameRegex, regexEscapedHostname, regexEscapedPath) repoRegexp, err := regexp.Compile(regexpStr) - if err != nil { return nil, fmt.Errorf("failed to compile regexp for repoURL '%s'", originalUrl) } @@ -223,7 +215,7 @@ func (webhook *Webhook) HandleRequest(writer http.ResponseWriter, request *http. if err != nil { // If the error is due to a large payload, return a more user-friendly error message. if err.Error() == "error parsing payload" { - msg := fmt.Sprintf("Webhook processing failed: The payload is either too large or corrupted. Please check the payload size (must be under %v MB) and ensure it is valid JSON", webhook.maxPayloadSize / 1024 / 1024) + msg := fmt.Sprintf("Webhook processing failed: The payload is either too large or corrupted. Please check the payload size (must be under %v MB) and ensure it is valid JSON", webhook.maxPayloadSize/1024/1024) log.WithField(common.SecurityField, common.SecurityHigh).Warn(msg) http.Error(writer, msg, http.StatusBadRequest) From 40a16ce36d4d49a206eaa941108880d1f7682017 Mon Sep 17 00:00:00 2001 From: Matthew Bennett Date: Mon, 12 Aug 2024 22:12:26 -0700 Subject: [PATCH 6/9] missing import Signed-off-by: Matthew Bennett --- applicationset/webhookhandler/webhookhandler.go | 1 + 1 file changed, 1 insertion(+) diff --git a/applicationset/webhookhandler/webhookhandler.go b/applicationset/webhookhandler/webhookhandler.go index ce49661d050f5..84f2a701358b8 100644 --- a/applicationset/webhookhandler/webhookhandler.go +++ b/applicationset/webhookhandler/webhookhandler.go @@ -18,6 +18,7 @@ import ( "k8s.io/apimachinery/pkg/types" "github.com/argoproj/argo-cd/v2/applicationset/generators" + "github.com/argoproj/argo-cd/v2/common" "github.com/argoproj/argo-cd/v2/pkg/apis/application/v1alpha1" "github.com/argoproj/argo-cd/v2/util/settings" "github.com/argoproj/argo-cd/v2/util/webhook" From f4e077344b700da0b81d20e1b1d3e6048c016873 Mon Sep 17 00:00:00 2001 From: Matthew Bennett Date: Tue, 13 Aug 2024 10:08:52 -0700 Subject: [PATCH 7/9] use numbered inputs so caller can provide regex format Signed-off-by: Matthew Bennett --- util/webhook/webhook.go | 20 +++++++------------- util/webhook/webhook_test.go | 28 +++++++++++++++++++++++++++- 2 files changed, 34 insertions(+), 14 deletions(-) diff --git a/util/webhook/webhook.go b/util/webhook/webhook.go index 0c51d1809a649..6429c522dc5a7 100644 --- a/util/webhook/webhook.go +++ b/util/webhook/webhook.go @@ -126,22 +126,16 @@ func (webhook *Webhook) startWorkerPool() { } } -func getUrlRegex(originalUrl string, includePath bool) (*regexp.Regexp, error) { +func getUrlRegex(originalUrl string, regexpFormat string) (*regexp.Regexp, error) { urlObj, err := url.Parse(originalUrl) if err != nil { return nil, fmt.Errorf("failed to parse repoURL '%s'", originalUrl) } regexEscapedHostname := regexp.QuoteMeta(urlObj.Hostname()) - regexEscapedPath := "" + regexEscapedPath := regexp.QuoteMeta(urlObj.EscapedPath()[1:]) + `(\.git)?` - if includePath { - regexEscapedPath = regexp.QuoteMeta(urlObj.EscapedPath()[1:]) + `(\.git)?` - } - - regexpStr := fmt.Sprintf( - `(?i)^(http://|https://|%s@|ssh://(%s@)?((alt)?ssh\.)?)%s(:[0-9]+|)[:/]%s$`, - usernameRegex, usernameRegex, regexEscapedHostname, regexEscapedPath) + regexpStr := fmt.Sprintf(regexpFormat, usernameRegex, regexEscapedHostname, regexEscapedPath) repoRegexp, err := regexp.Compile(regexpStr) if err != nil { @@ -151,12 +145,12 @@ func getUrlRegex(originalUrl string, includePath bool) (*regexp.Regexp, error) { return repoRegexp, nil } -func GetWebUrlRegex(webURL string) (*regexp.Regexp, error) { - return getUrlRegex(webURL, true) +func GetWebUrlRegex(originalUrl string) (*regexp.Regexp, error) { + return getUrlRegex(originalUrl, `(?i)^(https?://|%[1]s@|ssh://(%[1]s@)?((alt)?ssh\.)?)%[2]s(:[0-9]+)?[:/]%[3]s$`) } -func GetApiUrlRegex(apiURL string) (*regexp.Regexp, error) { - return getUrlRegex(apiURL, false) +func GetApiUrlRegex(originalUrl string) (*regexp.Regexp, error) { + return getUrlRegex(originalUrl, `(?i)^https?://%[2]s(:[0-9]+)?/?$`) } func (webhook *Webhook) HandleRequest(writer http.ResponseWriter, request *http.Request) { diff --git a/util/webhook/webhook_test.go b/util/webhook/webhook_test.go index 5544add157846..477aee91e7c20 100644 --- a/util/webhook/webhook_test.go +++ b/util/webhook/webhook_test.go @@ -41,7 +41,33 @@ func Test_GetWebUrlRegex(t *testing.T) { regexp, err := GetWebUrlRegex(testCopy.webURL) require.NoError(t, err) if matches := regexp.MatchString(testCopy.repo); matches != testCopy.shouldMatch { - t.Errorf("sourceRevisionHasChanged() = %v, want %v", matches, testCopy.shouldMatch) + t.Errorf("regexp.MatchString() = %v, want %v", matches, testCopy.shouldMatch) + } + }) + } +} + +func Test_GetApiUrlRegex(t *testing.T) { + tests := []struct { + shouldMatch bool + apiURL string + repo string + name string + }{ + // Ensure input is regex-escaped. + {false, "https://an.example.com/org/repo", "https://an-example.com/", "dots in domain names should not be treated as wildcards"}, + + // Standard cases. + {true, "https://example.com/org/repo", "https://example.com/", "exact hostname match should match"}, + } + for _, testCase := range tests { + testCopy := testCase + t.Run(testCopy.name, func(t *testing.T) { + t.Parallel() + regexp, err := GetApiUrlRegex(testCopy.apiURL) + require.NoError(t, err) + if matches := regexp.MatchString(testCopy.repo); matches != testCopy.shouldMatch { + t.Errorf("regexp.MatchString() = %v, want %v (%v)", matches, testCopy.shouldMatch, regexp.String()) } }) } From 7ba28750bfe0d723b388abeb0d170d3c1dac7c35 Mon Sep 17 00:00:00 2001 From: Matthew Bennett Date: Tue, 13 Aug 2024 19:23:18 -0700 Subject: [PATCH 8/9] tweak URL regular expressions Signed-off-by: Matthew Bennett --- util/webhook/webhook.go | 4 ++-- util/webhook/webhook_test.go | 5 +++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/util/webhook/webhook.go b/util/webhook/webhook.go index 6429c522dc5a7..ad83c9d98f064 100644 --- a/util/webhook/webhook.go +++ b/util/webhook/webhook.go @@ -146,11 +146,11 @@ func getUrlRegex(originalUrl string, regexpFormat string) (*regexp.Regexp, error } func GetWebUrlRegex(originalUrl string) (*regexp.Regexp, error) { - return getUrlRegex(originalUrl, `(?i)^(https?://|%[1]s@|ssh://(%[1]s@)?((alt)?ssh\.)?)%[2]s(:[0-9]+)?[:/]%[3]s$`) + return getUrlRegex(originalUrl, `(?i)^((https?|ssh)://)?(%[1]s@)?((alt)?ssh\.)?%[2]s(:[0-9]+)?[:/]%[3]s$`) } func GetApiUrlRegex(originalUrl string) (*regexp.Regexp, error) { - return getUrlRegex(originalUrl, `(?i)^https?://%[2]s(:[0-9]+)?/?$`) + return getUrlRegex(originalUrl, `(?i)^(https?://)?(%[1]s@)?%[2]s(:[0-9]+)?/?$`) } func (webhook *Webhook) HandleRequest(writer http.ResponseWriter, request *http.Request) { diff --git a/util/webhook/webhook_test.go b/util/webhook/webhook_test.go index 477aee91e7c20..224a887b66aa3 100644 --- a/util/webhook/webhook_test.go +++ b/util/webhook/webhook_test.go @@ -33,6 +33,11 @@ func Test_GetWebUrlRegex(t *testing.T) { {false, "https://example.com/org/repo", "ssh://-user-name@example.com/org/repo", "invalid usernames with hyphens in repo should not match"}, {true, "https://example.com:443/org/repo", "GIT@EXAMPLE.COM:22:ORG/REPO", "matches aren't case-sensitive"}, {true, "https://example.com/org/repo%20", "https://example.com/org/repo%20", "escape codes in path are preserved"}, + {true, "https://user@example.com/org/repo", "http://example.com/org/repo", "https+username should match http"}, + {true, "https://user@example.com/org/repo", "https://example.com/org/repo", "https+username should match https"}, + {true, "http://example.com/org/repo", "https://user@example.com/org/repo", "http should match https+username"}, + {true, "https://example.com/org/repo", "https://user@example.com/org/repo", "https should match https+username"}, + {true, "https://user@example.com/org/repo", "ssh://example.com/org/repo", "https+username should match ssh"}, } for _, testCase := range tests { testCopy := testCase From 27ef47452b3a8a4bfc5f157ee4d1f5c972581b22 Mon Sep 17 00:00:00 2001 From: Matthew Bennett Date: Thu, 5 Sep 2024 09:53:26 -0700 Subject: [PATCH 9/9] fix test data file indentation Signed-off-by: Matthew Bennett --- .../github-commit-event-feature-branch.json | 279 +++++++++--------- 1 file changed, 139 insertions(+), 140 deletions(-) diff --git a/applicationset/webhookhandler/testdata/github-commit-event-feature-branch.json b/applicationset/webhookhandler/testdata/github-commit-event-feature-branch.json index c859dc9163eeb..1a3be0e78524a 100644 --- a/applicationset/webhookhandler/testdata/github-commit-event-feature-branch.json +++ b/applicationset/webhookhandler/testdata/github-commit-event-feature-branch.json @@ -1,43 +1,14 @@ { - "ref": "refs/heads/env/dev", - "before": "d5c1ffa8e294bc18c639bfb4e0df499251034414", - "after": "63738bb582c8b540af7bcfc18f87c575c3ed66e0", - "created": false, - "deleted": false, - "forced": true, - "base_ref": null, - "compare": "https://github.com/org/repo/compare/d5c1ffa8e294...63738bb582c8", - "commits": [ - { - "id": "63738bb582c8b540af7bcfc18f87c575c3ed66e0", - "tree_id": "64897da445207e409ad05af93b1f349ad0a4ee19", - "distinct": true, - "message": "Add staging-argocd-demo environment", - "timestamp": "2018-05-04T15:40:02-07:00", - "url": "https://github.com/org/repo/commit/63738bb582c8b540af7bcfc18f87c575c3ed66e0", - "author": { - "name": "Jesse Suen", - "email": "Jesse_Suen@example.com", - "username": "org" - }, - "committer": { - "name": "Jesse Suen", - "email": "Jesse_Suen@example.com", - "username": "org" - }, - "added": [ - "ksapps/test-app/environments/staging-argocd-demo/main.jsonnet", - "ksapps/test-app/environments/staging-argocd-demo/params.libsonnet" - ], - "removed": [ - - ], - "modified": [ - "ksapps/test-app/app.yaml" - ] - } - ], - "head_commit": { + "ref": "refs/heads/env/dev", + "before": "d5c1ffa8e294bc18c639bfb4e0df499251034414", + "after": "63738bb582c8b540af7bcfc18f87c575c3ed66e0", + "created": false, + "deleted": false, + "forced": true, + "base_ref": null, + "compare": "https://github.com/org/repo/compare/d5c1ffa8e294...63738bb582c8", + "commits": [ + { "id": "63738bb582c8b540af7bcfc18f87c575c3ed66e0", "tree_id": "64897da445207e409ad05af93b1f349ad0a4ee19", "distinct": true, @@ -59,112 +30,48 @@ "ksapps/test-app/environments/staging-argocd-demo/params.libsonnet" ], "removed": [ - + ], "modified": [ "ksapps/test-app/app.yaml" ] + } + ], + "head_commit": { + "id": "63738bb582c8b540af7bcfc18f87c575c3ed66e0", + "tree_id": "64897da445207e409ad05af93b1f349ad0a4ee19", + "distinct": true, + "message": "Add staging-argocd-demo environment", + "timestamp": "2018-05-04T15:40:02-07:00", + "url": "https://github.com/org/repo/commit/63738bb582c8b540af7bcfc18f87c575c3ed66e0", + "author": { + "name": "Jesse Suen", + "email": "Jesse_Suen@example.com", + "username": "org" }, - "repository": { - "id": 123060978, - "name": "repo", - "full_name": "org/repo", - "owner": { - "name": "org", - "email": "org@users.noreply.github.com", - "login": "org", - "id": 12677113, - "avatar_url": "https://avatars0.githubusercontent.com/u/12677113?v=4", - "gravatar_id": "", - "url": "https://api.github.com/users/org", - "html_url": "https://github.com/org", - "followers_url": "https://api.github.com/users/org/followers", - "following_url": "https://api.github.com/users/org/following{/other_user}", - "gists_url": "https://api.github.com/users/org/gists{/gist_id}", - "starred_url": "https://api.github.com/users/org/starred{/owner}{/repo}", - "subscriptions_url": "https://api.github.com/users/org/subscriptions", - "organizations_url": "https://api.github.com/users/org/orgs", - "repos_url": "https://api.github.com/users/org/repos", - "events_url": "https://api.github.com/users/org/events{/privacy}", - "received_events_url": "https://api.github.com/users/org/received_events", - "type": "User", - "site_admin": false - }, - "private": false, - "html_url": "https://github.com/org/repo", - "description": "Test Repository", - "fork": false, - "url": "https://github.com/org/repo", - "forks_url": "https://api.github.com/repos/org/repo/forks", - "keys_url": "https://api.github.com/repos/org/repo/keys{/key_id}", - "collaborators_url": "https://api.github.com/repos/org/repo/collaborators{/collaborator}", - "teams_url": "https://api.github.com/repos/org/repo/teams", - "hooks_url": "https://api.github.com/repos/org/repo/hooks", - "issue_events_url": "https://api.github.com/repos/org/repo/issues/events{/number}", - "events_url": "https://api.github.com/repos/org/repo/events", - "assignees_url": "https://api.github.com/repos/org/repo/assignees{/user}", - "branches_url": "https://api.github.com/repos/org/repo/branches{/branch}", - "tags_url": "https://api.github.com/repos/org/repo/tags", - "blobs_url": "https://api.github.com/repos/org/repo/git/blobs{/sha}", - "git_tags_url": "https://api.github.com/repos/org/repo/git/tags{/sha}", - "git_refs_url": "https://api.github.com/repos/org/repo/git/refs{/sha}", - "trees_url": "https://api.github.com/repos/org/repo/git/trees{/sha}", - "statuses_url": "https://api.github.com/repos/org/repo/statuses/{sha}", - "languages_url": "https://api.github.com/repos/org/repo/languages", - "stargazers_url": "https://api.github.com/repos/org/repo/stargazers", - "contributors_url": "https://api.github.com/repos/org/repo/contributors", - "subscribers_url": "https://api.github.com/repos/org/repo/subscribers", - "subscription_url": "https://api.github.com/repos/org/repo/subscription", - "commits_url": "https://api.github.com/repos/org/repo/commits{/sha}", - "git_commits_url": "https://api.github.com/repos/org/repo/git/commits{/sha}", - "comments_url": "https://api.github.com/repos/org/repo/comments{/number}", - "issue_comment_url": "https://api.github.com/repos/org/repo/issues/comments{/number}", - "contents_url": "https://api.github.com/repos/org/repo/contents/{+path}", - "compare_url": "https://api.github.com/repos/org/repo/compare/{base}...{head}", - "merges_url": "https://api.github.com/repos/org/repo/merges", - "archive_url": "https://api.github.com/repos/org/repo/{archive_format}{/ref}", - "downloads_url": "https://api.github.com/repos/org/repo/downloads", - "issues_url": "https://api.github.com/repos/org/repo/issues{/number}", - "pulls_url": "https://api.github.com/repos/org/repo/pulls{/number}", - "milestones_url": "https://api.github.com/repos/org/repo/milestones{/number}", - "notifications_url": "https://api.github.com/repos/org/repo/notifications{?since,all,participating}", - "labels_url": "https://api.github.com/repos/org/repo/labels{/name}", - "releases_url": "https://api.github.com/repos/org/repo/releases{/id}", - "deployments_url": "https://api.github.com/repos/org/repo/deployments", - "created_at": 1519698615, - "updated_at": "2018-05-04T22:37:55Z", - "pushed_at": 1525473610, - "git_url": "git://github.com/org/repo.git", - "ssh_url": "git@github.com:org/repo.git", - "clone_url": "https://github.com/org/repo.git", - "svn_url": "https://github.com/org/repo", - "homepage": null, - "size": 538, - "stargazers_count": 0, - "watchers_count": 0, - "language": null, - "has_issues": true, - "has_projects": true, - "has_downloads": true, - "has_wiki": true, - "has_pages": false, - "forks_count": 1, - "mirror_url": null, - "archived": false, - "open_issues_count": 0, - "license": null, - "forks": 1, - "open_issues": 0, - "watchers": 0, - "default_branch": "master", - "stargazers": 0, - "master_branch": "master" + "committer": { + "name": "Jesse Suen", + "email": "Jesse_Suen@example.com", + "username": "org" }, - "pusher": { + "added": [ + "ksapps/test-app/environments/staging-argocd-demo/main.jsonnet", + "ksapps/test-app/environments/staging-argocd-demo/params.libsonnet" + ], + "removed": [ + + ], + "modified": [ + "ksapps/test-app/app.yaml" + ] + }, + "repository": { + "id": 123060978, + "name": "repo", + "full_name": "org/repo", + "owner": { "name": "org", - "email": "org@users.noreply.github.com" - }, - "sender": { + "email": "org@users.noreply.github.com", "login": "org", "id": 12677113, "avatar_url": "https://avatars0.githubusercontent.com/u/12677113?v=4", @@ -182,6 +89,98 @@ "received_events_url": "https://api.github.com/users/org/received_events", "type": "User", "site_admin": false - } + }, + "private": false, + "html_url": "https://github.com/org/repo", + "description": "Test Repository", + "fork": false, + "url": "https://github.com/org/repo", + "forks_url": "https://api.github.com/repos/org/repo/forks", + "keys_url": "https://api.github.com/repos/org/repo/keys{/key_id}", + "collaborators_url": "https://api.github.com/repos/org/repo/collaborators{/collaborator}", + "teams_url": "https://api.github.com/repos/org/repo/teams", + "hooks_url": "https://api.github.com/repos/org/repo/hooks", + "issue_events_url": "https://api.github.com/repos/org/repo/issues/events{/number}", + "events_url": "https://api.github.com/repos/org/repo/events", + "assignees_url": "https://api.github.com/repos/org/repo/assignees{/user}", + "branches_url": "https://api.github.com/repos/org/repo/branches{/branch}", + "tags_url": "https://api.github.com/repos/org/repo/tags", + "blobs_url": "https://api.github.com/repos/org/repo/git/blobs{/sha}", + "git_tags_url": "https://api.github.com/repos/org/repo/git/tags{/sha}", + "git_refs_url": "https://api.github.com/repos/org/repo/git/refs{/sha}", + "trees_url": "https://api.github.com/repos/org/repo/git/trees{/sha}", + "statuses_url": "https://api.github.com/repos/org/repo/statuses/{sha}", + "languages_url": "https://api.github.com/repos/org/repo/languages", + "stargazers_url": "https://api.github.com/repos/org/repo/stargazers", + "contributors_url": "https://api.github.com/repos/org/repo/contributors", + "subscribers_url": "https://api.github.com/repos/org/repo/subscribers", + "subscription_url": "https://api.github.com/repos/org/repo/subscription", + "commits_url": "https://api.github.com/repos/org/repo/commits{/sha}", + "git_commits_url": "https://api.github.com/repos/org/repo/git/commits{/sha}", + "comments_url": "https://api.github.com/repos/org/repo/comments{/number}", + "issue_comment_url": "https://api.github.com/repos/org/repo/issues/comments{/number}", + "contents_url": "https://api.github.com/repos/org/repo/contents/{+path}", + "compare_url": "https://api.github.com/repos/org/repo/compare/{base}...{head}", + "merges_url": "https://api.github.com/repos/org/repo/merges", + "archive_url": "https://api.github.com/repos/org/repo/{archive_format}{/ref}", + "downloads_url": "https://api.github.com/repos/org/repo/downloads", + "issues_url": "https://api.github.com/repos/org/repo/issues{/number}", + "pulls_url": "https://api.github.com/repos/org/repo/pulls{/number}", + "milestones_url": "https://api.github.com/repos/org/repo/milestones{/number}", + "notifications_url": "https://api.github.com/repos/org/repo/notifications{?since,all,participating}", + "labels_url": "https://api.github.com/repos/org/repo/labels{/name}", + "releases_url": "https://api.github.com/repos/org/repo/releases{/id}", + "deployments_url": "https://api.github.com/repos/org/repo/deployments", + "created_at": 1519698615, + "updated_at": "2018-05-04T22:37:55Z", + "pushed_at": 1525473610, + "git_url": "git://github.com/org/repo.git", + "ssh_url": "git@github.com:org/repo.git", + "clone_url": "https://github.com/org/repo.git", + "svn_url": "https://github.com/org/repo", + "homepage": null, + "size": 538, + "stargazers_count": 0, + "watchers_count": 0, + "language": null, + "has_issues": true, + "has_projects": true, + "has_downloads": true, + "has_wiki": true, + "has_pages": false, + "forks_count": 1, + "mirror_url": null, + "archived": false, + "open_issues_count": 0, + "license": null, + "forks": 1, + "open_issues": 0, + "watchers": 0, + "default_branch": "master", + "stargazers": 0, + "master_branch": "master" + }, + "pusher": { + "name": "org", + "email": "org@users.noreply.github.com" + }, + "sender": { + "login": "org", + "id": 12677113, + "avatar_url": "https://avatars0.githubusercontent.com/u/12677113?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/org", + "html_url": "https://github.com/org", + "followers_url": "https://api.github.com/users/org/followers", + "following_url": "https://api.github.com/users/org/following{/other_user}", + "gists_url": "https://api.github.com/users/org/gists{/gist_id}", + "starred_url": "https://api.github.com/users/org/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/org/subscriptions", + "organizations_url": "https://api.github.com/users/org/orgs", + "repos_url": "https://api.github.com/users/org/repos", + "events_url": "https://api.github.com/users/org/events{/privacy}", + "received_events_url": "https://api.github.com/users/org/received_events", + "type": "User", + "site_admin": false } - \ No newline at end of file +}