From ffea2d3decc54452d482a1654812c3c556066545 Mon Sep 17 00:00:00 2001 From: Dave Bauman Date: Tue, 26 Mar 2024 09:28:50 -0400 Subject: [PATCH] fix: Reduce iex-webhooks index size to critical fields only --- packages/backend/indices/iex-webhooks.json | 1274 ----------------- .../backend/src/controllers/webhook.v1.ts | 13 +- 2 files changed, 7 insertions(+), 1280 deletions(-) diff --git a/packages/backend/indices/iex-webhooks.json b/packages/backend/indices/iex-webhooks.json index a6e12579f..bdbf9b8a8 100644 --- a/packages/backend/indices/iex-webhooks.json +++ b/packages/backend/indices/iex-webhooks.json @@ -5,164 +5,6 @@ "type": "keyword", "ignore_above": 256 }, - "after": { - "type": "keyword", - "ignore_above": 256 - }, - "before": { - "type": "keyword", - "ignore_above": 256 - }, - "changes": { - "properties": { - "description": { - "properties": { - "from": { - "type": "text" - } - } - }, - "permission": { - "properties": { - "from": { - "type": "text" - }, - "to": { - "type": "text" - } - } - } - } - }, - "commits": { - "properties": { - "added": { - "type": "text" - }, - "author": { - "properties": { - "email": { - "type": "text" - }, - "name": { - "type": "text" - }, - "username": { - "type": "text" - } - } - }, - "committer": { - "properties": { - "email": { - "type": "text" - }, - "name": { - "type": "text" - }, - "username": { - "type": "text" - } - } - }, - "distinct": { - "type": "boolean" - }, - "id": { - "type": "text" - }, - "message": { - "type": "text" - }, - "modified": { - "type": "text" - }, - "removed": { - "type": "text" - }, - "timestamp": { - "type": "date" - }, - "tree_id": { - "type": "text" - }, - "url": { - "type": "text" - } - } - }, - "compare": { - "type": "text" - }, - "created": { - "type": "boolean" - }, - "deleted": { - "type": "boolean" - }, - "description": { - "type": "text" - }, - "forced": { - "type": "boolean" - }, - "head_commit": { - "properties": { - "added": { - "type": "text" - }, - "author": { - "properties": { - "email": { - "type": "text" - }, - "name": { - "type": "text" - }, - "username": { - "type": "text" - } - } - }, - "committer": { - "properties": { - "email": { - "type": "text" - }, - "name": { - "type": "text" - }, - "username": { - "type": "text" - } - } - }, - "distinct": { - "type": "boolean" - }, - "id": { - "type": "text" - }, - "message": { - "type": "text" - }, - "modified": { - "type": "text" - }, - "removed": { - "type": "text" - }, - "timestamp": { - "type": "date" - }, - "tree_id": { - "type": "text" - }, - "url": { - "type": "text" - } - } - }, "headers": { "properties": { "accept": { @@ -227,1119 +69,6 @@ } } }, - "hook": { - "properties": { - "active": { - "type": "boolean" - }, - "config": { - "properties": { - "content_type": { - "type": "keyword", - "ignore_above": 256 - }, - "insecure_ssl": { - "type": "keyword", - "ignore_above": 256 - }, - "url": { - "type": "text" - } - } - }, - "created_at": { - "type": "date" - }, - "events": { - "type": "text" - }, - "id": { - "type": "long" - }, - "last_response": { - "properties": { - "status": { - "type": "keyword", - "ignore_above": 256 - } - } - }, - "name": { - "type": "text" - }, - "ping_url": { - "type": "text" - }, - "test_url": { - "type": "text" - }, - "type": { - "type": "keyword", - "ignore_above": 256 - }, - "updated_at": { - "type": "date" - }, - "url": { - "type": "text" - } - } - }, - "hook_id": { - "type": "long" - }, - "member": { - "properties": { - "avatar_url": { - "type": "text" - }, - "events_url": { - "type": "text" - }, - "followers_url": { - "type": "text" - }, - "following_url": { - "type": "text" - }, - "gists_url": { - "type": "text" - }, - "gravatar_id": { - "type": "text" - }, - "html_url": { - "type": "text" - }, - "id": { - "type": "long" - }, - "ldap_dn": { - "type": "text" - }, - "login": { - "type": "text" - }, - "node_id": { - "type": "text" - }, - "organizations_url": { - "type": "text" - }, - "received_events_url": { - "type": "text" - }, - "repos_url": { - "type": "text" - }, - "site_admin": { - "type": "boolean" - }, - "starred_url": { - "type": "text" - }, - "subscriptions_url": { - "type": "text" - }, - "type": { - "type": "keyword", - "ignore_above": 256 - }, - "url": { - "type": "text" - } - } - }, - "organization": { - "properties": { - "avatar_url": { - "type": "text" - }, - "description": { - "type": "text" - }, - "events_url": { - "type": "text" - }, - "hooks_url": { - "type": "text" - }, - "id": { - "type": "long" - }, - "issues_url": { - "type": "text" - }, - "login": { - "type": "text" - }, - "members_url": { - "type": "text" - }, - "node_id": { - "type": "text" - }, - "public_members_url": { - "type": "text" - }, - "repos_url": { - "type": "text" - }, - "url": { - "type": "text" - } - } - }, - "pull_request": { - "properties": { - "_links": { - "properties": { - "comments": { - "properties": { - "href": { - "type": "text" - } - } - }, - "commits": { - "properties": { - "href": { - "type": "text" - } - } - }, - "html": { - "properties": { - "href": { - "type": "text" - } - } - }, - "issue": { - "properties": { - "href": { - "type": "text" - } - } - }, - "review_comment": { - "properties": { - "href": { - "type": "text" - } - } - }, - "review_comments": { - "properties": { - "href": { - "type": "text" - } - } - }, - "self": { - "properties": { - "href": { - "type": "text" - } - } - }, - "statuses": { - "properties": { - "href": { - "type": "text" - } - } - } - } - }, - "additions": { - "type": "long" - }, - "author_association": { - "type": "text" - }, - "base": { - "properties": { - "label": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 - } - } - }, - "ref": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 - } - } - }, - "repo": { - "properties": { - "allow_merge_commit": { - "type": "boolean" - }, - "allow_rebase_merge": { - "type": "boolean" - }, - "allow_squash_merge": { - "type": "boolean" - }, - "archive_url": { - "type": "text" - }, - "archived": { - "type": "boolean" - }, - "assignees_url": { - "type": "text" - }, - "blobs_url": { - "type": "text" - }, - "branches_url": { - "type": "text" - }, - "clone_url": { - "type": "text" - }, - "collaborators_url": { - "type": "text" - }, - "comments_url": { - "type": "text" - }, - "commits_url": { - "type": "text" - }, - "compare_url": { - "type": "text" - }, - "contents_url": { - "type": "text" - }, - "contributors_url": { - "type": "text" - }, - "created_at": { - "type": "date" - }, - "default_branch": { - "type": "text" - }, - "delete_branch_on_merge": { - "type": "boolean" - }, - "deployments_url": { - "type": "text" - }, - "description": { - "type": "text" - }, - "disabled": { - "type": "boolean" - }, - "downloads_url": { - "type": "text" - }, - "events_url": { - "type": "text" - }, - "fork": { - "type": "boolean" - }, - "forks": { - "type": "long" - }, - "forks_count": { - "type": "long" - }, - "forks_url": { - "type": "text" - }, - "full_name": { - "type": "text" - }, - "git_commits_url": { - "type": "text" - }, - "git_refs_url": { - "type": "text" - }, - "git_tags_url": { - "type": "text" - }, - "git_url": { - "type": "text" - }, - "has_downloads": { - "type": "boolean" - }, - "has_issues": { - "type": "boolean" - }, - "has_pages": { - "type": "boolean" - }, - "has_projects": { - "type": "boolean" - }, - "has_wiki": { - "type": "boolean" - }, - "hooks_url": { - "type": "text" - }, - "html_url": { - "type": "text" - }, - "id": { - "type": "long" - }, - "issue_comment_url": { - "type": "text" - }, - "issue_events_url": { - "type": "text" - }, - "issues_url": { - "type": "text" - }, - "keys_url": { - "type": "text" - }, - "labels_url": { - "type": "text" - }, - "language": { - "type": "text" - }, - "languages_url": { - "type": "text" - }, - "merges_url": { - "type": "text" - }, - "milestones_url": { - "type": "text" - }, - "name": { - "type": "text" - }, - "node_id": { - "type": "text" - }, - "notifications_url": { - "type": "text" - }, - "open_issues": { - "type": "long" - }, - "open_issues_count": { - "type": "long" - }, - "owner": { - "properties": { - "avatar_url": { - "type": "text" - }, - "events_url": { - "type": "text" - }, - "followers_url": { - "type": "text" - }, - "following_url": { - "type": "text" - }, - "gists_url": { - "type": "text" - }, - "gravatar_id": { - "type": "text" - }, - "html_url": { - "type": "text" - }, - "id": { - "type": "long" - }, - "login": { - "type": "text" - }, - "node_id": { - "type": "text" - }, - "organizations_url": { - "type": "text" - }, - "received_events_url": { - "type": "text" - }, - "repos_url": { - "type": "text" - }, - "site_admin": { - "type": "boolean" - }, - "starred_url": { - "type": "text" - }, - "subscriptions_url": { - "type": "text" - }, - "type": { - "type": "keyword", - "ignore_above": 256 - }, - "url": { - "type": "text" - } - } - }, - "private": { - "type": "boolean" - }, - "pulls_url": { - "type": "text" - }, - "pushed_at": { - "type": "date" - }, - "releases_url": { - "type": "text" - }, - "size": { - "type": "long" - }, - "ssh_url": { - "type": "text" - }, - "stargazers_count": { - "type": "long" - }, - "stargazers_url": { - "type": "text" - }, - "statuses_url": { - "type": "text" - }, - "subscribers_url": { - "type": "text" - }, - "subscription_url": { - "type": "text" - }, - "svn_url": { - "type": "text" - }, - "tags_url": { - "type": "text" - }, - "teams_url": { - "type": "text" - }, - "trees_url": { - "type": "text" - }, - "updated_at": { - "type": "date" - }, - "url": { - "type": "text" - }, - "watchers": { - "type": "long" - }, - "watchers_count": { - "type": "long" - } - } - }, - "sha": { - "type": "text" - }, - "user": { - "properties": { - "avatar_url": { - "type": "text" - }, - "events_url": { - "type": "text" - }, - "followers_url": { - "type": "text" - }, - "following_url": { - "type": "text" - }, - "gists_url": { - "type": "text" - }, - "gravatar_id": { - "type": "text" - }, - "html_url": { - "type": "text" - }, - "id": { - "type": "long" - }, - "login": { - "type": "text" - }, - "node_id": { - "type": "text" - }, - "organizations_url": { - "type": "text" - }, - "received_events_url": { - "type": "text" - }, - "repos_url": { - "type": "text" - }, - "site_admin": { - "type": "boolean" - }, - "starred_url": { - "type": "text" - }, - "subscriptions_url": { - "type": "text" - }, - "type": { - "type": "keyword", - "ignore_above": 256 - }, - "url": { - "type": "text" - } - } - } - } - }, - "body": { - "type": "text" - }, - "changed_files": { - "type": "long" - }, - "comments": { - "type": "long" - }, - "comments_url": { - "type": "text" - }, - "commits": { - "type": "long" - }, - "commits_url": { - "type": "text" - }, - "created_at": { - "type": "date" - }, - "deletions": { - "type": "long" - }, - "diff_url": { - "type": "text" - }, - "draft": { - "type": "boolean" - }, - "head": { - "properties": { - "label": { - "type": "text" - }, - "ref": { - "type": "text" - }, - "repo": { - "properties": { - "allow_merge_commit": { - "type": "boolean" - }, - "allow_rebase_merge": { - "type": "boolean" - }, - "allow_squash_merge": { - "type": "boolean" - }, - "archive_url": { - "type": "text" - }, - "archived": { - "type": "boolean" - }, - "assignees_url": { - "type": "text" - }, - "blobs_url": { - "type": "text" - }, - "branches_url": { - "type": "text" - }, - "clone_url": { - "type": "text" - }, - "collaborators_url": { - "type": "text" - }, - "comments_url": { - "type": "text" - }, - "commits_url": { - "type": "text" - }, - "compare_url": { - "type": "text" - }, - "contents_url": { - "type": "text" - }, - "contributors_url": { - "type": "text" - }, - "created_at": { - "type": "date" - }, - "default_branch": { - "type": "text" - }, - "delete_branch_on_merge": { - "type": "boolean" - }, - "deployments_url": { - "type": "text" - }, - "description": { - "type": "text" - }, - "disabled": { - "type": "boolean" - }, - "downloads_url": { - "type": "text" - }, - "events_url": { - "type": "text" - }, - "fork": { - "type": "boolean" - }, - "forks": { - "type": "long" - }, - "forks_count": { - "type": "long" - }, - "forks_url": { - "type": "text" - }, - "full_name": { - "type": "text" - }, - "git_commits_url": { - "type": "text" - }, - "git_refs_url": { - "type": "text" - }, - "git_tags_url": { - "type": "text" - }, - "git_url": { - "type": "text" - }, - "has_downloads": { - "type": "boolean" - }, - "has_issues": { - "type": "boolean" - }, - "has_pages": { - "type": "boolean" - }, - "has_projects": { - "type": "boolean" - }, - "has_wiki": { - "type": "boolean" - }, - "hooks_url": { - "type": "text" - }, - "html_url": { - "type": "text" - }, - "id": { - "type": "long" - }, - "issue_comment_url": { - "type": "text" - }, - "issue_events_url": { - "type": "text" - }, - "issues_url": { - "type": "text" - }, - "keys_url": { - "type": "text" - }, - "labels_url": { - "type": "text" - }, - "language": { - "type": "text" - }, - "languages_url": { - "type": "text" - }, - "merges_url": { - "type": "text" - }, - "milestones_url": { - "type": "text" - }, - "name": { - "type": "text" - }, - "node_id": { - "type": "text" - }, - "notifications_url": { - "type": "text" - }, - "open_issues": { - "type": "long" - }, - "open_issues_count": { - "type": "long" - }, - "owner": { - "properties": { - "avatar_url": { - "type": "text" - }, - "events_url": { - "type": "text" - }, - "followers_url": { - "type": "text" - }, - "following_url": { - "type": "text" - }, - "gists_url": { - "type": "text" - }, - "gravatar_id": { - "type": "text" - }, - "html_url": { - "type": "text" - }, - "id": { - "type": "long" - }, - "login": { - "type": "text" - }, - "node_id": { - "type": "text" - }, - "organizations_url": { - "type": "text" - }, - "received_events_url": { - "type": "text" - }, - "repos_url": { - "type": "text" - }, - "site_admin": { - "type": "boolean" - }, - "starred_url": { - "type": "text" - }, - "subscriptions_url": { - "type": "text" - }, - "type": { - "type": "keyword", - "ignore_above": 256 - }, - "url": { - "type": "text" - } - } - }, - "private": { - "type": "boolean" - }, - "pulls_url": { - "type": "text" - }, - "pushed_at": { - "type": "date" - }, - "releases_url": { - "type": "text" - }, - "size": { - "type": "long" - }, - "ssh_url": { - "type": "text" - }, - "stargazers_count": { - "type": "long" - }, - "stargazers_url": { - "type": "text" - }, - "statuses_url": { - "type": "text" - }, - "subscribers_url": { - "type": "text" - }, - "subscription_url": { - "type": "text" - }, - "svn_url": { - "type": "text" - }, - "tags_url": { - "type": "text" - }, - "teams_url": { - "type": "text" - }, - "trees_url": { - "type": "text" - }, - "updated_at": { - "type": "date" - }, - "url": { - "type": "text" - }, - "watchers": { - "type": "long" - }, - "watchers_count": { - "type": "long" - } - } - }, - "sha": { - "type": "text" - }, - "user": { - "properties": { - "avatar_url": { - "type": "text" - }, - "events_url": { - "type": "text" - }, - "followers_url": { - "type": "text" - }, - "following_url": { - "type": "text" - }, - "gists_url": { - "type": "text" - }, - "gravatar_id": { - "type": "text" - }, - "html_url": { - "type": "text" - }, - "id": { - "type": "long" - }, - "login": { - "type": "text" - }, - "node_id": { - "type": "text" - }, - "organizations_url": { - "type": "text" - }, - "received_events_url": { - "type": "text" - }, - "repos_url": { - "type": "text" - }, - "site_admin": { - "type": "boolean" - }, - "starred_url": { - "type": "text" - }, - "subscriptions_url": { - "type": "text" - }, - "type": { - "type": "keyword", - "ignore_above": 256 - }, - "url": { - "type": "text" - } - } - } - } - }, - "html_url": { - "type": "text" - }, - "id": { - "type": "long" - }, - "issue_url": { - "type": "text" - }, - "locked": { - "type": "boolean" - }, - "maintainer_can_modify": { - "type": "boolean" - }, - "mergeable_state": { - "type": "text" - }, - "merged": { - "type": "boolean" - }, - "node_id": { - "type": "text" - }, - "number": { - "type": "long" - }, - "patch_url": { - "type": "text" - }, - "review_comment_url": { - "type": "text" - }, - "review_comments": { - "type": "long" - }, - "review_comments_url": { - "type": "text" - }, - "state": { - "type": "text" - }, - "statuses_url": { - "type": "text" - }, - "title": { - "type": "text" - }, - "updated_at": { - "type": "date" - }, - "url": { - "type": "text" - }, - "user": { - "properties": { - "avatar_url": { - "type": "text" - }, - "events_url": { - "type": "text" - }, - "followers_url": { - "type": "text" - }, - "following_url": { - "type": "text" - }, - "gists_url": { - "type": "text" - }, - "gravatar_id": { - "type": "text" - }, - "html_url": { - "type": "text" - }, - "id": { - "type": "long" - }, - "ldap_dn": { - "type": "text" - }, - "login": { - "type": "text" - }, - "node_id": { - "type": "text" - }, - "organizations_url": { - "type": "text" - }, - "received_events_url": { - "type": "text" - }, - "repos_url": { - "type": "text" - }, - "site_admin": { - "type": "boolean" - }, - "starred_url": { - "type": "text" - }, - "subscriptions_url": { - "type": "text" - }, - "type": { - "type": "keyword", - "ignore_above": 256 - }, - "url": { - "type": "text" - } - } - } - } - }, - "pusher": { - "properties": { - "email": { - "type": "text" - }, - "name": { - "type": "text" - } - } - }, - "pusher_type": { - "type": "keyword", - "ignore_above": 256 - }, "ref": { "type": "text" }, @@ -1613,9 +342,6 @@ "type": "long" } } - }, - "starred_at": { - "type": "date" } } }, diff --git a/packages/backend/src/controllers/webhook.v1.ts b/packages/backend/src/controllers/webhook.v1.ts index 9befd0fb3..c4aabd52b 100644 --- a/packages/backend/src/controllers/webhook.v1.ts +++ b/packages/backend/src/controllers/webhook.v1.ts @@ -41,12 +41,13 @@ export const hook = (req: Request, res: Response): void => { } }; - // Remove some un-needed fields - delete webhook.enterprise; - delete webhook.master_branch; - delete webhook.sender; - delete webhook.team; - delete webhook.zen; + // Create a slim version with whitelisted fields only + const allowedFields = new Set(['action', 'headers', 'ref', 'ref_type', 'repository']); + for (const key in webhook) { + if (!allowedFields.has(key)) { + delete webhook[key]; + } + } // Track webhooks in Elasticsearch defaultElasticsearchClient.index({