From b07606ea1895f7a9ebfe5957d4e89ca82e3180d2 Mon Sep 17 00:00:00 2001 From: ukrocks007 Date: Fri, 17 Mar 2023 16:08:51 +0530 Subject: [PATCH 01/15] sending to datadog using triggermesh --- .../workers/saveEventToElasticsearch.ts | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/src/_processor/workers/saveEventToElasticsearch.ts b/src/_processor/workers/saveEventToElasticsearch.ts index f89eb9b1e..44cb53e62 100644 --- a/src/_processor/workers/saveEventToElasticsearch.ts +++ b/src/_processor/workers/saveEventToElasticsearch.ts @@ -4,6 +4,8 @@ import { Registry, getRegistry, instrumented } from "monkit"; import { Clock } from "../common"; import { ClientWithRetry, getESWithRetry } from "../../persistence/elasticsearch"; +import axios from "axios"; +import { v4 } from "uuid"; export class ElasticsearchSaver { public static getDefault(): ElasticsearchSaver { @@ -30,6 +32,7 @@ export class ElasticsearchSaver { const alias = `retraced.${jobObj.projectId}.${jobObj.environmentId}.current`; try { await this.esIndex(event, alias); + this.sendToWebhook(event); } catch (e) { e.retry = true; throw e; @@ -96,6 +99,28 @@ export class ElasticsearchSaver { return errString; } + + private async sendToWebhook(event: any): Promise { + await axios.post( + event.fields.webhookUrl || "http://localhost:63970", + { + ddsource: "local-dev-machine", + ddtags: "Audit-Logs, Retraced, BoxyHQ", + hostname: "127.0.0.1", + service: "Retraced-audit-logs", + message: JSON.stringify(event), + }, + { + headers: { + "Ce-Id": v4(), + "Ce-Specversion": "1.0", + "Ce-Type": "io.triggermesh.datadog.log.send", + "Ce-Source": "ocimetrics/adapter", + "Content-Type": "application/json", + }, + } + ); + } } export default async function saveEventToElasticsearch(job): Promise { From a2f90807a7f8cc6d588f1cb53eba22c2757eac61 Mon Sep 17 00:00:00 2001 From: ukrocks007 Date: Mon, 20 Mar 2023 13:06:11 +0530 Subject: [PATCH 02/15] not sending raw to datadog --- src/_processor/workers/saveEventToElasticsearch.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/_processor/workers/saveEventToElasticsearch.ts b/src/_processor/workers/saveEventToElasticsearch.ts index 44cb53e62..2718ecd5d 100644 --- a/src/_processor/workers/saveEventToElasticsearch.ts +++ b/src/_processor/workers/saveEventToElasticsearch.ts @@ -101,6 +101,7 @@ export class ElasticsearchSaver { } private async sendToWebhook(event: any): Promise { + delete event.raw; await axios.post( event.fields.webhookUrl || "http://localhost:63970", { From 10e99e9ed95fffa16b03f23f856c3f5aa31007b0 Mon Sep 17 00:00:00 2001 From: ukrocks007 Date: Thu, 23 Mar 2023 20:00:42 +0530 Subject: [PATCH 03/15] fix --- .../workers/saveEventToElasticsearch.ts | 39 ++++++++++--------- 1 file changed, 21 insertions(+), 18 deletions(-) diff --git a/src/_processor/workers/saveEventToElasticsearch.ts b/src/_processor/workers/saveEventToElasticsearch.ts index 2718ecd5d..739e93285 100644 --- a/src/_processor/workers/saveEventToElasticsearch.ts +++ b/src/_processor/workers/saveEventToElasticsearch.ts @@ -102,25 +102,28 @@ export class ElasticsearchSaver { private async sendToWebhook(event: any): Promise { delete event.raw; - await axios.post( - event.fields.webhookUrl || "http://localhost:63970", - { - ddsource: "local-dev-machine", - ddtags: "Audit-Logs, Retraced, BoxyHQ", - hostname: "127.0.0.1", - service: "Retraced-audit-logs", - message: JSON.stringify(event), - }, - { - headers: { - "Ce-Id": v4(), - "Ce-Specversion": "1.0", - "Ce-Type": "io.triggermesh.datadog.log.send", - "Ce-Source": "ocimetrics/adapter", - "Content-Type": "application/json", + axios + .post( + event?.fields?.webhookUrl || "http://localhost:63970", + { + ddsource: "local-dev-machine", + ddtags: "Audit-Logs, Retraced, BoxyHQ", + hostname: "127.0.0.1", + service: "Retraced-audit-logs", + message: JSON.stringify(event), }, - } - ); + { + headers: { + "Ce-Id": v4(), + "Ce-Specversion": "1.0", + "Ce-Type": "io.triggermesh.datadog.log.send", + "Ce-Source": "ocimetrics/adapter", + "Content-Type": "application/json", + }, + } + ) + .catch(console.log) + .then(console.log); } } From 8a4300ec65717272ae4fa5c2e2c4cdc64f646615 Mon Sep 17 00:00:00 2001 From: ukrocks007 Date: Thu, 30 Mar 2023 11:50:53 +0530 Subject: [PATCH 04/15] EE folder migration --- .env | 12 ++++++ docker-compose.yaml | 17 +++++++++ .../workers/saveEventToElasticsearch.ts | 31 +--------------- src/config.ts | 6 +++ src/ee/ENTERPRISE.md | 7 ++++ src/ee/LICENSE | 1 + src/ee/export/Readme.md | 3 ++ src/ee/export/index.ts | 31 ++++++++++++++++ vector.toml | 37 +++++++++++++++++++ 9 files changed, 116 insertions(+), 29 deletions(-) create mode 100644 src/ee/ENTERPRISE.md create mode 100644 src/ee/LICENSE create mode 100644 src/ee/export/Readme.md create mode 100644 src/ee/export/index.ts create mode 100644 vector.toml diff --git a/.env b/.env index cb1cf5e76..d722dfe41 100644 --- a/.env +++ b/.env @@ -26,3 +26,15 @@ NEXTAUTH_URL=http://localhost:5225 NEXTAUTH_SECRET=secret RETRACED_HOST_URL=http://retraced-api:3000/auditlog RETRACED_EXTERNAL_URL=http://localhost:3000/auditlog + +# EE +EXPORT_VECTOR_DATADOG_API_KEY= +EXPORT_VECTOR_WEBHOOK_URL=http://vector:9000 +EXPORT_VECTOR_WEBHOOK_USERNAME=admin +EXPORT_VECTOR_WEBHOOK_PASSWORD=admin +EXPORT_VECTOR_DDSOURCE=local-dev-machine +EXPORT_VECTOR_DDTAGS="Audit-Logs, Retraced, BoxyHQ" +EXPORT_VECTOR_HOSTNAME="127.0.0.1" +EXPORT_VECTOR_SERVICE="Retraced-audit-logs" +EXPORT_VECTOR_DATADOG_REGION=us +EXPORT_VECTOR_DATADOG_SITE=datadoghq.com \ No newline at end of file diff --git a/docker-compose.yaml b/docker-compose.yaml index d4ec3fa12..cfb407ca8 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -197,3 +197,20 @@ services: depends_on: - "retraced-api" restart: "always" + + vector: + image: timberio/vector:0.X-alpine + environment: + - EXPORT_VECTOR_WEBHOOK_USERNAME=${EXPORT_VECTOR_WEBHOOK_USERNAME} + - EXPORT_VECTOR_WEBHOOK_PASSWORD=${EXPORT_VECTOR_WEBHOOK_PASSWORD} + - EXPORT_VECTOR_DDSOURCE=${EXPORT_VECTOR_DDSOURCE} + - EXPORT_VECTOR_DDTAGS=${EXPORT_VECTOR_DDTAGS} + - EXPORT_VECTOR_HOSTNAME=${EXPORT_VECTOR_HOSTNAME} + - EXPORT_VECTOR_SERVICE=${EXPORT_VECTOR_SERVICE} + - EXPORT_VECTOR_DATADOG_API_KEY=${EXPORT_VECTOR_DATADOG_API_KEY} + - EXPORT_VECTOR_DATADOG_REGION=${EXPORT_VECTOR_DATADOG_REGION} + - EXPORT_VECTOR_DATADOG_SITE=${EXPORT_VECTOR_DATADOG_SITE} + volumes: + - ./vector.toml:/etc/vector/vector.toml + networks: + - retraced diff --git a/src/_processor/workers/saveEventToElasticsearch.ts b/src/_processor/workers/saveEventToElasticsearch.ts index 739e93285..91227ded0 100644 --- a/src/_processor/workers/saveEventToElasticsearch.ts +++ b/src/_processor/workers/saveEventToElasticsearch.ts @@ -4,8 +4,7 @@ import { Registry, getRegistry, instrumented } from "monkit"; import { Clock } from "../common"; import { ClientWithRetry, getESWithRetry } from "../../persistence/elasticsearch"; -import axios from "axios"; -import { v4 } from "uuid"; +import sendToWebhook from "../../ee/export/index"; export class ElasticsearchSaver { public static getDefault(): ElasticsearchSaver { @@ -32,7 +31,7 @@ export class ElasticsearchSaver { const alias = `retraced.${jobObj.projectId}.${jobObj.environmentId}.current`; try { await this.esIndex(event, alias); - this.sendToWebhook(event); + sendToWebhook(event); } catch (e) { e.retry = true; throw e; @@ -99,32 +98,6 @@ export class ElasticsearchSaver { return errString; } - - private async sendToWebhook(event: any): Promise { - delete event.raw; - axios - .post( - event?.fields?.webhookUrl || "http://localhost:63970", - { - ddsource: "local-dev-machine", - ddtags: "Audit-Logs, Retraced, BoxyHQ", - hostname: "127.0.0.1", - service: "Retraced-audit-logs", - message: JSON.stringify(event), - }, - { - headers: { - "Ce-Id": v4(), - "Ce-Specversion": "1.0", - "Ce-Type": "io.triggermesh.datadog.log.send", - "Ce-Source": "ocimetrics/adapter", - "Content-Type": "application/json", - }, - } - ) - .catch(console.log) - .then(console.log); - } } export default async function saveEventToElasticsearch(job): Promise { diff --git a/src/config.ts b/src/config.ts index 854e9ef29..ddb72e555 100644 --- a/src/config.ts +++ b/src/config.ts @@ -79,4 +79,10 @@ export default { env.RETRACED_NO_ANALYTICS || process.env.DO_NOT_TRACK || env.DO_NOT_TRACK, + //EXPORT CONFIG + EXPORT_VECTOR_WEBHOOK_URL: process.env.EXPORT_VECTOR_WEBHOOK_URL || env.EXPORT_VECTOR_WEBHOOK_URL, + EXPORT_VECTOR_WEBHOOK_USERNAME: + process.env.EXPORT_VECTOR_WEBHOOK_USERNAME || env.EXPORT_VECTOR_WEBHOOK_USERNAME, + EXPORT_VECTOR_WEBHOOK_PASSWORD: + process.env.EXPORT_VECTOR_WEBHOOK_PASSWORD || env.EXPORT_VECTOR_WEBHOOK_PASSWORD, }; diff --git a/src/ee/ENTERPRISE.md b/src/ee/ENTERPRISE.md new file mode 100644 index 000000000..86d37b62c --- /dev/null +++ b/src/ee/ENTERPRISE.md @@ -0,0 +1,7 @@ +# Enterprise Edition + +Welcome to the Enterprise Edition ("/ee") of BoxyHQ. + +The [/ee](https://github.com/retracedhq/retraced/tree/main/ee) subfolder is the place for all the **Enterprise** features for this repository. + +> _❗ NOTE: This section is copyrighted (unlike the rest of our [repository](https://github.com/retracedhq/retraced)). You are not allowed to use this code without obtaining a proper [license](https://boxyhq.com/pricing) first.❗_ diff --git a/src/ee/LICENSE b/src/ee/LICENSE new file mode 100644 index 000000000..e315ec834 --- /dev/null +++ b/src/ee/LICENSE @@ -0,0 +1 @@ +The BoxyHQ Enterprise Edition (EE) license (the “EE License”) diff --git a/src/ee/export/Readme.md b/src/ee/export/Readme.md new file mode 100644 index 000000000..673b763ee --- /dev/null +++ b/src/ee/export/Readme.md @@ -0,0 +1,3 @@ +# Export Audit-logs in real time using Vector + +This feature uses vector to export auditlogs as they get indexed to datadog & other destinations. diff --git a/src/ee/export/index.ts b/src/ee/export/index.ts new file mode 100644 index 000000000..d02c9ab71 --- /dev/null +++ b/src/ee/export/index.ts @@ -0,0 +1,31 @@ +import axios from "axios"; +import config from "../../config"; +import { logger } from "../../logger"; + +export default async function sendToWebhook(event: any): Promise { + if (config.EXPORT_VECTOR_WEBHOOK_URL) { + delete event.raw; + axios + .post( + config.EXPORT_VECTOR_WEBHOOK_URL, + { + message: JSON.stringify(event), + }, + { + auth: + config.EXPORT_VECTOR_WEBHOOK_USERNAME && config.EXPORT_VECTOR_WEBHOOK_PASSWORD + ? { + username: config.EXPORT_VECTOR_WEBHOOK_USERNAME, + password: config.EXPORT_VECTOR_WEBHOOK_PASSWORD, + } + : undefined, + } + ) + .catch(() => { + logger.info(`[VECTOR EXPORT] Failed to send to webhook`); + }) + .then(() => { + logger.info(`[VECTOR EXPORT] Sent to webhook`); + }); + } +} diff --git a/vector.toml b/vector.toml new file mode 100644 index 000000000..e8eb21818 --- /dev/null +++ b/vector.toml @@ -0,0 +1,37 @@ +# The data source that Vector will collect logs from +[sources.webhook] +type = "http_server" # The protocol to use +address = "0.0.0.0:9000" # The address to bind to +healthcheck = true # Enable built-in health checks +body_size_limit = "1mb" # Maximum size of request body +auth.password = "${EXPORT_VECTOR_WEBHOOK_PASSWORD}" +auth.username = "${EXPORT_VECTOR_WEBHOOK_USERNAME}" + +# The transformation(s) to apply to each event +[transforms.add_datadog_info] +type = "remap" +inputs = [ "webhook" ] +source = """ +# Set the values of the output object +.ddsource = "${EXPORT_VECTOR_DDSOURCE}" +.ddtags = "${EXPORT_VECTOR_DDTAGS}" +.hostname = "${EXPORT_VECTOR_HOSTNAME}" +.service = "${EXPORT_VECTOR_SERVICE}" +""" + +# The destination(s) to send the events to +[sinks.datadog_sink] +type = "datadog_logs" +inputs = [ "add_datadog_info" ] +default_api_key = "${EXPORT_VECTOR_DATADOG_API_KEY}" +compression = "gzip" +region = "${EXPORT_VECTOR_DATADOG_REGION}" +site = "${EXPORT_VECTOR_DATADOG_SITE}" +acknowledgements.enabled = true +healthcheck.enabled = true +request.concurrency = 10 +request.rate_limit_duration_secs = 1 +request.rate_limit_num = 10 +buffer.type = "disk" +# 1GB +buffer.max_size = 1073741952 From 48b23c0421e54a12c48c862f7a20d212a07a8b36 Mon Sep 17 00:00:00 2001 From: ukrocks007 Date: Thu, 30 Mar 2023 16:12:21 +0530 Subject: [PATCH 05/15] added environement variable for Volumes & data dir --- .env | 6 ++++-- .gitignore | 1 + docker-compose.yaml | 2 ++ vector.toml | 2 ++ 4 files changed, 9 insertions(+), 2 deletions(-) diff --git a/.env b/.env index d722dfe41..645dbcda7 100644 --- a/.env +++ b/.env @@ -27,7 +27,7 @@ NEXTAUTH_SECRET=secret RETRACED_HOST_URL=http://retraced-api:3000/auditlog RETRACED_EXTERNAL_URL=http://localhost:3000/auditlog -# EE +# Export Logs EXPORT_VECTOR_DATADOG_API_KEY= EXPORT_VECTOR_WEBHOOK_URL=http://vector:9000 EXPORT_VECTOR_WEBHOOK_USERNAME=admin @@ -37,4 +37,6 @@ EXPORT_VECTOR_DDTAGS="Audit-Logs, Retraced, BoxyHQ" EXPORT_VECTOR_HOSTNAME="127.0.0.1" EXPORT_VECTOR_SERVICE="Retraced-audit-logs" EXPORT_VECTOR_DATADOG_REGION=us -EXPORT_VECTOR_DATADOG_SITE=datadoghq.com \ No newline at end of file +EXPORT_VECTOR_DATADOG_SITE=datadoghq.com +EXPORT_VECTOR_DATA_DIR="/var/lib/vector/" +EXPORT_VECTOR_DATA_DIR_HOST="./vector/data" diff --git a/.gitignore b/.gitignore index 74791cb0c..07ed33130 100644 --- a/.gitignore +++ b/.gitignore @@ -106,3 +106,4 @@ test-results.xml .env.development.local .env.test.local .env.production.local +vector/* diff --git a/docker-compose.yaml b/docker-compose.yaml index cfb407ca8..65ee01cf1 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -210,7 +210,9 @@ services: - EXPORT_VECTOR_DATADOG_API_KEY=${EXPORT_VECTOR_DATADOG_API_KEY} - EXPORT_VECTOR_DATADOG_REGION=${EXPORT_VECTOR_DATADOG_REGION} - EXPORT_VECTOR_DATADOG_SITE=${EXPORT_VECTOR_DATADOG_SITE} + - EXPORT_VECTOR_DATA_DIR=${EXPORT_VECTOR_DATA_DIR} volumes: - ./vector.toml:/etc/vector/vector.toml + - ${EXPORT_VECTOR_DATA_DIR_HOST}:${EXPORT_VECTOR_DATA_DIR} networks: - retraced diff --git a/vector.toml b/vector.toml index e8eb21818..6a1aefdde 100644 --- a/vector.toml +++ b/vector.toml @@ -1,3 +1,5 @@ +data_dir = "${EXPORT_VECTOR_DATA_DIR}" + # The data source that Vector will collect logs from [sources.webhook] type = "http_server" # The protocol to use From 4973b5d7239d1bf291d549431a69561faa94ef37 Mon Sep 17 00:00:00 2001 From: ukrocks007 Date: Thu, 30 Mar 2023 17:15:21 +0530 Subject: [PATCH 06/15] fix --- src/config.ts | 1 - src/ee/export/index.ts | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/config.ts b/src/config.ts index ddb72e555..010c8e802 100644 --- a/src/config.ts +++ b/src/config.ts @@ -79,7 +79,6 @@ export default { env.RETRACED_NO_ANALYTICS || process.env.DO_NOT_TRACK || env.DO_NOT_TRACK, - //EXPORT CONFIG EXPORT_VECTOR_WEBHOOK_URL: process.env.EXPORT_VECTOR_WEBHOOK_URL || env.EXPORT_VECTOR_WEBHOOK_URL, EXPORT_VECTOR_WEBHOOK_USERNAME: process.env.EXPORT_VECTOR_WEBHOOK_USERNAME || env.EXPORT_VECTOR_WEBHOOK_USERNAME, diff --git a/src/ee/export/index.ts b/src/ee/export/index.ts index d02c9ab71..121587310 100644 --- a/src/ee/export/index.ts +++ b/src/ee/export/index.ts @@ -2,7 +2,7 @@ import axios from "axios"; import config from "../../config"; import { logger } from "../../logger"; -export default async function sendToWebhook(event: any): Promise { +export default function sendToWebhook(event: any): void { if (config.EXPORT_VECTOR_WEBHOOK_URL) { delete event.raw; axios From e3d9a5aaa8170f4e188131befe48e9d58c97007e Mon Sep 17 00:00:00 2001 From: ukrocks007 Date: Thu, 30 Mar 2023 20:11:36 +0530 Subject: [PATCH 07/15] fix --- .env | 24 ++++++++++++------------ docker-compose.yaml | 22 +++++++++++----------- src/config.ts | 8 +++----- vector.toml | 20 ++++++++++---------- 4 files changed, 36 insertions(+), 38 deletions(-) diff --git a/.env b/.env index 645dbcda7..69eda8562 100644 --- a/.env +++ b/.env @@ -28,15 +28,15 @@ RETRACED_HOST_URL=http://retraced-api:3000/auditlog RETRACED_EXTERNAL_URL=http://localhost:3000/auditlog # Export Logs -EXPORT_VECTOR_DATADOG_API_KEY= -EXPORT_VECTOR_WEBHOOK_URL=http://vector:9000 -EXPORT_VECTOR_WEBHOOK_USERNAME=admin -EXPORT_VECTOR_WEBHOOK_PASSWORD=admin -EXPORT_VECTOR_DDSOURCE=local-dev-machine -EXPORT_VECTOR_DDTAGS="Audit-Logs, Retraced, BoxyHQ" -EXPORT_VECTOR_HOSTNAME="127.0.0.1" -EXPORT_VECTOR_SERVICE="Retraced-audit-logs" -EXPORT_VECTOR_DATADOG_REGION=us -EXPORT_VECTOR_DATADOG_SITE=datadoghq.com -EXPORT_VECTOR_DATA_DIR="/var/lib/vector/" -EXPORT_VECTOR_DATA_DIR_HOST="./vector/data" +EXPORT_DATADOG_API_KEY= +EXPORT_WEBHOOK_URL=http://vector:9000 +EXPORT_WEBHOOK_USERNAME=admin +EXPORT_WEBHOOK_PASSWORD=admin +EXPORT_DDSOURCE=local-dev-machine +EXPORT_DDTAGS="Audit-Logs, Retraced, BoxyHQ" +EXPORT_HOSTNAME="127.0.0.1" +EXPORT_SERVICE="Retraced-audit-logs" +EXPORT_DATADOG_REGION=us +EXPORT_DATADOG_SITE=datadoghq.com +EXPORT_DATA_DIR="/var/lib/vector/" +EXPORT_DATA_DIR_HOST="./vector/data" diff --git a/docker-compose.yaml b/docker-compose.yaml index 65ee01cf1..d0d5e771b 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -201,18 +201,18 @@ services: vector: image: timberio/vector:0.X-alpine environment: - - EXPORT_VECTOR_WEBHOOK_USERNAME=${EXPORT_VECTOR_WEBHOOK_USERNAME} - - EXPORT_VECTOR_WEBHOOK_PASSWORD=${EXPORT_VECTOR_WEBHOOK_PASSWORD} - - EXPORT_VECTOR_DDSOURCE=${EXPORT_VECTOR_DDSOURCE} - - EXPORT_VECTOR_DDTAGS=${EXPORT_VECTOR_DDTAGS} - - EXPORT_VECTOR_HOSTNAME=${EXPORT_VECTOR_HOSTNAME} - - EXPORT_VECTOR_SERVICE=${EXPORT_VECTOR_SERVICE} - - EXPORT_VECTOR_DATADOG_API_KEY=${EXPORT_VECTOR_DATADOG_API_KEY} - - EXPORT_VECTOR_DATADOG_REGION=${EXPORT_VECTOR_DATADOG_REGION} - - EXPORT_VECTOR_DATADOG_SITE=${EXPORT_VECTOR_DATADOG_SITE} - - EXPORT_VECTOR_DATA_DIR=${EXPORT_VECTOR_DATA_DIR} + - EXPORT_WEBHOOK_USERNAME=${EXPORT_WEBHOOK_USERNAME} + - EXPORT_WEBHOOK_PASSWORD=${EXPORT_WEBHOOK_PASSWORD} + - EXPORT_DDSOURCE=${EXPORT_DDSOURCE} + - EXPORT_DDTAGS=${EXPORT_DDTAGS} + - EXPORT_HOSTNAME=${EXPORT_HOSTNAME} + - EXPORT_SERVICE=${EXPORT_SERVICE} + - EXPORT_DATADOG_API_KEY=${EXPORT_DATADOG_API_KEY} + - EXPORT_DATADOG_REGION=${EXPORT_DATADOG_REGION} + - EXPORT_DATADOG_SITE=${EXPORT_DATADOG_SITE} + - EXPORT_DATA_DIR=${EXPORT_DATA_DIR} volumes: - ./vector.toml:/etc/vector/vector.toml - - ${EXPORT_VECTOR_DATA_DIR_HOST}:${EXPORT_VECTOR_DATA_DIR} + - ${EXPORT_DATA_DIR_HOST}:${EXPORT_DATA_DIR} networks: - retraced diff --git a/src/config.ts b/src/config.ts index 010c8e802..6e35a2b77 100644 --- a/src/config.ts +++ b/src/config.ts @@ -79,9 +79,7 @@ export default { env.RETRACED_NO_ANALYTICS || process.env.DO_NOT_TRACK || env.DO_NOT_TRACK, - EXPORT_VECTOR_WEBHOOK_URL: process.env.EXPORT_VECTOR_WEBHOOK_URL || env.EXPORT_VECTOR_WEBHOOK_URL, - EXPORT_VECTOR_WEBHOOK_USERNAME: - process.env.EXPORT_VECTOR_WEBHOOK_USERNAME || env.EXPORT_VECTOR_WEBHOOK_USERNAME, - EXPORT_VECTOR_WEBHOOK_PASSWORD: - process.env.EXPORT_VECTOR_WEBHOOK_PASSWORD || env.EXPORT_VECTOR_WEBHOOK_PASSWORD, + EXPORT_WEBHOOK_URL: process.env.EXPORT_WEBHOOK_URL || env.EXPORT_WEBHOOK_URL, + EXPORT_WEBHOOK_USERNAME: process.env.EXPORT_WEBHOOK_USERNAME || env.EXPORT_WEBHOOK_USERNAME, + EXPORT_WEBHOOK_PASSWORD: process.env.EXPORT_WEBHOOK_PASSWORD || env.EXPORT_WEBHOOK_PASSWORD, }; diff --git a/vector.toml b/vector.toml index 6a1aefdde..b97c8eb7b 100644 --- a/vector.toml +++ b/vector.toml @@ -1,4 +1,4 @@ -data_dir = "${EXPORT_VECTOR_DATA_DIR}" +data_dir = "${EXPORT_DATA_DIR}" # The data source that Vector will collect logs from [sources.webhook] @@ -6,8 +6,8 @@ type = "http_server" # The protocol to use address = "0.0.0.0:9000" # The address to bind to healthcheck = true # Enable built-in health checks body_size_limit = "1mb" # Maximum size of request body -auth.password = "${EXPORT_VECTOR_WEBHOOK_PASSWORD}" -auth.username = "${EXPORT_VECTOR_WEBHOOK_USERNAME}" +auth.password = "${EXPORT_WEBHOOK_PASSWORD}" +auth.username = "${EXPORT_WEBHOOK_USERNAME}" # The transformation(s) to apply to each event [transforms.add_datadog_info] @@ -15,20 +15,20 @@ type = "remap" inputs = [ "webhook" ] source = """ # Set the values of the output object -.ddsource = "${EXPORT_VECTOR_DDSOURCE}" -.ddtags = "${EXPORT_VECTOR_DDTAGS}" -.hostname = "${EXPORT_VECTOR_HOSTNAME}" -.service = "${EXPORT_VECTOR_SERVICE}" +.ddsource = "${EXPORT_DDSOURCE}" +.ddtags = "${EXPORT_DDTAGS}" +.hostname = "${EXPORT_HOSTNAME}" +.service = "${EXPORT_SERVICE}" """ # The destination(s) to send the events to [sinks.datadog_sink] type = "datadog_logs" inputs = [ "add_datadog_info" ] -default_api_key = "${EXPORT_VECTOR_DATADOG_API_KEY}" +default_api_key = "${EXPORT_DATADOG_API_KEY}" compression = "gzip" -region = "${EXPORT_VECTOR_DATADOG_REGION}" -site = "${EXPORT_VECTOR_DATADOG_SITE}" +region = "${EXPORT_DATADOG_REGION}" +site = "${EXPORT_DATADOG_SITE}" acknowledgements.enabled = true healthcheck.enabled = true request.concurrency = 10 From d71452836d0a307f30a816192c25897790f70cf1 Mon Sep 17 00:00:00 2001 From: ukrocks007 Date: Thu, 30 Mar 2023 20:11:51 +0530 Subject: [PATCH 08/15] fix --- src/ee/export/index.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/ee/export/index.ts b/src/ee/export/index.ts index 121587310..2338875d7 100644 --- a/src/ee/export/index.ts +++ b/src/ee/export/index.ts @@ -3,20 +3,20 @@ import config from "../../config"; import { logger } from "../../logger"; export default function sendToWebhook(event: any): void { - if (config.EXPORT_VECTOR_WEBHOOK_URL) { + if (config.EXPORT_WEBHOOK_URL) { delete event.raw; axios .post( - config.EXPORT_VECTOR_WEBHOOK_URL, + config.EXPORT_WEBHOOK_URL, { message: JSON.stringify(event), }, { auth: - config.EXPORT_VECTOR_WEBHOOK_USERNAME && config.EXPORT_VECTOR_WEBHOOK_PASSWORD + config.EXPORT_WEBHOOK_USERNAME && config.EXPORT_WEBHOOK_PASSWORD ? { - username: config.EXPORT_VECTOR_WEBHOOK_USERNAME, - password: config.EXPORT_VECTOR_WEBHOOK_PASSWORD, + username: config.EXPORT_WEBHOOK_USERNAME, + password: config.EXPORT_WEBHOOK_PASSWORD, } : undefined, } From af654b04303092230980a92048da9caad1f254f1 Mon Sep 17 00:00:00 2001 From: ukrocks007 Date: Thu, 30 Mar 2023 20:30:00 +0530 Subject: [PATCH 09/15] fixes --- .env | 2 -- docker-compose.yaml | 3 +-- vector.toml | 2 +- 3 files changed, 2 insertions(+), 5 deletions(-) diff --git a/.env b/.env index 69eda8562..8a6ca1884 100644 --- a/.env +++ b/.env @@ -38,5 +38,3 @@ EXPORT_HOSTNAME="127.0.0.1" EXPORT_SERVICE="Retraced-audit-logs" EXPORT_DATADOG_REGION=us EXPORT_DATADOG_SITE=datadoghq.com -EXPORT_DATA_DIR="/var/lib/vector/" -EXPORT_DATA_DIR_HOST="./vector/data" diff --git a/docker-compose.yaml b/docker-compose.yaml index d0d5e771b..a2f55fe1e 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -210,9 +210,8 @@ services: - EXPORT_DATADOG_API_KEY=${EXPORT_DATADOG_API_KEY} - EXPORT_DATADOG_REGION=${EXPORT_DATADOG_REGION} - EXPORT_DATADOG_SITE=${EXPORT_DATADOG_SITE} - - EXPORT_DATA_DIR=${EXPORT_DATA_DIR} volumes: - ./vector.toml:/etc/vector/vector.toml - - ${EXPORT_DATA_DIR_HOST}:${EXPORT_DATA_DIR} + - ./vector/data:/var/lib/vector/ networks: - retraced diff --git a/vector.toml b/vector.toml index b97c8eb7b..3305c18ca 100644 --- a/vector.toml +++ b/vector.toml @@ -1,4 +1,4 @@ -data_dir = "${EXPORT_DATA_DIR}" +data_dir = "/var/lib/vector/" # The data source that Vector will collect logs from [sources.webhook] From 118c9b271db98698718b8987b6ea0424345b1225 Mon Sep 17 00:00:00 2001 From: ukrocks007 Date: Thu, 30 Mar 2023 20:48:39 +0530 Subject: [PATCH 10/15] fix --- .env | 2 +- docker-compose.yaml | 2 +- vector.toml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.env b/.env index 8a6ca1884..1a90bf12d 100644 --- a/.env +++ b/.env @@ -34,7 +34,7 @@ EXPORT_WEBHOOK_USERNAME=admin EXPORT_WEBHOOK_PASSWORD=admin EXPORT_DDSOURCE=local-dev-machine EXPORT_DDTAGS="Audit-Logs, Retraced, BoxyHQ" -EXPORT_HOSTNAME="127.0.0.1" +EXPORT_DDHOSTNAME="127.0.0.1" EXPORT_SERVICE="Retraced-audit-logs" EXPORT_DATADOG_REGION=us EXPORT_DATADOG_SITE=datadoghq.com diff --git a/docker-compose.yaml b/docker-compose.yaml index a2f55fe1e..12ea7fa29 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -205,7 +205,7 @@ services: - EXPORT_WEBHOOK_PASSWORD=${EXPORT_WEBHOOK_PASSWORD} - EXPORT_DDSOURCE=${EXPORT_DDSOURCE} - EXPORT_DDTAGS=${EXPORT_DDTAGS} - - EXPORT_HOSTNAME=${EXPORT_HOSTNAME} + - EXPORT_DDHOSTNAME=${EXPORT_DDHOSTNAME} - EXPORT_SERVICE=${EXPORT_SERVICE} - EXPORT_DATADOG_API_KEY=${EXPORT_DATADOG_API_KEY} - EXPORT_DATADOG_REGION=${EXPORT_DATADOG_REGION} diff --git a/vector.toml b/vector.toml index 3305c18ca..bf3773da1 100644 --- a/vector.toml +++ b/vector.toml @@ -17,7 +17,7 @@ source = """ # Set the values of the output object .ddsource = "${EXPORT_DDSOURCE}" .ddtags = "${EXPORT_DDTAGS}" -.hostname = "${EXPORT_HOSTNAME}" +.hostname = "${EXPORT_DDHOSTNAME}" .service = "${EXPORT_SERVICE}" """ From 0b80d373122fa8f915ff5dc72494931755a0e9d9 Mon Sep 17 00:00:00 2001 From: ukrocks007 Date: Thu, 30 Mar 2023 21:04:04 +0530 Subject: [PATCH 11/15] fix --- .env | 2 +- docker-compose.yaml | 2 +- vector.toml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.env b/.env index 1a90bf12d..ebb2fefe1 100644 --- a/.env +++ b/.env @@ -34,7 +34,7 @@ EXPORT_WEBHOOK_USERNAME=admin EXPORT_WEBHOOK_PASSWORD=admin EXPORT_DDSOURCE=local-dev-machine EXPORT_DDTAGS="Audit-Logs, Retraced, BoxyHQ" -EXPORT_DDHOSTNAME="127.0.0.1" +EXPORT_DATADOG_HOSTNAME="127.0.0.1" EXPORT_SERVICE="Retraced-audit-logs" EXPORT_DATADOG_REGION=us EXPORT_DATADOG_SITE=datadoghq.com diff --git a/docker-compose.yaml b/docker-compose.yaml index 12ea7fa29..5e92c4f21 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -205,7 +205,7 @@ services: - EXPORT_WEBHOOK_PASSWORD=${EXPORT_WEBHOOK_PASSWORD} - EXPORT_DDSOURCE=${EXPORT_DDSOURCE} - EXPORT_DDTAGS=${EXPORT_DDTAGS} - - EXPORT_DDHOSTNAME=${EXPORT_DDHOSTNAME} + - EXPORT_DATADOG_HOSTNAME=${EXPORT_DATADOG_HOSTNAME} - EXPORT_SERVICE=${EXPORT_SERVICE} - EXPORT_DATADOG_API_KEY=${EXPORT_DATADOG_API_KEY} - EXPORT_DATADOG_REGION=${EXPORT_DATADOG_REGION} diff --git a/vector.toml b/vector.toml index bf3773da1..e2673ecb8 100644 --- a/vector.toml +++ b/vector.toml @@ -17,7 +17,7 @@ source = """ # Set the values of the output object .ddsource = "${EXPORT_DDSOURCE}" .ddtags = "${EXPORT_DDTAGS}" -.hostname = "${EXPORT_DDHOSTNAME}" +.hostname = "${EXPORT_DATADOG_HOSTNAME}" .service = "${EXPORT_SERVICE}" """ From 1a21e3d40b7e27b7651c97ecdcc683d363e17ca3 Mon Sep 17 00:00:00 2001 From: ukrocks007 Date: Fri, 31 Mar 2023 13:44:20 +0530 Subject: [PATCH 12/15] added s3 sink and tested with minio --- docker-compose.yaml | 9 +++++++++ vector.toml | 11 +++++++++++ 2 files changed, 20 insertions(+) diff --git a/docker-compose.yaml b/docker-compose.yaml index 5e92c4f21..9b66cc177 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -215,3 +215,12 @@ services: - ./vector/data:/var/lib/vector/ networks: - retraced + + minio: + image: quay.io/minio/minio:latest + ports: + - "9002:9000" + - "9001:9001" + networks: + - retraced + command: server /data --console-address ":9001" diff --git a/vector.toml b/vector.toml index e2673ecb8..1b22608ee 100644 --- a/vector.toml +++ b/vector.toml @@ -37,3 +37,14 @@ request.rate_limit_num = 10 buffer.type = "disk" # 1GB buffer.max_size = 1073741952 + +[sinks.s3_sink] +type = "aws_s3" +inputs = ["webhook"] +bucket = "test" +region = "us-east-1" +endpoint = "http://minio:9000" +encoding.codec = "json" +acknowledgements.enabled = true +auth.access_key_id="GB7IVdZwvHYOLkhr" +auth.secret_access_key="aDmgK7w9hm3HPgXY9EcV4kNDUrO3Hsyv" From 157bc675406ccfdc799ae843cd6574b66f00b452 Mon Sep 17 00:00:00 2001 From: ukrocks007 Date: Fri, 31 Mar 2023 19:45:42 +0530 Subject: [PATCH 13/15] S3 config throught environment variables --- .env | 5 +++++ .gitignore | 1 + docker-compose.yaml | 8 ++++++++ vector.toml | 20 +++++++++++++++----- 4 files changed, 29 insertions(+), 5 deletions(-) diff --git a/.env b/.env index ebb2fefe1..091fec7e1 100644 --- a/.env +++ b/.env @@ -38,3 +38,8 @@ EXPORT_DATADOG_HOSTNAME="127.0.0.1" EXPORT_SERVICE="Retraced-audit-logs" EXPORT_DATADOG_REGION=us EXPORT_DATADOG_SITE=datadoghq.com +# S3 Bucket +EXPORT_S3_BUCKET= +EXPORT_S3_REGION= +EXPORT_S3_ACCESS_KEY_ID= +EXPORT_S3_SECRET_ACCESS_KEY= diff --git a/.gitignore b/.gitignore index 07ed33130..f94b9975f 100644 --- a/.gitignore +++ b/.gitignore @@ -107,3 +107,4 @@ test-results.xml .env.test.local .env.production.local vector/* +minio/* diff --git a/docker-compose.yaml b/docker-compose.yaml index 9b66cc177..f01e9a4d0 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -210,17 +210,25 @@ services: - EXPORT_DATADOG_API_KEY=${EXPORT_DATADOG_API_KEY} - EXPORT_DATADOG_REGION=${EXPORT_DATADOG_REGION} - EXPORT_DATADOG_SITE=${EXPORT_DATADOG_SITE} + - EXPORT_S3_BUCKET=${EXPORT_S3_BUCKET} + - EXPORT_S3_REGION=${EXPORT_S3_REGION} + - EXPORT_S3_ACCESS_KEY_ID=${EXPORT_S3_ACCESS_KEY_ID} + - EXPORT_S3_SECRET_ACCESS_KEY=${EXPORT_S3_SECRET_ACCESS_KEY} volumes: - ./vector.toml:/etc/vector/vector.toml - ./vector/data:/var/lib/vector/ networks: - retraced + depends_on: + - "minio" minio: image: quay.io/minio/minio:latest ports: - "9002:9000" - "9001:9001" + volumes: + - ./minio/data:/data networks: - retraced command: server /data --console-address ":9001" diff --git a/vector.toml b/vector.toml index 1b22608ee..842613609 100644 --- a/vector.toml +++ b/vector.toml @@ -9,7 +9,7 @@ body_size_limit = "1mb" # Maximum size of request body auth.password = "${EXPORT_WEBHOOK_PASSWORD}" auth.username = "${EXPORT_WEBHOOK_USERNAME}" -# The transformation(s) to apply to each event +# The transformation(s) to apply to each event for DataDog [transforms.add_datadog_info] type = "remap" inputs = [ "webhook" ] @@ -21,6 +21,15 @@ source = """ .service = "${EXPORT_SERVICE}" """ +# The transformation(s) to apply to each event for s3 +[transforms.s3_transform] +type = "remap" +inputs = [ "webhook" ] +source = """ +# Set the values of the output object +. = parse_json!(parse_json!(.message).message) +""" + # The destination(s) to send the events to [sinks.datadog_sink] type = "datadog_logs" @@ -40,11 +49,12 @@ buffer.max_size = 1073741952 [sinks.s3_sink] type = "aws_s3" -inputs = ["webhook"] +inputs = ["s3_transform"] bucket = "test" -region = "us-east-1" +region = "${EXPORT_S3_REGION}" endpoint = "http://minio:9000" encoding.codec = "json" acknowledgements.enabled = true -auth.access_key_id="GB7IVdZwvHYOLkhr" -auth.secret_access_key="aDmgK7w9hm3HPgXY9EcV4kNDUrO3Hsyv" +auth.access_key_id="${EXPORT_S3_ACCESS_KEY_ID}" +auth.secret_access_key="${EXPORT_S3_SECRET_ACCESS_KEY}" +batch.max_events = 10 From 71f42d39c089c14dacee9839f94af5f49e13bcef Mon Sep 17 00:00:00 2001 From: ukrocks007 Date: Wed, 26 Apr 2023 16:05:18 +0530 Subject: [PATCH 14/15] ECS VRL changes --- docker-compose.yaml | 21 ++++++------ src/ee/export/index.ts | 24 ++++++-------- vector.toml | 72 ++++++++++++++++++++++++++++++++++++++++-- 3 files changed, 90 insertions(+), 27 deletions(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index f01e9a4d0..0783b1f9e 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -124,16 +124,16 @@ services: # networks: # - retraced - # kibana: - # image: docker.elastic.co/kibana/kibana:7.8.0 - # environment: - # - ELASTICSEARCH_HOSTS=http://elasticsearch:9200 - # networks: - # - retraced - # depends_on: - # - elasticsearch - # ports: - # - 5601:5601 + kibana: + image: docker.elastic.co/kibana/kibana:7.8.0 + environment: + - ELASTICSEARCH_HOSTS=http://elasticsearch:9200 + networks: + - retraced + depends_on: + - elasticsearch + ports: + - 5601:5601 retraced-dev-bootstrap: build: @@ -221,6 +221,7 @@ services: - retraced depends_on: - "minio" + - "elasticsearch" minio: image: quay.io/minio/minio:latest diff --git a/src/ee/export/index.ts b/src/ee/export/index.ts index 2338875d7..fb4a49cff 100644 --- a/src/ee/export/index.ts +++ b/src/ee/export/index.ts @@ -6,21 +6,15 @@ export default function sendToWebhook(event: any): void { if (config.EXPORT_WEBHOOK_URL) { delete event.raw; axios - .post( - config.EXPORT_WEBHOOK_URL, - { - message: JSON.stringify(event), - }, - { - auth: - config.EXPORT_WEBHOOK_USERNAME && config.EXPORT_WEBHOOK_PASSWORD - ? { - username: config.EXPORT_WEBHOOK_USERNAME, - password: config.EXPORT_WEBHOOK_PASSWORD, - } - : undefined, - } - ) + .post(config.EXPORT_WEBHOOK_URL, event, { + auth: + config.EXPORT_WEBHOOK_USERNAME && config.EXPORT_WEBHOOK_PASSWORD + ? { + username: config.EXPORT_WEBHOOK_USERNAME, + password: config.EXPORT_WEBHOOK_PASSWORD, + } + : undefined, + }) .catch(() => { logger.info(`[VECTOR EXPORT] Failed to send to webhook`); }) diff --git a/vector.toml b/vector.toml index 842613609..2220ecfbb 100644 --- a/vector.toml +++ b/vector.toml @@ -1,5 +1,6 @@ data_dir = "/var/lib/vector/" - +[api] + enabled = true # The data source that Vector will collect logs from [sources.webhook] type = "http_server" # The protocol to use @@ -27,7 +28,60 @@ type = "remap" inputs = [ "webhook" ] source = """ # Set the values of the output object -. = parse_json!(parse_json!(.message).message) +. = parse_json!(.message) +""" + +[transforms.ecs_transform] +type = "remap" +inputs = [ "webhook" ] +source = """ +# Set the values of the output object +. = parse_json!(.message) +.host.name = "localhost" +.host.ip = .source_ip +.event.action = .action +.event.code = .crud +.event.module = .component +if .event.received == null { + .event.received = now() +} else { + .event.received = format_timestamp!(.event.received, format: "%+") +} +.event.dataset = "Audit Log" +.user.id = .actor.id +.user.name = .actor.name +.user.domain = .actor.href +.service.id = .target.id +.service.name = .target.name +.service.address = .target.href +.service.type = .target.type +.group.id = .group.id +.group.name = .group.name +.group.domain = .group.href +.source.ip = .source_ip +.message = .description +if .event.is_failure == true { + .event.outcome = "failure" + } else { + .event.outcome = "success" +} +if .event.created == null { + .@timestamp = now() +} else { + .@timestamp = format_timestamp!(.event.created, format: "%+") +} +del(.actor) +del(.target) +del(.received) +del(.action) +del(.crud) +del(.is_failure) +del(.component) +del(.group) +del(.source_ip) +del(.description) +del(.created) +del(.canonical_time) """ # The destination(s) to send the events to @@ -58,3 +112,17 @@ acknowledgements.enabled = true auth.access_key_id="${EXPORT_S3_ACCESS_KEY_ID}" auth.secret_access_key="${EXPORT_S3_SECRET_ACCESS_KEY}" batch.max_events = 10 + +[sinks.ecs_sink] +type = "elasticsearch" +inputs = [ "ecs_transform" ] +acknowledgements.enabled = true +api_version = "v7" +auth.strategy = "basic" +auth.user = "elastic" +auth.password = "changeme" +buffer.type = "memory" +buffer.max_events = 10 +bulk.action = "index" +bulk.index = "vector-%Y-%m-%d" +endpoints = ["http://elasticsearch:9200"] \ No newline at end of file From 534fac56a59562cd5fc93a858251c84bd1c961ce Mon Sep 17 00:00:00 2001 From: ukrocks007 Date: Fri, 28 Apr 2023 12:48:47 +0530 Subject: [PATCH 15/15] fix --- vector.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vector.toml b/vector.toml index 2220ecfbb..4dc878d89 100644 --- a/vector.toml +++ b/vector.toml @@ -111,7 +111,7 @@ encoding.codec = "json" acknowledgements.enabled = true auth.access_key_id="${EXPORT_S3_ACCESS_KEY_ID}" auth.secret_access_key="${EXPORT_S3_SECRET_ACCESS_KEY}" -batch.max_events = 10 +batch.max_events = 1000 [sinks.ecs_sink] type = "elasticsearch"