From b3154fe9abf0f8032ce7bddd6a98aa1b2cd52285 Mon Sep 17 00:00:00 2001 From: GabrielEluan Date: Wed, 9 Feb 2022 18:57:26 -0300 Subject: [PATCH 1/6] Add masterdata schema and client for EventRegistry --- manifest.json | 11 +++++++++++ masterdata/eventRegistry/schema.json | 12 ++++++++++++ node/clients/index.ts | 11 ++++++++++- 3 files changed, 33 insertions(+), 1 deletion(-) create mode 100644 masterdata/eventRegistry/schema.json diff --git a/manifest.json b/manifest.json index 62650a1..7d82036 100644 --- a/manifest.json +++ b/manifest.json @@ -9,6 +9,7 @@ "dependencies": {}, "builders": { "node": "6.x", + "masterdata": "1.x", "docs": "0.x" }, "scripts": { @@ -16,6 +17,16 @@ }, "credentialType": "absolute", "policies": [ + { + "name": "ADMIN_DS" + }, + { + "name": "outbound-access", + "attrs": { + "host": "api.vtex.com", + "path": "/api/dataentities/*" + } + }, { "name": "colossus-fire-event" }, diff --git a/masterdata/eventRegistry/schema.json b/masterdata/eventRegistry/schema.json new file mode 100644 index 0000000..33a48db --- /dev/null +++ b/masterdata/eventRegistry/schema.json @@ -0,0 +1,12 @@ +{ + "$schema": "http://json-schema.org/schema#", + "title": "Event Registry", + "type": "object", + "properties": { + "hasBeenProcessed": { + "type": "boolean" + } + }, + "required": ["hasBeenProcessed"], + "v-cache": false +} diff --git a/node/clients/index.ts b/node/clients/index.ts index 929cd03..f360536 100644 --- a/node/clients/index.ts +++ b/node/clients/index.ts @@ -1,3 +1,12 @@ import { IOClients } from '@vtex/api' +import { masterDataFor } from '@vtex/clients' +import type { EventRegistry } from 'vtex.spreadsheet-event-broadcaster' -export class Clients extends IOClients {} +export class Clients extends IOClients { + public get eventRegistry() { + return this.getOrSet( + 'eventRegistry', + masterDataFor('eventRegistry') + ) + } +} From a4f3406fdcef054a8dbe1e35baa6e949f384973a Mon Sep 17 00:00:00 2001 From: GabrielEluan Date: Wed, 9 Feb 2022 18:59:31 -0300 Subject: [PATCH 2/6] Add eventBroadcaster handler and middlewares --- node/index.ts | 21 +++- node/middlewares/broadcaster/broadcast.ts | 19 +++ node/middlewares/broadcaster/splitPayload.ts | 38 ++++++ .../broadcaster/verifyUniqueness.ts | 19 +++ node/middlewares/notify/startEventChain.ts | 7 +- node/package.json | 4 +- node/service.json | 8 +- node/services/eventRegistry.ts | 26 ++++ node/utils/constants.ts | 12 +- node/yarn.lock | 116 +++++++++++++++++- 10 files changed, 262 insertions(+), 8 deletions(-) create mode 100644 node/middlewares/broadcaster/broadcast.ts create mode 100644 node/middlewares/broadcaster/splitPayload.ts create mode 100644 node/middlewares/broadcaster/verifyUniqueness.ts create mode 100644 node/services/eventRegistry.ts diff --git a/node/index.ts b/node/index.ts index 96ea897..77485df 100644 --- a/node/index.ts +++ b/node/index.ts @@ -1,7 +1,15 @@ -import type { ClientsConfig, ServiceContext, RecorderState } from '@vtex/api' +import type { + ClientsConfig, + ServiceContext, + RecorderState, + EventContext, +} from '@vtex/api' import { method, Service } from '@vtex/api' import { Clients } from './clients' +import { broadcast } from './middlewares/broadcaster/broadcast' +import { splitPayload } from './middlewares/broadcaster/splitPayload' +import { verifyUniqueness } from './middlewares/broadcaster/verifyUniqueness' import { parseFile } from './middlewares/notify/parseFile' import { startEventChain } from './middlewares/notify/startEventChain' @@ -24,6 +32,14 @@ declare global { payload: unknown[] appId: string } + + interface BroadcasterEventContext extends EventContext { + body: { + eventId: string + payload: unknown[] + clientAppId: string + } + } } export default new Service({ @@ -33,4 +49,7 @@ export default new Service({ POST: [parseFile, startEventChain], }), }, + events: { + eventBroadcaster: [verifyUniqueness, splitPayload, broadcast], + }, }) diff --git a/node/middlewares/broadcaster/broadcast.ts b/node/middlewares/broadcaster/broadcast.ts new file mode 100644 index 0000000..795698a --- /dev/null +++ b/node/middlewares/broadcaster/broadcast.ts @@ -0,0 +1,19 @@ +import { OUTPUT_EVENT_KEY as eventKey } from '../../utils/constants' + +export async function broadcast( + ctx: BroadcasterEventContext, + next: () => Promise +) { + const { + clients: { events }, + body: { payload, clientAppId }, + } = ctx + + payload.forEach((row) => { + events.sendEvent(clientAppId, eventKey, { + data: row, + }) + }) + + await next() +} diff --git a/node/middlewares/broadcaster/splitPayload.ts b/node/middlewares/broadcaster/splitPayload.ts new file mode 100644 index 0000000..b14aa89 --- /dev/null +++ b/node/middlewares/broadcaster/splitPayload.ts @@ -0,0 +1,38 @@ +import { v4 as uuid } from 'uuid' + +import { + APP_ID as thisAppId, + EVENT_CHAIN_KEY as eventKey, + ARRAY_SIZE_TARGET, + ARRAY_SPLIT_FACTOR, +} from '../../utils/constants' + +export async function splitPayload( + ctx: BroadcasterEventContext, + next: () => Promise +) { + const { + clients: { events }, + body: { payload, clientAppId }, + } = ctx + + if (payload.length > ARRAY_SIZE_TARGET) { + const chunkSize = Math.ceil(payload.length / ARRAY_SPLIT_FACTOR) + + for (let i = 0; i < payload.length; i += chunkSize) { + const eventId = uuid() + + const chunk = payload.slice(i, i + chunkSize) + + events.sendEvent(thisAppId, eventKey, { + eventId, + payload: chunk, + clientAppId, + }) + } + + return + } + + await next() +} diff --git a/node/middlewares/broadcaster/verifyUniqueness.ts b/node/middlewares/broadcaster/verifyUniqueness.ts new file mode 100644 index 0000000..c4638ed --- /dev/null +++ b/node/middlewares/broadcaster/verifyUniqueness.ts @@ -0,0 +1,19 @@ +import { EventRegistryService } from '../../services/eventRegistry' + +export async function verifyUniqueness( + ctx: BroadcasterEventContext, + next: () => Promise +) { + const { + clients: { eventRegistry }, + body: { eventId }, + } = ctx + + const registryService = new EventRegistryService(eventRegistry) + + const isUnique = await registryService.isEventUnique(eventId) + + if (!isUnique) return + + await next() +} diff --git a/node/middlewares/notify/startEventChain.ts b/node/middlewares/notify/startEventChain.ts index bfa3311..01236f7 100644 --- a/node/middlewares/notify/startEventChain.ts +++ b/node/middlewares/notify/startEventChain.ts @@ -1,6 +1,9 @@ import { v4 as uuid } from 'uuid' -import { APP_ID as thisAppId } from '../../utils/constants' +import { + APP_ID as thisAppId, + EVENT_CHAIN_KEY as eventKey, +} from '../../utils/constants' export async function startEventChain(ctx: Context) { const { @@ -10,7 +13,7 @@ export async function startEventChain(ctx: Context) { const eventId = uuid() - events.sendEvent(thisAppId, 'spreadsheet.event.broadcast', { + events.sendEvent(thisAppId, eventKey, { eventId, payload, clientAppId, diff --git a/node/package.json b/node/package.json index 51207ca..d7d23ee 100644 --- a/node/package.json +++ b/node/package.json @@ -1,5 +1,6 @@ { "dependencies": { + "@vtex/clients": "^2.19.4", "async-busboy": "^1.1.0", "co-body": "^6.0.0", "get-stream": "^6.0.1", @@ -17,7 +18,8 @@ "@vtex/api": "6.45.6", "@vtex/test-tools": "^1.0.0", "@vtex/tsconfig": "^0.5.6", - "typescript": "3.9.7" + "typescript": "3.9.7", + "vtex.spreadsheet-event-broadcaster": "https://dropper--sandboxbrdev.myvtex.com/_v/private/typings/linked/v1/vtex.spreadsheet-event-broadcaster@0.0.1+build1644428151/public/@types/vtex.spreadsheet-event-broadcaster" }, "scripts": { "lint": "tsc --noEmit --pretty" diff --git a/node/service.json b/node/service.json index f6e02f3..1f1c40a 100644 --- a/node/service.json +++ b/node/service.json @@ -1,7 +1,7 @@ { "memory": 256, "ttl": 10, - "timeout": 2, + "timeout": 30, "minReplicas": 2, "maxReplicas": 4, "workers": 1, @@ -18,5 +18,11 @@ } ] } + }, + "events": { + "eventBroadcaster": { + "sender": "vtex.spreadsheet-event-broadcaster", + "keys": ["event.chain"] + } } } diff --git a/node/services/eventRegistry.ts b/node/services/eventRegistry.ts new file mode 100644 index 0000000..945cbe0 --- /dev/null +++ b/node/services/eventRegistry.ts @@ -0,0 +1,26 @@ +import type { MasterDataEntity } from '@vtex/clients' +import type { EventRegistry } from 'vtex.spreadsheet-event-broadcaster' + +import { MASTER_DATA_NO_CHANGES_RESPONSE_STATUS } from '../utils/constants' + +export class EventRegistryService { + private client: MasterDataEntity + constructor(client: MasterDataEntity) { + this.client = client + } + + public async isEventUnique(eventId: string) { + try { + await this.client.saveOrUpdate({ + id: eventId, + hasBeenProcessed: true, + }) + } catch (error) { + if (error?.response?.status === MASTER_DATA_NO_CHANGES_RESPONSE_STATUS) { + return false + } + } + + return true + } +} diff --git a/node/utils/constants.ts b/node/utils/constants.ts index b94c1e0..3b08f64 100644 --- a/node/utils/constants.ts +++ b/node/utils/constants.ts @@ -1,7 +1,17 @@ -export const APP_ID = 'vtex.spreadsheet-event-broadcaster@0.x' +export const APP_ID = 'vtex.spreadsheet-event-broadcaster' + +export const EVENT_CHAIN_KEY = 'event.chain' + +export const OUTPUT_EVENT_KEY = 'row.output' + +export const MASTER_DATA_NO_CHANGES_RESPONSE_STATUS = 304 const _1GB = 1024 ** 3 export const MAXIMUM_FILE_SIZE = _1GB export const MAXIMUM_FILE_SIZE_STRING = '1GB' + +export const ARRAY_SIZE_TARGET = 1000 + +export const ARRAY_SPLIT_FACTOR = 100 diff --git a/node/yarn.lock b/node/yarn.lock index a4bd738..4d5018b 100644 --- a/node/yarn.lock +++ b/node/yarn.lock @@ -765,6 +765,19 @@ exec-sh "^0.3.2" minimist "^1.2.0" +"@gocommerce/utils@^0.7.3": + version "0.7.3" + resolved "https://registry.yarnpkg.com/@gocommerce/utils/-/utils-0.7.3.tgz#e1c43025d4d08d0a947fb6d3cc7e4e8a0fe097ca" + integrity sha512-cSrl0QzTWIvgweq6h73KFS7q9+95VUwfK1VeZ/PuOFdUrohKjRjUD9AYTtcCMgZvigvQr/X6xL2pWwBoKRwgWQ== + dependencies: + axios "0.18.0" + babel-runtime "^6.26.0" + cookie "0.3.1" + splunk-events "^1.3.1" + ts-node "5.0.1" + typescript "3.0.3" + uuid "3.2.1" + "@jest/console@^24.7.1", "@jest/console@^24.9.0": version "24.9.0" resolved "https://registry.yarnpkg.com/@jest/console/-/console-24.9.0.tgz#79b1bc06fb74a8cfb01cbdedf945584b1b9707f0" @@ -1301,6 +1314,13 @@ uuid "^3.3.3" xss "^1.0.6" +"@vtex/clients@^2.19.4": + version "2.19.4" + resolved "https://registry.yarnpkg.com/@vtex/clients/-/clients-2.19.4.tgz#accabee2352f3863f5be906c6e2ee0bdb81b100c" + integrity sha512-Bdmi8yQBcj7woX/1QxAebVMOzal+6GWyQYIQNWQhyNqbKq0BnBjwxVGG44T0zxawBPto6PuULTEP8r1SI3hmWQ== + dependencies: + "@gocommerce/utils" "^0.7.3" + "@vtex/node-error-report@^0.0.3": version "0.0.3" resolved "https://registry.yarnpkg.com/@vtex/node-error-report/-/node-error-report-0.0.3.tgz#365a2652aeebbd6b51ddc5d64c3c0bc1a326dc71" @@ -1642,6 +1662,11 @@ array-unique@^0.3.2: resolved "https://registry.yarnpkg.com/array-unique/-/array-unique-0.3.2.tgz#a894b75d4bc4f6cd679ef3244a9fd8f46ae2d428" integrity sha1-qJS3XUvE9s1nnvMkSp/Y9Gri1Cg= +arrify@^1.0.0: + version "1.0.1" + resolved "https://registry.yarnpkg.com/arrify/-/arrify-1.0.1.tgz#898508da2226f380df904728456849c1501a4b0d" + integrity sha1-iYUI2iIm84DfkEcoRWhJwVAaSw0= + asn1@~0.2.3: version "0.2.4" resolved "https://registry.yarnpkg.com/asn1/-/asn1-0.2.4.tgz#8d2475dfab553bb33e77b54e59e880bb8ce23136" @@ -1714,6 +1739,14 @@ axios-retry@^3.1.2: dependencies: is-retry-allowed "^1.1.0" +axios@0.18.0: + version "0.18.0" + resolved "https://registry.yarnpkg.com/axios/-/axios-0.18.0.tgz#32d53e4851efdc0a11993b6cd000789d70c05102" + integrity sha1-MtU+SFHv3AoRmTts0AB4nXDAUQI= + dependencies: + follow-redirects "^1.3.0" + is-buffer "^1.1.5" + axios@^0.21.1: version "0.21.4" resolved "https://registry.yarnpkg.com/axios/-/axios-0.21.4.tgz#c67b90dc0568e5c1cf2b0b858c43ba28e2eda575" @@ -1766,6 +1799,14 @@ babel-preset-jest@^24.9.0: "@babel/plugin-syntax-object-rest-spread" "^7.0.0" babel-plugin-jest-hoist "^24.9.0" +babel-runtime@^6.26.0: + version "6.26.0" + resolved "https://registry.yarnpkg.com/babel-runtime/-/babel-runtime-6.26.0.tgz#965c7058668e82b55d7bfe04ff2337bc8b5647fe" + integrity sha1-llxwWGaOgrVde/4E/yM3vItWR/4= + dependencies: + core-js "^2.4.0" + regenerator-runtime "^0.11.0" + balanced-match@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.0.tgz#89b4d199ab2bee49de164ea02b89ce462d71b767" @@ -1990,7 +2031,7 @@ cfb@^1.1.4: crc-32 "~1.2.0" printj "~1.3.0" -chalk@^2.0.0, chalk@^2.0.1, chalk@^2.4.1, chalk@^2.4.2: +chalk@^2.0.0, chalk@^2.0.1, chalk@^2.3.0, chalk@^2.4.1, chalk@^2.4.2: version "2.4.2" resolved "https://registry.yarnpkg.com/chalk/-/chalk-2.4.2.tgz#cd42541677a54333cf541a49108c1432b44c9424" integrity sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ== @@ -2137,7 +2178,7 @@ convert-source-map@^1.1.0, convert-source-map@^1.4.0: dependencies: safe-buffer "~5.1.1" -cookie@^0.3.1: +cookie@0.3.1, cookie@^0.3.1: version "0.3.1" resolved "https://registry.yarnpkg.com/cookie/-/cookie-0.3.1.tgz#e7e0a1f9ef43b4c8ba925c5c5a96e806d16873bb" integrity sha1-5+Ch+e9DtMi6klxcWpboBtFoc7s= @@ -2163,6 +2204,11 @@ core-js-compat@^3.1.1: browserslist "^4.6.6" semver "^6.3.0" +core-js@^2.4.0: + version "2.6.12" + resolved "https://registry.yarnpkg.com/core-js/-/core-js-2.6.12.tgz#d9333dfa7b065e347cc5682219d6f690859cc2ec" + integrity sha512-Kb2wC0fvsWfQrgk8HU5lW6U/Lcs8+9aaYcy4ZFc6DDlo4nZ7n70dEgE5rtR0oG6ufKDUnrwfWL1mXR5ljDatrQ== + core-util-is@1.0.2, core-util-is@~1.0.0: version "1.0.2" resolved "https://registry.yarnpkg.com/core-util-is/-/core-util-is-1.0.2.tgz#b5fd54220aa2bc5ab57aab7140c940754503c1a7" @@ -2388,6 +2434,11 @@ diff-sequences@^24.9.0: resolved "https://registry.yarnpkg.com/diff-sequences/-/diff-sequences-24.9.0.tgz#5715d6244e2aa65f48bba0bc972db0b0b11e95b5" integrity sha512-Dj6Wk3tWyTE+Fo1rW8v0Xhwk80um6yFYKbuAxc9c3EZxIHFDYwbi34Uk42u1CdnIiVorvt4RmlSDjIPyzGC2ew== +diff@^3.1.0: + version "3.5.0" + resolved "https://registry.yarnpkg.com/diff/-/diff-3.5.0.tgz#800c0dd1e0a8bfbc95835c202ad220fe317e5a12" + integrity sha512-A46qtFgd+g7pDZinpnwiRJtxbC1hpgf0uzP3iG89scHk0AUC7A1TGxf5OiiOUv/JMZR8GOt8hL900hV0bOy5xA== + domexception@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/domexception/-/domexception-1.0.1.tgz#937442644ca6a31261ef36e3ec677fe805582c90" @@ -2650,6 +2701,11 @@ follow-redirects@^1.14.0: resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.14.7.tgz#2004c02eb9436eee9a21446a6477debf17e81685" integrity sha512-+hbxoLbFMbRKDwohX8GkTataGqO6Jb7jGwpAlwgy2bIz25XtRm7KEzJM76R1WiNT5SwZkX4Y75SwBolkpmE7iQ== +follow-redirects@^1.3.0: + version "1.14.8" + resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.14.8.tgz#016996fb9a11a100566398b1c6839337d7bfa8fc" + integrity sha512-1x0S9UVJHsQprFcEC/qnNzBLcIxsjAV905f/UkQxbclCsoTWlacCNOpQa/anodLl2uaEKFhfWOvM2Qg77+15zA== + for-in@^1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/for-in/-/for-in-1.0.2.tgz#81068d295a8142ec0ac726c6e2200c30fb6d5e80" @@ -4070,6 +4126,11 @@ make-dir@^2.1.0: pify "^4.0.1" semver "^5.6.0" +make-error@^1.1.1: + version "1.3.6" + resolved "https://registry.yarnpkg.com/make-error/-/make-error-1.3.6.tgz#2eb2e37ea9b67c4891f684a1394799af484cf7a2" + integrity sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw== + makeerror@1.0.x: version "1.0.11" resolved "https://registry.yarnpkg.com/makeerror/-/makeerror-1.0.11.tgz#e01a5c9109f2af79660e4e8b9587790184f5a96c" @@ -4915,6 +4976,11 @@ regenerate@^1.4.0: resolved "https://registry.yarnpkg.com/regenerate/-/regenerate-1.4.0.tgz#4a856ec4b56e4077c557589cae85e7a4c8869a11" integrity sha512-1G6jJVDWrt0rK99kBjvEtziZNCICAuvIPkSiUFIQxVP06RCVpq3dmDo2oi6ABpYaDYaTRr67BEhL8r1wgEZZKg== +regenerator-runtime@^0.11.0: + version "0.11.1" + resolved "https://registry.yarnpkg.com/regenerator-runtime/-/regenerator-runtime-0.11.1.tgz#be05ad7f9bf7d22e056f9726cee5017fbf19e2e9" + integrity sha512-MguG95oij0fC3QV3URf4V2SDYGJhJnJGqvIIgdECeODCT98wSWDAJ94SSuVpYQUoTcGUIL6L4yNB7j1DFFHSBg== + regenerator-runtime@^0.13.1, regenerator-runtime@^0.13.2: version "0.13.3" resolved "https://registry.yarnpkg.com/regenerator-runtime/-/regenerator-runtime-0.13.3.tgz#7cf6a77d8f5c6f60eb73c5fc1955b2ceb01e6bf5" @@ -5245,6 +5311,14 @@ source-map-resolve@^0.5.0, source-map-resolve@^0.5.2: source-map-url "^0.4.0" urix "^0.1.0" +source-map-support@^0.5.3: + version "0.5.21" + resolved "https://registry.yarnpkg.com/source-map-support/-/source-map-support-0.5.21.tgz#04fe7c7f9e1ed2d662233c28cb2b35b9f63f6e4f" + integrity sha512-uBHU3L3czsIyYXKX88fdrGovxdSCoTGDRZ6SYXtSRxLZUzHg5P/66Ht6uoUlHu9EZod+inXhKo3qQgwXUT/y1w== + dependencies: + buffer-from "^1.0.0" + source-map "^0.6.0" + source-map-support@^0.5.6: version "0.5.13" resolved "https://registry.yarnpkg.com/source-map-support/-/source-map-support-0.5.13.tgz#31b24a9c2e73c2de85066c0feb7d44767ed52932" @@ -5301,6 +5375,11 @@ split-string@^3.0.1, split-string@^3.0.2: dependencies: extend-shallow "^3.0.0" +splunk-events@^1.3.1: + version "1.7.0" + resolved "https://registry.yarnpkg.com/splunk-events/-/splunk-events-1.7.0.tgz#4400c2b224387e598806c8fb7e6e687cf9143fe4" + integrity sha512-OdqBiyE45GIYWWzeNZVcZHqvagRwBINy26+u2Iu2nWQYzPtD7HnL6hSgWzFXNBzRxdQuUQWeAKE58I6pvJ9xjA== + ssf@~0.11.2: version "0.11.2" resolved "https://registry.yarnpkg.com/ssf/-/ssf-0.11.2.tgz#0b99698b237548d088fc43cdf2b70c1a7512c06c" @@ -5644,6 +5723,20 @@ ts-invariant@^0.4.0, ts-invariant@^0.4.2: dependencies: tslib "^1.9.3" +ts-node@5.0.1: + version "5.0.1" + resolved "https://registry.yarnpkg.com/ts-node/-/ts-node-5.0.1.tgz#78e5d1cb3f704de1b641e43b76be2d4094f06f81" + integrity sha512-XK7QmDcNHVmZkVtkiwNDWiERRHPyU8nBqZB1+iv2UhOG0q3RQ9HsZ2CMqISlFbxjrYFGfG2mX7bW4dAyxBVzUw== + dependencies: + arrify "^1.0.0" + chalk "^2.3.0" + diff "^3.1.0" + make-error "^1.1.1" + minimist "^1.2.0" + mkdirp "^0.5.1" + source-map-support "^0.5.3" + yn "^2.0.0" + tslib@^1.9.3: version "1.9.3" resolved "https://registry.yarnpkg.com/tslib/-/tslib-1.9.3.tgz#d7e4dd79245d85428c4d7e4822a79917954ca286" @@ -5681,6 +5774,11 @@ type-is@^1.6.16: media-typer "0.3.0" mime-types "~2.1.18" +typescript@3.0.3: + version "3.0.3" + resolved "https://registry.yarnpkg.com/typescript/-/typescript-3.0.3.tgz#4853b3e275ecdaa27f78fda46dc273a7eb7fc1c8" + integrity sha512-kk80vLW9iGtjMnIv11qyxLqZm20UklzuR2tL0QAnDIygIUIemcZMxlMWudl9OOt76H3ntVzcTiddQ1/pAAJMYg== + typescript@3.9.7: version "3.9.7" resolved "https://registry.yarnpkg.com/typescript/-/typescript-3.9.7.tgz#98d600a5ebdc38f40cb277522f12dc800e9e25fa" @@ -5781,6 +5879,11 @@ util.promisify@^1.0.0: define-properties "^1.1.2" object.getownpropertydescriptors "^2.0.3" +uuid@3.2.1: + version "3.2.1" + resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.2.1.tgz#12c528bb9d58d0b9265d9a2f6f0fe8be17ff1f14" + integrity sha512-jZnMwlb9Iku/O3smGWvZhauCf6cvvpKi4BKRiliS3cxnI+Gz9j5MEpTz2UFuXiKPJocb7gnsLHwiS05ige5BEA== + uuid@^3.1.0: version "3.3.2" resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.3.2.tgz#1b4af4955eb3077c501c23872fc6513811587131" @@ -5823,6 +5926,10 @@ verror@1.10.0: core-util-is "1.0.2" extsprintf "^1.2.0" +"vtex.spreadsheet-event-broadcaster@https://dropper--sandboxbrdev.myvtex.com/_v/private/typings/linked/v1/vtex.spreadsheet-event-broadcaster@0.0.1+build1644428151/public/@types/vtex.spreadsheet-event-broadcaster": + version "0.0.1" + resolved "https://dropper--sandboxbrdev.myvtex.com/_v/private/typings/linked/v1/vtex.spreadsheet-event-broadcaster@0.0.1+build1644428151/public/@types/vtex.spreadsheet-event-broadcaster#daaba24def8533f145f8cbcc110d5242f9747afd" + w3c-hr-time@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/w3c-hr-time/-/w3c-hr-time-1.0.1.tgz#82ac2bff63d950ea9e3189a58a65625fedf19045" @@ -6024,6 +6131,11 @@ ylru@^1.2.0: resolved "https://registry.npmjs.org/ylru/-/ylru-1.2.1.tgz#f576b63341547989c1de7ba288760923b27fe84f" integrity sha512-faQrqNMzcPCHGVC2aaOINk13K+aaBDUPjGWl0teOXywElLjyVAB6Oe2jj62jHYtwsU49jXhScYbvPENK+6zAvQ== +yn@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/yn/-/yn-2.0.0.tgz#e5adabc8acf408f6385fc76495684c88e6af689a" + integrity sha1-5a2ryKz0CPY4X8dklWhMiOavaJo= + zen-observable-ts@^0.8.20: version "0.8.20" resolved "https://registry.yarnpkg.com/zen-observable-ts/-/zen-observable-ts-0.8.20.tgz#44091e335d3fcbc97f6497e63e7f57d5b516b163" From 9cd726384fdf29d87ab25e1b664c1237218217d2 Mon Sep 17 00:00:00 2001 From: GabrielEluan Date: Wed, 9 Feb 2022 19:00:46 -0300 Subject: [PATCH 3/6] Update CHANGELOG.md --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8d09ea0..af1a152 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## [Unreleased] +### Added + +- EventBroadcaster handler and middlewares + ## [0.1.0] - 2022-02-09 ### Added From ad049cd27f5371a666019157b395738106d50b28 Mon Sep 17 00:00:00 2001 From: GabrielEluan Date: Thu, 10 Feb 2022 10:14:41 -0300 Subject: [PATCH 4/6] vtex setup --- node/package.json | 2 +- node/yarn.lock | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/node/package.json b/node/package.json index d7d23ee..f186ce1 100644 --- a/node/package.json +++ b/node/package.json @@ -19,7 +19,7 @@ "@vtex/test-tools": "^1.0.0", "@vtex/tsconfig": "^0.5.6", "typescript": "3.9.7", - "vtex.spreadsheet-event-broadcaster": "https://dropper--sandboxbrdev.myvtex.com/_v/private/typings/linked/v1/vtex.spreadsheet-event-broadcaster@0.0.1+build1644428151/public/@types/vtex.spreadsheet-event-broadcaster" + "vtex.spreadsheet-event-broadcaster": "https://dropper--sandboxbrdev.myvtex.com/_v/private/typings/linked/v1/vtex.spreadsheet-event-broadcaster@0.1.0+build1644442090/public/@types/vtex.spreadsheet-event-broadcaster" }, "scripts": { "lint": "tsc --noEmit --pretty" diff --git a/node/yarn.lock b/node/yarn.lock index 4d5018b..c6427db 100644 --- a/node/yarn.lock +++ b/node/yarn.lock @@ -5926,9 +5926,9 @@ verror@1.10.0: core-util-is "1.0.2" extsprintf "^1.2.0" -"vtex.spreadsheet-event-broadcaster@https://dropper--sandboxbrdev.myvtex.com/_v/private/typings/linked/v1/vtex.spreadsheet-event-broadcaster@0.0.1+build1644428151/public/@types/vtex.spreadsheet-event-broadcaster": - version "0.0.1" - resolved "https://dropper--sandboxbrdev.myvtex.com/_v/private/typings/linked/v1/vtex.spreadsheet-event-broadcaster@0.0.1+build1644428151/public/@types/vtex.spreadsheet-event-broadcaster#daaba24def8533f145f8cbcc110d5242f9747afd" +"vtex.spreadsheet-event-broadcaster@https://dropper--sandboxbrdev.myvtex.com/_v/private/typings/linked/v1/vtex.spreadsheet-event-broadcaster@0.1.0+build1644442090/public/@types/vtex.spreadsheet-event-broadcaster": + version "0.1.0" + resolved "https://dropper--sandboxbrdev.myvtex.com/_v/private/typings/linked/v1/vtex.spreadsheet-event-broadcaster@0.1.0+build1644442090/public/@types/vtex.spreadsheet-event-broadcaster#2d3e6dbe41ed44968e5c4ab23d04767f429f697f" w3c-hr-time@^1.0.1: version "1.0.1" From da1916da0712edc2b404cbe1f3015d7eb705514e Mon Sep 17 00:00:00 2001 From: GabrielEluan Date: Fri, 11 Feb 2022 11:24:18 -0300 Subject: [PATCH 5/6] vtex setup -i --- node/package.json | 2 +- node/yarn.lock | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/node/package.json b/node/package.json index f186ce1..b945cc6 100644 --- a/node/package.json +++ b/node/package.json @@ -19,7 +19,7 @@ "@vtex/test-tools": "^1.0.0", "@vtex/tsconfig": "^0.5.6", "typescript": "3.9.7", - "vtex.spreadsheet-event-broadcaster": "https://dropper--sandboxbrdev.myvtex.com/_v/private/typings/linked/v1/vtex.spreadsheet-event-broadcaster@0.1.0+build1644442090/public/@types/vtex.spreadsheet-event-broadcaster" + "vtex.spreadsheet-event-broadcaster": "http://vtex.vtexassets.com/_v/public/typings/v1/vtex.spreadsheet-event-broadcaster@0.1.0/public/_types/react" }, "scripts": { "lint": "tsc --noEmit --pretty" diff --git a/node/yarn.lock b/node/yarn.lock index c6427db..4a44788 100644 --- a/node/yarn.lock +++ b/node/yarn.lock @@ -5926,9 +5926,9 @@ verror@1.10.0: core-util-is "1.0.2" extsprintf "^1.2.0" -"vtex.spreadsheet-event-broadcaster@https://dropper--sandboxbrdev.myvtex.com/_v/private/typings/linked/v1/vtex.spreadsheet-event-broadcaster@0.1.0+build1644442090/public/@types/vtex.spreadsheet-event-broadcaster": - version "0.1.0" - resolved "https://dropper--sandboxbrdev.myvtex.com/_v/private/typings/linked/v1/vtex.spreadsheet-event-broadcaster@0.1.0+build1644442090/public/@types/vtex.spreadsheet-event-broadcaster#2d3e6dbe41ed44968e5c4ab23d04767f429f697f" +"vtex.spreadsheet-event-broadcaster@http://vtex.vtexassets.com/_v/public/typings/v1/vtex.spreadsheet-event-broadcaster@0.1.0/public/_types/react": + version "0.0.0" + resolved "http://vtex.vtexassets.com/_v/public/typings/v1/vtex.spreadsheet-event-broadcaster@0.1.0/public/_types/react#fa7a0347e046eab3dd768998fc9252b2c0dd5aef" w3c-hr-time@^1.0.1: version "1.0.1" From 381e0ef38ddacb5e8e09cd483bc2c3115a0bc1ab Mon Sep 17 00:00:00 2001 From: GabrielEluan Date: Fri, 11 Feb 2022 12:00:17 -0300 Subject: [PATCH 6/6] Add senderAppId param to broadcaster --- node/index.ts | 4 +++- node/middlewares/broadcaster/broadcast.ts | 3 ++- node/middlewares/broadcaster/splitPayload.ts | 3 ++- node/middlewares/notify/parseFile.ts | 9 ++++++--- node/middlewares/notify/startEventChain.ts | 3 ++- node/utils/parsing.ts | 6 ++++++ 6 files changed, 21 insertions(+), 7 deletions(-) diff --git a/node/index.ts b/node/index.ts index 77485df..cc598f8 100644 --- a/node/index.ts +++ b/node/index.ts @@ -30,13 +30,15 @@ declare global { interface State extends RecorderState { payload: unknown[] - appId: string + senderAppId: string + clientAppId: string } interface BroadcasterEventContext extends EventContext { body: { eventId: string payload: unknown[] + senderAppId: string clientAppId: string } } diff --git a/node/middlewares/broadcaster/broadcast.ts b/node/middlewares/broadcaster/broadcast.ts index 795698a..6755da3 100644 --- a/node/middlewares/broadcaster/broadcast.ts +++ b/node/middlewares/broadcaster/broadcast.ts @@ -6,12 +6,13 @@ export async function broadcast( ) { const { clients: { events }, - body: { payload, clientAppId }, + body: { payload, senderAppId, clientAppId }, } = ctx payload.forEach((row) => { events.sendEvent(clientAppId, eventKey, { data: row, + senderAppId, }) }) diff --git a/node/middlewares/broadcaster/splitPayload.ts b/node/middlewares/broadcaster/splitPayload.ts index b14aa89..dd3c231 100644 --- a/node/middlewares/broadcaster/splitPayload.ts +++ b/node/middlewares/broadcaster/splitPayload.ts @@ -13,7 +13,7 @@ export async function splitPayload( ) { const { clients: { events }, - body: { payload, clientAppId }, + body: { payload, senderAppId, clientAppId }, } = ctx if (payload.length > ARRAY_SIZE_TARGET) { @@ -27,6 +27,7 @@ export async function splitPayload( events.sendEvent(thisAppId, eventKey, { eventId, payload: chunk, + senderAppId, clientAppId, }) } diff --git a/node/middlewares/notify/parseFile.ts b/node/middlewares/notify/parseFile.ts index 623798b..041b02b 100644 --- a/node/middlewares/notify/parseFile.ts +++ b/node/middlewares/notify/parseFile.ts @@ -6,7 +6,7 @@ import { MAXIMUM_FILE_SIZE, MAXIMUM_FILE_SIZE_STRING, } from '../../utils/constants' -import { asyncBusboyWrapper } from '../../utils/parsing' +import { asyncBusboyWrapper, senderAppIdFromHeaders } from '../../utils/parsing' export async function parseFile(ctx: Context, next: () => Promise) { const { @@ -14,8 +14,10 @@ export async function parseFile(ctx: Context, next: () => Promise) { vtex: { logger }, } = ctx + const senderAppId = senderAppIdFromHeaders(req.headers) + const { - fields: { appId }, + fields: { appId: clientAppId }, files: [file], } = await asyncBusboyWrapper(req) @@ -55,7 +57,8 @@ export async function parseFile(ctx: Context, next: () => Promise) { const payload = utils.sheet_to_json(sheet) ctx.state.payload = payload - ctx.state.appId = appId ?? '' + ctx.state.senderAppId = senderAppId + ctx.state.clientAppId = clientAppId ?? '' } catch (error) { logger.error(error.message) diff --git a/node/middlewares/notify/startEventChain.ts b/node/middlewares/notify/startEventChain.ts index 01236f7..091dce4 100644 --- a/node/middlewares/notify/startEventChain.ts +++ b/node/middlewares/notify/startEventChain.ts @@ -7,7 +7,7 @@ import { export async function startEventChain(ctx: Context) { const { - state: { payload, appId: clientAppId }, + state: { payload, senderAppId, clientAppId }, clients: { events }, } = ctx @@ -16,6 +16,7 @@ export async function startEventChain(ctx: Context) { events.sendEvent(thisAppId, eventKey, { eventId, payload, + senderAppId, clientAppId, }) diff --git a/node/utils/parsing.ts b/node/utils/parsing.ts index ea1233f..3628c1b 100644 --- a/node/utils/parsing.ts +++ b/node/utils/parsing.ts @@ -7,3 +7,9 @@ export async function asyncBusboyWrapper(req: IncomingMessage): Promise { return (result as unknown) as T } + +export function senderAppIdFromHeaders(headers: IncomingMessage['headers']) { + const appIdWithVersion = headers['x-vtex-caller'] as string + + return appIdWithVersion.split('@')[0] +}