From c0fca2e5a355ba1da97b5127d4ef6508e4c43690 Mon Sep 17 00:00:00 2001 From: Steven Date: Wed, 22 Apr 2020 16:16:49 +0100 Subject: [PATCH 1/4] Add message transformer for Pay payment events Flat message objects from the S3 stream will be formatted into GOV.UK Pay payment events that can be consumed by Ledger. This is the first functionality that will apply custom transforms on top the upstream fork. --- .../GovUKPayPaymentEventMessage.test.ts | 40 ++++++++++++++ .../GovUKPayPaymentEventMessage.ts | 55 +++++++++++++++++++ 2 files changed, 95 insertions(+) create mode 100644 src/transformers/GovUKPayPaymentEventMessage.test.ts create mode 100644 src/transformers/GovUKPayPaymentEventMessage.ts diff --git a/src/transformers/GovUKPayPaymentEventMessage.test.ts b/src/transformers/GovUKPayPaymentEventMessage.test.ts new file mode 100644 index 0000000..cde7dfa --- /dev/null +++ b/src/transformers/GovUKPayPaymentEventMessage.test.ts @@ -0,0 +1,40 @@ +import GovUKPayPaymentEventMessage from './GovUKPayPaymentEventMessage' +describe('message formatter', () => { + const messageFromS3Csv = { + 'transaction_id': 'some-transaction-id', + 'event_date': 'some-event-date', + 'parent_transaction_id': 'some-parent-transaction-id', + 'event_type': 'some-event-type', + 'resource_type': 'some-resource-type', + 'reference': 'some-reference', + 'amount': 'some-amount' + } + const messageBuilder = new GovUKPayPaymentEventMessage() + + test('correctly transforms known reserved columns', () => { + const formatted = messageBuilder.transform(messageFromS3Csv) + const body = JSON.parse(formatted.MessageBody) + + expect(body).toHaveProperty('resource_external_id') + expect(body).toHaveProperty('parent_resource_external_id') + expect(body).toHaveProperty('resource_type') + expect(body).toHaveProperty('event_date') + expect(body).toHaveProperty('event_data') + }) + + test('ignores reserved properties if not needed on transaction', () => { + const formatted = messageBuilder.transform({ 'transaction_id': 'some-transaction-id' }) + const body = JSON.parse(formatted.MessageBody) + + expect(body).toHaveProperty('resource_external_id') + expect(body).not.toHaveProperty('parent_resource_external_id') + }) + + test('correctly formats well formatted flat message', () => { + const formatted = messageBuilder.transform(messageFromS3Csv) + const body = JSON.parse(formatted.MessageBody) + + expect(body.event_data.reference).toBe('some-reference') + expect(body.event_data.amount).toBe('some-amount') + }) +}) \ No newline at end of file diff --git a/src/transformers/GovUKPayPaymentEventMessage.ts b/src/transformers/GovUKPayPaymentEventMessage.ts new file mode 100644 index 0000000..81d947b --- /dev/null +++ b/src/transformers/GovUKPayPaymentEventMessage.ts @@ -0,0 +1,55 @@ +/* eslint-disable @typescript-eslint/camelcase */ +import crypto from 'crypto' +import SQS = require("aws-sdk/clients/sqs") +import Transformer from './Transformer' +import { Message } from './../providers/Provider' + +interface PaymentEventMessage { + event_data: { [key: string]: string }; + resource_type?: string; + resource_external_id?: string; + event_date?: string; + event_type?: string; + parent_resource_external_id?: string; + [key: string]: string | { [key: string]: string }; +} + +// we can gaurantee the existence of required fields as anything with permissions +// to this resource will be validating data entry +function formatPaymentEventMessage(message: Message): PaymentEventMessage { + const reservedKeys = [ + { key: 'transaction_id', target: 'resource_external_id' }, + { key: 'parent_transaction_id', target: 'parent_resource_external_id' }, + { key: 'resource_type', target: 'resource_type' }, + { key: 'event_date', target: 'event_date' }, + { key: 'event_type', target: 'event_type' } + ] + const formatted: PaymentEventMessage = { event_data: {} } + + // initially extract the reserved properties + for (const reserved of reservedKeys) { + const reservedEntry = message[reserved.key] + if (reservedEntry) { + formatted[reserved.target] = reservedEntry + delete message[reserved.key] + } + } + + // any remaining properties will override attributes of the transaction itself + // put these in `event_data` + for (const paymentEventMessageKey in message) { + formatted.event_data[paymentEventMessageKey] = message[paymentEventMessageKey] + } + return formatted +} + +export default class GovUkPayPaymentEventMessage implements Transformer { + transform(message: Message): SQS.SendMessageBatchRequestEntry { + return { + Id: crypto.randomBytes(16).toString('hex'), + MessageBody: JSON.stringify( + formatPaymentEventMessage(message) + ) + } + } +} \ No newline at end of file From e145346c5bf2073cd645129e7751352323d3ac40 Mon Sep 17 00:00:00 2001 From: Steven Date: Wed, 22 Apr 2020 16:19:19 +0100 Subject: [PATCH 2/4] Use GovUKPayPaymentEventMessage transformer GOV.UK Pay will use a custom message transform for this use case, code kept up to date with fork. --- src/process.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/process.ts b/src/process.ts index 8c548cb..73d30e2 100644 --- a/src/process.ts +++ b/src/process.ts @@ -1,9 +1,10 @@ import S3Provider from './providers/S3Provider' import SQSConsumer from './consumers/SQSConsumer' +import GovUkPayPaymentEventMessage from './transformers/GovUKPayPaymentEventMessage' const { PROVIDER_S3_BUCKET_NAME, PROVIDER_S3_SOURCE_FILE, CONSUMER_SQS_QUEUE_URL } = process.env const s3Provider = new S3Provider(PROVIDER_S3_BUCKET_NAME, PROVIDER_S3_SOURCE_FILE) -const sqs = new SQSConsumer(CONSUMER_SQS_QUEUE_URL).writer() +const sqs = new SQSConsumer(CONSUMER_SQS_QUEUE_URL, new GovUkPayPaymentEventMessage()).writer() s3Provider .stream() From 809c5555f8a72afd0a9ba496beacd630eb270460 Mon Sep 17 00:00:00 2001 From: Steven Date: Wed, 22 Apr 2020 17:13:33 +0100 Subject: [PATCH 3/4] Fix `event_data` -> expected `event_details` --- src/transformers/GovUKPayPaymentEventMessage.test.ts | 8 ++++---- src/transformers/GovUKPayPaymentEventMessage.ts | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/transformers/GovUKPayPaymentEventMessage.test.ts b/src/transformers/GovUKPayPaymentEventMessage.test.ts index cde7dfa..10873cc 100644 --- a/src/transformers/GovUKPayPaymentEventMessage.test.ts +++ b/src/transformers/GovUKPayPaymentEventMessage.test.ts @@ -18,8 +18,8 @@ describe('message formatter', () => { expect(body).toHaveProperty('resource_external_id') expect(body).toHaveProperty('parent_resource_external_id') expect(body).toHaveProperty('resource_type') - expect(body).toHaveProperty('event_date') - expect(body).toHaveProperty('event_data') + expect(body).toHaveProperty('timestamp') + expect(body).toHaveProperty('event_details') }) test('ignores reserved properties if not needed on transaction', () => { @@ -34,7 +34,7 @@ describe('message formatter', () => { const formatted = messageBuilder.transform(messageFromS3Csv) const body = JSON.parse(formatted.MessageBody) - expect(body.event_data.reference).toBe('some-reference') - expect(body.event_data.amount).toBe('some-amount') + expect(body.event_details.reference).toBe('some-reference') + expect(body.event_details.amount).toBe('some-amount') }) }) \ No newline at end of file diff --git a/src/transformers/GovUKPayPaymentEventMessage.ts b/src/transformers/GovUKPayPaymentEventMessage.ts index 81d947b..754af05 100644 --- a/src/transformers/GovUKPayPaymentEventMessage.ts +++ b/src/transformers/GovUKPayPaymentEventMessage.ts @@ -5,7 +5,7 @@ import Transformer from './Transformer' import { Message } from './../providers/Provider' interface PaymentEventMessage { - event_data: { [key: string]: string }; + event_details: { [key: string]: string }; resource_type?: string; resource_external_id?: string; event_date?: string; @@ -21,10 +21,10 @@ function formatPaymentEventMessage(message: Message): PaymentEventMessage { { key: 'transaction_id', target: 'resource_external_id' }, { key: 'parent_transaction_id', target: 'parent_resource_external_id' }, { key: 'resource_type', target: 'resource_type' }, - { key: 'event_date', target: 'event_date' }, + { key: 'event_date', target: 'timestamp' }, { key: 'event_type', target: 'event_type' } ] - const formatted: PaymentEventMessage = { event_data: {} } + const formatted: PaymentEventMessage = { event_details: {} } // initially extract the reserved properties for (const reserved of reservedKeys) { @@ -38,7 +38,7 @@ function formatPaymentEventMessage(message: Message): PaymentEventMessage { // any remaining properties will override attributes of the transaction itself // put these in `event_data` for (const paymentEventMessageKey in message) { - formatted.event_data[paymentEventMessageKey] = message[paymentEventMessageKey] + formatted.event_details[paymentEventMessageKey] = message[paymentEventMessageKey] } return formatted } From 313b8ecd406e24d35dbd1031698cc8f254867332 Mon Sep 17 00:00:00 2001 From: Steven Date: Thu, 23 Apr 2020 12:39:13 +0100 Subject: [PATCH 4/4] Map expected values to more meaninful names --- src/transformers/GovUKPayPaymentEventMessage.test.ts | 5 +++-- src/transformers/GovUKPayPaymentEventMessage.ts | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/transformers/GovUKPayPaymentEventMessage.test.ts b/src/transformers/GovUKPayPaymentEventMessage.test.ts index 10873cc..a0c5f5e 100644 --- a/src/transformers/GovUKPayPaymentEventMessage.test.ts +++ b/src/transformers/GovUKPayPaymentEventMessage.test.ts @@ -4,8 +4,8 @@ describe('message formatter', () => { 'transaction_id': 'some-transaction-id', 'event_date': 'some-event-date', 'parent_transaction_id': 'some-parent-transaction-id', - 'event_type': 'some-event-type', - 'resource_type': 'some-resource-type', + 'event_name': 'some-event-type', + 'transaction_type': 'some-resource-type', 'reference': 'some-reference', 'amount': 'some-amount' } @@ -20,6 +20,7 @@ describe('message formatter', () => { expect(body).toHaveProperty('resource_type') expect(body).toHaveProperty('timestamp') expect(body).toHaveProperty('event_details') + expect(body).toHaveProperty('event_type') }) test('ignores reserved properties if not needed on transaction', () => { diff --git a/src/transformers/GovUKPayPaymentEventMessage.ts b/src/transformers/GovUKPayPaymentEventMessage.ts index 754af05..7c5acfc 100644 --- a/src/transformers/GovUKPayPaymentEventMessage.ts +++ b/src/transformers/GovUKPayPaymentEventMessage.ts @@ -20,9 +20,9 @@ function formatPaymentEventMessage(message: Message): PaymentEventMessage { const reservedKeys = [ { key: 'transaction_id', target: 'resource_external_id' }, { key: 'parent_transaction_id', target: 'parent_resource_external_id' }, - { key: 'resource_type', target: 'resource_type' }, + { key: 'transaction_type', target: 'resource_type' }, { key: 'event_date', target: 'timestamp' }, - { key: 'event_type', target: 'event_type' } + { key: 'event_name', target: 'event_type' } ] const formatted: PaymentEventMessage = { event_details: {} }