feat: upgrade to response-rejection plugin (#324)

* feat: upgrade to response-rejection plugin

* fix: import

* feat: update test

* chore: cleanup

* feat: change view rejections filter

* feat: update wording

.changeset/empty-crews-attack.md

@@ -0,0 +1,6 @@
+---
+'@flatfile/util-response-rejection': major
+'@flatfile/plugin-webhook-egress': major
+---
+
+Upgrade to the @flatfile/util-response-rejection plugin to support deleting successfully submitted records or adding a status column to indicate successful/rejected records.

plugins/webhook-egress/src/webhook.egress.spec.ts

@@ -6,12 +6,15 @@ import {
   setupSimpleWorkbook,
   setupSpace,
 } from '@flatfile/utils-testing'
+import axios from 'axios'
 import { webhookEgress } from './webhook.egress'
 
 jest.setTimeout(10_000)
+jest.mock('axios')
 
 describe('webhookEgress() e2e', () => {
   const listener = setupListener()
+  const mockedAxiosPost = axios.post as jest.MockedFunction<typeof axios.post>
 
   let spaceId
   let workbookId
@@ -46,8 +49,11 @@ describe('webhookEgress() e2e', () => {
   })
 
   it('returns successful outcome message', async () => {
-    const webhookUrl = process.env.WEBHOOK_SITE_URL
-    listener.use(webhookEgress('workbook:egressTestSuccess', webhookUrl))
+    mockedAxiosPost.mockResolvedValue({
+      status: 200,
+      data: {},
+    })
+    listener.use(webhookEgress('workbook:egressTestSuccess', 'example.com'))
 
     const { data: successfulJob } = await api.jobs.create({
       type: 'workbook',
@@ -61,15 +67,20 @@ describe('webhookEgress() e2e', () => {
 
     const response = await api.jobs.get(successfulJobId)
     expect(response.data.outcome.message).toEqual(
-      `Data was successfully submitted to the provided webhook. Go check it out at ${webhookUrl}.`
+      `Data was successfully submitted to the provided webhook. Go check it out at example.com.`
     )
   })
 
   it('returns failure outcome message', async () => {
+    mockedAxiosPost.mockResolvedValue({
+      status: 400,
+      statusText: 'Bad Request',
+      data: {},
+    })
+
     const logErrorSpy = jest.spyOn(global.console, 'error')
 
-    const invalidWebhookUrl = 'https://webhook.site'
-    listener.use(webhookEgress('workbook:egressTestFailure', invalidWebhookUrl))
+    listener.use(webhookEgress('workbook:egressTestFailure', 'example.com'))
 
     const { data: failedJob } = await api.jobs.create({
       type: 'workbook',
@@ -82,13 +93,90 @@ describe('webhookEgress() e2e', () => {
     await listener.waitFor('job:ready', 1, 'workbook:egressTestFailure')
 
     expect(logErrorSpy).toHaveBeenCalledWith(
-      '[@flatfile/plugin-webhook-egress]:[FATAL] {}'
+      expect.stringContaining(
+        '[@flatfile/plugin-webhook-egress]:[FATAL] Failed to submit data to example.com. Status: 400 Bad Request'
+      )
     )
 
     const response = await api.jobs.get(failedJobId)
     expect(response.data.outcome).toEqual({
       message:
-        "This job failed probably because it couldn't find the webhook URL.",
+        'Data was not successfully submitted to the provided webhook. Status: 400 Bad Request',
+    })
+  })
+
+  describe('webhookEgress() e2e w/ response rejection', () => {
+    it('returns no rejections', async () => {
+      mockedAxiosPost.mockResolvedValue({
+        status: 200,
+        data: {
+          rejections: {
+            deleteSubmitted: true,
+          },
+        },
+      })
+      listener.use(webhookEgress('workbook:egressTestSuccess', 'example.com'))
+
+      const { data: successfulJob } = await api.jobs.create({
+        type: 'workbook',
+        operation: 'egressTestSuccess',
+        source: workbookId,
+      })
+      const successfulJobId = successfulJob.id
+      await api.jobs.execute(successfulJobId)
+
+      await listener.waitFor('job:ready', 1, 'workbook:egressTestSuccess')
+
+      const response = await api.jobs.get(successfulJobId)
+      expect(response.data.outcome.message).toEqual(
+        'The data has been successfully submitted without any rejections. This task is now complete.'
+      )
+      expect(response.data.outcome.heading).toEqual('Success!')
+    })
+
+    it('returns rejections', async () => {
+      mockedAxiosPost.mockResolvedValue({
+        status: 200,
+        data: {
+          rejections: {
+            id: workbookId,
+            deleteSubmitted: true,
+            sheets: [
+              {
+                sheetId: sheetId,
+                rejectedRecords: [
+                  {
+                    id: 'dev_rc_91d271c823d84a378ec0165ddc886864',
+                    values: [
+                      {
+                        field: 'email',
+                        message: 'Not a valid Flatfile email address',
+                      },
+                    ],
+                  },
+                ],
+              },
+            ],
+          },
+        },
+      })
+      listener.use(webhookEgress('workbook:egressTestSuccess', 'example.com'))
+
+      const { data: successfulJob } = await api.jobs.create({
+        type: 'workbook',
+        operation: 'egressTestSuccess',
+        source: workbookId,
+      })
+      const successfulJobId = successfulJob.id
+      await api.jobs.execute(successfulJobId)
+
+      await listener.waitFor('job:ready', 1, 'workbook:egressTestSuccess')
+
+      const response = await api.jobs.get(successfulJobId)
+      expect(response.data.outcome.message).toEqual(
+        'During the data submission process, 1 records were rejected. Please review and correct these records before resubmitting.'
+      )
+      expect(response.data.outcome.heading).toEqual('Rejected Records')
     })
   })
 })

plugins/webhook-egress/src/webhook.egress.ts

@@ -3,7 +3,7 @@ import { FlatfileListener } from '@flatfile/listener'
 import { jobHandler } from '@flatfile/plugin-job-handler'
 import { logError } from '@flatfile/util-common'
 import {
-  ResponseRejection,
+  RejectionResponse,
   responseRejectionHandler,
 } from '@flatfile/util-response-rejection'
 import axios from 'axios'
@@ -12,7 +12,7 @@ export function webhookEgress(job: string, webhookUrl?: string) {
   return function (listener: FlatfileListener) {
     listener.use(
       jobHandler(job, async (event, tick) => {
-        const { workbookId, payload } = event.context
+        const { workbookId } = event.context
         const { data: workbook } = await api.workbooks.get(workbookId)
         const { data: workbookSheets } = await api.sheets.list({ workbookId })
 
@@ -34,7 +34,6 @@ export function webhookEgress(job: string, webhookUrl?: string) {
           const response = await axios.post(
             webhookReceiver,
             {
-              ...payload,
               workbook: {
                 ...workbook,
                 sheets,
@@ -48,22 +47,11 @@ export function webhookEgress(job: string, webhookUrl?: string) {
           )
 
           if (response.status === 200) {
-            const rejections: ResponseRejection = response.data.rejections
+            const rejections: RejectionResponse = response.data.rejections
             if (rejections) {
-              const totalRejectedRecords = await responseRejectionHandler(
-                rejections
-              )
-              return {
-                outcome: {
-                  next: {
-                    type: 'id',
-                    id: rejections.id,
-                    label: 'See rejections',
-                  },
-                  message: `${totalRejectedRecords} record(s) were rejected during data submission. Review the rejection notes, fix, then resubmit.`,
-                },
-              }
+              return await responseRejectionHandler(rejections)
             }
+
             return {
               outcome: {
                 message: `Data was successfully submitted to the provided webhook. Go check it out at ${webhookReceiver}.`,
@@ -85,13 +73,8 @@ export function webhookEgress(job: string, webhookUrl?: string) {
             '@flatfile/plugin-webhook-egress',
             JSON.stringify(error, null, 2)
           )
-
-          return {
-            outcome: {
-              message:
-                "This job failed probably because it couldn't find the webhook URL.",
-            },
-          }
+          // Throw error to fail job
+          throw new Error(`Error posting data to webhook`)
         }
       })
     )

utils/response-rejection/package.json

@@ -27,6 +27,7 @@
   },
   "license": "ISC",
   "dependencies": {
-    "@flatfile/api": "^1.6.0"
+    "@flatfile/api": "^1.6.0",
+    "@flatfile/util-common": "^0.2.3"
   }
 }