Campaign sending improvements (#150)

apps/platform/db/migrations/20230504131759_add_user_indexes.js

@@ -0,0 +1,13 @@
+exports.up = async function(knex) {
+    await knex.schema.table('users', function(table) {
+        table.index('email')
+        table.index('phone')
+    })
+}
+
+exports.down = async function(knex) {
+    await knex.schema.table('users', function(table) {
+        table.dropIndex('email')
+        table.dropIndex('phone')
+    })
+}

apps/platform/src/campaigns/CampaignService.ts

@@ -11,7 +11,7 @@ import { RequestError } from '../core/errors'
 import App from '../app'
 
 import { SearchParams } from '../core/searchParams'
-import { allLists, listUserCount } from '../lists/ListService'
+import { allLists } from '../lists/ListService'
 import { allTemplates, duplicateTemplate, validateTemplates } from '../render/TemplateService'
 import { getSubscription } from '../subscriptions/SubscriptionService'
 import { crossTimezoneCopy, pick } from '../utilities'
@@ -67,31 +67,32 @@ export const createCampaign = async (projectId: number, { tags, ...params }: Cam
     }
 
     const delivery = { sent: 0, total: 0, opens: 0, clicks: 0 }
-    if (params.list_ids) {
-        delivery.total = await totalUsersCount(
-            params.list_ids,
-            params.exclusion_list_ids ?? [],
-        )
-    }
-
-    const id = await Campaign.insert({
+    const campaign = await Campaign.insertAndFetch({
         ...params,
         state: 'draft',
         delivery,
         channel: subscription.channel,
         project_id: projectId,
     })
 
+    // Calculate initial users count
+    await Campaign.update(qb => qb.where('id', campaign.id), {
+        delivery: {
+            ...campaign.delivery,
+            total: await initialUsersCount(campaign),
+        },
+    })
+
     if (tags?.length) {
         await setTags({
             project_id: projectId,
             entity: Campaign.tableName,
-            entity_id: id,
+            entity_id: campaign.id,
             names: tags,
         })
     }
 
-    return await getCampaign(id, projectId) as Campaign
+    return await getCampaign(campaign.id, projectId) as Campaign
 }
 
 export const updateCampaign = async (id: number, projectId: number, { tags, ...params }: Partial<CampaignParams>): Promise<Campaign | undefined> => {
@@ -230,19 +231,6 @@ export const generateSendList = async (campaign: SentCampaign) => {
         throw new RequestError('Unable to send to a campaign that does not have an associated list', 404)
     }
 
-    // Update campaign state to show it's processing
-    await Campaign.update(qb => qb.where('id', campaign.id), {
-        delivery: {
-            sent: 0,
-            total: await totalUsersCount(
-                campaign.list_ids,
-                campaign.exclusion_list_ids ?? [],
-            ),
-            opens: 0,
-            clicks: 0,
-        },
-    })
-
     const insertChunk = async (chunk: CampaignSendParams[]) => {
         if (chunk.length <= 0) return
         return await CampaignSend.query()
@@ -298,7 +286,6 @@ export const campaignSendReadyQuery = (campaignId: number) => {
 }
 
 export const recipientQuery = (campaign: Campaign) => {
-
     return UserList.query()
         .select('user_list.user_id', 'users.timezone')
 
@@ -308,6 +295,12 @@ export const recipientQuery = (campaign: Campaign) => {
                 .andOn('user_subscription.subscription_id', '=', UserList.raw(campaign.subscription_id))
         })
 
+        // Join in the exclusion list
+        .leftJoin('user_list as ul2', qb => {
+            qb.on('ul2.user_id', 'user_list.user_id')
+                .onIn('ul2.list_id', campaign.exclusion_list_ids ?? [])
+        })
+
         // Join users to get the timezone field
         .leftJoin('users', 'users.id', 'user_list.user_id')
 
@@ -325,11 +318,21 @@ export const recipientQuery = (campaign: Campaign) => {
         })
 
         // Find all users in provided lists, removing ones in exclusion list
-        .whereIn('list_id', campaign.list_ids ?? [])
-        .whereNotIn('list_id', campaign.exclusion_list_ids ?? [])
+        .whereIn('user_list.list_id', campaign.list_ids ?? [])
+        .whereNull('ul2.list_id')
 
         // Filter out existing sends (handle aborts & reschedules)
         .whereNull('campaign_sends.id')
+
+        // Based on campaign type, filter out based on missing user
+        // criteria
+        .where(qb => {
+            if (campaign.channel === 'email') {
+                qb.whereNotNull('users.email')
+            } else if (campaign.channel === 'text') {
+                qb.whereNotNull('users.phone')
+            }
+        })
 }
 
 export const abortCampaign = async (campaign: Campaign) => {
@@ -350,10 +353,12 @@ export const duplicateCampaign = async (campaign: Campaign) => {
     return await getCampaign(cloneId, campaign.project_id)
 }
 
-const totalUsersCount = async (listIds: number[], exclusionListIds: number[]): Promise<number> => {
-    const totalIncluded = await listUserCount(listIds)
-    const totalExcluded = await listUserCount(exclusionListIds)
-    return Math.max(0, (totalIncluded - totalExcluded))
+const initialUsersCount = async (campaign: Campaign): Promise<number> => {
+    const response = await recipientQuery(campaign)
+        .clearSelect()
+        .select(UserList.raw('COUNT(DISTINCT(users.id)) as count'))
+    const { count } = response[0]
+    return Math.max(0, count)
 }
 
 export const campaignProgress = async (campaign: Campaign): Promise<CampaignProgress> => {

apps/platform/src/campaigns/__tests__/CampaignService.spec.ts

@@ -69,6 +69,7 @@ describe('CampaignService', () => {
         return await User.insertAndFetch({
             project_id,
             external_id: uuid(),
+            email: `${uuid()}@test.com`,
             data: {},
         })
     }

apps/ui/src/views/campaign/CampaignOverview.tsx

@@ -9,7 +9,7 @@ import Modal from '../../ui/Modal'
 import { PreferencesContext } from '../../ui/PreferencesContext'
 import { formatDate } from '../../utils'
 import { CampaignForm } from './CampaignForm'
-import { CampaignTag } from './Campaigns'
+import { CampaignTag, DeliveryRatio } from './Campaigns'
 import ChannelTag from './ChannelTag'
 
 export default function CampaignOverview() {
@@ -56,7 +56,7 @@ export default function CampaignOverview() {
                 in_timezone: campaign.send_in_user_timezone ? 'Yes' : 'No',
                 send_lists: DelimitedLists({ lists: campaign.lists }),
                 exclusion_lists: DelimitedLists({ lists: campaign.exclusion_lists }),
-                delivery: `${campaign.delivery?.sent ?? 0} / ${campaign.delivery?.total ?? 0}`,
+                delivery: DeliveryRatio({ delivery: campaign.delivery }),
             }} />
             <Modal
                 open={isEditOpen}

apps/ui/src/views/campaign/Campaigns.tsx

@@ -1,7 +1,7 @@
 import { useCallback, useState } from 'react'
 import { useNavigate, useParams } from 'react-router'
 import api from '../../api'
-import { Campaign, CampaignState } from '../../types'
+import { Campaign, CampaignDelivery, CampaignState } from '../../types'
 import Button from '../../ui/Button'
 import { ArchiveIcon, DuplicateIcon, EditIcon, PlusIcon } from '../../ui/icons'
 import Menu, { MenuItem } from '../../ui/Menu'
@@ -28,6 +28,12 @@ export const CampaignTag = ({ state }: { state: CampaignState }) => {
     </Tag>
 }
 
+export const DeliveryRatio = ({ delivery }: { delivery: CampaignDelivery }) => {
+    const sent = (delivery?.sent ?? 0).toLocaleString()
+    const total = (delivery?.total ?? 0).toLocaleString()
+    return `${sent} / ${total}`
+}
+
 export default function Campaigns() {
     const { projectId = '' } = useParams()
     const navigate = useNavigate()
@@ -74,7 +80,7 @@ export default function Campaigns() {
                         },
                         {
                             key: 'delivery',
-                            cell: ({ item }) => `${item.delivery?.sent ?? 0} / ${item.delivery?.total ?? 0}`,
+                            cell: ({ item: { delivery } }) => DeliveryRatio({ delivery }),
                         },
                         {
                             key: 'send_at',