Automated retries

api/paidAction/index.js

@@ -419,7 +419,7 @@ async function createSNInvoice (actionType, args, context) {
 }
 
 async function createDbInvoice (actionType, args, context) {
-  const { me, models, tx, cost, optimistic, actionId, invoiceArgs, predecessorId } = context
+  const { me, models, tx, cost, optimistic, actionId, invoiceArgs, paymentAttempt, predecessorId } = context
   const { bolt11, wrappedBolt11, preimage, wallet, maxFee } = invoiceArgs
 
   const db = tx ?? models
@@ -445,6 +445,7 @@ async function createDbInvoice (actionType, args, context) {
     actionArgs: args,
     expiresAt,
     actionId,
+    paymentAttempt,
     predecessorId
   }
 

api/resolvers/notifications.js

@@ -5,6 +5,7 @@ import { pushSubscriptionSchema, validateSchema } from '@/lib/validate'
 import { replyToSubscription } from '@/lib/webPush'
 import { getSub } from './sub'
 import { GqlAuthenticationError, GqlInputError } from '@/lib/error'
+import { WALLET_MAX_RETRIES } from '@/lib/constants'
 
 export default {
   Query: {
@@ -350,6 +351,7 @@ export default {
         WHERE "Invoice"."userId" = $1
         AND "Invoice"."updated_at" < $2
         AND "Invoice"."actionState" = 'FAILED'
+        AND "Invoice"."paymentAttempt" >= ${WALLET_MAX_RETRIES}
         AND (
           "Invoice"."actionType" = 'ITEM_CREATE' OR
           "Invoice"."actionType" = 'ZAP' OR

api/resolvers/paidAction.js

@@ -1,5 +1,5 @@
 import { retryPaidAction } from '../paidAction'
-import { USER_ID } from '@/lib/constants'
+import { USER_ID, WALLET_MAX_RETRIES } from '@/lib/constants'
 
 function paidActionType (actionType) {
   switch (actionType) {
@@ -67,7 +67,14 @@ export default {
         throw new Error(`Invoice is not in failed state: ${invoice.actionState}`)
       }
 
-      const result = await retryPaidAction(invoice.actionType, { invoice }, { models, me, lnd })
+      // a locked invoice means we want to retry a payment from the beginning
+      // with all sender and receiver wallets so we need to increment the retry count
+      const paymentAttempt = invoice.lockedAt ? invoice.paymentAttempt + 1 : invoice.paymentAttempt
+      if (paymentAttempt > WALLET_MAX_RETRIES) {
+        throw new Error('Payment has been retried too many times')
+      }
+
+      const result = await retryPaidAction(invoice.actionType, { invoice }, { paymentAttempt, models, me, lnd })
 
       return {
         ...result,

api/resolvers/wallet.js

@@ -9,7 +9,10 @@ import { formatMsats, msatsToSats, msatsToSatsDecimal, satsToMsats } from '@/lib
 import {
   USER_ID, INVOICE_RETENTION_DAYS,
   PAID_ACTION_PAYMENT_METHODS,
-  WALLET_CREATE_INVOICE_TIMEOUT_MS
+  WALLET_CREATE_INVOICE_TIMEOUT_MS,
+  WALLET_RETRY_AFTER_MS,
+  WALLET_RETRY_BEFORE_MS,
+  WALLET_MAX_RETRIES
 } from '@/lib/constants'
 import { amountSchema, validateSchema, withdrawlSchema, lnAddrSchema } from '@/lib/validate'
 import assertGofacYourself from './ofac'
@@ -456,6 +459,29 @@ const resolvers = {
         cursor: nextCursor,
         entries: logs
       }
+    },
+    failedInvoices: async (parent, args, { me, models }) => {
+      if (!me) {
+        throw new GqlAuthenticationError()
+      }
+      // make sure each invoice is only returned once via visibility timeouts and SKIP LOCKED
+      // we use queryRawUnsafe because Prisma does not support tsrange
+      // see https://www.prisma.io/docs/orm/overview/databases/postgresql
+      return await models.$queryRaw`
+        UPDATE "Invoice"
+        SET "lockedAt" = now()
+        WHERE id IN (
+          SELECT id FROM "Invoice"
+          WHERE "userId" = ${me.id}
+          AND "actionState" = 'FAILED'
+          AND "cancelledAt" < now() - ${`${WALLET_RETRY_AFTER_MS} milliseconds`}::interval
+          AND "cancelledAt" > now() - ${`${WALLET_RETRY_BEFORE_MS} milliseconds`}::interval
+          AND ("lockedAt" IS NULL OR "lockedAt" < now() - interval '1 minute')
+          AND "paymentAttempt" < ${WALLET_MAX_RETRIES}
+          ORDER BY id DESC
+          FOR UPDATE SKIP LOCKED
+        )
+        RETURNING *`
     }
   },
   Wallet: {

api/typeDefs/wallet.js

@@ -72,6 +72,7 @@ const typeDefs = `
     wallet(id: ID!): Wallet
     walletByType(type: String!): Wallet
     walletLogs(type: String, from: String, to: String, cursor: String): WalletLog!
+    failedInvoices: [Invoice!]!
   }
 
   extend type Mutation {

components/use-invoice.js

@@ -1,5 +1,5 @@
 import { useApolloClient, useMutation } from '@apollo/client'
-import { useCallback } from 'react'
+import { useCallback, useMemo } from 'react'
 import { InvoiceCanceledError, InvoiceExpiredError, WalletReceiverError } from '@/wallets/errors'
 import { RETRY_PAID_ACTION } from '@/fragments/paidAction'
 import { INVOICE, CANCEL_INVOICE } from '@/fragments/wallet'
@@ -42,7 +42,7 @@ export default function useInvoice () {
     return data.cancelInvoice
   }, [cancelInvoice])
 
-  const retry = useCallback(async ({ id, hash, hmac }, { update }) => {
+  const retry = useCallback(async ({ id, hash, hmac }, { update } = {}) => {
     console.log('retrying invoice:', hash)
     const { data, error } = await retryPaidAction({ variables: { invoiceId: Number(id) }, update })
     if (error) throw error
@@ -53,5 +53,5 @@ export default function useInvoice () {
     return newInvoice
   }, [retryPaidAction])
 
-  return { cancel, retry, isInvoice }
+  return useMemo(() => ({ cancel, retry, isInvoice }), [cancel, retry, isInvoice])
 }

fragments/wallet.js

@@ -231,3 +231,12 @@ export const CANCEL_INVOICE = gql`
     }
   }
 `
+
+export const FAILED_INVOICES = gql`
+  ${INVOICE_FIELDS}
+  query FailedInvoices {
+    failedInvoices {
+      ...InvoiceFields
+    }
+  }
+`

lib/apollo.js

@@ -272,6 +272,12 @@ function getClient (uri) {
                   facts: [...(existing?.facts || []), ...incoming.facts]
                 }
               }
+            },
+            failedInvoices: {
+              keyArgs: [],
+              merge (existing, incoming) {
+                return incoming
+              }
             }
           }
         },

lib/constants.js

@@ -194,3 +194,10 @@ export const ZAP_UNDO_DELAY_MS = 5_000
 
 export const WALLET_SEND_PAYMENT_TIMEOUT_MS = 150_000
 export const WALLET_CREATE_INVOICE_TIMEOUT_MS = 45_000
+// When should failed invoices be returned to a client to retry?
+// This must be high enough such that intermediate failed invoices that will be retried
+// by the client due to sender or receiver fallbacks are not returned to the client.
+export const WALLET_RETRY_AFTER_MS = 60_000
+export const WALLET_RETRY_BEFORE_MS = 86_400_000 // 24 hours
+// we want to attempt a payment three times so we retry two times
+export const WALLET_MAX_RETRIES = 2

prisma/migrations/20250107084543_automated_retries/migration.sql

@@ -0,0 +1,4 @@
+-- AlterTable
+ALTER TABLE "Invoice" ADD COLUMN     "paymentAttempt" INTEGER NOT NULL DEFAULT 0;
+ALTER TABLE "Invoice" ADD COLUMN     "lockedAt" TIMESTAMP(3);
+CREATE INDEX "Invoice_cancelledAt_idx" ON "Invoice"("cancelledAt");

prisma/schema.prisma

@@ -924,6 +924,8 @@ model Invoice {
   cancelled          Boolean             @default(false)
   cancelledAt        DateTime?
   userCancel         Boolean?
+  lockedAt           DateTime?
+  paymentAttempt     Int                 @default(0)
   msatsRequested     BigInt
   msatsReceived      BigInt?
   desc               String?
@@ -952,6 +954,7 @@ model Invoice {
   @@index([confirmedIndex], map: "Invoice.confirmedIndex_index")
   @@index([isHeld])
   @@index([confirmedAt])
+  @@index([cancelledAt])
   @@index([actionType])
   @@index([actionState])
 }

wallets/index.js

@@ -1,12 +1,14 @@
 import { useMe } from '@/components/me'
-import { SET_WALLET_PRIORITY, WALLETS } from '@/fragments/wallet'
-import { SSR } from '@/lib/constants'
+import { FAILED_INVOICES, SET_WALLET_PRIORITY, WALLETS } from '@/fragments/wallet'
+import { NORMAL_POLL_INTERVAL, SSR } from '@/lib/constants'
 import { useApolloClient, useMutation, useQuery } from '@apollo/client'
 import { createContext, useCallback, useContext, useEffect, useMemo, useState } from 'react'
 import { getStorageKey, getWalletByType, walletPrioritySort, canSend, isConfigured, upsertWalletVariables, siftConfig, saveWalletLocally } from './common'
 import useVault from '@/components/vault/use-vault'
 import walletDefs from '@/wallets/client'
 import { generateMutation } from './graphql'
+import { useWalletPayment } from './payment'
+import useInvoice from '@/components/use-invoice'
 
 const WalletsContext = createContext({
   wallets: []
@@ -204,7 +206,9 @@ export function WalletsProvider ({ children }) {
         removeLocalWallets
       }}
     >
-      {children}
+      <RetryHandler>
+        {children}
+      </RetryHandler>
     </WalletsContext.Provider>
   )
 }
@@ -221,7 +225,36 @@ export function useWallet (name) {
 export function useSendWallets () {
   const { wallets } = useWallets()
   // return all enabled wallets that are available and can send
-  return wallets
+  return useMemo(() => wallets
     .filter(w => !w.def.isAvailable || w.def.isAvailable())
-    .filter(w => w.config?.enabled && canSend(w))
+    .filter(w => w.config?.enabled && canSend(w)), [wallets])
+}
+
+function RetryHandler ({ children }) {
+  const failedInvoices = useFailedInvoices()
+  const waitForWalletPayment = useWalletPayment()
+  const invoiceHelper = useInvoice()
+
+  useEffect(() => {
+    (async () => {
+      for (const invoice of failedInvoices) {
+        const newInvoice = await invoiceHelper.retry(invoice)
+        waitForWalletPayment(newInvoice).catch(console.error)
+      }
+    })()
+  }, [failedInvoices, invoiceHelper, waitForWalletPayment])
+
+  return children
+}
+
+function useFailedInvoices () {
+  const wallets = useSendWallets()
+
+  const { data } = useQuery(FAILED_INVOICES, {
+    pollInterval: NORMAL_POLL_INTERVAL,
+    skip: wallets.length === 0,
+    notifyOnNetworkStatusChange: true
+  })
+
+  return data?.failedInvoices ?? []
 }

wallets/payment.js

@@ -17,7 +17,7 @@ export function useWalletPayment () {
   const loggerFactory = useWalletLoggerFactory()
   const invoiceHelper = useInvoice()
 
-  return useCallback(async (invoice, { waitFor, updateOnFallback }) => {
+  return useCallback(async (invoice, { waitFor, updateOnFallback } = {}) => {
     let aggregateError = new WalletAggregateError([])
     let latestInvoice = invoice
 

wallets/server.js

@@ -24,9 +24,13 @@ export default [lnd, cln, lnAddr, lnbits, nwc, phoenixd, blink, lnc, webln]
 
 const MAX_PENDING_INVOICES_PER_WALLET = 25
 
-export async function createInvoice (userId, { msats, description, descriptionHash, expiry = 360 }, { predecessorId, models }) {
+export async function createInvoice (userId, { msats, description, descriptionHash, expiry = 360 }, { paymentAttempt, predecessorId, models }) {
   // get the wallets in order of priority
-  const wallets = await getInvoiceableWallets(userId, { predecessorId, models })
+  const wallets = await getInvoiceableWallets(userId, {
+    paymentAttempt,
+    predecessorId,
+    models
+  })
 
   msats = toPositiveNumber(msats)
 
@@ -81,7 +85,7 @@ export async function createInvoice (userId, { msats, description, descriptionHa
 
 export async function createWrappedInvoice (userId,
   { msats, feePercent, description, descriptionHash, expiry = 360 },
-  { predecessorId, models, me, lnd }) {
+  { paymentAttempt, predecessorId, models, me, lnd }) {
   let logger, bolt11
   try {
     const { invoice, wallet } = await createInvoice(userId, {
@@ -90,7 +94,7 @@ export async function createWrappedInvoice (userId,
       description,
       descriptionHash,
       expiry
-    }, { predecessorId, models })
+    }, { paymentAttempt, predecessorId, models })
 
     logger = walletLogger({ wallet, models })
     bolt11 = invoice
@@ -110,7 +114,7 @@ export async function createWrappedInvoice (userId,
   }
 }
 
-export async function getInvoiceableWallets (userId, { predecessorId, models }) {
+export async function getInvoiceableWallets (userId, { paymentAttempt, predecessorId, models }) {
   // filter out all wallets that have already been tried by recursively following the retry chain of predecessor invoices.
   // the current predecessor invoice is in state 'FAILED' and not in state 'RETRYING' because we are currently retrying it
   // so it has not been updated yet.
@@ -143,6 +147,7 @@ export async function getInvoiceableWallets (userId, { predecessorId, models })
           FROM "Invoice"
           JOIN "Retries" ON "Invoice"."id" = "Retries"."predecessorId"
           WHERE "Invoice"."actionState" = 'RETRYING'
+          AND "Invoice"."paymentAttempt" = ${paymentAttempt}
         )
         SELECT
           "InvoiceForward"."walletId"