diff --git a/api/paidAction/index.js b/api/paidAction/index.js index ad067a3ca..8feabc306 100644 --- a/api/paidAction/index.js +++ b/api/paidAction/index.js @@ -419,7 +419,7 @@ async function createSNInvoice (actionType, args, context) { } async function createDbInvoice (actionType, args, context) { - const { me, models, tx, cost, optimistic, actionId, invoiceArgs, predecessorId } = context + const { me, models, tx, cost, optimistic, actionId, invoiceArgs, paymentAttempt, predecessorId } = context const { bolt11, wrappedBolt11, preimage, wallet, maxFee } = invoiceArgs const db = tx ?? models @@ -445,6 +445,7 @@ async function createDbInvoice (actionType, args, context) { actionArgs: args, expiresAt, actionId, + paymentAttempt, predecessorId } diff --git a/api/resolvers/notifications.js b/api/resolvers/notifications.js index 26e8c4872..eec24f60a 100644 --- a/api/resolvers/notifications.js +++ b/api/resolvers/notifications.js @@ -5,6 +5,7 @@ import { pushSubscriptionSchema, validateSchema } from '@/lib/validate' import { replyToSubscription } from '@/lib/webPush' import { getSub } from './sub' import { GqlAuthenticationError, GqlInputError } from '@/lib/error' +import { WALLET_MAX_RETRIES } from '@/lib/constants' export default { Query: { @@ -350,6 +351,7 @@ export default { WHERE "Invoice"."userId" = $1 AND "Invoice"."updated_at" < $2 AND "Invoice"."actionState" = 'FAILED' + AND "Invoice"."paymentAttempt" >= ${WALLET_MAX_RETRIES} AND ( "Invoice"."actionType" = 'ITEM_CREATE' OR "Invoice"."actionType" = 'ZAP' OR diff --git a/api/resolvers/paidAction.js b/api/resolvers/paidAction.js index 2b993c1f0..7a5184181 100644 --- a/api/resolvers/paidAction.js +++ b/api/resolvers/paidAction.js @@ -1,5 +1,5 @@ import { retryPaidAction } from '../paidAction' -import { USER_ID } from '@/lib/constants' +import { USER_ID, WALLET_MAX_RETRIES } from '@/lib/constants' function paidActionType (actionType) { switch (actionType) { @@ -67,7 +67,14 @@ export default { throw new Error(`Invoice is not in failed state: ${invoice.actionState}`) } - const result = await retryPaidAction(invoice.actionType, { invoice }, { models, me, lnd }) + // a locked invoice means we want to retry a payment from the beginning + // with all sender and receiver wallets so we need to increment the retry count + const paymentAttempt = invoice.lockedAt ? invoice.paymentAttempt + 1 : invoice.paymentAttempt + if (paymentAttempt > WALLET_MAX_RETRIES) { + throw new Error('Payment has been retried too many times') + } + + const result = await retryPaidAction(invoice.actionType, { invoice }, { paymentAttempt, models, me, lnd }) return { ...result, diff --git a/api/resolvers/wallet.js b/api/resolvers/wallet.js index c71749ac0..eeaa3ebc5 100644 --- a/api/resolvers/wallet.js +++ b/api/resolvers/wallet.js @@ -9,7 +9,10 @@ import { formatMsats, msatsToSats, msatsToSatsDecimal, satsToMsats } from '@/lib import { USER_ID, INVOICE_RETENTION_DAYS, PAID_ACTION_PAYMENT_METHODS, - WALLET_CREATE_INVOICE_TIMEOUT_MS + WALLET_CREATE_INVOICE_TIMEOUT_MS, + WALLET_RETRY_AFTER_MS, + WALLET_RETRY_BEFORE_MS, + WALLET_MAX_RETRIES } from '@/lib/constants' import { amountSchema, validateSchema, withdrawlSchema, lnAddrSchema } from '@/lib/validate' import assertGofacYourself from './ofac' @@ -456,6 +459,29 @@ const resolvers = { cursor: nextCursor, entries: logs } + }, + failedInvoices: async (parent, args, { me, models }) => { + if (!me) { + throw new GqlAuthenticationError() + } + // make sure each invoice is only returned once via visibility timeouts and SKIP LOCKED + // we use queryRawUnsafe because Prisma does not support tsrange + // see https://www.prisma.io/docs/orm/overview/databases/postgresql + return await models.$queryRaw` + UPDATE "Invoice" + SET "lockedAt" = now() + WHERE id IN ( + SELECT id FROM "Invoice" + WHERE "userId" = ${me.id} + AND "actionState" = 'FAILED' + AND "cancelledAt" < now() - ${`${WALLET_RETRY_AFTER_MS} milliseconds`}::interval + AND "cancelledAt" > now() - ${`${WALLET_RETRY_BEFORE_MS} milliseconds`}::interval + AND ("lockedAt" IS NULL OR "lockedAt" < now() - interval '1 minute') + AND "paymentAttempt" < ${WALLET_MAX_RETRIES} + ORDER BY id DESC + FOR UPDATE SKIP LOCKED + ) + RETURNING *` } }, Wallet: { diff --git a/api/typeDefs/wallet.js b/api/typeDefs/wallet.js index 3006fc7a4..efc49130b 100644 --- a/api/typeDefs/wallet.js +++ b/api/typeDefs/wallet.js @@ -72,6 +72,7 @@ const typeDefs = ` wallet(id: ID!): Wallet walletByType(type: String!): Wallet walletLogs(type: String, from: String, to: String, cursor: String): WalletLog! + failedInvoices: [Invoice!]! } extend type Mutation { diff --git a/components/use-invoice.js b/components/use-invoice.js index 1cbe94dfd..534426358 100644 --- a/components/use-invoice.js +++ b/components/use-invoice.js @@ -1,5 +1,5 @@ import { useApolloClient, useMutation } from '@apollo/client' -import { useCallback } from 'react' +import { useCallback, useMemo } from 'react' import { InvoiceCanceledError, InvoiceExpiredError, WalletReceiverError } from '@/wallets/errors' import { RETRY_PAID_ACTION } from '@/fragments/paidAction' import { INVOICE, CANCEL_INVOICE } from '@/fragments/wallet' @@ -42,7 +42,7 @@ export default function useInvoice () { return data.cancelInvoice }, [cancelInvoice]) - const retry = useCallback(async ({ id, hash, hmac }, { update }) => { + const retry = useCallback(async ({ id, hash, hmac }, { update } = {}) => { console.log('retrying invoice:', hash) const { data, error } = await retryPaidAction({ variables: { invoiceId: Number(id) }, update }) if (error) throw error @@ -53,5 +53,5 @@ export default function useInvoice () { return newInvoice }, [retryPaidAction]) - return { cancel, retry, isInvoice } + return useMemo(() => ({ cancel, retry, isInvoice }), [cancel, retry, isInvoice]) } diff --git a/fragments/wallet.js b/fragments/wallet.js index f75d6547e..b4d34d30a 100644 --- a/fragments/wallet.js +++ b/fragments/wallet.js @@ -231,3 +231,12 @@ export const CANCEL_INVOICE = gql` } } ` + +export const FAILED_INVOICES = gql` + ${INVOICE_FIELDS} + query FailedInvoices { + failedInvoices { + ...InvoiceFields + } + } +` diff --git a/lib/apollo.js b/lib/apollo.js index 6e2eeab1a..96fb0737b 100644 --- a/lib/apollo.js +++ b/lib/apollo.js @@ -272,6 +272,12 @@ function getClient (uri) { facts: [...(existing?.facts || []), ...incoming.facts] } } + }, + failedInvoices: { + keyArgs: [], + merge (existing, incoming) { + return incoming + } } } }, diff --git a/lib/constants.js b/lib/constants.js index 3372d9942..43d892acd 100644 --- a/lib/constants.js +++ b/lib/constants.js @@ -194,3 +194,10 @@ export const ZAP_UNDO_DELAY_MS = 5_000 export const WALLET_SEND_PAYMENT_TIMEOUT_MS = 150_000 export const WALLET_CREATE_INVOICE_TIMEOUT_MS = 45_000 +// When should failed invoices be returned to a client to retry? +// This must be high enough such that intermediate failed invoices that will be retried +// by the client due to sender or receiver fallbacks are not returned to the client. +export const WALLET_RETRY_AFTER_MS = 60_000 +export const WALLET_RETRY_BEFORE_MS = 86_400_000 // 24 hours +// we want to attempt a payment three times so we retry two times +export const WALLET_MAX_RETRIES = 2 diff --git a/prisma/migrations/20250107084543_automated_retries/migration.sql b/prisma/migrations/20250107084543_automated_retries/migration.sql new file mode 100644 index 000000000..17770e9e3 --- /dev/null +++ b/prisma/migrations/20250107084543_automated_retries/migration.sql @@ -0,0 +1,4 @@ +-- AlterTable +ALTER TABLE "Invoice" ADD COLUMN "paymentAttempt" INTEGER NOT NULL DEFAULT 0; +ALTER TABLE "Invoice" ADD COLUMN "lockedAt" TIMESTAMP(3); +CREATE INDEX "Invoice_cancelledAt_idx" ON "Invoice"("cancelledAt"); \ No newline at end of file diff --git a/prisma/schema.prisma b/prisma/schema.prisma index dcdb3b938..f1a189c1e 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -924,6 +924,8 @@ model Invoice { cancelled Boolean @default(false) cancelledAt DateTime? userCancel Boolean? + lockedAt DateTime? + paymentAttempt Int @default(0) msatsRequested BigInt msatsReceived BigInt? desc String? @@ -952,6 +954,7 @@ model Invoice { @@index([confirmedIndex], map: "Invoice.confirmedIndex_index") @@index([isHeld]) @@index([confirmedAt]) + @@index([cancelledAt]) @@index([actionType]) @@index([actionState]) } diff --git a/wallets/index.js b/wallets/index.js index bdf00083b..83c6c13a0 100644 --- a/wallets/index.js +++ b/wallets/index.js @@ -1,12 +1,14 @@ import { useMe } from '@/components/me' -import { SET_WALLET_PRIORITY, WALLETS } from '@/fragments/wallet' -import { SSR } from '@/lib/constants' +import { FAILED_INVOICES, SET_WALLET_PRIORITY, WALLETS } from '@/fragments/wallet' +import { NORMAL_POLL_INTERVAL, SSR } from '@/lib/constants' import { useApolloClient, useMutation, useQuery } from '@apollo/client' import { createContext, useCallback, useContext, useEffect, useMemo, useState } from 'react' import { getStorageKey, getWalletByType, walletPrioritySort, canSend, isConfigured, upsertWalletVariables, siftConfig, saveWalletLocally } from './common' import useVault from '@/components/vault/use-vault' import walletDefs from '@/wallets/client' import { generateMutation } from './graphql' +import { useWalletPayment } from './payment' +import useInvoice from '@/components/use-invoice' const WalletsContext = createContext({ wallets: [] @@ -204,7 +206,9 @@ export function WalletsProvider ({ children }) { removeLocalWallets }} > - {children} + + {children} + ) } @@ -221,7 +225,36 @@ export function useWallet (name) { export function useSendWallets () { const { wallets } = useWallets() // return all enabled wallets that are available and can send - return wallets + return useMemo(() => wallets .filter(w => !w.def.isAvailable || w.def.isAvailable()) - .filter(w => w.config?.enabled && canSend(w)) + .filter(w => w.config?.enabled && canSend(w)), [wallets]) +} + +function RetryHandler ({ children }) { + const failedInvoices = useFailedInvoices() + const waitForWalletPayment = useWalletPayment() + const invoiceHelper = useInvoice() + + useEffect(() => { + (async () => { + for (const invoice of failedInvoices) { + const newInvoice = await invoiceHelper.retry(invoice) + waitForWalletPayment(newInvoice).catch(console.error) + } + })() + }, [failedInvoices, invoiceHelper, waitForWalletPayment]) + + return children +} + +function useFailedInvoices () { + const wallets = useSendWallets() + + const { data } = useQuery(FAILED_INVOICES, { + pollInterval: NORMAL_POLL_INTERVAL, + skip: wallets.length === 0, + notifyOnNetworkStatusChange: true + }) + + return data?.failedInvoices ?? [] } diff --git a/wallets/payment.js b/wallets/payment.js index 043d57f89..7cb0bec64 100644 --- a/wallets/payment.js +++ b/wallets/payment.js @@ -17,7 +17,7 @@ export function useWalletPayment () { const loggerFactory = useWalletLoggerFactory() const invoiceHelper = useInvoice() - return useCallback(async (invoice, { waitFor, updateOnFallback }) => { + return useCallback(async (invoice, { waitFor, updateOnFallback } = {}) => { let aggregateError = new WalletAggregateError([]) let latestInvoice = invoice diff --git a/wallets/server.js b/wallets/server.js index f14e9fb36..bdb6448e2 100644 --- a/wallets/server.js +++ b/wallets/server.js @@ -24,9 +24,13 @@ export default [lnd, cln, lnAddr, lnbits, nwc, phoenixd, blink, lnc, webln] const MAX_PENDING_INVOICES_PER_WALLET = 25 -export async function createInvoice (userId, { msats, description, descriptionHash, expiry = 360 }, { predecessorId, models }) { +export async function createInvoice (userId, { msats, description, descriptionHash, expiry = 360 }, { paymentAttempt, predecessorId, models }) { // get the wallets in order of priority - const wallets = await getInvoiceableWallets(userId, { predecessorId, models }) + const wallets = await getInvoiceableWallets(userId, { + paymentAttempt, + predecessorId, + models + }) msats = toPositiveNumber(msats) @@ -81,7 +85,7 @@ export async function createInvoice (userId, { msats, description, descriptionHa export async function createWrappedInvoice (userId, { msats, feePercent, description, descriptionHash, expiry = 360 }, - { predecessorId, models, me, lnd }) { + { paymentAttempt, predecessorId, models, me, lnd }) { let logger, bolt11 try { const { invoice, wallet } = await createInvoice(userId, { @@ -90,7 +94,7 @@ export async function createWrappedInvoice (userId, description, descriptionHash, expiry - }, { predecessorId, models }) + }, { paymentAttempt, predecessorId, models }) logger = walletLogger({ wallet, models }) bolt11 = invoice @@ -110,7 +114,7 @@ export async function createWrappedInvoice (userId, } } -export async function getInvoiceableWallets (userId, { predecessorId, models }) { +export async function getInvoiceableWallets (userId, { paymentAttempt, predecessorId, models }) { // filter out all wallets that have already been tried by recursively following the retry chain of predecessor invoices. // the current predecessor invoice is in state 'FAILED' and not in state 'RETRYING' because we are currently retrying it // so it has not been updated yet. @@ -143,6 +147,7 @@ export async function getInvoiceableWallets (userId, { predecessorId, models }) FROM "Invoice" JOIN "Retries" ON "Invoice"."id" = "Retries"."predecessorId" WHERE "Invoice"."actionState" = 'RETRYING' + AND "Invoice"."paymentAttempt" = ${paymentAttempt} ) SELECT "InvoiceForward"."walletId"