From 64591552a6b29c6d66b1bac310924b61ce0c7eb2 Mon Sep 17 00:00:00 2001 From: nikhil kumar Date: Wed, 18 Dec 2024 14:57:04 +0530 Subject: [PATCH 1/3] ft: adding cron jobs to increase cache hit ratio for oracle data --- backend/src/paymaster/index.ts | 29 ++++--- backend/src/plugins/config.ts | 12 ++- backend/src/routes/paymaster-routes.ts | 7 +- backend/src/server.ts | 103 ++++++++++++++++++++++++- backend/src/types/arka-config-dto.ts | 4 + 5 files changed, 135 insertions(+), 20 deletions(-) diff --git a/backend/src/paymaster/index.ts b/backend/src/paymaster/index.ts index ce632a1..58be5b6 100644 --- a/backend/src/paymaster/index.ts +++ b/backend/src/paymaster/index.ts @@ -273,14 +273,16 @@ export class Paymaster { return provider.send('eth_estimateUserOperationGas', [userOp, entryPoint]); } - private async getLatestAnswerAndDecimals( + async getLatestAnswerAndDecimals( provider: providers.JsonRpcProvider, nativeOracleAddress: string, - chainId: number + chainId: number, + useCache = true ) { const cacheKey = `${chainId}-${nativeOracleAddress}`; const cache = this.nativeCurrencyPrice.get(cacheKey); - if(cache && cache.expiry > Date.now()) { + + if(useCache && cache && cache.expiry > Date.now()) { return { latestAnswer: cache.data, decimals: NativeOracleDecimals @@ -336,15 +338,16 @@ export class Paymaster { }) } - private async getPriceFromOrochi( + async getPriceFromOrochi( oracleAddress: string, provider: providers.JsonRpcProvider, gasToken: string, - chainId: number + chainId: number, + useCache = true ) { const cacheKey = `${chainId}-${oracleAddress}-${gasToken}`; const cache = this.priceAndMetadata.get(cacheKey); - if(cache && cache.expiry > Date.now()) { + if(useCache && cache && cache.expiry > Date.now()) { return cache.data; } @@ -385,17 +388,18 @@ export class Paymaster { }); } - private async getPriceFromChainlink( + async getPriceFromChainlink( oracleAddress: string, provider: providers.JsonRpcProvider, gasToken: string, ethUsdPrice: any, ethUsdPriceDecimal: any, - chainId: number + chainId: number, + useCache = true ) { const cacheKey = `${chainId}-${oracleAddress}-${gasToken}`; const cache = this.priceAndMetadata.get(cacheKey); - if(cache && cache.expiry > Date.now()) { + if(useCache && cache && cache.expiry > Date.now()) { return cache.data; } @@ -444,15 +448,16 @@ export class Paymaster { }); } - private async getPriceFromEtherspotChainlink( + async getPriceFromEtherspotChainlink( oracleAddress: string, provider: providers.JsonRpcProvider, gasToken: string, - chainId: number + chainId: number, + useCache = true ) { const cacheKey = `${chainId}-${oracleAddress}-${gasToken}`; const cache = this.priceAndMetadata.get(cacheKey); - if(cache && cache.expiry > Date.now()) { + if(useCache && cache && cache.expiry > Date.now()) { return cache.data; } const ecContract = new ethers.Contract(oracleAddress, EtherspotChainlinkOracleAbi, provider); diff --git a/backend/src/plugins/config.ts b/backend/src/plugins/config.ts index c0a7d87..196f053 100644 --- a/backend/src/plugins/config.ts +++ b/backend/src/plugins/config.ts @@ -32,7 +32,9 @@ const ConfigSchema = Type.Strict( DELETE_KEY_RECOVER_WINDOW: Type.Number(), KMS_KEY_ID: Type.String() || undefined, USE_KMS: Type.Boolean() || false, - DEFAULT_BUNDLER_API_KEY: Type.String() + DEFAULT_BUNDLER_API_KEY: Type.String(), + MULTI_TOKEN_PAYMASTERS: Type.String(), + MULTI_TOKEN_ORACLES: Type.String(), }) ); @@ -69,7 +71,9 @@ const configPlugin: FastifyPluginAsync = async (server) => { DELETE_KEY_RECOVER_WINDOW: process.env.DELETE_KEY_RECOVER_WINDOW, KMS_KEY_ID: process.env.KMS_KEY_ID, USE_KMS: process.env.USE_KMS, - DEFAULT_BUNDLER_API_KEY: process.env.DEFAULT_BUNDLER_API_KEY + DEFAULT_BUNDLER_API_KEY: process.env.DEFAULT_BUNDLER_API_KEY, + MULTI_TOKEN_PAYMASTERS: process.env.MULTI_TOKEN_PAYMASTERS, + MULTI_TOKEN_ORACLES: process.env.MULTI_TOKEN_ORACLES } const valid = validate(envVar); @@ -102,7 +106,9 @@ const configPlugin: FastifyPluginAsync = async (server) => { DELETE_KEY_RECOVER_WINDOW: parseInt(process.env.DELETE_KEY_RECOVER_WINDOW || '7'), KMS_KEY_ID: process.env.KMS_KEY_ID ?? '', USE_KMS: process.env.USE_KMS === 'true', - DEFAULT_BUNDLER_API_KEY: process.env.DEFAULT_BUNDLER_API_KEY ?? '' + DEFAULT_BUNDLER_API_KEY: process.env.DEFAULT_BUNDLER_API_KEY ?? '', + MULTI_TOKEN_PAYMASTERS: process.env.MULTI_TOKEN_PAYMASTERS ?? '', + MULTI_TOKEN_ORACLES: process.env.MULTI_TOKEN_ORACLES ?? '', } server.log.info(config, "config:"); diff --git a/backend/src/routes/paymaster-routes.ts b/backend/src/routes/paymaster-routes.ts index a03bf6a..a842edc 100644 --- a/backend/src/routes/paymaster-routes.ts +++ b/backend/src/routes/paymaster-routes.ts @@ -3,7 +3,6 @@ import { FastifyPluginAsync } from "fastify"; import { BigNumber, Wallet, ethers, providers } from "ethers"; import { gql, request as GLRequest } from "graphql-request"; import { GetSecretValueCommand, SecretsManagerClient } from "@aws-sdk/client-secrets-manager"; -import { Paymaster } from "../paymaster/index.js"; import SupportedNetworks from "../../config.json" assert { type: "json" }; import { PAYMASTER_ADDRESS } from "../constants/Pimlico.js"; import ErrorMessage, { generateErrorMessage } from "../constants/ErrorMessage.js"; @@ -13,9 +12,11 @@ import { printRequest, getNetworkConfig } from "../utils/common.js"; import { SponsorshipPolicy } from "../models/sponsorship-policy.js"; import { DEFAULT_EP_VERSION, EPVersions, getEPVersion } from "../types/sponsorship-policy-dto.js"; import { NativeOracles } from "../constants/ChainlinkOracles.js"; +import { PaymasterRoutesOpts } from "../types/arka-config-dto.js"; -const paymasterRoutes: FastifyPluginAsync = async (server) => { - const paymaster = new Paymaster(server.config.FEE_MARKUP, server.config.MULTI_TOKEN_MARKUP, server.config.EP7_TOKEN_VGL, server.config.EP7_TOKEN_PGL); +const paymasterRoutes: FastifyPluginAsync = async (server, options: PaymasterRoutesOpts) => { + + const {paymaster} = options; const SUPPORTED_ENTRYPOINTS = { EPV_06: server.config.EPV_06, diff --git a/backend/src/server.ts b/backend/src/server.ts index 940b854..1b086f6 100644 --- a/backend/src/server.ts +++ b/backend/src/server.ts @@ -15,7 +15,6 @@ import { getNetworkConfig } from './utils/common.js'; import { checkDeposit } from './utils/monitorTokenPaymaster.js'; import { APIKey } from './models/api-key.js'; import { APIKeyRepository } from './repository/api-key-repository.js'; -import { ArkaConfig } from './models/arka-config.js'; import { ArkaConfigRepository } from './repository/arka-config-repository.js'; import adminRoutes from './routes/admin-routes.js'; import depositRoutes from './routes/deposit-route.js'; @@ -25,6 +24,8 @@ import pimlicoRoutes from './routes/pimlico-routes.js'; import whitelistRoutes from './routes/whitelist-routes.js'; import sponsorshipPolicyRoutes from './routes/sponsorship-policy-routes.js'; import SupportedNetworks from "../config.json" assert { type: "json" }; +import { Paymaster } from './paymaster/index.js'; +import { NativeOracles } from './constants/ChainlinkOracles.js'; let server: FastifyInstance; @@ -55,7 +56,9 @@ const initializeServer = async (): Promise => { logLevel: "warn" }); - await server.register(paymasterRoutes); + const paymaster = new Paymaster(server.config.FEE_MARKUP, server.config.MULTI_TOKEN_MARKUP, server.config.EP7_TOKEN_VGL, server.config.EP7_TOKEN_PGL); + + await server.register(paymasterRoutes, {paymaster}); await server.register(adminRoutes); @@ -317,6 +320,102 @@ const initializeServer = async (): Promise => { } } } + }, + { + name: 'updateNativeTokenOracleData', + cronTime: process.env.NATIVE_ORACLE_UPDATE_CRON_EXP || "*/2 * * * *", // every 2 mins. + onTick: async () => { + const chainIds = Object.keys(NativeOracles); + for(const chainId of chainIds) { + const networkConfig = getNetworkConfig(chainId, '', server.config.EPV_06); + if(!networkConfig) { + continue; + } + const provider = new ethers.providers.JsonRpcProvider(networkConfig.bundler); + paymaster.getLatestAnswerAndDecimals(provider, NativeOracles[+chainId], +chainId, false) + .catch((error) => { + server.log.error(`Failed to update native oracle price for chain id: ${chainId}, ${error}`); + }); + } + } + }, + { + name: 'updateTokenOracleData', + cronTime: process.env.TOKEN_ORACLE_UPDATE_CRON_EXP || '*/5 * * * *', // every 5 mins. + onTick: async () => { + let buffer = Buffer.from(server.config.MULTI_TOKEN_PAYMASTERS, 'base64'); + const multiTokenPaymasters = JSON.parse(buffer.toString()); + + buffer = Buffer.from(server.config.MULTI_TOKEN_ORACLES, 'base64'); + const multiTokenOracles = JSON.parse(buffer.toString()); + + const chainIds = Object.keys(multiTokenPaymasters); + + for(const chainId of chainIds) { + const networkConfig = getNetworkConfig(chainId, '', server.config.EPV_06); + if(!networkConfig) { + continue; + } + const provider = new ethers.providers.JsonRpcProvider(networkConfig.bundler) + paymaster.getLatestAnswerAndDecimals( + provider, + NativeOracles[+chainId], + +chainId + ).then((data) => { + const {latestAnswer, decimals} = data; + if(!latestAnswer || !decimals) { + return; + } + const tokens = Object.keys(multiTokenPaymasters[chainId]); + for(const token of tokens) { + if(networkConfig.MultiTokenPaymasterOracleUsed === 'chainlink') { + paymaster.getPriceFromChainlink( + multiTokenOracles[chainId][token], + provider, + token, + latestAnswer, + decimals, + +chainId, + false + ).catch((error) => { + server.log.error( + `Failed to update oracle data for token: ${token}, chain id: ${chainId}, ${error}` + ); + }); + } else if (networkConfig.MultiTokenPaymasterOracleUsed === 'orochi') { + paymaster.getPriceFromOrochi( + multiTokenOracles[chainId][token], + provider, + token, + +chainId, + false + ).catch((error) => { + server.log.error( + `Failed to update oracle data for token: ${token}, chain id: ${chainId}, ${error}` + ); + }); + } else { + paymaster.getPriceFromEtherspotChainlink( + multiTokenOracles[chainId][token], + provider, + token, + +chainId, + false + ).catch((error) => { + server.log.error( + `Failed to update oracle data for token: ${token}, chain id: ${chainId}, ${error}` + ); + }); + } + } + }) + .catch((error) => { + server.log.error( + `Failed to get native prices while updating oracle token data: chain id ${chainId}, ${error}` + ); + }); + } + } } ] }); diff --git a/backend/src/types/arka-config-dto.ts b/backend/src/types/arka-config-dto.ts index 72947be..1cd5860 100644 --- a/backend/src/types/arka-config-dto.ts +++ b/backend/src/types/arka-config-dto.ts @@ -1,3 +1,4 @@ +import { Paymaster } from "../paymaster/index.js"; export interface ArkaConfigUpdateData { deployedErc20Paymasters: string; @@ -11,3 +12,6 @@ export interface ArkaConfigUpdateData { coingeckoApiUrl: string; } +export interface PaymasterRoutesOpts { + paymaster: Paymaster; +} \ No newline at end of file From 3934afa60fdc6aeee9619919a7e15ba357937c32 Mon Sep 17 00:00:00 2001 From: nikhil kumar Date: Wed, 18 Dec 2024 15:06:00 +0530 Subject: [PATCH 2/3] update version --- backend/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/package.json b/backend/package.json index d818b25..c132096 100644 --- a/backend/package.json +++ b/backend/package.json @@ -1,6 +1,6 @@ { "name": "arka", - "version": "1.7.2", + "version": "1.7.3", "description": "ARKA - (Albanian for Cashier's case) is the first open source Paymaster as a service software", "type": "module", "directories": { From b9d337354262b4082ae35e910198f42ccc02cf92 Mon Sep 17 00:00:00 2001 From: nikhil kumar Date: Wed, 18 Dec 2024 19:11:45 +0530 Subject: [PATCH 3/3] changes to make native oracle calls only for chainlink --- backend/src/server.ts | 95 +++++++++++++++++++++++-------------------- 1 file changed, 51 insertions(+), 44 deletions(-) diff --git a/backend/src/server.ts b/backend/src/server.ts index 1b086f6..4134899 100644 --- a/backend/src/server.ts +++ b/backend/src/server.ts @@ -356,19 +356,20 @@ const initializeServer = async (): Promise => { if(!networkConfig) { continue; } - const provider = new ethers.providers.JsonRpcProvider(networkConfig.bundler) - paymaster.getLatestAnswerAndDecimals( - provider, - NativeOracles[+chainId], - +chainId - ).then((data) => { - const {latestAnswer, decimals} = data; - if(!latestAnswer || !decimals) { - return; - } - const tokens = Object.keys(multiTokenPaymasters[chainId]); - for(const token of tokens) { - if(networkConfig.MultiTokenPaymasterOracleUsed === 'chainlink') { + const provider = new ethers.providers.JsonRpcProvider(networkConfig.bundler); + + if(networkConfig.MultiTokenPaymasterOracleUsed === 'chainlink') { + paymaster.getLatestAnswerAndDecimals( + provider, + NativeOracles[+chainId], + +chainId + ).then((data) => { + const {latestAnswer, decimals} = data; + if(!latestAnswer || !decimals) { + return; + } + const tokens = Object.keys(multiTokenPaymasters[chainId]); + for(const token of tokens) { paymaster.getPriceFromChainlink( multiTokenOracles[chainId][token], provider, @@ -382,38 +383,44 @@ const initializeServer = async (): Promise => { `Failed to update oracle data for token: ${token}, chain id: ${chainId}, ${error}` ); }); - } else if (networkConfig.MultiTokenPaymasterOracleUsed === 'orochi') { - paymaster.getPriceFromOrochi( - multiTokenOracles[chainId][token], - provider, - token, - +chainId, - false - ).catch((error) => { - server.log.error( - `Failed to update oracle data for token: ${token}, chain id: ${chainId}, ${error}` - ); - }); - } else { - paymaster.getPriceFromEtherspotChainlink( - multiTokenOracles[chainId][token], - provider, - token, - +chainId, - false - ).catch((error) => { - server.log.error( - `Failed to update oracle data for token: ${token}, chain id: ${chainId}, ${error}` - ); - }); } - } - }) - .catch((error) => { - server.log.error( - `Failed to get native prices while updating oracle token data: chain id ${chainId}, ${error}` - ); - }); + }) + .catch((error) => { + server.log.error( + `Failed to get native prices while updating oracle token data: chain id ${chainId}, ${error}` + ); + }); + } else if (networkConfig.MultiTokenPaymasterOracleUsed === 'orochi') { + const tokens = Object.keys(multiTokenPaymasters[chainId]); + for(const token of tokens) { + paymaster.getPriceFromOrochi( + multiTokenOracles[chainId][token], + provider, + token, + +chainId, + false + ).catch((error) => { + server.log.error( + `Failed to update oracle data for token: ${token}, chain id: ${chainId}, ${error}` + ); + }); + } + } else { + const tokens = Object.keys(multiTokenPaymasters[chainId]); + for(const token of tokens) { + paymaster.getPriceFromEtherspotChainlink( + multiTokenOracles[chainId][token], + provider, + token, + +chainId, + false + ).catch((error) => { + server.log.error( + `Failed to update oracle data for token: ${token}, chain id: ${chainId}, ${error}` + ); + }); + } + } } } }