From fb897eb0ea8a422f7adc67280fa31a5fa7734c28 Mon Sep 17 00:00:00 2001 From: marie-fourier Date: Tue, 20 Feb 2024 19:01:10 +0500 Subject: [PATCH 1/3] feat: merkle-io integration --- README.md | 2 +- packages/executor/src/config.ts | 9 + packages/executor/src/executor.ts | 10 + packages/executor/src/interfaces.ts | 2 + .../BundlingService/relayers/index.ts | 1 + .../BundlingService/relayers/merkle.ts | 194 ++++++++++++++++++ .../src/services/BundlingService/service.ts | 18 +- packages/types/src/executor/index.ts | 2 +- 8 files changed, 233 insertions(+), 5 deletions(-) create mode 100644 packages/executor/src/services/BundlingService/relayers/merkle.ts diff --git a/README.md b/README.md index 82cf967d..51bfd442 100644 --- a/README.md +++ b/README.md @@ -120,7 +120,7 @@ Or follow the steps below: "account": [] }, "bundleGasLimitMarkup": 25000, # optional, adds some amount of additional gas to a bundle tx - "relayingMode": "classic"; # optional, allows to switch to Flashbots Builder api if set to "flashbots", see packages/executor/src/interfaces.ts for more + "relayingMode": "classic"; # optional, "flashbots" for Flashbots Builder API, "merkle" for Merkle.io "bundleInterval": 10000, # bundle creation interval "bundleSize": 4, # optional, max size of a bundle, 4 userops by default "pvgMarkup": 0 # optional, adds some gas on top of estimated PVG diff --git a/packages/executor/src/config.ts b/packages/executor/src/config.ts index a0dad6f3..91b68d5c 100644 --- a/packages/executor/src/config.ts +++ b/packages/executor/src/config.ts @@ -333,6 +333,14 @@ export class Config { ) ); + conf.merkleApiURL = String( + fromEnvVar( + network, + "MERKLE_API_URL", + conf.merkleApiURL || bundlerDefaultConfigs.merkleApiURL + ) + ); + // eslint-disable-next-line @typescript-eslint/strict-boolean-expressions if (!conf.whitelistedEntities) { conf.whitelistedEntities = bundlerDefaultConfigs.whitelistedEntities; @@ -391,6 +399,7 @@ const bundlerDefaultConfigs: BundlerConfig = { relayingMode: "classic", pvgMarkup: 0, gasFeeInSimulation: false, + merkleApiURL: "https://pool.merkle.io", }; const NETWORKS_ENV = (): string[] | undefined => { diff --git a/packages/executor/src/executor.ts b/packages/executor/src/executor.ts index e3fb2d92..7e480329 100644 --- a/packages/executor/src/executor.ts +++ b/packages/executor/src/executor.ts @@ -167,6 +167,16 @@ export class Executor { ); this.logger.info(`${this.networkName}: [X] FLASHBOTS BUIDLER API`); } + if (this.networkConfig.relayingMode === "merkle") { + if ( + !this.networkConfig.rpcEndpointSubmit || + !this.networkConfig.merkleApiURL + ) + throw Error( + "If you want to use Merkle API, please set RPC url in 'rpcEndpointSubmit' and API url in `merkleApiURL` in config file" + ); + this.logger.info(`${this.networkName}: [X] Merkle API`); + } if (this.networkConfig.conditionalTransactions) { this.logger.info(`${this.networkName}: [x] CONDITIONAL TRANSACTIONS`); diff --git a/packages/executor/src/interfaces.ts b/packages/executor/src/interfaces.ts index 932b8957..559fc76d 100644 --- a/packages/executor/src/interfaces.ts +++ b/packages/executor/src/interfaces.ts @@ -148,6 +148,8 @@ export interface NetworkConfig { pvgMarkup: number; // add gas fee in simulated transactions (may be required for some rpc providers) gasFeeInSimulation: boolean; + // api url of Merkle.io (by default https://pool.merkle.io) + merkleApiURL: string; } export type BundlerConfig = Omit< diff --git a/packages/executor/src/services/BundlingService/relayers/index.ts b/packages/executor/src/services/BundlingService/relayers/index.ts index 15414380..713a5b1c 100644 --- a/packages/executor/src/services/BundlingService/relayers/index.ts +++ b/packages/executor/src/services/BundlingService/relayers/index.ts @@ -1,2 +1,3 @@ export * from "./classic"; export * from "./flashbots"; +export * from "./merkle"; diff --git a/packages/executor/src/services/BundlingService/relayers/merkle.ts b/packages/executor/src/services/BundlingService/relayers/merkle.ts new file mode 100644 index 00000000..37fef4e0 --- /dev/null +++ b/packages/executor/src/services/BundlingService/relayers/merkle.ts @@ -0,0 +1,194 @@ +import path from "node:path"; +import { providers } from "ethers"; +import { PerChainMetrics } from "monitoring/lib"; +import { Logger, NetworkName } from "types/lib"; +import { IEntryPoint__factory } from "types/lib/executor/contracts"; +import { AccessList, fetchJson } from "ethers/lib/utils"; +import { MempoolEntryStatus } from "types/lib/executor"; +import { Config } from "../../../config"; +import { Bundle, NetworkConfig } from "../../../interfaces"; +import { MempoolService } from "../../MempoolService"; +import { ReputationService } from "../../ReputationService"; +import { estimateBundleGasLimit } from "../utils"; +import { now } from "../../../utils"; +import { BaseRelayer } from "./base"; + +export class MerkleRelayer extends BaseRelayer { + private submitTimeout = 2 * 60 * 1000; // 2 minutes + + constructor( + logger: Logger, + chainId: number, + network: NetworkName, + provider: providers.JsonRpcProvider, + config: Config, + networkConfig: NetworkConfig, + mempoolService: MempoolService, + reputationService: ReputationService, + metrics: PerChainMetrics | null + ) { + super( + logger, + chainId, + network, + provider, + config, + networkConfig, + mempoolService, + reputationService, + metrics + ); + } + + async sendBundle(bundle: Bundle): Promise { + const availableIndex = this.getAvailableRelayerIndex(); + if (availableIndex == null) return; + + const relayer = this.relayers[availableIndex]; + const mutex = this.mutexes[availableIndex]; + + const { entries, storageMap } = bundle; + if (!bundle.entries.length) return; + + await mutex.runExclusive(async () => { + const beneficiary = await this.selectBeneficiary(relayer); + const entryPoint = entries[0]!.entryPoint; + const entryPointContract = IEntryPoint__factory.connect( + entryPoint, + this.provider + ); + + const txRequest = entryPointContract.interface.encodeFunctionData( + "handleOps", + [entries.map((entry) => entry.userOp), beneficiary] + ); + + const transactionRequest: providers.TransactionRequest = { + to: entryPoint, + data: txRequest, + type: 2, + maxPriorityFeePerGas: bundle.maxPriorityFeePerGas, + maxFeePerGas: bundle.maxFeePerGas, + gasLimit: estimateBundleGasLimit( + this.networkConfig.bundleGasLimitMarkup, + bundle.entries + ), + chainId: this.provider._network.chainId, + nonce: await relayer.getTransactionCount(), + }; + + if (this.networkConfig.eip2930) { + const { storageMap } = bundle; + const addresses = Object.keys(storageMap); + if (addresses.length) { + const accessList: AccessList = []; + for (const address of addresses) { + const storageKeys = storageMap[address]; + if (typeof storageKeys == "object") { + accessList.push({ + address, + storageKeys: Object.keys(storageKeys), + }); + } + } + transactionRequest.accessList = accessList; + } + } + + try { + // checking for tx revert + await relayer.estimateGas(transactionRequest); + } catch (err) { + this.logger.error(err); + await this.mempoolService.removeAll(entries); + return; + } + + this.logger.debug(transactionRequest, "Merkle: Submitting"); + const merkleProvider = new providers.JsonRpcProvider( + this.networkConfig.rpcEndpointSubmit + ); + const signedRawTx = await relayer.signTransaction(transactionRequest); + const params = !this.networkConfig.conditionalTransactions + ? [signedRawTx] + : [signedRawTx, { knownAccounts: storageMap }]; + try { + const hash = await merkleProvider.send( + "eth_sendRawTransaction", + params + ); + this.logger.debug(`Bundle submitted: ${hash}`); + this.logger.debug( + `User op hashes ${entries.map((entry) => entry.userOpHash)}` + ); + await this.mempoolService.setStatus( + entries, + MempoolEntryStatus.Submitted, + hash + ); + await this.waitForTransaction(hash); + } catch (err) { + await this.mempoolService.setStatus(entries, MempoolEntryStatus.New); + await this.handleUserOpFail(entries, err); + } + }); + } + + async waitForTransaction(hash: string): Promise { + const txStatusUrl = new URL( + path.join("transaction", hash), + this.networkConfig.merkleApiURL + ).toString(); + const submitStart = now(); + return new Promise((resolve, reject) => { + let lock = false; + const handler = async (): Promise => { + this.logger.debug("Merkle: Fetching tx status"); + if (now() - submitStart > this.submitTimeout) return reject("timeout"); + if (lock) return; + lock = true; + try { + // https://docs.merkle.io/private-pool/wallets/transaction-status + const status = await fetchJson(txStatusUrl); + this.logger.debug(status, `Merkle: ${hash}`); + switch (status.status) { + case "nonce_too_low": + case "not_enough_funds": + case "base_fee_low": + case "low_priority_fee": + case "not_enough_gas": + case "sanctioned": + case "gas_limit_too_high": + case "invalid_signature": + case "nonce_gapped": + reject("rebundle"); // the bundle can be submitted again, no need to delete userops + break; + default: { + const response = await this.provider.getTransaction(hash); + if (response == null) { + this.logger.debug( + "Transaction not found yet. Trying again in 2 seconds" + ); + setTimeout(() => handler(), 2000); // fetch status again in 2 seconds + lock = false; + return; + } + this.logger.debug("Transaction is found"); + resolve(true); // transaction is found + } + } + } catch (err: any) { + this.logger.debug(err, "Could not fetch transaction status"); + // transaction is not found, but not necessarily failed + if (err.status === 400) { + setTimeout(() => handler(), 2000); // fetch status again in 2 seconds + lock = false; + return; + } + reject(err); + } + }; + void handler(); + }); + } +} diff --git a/packages/executor/src/services/BundlingService/service.ts b/packages/executor/src/services/BundlingService/service.ts index 46e7a194..c4b8975a 100644 --- a/packages/executor/src/services/BundlingService/service.ts +++ b/packages/executor/src/services/BundlingService/service.ts @@ -24,7 +24,7 @@ import { mergeStorageMap } from "../../utils/mergeStorageMap"; import { getAddr, wait } from "../../utils"; import { MempoolEntry } from "../../entities/MempoolEntry"; import { IRelayingMode } from "./interfaces"; -import { ClassicRelayer, FlashbotsRelayer } from "./relayers"; +import { ClassicRelayer, FlashbotsRelayer, MerkleRelayer } from "./relayers"; export class BundlingService { private mutex: Mutex; @@ -64,6 +64,18 @@ export class BundlingService { this.reputationService, this.metrics ); + } else if (relayingMode === "merkle") { + this.relayer = new MerkleRelayer( + this.logger, + this.chainId, + this.network, + this.provider, + this.config, + this.networkConfig, + this.mempoolService, + this.reputationService, + this.metrics + ); } else { this.relayer = new ClassicRelayer( this.logger, @@ -341,7 +353,7 @@ export class BundlingService { if (!entries.length) { this.logger.debug("No new entries"); return; - }; + } // remove entries from mempool if submitAttempts are greater than maxAttemps const invalidEntries = entries.filter( (entry) => entry.submitAttempts >= this.maxSubmitAttempts @@ -358,7 +370,7 @@ export class BundlingService { if (!entries.length) { this.logger.debug("No entries left"); return; - }; + } const gasFee = await getGasFee( this.chainId, this.provider, diff --git a/packages/types/src/executor/index.ts b/packages/types/src/executor/index.ts index 89699564..53ffb2c5 100644 --- a/packages/types/src/executor/index.ts +++ b/packages/types/src/executor/index.ts @@ -1,4 +1,4 @@ -export type RelayingMode = "flashbots" | "classic"; +export type RelayingMode = "merkle" | "flashbots" | "classic"; export interface SendBundleReturn { transactionHash: string; userOpHashes: string[]; From dcfc3ea473753043d736a37d164645a30eb827ae Mon Sep 17 00:00:00 2001 From: marie-fourier Date: Tue, 20 Feb 2024 19:11:30 +0500 Subject: [PATCH 2/3] uncomment max attempts to bundle --- packages/executor/src/services/BundlingService/service.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/executor/src/services/BundlingService/service.ts b/packages/executor/src/services/BundlingService/service.ts index c4b8975a..4c78ccc5 100644 --- a/packages/executor/src/services/BundlingService/service.ts +++ b/packages/executor/src/services/BundlingService/service.ts @@ -34,7 +34,7 @@ export class BundlingService { private maxBundleSize: number; private networkConfig: NetworkConfig; private relayer: IRelayingMode; - private maxSubmitAttempts = 10; + private maxSubmitAttempts = 3; constructor( private chainId: number, @@ -391,7 +391,7 @@ export class BundlingService { bundle.entries, MempoolEntryStatus.Pending ); - // await this.mempoolService.attemptToBundle(bundle.entries); + await this.mempoolService.attemptToBundle(bundle.entries); void this.relayer.sendBundle(bundle).catch((err) => { this.logger.error(err); }); From 2a10230ab00d6e2a0956f0e90a5f84cb834ff64f Mon Sep 17 00:00:00 2001 From: marie-fourier Date: Thu, 29 Feb 2024 14:23:03 +0500 Subject: [PATCH 3/3] chore: release(1.0.40-alpha) --- lerna.json | 2 +- package.json | 2 +- packages/api/package.json | 8 ++++---- packages/cli/package.json | 14 +++++++------- packages/db/package.json | 4 ++-- packages/executor/package.json | 8 ++++---- packages/monitoring/package.json | 4 ++-- packages/node/package.json | 16 ++++++++-------- packages/params/package.json | 6 +++--- packages/types/package.json | 2 +- packages/utils/package.json | 4 ++-- 11 files changed, 35 insertions(+), 35 deletions(-) diff --git a/lerna.json b/lerna.json index 7d0c4b18..94ed2541 100644 --- a/lerna.json +++ b/lerna.json @@ -4,7 +4,7 @@ ], "npmClient": "yarn", "useWorkspaces": true, - "version": "1.0.39-alpha", + "version": "1.0.40-alpha", "stream": "true", "command": { "version": { diff --git a/package.json b/package.json index fa88b233..53dd9d45 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "root", "private": true, - "version": "1.0.39-alpha", + "version": "1.0.40-alpha", "engines": { "node": ">=18.0.0" }, diff --git a/packages/api/package.json b/packages/api/package.json index 4f9d34e6..fba01a47 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -1,6 +1,6 @@ { "name": "api", - "version": "1.0.39-alpha", + "version": "1.0.40-alpha", "description": "The API module of Etherspot bundler client", "author": "Etherspot", "homepage": "https://https://github.com/etherspot/skandha#readme", @@ -35,13 +35,13 @@ "class-transformer": "0.5.1", "class-validator": "0.14.0", "ethers": "5.7.2", - "executor": "^1.0.39-alpha", + "executor": "^1.0.40-alpha", "fastify": "4.14.1", - "monitoring": "^1.0.39-alpha", + "monitoring": "^1.0.40-alpha", "pino": "8.11.0", "pino-pretty": "10.0.0", "reflect-metadata": "0.1.13", - "types": "^1.0.39-alpha" + "types": "^1.0.40-alpha" }, "devDependencies": { "@types/connect": "3.4.35" diff --git a/packages/cli/package.json b/packages/cli/package.json index a2fbadc2..bcffb068 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "cli", - "version": "1.0.39-alpha", + "version": "1.0.40-alpha", "description": "> TODO: description", "author": "zincoshine ", "homepage": "https://https://github.com/etherspot/skandha#readme", @@ -38,15 +38,15 @@ "@libp2p/peer-id-factory": "2.0.1", "@libp2p/prometheus-metrics": "1.1.3", "@multiformats/multiaddr": "12.1.3", - "api": "^1.0.39-alpha", - "db": "^1.0.39-alpha", - "executor": "^1.0.39-alpha", + "api": "^1.0.40-alpha", + "db": "^1.0.40-alpha", + "executor": "^1.0.40-alpha", "find-up": "5.0.0", "got": "12.5.3", "js-yaml": "4.1.0", - "monitoring": "^1.0.39-alpha", - "node": "^1.0.39-alpha", - "types": "^1.0.39-alpha", + "monitoring": "^1.0.40-alpha", + "node": "^1.0.40-alpha", + "types": "^1.0.40-alpha", "yargs": "17.6.2" }, "devDependencies": { diff --git a/packages/db/package.json b/packages/db/package.json index 9e48f07e..d7d34e8d 100644 --- a/packages/db/package.json +++ b/packages/db/package.json @@ -1,6 +1,6 @@ { "name": "db", - "version": "1.0.39-alpha", + "version": "1.0.40-alpha", "description": "The DB module of Etherspot bundler client", "author": "Etherspot", "homepage": "https://github.com/etherspot/etherspot-bundler#readme", @@ -33,7 +33,7 @@ "dependencies": { "@chainsafe/ssz": "0.10.1", "@farcaster/rocksdb": "5.5.0", - "types": "^1.0.39-alpha" + "types": "^1.0.40-alpha" }, "devDependencies": { "@types/rocksdb": "3.0.1", diff --git a/packages/executor/package.json b/packages/executor/package.json index 1045c107..8b145697 100644 --- a/packages/executor/package.json +++ b/packages/executor/package.json @@ -1,6 +1,6 @@ { "name": "executor", - "version": "1.0.39-alpha", + "version": "1.0.40-alpha", "description": "The Relayer module of Etherspot bundler client", "author": "Etherspot", "homepage": "https://https://github.com/etherspot/skandha#readme", @@ -34,8 +34,8 @@ "@flashbots/ethers-provider-bundle": "0.6.2", "async-mutex": "0.4.0", "ethers": "5.7.2", - "monitoring": "^1.0.39-alpha", - "params": "^1.0.39-alpha", - "types": "^1.0.39-alpha" + "monitoring": "^1.0.40-alpha", + "params": "^1.0.40-alpha", + "types": "^1.0.40-alpha" } } diff --git a/packages/monitoring/package.json b/packages/monitoring/package.json index 959e0ad7..60d2c4e9 100644 --- a/packages/monitoring/package.json +++ b/packages/monitoring/package.json @@ -1,6 +1,6 @@ { "name": "monitoring", - "version": "1.0.39-alpha", + "version": "1.0.40-alpha", "description": "The Monitoring module of Etherspot bundler client", "author": "Etherspot", "homepage": "https://github.com/etherspot/etherspot-bundler#readme", @@ -32,6 +32,6 @@ }, "dependencies": { "prom-client": "15.1.0", - "types": "^1.0.39-alpha" + "types": "^1.0.40-alpha" } } diff --git a/packages/node/package.json b/packages/node/package.json index 579b2f45..0c55e2bf 100644 --- a/packages/node/package.json +++ b/packages/node/package.json @@ -1,6 +1,6 @@ { "name": "node", - "version": "1.0.39-alpha", + "version": "1.0.40-alpha", "description": "The bundler node module of Etherspot bundler client", "author": "Etherspot", "homepage": "https://https://github.com/etherspot/skandha#readme", @@ -56,25 +56,25 @@ "@libp2p/tcp": "6.1.0", "@multiformats/multiaddr": "11.4.0", "abstract-leveldown": "7.2.0", - "api": "^1.0.39-alpha", + "api": "^1.0.40-alpha", "datastore-core": "8.0.1", - "db": "^1.0.39-alpha", + "db": "^1.0.40-alpha", "ethers": "5.7.2", - "executor": "^1.0.39-alpha", + "executor": "^1.0.40-alpha", "it-filter": "1.0.2", "it-map": "1.0.5", "it-sort": "1.0.0", "it-take": "1.0.1", "libp2p": "0.42.2", - "monitoring": "^1.0.39-alpha", - "params": "^1.0.39-alpha", + "monitoring": "^1.0.40-alpha", + "params": "^1.0.40-alpha", "prettier": "2.8.4", "snappy": "7.2.2", "snappyjs": "0.7.0", "stream-to-it": "0.2.4", "strict-event-emitter-types": "2.0.0", - "types": "^1.0.39-alpha", - "utils": "^1.0.39-alpha", + "types": "^1.0.40-alpha", + "utils": "^1.0.40-alpha", "varint": "6.0.0", "xxhash-wasm": "1.0.2" }, diff --git a/packages/params/package.json b/packages/params/package.json index a5219bf1..11775a6c 100644 --- a/packages/params/package.json +++ b/packages/params/package.json @@ -1,6 +1,6 @@ { "name": "params", - "version": "1.0.39-alpha", + "version": "1.0.40-alpha", "description": "Various bundler parameters", "author": "Etherspot", "homepage": "https://github.com/etherspot/skandha#readme", @@ -26,8 +26,8 @@ "@eth-optimism/sdk": "3.0.0", "@mantleio/sdk": "0.2.1", "ethers": "5.7.2", - "types": "^1.0.39-alpha", - "utils": "^1.0.39-alpha" + "types": "^1.0.40-alpha", + "utils": "^1.0.40-alpha" }, "scripts": { "clean": "rm -rf lib && rm -f *.tsbuildinfo", diff --git a/packages/types/package.json b/packages/types/package.json index cecad1b3..40f31d83 100644 --- a/packages/types/package.json +++ b/packages/types/package.json @@ -1,6 +1,6 @@ { "name": "types", - "version": "1.0.39-alpha", + "version": "1.0.40-alpha", "description": "The types of Etherspot bundler client", "author": "Etherspot", "homepage": "https://https://github.com/etherspot/skandha#readme", diff --git a/packages/utils/package.json b/packages/utils/package.json index 1b3958ee..766af9a3 100644 --- a/packages/utils/package.json +++ b/packages/utils/package.json @@ -1,6 +1,6 @@ { "name": "utils", - "version": "1.0.39-alpha", + "version": "1.0.40-alpha", "description": "utils of Etherspot bundler client", "author": "Etherspot", "homepage": "https://https://github.com/etherspot/skandha#readme", @@ -37,6 +37,6 @@ "case": "^1.6.3", "pino": "8.11.0", "pino-pretty": "10.0.0", - "types": "^1.0.39-alpha" + "types": "^1.0.40-alpha" } }