New metrics (#150)

README.md

@@ -73,7 +73,9 @@ Or follow the steps below:
       "entryPoints": [
         "0x5FF137D4b0FDCD49DcA30c7CF57E578a026d2789"
       ],
-      "relayer": "0x{RELAYER-PRIVATE-KEY}",
+      "relayers": [
+        "0x{RELAYER-PRIVATE-KEY}"
+      ],
       "beneficiary": "0x{BENEFICIARY-ADDRESS}",
       "rpcEndpoint": "https://polygon-mumbai.blockpi.network/v1/rpc/public"
     }

lerna.json

@@ -4,7 +4,7 @@
   ],
   "npmClient": "yarn",
   "useWorkspaces": true,
-  "version": "1.0.36-alpha",
+  "version": "1.0.37-alpha",
   "stream": "true",
   "command": {
     "version": {

package.json

@@ -1,7 +1,7 @@
 {
   "name": "root",
   "private": true,
-  "version": "1.0.36-alpha",
+  "version": "1.0.37-alpha",
   "engines": {
     "node": ">=18.0.0"
   },

packages/api/package.json

@@ -1,6 +1,6 @@
 {
   "name": "api",
-  "version": "1.0.36-alpha",
+  "version": "1.0.37-alpha",
   "description": "The API module of Etherspot bundler client",
   "author": "Etherspot",
   "homepage": "https://https://github.com/etherspot/skandha#readme",
@@ -35,13 +35,13 @@
     "class-transformer": "0.5.1",
     "class-validator": "0.14.0",
     "ethers": "5.7.2",
-    "executor": "^1.0.36-alpha",
+    "executor": "^1.0.37-alpha",
     "fastify": "4.14.1",
-    "monitoring": "^1.0.36-alpha",
+    "monitoring": "^1.0.37-alpha",
     "pino": "8.11.0",
     "pino-pretty": "10.0.0",
     "reflect-metadata": "0.1.13",
-    "types": "^1.0.36-alpha"
+    "types": "^1.0.37-alpha"
   },
   "devDependencies": {
     "@types/connect": "3.4.35"

packages/cli/package.json

@@ -1,6 +1,6 @@
 {
   "name": "cli",
-  "version": "1.0.36-alpha",
+  "version": "1.0.37-alpha",
   "description": "> TODO: description",
   "author": "zincoshine <[email protected]>",
   "homepage": "https://https://github.com/etherspot/skandha#readme",
@@ -38,15 +38,15 @@
     "@libp2p/peer-id-factory": "2.0.1",
     "@libp2p/prometheus-metrics": "1.1.3",
     "@multiformats/multiaddr": "12.1.3",
-    "api": "^1.0.36-alpha",
-    "db": "^1.0.36-alpha",
-    "executor": "^1.0.36-alpha",
+    "api": "^1.0.37-alpha",
+    "db": "^1.0.37-alpha",
+    "executor": "^1.0.37-alpha",
     "find-up": "5.0.0",
     "got": "12.5.3",
     "js-yaml": "4.1.0",
-    "monitoring": "^1.0.36-alpha",
-    "node": "^1.0.36-alpha",
-    "types": "^1.0.36-alpha",
+    "monitoring": "^1.0.37-alpha",
+    "node": "^1.0.37-alpha",
+    "types": "^1.0.37-alpha",
     "yargs": "17.6.2"
   },
   "devDependencies": {

packages/db/package.json

@@ -1,6 +1,6 @@
 {
   "name": "db",
-  "version": "1.0.36-alpha",
+  "version": "1.0.37-alpha",
   "description": "The DB module of Etherspot bundler client",
   "author": "Etherspot",
   "homepage": "https://github.com/etherspot/etherspot-bundler#readme",
@@ -33,7 +33,7 @@
   "dependencies": {
     "@chainsafe/ssz": "0.10.1",
     "@farcaster/rocksdb": "5.5.0",
-    "types": "^1.0.36-alpha"
+    "types": "^1.0.37-alpha"
   },
   "devDependencies": {
     "@types/rocksdb": "3.0.1",

packages/executor/package.json

@@ -1,6 +1,6 @@
 {
   "name": "executor",
-  "version": "1.0.36-alpha",
+  "version": "1.0.37-alpha",
   "description": "The Relayer module of Etherspot bundler client",
   "author": "Etherspot",
   "homepage": "https://https://github.com/etherspot/skandha#readme",
@@ -34,8 +34,8 @@
     "@flashbots/ethers-provider-bundle": "0.6.2",
     "async-mutex": "0.4.0",
     "ethers": "5.7.2",
-    "monitoring": "^1.0.36-alpha",
-    "params": "^1.0.36-alpha",
-    "types": "^1.0.36-alpha"
+    "monitoring": "^1.0.37-alpha",
+    "params": "^1.0.37-alpha",
+    "types": "^1.0.37-alpha"
   }
 }

packages/executor/src/entities/MempoolEntry.ts

@@ -21,6 +21,7 @@ export class MempoolEntry implements IMempoolEntry {
   hash?: string; // keccak256 of all referenced contracts
   transaction?: string; // hash of a submitted bundle
   submitAttempts: number;
+  submittedTime?: number; // timestamp when mempool was first put into the mempool
 
   constructor({
     chainId,
@@ -36,6 +37,7 @@ export class MempoolEntry implements IMempoolEntry {
     status,
     transaction,
     submitAttempts,
+    submittedTime,
   }: {
     chainId: number;
     userOp: UserOperationStruct;
@@ -50,6 +52,7 @@ export class MempoolEntry implements IMempoolEntry {
     status?: MempoolEntryStatus | undefined;
     transaction?: string | undefined;
     submitAttempts?: number | undefined;
+    submittedTime?: number | undefined;
   }) {
     this.chainId = chainId;
     this.userOp = userOp;
@@ -61,6 +64,7 @@ export class MempoolEntry implements IMempoolEntry {
     this.paymaster = paymaster;
     this.hash = hash;
     this.lastUpdatedTime = lastUpdatedTime ?? now();
+    this.submittedTime = submittedTime;
     this.status = status ?? MempoolEntryStatus.New;
     this.transaction = transaction;
     this.submitAttempts = submitAttempts ?? 0;
@@ -186,6 +190,7 @@ export class MempoolEntry implements IMempoolEntry {
       transaction: this.transaction,
       submitAttempts: this.submitAttempts,
       status: this.status,
+      submittedTime: this.submittedTime,
     };
   }
 }

packages/executor/src/entities/interfaces.ts

@@ -16,6 +16,7 @@ export interface IMempoolEntry {
   status: MempoolEntryStatus;
   transaction?: string;
   submitAttempts: number;
+  submittedTime?: number;
 }
 
 export interface MempoolEntrySerialized {
@@ -43,6 +44,7 @@ export interface MempoolEntrySerialized {
   transaction: string | undefined;
   submitAttempts: number;
   status: MempoolEntryStatus;
+  submittedTime: number | undefined;
 }
 
 export interface IReputationEntry {

packages/executor/src/services/BundlingService/relayers/base.ts

@@ -116,15 +116,25 @@ export abstract class BaseRelayer implements IRelayingMode {
   // metrics
   protected reportSubmittedUserops(txHash: string, bundle: Bundle): void {
     if (txHash && this.metrics) {
+      this.metrics.bundlesSubmitted.inc(1);
       this.metrics.useropsSubmitted.inc(bundle.entries.length);
+      this.metrics.useropsInBundle.observe(bundle.entries.length);
       bundle.entries.forEach((entry) => {
         this.metrics!.useropsTimeToProcess.observe(
-          now() - entry.lastUpdatedTime
+          Math.ceil(
+            (now() - (entry.submittedTime ?? entry.lastUpdatedTime)) / 1000
+          )
         );
       });
     }
   }
 
+  protected reportFailedBundle(): void {
+    if (this.metrics) {
+      this.metrics.bundlesFailed.inc(1);
+    }
+  }
+
   /**
    * determine who should receive the proceedings of the request.
    * if signer's balance is too low, send it to signer. otherwise, send to configured beneficiary.

packages/executor/src/services/BundlingService/relayers/classic.ts

@@ -123,6 +123,7 @@ export class ClassicRelayer extends BaseRelayer {
         } catch (err) {
           this.logger.error(err);
           await this.mempoolService.removeAll(entries);
+          this.reportFailedBundle();
           return;
         }
 
@@ -150,6 +151,7 @@ export class ClassicRelayer extends BaseRelayer {
             this.reportSubmittedUserops(txHash, bundle);
           })
           .catch(async (err: any) => {
+            this.reportFailedBundle();
             // Put all userops back to the mempool
             // if some userop failed, it will be deleted inside handleUserOpFail()
             await this.mempoolService.setStatus(

packages/executor/src/services/BundlingService/relayers/flashbots.ts

@@ -86,6 +86,7 @@ export class FlashbotsRelayer extends BaseRelayer {
       } catch (err) {
         this.logger.error(err);
         await this.mempoolService.removeAll(entries);
+        this.reportFailedBundle();
         return;
       }
 
@@ -109,6 +110,7 @@ export class FlashbotsRelayer extends BaseRelayer {
           this.reportSubmittedUserops(txHash, bundle);
         })
         .catch(async (err: any) => {
+          this.reportFailedBundle();
           // Put all userops back to the mempool
           // if some userop failed, it will be deleted inside handleUserOpFail()
           await this.mempoolService.setStatus(entries, MempoolEntryStatus.New);

packages/executor/src/services/MempoolService.ts

@@ -65,6 +65,7 @@ export class MempoolService {
       userOpHash,
       factory: getAddr(userOp.initCode),
       paymaster: getAddr(userOp.paymasterAndData),
+      submittedTime: now(),
     });
     await this.mutex.runExclusive(async () => {
       const existingEntry = await this.find(entry);
@@ -386,6 +387,7 @@ export class MempoolService {
       transaction: raw.transaction,
       status: raw.status,
       submitAttempts: raw.submitAttempts,
+      submittedTime: raw.submittedTime,
     });
   }
 

packages/monitoring/package.json

@@ -1,6 +1,6 @@
 {
   "name": "monitoring",
-  "version": "1.0.36-alpha",
+  "version": "1.0.37-alpha",
   "description": "The Monitoring module of Etherspot bundler client",
   "author": "Etherspot",
   "homepage": "https://github.com/etherspot/etherspot-bundler#readme",
@@ -31,7 +31,7 @@
     "check-readme": "typescript-docs-verifier"
   },
   "dependencies": {
-    "prom-client": "^14.2.0",
-    "types": "^1.0.36-alpha"
+    "prom-client": "15.1.0",
+    "types": "^1.0.37-alpha"
   }
 }

packages/monitoring/src/metrics/chain.ts

@@ -6,6 +6,9 @@ export interface IChainMetrics {
   useropsSubmitted: Counter.Internal;
   useropsEstimated: Counter.Internal;
   useropsTimeToProcess: Histogram.Internal<"chainId">;
+  bundlesSubmitted: Counter.Internal;
+  bundlesFailed: Counter.Internal;
+  useropsInBundle: Histogram.Internal<"chainId">;
 }
 
 const useropsInMempool = new Counter({
@@ -26,6 +29,18 @@ const useropsSubmitted = new Counter({
   labelNames: ["chainId"],
 });
 
+const bundlesSubmitted = new Counter({
+  name: "skandha_bundles_submitted_count",
+  help: "Number of bundles successfully submitted on-chain",
+  labelNames: ["chainId"],
+});
+
+const bundlesFailed = new Counter({
+  name: "skandha_bundles_failed_count",
+  help: "Number of bundles failed to submit",
+  labelNames: ["chainId"],
+});
+
 const useropsEstimated = new Counter({
   name: "skandha_user_ops_estimated_count",
   help: "Number of user ops estimated",
@@ -36,7 +51,16 @@ const useropsTimeToProcess = new Histogram({
   name: "skandha_user_op_time_to_process",
   help: "How long did it take for userop to get submitted",
   labelNames: ["chainId"],
-  buckets: [1, 2, 3, 5, 10, 12, 15, 17, 20, 25, 30, 60, 120, 180],
+  buckets: [
+    1, 2, 3, 5, 10, 12, 15, 17, 20, 25, 30, 60, 120, 180, 210, 240, 270, 300,
+  ],
+});
+
+const useropsInBundle = new Histogram({
+  name: "skandha_userops_in_bundle",
+  help: "Number of bundles with x userops",
+  labelNames: ["chainId"],
+  buckets: [1, 2, 3, 4],
 });
 
 /**
@@ -51,12 +75,18 @@ export function createChainMetrics(
   registry.registerMetric(useropsSubmitted);
   registry.registerMetric(useropsEstimated);
   registry.registerMetric(useropsTimeToProcess);
+  registry.registerMetric(bundlesFailed);
+  registry.registerMetric(bundlesSubmitted);
+  registry.registerMetric(useropsInBundle);
 
   return {
     useropsInMempool: useropsInMempool.labels({ chainId }),
     useropsAttempted: useropsAttempted.labels({ chainId }),
     useropsSubmitted: useropsSubmitted.labels({ chainId }),
     useropsEstimated: useropsEstimated.labels({ chainId }),
     useropsTimeToProcess: useropsTimeToProcess.labels({ chainId }),
+    bundlesSubmitted: bundlesSubmitted.labels({ chainId }),
+    bundlesFailed: bundlesFailed.labels({ chainId }),
+    useropsInBundle: useropsInBundle.labels({ chainId }),
   };
 }

packages/node/package.json

@@ -1,6 +1,6 @@
 {
   "name": "node",
-  "version": "1.0.36-alpha",
+  "version": "1.0.37-alpha",
   "description": "The bundler node module of Etherspot bundler client",
   "author": "Etherspot",
   "homepage": "https://https://github.com/etherspot/skandha#readme",
@@ -56,25 +56,25 @@
     "@libp2p/tcp": "6.1.0",
     "@multiformats/multiaddr": "11.4.0",
     "abstract-leveldown": "7.2.0",
-    "api": "^1.0.36-alpha",
+    "api": "^1.0.37-alpha",
     "datastore-core": "8.0.1",
-    "db": "^1.0.36-alpha",
+    "db": "^1.0.37-alpha",
     "ethers": "5.7.2",
-    "executor": "^1.0.36-alpha",
+    "executor": "^1.0.37-alpha",
     "it-filter": "1.0.2",
     "it-map": "1.0.5",
     "it-sort": "1.0.0",
     "it-take": "1.0.1",
     "libp2p": "0.42.2",
-    "monitoring": "^1.0.36-alpha",
-    "params": "^1.0.36-alpha",
+    "monitoring": "^1.0.37-alpha",
+    "params": "^1.0.37-alpha",
     "prettier": "2.8.4",
     "snappy": "7.2.2",
     "snappyjs": "0.7.0",
     "stream-to-it": "0.2.4",
     "strict-event-emitter-types": "2.0.0",
-    "types": "^1.0.36-alpha",
-    "utils": "^1.0.36-alpha",
+    "types": "^1.0.37-alpha",
+    "utils": "^1.0.37-alpha",
     "varint": "6.0.0",
     "xxhash-wasm": "1.0.2"
   },

packages/params/package.json

@@ -1,6 +1,6 @@
 {
   "name": "params",
-  "version": "1.0.36-alpha",
+  "version": "1.0.37-alpha",
   "description": "Various bundler parameters",
   "author": "Etherspot",
   "homepage": "https://github.com/etherspot/skandha#readme",
@@ -26,8 +26,8 @@
     "@eth-optimism/sdk": "3.0.0",
     "@mantleio/sdk": "0.2.1",
     "ethers": "5.7.2",
-    "types": "^1.0.36-alpha",
-    "utils": "^1.0.36-alpha"
+    "types": "^1.0.37-alpha",
+    "utils": "^1.0.37-alpha"
   },
   "scripts": {
     "clean": "rm -rf lib && rm -f *.tsbuildinfo",