feat: gen proofs with workers

Dockerfile

@@ -26,7 +26,7 @@ RUN mkdir -p ./storage/ && chown -R node:node ./storage/
 
 USER node
 
-HEALTHCHECK --interval=360s --timeout=120s --retries=3 \
+HEALTHCHECK --interval=60s --timeout=10s --retries=3 \
   CMD curl -f http://localhost:$HTTP_PORT/health || exit 1
 
 CMD ["yarn", "start:prod"]

package.json

@@ -7,8 +7,8 @@
   "private": true,
   "license": "GPL-3.0",
   "scripts": {
-    "prove": "NODE_OPTIONS=--max_old_space_size=8192 WORKING_MODE=cli node dist/main prove",
-    "prove:debug": "NODE_OPTIONS=--max_old_space_size=8192 WORKING_MODE=cli node --inspect dist/main prove",
+    "prove": "WORKING_MODE=cli node dist/main prove",
+    "prove:debug": "WORKING_MODE=cli node --inspect dist/main prove",
     "slashing": "yarn prove slashing",
     "slashing:debug": "yarn prove:debug slashing",
     "withdrawal": "yarn prove withdrawal",
@@ -19,7 +19,7 @@
     "start": "nest start",
     "start:dev": "nest start --watch",
     "start:debug": "nest start --debug --watch",
-    "start:prod": "NODE_OPTIONS=--max_old_space_size=8192 node dist/main",
+    "start:prod": "node dist/main",
     "lint": "eslint \"{src,apps,libs,test}/**/*.ts\"",
     "lint:fix": "eslint \"{src,apps,libs,test}/**/*.ts\" --fix",
     "test:debug": "node --inspect-brk -r tsconfig-paths/register -r ts-node/register node_modules/.bin/jest --runInBand",

src/common/prometheus/prometheus.service.ts

@@ -241,3 +241,41 @@ export function TrackTask(name: string) {
     };
   };
 }
+
+// Only for Workers service. The first argument in tracked runner should be the name of the worker
+export function TrackWorker() {
+  return (target: any, propertyKey: string, descriptor: PropertyDescriptor) => {
+    const originalValue = descriptor.value;
+
+    descriptor.value = function (...args: any[]) {
+      // "this" here will refer to the class instance
+      if (!this.prometheus) throw Error(`'${this.constructor.name}' class object must contain 'prometheus' property`);
+      const name = `run-worker-${args[0]}`;
+      const stop = this.prometheus.taskDuration.startTimer({
+        name: name,
+      });
+      this.logger.debug(`Worker '${name}' in progress`);
+      return originalValue
+        .apply(this, args)
+        .then((r: any) => {
+          this.prometheus.taskCount.inc({
+            name: name,
+            status: TaskStatus.COMPLETE,
+          });
+          return r;
+        })
+        .catch((e: Error) => {
+          this.logger.error(`Worker '${name}' ended with an error`, e.stack);
+          this.prometheus.taskCount.inc({
+            name: name,
+            status: TaskStatus.ERROR,
+          });
+          throw e;
+        })
+        .finally(() => {
+          const duration = stop();
+          this.logger.debug(`Worker '${name}' is complete. Duration: ${duration}`);
+        });
+    };
+  };
+}

src/common/prover/duties/slashings.ts → src/common/prover/duties/slashings.service.ts

@@ -1,23 +1,20 @@
-import { ContainerTreeViewType } from '@chainsafe/ssz/lib/view/container';
 import { LOGGER_PROVIDER } from '@lido-nestjs/logger';
 import { Inject, Injectable, LoggerService } from '@nestjs/common';
 
 import { CsmContract } from '../../contracts/csm-contract.service';
 import { VerifierContract } from '../../contracts/verifier-contract.service';
 import { Consensus } from '../../providers/consensus/consensus';
 import { BlockHeaderResponse, BlockInfoResponse } from '../../providers/consensus/response.interface';
-import { generateValidatorProof, toHex, verifyProof } from '../helpers/proofs';
-import { KeyInfo, KeyInfoFn, SlashingProofPayload } from '../types';
+import { WorkersService } from '../../workers/workers.service';
+import { KeyInfo, KeyInfoFn } from '../types';
 
-let ssz: typeof import('@lodestar/types').ssz;
-let anySsz: typeof ssz.phase0 | typeof ssz.altair | typeof ssz.bellatrix | typeof ssz.capella | typeof ssz.deneb;
-
-type InvolvedKeys = { [valIndex: string]: KeyInfo };
+export type InvolvedKeys = { [valIndex: string]: KeyInfo };
 
 @Injectable()
 export class SlashingsService {
   constructor(
     @Inject(LOGGER_PROVIDER) protected readonly logger: LoggerService,
+    protected readonly workers: WorkersService,
     protected readonly consensus: Consensus,
     protected readonly csm: CsmContract,
     protected readonly verifier: VerifierContract,
@@ -48,9 +45,13 @@ export class SlashingsService {
     const finalizedState = await this.consensus.getState(finalizedHeader.header.message.state_root);
     const nextHeader = (await this.consensus.getBeaconHeadersByParentRoot(finalizedHeader.root)).data[0];
     const nextHeaderTs = this.consensus.slotToTimestamp(Number(nextHeader.header.message.slot));
-    const stateView = this.consensus.stateToView(finalizedState.bodyBytes, finalizedState.forkName);
     this.logger.log(`Building slashing proof payloads`);
-    const payloads = this.buildSlashingsProofPayloads(finalizedHeader, nextHeaderTs, stateView, slashings);
+    const payloads = await this.workers.getSlashingProofPayloads({
+      currentHeader: finalizedHeader,
+      nextHeaderTimestamp: nextHeaderTs,
+      state: finalizedState,
+      slashings,
+    });
     for (const payload of payloads) {
       this.logger.log(`📡 Sending slashing proof payload for validator index: ${payload.witness.validatorIndex}`);
       await this.verifier.sendSlashingProof(payload);
@@ -87,43 +88,4 @@ export class SlashingsService {
     }
     return slashed;
   }
-
-  private *buildSlashingsProofPayloads(
-    currentHeader: BlockHeaderResponse,
-    nextHeaderTimestamp: number,
-    stateView: ContainerTreeViewType<typeof anySsz.BeaconState.fields>,
-    slashings: InvolvedKeys,
-  ): Generator<SlashingProofPayload> {
-    for (const [valIndex, keyInfo] of Object.entries(slashings)) {
-      const validator = stateView.validators.getReadonly(Number(valIndex));
-      this.logger.log(`Generating validator [${valIndex}] proof`);
-      const validatorProof = generateValidatorProof(stateView, Number(valIndex));
-      this.logger.log('Verifying validator proof locally');
-      verifyProof(stateView.hashTreeRoot(), validatorProof.gindex, validatorProof.witnesses, validator.hashTreeRoot());
-      yield {
-        keyIndex: keyInfo.keyIndex,
-        nodeOperatorId: keyInfo.operatorId,
-        beaconBlock: {
-          header: {
-            slot: currentHeader.header.message.slot,
-            proposerIndex: Number(currentHeader.header.message.proposer_index),
-            parentRoot: currentHeader.header.message.parent_root,
-            stateRoot: currentHeader.header.message.state_root,
-            bodyRoot: currentHeader.header.message.body_root,
-          },
-          rootsTimestamp: nextHeaderTimestamp,
-        },
-        witness: {
-          validatorIndex: Number(valIndex),
-          withdrawalCredentials: toHex(validator.withdrawalCredentials),
-          effectiveBalance: validator.effectiveBalance,
-          activationEligibilityEpoch: validator.activationEligibilityEpoch,
-          activationEpoch: validator.activationEpoch,
-          exitEpoch: validator.exitEpoch,
-          withdrawableEpoch: validator.withdrawableEpoch,
-          validatorProof: validatorProof.witnesses.map(toHex),
-        },
-      };
-    }
-  }
 }

src/common/prover/duties/withdrawals.service.ts

@@ -0,0 +1,173 @@
+import { LOGGER_PROVIDER } from '@lido-nestjs/logger';
+import { ForkName } from '@lodestar/params';
+import { Inject, Injectable, LoggerService } from '@nestjs/common';
+
+import { CsmContract } from '../../contracts/csm-contract.service';
+import { VerifierContract } from '../../contracts/verifier-contract.service';
+import { Consensus } from '../../providers/consensus/consensus';
+import {
+  BlockHeaderResponse,
+  BlockInfoResponse,
+  RootHex,
+  Withdrawal,
+} from '../../providers/consensus/response.interface';
+import { WorkersService } from '../../workers/workers.service';
+import { KeyInfo, KeyInfoFn } from '../types';
+
+// according to the research https://hackmd.io/1wM8vqeNTjqt4pC3XoCUKQ?view#Proposed-solution
+const FULL_WITHDRAWAL_MIN_AMOUNT = 8 * 10 ** 9; // 8 ETH in Gwei
+
+type WithdrawalWithOffset = Withdrawal & { offset: number };
+export type InvolvedKeysWithWithdrawal = { [valIndex: string]: KeyInfo & { withdrawal: WithdrawalWithOffset } };
+
+@Injectable()
+export class WithdrawalsService {
+  constructor(
+    @Inject(LOGGER_PROVIDER) protected readonly logger: LoggerService,
+    protected readonly workers: WorkersService,
+    protected readonly consensus: Consensus,
+    protected readonly csm: CsmContract,
+    protected readonly verifier: VerifierContract,
+  ) {}
+
+  public async getUnprovenWithdrawals(
+    blockInfo: BlockInfoResponse,
+    keyInfoFn: KeyInfoFn,
+  ): Promise<InvolvedKeysWithWithdrawal> {
+    const withdrawals = this.getFullWithdrawals(blockInfo, keyInfoFn);
+    if (!Object.keys(withdrawals).length) return {};
+    const unproven: InvolvedKeysWithWithdrawal = {};
+    for (const [valIndex, keyWithWithdrawalInfo] of Object.entries(withdrawals)) {
+      const proved = await this.csm.isWithdrawalProved(keyWithWithdrawalInfo);
+      if (!proved) unproven[valIndex] = keyWithWithdrawalInfo;
+    }
+    const unprovenCount = Object.keys(unproven).length;
+    if (!unprovenCount) {
+      this.logger.log('No full withdrawals to prove');
+      return {};
+    }
+    this.logger.warn(`🔍 Unproven full withdrawals: ${unprovenCount}`);
+    return unproven;
+  }
+
+  public async sendWithdrawalProofs(
+    blockRoot: RootHex,
+    blockInfo: BlockInfoResponse,
+    finalizedHeader: BlockHeaderResponse,
+    withdrawals: InvolvedKeysWithWithdrawal,
+  ): Promise<void> {
+    if (!Object.keys(withdrawals).length) return;
+    const blockHeader = await this.consensus.getBeaconHeader(blockRoot);
+    const state = await this.consensus.getState(blockHeader.header.message.state_root);
+    // There is a case when the block is not historical regarding the finalized block, but it is historical
+    // regarding the transaction execution time. This is possible when long finalization time
+    // The transaction will be reverted and the application will try to handle that block again
+    if (this.isHistoricalBlock(blockInfo, finalizedHeader)) {
+      this.logger.warn('It is historical withdrawal. Processing will take longer than usual');
+      await this.sendHistoricalWithdrawalProofs(blockHeader, blockInfo, state, finalizedHeader, withdrawals);
+    } else {
+      await this.sendGeneralWithdrawalProofs(blockHeader, blockInfo, state, withdrawals);
+    }
+  }
+
+  private async sendGeneralWithdrawalProofs(
+    blockHeader: BlockHeaderResponse,
+    blockInfo: BlockInfoResponse,
+    state: { bodyBytes: Uint8Array; forkName: keyof typeof ForkName },
+    withdrawals: InvolvedKeysWithWithdrawal,
+  ): Promise<void> {
+    // create proof against the state with withdrawals
+    const nextBlockHeader = (await this.consensus.getBeaconHeadersByParentRoot(blockHeader.root)).data[0];
+    const nextBlockTs = this.consensus.slotToTimestamp(Number(nextBlockHeader.header.message.slot));
+    this.logger.log(`Building withdrawal proof payloads`);
+    const payloads = await this.workers.getGeneralWithdrawalProofPayloads({
+      currentHeader: blockHeader,
+      nextHeaderTimestamp: nextBlockTs,
+      state,
+      currentBlock: blockInfo,
+      withdrawals,
+      epoch: this.consensus.slotToEpoch(Number(blockHeader.header.message.slot)),
+    });
+    for (const payload of payloads) {
+      this.logger.log(`📡 Sending withdrawal proof payload for validator index: ${payload.witness.validatorIndex}`);
+      await this.verifier.sendWithdrawalProof(payload);
+    }
+  }
+
+  private async sendHistoricalWithdrawalProofs(
+    blockHeader: BlockHeaderResponse,
+    blockInfo: BlockInfoResponse,
+    state: { bodyBytes: Uint8Array; forkName: keyof typeof ForkName },
+    finalizedHeader: BlockHeaderResponse,
+    withdrawals: InvolvedKeysWithWithdrawal,
+  ): Promise<void> {
+    // create proof against the historical state with withdrawals
+    const nextBlockHeader = (await this.consensus.getBeaconHeadersByParentRoot(finalizedHeader.root)).data[0];
+    const nextBlockTs = this.consensus.slotToTimestamp(Number(nextBlockHeader.header.message.slot));
+    const finalizedState = await this.consensus.getState(finalizedHeader.header.message.state_root);
+    const summaryIndex = this.calcSummaryIndex(blockInfo);
+    const summarySlot = this.calcSlotOfSummary(summaryIndex);
+    const summaryState = await this.consensus.getState(summarySlot);
+    this.logger.log('Building historical withdrawal proof payloads');
+    const payloads = await this.workers.getHistoricalWithdrawalProofPayloads({
+      headerWithWds: blockHeader,
+      finalHeader: finalizedHeader,
+      nextToFinalizedHeaderTimestamp: nextBlockTs,
+      finalizedState,
+      summaryState,
+      stateWithWds: state,
+      blockWithWds: blockInfo,
+      summaryIndex,
+      rootIndexInSummary: this.calcRootIndexInSummary(blockInfo),
+      withdrawals,
+      epoch: this.consensus.slotToEpoch(Number(blockHeader.header.message.slot)),
+    });
+    for (const payload of payloads) {
+      this.logger.log(
+        `📡 Sending historical withdrawal proof payload for validator index: ${payload.witness.validatorIndex}`,
+      );
+      await this.verifier.sendHistoricalWithdrawalProof(payload);
+    }
+  }
+
+  private getFullWithdrawals(
+    blockInfo: BlockInfoResponse,
+    keyInfoFn: (valIndex: number) => KeyInfo | undefined,
+  ): InvolvedKeysWithWithdrawal {
+    const fullWithdrawals: InvolvedKeysWithWithdrawal = {};
+    const withdrawals = blockInfo.message.body.execution_payload?.withdrawals ?? [];
+    for (let i = 0; i < withdrawals.length; i++) {
+      const keyInfo = keyInfoFn(Number(withdrawals[i].validator_index));
+      if (!keyInfo) continue;
+      if (Number(withdrawals[i].amount) < FULL_WITHDRAWAL_MIN_AMOUNT) continue;
+      fullWithdrawals[withdrawals[i].validator_index] = { ...keyInfo, withdrawal: { ...withdrawals[i], offset: i } };
+    }
+    return fullWithdrawals;
+  }
+
+  private isHistoricalBlock(blockInfo: BlockInfoResponse, finalizedHeader: BlockHeaderResponse): boolean {
+    const finalizationBufferEpochs = 2;
+    const finalizationBufferSlots = this.consensus.epochToSlot(finalizationBufferEpochs);
+    return (
+      Number(finalizedHeader.header.message.slot) - Number(blockInfo.message.slot) >=
+      Number(this.consensus.beaconConfig.SLOTS_PER_HISTORICAL_ROOT) - finalizationBufferSlots
+    );
+  }
+
+  private calcSummaryIndex(blockInfo: BlockInfoResponse): number {
+    const capellaForkSlot = this.consensus.epochToSlot(Number(this.consensus.beaconConfig.CAPELLA_FORK_EPOCH));
+    const slotsPerHistoricalRoot = Number(this.consensus.beaconConfig.SLOTS_PER_HISTORICAL_ROOT);
+    return Math.floor((Number(blockInfo.message.slot) - capellaForkSlot) / slotsPerHistoricalRoot);
+  }
+
+  private calcSlotOfSummary(summaryIndex: number): number {
+    const capellaForkSlot = this.consensus.epochToSlot(Number(this.consensus.beaconConfig.CAPELLA_FORK_EPOCH));
+    const slotsPerHistoricalRoot = Number(this.consensus.beaconConfig.SLOTS_PER_HISTORICAL_ROOT);
+    return capellaForkSlot + (summaryIndex + 1) * slotsPerHistoricalRoot;
+  }
+
+  private calcRootIndexInSummary(blockInfo: BlockInfoResponse): number {
+    const slotsPerHistoricalRoot = Number(this.consensus.beaconConfig.SLOTS_PER_HISTORICAL_ROOT);
+    return Number(blockInfo.message.slot) % slotsPerHistoricalRoot;
+  }
+}