From 4863847a195626ba7cc1b96e30b17e4adf61c4ab Mon Sep 17 00:00:00 2001 From: Vladimir Gorkavenko <32727352+vgorkavenko@users.noreply.github.com> Date: Thu, 24 Oct 2024 10:57:04 +0200 Subject: [PATCH] feat: gen proofs with workers (#47) * feat: workers * fix: healthcheck props * fix: remove `max_old_space_size` from yarn * feat: `TrackWorker` * fix: review * fix: linter --- Dockerfile | 2 +- package.json | 6 +- src/common/{prover => }/helpers/proofs.ts | 0 src/common/prometheus/prometheus.service.ts | 38 ++ .../{slashings.ts => slashings.service.ts} | 58 +-- .../prover/duties/withdrawals.service.ts | 173 +++++++++ src/common/prover/duties/withdrawals.ts | 352 ------------------ src/common/prover/prover.module.ts | 7 +- src/common/prover/prover.service.ts | 4 +- src/common/providers/consensus/consensus.ts | 33 +- .../items/build-general-wd-proof-payloads.ts | 107 ++++++ .../build-historical-wd-proof-payloads.ts | 168 +++++++++ .../items/build-slashing-proof-payloads.ts | 75 ++++ src/common/workers/items/get-validators.ts | 61 +++ src/common/workers/workers.module.ts | 10 + src/common/workers/workers.service.ts | 114 ++++++ src/daemon/daemon.module.ts | 3 +- src/daemon/services/keys-indexer.ts | 90 ++--- src/daemon/services/roots-processor.ts | 1 - 19 files changed, 803 insertions(+), 499 deletions(-) rename src/common/{prover => }/helpers/proofs.ts (100%) rename src/common/prover/duties/{slashings.ts => slashings.service.ts} (57%) create mode 100644 src/common/prover/duties/withdrawals.service.ts delete mode 100644 src/common/prover/duties/withdrawals.ts create mode 100644 src/common/workers/items/build-general-wd-proof-payloads.ts create mode 100644 src/common/workers/items/build-historical-wd-proof-payloads.ts create mode 100644 src/common/workers/items/build-slashing-proof-payloads.ts create mode 100644 src/common/workers/items/get-validators.ts create mode 100644 src/common/workers/workers.module.ts create mode 100644 src/common/workers/workers.service.ts diff --git a/Dockerfile b/Dockerfile index e85e311..71e31be 100644 --- a/Dockerfile +++ b/Dockerfile @@ -26,7 +26,7 @@ RUN mkdir -p ./storage/ && chown -R node:node ./storage/ USER node -HEALTHCHECK --interval=360s --timeout=120s --retries=3 \ +HEALTHCHECK --interval=60s --timeout=10s --retries=3 \ CMD curl -f http://localhost:$HTTP_PORT/health || exit 1 CMD ["yarn", "start:prod"] diff --git a/package.json b/package.json index 2056963..aa9ea20 100644 --- a/package.json +++ b/package.json @@ -7,8 +7,8 @@ "private": true, "license": "GPL-3.0", "scripts": { - "prove": "NODE_OPTIONS=--max_old_space_size=8192 WORKING_MODE=cli node dist/main prove", - "prove:debug": "NODE_OPTIONS=--max_old_space_size=8192 WORKING_MODE=cli node --inspect dist/main prove", + "prove": "WORKING_MODE=cli node dist/main prove", + "prove:debug": "WORKING_MODE=cli node --inspect dist/main prove", "slashing": "yarn prove slashing", "slashing:debug": "yarn prove:debug slashing", "withdrawal": "yarn prove withdrawal", @@ -19,7 +19,7 @@ "start": "nest start", "start:dev": "nest start --watch", "start:debug": "nest start --debug --watch", - "start:prod": "NODE_OPTIONS=--max_old_space_size=8192 node dist/main", + "start:prod": "node dist/main", "lint": "eslint \"{src,apps,libs,test}/**/*.ts\"", "lint:fix": "eslint \"{src,apps,libs,test}/**/*.ts\" --fix", "test:debug": "node --inspect-brk -r tsconfig-paths/register -r ts-node/register node_modules/.bin/jest --runInBand", diff --git a/src/common/prover/helpers/proofs.ts b/src/common/helpers/proofs.ts similarity index 100% rename from src/common/prover/helpers/proofs.ts rename to src/common/helpers/proofs.ts diff --git a/src/common/prometheus/prometheus.service.ts b/src/common/prometheus/prometheus.service.ts index 2786359..289cc4b 100644 --- a/src/common/prometheus/prometheus.service.ts +++ b/src/common/prometheus/prometheus.service.ts @@ -241,3 +241,41 @@ export function TrackTask(name: string) { }; }; } + +// Only for Workers service. The first argument in tracked runner should be the name of the worker +export function TrackWorker() { + return (target: any, propertyKey: string, descriptor: PropertyDescriptor) => { + const originalValue = descriptor.value; + + descriptor.value = function (...args: any[]) { + // "this" here will refer to the class instance + if (!this.prometheus) throw Error(`'${this.constructor.name}' class object must contain 'prometheus' property`); + const name = `run-worker-${args[0]}`; + const stop = this.prometheus.taskDuration.startTimer({ + name: name, + }); + this.logger.debug(`Worker '${name}' in progress`); + return originalValue + .apply(this, args) + .then((r: any) => { + this.prometheus.taskCount.inc({ + name: name, + status: TaskStatus.COMPLETE, + }); + return r; + }) + .catch((e: Error) => { + this.logger.error(`Worker '${name}' ended with an error`, e.stack); + this.prometheus.taskCount.inc({ + name: name, + status: TaskStatus.ERROR, + }); + throw e; + }) + .finally(() => { + const duration = stop(); + this.logger.debug(`Worker '${name}' is complete. Duration: ${duration}`); + }); + }; + }; +} diff --git a/src/common/prover/duties/slashings.ts b/src/common/prover/duties/slashings.service.ts similarity index 57% rename from src/common/prover/duties/slashings.ts rename to src/common/prover/duties/slashings.service.ts index 9be89a2..fcc8108 100644 --- a/src/common/prover/duties/slashings.ts +++ b/src/common/prover/duties/slashings.service.ts @@ -1,4 +1,3 @@ -import { ContainerTreeViewType } from '@chainsafe/ssz/lib/view/container'; import { LOGGER_PROVIDER } from '@lido-nestjs/logger'; import { Inject, Injectable, LoggerService } from '@nestjs/common'; @@ -6,18 +5,16 @@ import { CsmContract } from '../../contracts/csm-contract.service'; import { VerifierContract } from '../../contracts/verifier-contract.service'; import { Consensus } from '../../providers/consensus/consensus'; import { BlockHeaderResponse, BlockInfoResponse } from '../../providers/consensus/response.interface'; -import { generateValidatorProof, toHex, verifyProof } from '../helpers/proofs'; -import { KeyInfo, KeyInfoFn, SlashingProofPayload } from '../types'; +import { WorkersService } from '../../workers/workers.service'; +import { KeyInfo, KeyInfoFn } from '../types'; -let ssz: typeof import('@lodestar/types').ssz; -let anySsz: typeof ssz.phase0 | typeof ssz.altair | typeof ssz.bellatrix | typeof ssz.capella | typeof ssz.deneb; - -type InvolvedKeys = { [valIndex: string]: KeyInfo }; +export type InvolvedKeys = { [valIndex: string]: KeyInfo }; @Injectable() export class SlashingsService { constructor( @Inject(LOGGER_PROVIDER) protected readonly logger: LoggerService, + protected readonly workers: WorkersService, protected readonly consensus: Consensus, protected readonly csm: CsmContract, protected readonly verifier: VerifierContract, @@ -48,9 +45,13 @@ export class SlashingsService { const finalizedState = await this.consensus.getState(finalizedHeader.header.message.state_root); const nextHeader = (await this.consensus.getBeaconHeadersByParentRoot(finalizedHeader.root)).data[0]; const nextHeaderTs = this.consensus.slotToTimestamp(Number(nextHeader.header.message.slot)); - const stateView = this.consensus.stateToView(finalizedState.bodyBytes, finalizedState.forkName); this.logger.log(`Building slashing proof payloads`); - const payloads = this.buildSlashingsProofPayloads(finalizedHeader, nextHeaderTs, stateView, slashings); + const payloads = await this.workers.getSlashingProofPayloads({ + currentHeader: finalizedHeader, + nextHeaderTimestamp: nextHeaderTs, + state: finalizedState, + slashings, + }); for (const payload of payloads) { this.logger.log(`📡 Sending slashing proof payload for validator index: ${payload.witness.validatorIndex}`); await this.verifier.sendSlashingProof(payload); @@ -87,43 +88,4 @@ export class SlashingsService { } return slashed; } - - private *buildSlashingsProofPayloads( - currentHeader: BlockHeaderResponse, - nextHeaderTimestamp: number, - stateView: ContainerTreeViewType, - slashings: InvolvedKeys, - ): Generator { - for (const [valIndex, keyInfo] of Object.entries(slashings)) { - const validator = stateView.validators.getReadonly(Number(valIndex)); - this.logger.log(`Generating validator [${valIndex}] proof`); - const validatorProof = generateValidatorProof(stateView, Number(valIndex)); - this.logger.log('Verifying validator proof locally'); - verifyProof(stateView.hashTreeRoot(), validatorProof.gindex, validatorProof.witnesses, validator.hashTreeRoot()); - yield { - keyIndex: keyInfo.keyIndex, - nodeOperatorId: keyInfo.operatorId, - beaconBlock: { - header: { - slot: currentHeader.header.message.slot, - proposerIndex: Number(currentHeader.header.message.proposer_index), - parentRoot: currentHeader.header.message.parent_root, - stateRoot: currentHeader.header.message.state_root, - bodyRoot: currentHeader.header.message.body_root, - }, - rootsTimestamp: nextHeaderTimestamp, - }, - witness: { - validatorIndex: Number(valIndex), - withdrawalCredentials: toHex(validator.withdrawalCredentials), - effectiveBalance: validator.effectiveBalance, - activationEligibilityEpoch: validator.activationEligibilityEpoch, - activationEpoch: validator.activationEpoch, - exitEpoch: validator.exitEpoch, - withdrawableEpoch: validator.withdrawableEpoch, - validatorProof: validatorProof.witnesses.map(toHex), - }, - }; - } - } } diff --git a/src/common/prover/duties/withdrawals.service.ts b/src/common/prover/duties/withdrawals.service.ts new file mode 100644 index 0000000..de21d39 --- /dev/null +++ b/src/common/prover/duties/withdrawals.service.ts @@ -0,0 +1,173 @@ +import { LOGGER_PROVIDER } from '@lido-nestjs/logger'; +import { ForkName } from '@lodestar/params'; +import { Inject, Injectable, LoggerService } from '@nestjs/common'; + +import { CsmContract } from '../../contracts/csm-contract.service'; +import { VerifierContract } from '../../contracts/verifier-contract.service'; +import { Consensus } from '../../providers/consensus/consensus'; +import { + BlockHeaderResponse, + BlockInfoResponse, + RootHex, + Withdrawal, +} from '../../providers/consensus/response.interface'; +import { WorkersService } from '../../workers/workers.service'; +import { KeyInfo, KeyInfoFn } from '../types'; + +// according to the research https://hackmd.io/1wM8vqeNTjqt4pC3XoCUKQ?view#Proposed-solution +const FULL_WITHDRAWAL_MIN_AMOUNT = 8 * 10 ** 9; // 8 ETH in Gwei + +type WithdrawalWithOffset = Withdrawal & { offset: number }; +export type InvolvedKeysWithWithdrawal = { [valIndex: string]: KeyInfo & { withdrawal: WithdrawalWithOffset } }; + +@Injectable() +export class WithdrawalsService { + constructor( + @Inject(LOGGER_PROVIDER) protected readonly logger: LoggerService, + protected readonly workers: WorkersService, + protected readonly consensus: Consensus, + protected readonly csm: CsmContract, + protected readonly verifier: VerifierContract, + ) {} + + public async getUnprovenWithdrawals( + blockInfo: BlockInfoResponse, + keyInfoFn: KeyInfoFn, + ): Promise { + const withdrawals = this.getFullWithdrawals(blockInfo, keyInfoFn); + if (!Object.keys(withdrawals).length) return {}; + const unproven: InvolvedKeysWithWithdrawal = {}; + for (const [valIndex, keyWithWithdrawalInfo] of Object.entries(withdrawals)) { + const proved = await this.csm.isWithdrawalProved(keyWithWithdrawalInfo); + if (!proved) unproven[valIndex] = keyWithWithdrawalInfo; + } + const unprovenCount = Object.keys(unproven).length; + if (!unprovenCount) { + this.logger.log('No full withdrawals to prove'); + return {}; + } + this.logger.warn(`🔍 Unproven full withdrawals: ${unprovenCount}`); + return unproven; + } + + public async sendWithdrawalProofs( + blockRoot: RootHex, + blockInfo: BlockInfoResponse, + finalizedHeader: BlockHeaderResponse, + withdrawals: InvolvedKeysWithWithdrawal, + ): Promise { + if (!Object.keys(withdrawals).length) return; + const blockHeader = await this.consensus.getBeaconHeader(blockRoot); + const state = await this.consensus.getState(blockHeader.header.message.state_root); + // There is a case when the block is not historical regarding the finalized block, but it is historical + // regarding the transaction execution time. This is possible when long finalization time + // The transaction will be reverted and the application will try to handle that block again + if (this.isHistoricalBlock(blockInfo, finalizedHeader)) { + this.logger.warn('It is historical withdrawal. Processing will take longer than usual'); + await this.sendHistoricalWithdrawalProofs(blockHeader, blockInfo, state, finalizedHeader, withdrawals); + } else { + await this.sendGeneralWithdrawalProofs(blockHeader, blockInfo, state, withdrawals); + } + } + + private async sendGeneralWithdrawalProofs( + blockHeader: BlockHeaderResponse, + blockInfo: BlockInfoResponse, + state: { bodyBytes: Uint8Array; forkName: keyof typeof ForkName }, + withdrawals: InvolvedKeysWithWithdrawal, + ): Promise { + // create proof against the state with withdrawals + const nextBlockHeader = (await this.consensus.getBeaconHeadersByParentRoot(blockHeader.root)).data[0]; + const nextBlockTs = this.consensus.slotToTimestamp(Number(nextBlockHeader.header.message.slot)); + this.logger.log(`Building withdrawal proof payloads`); + const payloads = await this.workers.getGeneralWithdrawalProofPayloads({ + currentHeader: blockHeader, + nextHeaderTimestamp: nextBlockTs, + state, + currentBlock: blockInfo, + withdrawals, + epoch: this.consensus.slotToEpoch(Number(blockHeader.header.message.slot)), + }); + for (const payload of payloads) { + this.logger.log(`📡 Sending withdrawal proof payload for validator index: ${payload.witness.validatorIndex}`); + await this.verifier.sendWithdrawalProof(payload); + } + } + + private async sendHistoricalWithdrawalProofs( + blockHeader: BlockHeaderResponse, + blockInfo: BlockInfoResponse, + state: { bodyBytes: Uint8Array; forkName: keyof typeof ForkName }, + finalizedHeader: BlockHeaderResponse, + withdrawals: InvolvedKeysWithWithdrawal, + ): Promise { + // create proof against the historical state with withdrawals + const nextBlockHeader = (await this.consensus.getBeaconHeadersByParentRoot(finalizedHeader.root)).data[0]; + const nextBlockTs = this.consensus.slotToTimestamp(Number(nextBlockHeader.header.message.slot)); + const finalizedState = await this.consensus.getState(finalizedHeader.header.message.state_root); + const summaryIndex = this.calcSummaryIndex(blockInfo); + const summarySlot = this.calcSlotOfSummary(summaryIndex); + const summaryState = await this.consensus.getState(summarySlot); + this.logger.log('Building historical withdrawal proof payloads'); + const payloads = await this.workers.getHistoricalWithdrawalProofPayloads({ + headerWithWds: blockHeader, + finalHeader: finalizedHeader, + nextToFinalizedHeaderTimestamp: nextBlockTs, + finalizedState, + summaryState, + stateWithWds: state, + blockWithWds: blockInfo, + summaryIndex, + rootIndexInSummary: this.calcRootIndexInSummary(blockInfo), + withdrawals, + epoch: this.consensus.slotToEpoch(Number(blockHeader.header.message.slot)), + }); + for (const payload of payloads) { + this.logger.log( + `📡 Sending historical withdrawal proof payload for validator index: ${payload.witness.validatorIndex}`, + ); + await this.verifier.sendHistoricalWithdrawalProof(payload); + } + } + + private getFullWithdrawals( + blockInfo: BlockInfoResponse, + keyInfoFn: (valIndex: number) => KeyInfo | undefined, + ): InvolvedKeysWithWithdrawal { + const fullWithdrawals: InvolvedKeysWithWithdrawal = {}; + const withdrawals = blockInfo.message.body.execution_payload?.withdrawals ?? []; + for (let i = 0; i < withdrawals.length; i++) { + const keyInfo = keyInfoFn(Number(withdrawals[i].validator_index)); + if (!keyInfo) continue; + if (Number(withdrawals[i].amount) < FULL_WITHDRAWAL_MIN_AMOUNT) continue; + fullWithdrawals[withdrawals[i].validator_index] = { ...keyInfo, withdrawal: { ...withdrawals[i], offset: i } }; + } + return fullWithdrawals; + } + + private isHistoricalBlock(blockInfo: BlockInfoResponse, finalizedHeader: BlockHeaderResponse): boolean { + const finalizationBufferEpochs = 2; + const finalizationBufferSlots = this.consensus.epochToSlot(finalizationBufferEpochs); + return ( + Number(finalizedHeader.header.message.slot) - Number(blockInfo.message.slot) >= + Number(this.consensus.beaconConfig.SLOTS_PER_HISTORICAL_ROOT) - finalizationBufferSlots + ); + } + + private calcSummaryIndex(blockInfo: BlockInfoResponse): number { + const capellaForkSlot = this.consensus.epochToSlot(Number(this.consensus.beaconConfig.CAPELLA_FORK_EPOCH)); + const slotsPerHistoricalRoot = Number(this.consensus.beaconConfig.SLOTS_PER_HISTORICAL_ROOT); + return Math.floor((Number(blockInfo.message.slot) - capellaForkSlot) / slotsPerHistoricalRoot); + } + + private calcSlotOfSummary(summaryIndex: number): number { + const capellaForkSlot = this.consensus.epochToSlot(Number(this.consensus.beaconConfig.CAPELLA_FORK_EPOCH)); + const slotsPerHistoricalRoot = Number(this.consensus.beaconConfig.SLOTS_PER_HISTORICAL_ROOT); + return capellaForkSlot + (summaryIndex + 1) * slotsPerHistoricalRoot; + } + + private calcRootIndexInSummary(blockInfo: BlockInfoResponse): number { + const slotsPerHistoricalRoot = Number(this.consensus.beaconConfig.SLOTS_PER_HISTORICAL_ROOT); + return Number(blockInfo.message.slot) % slotsPerHistoricalRoot; + } +} diff --git a/src/common/prover/duties/withdrawals.ts b/src/common/prover/duties/withdrawals.ts deleted file mode 100644 index f22c433..0000000 --- a/src/common/prover/duties/withdrawals.ts +++ /dev/null @@ -1,352 +0,0 @@ -import { ContainerTreeViewType } from '@chainsafe/ssz/lib/view/container'; -import { LOGGER_PROVIDER } from '@lido-nestjs/logger'; -import { ForkName } from '@lodestar/params'; -import { Inject, Injectable, LoggerService } from '@nestjs/common'; - -import { CsmContract } from '../../contracts/csm-contract.service'; -import { VerifierContract } from '../../contracts/verifier-contract.service'; -import { Consensus } from '../../providers/consensus/consensus'; -import { - BlockHeaderResponse, - BlockInfoResponse, - RootHex, - Withdrawal, -} from '../../providers/consensus/response.interface'; -import { - generateHistoricalStateProof, - generateValidatorProof, - generateWithdrawalProof, - toHex, - verifyProof, -} from '../helpers/proofs'; -import { HistoricalWithdrawalsProofPayload, KeyInfo, KeyInfoFn, WithdrawalsProofPayload } from '../types'; - -let ssz: typeof import('@lodestar/types').ssz; -let anySsz: typeof ssz.phase0 | typeof ssz.altair | typeof ssz.bellatrix | typeof ssz.capella | typeof ssz.deneb; - -// according to the research https://hackmd.io/1wM8vqeNTjqt4pC3XoCUKQ?view#Proposed-solution -const FULL_WITHDRAWAL_MIN_AMOUNT = 8 * 10 ** 9; // 8 ETH in Gwei - -type WithdrawalWithOffset = Withdrawal & { offset: number }; -type InvolvedKeysWithWithdrawal = { [valIndex: string]: KeyInfo & { withdrawal: WithdrawalWithOffset } }; - -@Injectable() -export class WithdrawalsService { - constructor( - @Inject(LOGGER_PROVIDER) protected readonly logger: LoggerService, - protected readonly consensus: Consensus, - protected readonly csm: CsmContract, - protected readonly verifier: VerifierContract, - ) {} - - public async getUnprovenWithdrawals( - blockInfo: BlockInfoResponse, - keyInfoFn: KeyInfoFn, - ): Promise { - const withdrawals = this.getFullWithdrawals(blockInfo, keyInfoFn); - if (!Object.keys(withdrawals).length) return {}; - const unproven: InvolvedKeysWithWithdrawal = {}; - for (const [valIndex, keyWithWithdrawalInfo] of Object.entries(withdrawals)) { - const proved = await this.csm.isWithdrawalProved(keyWithWithdrawalInfo); - if (!proved) unproven[valIndex] = keyWithWithdrawalInfo; - } - const unprovenCount = Object.keys(unproven).length; - if (!unprovenCount) { - this.logger.log('No full withdrawals to prove'); - return {}; - } - this.logger.warn(`🔍 Unproven full withdrawals: ${unprovenCount}`); - return unproven; - } - - public async sendWithdrawalProofs( - blockRoot: RootHex, - blockInfo: BlockInfoResponse, - finalizedHeader: BlockHeaderResponse, - withdrawals: InvolvedKeysWithWithdrawal, - ): Promise { - if (!Object.keys(withdrawals).length) return; - const blockHeader = await this.consensus.getBeaconHeader(blockRoot); - const state = await this.consensus.getState(blockHeader.header.message.state_root); - // There is a case when the block is not historical regarding the finalized block, but it is historical - // regarding the transaction execution time. This is possible when long finalization time - // The transaction will be reverted and the application will try to handle that block again - if (this.isHistoricalBlock(blockInfo, finalizedHeader)) { - this.logger.warn('It is historical withdrawal. Processing will take longer than usual'); - await this.sendHistoricalWithdrawalProofs(blockHeader, blockInfo, state, finalizedHeader, withdrawals); - } else { - await this.sendGeneralWithdrawalProofs(blockHeader, blockInfo, state, withdrawals); - } - } - - private async sendGeneralWithdrawalProofs( - blockHeader: BlockHeaderResponse, - blockInfo: BlockInfoResponse, - state: { bodyBytes: Uint8Array; forkName: keyof typeof ForkName }, - withdrawals: InvolvedKeysWithWithdrawal, - ): Promise { - // create proof against the state with withdrawals - const nextBlockHeader = (await this.consensus.getBeaconHeadersByParentRoot(blockHeader.root)).data[0]; - const nextBlockTs = this.consensus.slotToTimestamp(Number(nextBlockHeader.header.message.slot)); - this.logger.log(`Building withdrawal proof payloads`); - const payloads = this.buildWithdrawalsProofPayloads( - blockHeader, - nextBlockTs, - this.consensus.stateToView(state.bodyBytes, state.forkName), - this.consensus.blockToView(blockInfo, state.forkName), - withdrawals, - ); - for (const payload of payloads) { - this.logger.log(`📡 Sending withdrawal proof payload for validator index: ${payload.witness.validatorIndex}`); - await this.verifier.sendWithdrawalProof(payload); - } - } - - private async sendHistoricalWithdrawalProofs( - blockHeader: BlockHeaderResponse, - blockInfo: BlockInfoResponse, - state: { bodyBytes: Uint8Array; forkName: keyof typeof ForkName }, - finalizedHeader: BlockHeaderResponse, - withdrawals: InvolvedKeysWithWithdrawal, - ): Promise { - // create proof against the historical state with withdrawals - const nextBlockHeader = (await this.consensus.getBeaconHeadersByParentRoot(finalizedHeader.root)).data[0]; - const nextBlockTs = this.consensus.slotToTimestamp(Number(nextBlockHeader.header.message.slot)); - const finalizedState = await this.consensus.getState(finalizedHeader.header.message.state_root); - const summaryIndex = this.calcSummaryIndex(blockInfo); - const summarySlot = this.calcSlotOfSummary(summaryIndex); - const summaryState = await this.consensus.getState(summarySlot); - this.logger.log(`Building historical withdrawal proof payloads`); - const payloads = this.buildHistoricalWithdrawalsProofPayloads( - blockHeader, - finalizedHeader, - nextBlockTs, - this.consensus.stateToView(finalizedState.bodyBytes, finalizedState.forkName), - this.consensus.stateToView(summaryState.bodyBytes, summaryState.forkName), - this.consensus.stateToView(state.bodyBytes, state.forkName), - this.consensus.blockToView(blockInfo, state.forkName), - summaryIndex, - this.calcRootIndexInSummary(blockInfo), - withdrawals, - ); - for (const payload of payloads) { - this.logger.log( - `📡 Sending historical withdrawal proof payload for validator index: ${payload.witness.validatorIndex}`, - ); - await this.verifier.sendHistoricalWithdrawalProof(payload); - } - } - - private getFullWithdrawals( - blockInfo: BlockInfoResponse, - keyInfoFn: (valIndex: number) => KeyInfo | undefined, - ): InvolvedKeysWithWithdrawal { - const fullWithdrawals: InvolvedKeysWithWithdrawal = {}; - const withdrawals = blockInfo.message.body.execution_payload?.withdrawals ?? []; - for (let i = 0; i < withdrawals.length; i++) { - const keyInfo = keyInfoFn(Number(withdrawals[i].validator_index)); - if (!keyInfo) continue; - if (Number(withdrawals[i].amount) < FULL_WITHDRAWAL_MIN_AMOUNT) continue; - fullWithdrawals[withdrawals[i].validator_index] = { ...keyInfo, withdrawal: { ...withdrawals[i], offset: i } }; - } - return fullWithdrawals; - } - - private *buildWithdrawalsProofPayloads( - currentHeader: BlockHeaderResponse, - nextHeaderTimestamp: number, - stateView: ContainerTreeViewType, - currentBlockView: ContainerTreeViewType, - withdrawals: InvolvedKeysWithWithdrawal, - ): Generator { - const epoch = this.consensus.slotToEpoch(Number(currentHeader.header.message.slot)); - for (const [valIndex, keyWithWithdrawalInfo] of Object.entries(withdrawals)) { - const validator = stateView.validators.getReadonly(Number(valIndex)); - if (epoch < validator.withdrawableEpoch) { - this.logger.warn(`Validator ${valIndex} is not full withdrawn. Just huge amount of ETH. Skipped`); - continue; - } - this.logger.log(`Generating validator [${valIndex}] proof`); - const validatorProof = generateValidatorProof(stateView, Number(valIndex)); - this.logger.log('Generating withdrawal proof'); - const withdrawalProof = generateWithdrawalProof( - stateView, - currentBlockView, - keyWithWithdrawalInfo.withdrawal.offset, - ); - this.logger.log('Verifying validator proof locally'); - verifyProof(stateView.hashTreeRoot(), validatorProof.gindex, validatorProof.witnesses, validator.hashTreeRoot()); - this.logger.log('Verifying withdrawal proof locally'); - verifyProof( - stateView.hashTreeRoot(), - withdrawalProof.gindex, - withdrawalProof.witnesses, - ( - currentBlockView as ContainerTreeViewType - ).body.executionPayload.withdrawals - .getReadonly(keyWithWithdrawalInfo.withdrawal.offset) - .hashTreeRoot(), - ); - yield { - keyIndex: keyWithWithdrawalInfo.keyIndex, - nodeOperatorId: keyWithWithdrawalInfo.operatorId, - beaconBlock: { - header: { - slot: currentHeader.header.message.slot, - proposerIndex: Number(currentHeader.header.message.proposer_index), - parentRoot: currentHeader.header.message.parent_root, - stateRoot: currentHeader.header.message.state_root, - bodyRoot: currentHeader.header.message.body_root, - }, - rootsTimestamp: nextHeaderTimestamp, - }, - witness: { - withdrawalOffset: Number(keyWithWithdrawalInfo.withdrawal.offset), - withdrawalIndex: Number(keyWithWithdrawalInfo.withdrawal.index), - validatorIndex: Number(keyWithWithdrawalInfo.withdrawal.validator_index), - amount: Number(keyWithWithdrawalInfo.withdrawal.amount), - withdrawalCredentials: toHex(validator.withdrawalCredentials), - effectiveBalance: validator.effectiveBalance, - slashed: Boolean(validator.slashed), - activationEligibilityEpoch: validator.activationEligibilityEpoch, - activationEpoch: validator.activationEpoch, - exitEpoch: validator.exitEpoch, - withdrawableEpoch: validator.withdrawableEpoch, - withdrawalProof: withdrawalProof.witnesses.map(toHex), - validatorProof: validatorProof.witnesses.map(toHex), - }, - }; - } - } - - private *buildHistoricalWithdrawalsProofPayloads( - headerWithWds: BlockHeaderResponse, - finalHeader: BlockHeaderResponse, - nextToFinalizedHeaderTimestamp: number, - finalizedStateView: ContainerTreeViewType, - summaryStateView: ContainerTreeViewType, - stateWithWdsView: ContainerTreeViewType, - blockWithWdsView: ContainerTreeViewType, - summaryIndex: number, - rootIndexInSummary: number, - withdrawals: InvolvedKeysWithWithdrawal, - ): Generator { - const epoch = this.consensus.slotToEpoch(Number(headerWithWds.header.message.slot)); - for (const [valIndex, keyWithWithdrawalInfo] of Object.entries(withdrawals)) { - const validator = stateWithWdsView.validators.getReadonly(Number(valIndex)); - if (epoch < validator.withdrawableEpoch) { - this.logger.warn(`Validator ${valIndex} is not full withdrawn. Just huge amount of ETH. Skipped`); - continue; - } - this.logger.log(`Generating validator [${valIndex}] proof`); - const validatorProof = generateValidatorProof(stateWithWdsView, Number(valIndex)); - this.logger.log('Generating withdrawal proof'); - const withdrawalProof = generateWithdrawalProof( - stateWithWdsView, - blockWithWdsView, - keyWithWithdrawalInfo.withdrawal.offset, - ); - this.logger.log('Generating historical state proof'); - const historicalStateProof = generateHistoricalStateProof( - finalizedStateView, - summaryStateView, - summaryIndex, - rootIndexInSummary, - ); - this.logger.log('Verifying validator proof locally'); - verifyProof( - stateWithWdsView.hashTreeRoot(), - validatorProof.gindex, - validatorProof.witnesses, - validator.hashTreeRoot(), - ); - this.logger.log('Verifying withdrawal proof locally'); - verifyProof( - stateWithWdsView.hashTreeRoot(), - withdrawalProof.gindex, - withdrawalProof.witnesses, - ( - blockWithWdsView as ContainerTreeViewType - ).body.executionPayload.withdrawals - .getReadonly(keyWithWithdrawalInfo.withdrawal.offset) - .hashTreeRoot(), - ); - this.logger.log('Verifying historical state proof locally'); - verifyProof( - finalizedStateView.hashTreeRoot(), - historicalStateProof.gindex, - historicalStateProof.witnesses, - (summaryStateView as ContainerTreeViewType).blockRoots.getReadonly( - rootIndexInSummary, - ), - ); - yield { - keyIndex: keyWithWithdrawalInfo.keyIndex, - nodeOperatorId: keyWithWithdrawalInfo.operatorId, - beaconBlock: { - header: { - slot: finalHeader.header.message.slot, - proposerIndex: Number(finalHeader.header.message.proposer_index), - parentRoot: finalHeader.header.message.parent_root, - stateRoot: finalHeader.header.message.state_root, - bodyRoot: finalHeader.header.message.body_root, - }, - rootsTimestamp: nextToFinalizedHeaderTimestamp, - }, - oldBlock: { - header: { - slot: headerWithWds.header.message.slot, - proposerIndex: Number(headerWithWds.header.message.proposer_index), - parentRoot: headerWithWds.header.message.parent_root, - stateRoot: headerWithWds.header.message.state_root, - bodyRoot: headerWithWds.header.message.body_root, - }, - // NOTE: the last byte can be changed due to `CSVerifier` implementation in the future - rootGIndex: '0x' + (historicalStateProof.gindex.toString(16) + '00').padStart(64, '0'), - proof: historicalStateProof.witnesses.map(toHex), - }, - witness: { - withdrawalOffset: Number(keyWithWithdrawalInfo.withdrawal.offset), - withdrawalIndex: Number(keyWithWithdrawalInfo.withdrawal.index), - validatorIndex: Number(keyWithWithdrawalInfo.withdrawal.validator_index), - amount: Number(keyWithWithdrawalInfo.withdrawal.amount), - withdrawalCredentials: toHex(validator.withdrawalCredentials), - effectiveBalance: validator.effectiveBalance, - slashed: Boolean(validator.slashed), - activationEligibilityEpoch: validator.activationEligibilityEpoch, - activationEpoch: validator.activationEpoch, - exitEpoch: validator.exitEpoch, - withdrawableEpoch: validator.withdrawableEpoch, - withdrawalProof: withdrawalProof.witnesses.map(toHex), - validatorProof: validatorProof.witnesses.map(toHex), - }, - }; - } - } - - private isHistoricalBlock(blockInfo: BlockInfoResponse, finalizedHeader: BlockHeaderResponse): boolean { - const finalizationBufferEpochs = 2; - const finalizationBufferSlots = this.consensus.epochToSlot(finalizationBufferEpochs); - return ( - Number(finalizedHeader.header.message.slot) - Number(blockInfo.message.slot) > - Number(this.consensus.beaconConfig.SLOTS_PER_HISTORICAL_ROOT) - finalizationBufferSlots - ); - } - - private calcSummaryIndex(blockInfo: BlockInfoResponse): number { - const capellaForkSlot = this.consensus.epochToSlot(Number(this.consensus.beaconConfig.CAPELLA_FORK_EPOCH)); - const slotsPerHistoricalRoot = Number(this.consensus.beaconConfig.SLOTS_PER_HISTORICAL_ROOT); - return Math.floor((Number(blockInfo.message.slot) - capellaForkSlot) / slotsPerHistoricalRoot); - } - - private calcSlotOfSummary(summaryIndex: number): number { - const capellaForkSlot = this.consensus.epochToSlot(Number(this.consensus.beaconConfig.CAPELLA_FORK_EPOCH)); - const slotsPerHistoricalRoot = Number(this.consensus.beaconConfig.SLOTS_PER_HISTORICAL_ROOT); - return capellaForkSlot + (summaryIndex + 1) * slotsPerHistoricalRoot; - } - - private calcRootIndexInSummary(blockInfo: BlockInfoResponse): number { - const slotsPerHistoricalRoot = Number(this.consensus.beaconConfig.SLOTS_PER_HISTORICAL_ROOT); - return Number(blockInfo.message.slot) % slotsPerHistoricalRoot; - } -} diff --git a/src/common/prover/prover.module.ts b/src/common/prover/prover.module.ts index a98901e..ea7aa44 100644 --- a/src/common/prover/prover.module.ts +++ b/src/common/prover/prover.module.ts @@ -1,13 +1,14 @@ import { Module } from '@nestjs/common'; -import { SlashingsService } from './duties/slashings'; -import { WithdrawalsService } from './duties/withdrawals'; +import { SlashingsService } from './duties/slashings.service'; +import { WithdrawalsService } from './duties/withdrawals.service'; import { ProverService } from './prover.service'; import { ContractsModule } from '../contracts/contracts.module'; import { ProvidersModule } from '../providers/providers.module'; +import { WorkersModule } from '../workers/workers.module'; @Module({ - imports: [ProvidersModule, ContractsModule], + imports: [ProvidersModule, ContractsModule, WorkersModule], providers: [ProverService, SlashingsService, WithdrawalsService], exports: [ProverService], }) diff --git a/src/common/prover/prover.service.ts b/src/common/prover/prover.service.ts index d7cd417..b23d0c1 100644 --- a/src/common/prover/prover.service.ts +++ b/src/common/prover/prover.service.ts @@ -1,8 +1,8 @@ import { LOGGER_PROVIDER } from '@lido-nestjs/logger'; import { Inject, Injectable, LoggerService } from '@nestjs/common'; -import { SlashingsService } from './duties/slashings'; -import { WithdrawalsService } from './duties/withdrawals'; +import { SlashingsService } from './duties/slashings.service'; +import { WithdrawalsService } from './duties/withdrawals.service'; import { KeyInfoFn } from './types'; import { Consensus } from '../providers/consensus/consensus'; import { BlockHeaderResponse, BlockInfoResponse, RootHex } from '../providers/consensus/response.interface'; diff --git a/src/common/providers/consensus/consensus.ts b/src/common/providers/consensus/consensus.ts index b171c67..6acfbf2 100644 --- a/src/common/providers/consensus/consensus.ts +++ b/src/common/providers/consensus/consensus.ts @@ -1,4 +1,3 @@ -import { ContainerTreeViewType } from '@chainsafe/ssz/lib/view/container'; import { LOGGER_PROVIDER } from '@lido-nestjs/logger'; import { Inject, Injectable, LoggerService, OnModuleInit, Optional } from '@nestjs/common'; import { promise as spinnerFor } from 'ora-classic'; @@ -20,10 +19,13 @@ import { DownloadProgress } from '../../utils/download-progress/download-progres import { BaseRestProvider } from '../base/rest-provider'; import { RequestOptions } from '../base/utils/func'; -let ssz: typeof import('@lodestar/types').ssz; -let anySsz: typeof ssz.phase0 | typeof ssz.altair | typeof ssz.bellatrix | typeof ssz.capella | typeof ssz.deneb; let ForkName: typeof import('@lodestar/params').ForkName; +export interface State { + bodyBytes: Uint8Array; + forkName: keyof typeof ForkName; +} + @Injectable() export class Consensus extends BaseRestProvider implements OnModuleInit { private readonly endpoints = { @@ -57,8 +59,6 @@ export class Consensus extends BaseRestProvider implements OnModuleInit { } public async onModuleInit(): Promise { - // ugly hack to import ESModule to CommonJS project - ssz = await eval(`import('@lodestar/types').then((m) => m.ssz)`); this.logger.log(`Getting genesis timestamp`); const genesis = await this.getGenesis(); this.genesisTimestamp = Number(genesis.genesis_time); @@ -110,10 +110,7 @@ export class Consensus extends BaseRestProvider implements OnModuleInit { return (await body.json()) as { finalized: boolean; data: BlockHeaderResponse[] }; } - public async getState( - stateId: StateId, - signal?: AbortSignal, - ): Promise<{ bodyBytes: Uint8Array; forkName: keyof typeof ForkName }> { + public async getState(stateId: StateId, signal?: AbortSignal): Promise { const requestPromise = this.retryRequest(async (baseUrl) => this.baseGet(baseUrl, this.endpoints.state(stateId), { signal, @@ -140,22 +137,4 @@ export class Consensus extends BaseRestProvider implements OnModuleInit { ): Promise<{ body: BodyReadable; headers: IncomingHttpHeaders }> { return super.baseGet(baseUrl, endpoint, options); } - - public stateToView( - bodyBytes: Uint8Array, - forkName: keyof typeof ForkName, - ): ContainerTreeViewType { - return ssz[forkName].BeaconState.deserializeToView(bodyBytes) as ContainerTreeViewType< - typeof anySsz.BeaconState.fields - >; - } - - public blockToView( - body: BlockInfoResponse, - forkName: keyof typeof ForkName, - ): ContainerTreeViewType { - return ssz[forkName].BeaconBlock.toView( - ssz[forkName].BeaconBlock.fromJson(body.message) as any, - ) as ContainerTreeViewType; - } } diff --git a/src/common/workers/items/build-general-wd-proof-payloads.ts b/src/common/workers/items/build-general-wd-proof-payloads.ts new file mode 100644 index 0000000..f6dda19 --- /dev/null +++ b/src/common/workers/items/build-general-wd-proof-payloads.ts @@ -0,0 +1,107 @@ +import { parentPort, workerData } from 'node:worker_threads'; + +import { ContainerTreeViewType } from '@chainsafe/ssz/lib/view/container'; + +import { generateValidatorProof, generateWithdrawalProof, toHex, verifyProof } from '../../helpers/proofs'; +import { InvolvedKeysWithWithdrawal } from '../../prover/duties/withdrawals.service'; +import { WithdrawalsProofPayload } from '../../prover/types'; +import { State } from '../../providers/consensus/consensus'; +import { BlockHeaderResponse, BlockInfoResponse } from '../../providers/consensus/response.interface'; +import { WorkerLogger } from '../workers.service'; + +let ssz: typeof import('@lodestar/types').ssz; +let anySsz: typeof ssz.phase0 | typeof ssz.altair | typeof ssz.bellatrix | typeof ssz.capella | typeof ssz.deneb; +let ForkName: typeof import('@lodestar/params').ForkName; + +export type BuildGeneralWithdrawalProofArgs = { + currentHeader: BlockHeaderResponse; + nextHeaderTimestamp: number; + state: State; + currentBlock: BlockInfoResponse; + withdrawals: InvolvedKeysWithWithdrawal; + epoch: number; +}; + +async function buildGeneralWithdrawalsProofPayloads(): Promise { + ssz = await eval(`import('@lodestar/types').then((m) => m.ssz)`); + const { currentHeader, nextHeaderTimestamp, state, currentBlock, withdrawals, epoch } = + workerData as BuildGeneralWithdrawalProofArgs; + // + // Get views + // + const stateView = ssz[state.forkName as keyof typeof ForkName].BeaconState.deserializeToView( + state.bodyBytes, + ) as ContainerTreeViewType; + const currentBlockView = ssz[state.forkName as keyof typeof ForkName].BeaconBlock.toView( + ssz[state.forkName as keyof typeof ForkName].BeaconBlock.fromJson(currentBlock.message) as any, + ) as ContainerTreeViewType; + // + // + // + const payloads = []; + for (const [valIndex, keyWithWithdrawalInfo] of Object.entries(withdrawals)) { + const validator = stateView.validators.getReadonly(Number(valIndex)); + if (epoch < validator.withdrawableEpoch) { + WorkerLogger.warn(`Validator ${valIndex} is not full withdrawn. Just huge amount of ETH. Skipped`); + continue; + } + WorkerLogger.log(`Generating validator [${valIndex}] proof`); + const validatorProof = generateValidatorProof(stateView, Number(valIndex)); + WorkerLogger.log('Generating withdrawal proof'); + const withdrawalProof = generateWithdrawalProof( + stateView, + currentBlockView, + keyWithWithdrawalInfo.withdrawal.offset, + ); + WorkerLogger.log('Verifying validator proof locally'); + verifyProof(stateView.hashTreeRoot(), validatorProof.gindex, validatorProof.witnesses, validator.hashTreeRoot()); + WorkerLogger.log('Verifying withdrawal proof locally'); + verifyProof( + stateView.hashTreeRoot(), + withdrawalProof.gindex, + withdrawalProof.witnesses, + ( + currentBlockView as ContainerTreeViewType + ).body.executionPayload.withdrawals + .getReadonly(keyWithWithdrawalInfo.withdrawal.offset) + .hashTreeRoot(), + ); + payloads.push({ + keyIndex: keyWithWithdrawalInfo.keyIndex, + nodeOperatorId: keyWithWithdrawalInfo.operatorId, + beaconBlock: { + header: { + slot: currentHeader.header.message.slot, + proposerIndex: Number(currentHeader.header.message.proposer_index), + parentRoot: currentHeader.header.message.parent_root, + stateRoot: currentHeader.header.message.state_root, + bodyRoot: currentHeader.header.message.body_root, + }, + rootsTimestamp: nextHeaderTimestamp, + }, + witness: { + withdrawalOffset: Number(keyWithWithdrawalInfo.withdrawal.offset), + withdrawalIndex: Number(keyWithWithdrawalInfo.withdrawal.index), + validatorIndex: Number(keyWithWithdrawalInfo.withdrawal.validator_index), + amount: Number(keyWithWithdrawalInfo.withdrawal.amount), + withdrawalCredentials: toHex(validator.withdrawalCredentials), + effectiveBalance: validator.effectiveBalance, + slashed: Boolean(validator.slashed), + activationEligibilityEpoch: validator.activationEligibilityEpoch, + activationEpoch: validator.activationEpoch, + exitEpoch: validator.exitEpoch, + withdrawableEpoch: validator.withdrawableEpoch, + withdrawalProof: withdrawalProof.witnesses.map(toHex), + validatorProof: validatorProof.witnesses.map(toHex), + }, + }); + } + return payloads; +} + +buildGeneralWithdrawalsProofPayloads() + .then((v) => parentPort?.postMessage(v)) + .catch((e) => { + console.error(e); + throw e; + }); diff --git a/src/common/workers/items/build-historical-wd-proof-payloads.ts b/src/common/workers/items/build-historical-wd-proof-payloads.ts new file mode 100644 index 0000000..8bf23a4 --- /dev/null +++ b/src/common/workers/items/build-historical-wd-proof-payloads.ts @@ -0,0 +1,168 @@ +import { parentPort, workerData } from 'node:worker_threads'; + +import { ContainerTreeViewType } from '@chainsafe/ssz/lib/view/container'; + +import { + generateHistoricalStateProof, + generateValidatorProof, + generateWithdrawalProof, + toHex, + verifyProof, +} from '../../helpers/proofs'; +import { InvolvedKeysWithWithdrawal } from '../../prover/duties/withdrawals.service'; +import { HistoricalWithdrawalsProofPayload } from '../../prover/types'; +import { State } from '../../providers/consensus/consensus'; +import { BlockHeaderResponse, BlockInfoResponse } from '../../providers/consensus/response.interface'; +import { WorkerLogger } from '../workers.service'; + +let ssz: typeof import('@lodestar/types').ssz; +let anySsz: typeof ssz.phase0 | typeof ssz.altair | typeof ssz.bellatrix | typeof ssz.capella | typeof ssz.deneb; +let ForkName: typeof import('@lodestar/params').ForkName; + +export type BuildHistoricalWithdrawalProofArgs = { + headerWithWds: BlockHeaderResponse; + finalHeader: BlockHeaderResponse; + nextToFinalizedHeaderTimestamp: number; + finalizedState: State; + summaryState: State; + stateWithWds: State; + blockWithWds: BlockInfoResponse; + summaryIndex: number; + rootIndexInSummary: number; + withdrawals: InvolvedKeysWithWithdrawal; + epoch: number; +}; + +async function buildHistoricalWithdrawalsProofPayloads(): Promise { + ssz = await eval(`import('@lodestar/types').then((m) => m.ssz)`); + const { + headerWithWds, + finalHeader, + nextToFinalizedHeaderTimestamp, + finalizedState, + summaryState, + stateWithWds, + blockWithWds, + summaryIndex, + rootIndexInSummary, + withdrawals, + epoch, + } = workerData as BuildHistoricalWithdrawalProofArgs; + // + // Get views + // + const finalizedStateView = ssz[finalizedState.forkName as keyof typeof ForkName].BeaconState.deserializeToView( + finalizedState.bodyBytes, + ) as ContainerTreeViewType; + const summaryStateView = ssz[summaryState.forkName as keyof typeof ForkName].BeaconState.deserializeToView( + summaryState.bodyBytes, + ) as ContainerTreeViewType; + const stateWithWdsView = ssz[stateWithWds.forkName as keyof typeof ForkName].BeaconState.deserializeToView( + stateWithWds.bodyBytes, + ) as ContainerTreeViewType; + const blockWithWdsView = ssz[stateWithWds.forkName as keyof typeof ForkName].BeaconBlock.toView( + ssz[stateWithWds.forkName as keyof typeof ForkName].BeaconBlock.fromJson(blockWithWds.message) as any, + ) as ContainerTreeViewType; + // + // + // + const payloads = []; + for (const [valIndex, keyWithWithdrawalInfo] of Object.entries(withdrawals)) { + const validator = stateWithWdsView.validators.getReadonly(Number(valIndex)); + if (epoch < validator.withdrawableEpoch) { + WorkerLogger.warn(`Validator ${valIndex} is not full withdrawn. Just huge amount of ETH. Skipped`); + continue; + } + WorkerLogger.log(`Generating validator [${valIndex}] proof`); + const validatorProof = generateValidatorProof(stateWithWdsView, Number(valIndex)); + WorkerLogger.log('Generating withdrawal proof'); + const withdrawalProof = generateWithdrawalProof( + stateWithWdsView, + blockWithWdsView, + keyWithWithdrawalInfo.withdrawal.offset, + ); + WorkerLogger.log('Generating historical state proof'); + const historicalStateProof = generateHistoricalStateProof( + finalizedStateView, + summaryStateView, + summaryIndex, + rootIndexInSummary, + ); + WorkerLogger.log('Verifying validator proof locally'); + verifyProof( + stateWithWdsView.hashTreeRoot(), + validatorProof.gindex, + validatorProof.witnesses, + validator.hashTreeRoot(), + ); + WorkerLogger.log('Verifying withdrawal proof locally'); + verifyProof( + stateWithWdsView.hashTreeRoot(), + withdrawalProof.gindex, + withdrawalProof.witnesses, + ( + blockWithWdsView as ContainerTreeViewType + ).body.executionPayload.withdrawals + .getReadonly(keyWithWithdrawalInfo.withdrawal.offset) + .hashTreeRoot(), + ); + WorkerLogger.log('Verifying historical state proof locally'); + verifyProof( + finalizedStateView.hashTreeRoot(), + historicalStateProof.gindex, + historicalStateProof.witnesses, + (summaryStateView as ContainerTreeViewType).blockRoots.getReadonly( + rootIndexInSummary, + ), + ); + payloads.push({ + keyIndex: keyWithWithdrawalInfo.keyIndex, + nodeOperatorId: keyWithWithdrawalInfo.operatorId, + beaconBlock: { + header: { + slot: finalHeader.header.message.slot, + proposerIndex: Number(finalHeader.header.message.proposer_index), + parentRoot: finalHeader.header.message.parent_root, + stateRoot: finalHeader.header.message.state_root, + bodyRoot: finalHeader.header.message.body_root, + }, + rootsTimestamp: nextToFinalizedHeaderTimestamp, + }, + oldBlock: { + header: { + slot: headerWithWds.header.message.slot, + proposerIndex: Number(headerWithWds.header.message.proposer_index), + parentRoot: headerWithWds.header.message.parent_root, + stateRoot: headerWithWds.header.message.state_root, + bodyRoot: headerWithWds.header.message.body_root, + }, + // NOTE: the last byte can be changed due to `CSVerifier` implementation in the future + rootGIndex: '0x' + (historicalStateProof.gindex.toString(16) + '00').padStart(64, '0'), + proof: historicalStateProof.witnesses.map(toHex), + }, + witness: { + withdrawalOffset: Number(keyWithWithdrawalInfo.withdrawal.offset), + withdrawalIndex: Number(keyWithWithdrawalInfo.withdrawal.index), + validatorIndex: Number(keyWithWithdrawalInfo.withdrawal.validator_index), + amount: Number(keyWithWithdrawalInfo.withdrawal.amount), + withdrawalCredentials: toHex(validator.withdrawalCredentials), + effectiveBalance: validator.effectiveBalance, + slashed: Boolean(validator.slashed), + activationEligibilityEpoch: validator.activationEligibilityEpoch, + activationEpoch: validator.activationEpoch, + exitEpoch: validator.exitEpoch, + withdrawableEpoch: validator.withdrawableEpoch, + withdrawalProof: withdrawalProof.witnesses.map(toHex), + validatorProof: validatorProof.witnesses.map(toHex), + }, + }); + } + return payloads; +} + +buildHistoricalWithdrawalsProofPayloads() + .then((v) => parentPort?.postMessage(v)) + .catch((e) => { + console.error(e); + throw e; + }); diff --git a/src/common/workers/items/build-slashing-proof-payloads.ts b/src/common/workers/items/build-slashing-proof-payloads.ts new file mode 100644 index 0000000..b9a978e --- /dev/null +++ b/src/common/workers/items/build-slashing-proof-payloads.ts @@ -0,0 +1,75 @@ +import { parentPort, workerData } from 'node:worker_threads'; + +import { ContainerTreeViewType } from '@chainsafe/ssz/lib/view/container'; + +import { generateValidatorProof, toHex, verifyProof } from '../../helpers/proofs'; +import { InvolvedKeys } from '../../prover/duties/slashings.service'; +import { SlashingProofPayload } from '../../prover/types'; +import { State } from '../../providers/consensus/consensus'; +import { BlockHeaderResponse } from '../../providers/consensus/response.interface'; +import { WorkerLogger } from '../workers.service'; + +let ssz: typeof import('@lodestar/types').ssz; +let anySsz: typeof ssz.phase0 | typeof ssz.altair | typeof ssz.bellatrix | typeof ssz.capella | typeof ssz.deneb; +let ForkName: typeof import('@lodestar/params').ForkName; + +export type BuildSlashingProofArgs = { + currentHeader: BlockHeaderResponse; + nextHeaderTimestamp: number; + state: State; + slashings: InvolvedKeys; +}; + +async function buildSlashingProofPayloads(): Promise { + ssz = await eval(`import('@lodestar/types').then((m) => m.ssz)`); + const { currentHeader, nextHeaderTimestamp, state, slashings } = workerData as BuildSlashingProofArgs; + // + // Get views + // + const stateView = ssz[state.forkName as keyof typeof ForkName].BeaconState.deserializeToView( + state.bodyBytes, + ) as ContainerTreeViewType; + // + // + // + const payloads = []; + for (const [valIndex, keyInfo] of Object.entries(slashings)) { + const validator = stateView.validators.getReadonly(Number(valIndex)); + WorkerLogger.log(`Generating validator [${valIndex}] proof`); + const validatorProof = generateValidatorProof(stateView, Number(valIndex)); + WorkerLogger.log('Verifying validator proof locally'); + verifyProof(stateView.hashTreeRoot(), validatorProof.gindex, validatorProof.witnesses, validator.hashTreeRoot()); + payloads.push({ + keyIndex: keyInfo.keyIndex, + nodeOperatorId: keyInfo.operatorId, + beaconBlock: { + header: { + slot: currentHeader.header.message.slot, + proposerIndex: Number(currentHeader.header.message.proposer_index), + parentRoot: currentHeader.header.message.parent_root, + stateRoot: currentHeader.header.message.state_root, + bodyRoot: currentHeader.header.message.body_root, + }, + rootsTimestamp: nextHeaderTimestamp, + }, + witness: { + validatorIndex: Number(valIndex), + withdrawalCredentials: toHex(validator.withdrawalCredentials), + effectiveBalance: validator.effectiveBalance, + activationEligibilityEpoch: validator.activationEligibilityEpoch, + activationEpoch: validator.activationEpoch, + exitEpoch: validator.exitEpoch, + withdrawableEpoch: validator.withdrawableEpoch, + validatorProof: validatorProof.witnesses.map(toHex), + }, + }); + } + return payloads; +} + +buildSlashingProofPayloads() + .then((v) => parentPort?.postMessage(v)) + .catch((e) => { + console.error(e); + throw e; + }); diff --git a/src/common/workers/items/get-validators.ts b/src/common/workers/items/get-validators.ts new file mode 100644 index 0000000..f8df1c8 --- /dev/null +++ b/src/common/workers/items/get-validators.ts @@ -0,0 +1,61 @@ +import { parentPort, workerData } from 'node:worker_threads'; + +import { iterateNodesAtDepth } from '@chainsafe/persistent-merkle-tree'; +import { ContainerTreeViewType } from '@chainsafe/ssz/lib/view/container'; + +import { toHex } from '../../helpers/proofs'; +import { State } from '../../providers/consensus/consensus'; + +let ssz: typeof import('@lodestar/types').ssz; +let anySsz: typeof ssz.phase0 | typeof ssz.altair | typeof ssz.bellatrix | typeof ssz.capella | typeof ssz.deneb; +let ForkName: typeof import('@lodestar/params').ForkName; + +export type GetValidatorsArgs = { + state: State; + lastValidatorsCount: number; +}; + +export type GetValidatorsResult = { + totalValLength: number; + valKeys: string[]; +}; + +async function getValidators(): Promise { + ssz = await eval(`import('@lodestar/types').then((m) => m.ssz)`); + const { state, lastValidatorsCount } = workerData as GetValidatorsArgs; + // + // Get views + // + const stateView = ssz[state.forkName as keyof typeof ForkName].BeaconState.deserializeToView( + state.bodyBytes, + ) as ContainerTreeViewType; + // + // + // + const totalValLength = stateView.validators.length; + const appearedValsCount = totalValLength - lastValidatorsCount; + if (appearedValsCount === 0) { + return { totalValLength, valKeys: [] }; + } + const iterator = iterateNodesAtDepth( + stateView.validators.type.tree_getChunksNode(stateView.validators.node), + stateView.validators.type.chunkDepth, + lastValidatorsCount, + appearedValsCount, + ); + const valKeys = []; + for (let i = lastValidatorsCount; i < totalValLength; i++) { + const node = iterator.next().value; + const v = stateView.validators.type.elementType.tree_toValue(node); + valKeys.push(toHex(v.pubkey)); + } + iterator.return && iterator.return(); + return { totalValLength, valKeys }; +} + +getValidators() + .then((v) => parentPort?.postMessage(v)) + .catch((e) => { + console.error(e); + throw e; + }); diff --git a/src/common/workers/workers.module.ts b/src/common/workers/workers.module.ts new file mode 100644 index 0000000..150b3f8 --- /dev/null +++ b/src/common/workers/workers.module.ts @@ -0,0 +1,10 @@ +import { Module } from '@nestjs/common'; + +import { WorkersService } from './workers.service'; + +@Module({ + imports: [], + providers: [WorkersService], + exports: [WorkersService], +}) +export class WorkersModule {} diff --git a/src/common/workers/workers.service.ts b/src/common/workers/workers.service.ts new file mode 100644 index 0000000..9a773b0 --- /dev/null +++ b/src/common/workers/workers.service.ts @@ -0,0 +1,114 @@ +import { Worker, parentPort } from 'node:worker_threads'; + +import { LOGGER_PROVIDER } from '@lido-nestjs/logger'; +import { Inject, Injectable, LoggerService, Optional } from '@nestjs/common'; +import { ConfigService } from '@nestjs/config'; + +import { WorkingMode } from '../config/env.validation'; +import { PrometheusService, TrackWorker } from '../prometheus'; +import { HistoricalWithdrawalsProofPayload, SlashingProofPayload, WithdrawalsProofPayload } from '../prover/types'; +import { BuildGeneralWithdrawalProofArgs } from './items/build-general-wd-proof-payloads'; +import { BuildHistoricalWithdrawalProofArgs } from './items/build-historical-wd-proof-payloads'; +import { BuildSlashingProofArgs } from './items/build-slashing-proof-payloads'; +import { GetValidatorsArgs, GetValidatorsResult } from './items/get-validators'; + +class ParentLoggerMessage { + __class: string; + level: string; + message: string; + + constructor(level: string, message: string) { + this.__class = ParentLoggerMessage.name; + this.level = level; + this.message = message; + } + + // override `instanceof` behavior to allow simple type checking + static get [Symbol.hasInstance]() { + return function (instance: any) { + return instance.__class === ParentLoggerMessage.name; + }; + } +} + +export class WorkerLogger { + public static warn(message: string): void { + parentPort?.postMessage(new ParentLoggerMessage('warn', message)); + } + + public static log(message: string): void { + parentPort?.postMessage(new ParentLoggerMessage('log', message)); + } +} + +@Injectable() +export class WorkersService { + constructor( + @Inject(LOGGER_PROVIDER) protected readonly logger: LoggerService, + @Optional() protected readonly prometheus: PrometheusService, + protected readonly config: ConfigService, + ) {} + + public async getValidators(args: GetValidatorsArgs): Promise { + return await this._run('get-validators', args); + } + + public async getSlashingProofPayloads(args: BuildSlashingProofArgs): Promise { + return await this._run('build-slashing-proof-payloads', args); + } + + public async getGeneralWithdrawalProofPayloads( + args: BuildGeneralWithdrawalProofArgs, + ): Promise { + return await this._run('build-general-wd-proof-payloads', args); + } + + public async getHistoricalWithdrawalProofPayloads( + args: BuildHistoricalWithdrawalProofArgs, + ): Promise { + return await this._run('build-historical-wd-proof-payloads', args); + } + + private async _run(name: string, data: any): Promise { + if (this.config.get('WORKING_MODE') == WorkingMode.Daemon) { + return await this._baseRunWithTracker(name, data); + } + return await this._baseRun(name, data); + } + + @TrackWorker() + private async _baseRunWithTracker(name: string, data: any): Promise { + return await this._baseRun(name, data); + } + + private async _baseRun(name: string, data: any): Promise { + return new Promise((resolve, reject) => { + const worker = new Worker(__dirname + `/items/${name}.js`, { + workerData: data, + resourceLimits: { + maxOldGenerationSizeMb: 8192, + }, + }); + worker.on('message', (msg) => { + if (msg instanceof ParentLoggerMessage) { + switch (msg.level) { + case 'warn': { + this.logger.warn(msg.message); + break; + } + case 'log': { + this.logger.log(msg.message); + break; + } + } + return; + } + resolve(msg); + }); + worker.on('error', (error) => reject(new Error(`Worker error: ${error}`))); + worker.on('exit', (code) => { + if (code !== 0) reject(new Error(`Worker stopped with exit code ${code}`)); + }); + }); + } +} diff --git a/src/daemon/daemon.module.ts b/src/daemon/daemon.module.ts index b4d7f02..7641a16 100644 --- a/src/daemon/daemon.module.ts +++ b/src/daemon/daemon.module.ts @@ -11,9 +11,10 @@ import { LoggerModule } from '../common/logger/logger.module'; import { PrometheusModule } from '../common/prometheus/prometheus.module'; import { ProverModule } from '../common/prover/prover.module'; import { ProvidersModule } from '../common/providers/providers.module'; +import { WorkersModule } from '../common/workers/workers.module'; @Module({ - imports: [LoggerModule, ConfigModule, HealthModule, PrometheusModule, ProvidersModule, ProverModule], + imports: [LoggerModule, ConfigModule, HealthModule, PrometheusModule, ProvidersModule, WorkersModule, ProverModule], providers: [DaemonService, KeysIndexer, RootsProvider, RootsProcessor, RootsStack], exports: [DaemonService], }) diff --git a/src/daemon/services/keys-indexer.ts b/src/daemon/services/keys-indexer.ts index 279c442..4e66236 100644 --- a/src/daemon/services/keys-indexer.ts +++ b/src/daemon/services/keys-indexer.ts @@ -1,6 +1,3 @@ -import { iterateNodesAtDepth } from '@chainsafe/persistent-merkle-tree'; -import { BooleanType, ByteVectorType, ContainerNodeStructType, UintNumberType } from '@chainsafe/ssz'; -import { ListCompositeTreeView } from '@chainsafe/ssz/lib/view/listComposite'; import { Low } from '@huanshiwushuang/lowdb'; import { JSONFile } from '@huanshiwushuang/lowdb/node'; import { LOGGER_PROVIDER } from '@lido-nestjs/logger'; @@ -14,12 +11,12 @@ import { PrometheusService, TrackTask, } from '../../common/prometheus'; -import { toHex } from '../../common/prover/helpers/proofs'; import { KeyInfo } from '../../common/prover/types'; -import { Consensus } from '../../common/providers/consensus/consensus'; +import { Consensus, State } from '../../common/providers/consensus/consensus'; import { BlockHeaderResponse, RootHex, Slot } from '../../common/providers/consensus/response.interface'; import { Keysapi } from '../../common/providers/keysapi/keysapi'; import { Key, Module } from '../../common/providers/keysapi/response.interface'; +import { WorkersService } from '../../common/workers/workers.service'; type KeysIndexerServiceInfo = { moduleAddress: string; @@ -32,19 +29,6 @@ type KeysIndexerServiceStorage = { [valIndex: number]: KeyInfo; }; -type Validators = ListCompositeTreeView< - ContainerNodeStructType<{ - pubkey: ByteVectorType; - withdrawalCredentials: ByteVectorType; - effectiveBalance: UintNumberType; - slashed: BooleanType; - activationEligibilityEpoch: UintNumberType; - activationEpoch: UintNumberType; - exitEpoch: UintNumberType; - withdrawableEpoch: UintNumberType; - }> ->; - // At one time only one task should be running function Single(target: any, propertyKey: string, descriptor: PropertyDescriptor) { const originalMethod = descriptor.value; @@ -74,6 +58,7 @@ export class KeysIndexer implements OnApplicationBootstrap { @Inject(LOGGER_PROVIDER) protected readonly logger: LoggerService, protected readonly config: ConfigService, protected readonly prometheus: PrometheusService, + protected readonly workers: WorkersService, protected readonly consensus: Consensus, protected readonly keysapi: Keysapi, ) {} @@ -111,19 +96,16 @@ export class KeysIndexer implements OnApplicationBootstrap { private async baseRun( stateRoot: RootHex, finalizedSlot: Slot, - stateDataProcessingCallback: (validators: Validators, finalizedSlot: Slot) => Promise, + stateDataProcessingCallback: (state: State, finalizedSlot: Slot) => Promise, ): Promise { this.logger.log(`🔑 Keys indexer is running`); this.logger.log(`Get validators. State root [${stateRoot}]`); const state = await this.consensus.getState(stateRoot); - const stateView = this.consensus.stateToView(state.bodyBytes, state.forkName); - this.logger.log(`Total validators count: ${stateView.validators.length}`); // TODO: do we need to store already full withdrawn keys ? - const currValidatorsCount = stateView.validators.length; - await stateDataProcessingCallback(stateView.validators, finalizedSlot); + const totalValLength = await stateDataProcessingCallback(state, finalizedSlot); this.logger.log(`CSM validators count: ${Object.keys(this.storage.data).length}`); this.info.data.storageStateSlot = finalizedSlot; - this.info.data.lastValidatorsCount = currValidatorsCount; + this.info.data.lastValidatorsCount = totalValLength; await this.info.write(); await this.storage.write(); } @@ -222,27 +204,23 @@ export class KeysIndexer implements OnApplicationBootstrap { await this.baseRun( stateRoot, finalizedSlot, - async (validators, finalizedSlot) => await this.initStorage(validators, finalizedSlot), + async (state, finalizedSlot): Promise => await this.initStorage(state, finalizedSlot), ); } } - private async initStorage(validators: Validators, finalizedSlot: Slot): Promise { + private async initStorage(state: State, finalizedSlot: Slot): Promise { const csmKeys = await this.keysapi.getModuleKeys(this.info.data.moduleId); this.keysapi.healthCheck(this.consensus.slotToTimestamp(finalizedSlot), csmKeys.meta); const keysMap = new Map(); csmKeys.data.keys.forEach((k: Key) => keysMap.set(k.key, { ...k })); - const valLength = validators.length; - const iterator = iterateNodesAtDepth( - validators.type.tree_getChunksNode(validators.node), - validators.type.chunkDepth, - 0, - valLength, - ); - for (let i = 0; i < valLength; i++) { - const node = iterator.next().value; - const v = node.value; - const pubKey = toHex(v.pubkey); + const { totalValLength, valKeys } = await this.workers.getValidators({ + state, + lastValidatorsCount: 0, + }); + this.logger.log(`Total validators count: ${totalValLength}`); + for (let i = 0; i < totalValLength; i++) { + const pubKey = valKeys[i]; const keyInfo = keysMap.get(pubKey); if (!keyInfo) continue; this.storage.data[i] = { @@ -251,39 +229,29 @@ export class KeysIndexer implements OnApplicationBootstrap { pubKey: pubKey, }; } - iterator.return && iterator.return(); + return totalValLength; } - private async updateStorage(validators: Validators, finalizedSlot: Slot): Promise { + private async updateStorage(state: State, finalizedSlot: Slot): Promise { // TODO: should we think about re-using validator indexes? // TODO: should we think about changing WC for existing old vaidators ? - const valLength = validators.length; - const appearedValsCount = valLength - this.info.data.lastValidatorsCount; - if (appearedValsCount == 0) { + const { totalValLength, valKeys: newValKeys } = await this.workers.getValidators({ + state, + lastValidatorsCount: this.info.data.lastValidatorsCount, + }); + this.logger.log(`Total validators count: ${totalValLength}`); + if (newValKeys.length == 0) { this.logger.log(`No new validators in the state`); - return; - } - this.logger.log(`New appeared validators count: ${appearedValsCount}`); - const iterator = iterateNodesAtDepth( - validators.type.tree_getChunksNode(validators.node), - validators.type.chunkDepth, - this.info.data.lastValidatorsCount, - appearedValsCount, - ); - const valKeys = []; - for (let i = this.info.data.lastValidatorsCount; i < valLength; i++) { - const node = iterator.next().value; - const v = validators.type.elementType.tree_toValue(node); - valKeys.push(toHex(v.pubkey)); + return totalValLength; } - // TODO: can be better - const csmKeys = await this.keysapi.findModuleKeys(this.info.data.moduleId, valKeys); + this.logger.log(`New appeared validators count: ${newValKeys.length}`); + const csmKeys = await this.keysapi.findModuleKeys(this.info.data.moduleId, newValKeys); this.keysapi.healthCheck(this.consensus.slotToTimestamp(finalizedSlot), csmKeys.meta); this.logger.log(`New appeared CSM validators count: ${csmKeys.data.keys.length}`); - const valKeysLength = valKeys.length; + const valKeysLength = newValKeys.length; for (const csmKey of csmKeys.data.keys) { for (let i = 0; i < valKeysLength; i++) { - if (valKeys[i] != csmKey.key) continue; + if (newValKeys[i] != csmKey.key) continue; const index = i + this.info.data.lastValidatorsCount; this.storage.data[index] = { operatorId: csmKey.operatorIndex, @@ -292,7 +260,7 @@ export class KeysIndexer implements OnApplicationBootstrap { }; } } - iterator.return && iterator.return(); + return totalValLength; } private setMetrics() { diff --git a/src/daemon/services/roots-processor.ts b/src/daemon/services/roots-processor.ts index a0b2f6d..11067c9 100644 --- a/src/daemon/services/roots-processor.ts +++ b/src/daemon/services/roots-processor.ts @@ -28,7 +28,6 @@ export class RootsProcessor { slotNumber: Number(blockInfoToProcess.message.slot), }; await this.rootsStack.push(rootSlot); // in case of revert we should reprocess the root - // TODO: need some protection from run out of account's balance when tx reverting for the same root await this.prover.handleBlock(blockRootToProcess, blockInfoToProcess, finalizedHeader, this.keysIndexer.getKey); const indexerIsTrusted = this.keysIndexer.isTrustedForEveryDuty(rootSlot.slotNumber); if (indexerIsTrusted) await this.rootsStack.purge(rootSlot);