diff --git a/src/module.api/actuator.e2e.ts b/src/module.api/actuator.e2e.ts index 5bce04422..e64bdf1c5 100644 --- a/src/module.api/actuator.e2e.ts +++ b/src/module.api/actuator.e2e.ts @@ -200,4 +200,46 @@ describe('with peers', () => { status: 'ok' }) }) + + it('/_actuator/probes/readiness stale', async () => { + // Time travel 100 mins + const mockTime = jest.spyOn(Date, 'now').mockImplementation(() => new Date().getTime() + 6_000_000) + + const res = await app.inject({ + method: 'GET', + url: '/_actuator/probes/readiness' + }) + expect(res.statusCode).toStrictEqual(503) + expect(res.json()).toStrictEqual({ + details: { + defid: { + blocks: expect.any(Number), + headers: expect.any(Number), + initialBlockDownload: false, + peers: 1, + status: 'up' + }, + model: { + status: 'down' + } + }, + error: { + model: { + status: 'down' + } + }, + info: { + defid: { + blocks: expect.any(Number), + headers: expect.any(Number), + initialBlockDownload: false, + peers: 1, + status: 'up' + } + }, + status: 'error' + }) + + mockTime.mockRestore() + }) }) diff --git a/src/module.model/_model.probes.ts b/src/module.model/_model.probes.ts index eb89a3197..a408861bf 100644 --- a/src/module.model/_model.probes.ts +++ b/src/module.model/_model.probes.ts @@ -1,7 +1,7 @@ import { ProbeIndicator } from '@src/module.health/probe.indicator' import { Injectable } from '@nestjs/common' import { HealthIndicatorResult } from '@nestjs/terminus' -import { BlockMapper } from '@src/module.model/block' +import { Block, BlockMapper } from '@src/module.model/block' import { JsonRpcClient } from '@defichain/jellyfish-api-jsonrpc' @Injectable() @@ -32,13 +32,16 @@ export class ModelProbeIndicator extends ProbeIndicator { * - unable to get the latest block * - synced blocks are undefined * - synced blocks are more than 2 blocks behind + * - synced latest block height is not more than 90 mins old */ async readiness (): Promise { + let highest: Block let index: number | undefined let defid: number | undefined try { - index = (await this.block.getHighest())?.height + highest = await this.block.getHighest() as Block + index = highest.height defid = await this.client.blockchain.getBlockCount() } catch (err) { return this.withDead('model', 'unable to get the latest block') @@ -59,6 +62,15 @@ export class ModelProbeIndicator extends ProbeIndicator { return this.withDead('model', 'synced blocks are more than 2 blocks behind', details) } + // index defid can experience rollback, so make sure the condition is only checked if `Model == DeFid` + if (index === defid && secondsSince(highest.time) >= 90 * 60) { + return this.withDead('model', 'defid chain is stale') + } + return this.withAlive('model', details) } } + +function secondsSince (timeInSeconds: number): number { + return (Math.floor(Date.now() / 1000)) - timeInSeconds +}