feat(module.defid): Detect tip stale by height (#975)

* check height after 90 mins

* revert unuse change

* move stale detection logic to model probe

* revert defid probe

* miss set index = highest.height

* pr req & add test

* add restore mock time

* minor changes

src/module.api/actuator.e2e.ts

@@ -200,4 +200,46 @@ describe('with peers', () => {
       status: 'ok'
     })
   })
+
+  it('/_actuator/probes/readiness stale', async () => {
+    // Time travel 100 mins
+    const mockTime = jest.spyOn(Date, 'now').mockImplementation(() => new Date().getTime() + 6_000_000)
+
+    const res = await app.inject({
+      method: 'GET',
+      url: '/_actuator/probes/readiness'
+    })
+    expect(res.statusCode).toStrictEqual(503)
+    expect(res.json()).toStrictEqual({
+      details: {
+        defid: {
+          blocks: expect.any(Number),
+          headers: expect.any(Number),
+          initialBlockDownload: false,
+          peers: 1,
+          status: 'up'
+        },
+        model: {
+          status: 'down'
+        }
+      },
+      error: {
+        model: {
+          status: 'down'
+        }
+      },
+      info: {
+        defid: {
+          blocks: expect.any(Number),
+          headers: expect.any(Number),
+          initialBlockDownload: false,
+          peers: 1,
+          status: 'up'
+        }
+      },
+      status: 'error'
+    })
+
+    mockTime.mockRestore()
+  })
 })

src/module.model/_model.probes.ts

@@ -1,7 +1,7 @@
 import { ProbeIndicator } from '@src/module.health/probe.indicator'
 import { Injectable } from '@nestjs/common'
 import { HealthIndicatorResult } from '@nestjs/terminus'
-import { BlockMapper } from '@src/module.model/block'
+import { Block, BlockMapper } from '@src/module.model/block'
 import { JsonRpcClient } from '@defichain/jellyfish-api-jsonrpc'
 
 @Injectable()
@@ -32,13 +32,16 @@ export class ModelProbeIndicator extends ProbeIndicator {
    * - unable to get the latest block
    * - synced blocks are undefined
    * - synced blocks are more than 2 blocks behind
+   * - synced latest block height is not more than 90 mins old
    */
   async readiness (): Promise<HealthIndicatorResult> {
+    let highest: Block
     let index: number | undefined
     let defid: number | undefined
 
     try {
-      index = (await this.block.getHighest())?.height
+      highest = await this.block.getHighest() as Block
+      index = highest.height
       defid = await this.client.blockchain.getBlockCount()
     } catch (err) {
       return this.withDead('model', 'unable to get the latest block')
@@ -59,6 +62,15 @@ export class ModelProbeIndicator extends ProbeIndicator {
       return this.withDead('model', 'synced blocks are more than 2 blocks behind', details)
     }
 
+    // index defid can experience rollback, so make sure the condition is only checked if `Model == DeFid`
+    if (index === defid && secondsSince(highest.time) >= 90 * 60) {
+      return this.withDead('model', 'defid chain is stale')
+    }
+
     return this.withAlive('model', details)
   }
 }
+
+function secondsSince (timeInSeconds: number): number {
+  return (Math.floor(Date.now() / 1000)) - timeInSeconds
+}