From ff346f27a6f8ca33a3d8f80d1c72f3efc508a971 Mon Sep 17 00:00:00 2001 From: Eugene Panteleymonchuk Date: Tue, 5 Dec 2023 11:09:08 +0200 Subject: [PATCH] Fix: Improve Milestone Feed consistency (#854) * Fix missed milestone index. Signed-off-by: Eugene Panteleymonchuk * Fix reset time when switch network. Signed-off-by: Eugene Panteleymonchuk * Fix comments. Signed-off-by: Eugene Panteleymonchuk --------- Signed-off-by: Eugene Panteleymonchuk --- .../routes/stardust/milestone/influx/get.ts | 7 +- .../stardust/influx/influxDbClient.ts | 119 +++++++++++------- .../stardust/influx/influxDbService.ts | 5 + .../services/stardust/influx/influxQueries.ts | 11 ++ client/src/helpers/hooks/useBlockFeed.ts | 7 +- 5 files changed, 99 insertions(+), 50 deletions(-) diff --git a/api/src/routes/stardust/milestone/influx/get.ts b/api/src/routes/stardust/milestone/influx/get.ts index 31a20e497..f509b87f0 100644 --- a/api/src/routes/stardust/milestone/influx/get.ts +++ b/api/src/routes/stardust/milestone/influx/get.ts @@ -36,7 +36,12 @@ export async function get( } const milestoneIndex = Number.parseInt(request.milestoneIndex, 10); - const maybeMsStats = await influxService.fetchAnalyticsForMilestoneWithRetries(milestoneIndex); + let maybeMsStats = await influxService.fetchAnalyticsForMilestoneWithRetries(milestoneIndex); + + if (!maybeMsStats) { + maybeMsStats = await influxService.fetchAnalyticsForMilestone(milestoneIndex); + } + return maybeMsStats ? { milestoneIndex: maybeMsStats.milestoneIndex, diff --git a/api/src/services/stardust/influx/influxDbClient.ts b/api/src/services/stardust/influx/influxDbClient.ts index f08cf8881..4334ea846 100644 --- a/api/src/services/stardust/influx/influxDbClient.ts +++ b/api/src/services/stardust/influx/influxDbClient.ts @@ -15,7 +15,8 @@ import { NFT_STAT_TOTAL_QUERY, SHIMMER_CLAIMED_TOTAL_QUERY, MILESTONE_STATS_QUERY, - STORAGE_DEPOSIT_TOTAL_QUERY + STORAGE_DEPOSIT_TOTAL_QUERY, + MILESTONE_STATS_QUERY_BY_INDEX } from "./influxQueries"; import logger from "../../../logger"; import { INetwork } from "../../../models/db/INetwork"; @@ -32,6 +33,15 @@ import { IUnclaimedGenesisOutputsDailyInflux, IUnclaimedTokensDailyInflux, IUnlockConditionsPerTypeDailyInflux } from "../../../models/influx/IInfluxTimedEntries"; +type MilestoneUpdate = ITimedEntry & { + milestoneIndex: number; + taggedData: number; + milestone: number; + transaction: number; + treasuryTransaction: number; + noPayload: number; +}; + /** * The collect graph data interval cron expression. * Every hour at 59 min 55 sec @@ -147,6 +157,23 @@ export abstract class InfluxDbClient { return false; } + /** + * Get the milestone analytics by index and set it in the cache. + * @param milestoneIndex - The milestone index. + */ + public async collectMilestoneStatsByIndex(milestoneIndex: number) { + console.log('--- request ', milestoneIndex); + try { + for (const update of await + this._client.query(MILESTONE_STATS_QUERY_BY_INDEX, { placeholders: { milestoneIndex } }) + ) { + this.updateMilestoneCache(update); + } + } catch (err) { + logger.warn(`[InfluxDb] Failed refreshing milestone stats for "${this._network.network}". Cause: ${err}`); + } + } + /** * Function to sort map entries in ascending order. * @param a The first entry @@ -187,7 +214,7 @@ export abstract class InfluxDbClient { void this.collectAnalytics(); }); - cron.schedule("*/5 * * * * *", async () => { + cron.schedule("*/4 * * * * *", async () => { // eslint-disable-next-line no-void void this.collectMilestoneStats(); }); @@ -349,60 +376,56 @@ export abstract class InfluxDbClient { logger.debug(`[InfluxDb] Collecting milestone stats for "${this._network.network}"`); try { for (const update of await - this.queryInflux( + this.queryInflux( MILESTONE_STATS_QUERY, null, this.getToNanoDate() - ) - ) { - if (update.milestoneIndex !== undefined && !this._milestoneCache.has(update.milestoneIndex)) { - const { - milestoneIndex, transaction, milestone, taggedData, treasuryTransaction, noPayload - } = update; - const blockCount = transaction + milestone + taggedData + treasuryTransaction + noPayload; - this._milestoneCache.set(milestoneIndex, { - milestoneIndex, - blockCount, - perPayloadType: { - transaction, - milestone, - taggedData, - treasuryTransaction, - noPayload - } - }); - - logger.debug( - `[InfluxDb] Added milestone index "${milestoneIndex}" to cache for "${this._network.network}"` - ); + )) { + this.updateMilestoneCache(update); + } + } catch (err) { + logger.warn(`[InfluxDb] Failed refreshing milestone stats for "${this._network.network}". Cause: ${err}`); + } + } - if (this._milestoneCache.size > MILESTONE_CACHE_MAX) { - let lowestIndex: number; - for (const index of this._milestoneCache.keys()) { - if (!lowestIndex) { - lowestIndex = index; - } + private updateMilestoneCache(update: MilestoneUpdate) { + if (update.milestoneIndex !== undefined && !this._milestoneCache.has(update.milestoneIndex)) { + const { + milestoneIndex, transaction, milestone, taggedData, treasuryTransaction, noPayload + } = update; + const blockCount = transaction + milestone + taggedData + treasuryTransaction + noPayload; + this._milestoneCache.set(milestoneIndex, { + milestoneIndex, + blockCount, + perPayloadType: { + transaction, + milestone, + taggedData, + treasuryTransaction, + noPayload + } + }); - if (milestoneIndex < lowestIndex) { - lowestIndex = index; - } - } + logger.debug( + `[InfluxDb] Added milestone index "${milestoneIndex}" to cache for "${this._network.network}"` + ); - logger.debug( - `[InfluxDb] Deleting milestone index "${lowestIndex}" ("${this._network.network}")` - ); + if (this._milestoneCache.size > MILESTONE_CACHE_MAX) { + let lowestIndex: number; + for (const index of this._milestoneCache.keys()) { + if (!lowestIndex) { + lowestIndex = index; + } - this._milestoneCache.delete(lowestIndex); + if (milestoneIndex < lowestIndex) { + lowestIndex = index; } } + + logger.debug( + `[InfluxDb] Deleting milestone index "${lowestIndex}" ("${this._network.network}")` + ); + + this._milestoneCache.delete(lowestIndex); } - } catch (err) { - logger.warn(`[InfluxDb] Failed refreshing milestone stats for "${this._network.network}". Cause: ${err}`); } } diff --git a/api/src/services/stardust/influx/influxDbService.ts b/api/src/services/stardust/influx/influxDbService.ts index a056209fb..e7c0c6ce2 100644 --- a/api/src/services/stardust/influx/influxDbService.ts +++ b/api/src/services/stardust/influx/influxDbService.ts @@ -121,6 +121,11 @@ export class InfluxDBService extends InfluxDbClient { return this._milestoneCache; } + public async fetchAnalyticsForMilestone(milestoneIndex: number) { + await this.collectMilestoneStatsByIndex(milestoneIndex); + return this._milestoneCache.get(milestoneIndex); + } + public async fetchAnalyticsForMilestoneWithRetries( milestoneIndex: number ): Promise { diff --git a/api/src/services/stardust/influx/influxQueries.ts b/api/src/services/stardust/influx/influxQueries.ts index ba6786da1..32085be19 100644 --- a/api/src/services/stardust/influx/influxQueries.ts +++ b/api/src/services/stardust/influx/influxQueries.ts @@ -335,4 +335,15 @@ export const MILESTONE_STATS_QUERY = ` last("no_payload_count") AS "noPayload" FROM "stardust_block_activity" `; +export const MILESTONE_STATS_QUERY_BY_INDEX = ` + SELECT + milestone_index AS "milestoneIndex", + tagged_data_count AS "taggedData", + milestone_count AS "milestone", + transaction_count AS "transaction", + treasury_transaction_count AS "treasuryTransaction", + no_payload_count AS "noPayload" + FROM "stardust_block_activity" + WHERE "milestone_index" = $milestoneIndex +`; diff --git a/client/src/helpers/hooks/useBlockFeed.ts b/client/src/helpers/hooks/useBlockFeed.ts index b23c53813..f2a7bf513 100644 --- a/client/src/helpers/hooks/useBlockFeed.ts +++ b/client/src/helpers/hooks/useBlockFeed.ts @@ -27,6 +27,7 @@ export function useBlockFeed(network: string): [ const resetCounter = useRef(0); const [milestones, setMilestones] = useState([]); const [latestMilestonIndex, setLatestMilestoneIndex] = useState(null); + const latestMilestoneIndexRef = useRef(latestMilestonIndex); const fetchLatestCachedMilestones = useCallback(async () => { if (apiClient) { @@ -60,6 +61,10 @@ export function useBlockFeed(network: string): [ }; }, [network, feedProbe]); + useEffect(() => { + latestMilestoneIndexRef.current = latestMilestonIndex; + }, [latestMilestonIndex]); + useEffect(() => { // eslint-disable-next-line no-void void fetchLatestCachedMilestones(); @@ -69,7 +74,7 @@ export function useBlockFeed(network: string): [ const onMilestoneUpdate = (newMilestone: IFeedMilestoneData) => { lastUpdateTime.current = Date.now(); if (isMounted) { - if (isMounted && (latestMilestonIndex ?? 0) < newMilestone.milestoneIndex) { + if (isMounted && (latestMilestoneIndexRef.current ?? 0) < newMilestone.milestoneIndex) { setLatestMilestoneIndex(newMilestone.milestoneIndex); } if (isMounted) {